diff --git a/backup-everything.sh b/backup-everything.sh index 6c38f8a..ebf2720 100755 --- a/backup-everything.sh +++ b/backup-everything.sh @@ -1,519 +1,387 @@ #!/bin/bash ### # Stupid script to backup some stuff # # Author: Valerio B. # Date: Wed 25 Mar 2020 ## -# try to proceed even in case of errors +# try to proceed even in case of errors (to do not skip any backup) # set -e -# current directory -DIR="${BASH_SOURCE%/*}" -if [[ ! -d "$DIR" ]]; then - DIR="$PWD"; -fi - -# check if the standard input is not a terminal -INTERACTIVE= -if [ -t 0 ]; then - INTERACTIVE=1 -fi - -# -# Check if this is the quiet mode -# -# Actually we are in quiet mode if it's not interactive. -# This lazy behavior is to avoid stupid emails from the crontab -# without the need to specify some --quiet etc. -# Note that in quiet mode only WARN and ERROR messages are shown. -# I've not created a --quiet flag because nobody is needing it. -QUIET= -if [ "$INTERACTIVE" != 1 ]; then - QUIET=1 -fi - -# path to the instructions file -INSTRUCTIONS="$DIR/backup-instructions.conf" - -# path to the configuration file -CONFIG="$DIR/options.conf" - -# no config no party -if [ ! -f $CONFIG ]; then - echo "missing options expected in $CONFIG" - exit 1 -fi - -# no instructions no party -if [ ! -f $INSTRUCTIONS ]; then - echo "missing instructions expected in $INSTRUCTIONS" - exit 1 -fi - -# default mysql commands -MYSQL="mysql" -MYSQLDUMP="mysqldump" - -# default rsync command -# --archive: Try to keep all the properties -# --fuzzy: Try to check if a file was renamed instead of delete and download a new one -# It's efficient for example with log rotated files. -# --delete: Delete the destination files if not present in the source -# NOTE: we want this behaviour but it's not a good idea toghether with --fuzzy -# that's why we do not use --delete but we use the next flags -# --delay-updates Put all updated files into place at end (useful with fuzzy and delete modes) -# --delete-delay Delete after everything (useful with fuzzy and delete modes) -RSYNC="rsync --archive --fuzzy --delay-updates --delete-delay" - -# default base backup directory for all backups -BASE="/home/backups" - -# default box name -BOX="$(hostname)" - -# set to 1 to avoid any disservice (e.g. systemctl stop/start) -NO_DISSERVICE= - -# set to 1 to do nothing -PORCELAIN= - -# include the configuration to eventually override some options -. "$CONFIG" - -# full pathnames to the backup directories -BASEBOX="$BASE/$BOX" -DAILY="$BASEBOX/daily" -DAILY_FILES="$DAILY/files" -DAILY_DATABASES="$DAILY/databases" -DAILY_LASTLOG="$DAILY/last.log" - -# apply the porcelain to the rsync command -if [ "$PORCELAIN" = 1 ]; then - RSYNC="$RSYNC --dry-run" -fi - -### -# Print something -# -# It also put the message in the backup directory -# -# @param string severity -# @param string message -# -function printthis() { - - local msg="[$(date)][$1] $2" - - # print to standard output if it's not in quiet mode - if [ "$QUIET" != 1 ]; then - echo "$msg" - fi - - # put in the log file if possible - if [ -f "$DAILY_LASTLOG" ]; then - echo "$msg" >> "$DAILY_LASTLOG" - fi -} - -### -# Print a information message -# -# @param msg Message -# -function log() { - printthis INFO "$1" -} - -### -# Print an error message -# -# @param msg Message -# -function error() { - printthis ERROR "$1" -} - -### -# Print a warning message -# -# @param msg Message -# -function warn() { - printthis WARN "$1" -} +# load useful stuff +. bootstrap.sh ### # Create a directory with safe permissions if it does not exist # # @param string path Pathname # @param int is_file_mode Check if we are in file mode (default: directory mode) # write_safe_dir_if_unexisting() { local path="$1" local is_file_mode="$2" # try to create the directory if it does not exists if [ ! -d "$path" ]; then warn "creating missing $path with 750" mkdir --parents "$path" chmod 750 "$path" fi # it must exists now require_existing_dir "$path" } ### # Create a directory for a filename with safe permissions if it does not exist # # @param string path Pathname to a filename # @param int is_file_mode Check if we are in file mode (default: directory mode) # write_safe_basedir_if_unexisting() { # first function argument local path="$1" # extract just the sub-directory local basepath=$(dirname "$path") # create that sub-directory write_safe_dir_if_unexisting "$basepath" } ### # Require an existing directory or die # # @param string Directory path # require_existing_dir() { if [ ! -d "$1" ]; then error "unexisting directory $1" exit 1 fi } # create these pathnames if they do not exist write_safe_dir_if_unexisting "$BASE" write_safe_dir_if_unexisting "$DAILY" write_safe_dir_if_unexisting "$DAILY_FILES" write_safe_dir_if_unexisting "$DAILY_DATABASES" # create or clean the last log file cat /dev/null > "$DAILY_LASTLOG" # # Dump and compress a database # # @param db string Database name # @param spaces string Cosmetic spaces # backup_database() { local db="$1" local spaces="$2" local path="$DAILY_DATABASES"/"$db".sql.gz log "$spaces""dumping database $db" # no database no party if ! $MYSQL "$db" -e exit > /dev/null 2>&1; then warn "$spaces skip unexisting database" return fi if [ "$PORCELAIN" != 1 ]; then $MYSQLDUMP "$db" | gzip > "$path" fi } # # Dump and compress some databases # # @param database... string Database names # backup_databases() { for database in $@; do backup_database "$database" done } # # Dump some databases with a shared prefix # # @param prefix string Database prefix # @param spaces string Cosmetic spaces # backup_databases_prefixed() { local prefix="$1" local spaces="$2" log "$spaces""backup all databases prefixed with: $prefix" databases=$($MYSQL -e 'SHOW DATABASES' | grep "$prefix") for database in $databases; do backup_database "$database" "$spaces " done } # # Backup every single database. That's easy. # backup_every_database() { # backup every damn database databases=$($MYSQL -e 'SHOW DATABASES') for database in $databases; do # just skip the information_schema and the performance_schema that cannot be locked if [[ "$database" != information_schema && "$database" != performance_schema ]]; then backup_database "$database" fi done } # # Backup a database that is used by a service # # The service will be stopped before dumping. # # @param service string Systemd unit file # @param db string Database name # backup_service_and_database() { local service="$1" local db="$2" local is_active= log "start backup service '$service' with database '$db'" # check if the service is running if systemctl is-active "$service" > /dev/null; then is_active=1 fi # eventually stop the service if [ $is_active == 1 ]; then if [ "$NO_DISSERVICE" == 1 ]; then warn " NOT stopping service: $service (to avoid any disservice)" else log " stopping service: $service" if [ "$PORCELAIN" != 1 ]; then systemctl stop "$service" fi fi else log " service already inactive: $service" fi # backup the database now that the service is down backup_database "$db" " " # eventually start again the service if [ $is_active == 1 ]; then if [ "$NO_DISSERVICE" == 1 ]; then warn " NOT starting again service: $service (to avoid any disservice)" else log " starting again service: $service" if [ "$PORCELAIN" != 1 ]; then systemctl start "$service" fi fi fi } # # Backup some databases used by a service # # The service will be stopped before dumping the databases # # @param service string Systemd unit file # @param db... string Database names # backup_service_and_databases() { backup_service_and_database $@ } # # Backup phabricator # # @param path string Phabricator webroot # @param dbprefix string Database prefix # backup_phabricator() { local path="$1" local dbprefix="$2" local configset="$path"/bin/config log "backup Phabricator databases" log " set Phabricator read only mode (and wait some time to make it effective)" $configset set cluster.read-only true > /dev/null sleep 5 backup_databases_prefixed "$dbprefix" " " log " revert Phabricator read only mode" $configset set cluster.read-only false > /dev/null } # # Backup a directory or a filename # # @param path string Pathname to be backupped # @param identifier string Optional identifier of this pathname # backup_path() { local path="$1" local identifier="$2" # create a default identifier if [ -z "$identifier" ]; then identifier="$path" fi # destination directory # note that the identifier may start with a slash but this is good, just don't care dest="$DAILY_FILES/$identifier" # tell were the backup will go if [ "$path" = "$identifier" ]; then log "backup $path" else log "backup $path with identifier $identifier" fi # check if the path exists if [ -e "$path" ]; then # check if it's a directory if [ -d "$path" ]; then # this is a directory # eventually create the destination if it does not exist write_safe_dir_if_unexisting "$dest" # backup this # force the source to end with a slash in order to copy the files inside the directory $RSYNC "$path/" "$dest" else # this is a filename # eventually create the base destination if it does not exist write_safe_basedir_if_unexisting "$dest" # backup this filename $RSYNC "$path" "$dest" fi else # no path no party warn " path not found: $path" fi } ## # Validate the backup host arguments # # validate_backup_host_args() { # function parameters local hostlink="$1" local identifier="$2" local port="$3" # no host no party if [ -z "$hostlink" ] || [ -z "$identifier" ]; then error "Bad usage: backup_host REMOTE_HOST:REMOTE_PATH IDENTIFIER [PORT]" exit 1 fi # tell what we will do if [ "$port" = "22" ]; then log "backup $hostlink in $identifier" else log "backup $hostlink (:$port) in $identifier" fi } ## # Backup another host directory via rsync # # @param string Rsync host # @param string Rsync path to a directory # backup_host_dir() { # function parameters local hostlink="$1" local identifier="$2" local port="$3" # eventually set default rsync port if [ -z "$port" ]; then port=22 fi # validate the arguments validate_backup_host_args "$hostlink" "$identifier" "$port" # destination path local dest="$BASE/$identifier" # create the destination if it does not exist write_safe_dir_if_unexisting "$dest" # backup everything # force the source to end with a slash to just copy the files inside it $RSYNC --compress --rsh="ssh -p $port" "$hostlink/" "$dest" } ## # Backup another host file via rsync # # @param string hostlink Rsync source host and pathname e.g. ravotti94:/tmp/asd.txt # @param string identifier Rsync destination directory # backup_host_file() { # function parameters local hostlink="$1" local identifier="$2" local port="$3" # filename local filename=$(basename "$hostlink") # eventually set default rsync port if [ -z "$port" ]; then port=22 fi # validate the arguments validate_backup_host_args "$hostlink" "$identifier" "$port" # destination path local dest="$BASE/$identifier/$filename" # create the destination directory if it does not exist write_safe_basedir_if_unexisting "$dest" # backup everything # force the source to end with a slash to just copy the files inside it $RSYNC --compress --rsh="ssh -p $port" "$hostlink" "$dest" } log "init backup in $DAILY" . "$INSTRUCTIONS" log "backup successfully concluded" diff --git a/bootstrap.sh b/bootstrap.sh new file mode 100644 index 0000000..2fd1d25 --- /dev/null +++ b/bootstrap.sh @@ -0,0 +1,144 @@ +#!/bin/bash +### +# Part of a stupid script to backup some stuff +# +# This bootstrap.sh file does nothing by itself but loads useful stuff. +# +# Author: Valerio B. +# Date: Wed 5 ago 2020 +## + +# current directory +DIR="${BASH_SOURCE%/*}" +if [[ ! -d "$DIR" ]]; then + DIR="$PWD"; +fi + +# check if the standard input is not a terminal +INTERACTIVE= +if [ -t 0 ]; then + INTERACTIVE=1 +fi + +# +# Check if this is the quiet mode +# +# Actually we are in quiet mode if it's not interactive. +# This lazy behavior is to avoid stupid emails from the crontab +# without the need to specify some --quiet etc. +# Note that in quiet mode only WARN and ERROR messages are shown. +# I've not created a --quiet flag because nobody is needing it. +QUIET= +if [ "$INTERACTIVE" != 1 ]; then + QUIET=1 +fi + +# path to the instructions file +INSTRUCTIONS="$DIR/backup-instructions.conf" + +# path to the configuration file +CONFIG="$DIR/options.conf" + +# no config no party +if [ ! -f $CONFIG ]; then + echo "missing options expected in $CONFIG" + exit 1 +fi + +# no instructions no party +if [ ! -f $INSTRUCTIONS ]; then + echo "missing instructions expected in $INSTRUCTIONS" + exit 1 +fi + +# default mysql commands +MYSQL="mysql" +MYSQLDUMP="mysqldump" + +# default rsync command +# --archive: Try to keep all the properties +# --fuzzy: Try to check if a file was renamed instead of delete and download a new one +# It's efficient for example with log rotated files. +# --delete: Delete the destination files if not present in the source +# NOTE: we want this behaviour but it's not a good idea toghether with --fuzzy +# that's why we do not use --delete but we use the next flags +# --delay-updates Put all updated files into place at end (useful with fuzzy and delete modes) +# --delete-delay Delete after everything (useful with fuzzy and delete modes) +RSYNC="rsync --archive --fuzzy --delay-updates --delete-delay" + +# default base backup directory for all backups +BASE="/home/backups" + +# default box name +BOX="$(hostname)" + +# set to 1 to avoid any disservice (e.g. systemctl stop/start) +NO_DISSERVICE= + +# set to 1 to do nothing +PORCELAIN= + +# include the configuration to eventually override some options +. "$CONFIG" + +# full pathnames to the backup directories +BASEBOX="$BASE/$BOX" +DAILY="$BASEBOX/daily" +DAILY_FILES="$DAILY/files" +DAILY_DATABASES="$DAILY/databases" +DAILY_LASTLOG="$DAILY/last.log" + +# apply the porcelain to the rsync command +if [ "$PORCELAIN" = 1 ]; then + RSYNC="$RSYNC --dry-run" +fi + +### +# Print something +# +# It also put the message in the backup directory +# +# @param string severity +# @param string message +# +function printthis() { + + local msg="[$(date)][$1] $2" + + # print to standard output if it's not in quiet mode + if [ "$QUIET" != 1 ]; then + echo "$msg" + fi + + # put in the log file if possible + if [ -f "$DAILY_LASTLOG" ]; then + echo "$msg" >> "$DAILY_LASTLOG" + fi +} + +### +# Print a information message +# +# @param msg Message +# +function log() { + printthis INFO "$1" +} + +### +# Print a warning message +# +# @param msg Message +# +function warn() { + printthis WARN "$1" +} + +### +# Print an error message +# +# @param msg Message +# +function error() { + printthis ERROR "$1" +}