diff --git a/backup-everything.sh b/backup-everything.sh index 51426fd..d290bbb 100755 --- a/backup-everything.sh +++ b/backup-everything.sh @@ -1,522 +1,524 @@ #!/bin/bash ### # Micro script to backup some stuff # # IMPORTANT: do not edit this file # and see 'backup-instructions.conf' instead! # # This file defines some functions and then it executes # your 'backup-instructions.conf' # # Author: Valerio Bozzolan # Date: Wed 25 Mar 2020 # License: CC 0 - public domain ## # try to proceed even in case of errors (to do not skip any backup) # set -e # current directory MYDIR="$(dirname "$(realpath "$0")")" # load useful stuff . "$MYDIR"/bootstrap.sh ### # Create a directory with safe permissions if it does not exist # # @param string path Pathname # @param int is_file_mode Check if we are in file mode (default: directory mode) # write_safe_dir_if_unexisting() { local path="$1" local is_file_mode="$2" # try to create the directory if it does not exists if [ ! -d "$path" ]; then warn "creating missing $path with 750" mkdir --parents "$path" chmod 750 "$path" fi # it must exists now require_existing_dir "$path" } ### # Create a directory for a filename with safe permissions if it does not exist # # @param string path Pathname to a filename # @param int is_file_mode Check if we are in file mode (default: directory mode) # write_safe_basedir_if_unexisting() { # first function argument local path="$1" # extract just the sub-directory local basepath=$(dirname "$path") # create that sub-directory write_safe_dir_if_unexisting "$basepath" } ### # Require an existing directory or die # # @param string Directory path # require_existing_dir() { if [ ! -d "$1" ]; then error "unexisting directory $1" exit 1 fi } # create these pathnames if they do not exist write_safe_dir_if_unexisting "$BASE" write_safe_dir_if_unexisting "$DAILY" write_safe_dir_if_unexisting "$DAILY_FILES" write_safe_dir_if_unexisting "$DAILY_DATABASES" # create or clean the last log file -cat /dev/null > "$DAILY_LASTLOG" +if [ "$WRITELOG" = 1 ]; then + cat /dev/null > "$DAILY_LASTLOG" +fi # # Dump and compress a database # # @param db string Database name # @param spaces string Cosmetic spaces # backup_database() { local db="$1" local spaces="$2" local path="$DAILY_DATABASES"/"$db".sql.gz log "$spaces""dumping database $db" # no database no party if ! $MYSQL "$db" -e exit > /dev/null 2>&1; then warn "$spaces skip unexisting database" return fi if [ "$PORCELAIN" != 1 ]; then $MYSQLDUMP "$db" | gzip > "$path" fi } # # Dump and compress some databases # # @param database... string Database names # backup_databases() { for database in $@; do backup_database "$database" done } # # Dump some databases with a shared prefix # # @param prefix string Database prefix # @param spaces string Cosmetic spaces # backup_databases_prefixed() { local prefix="$1" local spaces="$2" log "$spaces""backup all databases prefixed with: $prefix" databases=$($MYSQL -e 'SHOW DATABASES' | grep "$prefix") for database in $databases; do backup_database "$database" "$spaces " done } # default databases to be skipped DATABASE_SKIP_LIST=("^information_schema$" "^performance_schema$") # # Call this to skip a database # # @param string database Database pattern (Bash regex) # # This should be called before calling backup_every_database. # skip_database() { local db="$1" DATABASE_SKIP_LIST+=( "$db" ) } # # Backup every single database. That's easy. # backup_every_database() { # backup every damn database databases=$($MYSQL -e 'SHOW DATABASES') for database in $databases; do local do_backup=1 # just skip the information_schema and the performance_schema that cannot be locked for skip_entry in "${DATABASE_SKIP_LIST[@]}"; do if [[ "$database" =~ $skip_entry ]]; then # show a cute message log "skippin database $database matching blacklist" # do not backup this do_backup=0 # do not check other entries break fi done # backup if it does not match the above skip entries if [ "$do_backup" = 1 ]; then backup_database "$database" fi done } # # Backup a database that is used by a service # # The service will be stopped before dumping. # # @param service string Systemd unit file # @param db string Database name # backup_service_and_database() { local service="$1" local db="$2" local is_active= log "start backup service '$service' with database '$db'" # check if the service is running if systemctl is-active "$service" > /dev/null; then is_active=1 fi # eventually stop the service if [ $is_active == 1 ]; then if [ "$NO_DISSERVICE" == 1 ]; then warn " NOT stopping service: $service (to avoid any disservice)" else log " stopping service: $service" if [ "$PORCELAIN" != 1 ]; then systemctl stop "$service" fi fi else log " service already inactive: $service" fi # backup the database now that the service is down backup_database "$db" " " # eventually start again the service if [ $is_active == 1 ]; then if [ "$NO_DISSERVICE" == 1 ]; then warn " NOT starting again service: $service (to avoid any disservice)" else log " starting again service: $service" if [ "$PORCELAIN" != 1 ]; then systemctl start "$service" fi fi fi } # # Backup some databases used by a service # # The service will be stopped before dumping the databases # # @param service string Systemd unit file # @param db... string Database names # backup_service_and_databases() { backup_service_and_database $@ } # # Backup phabricator # # @param path string Phabricator webroot # @param dbprefix string Database prefix # backup_phabricator() { local path="$1" local dbprefix="$2" local configset="$path"/bin/config log "backup Phabricator databases" log " set Phabricator read only mode (and wait some time to make it effective)" $configset set cluster.read-only true > /dev/null sleep 5 backup_databases_prefixed "$dbprefix" " " log " revert Phabricator read only mode" $configset set cluster.read-only false > /dev/null } # # Backup a directory or a filename # # @param path string Pathname to be backupped # @param identifier string Optional identifier of this pathname # backup_path() { local path="$1" local identifier="$2" # create a default identifier if [ -z "$identifier" ]; then identifier="$path" fi # destination directory # note that the identifier may start with a slash but this is good, just don't care dest="$DAILY_FILES/$identifier" # tell were the backup will go if [ "$path" = "$identifier" ]; then log "backup $path" else log "backup $path with identifier $identifier" fi # check if the path exists if [ -e "$path" ]; then # check if it's a directory if [ -d "$path" ]; then # this is a directory # eventually create the destination if it does not exist write_safe_dir_if_unexisting "$dest" # backup this # force the source to end with a slash in order to copy the files inside the directory $RSYNC "$path/" "$dest" else # this is a filename # eventually create the base destination if it does not exist write_safe_basedir_if_unexisting "$dest" # backup this filename $RSYNC "$path" "$dest" fi else # no path no party warn " path not found: $path" fi } # # Backup some filenames ending with whatever # # @param path[...] string Pathname to be backupped # backup_paths() { # pathnames local paths="$@" # process every pathname for path in $paths; do # backup every single pathname backup_path "$path" done } ## # Validate the backup host arguments # # validate_backup_host_args() { # function parameters local port="$3" # no host no party if [ -z "$1" ] || [ -z "$2" ]; then error "Bad usage: backup_host REMOTE_HOST:REMOTE_PATH IDENTIFIER [PORT]" exit 1 fi # tell what we will do if [ "$port" = "22" ]; then log "backup $1 in $2" else log "backup $1 (:$port) in $2" fi } ## # Backup another host directory via rsync # # @param string Rsync host # @param string Rsync path to a directory # @param string Rsync host port # backup_host_dir() { # function parameters local hostlink="$1" local identifier="$2" local port="$3" # eventually set default rsync port if [ -z "$port" ]; then port=22 fi # validate the arguments validate_backup_host_args "$hostlink" "$identifier" "$port" # destination path local dest="$BASE/$identifier" # create the destination if it does not exist write_safe_dir_if_unexisting "$dest" # backup everything # force the source to end with a slash to just copy the files inside it $RSYNC --compress --rsh="ssh -p $port" "$hostlink/" "$dest" } ## # Backup another host directory via rsync # # @param string Rsync host # @param string Rsync path to a directory # push_path_host() { # function parameters local source="$1" local hostlink="$2" local port="$3" # eventually set default rsync port if [ -z "$port" ]; then port=22 fi # validate the arguments and print message validate_backup_host_args "$source" "$hostlink" "$port" # backup everything # force the source to end with a slash to just copy the files inside it $RSYNC --compress --rsh="ssh -p $port" "$source" "$hostlink" } ## # Push some pathnames into an host # # @param string Rsync host # @param string Rsync port # @param string... Rsync path to some directories # push_host_port_paths() { # function parameters local hostlink="$1" local port="$2" # strip the first two arguments shift shift local path="" # process every pathname while [[ $# -gt 0 ]]; do path="$1" shift # backup every single pathname push_path_host "$path" "$hostlink" "$port" done } ## # Push the local "daily/" directory into a remote host # # @param string Rsync host # @param string Rsync path to a directory # push_daily_directory() { # function parameters local hostlink="$1" local port="$2" push_path_host "$DAILY/" "$hostlink" "$port" } ## # Backup another host file via rsync # # @param string hostlink Rsync source host and pathname e.g. ravotti94:/tmp/asd.txt # @param string identifier Rsync destination directory # backup_host_file() { # function parameters local hostlink="$1" local identifier="$2" local port="$3" # filename local filename=$(basename "$hostlink") # eventually set default rsync port if [ -z "$port" ]; then port=22 fi # validate the arguments validate_backup_host_args "$hostlink" "$identifier" "$port" # destination path local dest="$BASE/$identifier/$filename" # create the destination directory if it does not exist write_safe_basedir_if_unexisting "$dest" # backup everything # force the source to end with a slash to just copy the files inside it $RSYNC --compress --rsh="ssh -p $port" "$hostlink" "$dest" } log "init backup in $DAILY" . "$INSTRUCTIONS" log "backup successfully concluded" # remember the last successfully concluded backup timestamp date +%s > "$DAILY_LASTTIME" diff --git a/bootstrap.sh b/bootstrap.sh index f5eae57..c9fb7fd 100644 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,187 +1,211 @@ #!/bin/bash ### # Part of a stupid script to backup some stuff # # This bootstrap.sh file does nothing by itself but loads useful stuff. # -# Author: Valerio B. -# Date: Wed 5 ago 2020 +# This file is loaded from 'backup-everything.sh' or 'rotate.sh' +# +# Author: Valerio B. +# Date: Wed 5 ago 2020 +# License: CC 0 - public domain ## # current directory DIR="${BASH_SOURCE%/*}" if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi # check if the standard input is not a terminal INTERACTIVE= if [ -t 0 ]; then INTERACTIVE=1 fi # # Check if this is the quiet mode # +# Default - not quite. +# # Actually we are in quiet mode if it's not interactive. # This lazy behavior is to avoid stupid emails from the crontab # without the need to specify some --quiet etc. # Note that in quiet mode only WARN and ERROR messages are shown. # I've not created a --quiet flag because nobody is needing it. -QUIET= -if [ "$INTERACTIVE" != 1 ]; then - QUIET=1 -fi +# +# Edit your options - do not edit here. +# +#QUIET= + +# +# Eventually write a log file +# +# Default - don't write. +# +# Edit your options - do not edit here. +# +#WRITELOG= # path to the instructions file INSTRUCTIONS="$DIR/backup-instructions.conf" # path to the configuration file CONFIG="$DIR/options.conf" # no config no party if [ ! -f $CONFIG ]; then echo "missing options expected in $CONFIG" exit 1 fi # no instructions no party if [ ! -f $INSTRUCTIONS ]; then echo "missing instructions expected in $INSTRUCTIONS" exit 1 fi # default mysql commands MYSQL="mysql" MYSQLDUMP="mysqldump" # default rsync command # --archive: Try to keep all the properties # --fuzzy: Try to check if a file was renamed instead of delete and download a new one # It's efficient for example with log rotated files. # --delete: Delete the destination files if not present in the source # NOTE: we want this behaviour but it's not a good idea toghether with --fuzzy # that's why we do not use --delete but we use the next flags # --delay-updates Put all updated files into place at end (useful with fuzzy and delete modes) # --delete-delay Delete after everything (useful with fuzzy and delete modes) # --hard-links Try to look for hard links during the transfer to do not copy separate files RSYNC="rsync --archive --fuzzy --delay-updates --delete-delay --hard-links" # default base backup directory for all backups BASE="/home/backups" # default box name BOX="$(hostname)" # set to 1 to avoid any disservice (e.g. systemctl stop/start) NO_DISSERVICE= # set to 1 to do nothing PORCELAIN= # include the configuration to eventually override some options . "$CONFIG" +# as default, if not interactive, set quite mode +if [ -z "$QUIET" ] && [ "$INTERACTIVE" != 1 ]; then + QUIET=1 +fi + +# as default, if quiet, write in the log +if [ -z "$WRITELOG" ] && [ "$QUIET" = 1 ]; then + WRITELOG=1 +fi + # full pathnames to the backup directories BASEBOX="$BASE/$BOX" DAILY="$BASEBOX/daily" DAILY_FILES="$DAILY/files" DAILY_DATABASES="$DAILY/databases" DAILY_LASTLOG="$DAILY/last.log" DAILY_LASTTIME="$DAILY/last.timestamp" # apply the porcelain to the rsync command if [ "$PORCELAIN" = 1 ]; then RSYNC="$RSYNC --dry-run" fi ### # Print something # # It also put the message in the backup directory # # @param string severity # @param string message # function printthis() { local msg="[$(date)][$1] $2" # print to standard output if it's not in quiet mode if [ "$QUIET" != 1 ]; then echo "$msg" fi # put in the log file if possible - if [ -f "$DAILY_LASTLOG" ]; then + if [ -f "$DAILY_LASTLOG" ] && [ "$WRITELOG" = 1 ]; then echo "$msg" >> "$DAILY_LASTLOG" fi } ### # Run an rsync # function copy() { # show what we are doing log "copy $*" # run the rsync command if [ "$PORCELAIN" != 1 ]; then $RSYNC $@ fi } ### # Remove a pathname # function drop() { # show what we are doing log "drop $*" # well, proceed... finger crossed... with some protections if [ "$PORCELAIN" != 1 ]; then rm --recursive --force --one-file-system --preserve-root -- $@ fi } ### # Move something somewhere # function move() { # show what we are doing log "move $*" if [ "$PORCELAIN" != 1 ]; then mv --force $@ fi } ### # Print a information message # # @param msg Message # function log() { printthis INFO "$1" } ### # Print a warning message # # @param msg Message # function warn() { printthis WARN "$1" } ### # Print an error message # # @param msg Message # function error() { printthis ERROR "$1" } diff --git a/options-example.conf b/options-example.conf index 3c05584..3975ad7 100644 --- a/options-example.conf +++ b/options-example.conf @@ -1,28 +1,30 @@ # # Generic options for 'micro-backup-script' # # See: # https://sviluppo.erinformatica.it/source/micro-backup-script/ # # Notes: # If you have not an options.conf then copy it from options-example.conf +# +# This file is loaded after 'bootstrap.sh' ####################################################################### # please uncomment and set here your hostname or something like that #BOX=my-important-server # eventually change the place where you want to put backups (no trailing slash) #BASE=/home/backups # eventually uncomment these if you have custom mysql commands to be root #MYSQL="mysql --defaults-file=/etc/mysql/debian.cnf" #MYSQLDUMP="mysqldump --defaults-file=/etc/mysql/debian.cnf" # uncomment to set bandwidth in KBps or other flags RSYNC="$RSYNC --bwlimit=900" # eventually uncomment to avoid any disservice (e.g. avoid systemctl stop/start) #NO_DISSERVICE=1 # eventually uncomment to test things and do nothing #PORCELAIN=1 diff --git a/rotate.sh b/rotate.sh index d05d52c..fc43250 100755 --- a/rotate.sh +++ b/rotate.sh @@ -1,124 +1,131 @@ #!/bin/bash ### # Stupid script to rotate a backup # # Author: Valerio B. # Date: Wed 4 Ago 2020 # License: CC 0 - public domain ## # do not proceed in case of errors set -e # current directory MYDIR="$(dirname "$(realpath "$0")")" +# as default don't be quiet while rotating +QUIET=0 + +# as default don't write in the log while rotating +WRITELOG=0 + # include all the stuff and useful functions . "$MYDIR"/bootstrap.sh # arguments place="$1" days="$2" max="$3" # expected file containing last timestamp last_timestamp_file="$place.timestamp" # current timestamp current_timestamp=$(date +%s) # show usage function show_help_rotate() { echo "USAGE" echo " $0 PATH DAYS MAX_ROTATIONS" echo "EXAMPLE" echo " $0 /home/backups 1 30" } # all the arguments must exist (just check the last one) if [ -z "$max" ]; then echo "Bad usage" show_help_rotate exit 1 fi # the place to be rotated must exist if [ ! -e "$place" ]; then error "unexisting directory '$place'" exit 2 fi # validate max parameter if [ "$max" -lt 2 ]; then echo "The MAX parameter must be greater than 1" show_help_rotate exit 3 fi # check if the last timestamp was writed if [ -f "$last_timestamp_file" ]; then # check the timestamp saved in the file timestamp=$(<"$last_timestamp_file") if [ "$timestamp" -lt 1000 ]; then echo "bad format in file $last_timestamp_file" exit fi # seconds spent from the last rotation diff_seconds=$(expr "$current_timestamp" - "$timestamp") # expected seconds from the last rotation before continuing # NOTE: leave the star escaped to avoid syntax error in expr expected_seconds=$(expr "$days" "*" 86400) # check if it's not passed enought time if [ "$diff_seconds" -lt "$expected_seconds" ]; then warn "Doing nothing: last rotation was executed $diff_seconds seconds ago (expected at least $expected_seconds)" exit fi fi # save the last timestamp before rotating everything # this will avoid even parallel rotations -echo $(date +%s) > "$last_timestamp_file" +echo "$(date +%s)" > "$last_timestamp_file" # eventually drop the last backup step # if it does not exist, don't care max_path="$place.$max" drop "$max_path" # shift all the backups after="$max" while [[ "$after" -gt 1 ]]; do before=$(expr "$after" - 1) # do not process the root directory for no reason in the world if you type that by mistake # the --preserve-root is already implicit but... let's be sure! asd before_path="$place.$before" after_path="$place.$after" # the source must exist. asd if [ -e "$before_path" ]; then # the trailing slash means: copy files and not just the directory move "$before_path/" "$after_path" fi # next after="$before" done # at the end, move the base forward # the trailing slash means: copy files and not just the directory copy "$place/" "$place.1" -# now you are ready to overwrite "$place" +# yeah! +log "rotation concluded"