diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..3ceed58 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,7 @@ +Copyright © 2022 Valerio Bozzolan, ER Informatica + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index e66c679..991e00a 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,30 @@ # Micro Backup script This is a keep it simple and stupid backup script. ## Installation ``` git clone this_repo_url ``` ## Configuration Enter in the cloned directory, then: ``` cp backup-instructions-example.conf backup-instructions.conf cp options-example.conf options.conf editor options.conf editor backup-instructions.conf ``` +## License + +ER Informatica, Valerio Bozzolan, MIT License + ## Usage ``` ./backup-everything.sh ``` diff --git a/backup-everything.sh b/backup-everything.sh index 28c5a59..0d9d93e 100755 --- a/backup-everything.sh +++ b/backup-everything.sh @@ -1,575 +1,574 @@ #!/bin/bash ### # Micro script to backup some stuff # # IMPORTANT: do not edit this file # and see 'backup-instructions.conf' instead! # # This file defines some functions and then it executes # your 'backup-instructions.conf' # -# Author: Valerio Bozzolan -# Date: Wed 25 Mar 2020 -# License: CC 0 - public domain +# Author: 2020, 2021 Valerio Bozzolan +# License: MIT ## # try to proceed even in case of errors (to do not skip any backup) # set -e # current directory MYDIR="$(dirname "$(realpath "$0")")" # load useful stuff . "$MYDIR"/bootstrap.sh ### # Create a directory with safe permissions if it does not exist # # @param string path Pathname # @param int is_file_mode Check if we are in file mode (default: directory mode) # write_safe_dir_if_unexisting() { local path="$1" local is_file_mode="$2" # try to create the directory if it does not exists if [ ! -d "$path" ]; then warn "creating missing $path with 750" mkdir --parents "$path" chmod 750 "$path" fi # it must exists now require_existing_dir "$path" } ### # Create a directory for a filename with safe permissions if it does not exist # # @param string path Pathname to a filename # @param int is_file_mode Check if we are in file mode (default: directory mode) # write_safe_basedir_if_unexisting() { # first function argument local path="$1" # extract just the sub-directory local basepath=$(dirname "$path") # create that sub-directory write_safe_dir_if_unexisting "$basepath" } ### # Require an existing directory or die # # @param string Directory path # require_existing_dir() { if [ ! -d "$1" ]; then error "unexisting directory $1" exit 1 fi } # create these pathnames if they do not exist write_safe_dir_if_unexisting "$BASE" write_safe_dir_if_unexisting "$DAILY" write_safe_dir_if_unexisting "$DAILY_FILES" write_safe_dir_if_unexisting "$DAILY_DATABASES" # create or clean the last log file if [ "$WRITELOG" = 1 ]; then cat /dev/null > "$DAILY_LASTLOG" fi # # Dump and compress a database # # @param db string Database name # @param spaces string Cosmetic spaces # backup_database() { local db="$1" local spaces="$2" local path="$DAILY_DATABASES"/"$db".sql.gz log "$spaces""dumping database $db" # no database no party if ! $MYSQL "$db" -e exit > /dev/null 2>&1; then warn "$spaces skip unexisting database" return fi if [ "$PORCELAIN" != 1 ]; then $MYSQLDUMP "$db" | gzip > "$path" fi } # # Dump and compress some databases # # @param database... string Database names # backup_databases() { for database in $@; do backup_database "$database" done } # # Dump some databases with a shared prefix # # @param prefix string Database prefix # @param spaces string Cosmetic spaces # backup_databases_prefixed() { local prefix="$1" local spaces="$2" log "$spaces""backup all databases prefixed with: $prefix" databases=$($MYSQL -e 'SHOW DATABASES' | grep "$prefix") for database in $databases; do backup_database "$database" "$spaces " done } # default databases to be skipped DATABASE_SKIP_LIST=("^information_schema$" "^performance_schema$") # # Call this to skip a database # # @param string database Database pattern (Bash regex) # # This should be called before calling backup_every_database. # skip_database() { local db="$1" DATABASE_SKIP_LIST+=( "$db" ) } # # Backup every single database. That's easy. # backup_every_database() { # backup every damn database databases=$($MYSQL -e 'SHOW DATABASES') for database in $databases; do local do_backup=1 # just skip the information_schema and the performance_schema that cannot be locked for skip_entry in "${DATABASE_SKIP_LIST[@]}"; do if [[ "$database" =~ $skip_entry ]]; then # show a cute message log "skippin database $database matching blacklist" # do not backup this do_backup=0 # do not check other entries break fi done # backup if it does not match the above skip entries if [ "$do_backup" = 1 ]; then backup_database "$database" fi done } # # Backup a database that is used by a service # # The service will be stopped before dumping. # # @param service string Systemd unit file # @param db string Database name # backup_service_and_database() { local service="$1" local db="$2" local is_active= log "start backup service '$service' with database '$db'" # check if the service is running if systemctl is-active "$service" > /dev/null; then is_active=1 fi # eventually stop the service if [ $is_active == 1 ]; then if [ "$NO_DISSERVICE" == 1 ]; then warn " NOT stopping service: $service (to avoid any disservice)" else log " stopping service: $service" if [ "$PORCELAIN" != 1 ]; then systemctl stop "$service" fi fi else log " service already inactive: $service" fi # backup the database now that the service is down backup_database "$db" " " # eventually start again the service if [ $is_active == 1 ]; then if [ "$NO_DISSERVICE" == 1 ]; then warn " NOT starting again service: $service (to avoid any disservice)" else log " starting again service: $service" if [ "$PORCELAIN" != 1 ]; then systemctl start "$service" fi fi fi } # # Backup some databases used by a service # # The service will be stopped before dumping the databases # # @param service string Systemd unit file # @param db... string Database names # backup_service_and_databases() { backup_service_and_database $@ } # # Backup phabricator # # @param path string Phabricator webroot # @param dbprefix string Database prefix # backup_phabricator() { local path="$1" local dbprefix="$2" local configset="$path"/bin/config log "backup Phabricator databases" log " set Phabricator read only mode (and wait some time to make it effective)" $configset set cluster.read-only true > /dev/null sleep 5 backup_databases_prefixed "$dbprefix" " " log " revert Phabricator read only mode" $configset set cluster.read-only false > /dev/null } # # Backup a directory or a filename # # @param path string Pathname to be backupped # @param identifier string Optional identifier of this pathname # backup_path() { local path="$1" local identifier="$2" # create a default identifier if [ -z "$identifier" ]; then identifier="$path" fi # destination directory # note that the identifier may start with a slash but this is good, just don't care dest="$DAILY_FILES/$identifier" # tell were the backup will go if [ "$path" = "$identifier" ]; then log "backup $path" else log "backup $path with identifier $identifier" fi # check if the path exists if [ -e "$path" ]; then # check if it's a directory if [ -d "$path" ]; then # this is a directory # eventually create the destination if it does not exist write_safe_dir_if_unexisting "$dest" # backup this # force the source to end with a slash in order to copy the files inside the directory $RSYNC "$path/" "$dest" else # this is a filename # eventually create the base destination if it does not exist write_safe_basedir_if_unexisting "$dest" # backup this filename $RSYNC "$path" "$dest" fi else # no path no party warn " path not found: $path" fi } # # Backup last lines of a filename # # @param path string Pathname to be backupped # @param identifier string Optional identifier of this pathname (it's considered a file) # @param lines string Lines to be saved # backup_last_log_lines() { local path="$1" local identifier="$2" local lines="$3" # create a default identifier if [ -z "$identifier" ]; then identifier="$path" fi # default lines if [ -z "$lines" ]; then lines="$BACKUP_LAST_LOG_LINES" fi # destination directory # note that the identifier may start with a slash but this is good, just don't care dest="$DAILY_FILES/$identifier" # tell were the backup will go if [ "$path" = "$identifier" ]; then log "backup $path last $lines log lines" else log "backup $path last $lines log lines with identifier $identifier" fi # check if the path exists if [ -e "$path" ]; then # this is a filename # eventually create the base destination if it does not exist write_safe_basedir_if_unexisting "$dest" # backup this file tail -n "$lines" "$path" > "$dest" else # no path no party warn " path not found: $path" fi } # # Backup some filenames ending with whatever # # @param path[...] string Pathname to be backupped # backup_paths() { # pathnames local paths="$@" # process every pathname for path in $paths; do # backup every single pathname backup_path "$path" done } ## # Validate the backup host arguments # # validate_backup_host_args() { # function parameters local port="$3" # no host no party if [ -z "$1" ] || [ -z "$2" ]; then error "Bad usage: backup_host REMOTE_HOST:REMOTE_PATH IDENTIFIER [PORT]" exit 1 fi # tell what we will do if [ "$port" = "22" ]; then log "backup $1 in $2" else log "backup $1 (:$port) in $2" fi } ## # Backup another host directory via rsync # # @param string Rsync host # @param string Rsync path to a directory # @param string Rsync host port # backup_host_dir() { # function parameters local hostlink="$1" local identifier="$2" local port="$3" # eventually set default rsync port if [ -z "$port" ]; then port=22 fi # validate the arguments validate_backup_host_args "$hostlink" "$identifier" "$port" # destination path local dest="$BASE/$identifier" # create the destination if it does not exist write_safe_dir_if_unexisting "$dest" # backup everything # force the source to end with a slash to just copy the files inside it $RSYNC_REMOTE --rsh="ssh -p $port" "$hostlink/" "$dest" } ## # Backup another host directory via rsync # # @param string Rsync host # @param string Rsync path to a directory # push_path_host() { # function parameters local source="$1" local hostlink="$2" local port="$3" # eventually set default rsync port if [ -z "$port" ]; then port=22 fi # validate the arguments and print message validate_backup_host_args "$source" "$hostlink" "$port" # backup everything # force the source to end with a slash to just copy the files inside it $RSYNC_REMOTE --rsh="ssh -p $port" "$source" "$hostlink" } ## # Push some pathnames into an host # # @param string Rsync host # @param string Rsync port # @param string... Rsync path to some directories # push_host_port_paths() { # function parameters local hostlink="$1" local port="$2" # strip the first two arguments shift shift local path="" # process every pathname while [[ $# -gt 0 ]]; do path="$1" shift # backup every single pathname push_path_host "$path" "$hostlink" "$port" done } ## # Push the local "daily/" directory into a remote host # # @param string Rsync host # @param string Rsync path to a directory # push_daily_directory() { # function parameters local hostlink="$1" local port="$2" push_path_host "$DAILY/" "$hostlink" "$port" } ## # Backup another host file via rsync # # @param string hostlink Rsync source host and pathname e.g. ravotti94:/tmp/asd.txt # @param string identifier Rsync destination directory # backup_host_file() { # function parameters local hostlink="$1" local identifier="$2" local port="$3" # filename local filename=$(basename "$hostlink") # eventually set default rsync port if [ -z "$port" ]; then port=22 fi # validate the arguments validate_backup_host_args "$hostlink" "$identifier" "$port" # destination path local dest="$BASE/$identifier/$filename" # create the destination directory if it does not exist write_safe_basedir_if_unexisting "$dest" # backup everything # force the source to end with a slash to just copy the files inside it $RSYNC_REMOTE --rsh="ssh -p $port" "$hostlink" "$dest" } log "init backup in $DAILY" . "$INSTRUCTIONS" log "backup successfully concluded" # remember the last successfully concluded backup timestamp date +%s > "$DAILY_LASTTIME" diff --git a/bootstrap.sh b/bootstrap.sh index 0ecf18a..76d39fd 100644 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,223 +1,222 @@ #!/bin/bash ### # Part of a stupid script to backup some stuff # # This bootstrap.sh file does nothing by itself but loads useful stuff. # # This file is loaded from 'backup-everything.sh' or 'rotate.sh' # -# Author: Valerio B. -# Date: Wed 5 ago 2020 -# License: CC 0 - public domain +# Author: 2020, 2021 Valerio Bozzolan +# License: MIT ## # current directory DIR="${BASH_SOURCE%/*}" if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi # check if the standard input is not a terminal INTERACTIVE= if [ -t 0 ]; then INTERACTIVE=1 fi # # Check if this is the quiet mode # # Default - not quite. # # Actually we are in quiet mode if it's not interactive. # This lazy behavior is to avoid stupid emails from the crontab # without the need to specify some --quiet etc. # Note that in quiet mode only WARN and ERROR messages are shown. # I've not created a --quiet flag because nobody is needing it. # # Edit your options - do not edit here. # #QUIET= # # Eventually write a log file # # Default - don't write. # # Edit your options - do not edit here. # #WRITELOG= # path to the instructions file INSTRUCTIONS="$DIR/backup-instructions.conf" # path to the configuration file CONFIG="$DIR/options.conf" # no config no party if [ ! -f $CONFIG ]; then echo "missing options expected in $CONFIG" exit 1 fi # no instructions no party if [ ! -f $INSTRUCTIONS ]; then echo "missing instructions expected in $INSTRUCTIONS" exit 1 fi # default mysql commands # --batch: avoid fancy columns (auto-enabled, but better to specify it) # --silent: avoid the column name to be included MYSQL="mysql --batch --silent" MYSQLDUMP="mysqldump" # default rsync command # --archive: Try to keep all the properties # --fuzzy: Try to check if a file was renamed instead of delete and download a new one # It's efficient for example with log rotated files. # --delete: Delete the destination files if not present in the source # NOTE: we want this behaviour but it's not a good idea toghether with --fuzzy # that's why we do not use --delete but we use the next flags # --delay-updates Put all updated files into place at end (useful with fuzzy and delete modes) # --delete-delay Delete after everything (useful with fuzzy and delete modes) # --hard-links Try to look for hard links during the transfer to do not copy separate files RSYNC="rsync --archive --fuzzy --delay-updates --delete-delay --hard-links" # rsync used in remote transfers # --compress Use more CPU to save network bandwidth RSYNC_REMOTE="$RSYNC --compress" # default base backup directory for all backups BASE="/home/backups" # default box name BOX="$(hostname)" # set to 1 to avoid any disservice (e.g. systemctl stop/start) NO_DISSERVICE= # set to 1 to do nothing PORCELAIN= # include the configuration to eventually override some options . "$CONFIG" # as default, if not interactive, set quite mode if [ -z "$QUIET" ] && [ "$INTERACTIVE" != 1 ]; then QUIET=1 fi # as default, if quiet, write in the log if [ -z "$WRITELOG" ] && [ "$QUIET" = 1 ]; then WRITELOG=1 fi # full pathnames to the backup directories BASEBOX="$BASE/$BOX" DAILY="$BASEBOX/daily" DAILY_FILES="$DAILY/files" DAILY_DATABASES="$DAILY/databases" DAILY_LASTLOG="$DAILY/last.log" DAILY_LASTTIME="$DAILY/last.timestamp" # apply the porcelain to the rsync command if [ "$PORCELAIN" = 1 ]; then RSYNC="$RSYNC --dry-run" RSYNC_REMOTE="$RSYNC_REMOTE --dry-run" fi # set default backup_last_log() lines if [ -z "$BACKUP_LAST_LOG_LINES" ]; then BACKUP_LAST_LOG_LINES=2000 fi ### # Print something # # It also put the message in the backup directory # # @param string severity # @param string message # function printthis() { local msg="[$(date)][$1] $2" # print to standard output if it's not in quiet mode if [ "$QUIET" != 1 ]; then echo "$msg" fi # put in the log file if possible if [ -f "$DAILY_LASTLOG" ] && [ "$WRITELOG" = 1 ]; then echo "$msg" >> "$DAILY_LASTLOG" fi } ### # Run an rsync # function copy() { # show what we are doing log "copy $*" # run the rsync command if [ "$PORCELAIN" != 1 ]; then $RSYNC $@ fi } ### # Remove a pathname # function drop() { # show what we are doing log "drop $*" # well, proceed... finger crossed... with some protections if [ "$PORCELAIN" != 1 ]; then rm --recursive --force --one-file-system --preserve-root -- $@ fi } ### # Move something somewhere # function move() { # show what we are doing log "move $*" if [ "$PORCELAIN" != 1 ]; then mv --force $@ fi } ### # Print a information message # # @param msg Message # function log() { printthis INFO "$1" } ### # Print a warning message # # @param msg Message # function warn() { printthis WARN "$1" } ### # Print an error message # # @param msg Message # function error() { printthis ERROR "$1" }