commit 2fa4572b20835518a8a2697811a2f1a019885552 Author: q <diskofgrain.xyz> Date: Sun, 25 Sep 2022 10:37:44 -0400 added script Diffstat:
| A | rwrapper | | | 251 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
1 file changed, 251 insertions(+), 0 deletions(-)
diff --git a/rwrapper b/rwrapper @@ -0,0 +1,250 @@ +#!/bin/bash + +# TODO make some way of having this script communicate with scripts on the backup server to handle compression +# and pruning on the local machine rather than over an ssh connection. Do this in a non votile way + +# stop on errors +set -e -o pipefail + +# Where logs are kept +logfolder="/tmp/logs" +# How long to keep backups in destination until they are deleted +retention=30 +# The log name for this execution of the script +logfile="$logfolder/log-$(date +"%Y-%m-%dT%H:%M:%S%z").log" +# Start time of script +start=$(date +%s.%N) + +# Prints message to stdout and appends to log file +# Takes message as argument +message() { echo "$1" | tee -a "$logfile"; } + +# Depending on use flags prune either remotely or locally and prune either directories or archives +prune() { \ + # Check if user specified prune + if [ -n "${prune+x}" ]; then + message "Pruning files..." + # if user is backing up locally and specified some sort of compression + if [ -n "${compress_diff+x}" ] || [ -n "${compress+x}" ] || [ -n "${encryptor_name}" ] && [ -n "${local_backup+x}" ]; then + message "Pruning old files on local ${dst} locally (compressed)" + # look for files that end in compression extension or gpg extension and delete of over retention days + find "${dst}" -mindepth 1 -maxdepth 2 -name '*.tar.gz' -mtime +${retention} -o -name "*.tar.gz.gpg" -mtime +${retention} -not -path "${dst}/full/*" -exec rm -rfv {} \; 2>&1 | tee -a "$logfile" + # if user is backing to remote server and specified some sort of compression + elif [ -n "${compress_diff+x}" ] || [ -n "${compress+x}" ] || [ -n "${encryptor_name}" ]; then + # find command same as above but does not use -exec because that does not work over ssh + remote_command="find ${dst#*:} -mindepth 1 -maxdepth 2 -name '*.tar.gz' -mtime +${retention} -o -name '*.tar.gz.gpg' -mtime +${retention} -not -path ${dst#*:}/full/* | xargs rm -rfv" + message "Pruning old files on ${dst%%:*} located at ${dst#*:} (compressed)" + # run the command over ssh + ssh "${dst%%:*}" "eval ${remote_command}" 2>&1 | tee -a "$logfile" + # if user is backing up locally and did not specify and compression + elif [ -n "${local_backup+x}" ]; then + message "Pruning old files on local ${dst} locally (not compressed)" + # look for directiories over retention + find "${dst}" -mindepth 1 -maxdepth 1 -type d -mtime +${retention} -not -path "${dst}/full" -exec rm -rfv {} \; 2>&1 | tee -a "$logfile" + # must be backing up remotley and did not specify any compression + else + # look for directiories over retention + remote_command="find ${dst#*:} -mindepth 1 -maxdepth 1 -type d -mtime +${retention} -not -path ${dst#*:}/full | xargs rm -rfv" + message "Pruning old files on ${dst%%:*} located at ${dst#*:} (not compressed)" + # run the command over ssh + ssh "${dst%%:*}" "eval ${remote_command}" 2>&1 | tee -a "$logfile" + fi + fi ;} + +# Run ending messages and end with success +end_backup() { \ + message "Backup complete!" + message "Finished at: $(date +%Y-%m-%d-%H%M)" + message "Execution time: $(echo "$(date +%s.%N) - $start" | bc) seconds" + message "##### Rsync Backup Finished ######" + exit 0 +} + +# Help screen +info() { cat << EOF +rwrapper: bash wrapper script for rsync +Main actions: + -s source_dir provide source address for rsync (REQUIRED) + -d destination_dir provide destination address for rsync can be remote or local (REQUIRED) + NOTE: if local add -m flag! + -r remove ALL source files after transfer + -x compress entire source folder then sync + -l log_dir folder to keep logs. Default: /tmp/logs + -m local backup. Script wont work from local dir to local dir without this set. + NOTE: do not have this set when doing remote transfers! + -z compress incremental backups need -i as well + -p prune files if older than retention value + -t number set retention (days). Default: 30 + -e name specify key name to encrypt backup archive with (implies -x) + -i Do incremental backups needed for -z to work + -b rate limit socket I/O bandwidth when running rsync + +Examples: + + ./rwrapper -s src -d fire@7.7.7.7:/home/keeper/backups -p -e fire + +Compress and encrypt file, enable pruning and send over to remote host. +(space efficient, less likley to get corrupted data, and secure for off site backups) + + ./rwrapper -s src -d fire@7.7.7.7:/home/keeper/backups -px + +Same as above just don't encrypt data at rest (Best for space, less likley to get corrupted data) + + ./rwrapper -s /data/src -d /two/data/dst -irzpm + +Rsync files to local directory while keeping compressed incremental backups and removing source files +(for local to work -m MUST be specified) +(Most likley to have corrupted data over a remote connection because tar is being run over ssh) + + ./rwrapper -s /data/src -d cool@2.2.2.2:/home/icebox + +Just normal old rsync no compression or anything (good for not getting corrupted data, bad for space) + + ./rwrapper -s /data/src -d cool@2.2.2.2:/home/coolbox -pi -t 100 -b 50 + +Set retention of (incremental backup) directories (not using compression) to 100 days and +set maximum transfer spped to 50KB/s + +EOF +} + +# Handles reading the users use flags and arguments entered +while getopts "hrnxlmzpis:d:l:t:e:b:" o; do case "${o}" in + s) src="${OPTARG%/}/" ;; + d) dst="${OPTARG%/}" ;; + l) logfolder="${OPTARG%/}" ;; + n) dryrun=True ;; + x) compress=True ;; + m) local_backup=True ;; + z) compress_diff=True ;; + r) remove=True ;; + t) retention="${OPTARG}" ;; + p) prune=True ;; + e) encryptor_name="${OPTARG}" ;; + i) incremental_backups=True ;; + b) rate_limit="${OPTARG}";; + *) info; exit 1 ;; +esac done + +# Wherever you see 2>&1 | tee -a "$logfile" that is a log to stdout and log file + +# Check for no options +[ $OPTIND -eq 1 ] && info && exit 1 + +message "##### Rsync Backup Started ######" +message "Started at: $(date +%Y-%m-%d-%H%M)" + +# check of source and destinations fields are entered +# rsync and other programs handle the errors from this point on +[ -z "$src" ] && message 'Missing source directory (missing flag -s) Exiting ...' >&2 && exit 1 +[ -z "$dst" ] && message 'Missing destination directory (missing flag -d) Exiting ...' >&2 && exit 1 + +# Rsync config +# -a archive mode save most attributes of a file including permissions +# -P show progress of file, good if you are watching in the terminal +# -v verbose +# -h times in human readable format +# -i show transferring files in list format +# -O dont worry about changing directory times (without this rsync will throw a minor error about not being able to change directory times unless user owns file) +# -A preseve acls +# -X preserver xattrs +# -z compress when transferring +# --delete delete extraneous files from the receiving side (this is mostly to keep an up to date exact copy of the full directory) +# --stats display and log stats after execution +# --exclude exclude files maching pattern +OPT="-aPvhiOAXz --delete --stats --exclude='*.incomplete'" +# if user specifies -r remove source files after transfer successful to destination +[ -n "${remove+x}" ] && OPT="$OPT --remove-source-files" +# run in dryrun mode if specified +[ -n "${dryrun+x}" ] && OPT="$OPT -n" +# rsync will log to this programs log file +LOG="--log-file=$logfile" +# Keep the date dor incremental backups +DATE=$(date +%Y-%m-%d-%H%M) +# do incremental backups if user specified -i +BACKUP="" +[ -n "${incremental_backups}" ] && BACKUP="--backup-dir=../$DATE/" +# allow limiting of I/O socket bandwidth (read rsync man page for more on this option) +RATE="" +[ -n "${rate_limit}" ] && RATE="--bwlimit=${rate_limit}" + +# concatenated rsync command. Notice destination goes to full directory thats so delete doesnt remove our +# incremental backups or other backups +command="rsync ${OPT} ${LOG} ${RATE} ${src} ${dst}/full/ ${BACKUP}" +# Rsync Notes: +# +# About checksums (from man page): +# Note that rsync always verifies that each transferred file was correctly reconstructed on the receiving side +# by checking a whole-file checksum that is generated as the file is transferred, but that automatic after-the-transfer +# verification has nothing to do with this option's before-the-transfer "Does this file need to be updated?" check. +# +# Sync transfer is encrypted as it happens through SSH, unles rsync is daemonized, but thats why this script exists + +# Compress entire source directory (-x or -e) +if [ -n "${compress+x}" ] || [ -n "${encryptor_name+x}" ]; then + # backup a directory up from the source directory just to avoid any problems + backupfile="${src}../full-backup-$(date +%Y-%m-%d-%H%M).tar.gz" + # if a local backup the archive can just be made in the destiantion directory + [ -n "${local_backup+x}" ] && backupfile="$dst/full-backup-$(date +%Y-%m-%d-%H%M).tar.gz" + # dry run will just make the archive command do nothing + [ -n "${dryrun+x}" ] && backupfile="/dev/null" + message "Compressing entire ${src} directory to ${backupfile}" + # compress entire archive. Excluding .incomplete files keeping all permissions (--acls --xattrs -p) + # --totals just logs the total size after compressed + tar --exclude '*.incomplete' --acls --xattrs -cpzf "${backupfile}" --totals "${src}" 2>&1 | tee -a "$logfile" + # check if we are encrypting if we are encrypt with specified public key and remove unencrypted archive + [ -n "${encryptor_name+x}" ] && message "Encrypting archive with ${encryptor_name} key" \ + && gpg -e --batch --yes --no-symkey-cache -r "${encryptor_name}" "${backupfile}" 2>&1 | tee -a "$logfile" \ + && rm -fv "${backupfile}" 2>&1 | tee -a "$logfile" && backupfile="${backupfile}.gpg" + # if user wants to remove source directory contents do so now we are done backing it up + [ -n "${remove}" ] && message "Removing source directory" && rm -rfv "${src}"* 2>&1 | tee -a "$logfile" + # local has no need for rsync, just end + [ -n "${local_backup+x}" ] && prune && end_backup + # rsync the archive to the remote server + # change destination for rsync command and only sync the file + # that way other files arent deleted by --delete option + command="rsync ${OPT} ${LOG} ${RATE} ${backupfile} ${dst}" + # since we just moving one file from outside the source directory lets get rsync to remove it when its done tranferring + [ -z "${BACKUP}" ] && command="rsync ${OPT} --remove-source-files ${LOG} ${RATE} ${backupfile} ${dst}" +fi + +# Run rsync +message "----- RSYNC START -----" +message "Running: $command" +# This loop will just keep going if rsync gets network errors as that will likley be resolved with time +# if it succeeds the script continues if it doesnt it exits with error +while true; do + eval "${command}" 2>&1 | tee -a "$logfile" + exit_code=$? + case $exit_code in + 10|23|30|35) message "${DATE}: rsync failed with a network type error: ${exit_code}!" && DATE=$(date +%Y-%m-%d-%H%M) ;; + 0) message "${DATE}: rsync had no errors!" && break ;; + *) message "${DATE}: rsync failed with unexpected error: ${exit_code}!" && exit 1 ;; +esac done +message "----- RSYNC END -----" + +# Compress incremental backups created by rsync with the --backup command (-z and -i) +if [ -n "${incremental_backups}" ]; then + if [ -n "${compress_diff+x}" ] && [ -z "${local_backup+x}" ]; then + # Compress command to run on the remote machine + # {dst#*:} takes data after the semi-colon i.e the directory + # {dst%%:*} takes data before the semi colon i.e the host + remote_command="[ -d '${dst#*:}/${DATE}' ] && tar --acls --xattrs -cpzf ${dst#*:}/${DATE}.tar.gz --totals ${dst#*:}/${DATE} && rm -rf ${dst#*:}/${DATE:?} || :" + message "Compressing incremental backups on ${dst%%:*} located at ${dst#*:}" + # Run the remote command over ssh + ssh "${dst%%:*}" "eval ${remote_command}" 2>&1 | tee -a "$logfile" + elif [ -n "${compress_diff+x}" ] && [ -n "${local_backup+x}" ] && [ -d "${dst}/${DATE}" ]; then + message "Compressing incremental backups at ${dst} locally" + # Just archive the incremental backup locally + tar --acls --xattrs -cpzf "${dst}/${DATE}.tar.gz" --totals "${dst}/${DATE}" && rm -rf "${dst:?}/${DATE:?}" 2>&1 | tee -a "$logfile" + fi +fi + +# files all transferred clean up +prune +end_backup + +# To decrypt if specified -e +# 1. Be on machine that has the key that matches the name you used to encrypt +# 2. run: gpg -d --output <outfile> <infile> +# This will decrypt with the private key as we encrypt with the public key+ \ No newline at end of file