rwrapper (12275B)
1 #!/bin/bash 2 3 # TODO make some way of having this script communicate with scripts on the backup server to handle compression 4 # and pruning on the local machine rather than over an ssh connection. Do this in a non votile way 5 6 # stop on errors 7 set -e -o pipefail 8 9 # Where logs are kept 10 logfolder="/tmp/logs" 11 # How long to keep backups in destination until they are deleted 12 retention=30 13 # The log name for this execution of the script 14 logfile="$logfolder/log-$(date +"%Y-%m-%dT%H:%M:%S%z").log" 15 # Start time of script 16 start=$(date +%s.%N) 17 18 # Prints message to stdout and appends to log file 19 # Takes message as argument 20 message() { echo "$1" | tee -a "$logfile"; } 21 22 # Depending on use flags prune either remotely or locally and prune either directories or archives 23 prune() { \ 24 # Check if user specified prune 25 if [ -n "${prune+x}" ]; then 26 message "Pruning files..." 27 # if user is backing up locally and specified some sort of compression 28 if [ -n "${compress_diff+x}" ] || [ -n "${compress+x}" ] || [ -n "${encryptor_name}" ] && [ -n "${local_backup+x}" ]; then 29 message "Pruning old files on local ${dst} locally (compressed)" 30 # look for files that end in compression extension or gpg extension and delete of over retention days 31 find "${dst}" -mindepth 1 -maxdepth 2 -name '*.tar.gz' -mtime +${retention} -o -name "*.tar.gz.gpg" -mtime +${retention} -not -path "${dst}/full/*" -exec rm -rfv {} \; 2>&1 | tee -a "$logfile" 32 # if user is backing to remote server and specified some sort of compression 33 elif [ -n "${compress_diff+x}" ] || [ -n "${compress+x}" ] || [ -n "${encryptor_name}" ]; then 34 # find command same as above but does not use -exec because that does not work over ssh 35 remote_command="find ${dst#*:} -mindepth 1 -maxdepth 2 -name '*.tar.gz' -mtime +${retention} -o -name '*.tar.gz.gpg' -mtime +${retention} -not -path ${dst#*:}/full/* | xargs rm -rfv" 36 message "Pruning old files on ${dst%%:*} located at ${dst#*:} (compressed)" 37 # run the command over ssh 38 ssh "${dst%%:*}" "eval ${remote_command}" 2>&1 | tee -a "$logfile" 39 # if user is backing up locally and did not specify and compression 40 elif [ -n "${local_backup+x}" ]; then 41 message "Pruning old files on local ${dst} locally (not compressed)" 42 # look for directiories over retention 43 find "${dst}" -mindepth 1 -maxdepth 1 -type d -mtime +${retention} -not -path "${dst}/full" -exec rm -rfv {} \; 2>&1 | tee -a "$logfile" 44 # must be backing up remotley and did not specify any compression 45 else 46 # look for directiories over retention 47 remote_command="find ${dst#*:} -mindepth 1 -maxdepth 1 -type d -mtime +${retention} -not -path ${dst#*:}/full | xargs rm -rfv" 48 message "Pruning old files on ${dst%%:*} located at ${dst#*:} (not compressed)" 49 # run the command over ssh 50 ssh "${dst%%:*}" "eval ${remote_command}" 2>&1 | tee -a "$logfile" 51 fi 52 fi ;} 53 54 # Run ending messages and end with success 55 end_backup() { \ 56 message "Backup complete!" 57 message "Finished at: $(date +%Y-%m-%d-%H%M)" 58 message "Execution time: $(echo "$(date +%s.%N) - $start" | bc) seconds" 59 message "##### Rsync Backup Finished ######" 60 exit 0 61 } 62 63 # Help screen 64 info() { cat << EOF 65 rwrapper: bash wrapper script for rsync 66 Main actions: 67 -s source_dir provide source address for rsync (REQUIRED) 68 -d destination_dir provide destination address for rsync can be remote or local (REQUIRED) 69 NOTE: if local add -m flag! 70 -r remove ALL source files after transfer 71 -x compress entire source folder then sync 72 -l log_dir folder to keep logs. Default: /tmp/logs 73 -m local backup. Script wont work from local dir to local dir without this set. 74 NOTE: do not have this set when doing remote transfers! 75 -z compress incremental backups need -i as well 76 -p prune files if older than retention value 77 -t number set retention (days). Default: 30 78 -e name specify key name to encrypt backup archive with (implies -x) 79 -i Do incremental backups needed for -z to work 80 -b rate limit socket I/O bandwidth when running rsync 81 82 Examples: 83 84 ./rwrapper -s src -d fire@7.7.7.7:/home/keeper/backups -p -e fire 85 86 Compress and encrypt file, enable pruning and send over to remote host. 87 (space efficient, less likley to get corrupted data, and secure for off site backups) 88 89 ./rwrapper -s src -d fire@7.7.7.7:/home/keeper/backups -px 90 91 Same as above just don't encrypt data at rest (Best for space, less likley to get corrupted data) 92 93 ./rwrapper -s /data/src -d /two/data/dst -irzpm 94 95 Rsync files to local directory while keeping compressed incremental backups and removing source files 96 (for local to work -m MUST be specified) 97 (Most likley to have corrupted data over a remote connection because tar is being run over ssh) 98 99 ./rwrapper -s /data/src -d cool@2.2.2.2:/home/icebox 100 101 Just normal old rsync no compression or anything (good for not getting corrupted data, bad for space) 102 103 ./rwrapper -s /data/src -d cool@2.2.2.2:/home/coolbox -pi -t 100 -b 50 104 105 Set retention of (incremental backup) directories (not using compression) to 100 days and 106 set maximum transfer spped to 50KB/s 107 108 EOF 109 } 110 111 # Handles reading the users use flags and arguments entered 112 while getopts "hrnxlmzpis:d:l:t:e:b:" o; do case "${o}" in 113 s) src="${OPTARG%/}/" ;; 114 d) dst="${OPTARG%/}" ;; 115 l) logfolder="${OPTARG%/}" ;; 116 n) dryrun=True ;; 117 x) compress=True ;; 118 m) local_backup=True ;; 119 z) compress_diff=True ;; 120 r) remove=True ;; 121 t) retention="${OPTARG}" ;; 122 p) prune=True ;; 123 e) encryptor_name="${OPTARG}" ;; 124 i) incremental_backups=True ;; 125 b) rate_limit="${OPTARG}";; 126 *) info; exit 1 ;; 127 esac done 128 129 # Wherever you see 2>&1 | tee -a "$logfile" that is a log to stdout and log file 130 131 # Check for no options 132 [ $OPTIND -eq 1 ] && info && exit 1 133 134 message "##### Rsync Backup Started ######" 135 message "Started at: $(date +%Y-%m-%d-%H%M)" 136 137 # check of source and destinations fields are entered 138 # rsync and other programs handle the errors from this point on 139 [ -z "$src" ] && message 'Missing source directory (missing flag -s) Exiting ...' >&2 && exit 1 140 [ -z "$dst" ] && message 'Missing destination directory (missing flag -d) Exiting ...' >&2 && exit 1 141 142 # Rsync config 143 # -a archive mode save most attributes of a file including permissions 144 # -P show progress of file, good if you are watching in the terminal 145 # -v verbose 146 # -h times in human readable format 147 # -i show transferring files in list format 148 # -O dont worry about changing directory times (without this rsync will throw a minor error about not being able to change directory times unless user owns file) 149 # -A preseve acls 150 # -X preserver xattrs 151 # -z compress when transferring 152 # --delete delete extraneous files from the receiving side (this is mostly to keep an up to date exact copy of the full directory) 153 # --stats display and log stats after execution 154 # --exclude exclude files maching pattern 155 OPT="-aPvhiOAXz --delete --stats --exclude='*.incomplete'" 156 # if user specifies -r remove source files after transfer successful to destination 157 [ -n "${remove+x}" ] && OPT="$OPT --remove-source-files" 158 # run in dryrun mode if specified 159 [ -n "${dryrun+x}" ] && OPT="$OPT -n" 160 # rsync will log to this programs log file 161 LOG="--log-file=$logfile" 162 # Keep the date dor incremental backups 163 DATE=$(date +%Y-%m-%d-%H%M) 164 # do incremental backups if user specified -i 165 BACKUP="" 166 [ -n "${incremental_backups}" ] && BACKUP="--backup-dir=../$DATE/" 167 # allow limiting of I/O socket bandwidth (read rsync man page for more on this option) 168 RATE="" 169 [ -n "${rate_limit}" ] && RATE="--bwlimit=${rate_limit}" 170 171 # concatenated rsync command. Notice destination goes to full directory thats so delete doesnt remove our 172 # incremental backups or other backups 173 command="rsync ${OPT} ${LOG} ${RATE} ${src} ${dst}/full/ ${BACKUP}" 174 # Rsync Notes: 175 # 176 # About checksums (from man page): 177 # Note that rsync always verifies that each transferred file was correctly reconstructed on the receiving side 178 # by checking a whole-file checksum that is generated as the file is transferred, but that automatic after-the-transfer 179 # verification has nothing to do with this option's before-the-transfer "Does this file need to be updated?" check. 180 # 181 # Sync transfer is encrypted as it happens through SSH, unles rsync is daemonized, but thats why this script exists 182 183 # Compress entire source directory (-x or -e) 184 if [ -n "${compress+x}" ] || [ -n "${encryptor_name+x}" ]; then 185 # backup a directory up from the source directory just to avoid any problems 186 backupfile="${src}../full-backup-$(date +%Y-%m-%d-%H%M).tar.gz" 187 # if a local backup the archive can just be made in the destiantion directory 188 [ -n "${local_backup+x}" ] && backupfile="$dst/full-backup-$(date +%Y-%m-%d-%H%M).tar.gz" 189 # dry run will just make the archive command do nothing 190 [ -n "${dryrun+x}" ] && backupfile="/dev/null" 191 message "Compressing entire ${src} directory to ${backupfile}" 192 # compress entire archive. Excluding .incomplete files keeping all permissions (--acls --xattrs -p) 193 # --totals just logs the total size after compressed 194 tar --exclude '*.incomplete' --acls --xattrs -cpzf "${backupfile}" --totals "${src}" 2>&1 | tee -a "$logfile" 195 # check if we are encrypting if we are encrypt with specified public key and remove unencrypted archive 196 [ -n "${encryptor_name+x}" ] && message "Encrypting archive with ${encryptor_name} key" \ 197 && gpg -e --batch --yes --no-symkey-cache -r "${encryptor_name}" "${backupfile}" 2>&1 | tee -a "$logfile" \ 198 && rm -fv "${backupfile}" 2>&1 | tee -a "$logfile" && backupfile="${backupfile}.gpg" 199 # if user wants to remove source directory contents do so now we are done backing it up 200 [ -n "${remove}" ] && message "Removing source directory" && rm -rfv "${src}"* 2>&1 | tee -a "$logfile" 201 # local has no need for rsync, just end 202 [ -n "${local_backup+x}" ] && prune && end_backup 203 # rsync the archive to the remote server 204 # change destination for rsync command and only sync the file 205 # that way other files arent deleted by --delete option 206 command="rsync ${OPT} ${LOG} ${RATE} ${backupfile} ${dst}" 207 # since we just moving one file from outside the source directory lets get rsync to remove it when its done tranferring 208 [ -z "${BACKUP}" ] && command="rsync ${OPT} --remove-source-files ${LOG} ${RATE} ${backupfile} ${dst}" 209 fi 210 211 # Run rsync 212 message "----- RSYNC START -----" 213 message "Running: $command" 214 # This loop will just keep going if rsync gets network errors as that will likley be resolved with time 215 # if it succeeds the script continues if it doesnt it exits with error 216 while true; do 217 eval "${command}" 2>&1 | tee -a "$logfile" 218 exit_code=$? 219 case $exit_code in 220 10|23|30|35) message "${DATE}: rsync failed with a network type error: ${exit_code}!" && DATE=$(date +%Y-%m-%d-%H%M) ;; 221 0) message "${DATE}: rsync had no errors!" && break ;; 222 *) message "${DATE}: rsync failed with unexpected error: ${exit_code}!" && exit 1 ;; 223 esac done 224 message "----- RSYNC END -----" 225 226 # Compress incremental backups created by rsync with the --backup command (-z and -i) 227 if [ -n "${incremental_backups}" ]; then 228 if [ -n "${compress_diff+x}" ] && [ -z "${local_backup+x}" ]; then 229 # Compress command to run on the remote machine 230 # {dst#*:} takes data after the semi-colon i.e the directory 231 # {dst%%:*} takes data before the semi colon i.e the host 232 remote_command="[ -d '${dst#*:}/${DATE}' ] && tar --acls --xattrs -cpzf ${dst#*:}/${DATE}.tar.gz --totals ${dst#*:}/${DATE} && rm -rf ${dst#*:}/${DATE:?} || :" 233 message "Compressing incremental backups on ${dst%%:*} located at ${dst#*:}" 234 # Run the remote command over ssh 235 ssh "${dst%%:*}" "eval ${remote_command}" 2>&1 | tee -a "$logfile" 236 elif [ -n "${compress_diff+x}" ] && [ -n "${local_backup+x}" ] && [ -d "${dst}/${DATE}" ]; then 237 message "Compressing incremental backups at ${dst} locally" 238 # Just archive the incremental backup locally 239 tar --acls --xattrs -cpzf "${dst}/${DATE}.tar.gz" --totals "${dst}/${DATE}" && rm -rf "${dst:?}/${DATE:?}" 2>&1 | tee -a "$logfile" 240 fi 241 fi 242 243 # files all transferred clean up 244 prune 245 end_backup 246 247 # To decrypt if specified -e 248 # 1. Be on machine that has the key that matches the name you used to encrypt 249 # 2. run: gpg -d --output <outfile> <infile> 250 # This will decrypt with the private key as we encrypt with the public key 251 # To make gpg key run gpg --full-generate-key