From 8ce8ceff3f4f7776f00aa9bfa75bdc124c42c8f3 Mon Sep 17 00:00:00 2001 From: Tamas Gerczei Date: Sun, 26 Apr 2020 10:46:05 +0200 Subject: [PATCH] implemented sync mode --- zfs-backup.sh | 163 ++++++++++++++++++++++++++++---------------------- 1 file changed, 93 insertions(+), 70 deletions(-) diff --git a/zfs-backup.sh b/zfs-backup.sh index fee758b..3a4f563 100755 --- a/zfs-backup.sh +++ b/zfs-backup.sh @@ -22,18 +22,18 @@ function snapuse() { function human() { # found at http://unix.stackexchange.com/a/191787 nawk 'function human(x) { - x[1]/=1024; - if (x[1]>=1000) { x[2]++; human(x); } - } - {a[1]=$1; a[2]=0; human(a); printf "%.2f%s\n",a[1],substr("KMGTEPYZ",a[2]+1,1)}' <<< $1 + x[1]/=1024; + if (x[1]>=1000) { x[2]++; human(x); } + } + {a[1]=$1; a[2]=0; human(a); printf "%.2f%s\n",a[1],substr("KMGTEPYZ",a[2]+1,1)}' <<< $1 } function backup() { # check source check_dataset ${DATASET} || { - logger -t $(basename ${0%.sh}) -p user.notice "source dataset \"${DATASET}\" in configuration entry #${COUNTER} does not exist; omitting" - continue 2 - } + logger -t $(basename ${0%.sh}) -p user.notice "source dataset \"${DATASET}\" in configuration entry #${COUNTER} does not exist; omitting" + continue 2 + } # determine which local snapshots exist already SNAPSHOTS=( $(zfs list -rt snapshot -d1 -Ho name -S creation ${DATASET} 2>/dev/null) ) @@ -69,21 +69,24 @@ function backup() { # check target check_dataset ${SAVETO} || { - logger -t $(basename ${0%.sh}) -p user.notice "target dataset \"${SAVETO}\" in configuration entry #${COUNTER} does not exist; omitting" - continue 2 - } + logger -t $(basename ${0%.sh}) -p user.notice "target dataset \"${SAVETO}\" in configuration entry #${COUNTER} does not exist; omitting" + continue 2 + } R_SNAPSHOTS=( $(${R_RMOD} zfs list -rt snapshot -d1 -Ho name -S creation ${SAVETO}/$( basename ${DATASET}) 2>/dev/null) ) check_dataset ${SAVETO}/$( basename ${DATASET}) && R_USED_BEFORE=$(snapuse ${SAVETO}/$( basename ${DATASET})) - # determine current timestamp - DATE=$(date +%Y-%m-%d-%H%M) + if [ ${MODE:-backup} == "backup" ] + then + # determine current timestamp + DATE=$(date +%Y-%m-%d-%H%M) - # determine the name of the current snapshot to create - NEWSNAP="${DATASET}@${DATE}" - - # take a snapshot - zfs snapshot -r ${NEWSNAP} + # determine the name of the current snapshot to create + NEWSNAP="${DATASET}@${DATE}" + + # take a snapshot + zfs snapshot -r ${NEWSNAP} + fi # check if source is encrypted if [ $ENCRYPTION_FEATURE != "disabled" ] @@ -103,40 +106,50 @@ function backup() { # local snapshot(s) found SNAPMODIFIER="i ${LASTSNAP}" check_dataset ${SAVETO}/$(basename ${LASTSNAP}) || { - # last local snapshot is not available at the destination location - if [ ${#R_SNAPSHOTS[*]} -ge 1 ] - then - # remote snapshot(s) found - R_SNAPMODIFIER="I $(dirname ${DATASET})/$(basename ${R_SNAPSHOTS[*]:(-1)})" - fi - # send any previous snapshots - zfs send -Rcv${RAW_MOD}${R_SNAPMODIFIER} ${LASTSNAP} | ${RMOD:-$R_RMOD} zfs recv -Feu${RESUME_MOD}v ${SAVETO} 2>&1 >> ${LOGFILE} - } + # last local snapshot is not available at the destination location + if [ ${#R_SNAPSHOTS[*]} -ge 1 ] + then + # remote snapshot(s) found + R_SNAPMODIFIER="I $(dirname ${DATASET})/$(basename ${R_SNAPSHOTS[0]})" + fi + # send any previous snapshots + zfs send -Rcv${RAW_MOD}${R_SNAPMODIFIER} ${LASTSNAP} | ${RMOD:-$R_RMOD} zfs recv -Feu${RESUME_MOD}v ${SAVETO} 2>&1 >> ${LOGFILE} + if [[ ${PIPESTATUS[*]} =~ [1-9]+ ]] + then + # replication failure + logger -t $(basename ${0%.sh}) -p user.notice "failed to replicate ${LASTSNAP} to ${TARGET:-local}:${SAVETO}" + else + let CHANGES++ + fi + } else # ensure this does not remain in effect unset SNAPMODIFIER R_SNAPMODIFIER fi - # send backup - zfs send -Rcv${RAW_MOD}${SNAPMODIFIER} ${NEWSNAP} | ${RMOD:-$R_RMOD} zfs recv -Feu${RESUME_MOD}v ${SAVETO} 2>&1 >> ${LOGFILE} - - # if replication is unsuccessful, omit the aging check so as to prevent data loss - if [ $? -eq 0 ] + if [ ${MODE:-backup} == "backup" ] then - THRESHOLD=$(( $KEEP * 24 * 3600 )) - for SNAPSHOT in ${SNAPSHOTS[*]} - do - TIMESTAMP=$(zfs get -pHo value creation "${SNAPSHOT}") - AGE=$(( $NOW - $TIMESTAMP )) - if [ $AGE -ge $THRESHOLD ] - then - zfs destroy -r ${SNAPSHOT} 2>&1 >> ${LOGFILE} - ${RMOD:-$R_RMOD} zfs destroy -r ${SAVETO}/$(basename ${SNAPSHOT}) 2>&1 >> ${LOGFILE} - fi - done - else - MY_EXIT_CODE=$? - logger -t $(basename ${0%.sh}) -p user.notice "failed to replicate ${NEWSNAP} (rc: $MY_EXIT_CODE) to ${TARGET:-local}:${SAVETO}, no aging" + # send backup + zfs send -Rcv${RAW_MOD}${SNAPMODIFIER} ${NEWSNAP} | ${RMOD:-$R_RMOD} zfs recv -Feu${RESUME_MOD}v ${SAVETO} 2>&1 >> ${LOGFILE} + + # if replication is unsuccessful, omit the aging check so as to prevent data loss + if [[ ! ${PIPESTATUS[*]} =~ [1-9]+ ]] + then + let CHANGES++ + THRESHOLD=$(( $KEEP * 24 * 3600 )) + for SNAPSHOT in ${SNAPSHOTS[*]} + do + TIMESTAMP=$(zfs get -pHo value creation "${SNAPSHOT}") + AGE=$(( $NOW - $TIMESTAMP )) + if [ $AGE -ge $THRESHOLD ] + then + zfs destroy -r ${SNAPSHOT} 2>&1 >> ${LOGFILE} + ${RMOD:-$R_RMOD} zfs destroy -r ${SAVETO}/$(basename ${SNAPSHOT}) 2>&1 >> ${LOGFILE} + fi + done + else + logger -t $(basename ${0%.sh}) -p user.notice "failed to replicate ${NEWSNAP} to ${TARGET:-local}:${SAVETO}, no aging" + fi fi # re-evaluate snapshot data usage @@ -148,34 +161,38 @@ function backup() { R_RMOD="ssh ${USER}@${TARGET}" fi - R_USED_AFTER=$(snapuse ${SAVETO}/$( basename ${DATASET})) - L_DELTA=$(( $L_USED_AFTER - $L_USED_BEFORE )) - R_DELTA=$(( $R_USED_AFTER - ${R_USED_BEFORE:-0} )) + if [ ${CHANGES:-0} -gt 0 ] + then + # calculate data volume change + R_USED_AFTER=$(snapuse ${SAVETO}/$( basename ${DATASET})) + L_DELTA=$(( $L_USED_AFTER - $L_USED_BEFORE )) + R_DELTA=$(( $R_USED_AFTER - ${R_USED_BEFORE:-0} )) - for DELTA in L_DELTA R_DELTA - do - unset WHAT WHERE - eval _DELTA=\$$DELTA + for DELTA in L_DELTA R_DELTA + do + unset WHAT WHERE + eval _DELTA=\$$DELTA - if [ $_DELTA -lt 0 ] - then - _DELTA=$(( $_DELTA * -1 )) - WHAT="freed" - fi + if [ $_DELTA -lt 0 ] + then + _DELTA=$(( $_DELTA * -1 )) + WHAT="freed" + fi - if [[ $DELTA =~ ^L ]] - then - WHERE="$(dirname ${DATASET})" - else - WHERE="${SAVETO}" - fi + if [[ $DELTA =~ ^L ]] + then + WHERE="$(dirname ${DATASET})" + else + WHERE="${SAVETO}" + fi - if [ $_DELTA -ne 0 ] - then - _DELTA=$(human $_DELTA) - logger -t $(basename ${0%.sh}) -p user.notice "$_DELTA ${WHAT:-allocated} in \"${WHERE}\" by backing up \"${DATASET}\"" - fi - done + if [ $_DELTA -ne 0 ] + then + _DELTA=$(human $_DELTA) + logger -t $(basename ${0%.sh}) -p user.notice "$_DELTA ${WHAT:-allocated} in \"${WHERE}\" by backing up \"${DATASET}\"" + fi + done + fi # reset remote configuration unset R_RMOD RMOD RAW_MOD @@ -185,7 +202,7 @@ function backup() { #### BEGIN LOGIC #### -while getopts hf:m: OPTION +while getopts hf:m:s OPTION do case "$OPTION" in f) @@ -198,6 +215,11 @@ while getopts hf:m: OPTION RECIPIENT="$OPTARG" ;; + s) + # sync mode | this could also be a per-target setting to mitigate the need to call multiple jobs + MODE="sync" + ;; + h|\?) # display help usage @@ -280,6 +302,7 @@ while read -u 4 DATASET SAVETO KEEP ENABLED let COUNTER++ if [ ${ENABLED} == "Y" ] then + # split this function -> snapshot(), sync(), age() ? -> a new backup() would call all three backup fi done 4<<< "$(egrep -v '^(#|$)' "${CFGFILE}")" # graciously overlook any comments or blank lines