home-brew solution for ZFS snapshot replication and rotation
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

317 lines
8.5 KiB

  1. #!/usr/bin/env bash
  2. # create and rotate ZFS backups | 150415 | tamas@gerczei.eu
  3. #### BEGIN FUNCTIONS ####
  4. function usage() {
  5. # print synopsis
  6. printf "\n-=[ ZFS back-up utility ]=-\n\n\
  7. usage: $(basename $0) [-h] [-m <e-mail_address>] -f <fully_qualified_configuration_file_name>\n\n\
  8. A configuration (f)ile must be supplied. An e-(m)ail address for reporting is optional.\n"
  9. }
  10. function check_dataset() {
  11. ${R_RMOD} zfs get -pHo value creation ${1} &>/dev/null
  12. return $?
  13. }
  14. function snapuse() {
  15. ${R_RMOD} zfs get -Hpo value usedbysnapshots ${1}
  16. }
  17. function human() {
  18. # found at http://unix.stackexchange.com/a/191787
  19. nawk 'function human(x) {
  20. x[1]/=1024;
  21. if (x[1]>=1000) { x[2]++; human(x); }
  22. }
  23. {a[1]=$1; a[2]=0; human(a); printf "%.2f%s\n",a[1],substr("KMGTEPYZ",a[2]+1,1)}' <<< $1
  24. }
  25. function backup() {
  26. # check source
  27. check_dataset ${DATASET} || {
  28. logger -t $(basename ${0%.sh}) -p user.notice "source dataset \"${DATASET}\" in configuration entry #${COUNTER} does not exist; omitting"
  29. continue 2
  30. }
  31. # determine which local snapshots exist already
  32. SNAPSHOTS=( $(zfs list -rt snapshot -d1 -Ho name -S creation ${DATASET} 2>/dev/null) )
  33. LASTSNAP=${SNAPSHOTS[0]}
  34. L_USED_BEFORE=$(snapuse ${DATASET})
  35. # check target configuration
  36. if [[ ${SAVETO} =~ ^[a-zA-z0-9]+@ ]]
  37. then
  38. # remote target
  39. NETLOC=$(cut -d: -f1 <<< ${SAVETO}) # strip dataset name
  40. USER=$(cut -d@ -f1 <<< ${NETLOC}) # obtain user
  41. TARGET=$(cut -d@ -f2 <<< ${NETLOC}) # obtain host
  42. SAVETO=$(cut -d: -f2 <<< ${SAVETO}) # drop parsed data
  43. unset NETLOC
  44. # check remote host availability and user validity
  45. /usr/bin/ssh ${USER}@${TARGET} true &>/dev/null
  46. if [ $? -ne 0 ]
  47. then
  48. # failed to connect, bail out
  49. logger -t $(basename ${0%.sh}) -p user.notice "${TARGET} is unreachable as ${USER}"
  50. continue 2
  51. else
  52. # configure for remote access
  53. R_RMOD="ssh ${USER}@${TARGET}"
  54. if [ $USER != 'root' ]
  55. then
  56. RMOD="${R_RMOD} sudo"
  57. fi
  58. fi
  59. fi
  60. # check target
  61. check_dataset ${SAVETO} || {
  62. logger -t $(basename ${0%.sh}) -p user.notice "target dataset \"${SAVETO}\" in configuration entry #${COUNTER} does not exist; omitting"
  63. continue 2
  64. }
  65. R_SNAPSHOTS=( $(${R_RMOD} zfs list -rt snapshot -d1 -Ho name -S creation ${SAVETO}/$( basename ${DATASET}) 2>/dev/null) )
  66. check_dataset ${SAVETO}/$( basename ${DATASET}) && R_USED_BEFORE=$(snapuse ${SAVETO}/$( basename ${DATASET}))
  67. if [ $MODE == "backup" ]
  68. then
  69. # determine current timestamp
  70. DATE=$(date +%Y-%m-%d-%H%M)
  71. # determine the name of the current snapshot to create
  72. NEWSNAP="${DATASET}@${DATE}"
  73. # take a snapshot
  74. zfs snapshot -r ${NEWSNAP}
  75. fi
  76. # check if source is encrypted
  77. if [ $ENCRYPTION_FEATURE != "disabled" ]
  78. then
  79. # encryption feature available
  80. ENCRYPTION=$(zfs get -Ho value encryption ${DATASET})
  81. if [[ ${ENCRYPTION} != "off" ]]
  82. then
  83. # encryption in use, send raw stream
  84. RAW_MOD="w"
  85. fi
  86. fi
  87. # determine whether to do differential send or not
  88. if [ ! -z ${LASTSNAP} ]
  89. then
  90. # local snapshot(s) found
  91. SNAPMODIFIER="i ${LASTSNAP}"
  92. check_dataset ${SAVETO}/$(basename ${LASTSNAP}) || {
  93. # last local snapshot is not available at the destination location
  94. if [ ${#R_SNAPSHOTS[*]} -ge 1 ]
  95. then
  96. # remote snapshot(s) found
  97. R_SNAPMODIFIER="I $(dirname ${DATASET})/$(basename ${R_SNAPSHOTS[0]})"
  98. fi
  99. # send any previous snapshots
  100. zfs send -Rcv${RAW_MOD}${R_SNAPMODIFIER} ${LASTSNAP} | ${RMOD:-$R_RMOD} zfs recv -Feu${RESUME_MOD}v ${SAVETO} 2>&1 >> ${LOGFILE}
  101. if [[ ${PIPESTATUS[*]} =~ [1-9]+ ]]
  102. then
  103. # replication failure
  104. logger -t $(basename ${0%.sh}) -p user.notice "failed to replicate ${LASTSNAP} to ${TARGET:-local}:${SAVETO}"
  105. else
  106. let CHANGES++
  107. fi
  108. }
  109. else
  110. # ensure this does not remain in effect
  111. unset SNAPMODIFIER R_SNAPMODIFIER
  112. fi
  113. if [ $MODE == "backup" ]
  114. then
  115. # send backup
  116. zfs send -Rcv${RAW_MOD}${SNAPMODIFIER} ${NEWSNAP} | ${RMOD:-$R_RMOD} zfs recv -Feu${RESUME_MOD}v ${SAVETO} 2>&1 >> ${LOGFILE}
  117. # if replication is unsuccessful, omit the aging check so as to prevent data loss
  118. if [[ ! ${PIPESTATUS[*]} =~ [1-9]+ ]]
  119. then
  120. let CHANGES++
  121. THRESHOLD=$(( $KEEP * 24 * 3600 ))
  122. for SNAPSHOT in ${SNAPSHOTS[*]}
  123. do
  124. TIMESTAMP=$(zfs get -pHo value creation "${SNAPSHOT}")
  125. AGE=$(( $NOW - $TIMESTAMP ))
  126. if [ $AGE -ge $THRESHOLD ]
  127. then
  128. zfs destroy -r ${SNAPSHOT} 2>&1 >> ${LOGFILE}
  129. ${RMOD:-$R_RMOD} zfs destroy -r ${SAVETO}/$(basename ${SNAPSHOT}) 2>&1 >> ${LOGFILE}
  130. fi
  131. done
  132. else
  133. logger -t $(basename ${0%.sh}) -p user.notice "failed to replicate ${NEWSNAP} to ${TARGET:-local}:${SAVETO}, no aging"
  134. fi
  135. fi
  136. # re-evaluate snapshot data usage
  137. unset R_RMOD RMOD
  138. L_USED_AFTER=$(snapuse ${DATASET})
  139. if [ ! -z $TARGET ] && [ ! -z $USER ]
  140. then
  141. R_RMOD="ssh ${USER}@${TARGET}"
  142. fi
  143. if [ ${CHANGES:-0} -gt 0 ]
  144. then
  145. # calculate data volume change
  146. R_USED_AFTER=$(snapuse ${SAVETO}/$( basename ${DATASET}))
  147. L_DELTA=$(( $L_USED_AFTER - $L_USED_BEFORE ))
  148. R_DELTA=$(( $R_USED_AFTER - ${R_USED_BEFORE:-0} ))
  149. for DELTA in L_DELTA R_DELTA
  150. do
  151. unset WHAT WHERE
  152. eval _DELTA=\$$DELTA
  153. if [ $_DELTA -lt 0 ]
  154. then
  155. _DELTA=$(( $_DELTA * -1 ))
  156. WHAT="freed"
  157. fi
  158. if [[ $DELTA =~ ^L ]]
  159. then
  160. WHERE="$(dirname ${DATASET})"
  161. else
  162. WHERE="${SAVETO}"
  163. fi
  164. if [ $_DELTA -ne 0 ]
  165. then
  166. _DELTA=$(human $_DELTA)
  167. logger -t $(basename ${0%.sh}) -p user.notice "$_DELTA ${WHAT:-allocated} in \"${WHERE}\" by backing up \"${DATASET}\""
  168. fi
  169. done
  170. fi
  171. # reset remote configuration
  172. unset R_RMOD RMOD RAW_MOD
  173. }
  174. #### END FUNCTIONS ####
  175. #### BEGIN LOGIC ####
  176. while getopts hf:m: OPTION
  177. do
  178. case "$OPTION" in
  179. f)
  180. # configuration file
  181. CFGFILE="$OPTARG"
  182. ;;
  183. m)
  184. # e-mail recipient for log
  185. RECIPIENT="$OPTARG"
  186. ;;
  187. h|\?)
  188. # display help
  189. usage
  190. exit 1
  191. ;;
  192. esac
  193. done
  194. if [ -z "${CFGFILE}" ]
  195. then
  196. echo "No configuration file supplied!"
  197. exit 1
  198. fi
  199. ## sanity checks
  200. # verify configuration file
  201. if [ ! -f "${CFGFILE}" ]
  202. then
  203. # cannot proceed
  204. logger -t $(basename ${0%.sh}) -p user.notice "can not access configuration file \""${CFGFILE}"\""
  205. exit 1
  206. fi
  207. # determine platform
  208. PLATFORM_VERSION=$(uname -v)
  209. if [[ "$PLATFORM_VERSION" =~ ^joyent ]]
  210. then
  211. # SmartOS GZ has GNU date shipped by default but no perl interpreter on-board
  212. TIMECMD="\$(date +%s)"
  213. # determine pool name
  214. POOL_NAME=$(/usr/bin/sysinfo | /usr/bin/json Zpool)
  215. ENCRYPTION_FEATURE=$(/usr/sbin/zpool get -Ho value feature@encryption ${POOL_NAME})
  216. if [ $? -ne 0 ]
  217. then
  218. # feature unknown, outdated PI
  219. logger -t $(basename ${0%.sh}) -p user.notice "ZFS encryption is not supported on $PLATFORM_VERSION"
  220. ENCRYPTION_FEATURE="disabled"
  221. fi
  222. EXTENSION_FEATURE=$(/usr/sbin/zpool get -Ho value feature@extensible_dataset ${POOL_NAME})
  223. if [ $? -ne 0 ]
  224. then
  225. # feature unknown, outdated PI
  226. logger -t $(basename ${0%.sh}) -p user.notice "extensible datasets are not supported on $PLATFORM_VERSION"
  227. EXTENSION_FEATURE="disabled"
  228. else
  229. # check if resuming is supported
  230. if [ $EXTENSION_FEATURE == "active" ]
  231. then
  232. # extensible_dataset feature available
  233. RESUME_MOD="s"
  234. fi
  235. fi
  236. fi
  237. # determine current timestamp
  238. eval NOW="${TIMECMD:-\$(perl -e 'print time')}"
  239. if [ $? -ne 0 ]
  240. then
  241. # cannot proceed
  242. logger -t $(basename ${0%.sh}) -p user.notice "failed to determine current time on $PLATFORM_VERSION"
  243. exit 1
  244. fi
  245. # determine current timestamp
  246. RUNDATE=$(date +%Y-%m-%d-%H%M)
  247. # determine the name of this script
  248. ME=$(basename ${0%.sh})
  249. # define logging directory, defaulting to "/tmp" if omitted
  250. LOGDIR="/var/log"
  251. # determine session logfile
  252. LOGFILE="${LOGDIR:-/tmp}/${ME}_${RUNDATE}.txt"
  253. while read -u 4 DATASET SAVETO KEEP MODE ENABLED
  254. # alternate file descriptor in use because SSH might be involved and we can not pass '-n' to it because we need stdin for 'zfs recv'
  255. do
  256. let COUNTER++
  257. if [ ${ENABLED} == "Y" ]
  258. then
  259. # split this function -> snapshot(), sync(), age() ? -> a new backup() would call all three
  260. backup
  261. fi
  262. done 4<<< "$(egrep -v '^(#|$)' "${CFGFILE}")" # graciously overlook any comments or blank lines
  263. if [ ! -z "${RECIPIENT}" ]
  264. then
  265. # report the outcome by e-mailing the session log
  266. sendmail ${RECIPIENT} <<- EOF
  267. Subject: ${ME}_${RUNDATE}
  268. Auto-Submitted: auto-generated
  269. $(cat ${LOGFILE})
  270. EOF
  271. fi
  272. #### END LOGIC ####