### Determine the name of the backup archive ### This needs to be after we special case USB devices. # FIXME: backuparchive is no local variable (regardless that it is lowercased) # To avoid logging of 'Using backup archive' on the screen and in the log file # during a mkrescue run we skip what this script does for the mkrescue workflow # see https://github.com/rear/rear/issues/1363 # There is a relation to output/ISO/Linux-i386/810_prepare_multiple_iso.sh # that normally needs the backuparchive variable value # except for the mkrescue workflow so that this here is safe # see https://github.com/rear/rear/pull/2063#issuecomment-469222487 [[ "$WORKFLOW" = "mkrescue" ]] && return # If TAPE_DEVICE is specified, use that: if test "$TAPE_DEVICE" ; then backuparchive="$TAPE_DEVICE" LogPrint "Using backup archive '$backuparchive'" return fi local backup_file_suffix="$BACKUP_PROG_SUFFIX$BACKUP_PROG_COMPRESS_SUFFIX" local backup_file_name="$BACKUP_PROG_ARCHIVE$backup_file_suffix" local scheme=$( url_scheme $BACKUP_URL ) local path=$( url_path $BACKUP_URL ) case "$scheme" in (file|iso) # Define the output path according to the scheme local outputpath=$( backup_path $scheme $path ) backuparchive="$outputpath/$backup_file_name" LogPrint "Using backup archive '$backuparchive'" return ;; (tape) # TODO: Check if that case is really needed. # Perhaps prep/default/030_translate_tape.sh does already all what is needed. backuparchive=$path LogPrint "Using backup archive '$backuparchive'" return ;; esac local backup_directory=$BUILD_DIR/outputfs/$NETFS_PREFIX # Normal (i.e. non-incremental/non-differential) backup: if ! test "incremental" = "$BACKUP_TYPE" -o "differential" = "$BACKUP_TYPE" ; then # In case of normal (i.e. non-incremental) backup there is only one restore archive # and its name is the same as the backup archive (usually 'backup.tar.gz'): backuparchive="$backup_directory/$backup_file_name" LogPrint "Using backup archive '$backuparchive'" # This script is also run during "rear recover/restoreonly" where RESTORE_ARCHIVES must be set. local backup_restore_workflows=( "recover" "restoreonly" ) if IsInArray "$WORKFLOW" "${backup_restore_workflows[@]}" ; then # Only set RESTORE_ARCHIVES when the backup archive is actually accessible # cf. https://github.com/rear/rear/issues/1166 if test -r "$backuparchive" ; then RESTORE_ARCHIVES=( "$backuparchive" ) else # In case of USB backup there is the subsequent 540_choose_backup_archive.sh script # that shows a backup selection dialog when RESTORE_ARCHIVES is not already set. if test "usb" = "$scheme" ; then LogPrint "Backup archive '$backuparchive' not readable. Need to select another one." else Error "Backup archive '$backuparchive' not readable." fi fi fi return fi # Incremental or differential backup: set -e -u -o pipefail # Incremental or differential backup only works for the NETFS backup method # and only with the 'tar' backup program: if ! test "NETFS" = "$BACKUP" -a "tar" = "$BACKUP_PROG" ; then Error "BACKUP_TYPE incremental or differential only works with BACKUP=NETFS and BACKUP_PROG=tar" fi # Incremental or differential backup is currently only known to work with BACKUP_URL=nfs://. # Other BACKUP_URL schemes may work and at least BACKUP_URL=usb:///... needs special setup # to work with incremental or differential backup (see https://github.com/rear/rear/issues/1145): if test "usb" = "$scheme" ; then # When USB_SUFFIX is set the compliance mode is used where # backup on USB works in compliance with backup on NFS which means # a fixed backup directory where incremental or differential backups work. # Use plain $USB_SUFFIX and not "$USB_SUFFIX" because when USB_SUFFIX contains only blanks # test "$USB_SUFFIX" would result true because test " " results true: test $USB_SUFFIX || Error "BACKUP_TYPE incremental or differential requires USB_SUFFIX for BACKUP_URL=usb" fi # Incremental or differential backup and keeping old backup contradict each other (mutual exclusive) # so that NETFS_KEEP_OLD_BACKUP_COPY must not be 'true' in case of incremental or differential backup: if test "$NETFS_KEEP_OLD_BACKUP_COPY" ; then NETFS_KEEP_OLD_BACKUP_COPY="" LogPrint "Disabled NETFS_KEEP_OLD_BACKUP_COPY because BACKUP_TYPE incremental or differential does not work with that" fi # For incremental or differential backup some date values (weekday, YYYY-MM-DD, HHMM) are needed # that must be consistent for one single point of the current time which means # one cannot call the 'date' command several times because then there would be # a small probability that e.g. weekday, YYYY-MM-DD, HHMM do not match # one single point in time (in particular when midnight passes in between). # Therefore the output of one single 'date' call is storend in an array and # the array elements are then assinged to individual variables as needed: local current_date_output=( $( date '+%a %Y-%m-%d %H%M' ) ) local current_weekday="${current_date_output[0]}" local current_yyyy_mm_dd="${current_date_output[1]}" local current_hhmm="${current_date_output[2]}" # The date FULLBACKUP_OUTDATED_DAYS ago is needed to check if the latest full backup is too old. # When the latest full backup is more than FULLBACKUP_OUTDATED_DAYS ago a new full backup is made. # This separated call of the 'date' command which is technically needed because it is # for another point in time (e.g. 7 days ago) is run after the above call of the 'date' # command for the current time to be on the safe side when midnight passes in between # both 'date' commands which would then result that a new full backup is made # when the latest full backup is basically right now FULLBACKUP_OUTDATED_DAYS ago because # the stored date of the latest full backup is the current date at the time when it was made. # Example (assuming FULLBACKUP_OUTDATED_DAYS=7 ): # The latest full backup was made on Sunday January 10 in 2016 (just before midnight). # One week later this script runs again while midnight passes between the two 'date' calls # so that current_date_output[@]="Sun 2016-01-17 0000" (still Sunday January 17 in 2016) # and yyyymmdd_max_days_ago=20160111 (already Monday January 11 in 2016), then # Sunday January 10 is older than Monday January 11 so that a new full backup is made: test "$FULLBACKUP_OUTDATED_DAYS" || FULLBACKUP_OUTDATED_DAYS="7" local yyyymmdd_max_days_ago=$( date '+%Y%m%d' --date="$FULLBACKUP_OUTDATED_DAYS days ago" ) # Full backup file names are of the form YYYY-MM-DD-HHMM-F.tar.gz # where the 'F' denotes a full backup: local full_backup_marker="F" # Incremental backup file names are of the form YYYY-MM-DD-HHMM-I.tar.gz # where the 'I' denotes an incremental backup: local incremental_backup_marker="I" # Differential backup file names are of the form YYYY-MM-DD-HHMM-D.tar.gz # where the last 'D' denotes a differential backup: local differential_backup_marker="D" # In case of incremental or differential backup the RESTORE_ARCHIVES contains # first the latest full backup file. # In case of incremental backup the RESTORE_ARCHIVES contains # after the latest full backup file each incremental backup # in the ordering how they must be restored. # For example when the latest full backup was made on Sunday # plus each subsequent weekday a separated incremental backup was made, # then during a "rear recover" on Wednesday morning # first the full backup from Sunday has to be restored, # then the incremental backup from Monday, and # finally the incremental backup from Tuesday. # In case of differential backup the RESTORE_ARCHIVES contains # after the latest full backup file the latest differential backup. # For example when the latest full backup was made on Sunday # plus each subsequent weekday a separated differential backup was made, # then during a "rear recover" on Wednesday morning # first the full backup from Sunday has to be restored, # and finally the differential backup from Tuesday # (i.e. the differential backup from Monday is skipped). # The date format YYYY-MM-DD that is used here is crucial. # It is the ISO 8601 format 'year-month-day' to specify a day of a year # that is accepted by 'tar' for the '--newer' option, # see the GNU tar manual section "Operating Only on New Files" # at https://www.gnu.org/software/tar/manual/html_node/after.html # and the GNU tar manual section "Calendar date items" # at https://www.gnu.org/software/tar/manual/html_node/Calendar-date-items.html#SEC124 local date_glob_regex="[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]" local date_time_glob_regex="$date_glob_regex-[0-9][0-9][0-9][0-9]" # Determine what kind of backup must be created, 'full' or 'incremental' or 'differential' # (the empty default means it is undecided what kind of backup must be created): local create_backup_type="" # Code regarding creating a backup is useless during "rear recover" and # messages about creating a backup are misleading during "rear recover": local recovery_workflows=( "recover" "layoutonly" "restoreonly" ) if ! IsInArray "$WORKFLOW" "${recovery_workflows[@]}" ; then # When today is a specified full backup day, do a full backup in any case # (regardless if there is already a full backup of this day): if IsInArray "$current_weekday" "${FULLBACKUPDAY[@]}" ; then create_backup_type="full" LogPrint "Today's weekday ('$current_weekday') is a full backup day that triggers a new full backup in any case" fi fi # Get the latest full backup (if exists): local full_backup_glob_regex="$date_time_glob_regex-$full_backup_marker$backup_file_suffix" # Here things like 'find /path/to/dir -name '*.tar.gz' | sort' are used because # one cannot use bash globbing via commands like 'ls /path/to/dir/*.tar.gz' # because /usr/sbin/rear sets the nullglob bash option which leads to plain 'ls' # when '/path/to/dir/*.tar.gz' matches nothing (i.e. when no backup file exists) # so that then plain 'ls' would result nonsense. local latest_full_backup=$( find $backup_directory -name "$full_backup_glob_regex" | sort | tail -n1 ) # A latest full backup is found: if test "$latest_full_backup" ; then local latest_full_backup_file_name=$( basename "$latest_full_backup" ) # The full_or_incremental_backup_glob_regex is also needed below for non-"recover" WORKFLOWs # to set the right variables for creating an incremental backup: local full_or_incremental_backup_glob_regex="$date_time_glob_regex-[$full_backup_marker$incremental_backup_marker]$backup_file_suffix" # Code regarding creating a backup is useless during "rear recover" and # messages about creating a backup are misleading during "rear recover": if ! IsInArray "$WORKFLOW" "${recovery_workflows[@]}" ; then # There is nothing to do here if it is already decided that # a full backup must be created (see "full backup day" above"): if ! test "full" = "$create_backup_type" ; then local latest_full_backup_date=$( echo $latest_full_backup_file_name | grep -o "$date_glob_regex" ) local yyyymmdd_latest_full_backup=$( echo $latest_full_backup_date | tr -d '-' ) # Check if the latest full backup is too old: if test $yyyymmdd_latest_full_backup -lt $yyyymmdd_max_days_ago ; then create_backup_type="full" LogPrint "Latest full backup date '$latest_full_backup_date' too old (more than $FULLBACKUP_OUTDATED_DAYS days ago) triggers new full backup" else # When a latest full backup is found that is not too old # a BACKUP_TYPE (incremental or differential) backup will be created: create_backup_type="$BACKUP_TYPE" LogPrint "Latest full backup found ($latest_full_backup_file_name) triggers $BACKUP_TYPE backup" fi fi else # This script is also run during "rear recover" where RESTORE_ARCHIVES must be set: case "$BACKUP_TYPE" in (incremental) # When a latest full backup is found use that plus all later incremental backups for restore: # The following command is a bit tricky: # It lists all YYYY-MM-DD-HHMM-F.tar.gz and all YYYY-MM-DD-HHMM-I.tar.gz files in the backup directory and sorts them # and finally it outputs only those that match the latest full backup file name and incremental backups that got sorted after that # where it is mandatory that the backup file names sort by date (i.e. date must be the leading part of the backup file names): RESTORE_ARCHIVES=( $( find $backup_directory -name "$full_or_incremental_backup_glob_regex" | sort | sed -n -e "/$latest_full_backup_file_name/,\$p" ) ) ;; (differential) # For differential backup use the latest full backup plus the one latest differential backup for restore: # The following command is a bit tricky: # It lists all YYYY-MM-DD-HHMM-F.tar.gz and all YYYY-MM-DD-HHMM-D.tar.gz files in the backup directory and sorts them # then it outputs only those that match the latest full backup file name and all differential backups that got sorted after that # and then it outputs only the first line (i.e. the full backup) and the last line (i.e. the latest differential backup) # but when no differential backup exists (i.e. when only the full backup exists) the first line is also the last line # so that "sed -n -e '1p;$p'" outputs the full backup twice which is corrected by the final "sort -u": local full_or_differential_backup_glob_regex="$date_time_glob_regex-[$full_backup_marker$differential_backup_marker]$backup_file_suffix" RESTORE_ARCHIVES=( $( find $backup_directory -name "$full_or_differential_backup_glob_regex" | sort | sed -n -e "/$latest_full_backup_file_name/,\$p" | sed -n -e '1p;$p' | sort -u ) ) ;; (*) BugError "Unexpected BACKUP_TYPE '$BACKUP_TYPE'" ;; esac # Tell the user what will be restored: local restore_archives_file_names="" for restore_archive in "${RESTORE_ARCHIVES[@]}" ; do restore_archives_file_names+=" $( basename "$restore_archive" )" done LogPrint "For backup restore using $restore_archives_file_names" fi # No latest full backup is found: else # Code regarding creating a backup is useless during "rear recover" and # messages about creating a backup are misleading during "rear recover": if ! IsInArray "$WORKFLOW" "${recovery_workflows[@]}" ; then # If no latest full backup is found create one during "rear mkbackup": create_backup_type="full" LogPrint "No full backup found (YYYY-MM-DD-HHMM-F.tar.gz) triggers full backup" else # This script is also run during "rear recover" where RESTORE_ARCHIVES must be set: # If no latest full backup is found (i.e. no file name matches the YYYY-MM-DD-HHMM-F.tar.gz form) # fall back to what is done in case of normal (i.e. non-incremental/non-differential) backup # and hope for the best (i.e. that a backup_directory/backup_file_name actually exists). # In case of normal (i.e. non-incremental/non-differential) backup there is only one restore archive # and its name is the same as the backup archive (usually 'backup.tar.gz'). # This is only a fallback setting to be more on the safe side for "rear recover". # Initially for the very fist run of incremental backup during "rear mkbackup" # a full backup file of the YYYY-MM-DD-HHMM-F.tar.gz form will be created. RESTORE_ARCHIVES=( "$backup_directory/$backup_file_name" ) LogPrint "Using $backup_file_name for backup restore" fi fi # Code regarding creating a backup is useless during "rear recover" and # messages about creating a backup are misleading during "rear recover": if ! IsInArray "$WORKFLOW" "${recovery_workflows[@]}" ; then # Set the right variables for creating a backup (but do not actually do anything at this point): case "$create_backup_type" in (full) local new_full_backup_file_name="$current_yyyy_mm_dd-$current_hhmm-$full_backup_marker$backup_file_suffix" backuparchive="$backup_directory/$new_full_backup_file_name" BACKUP_PROG_CREATE_NEWER_OPTIONS="-V $new_full_backup_file_name" LogPrint "Performing full backup using backup archive '$new_full_backup_file_name'" ;; (incremental) local new_incremental_backup_file_name="$current_yyyy_mm_dd-$current_hhmm-$incremental_backup_marker$backup_file_suffix" backuparchive="$backup_directory/$new_incremental_backup_file_name" # Get the latest incremental backup that is based on the latest full backup (if exists): local incremental_backup_glob_regex="$date_time_glob_regex-$incremental_backup_marker$backup_file_suffix" # First get the latest full backup plus all later incremental backups (cf. how RESTORE_ARCHIVES is set in case of incremental backup) # then grep only the incremental backups and from the incremental backups use only the last one (if exists): local latest_incremental_backup=$( find $backup_directory -name "$full_or_incremental_backup_glob_regex" | sort | sed -n -e "/$latest_full_backup_file_name/,\$p" | grep "$incremental_backup_glob_regex" | tail -n1 ) if test "$latest_incremental_backup" ; then # A latest incremental backup that is based on the latest full backup is found: local latest_incremental_backup_file_name=$( basename $latest_incremental_backup ) LogPrint "Latest incremental backup found ($latest_incremental_backup_file_name) that is newer than the latest full backup" local latest_incremental_backup_date=$( echo $latest_incremental_backup_file_name | grep -o "$date_glob_regex" ) BACKUP_PROG_CREATE_NEWER_OPTIONS="--newer=$latest_incremental_backup_date -V $latest_incremental_backup_file_name" LogPrint "Performing incremental backup for files newer than $latest_incremental_backup_date using backup archive '$new_incremental_backup_file_name'" else # When there is not yet an incremental backup that is based on the latest full backup # the new created incremental backup must be based on the latest full backup: BACKUP_PROG_CREATE_NEWER_OPTIONS="--newer=$latest_full_backup_date -V $latest_full_backup_file_name" LogPrint "Performing incremental backup for files newer than $latest_full_backup_date using backup archive '$new_incremental_backup_file_name'" fi ;; (differential) local new_differential_backup_file_name="$current_yyyy_mm_dd-$current_hhmm-$differential_backup_marker$backup_file_suffix" backuparchive="$backup_directory/$new_differential_backup_file_name" BACKUP_PROG_CREATE_NEWER_OPTIONS="--newer=$latest_full_backup_date -V $latest_full_backup_file_name" LogPrint "Performing differential backup for files newer than $latest_full_backup_date using backup archive '$new_differential_backup_file_name'" ;; (*) BugError "Unexpected create_backup_type '$create_backup_type'" ;; esac fi # Go back from "set -e -u -o pipefail" to the defaults: apply_bash_flags_and_options_commands "$DEFAULT_BASH_FLAGS_AND_OPTIONS_COMMANDS"