#!/bin/bash
# $Id$
#
# Relax-and-Recover
#
#    Relax-and-Recover is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 3 of the License, or
#    (at your option) any later version.

#    Relax-and-Recover is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.

#    You should have received a copy of the GNU General Public License
#    along with Relax-and-Recover; if not, write to the Free Software
#    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
#
# Authors:
# See https://github.com/rear/rear/graphs/contributors for full list of authors

# the following Bash minimum version check is so essential that it has to come first
# and that is why it doesn't use anything from our framework or functions.
((BASH_VERSINFO[0] >= 4)) || {
    echo "ERROR: ReaR needs bash version 4 or higher" >&2
    exit 1
}

# Usually functions belong to the library scripts (i.e. $SHARE_DIR/lib/*.sh),
# but these 2 are exceptions on the rule because the complementary global functions
# to get and apply bash flags and options commands are needed from the very beginning
# (the usual functions are sourced at "Include functions" below):
function get_bash_flags_and_options_commands () {
    # Output bash commands that set the currently set flags
    # in reverse ordering to have "set ... xtrace" (i.e. "set +/- x") first
    # so that this flag is set first via apply_bash_flags_and_options_commands
    # which results better matching logging output in debugscript mode:
    set +o | tac | tr '\n' ';'
    # Output bash commands that set the currently set options:
    shopt -p | tr '\n' ';'
}
# This function exists only to provide a syntactically matching counterpart of get_bash_flags_and_options_commands:
function apply_bash_flags_and_options_commands () {
    # Must be called with $1 that is the output of a previous get_bash_flags_and_options_commands:
    eval "$1"
}
# At the very beginning save initial bash flags and options in a global readonly variable
# so that exactly the initial bash flags and options can be restored if needed via
#   apply_bash_flags_and_options_commands "$INITIAL_BASH_FLAGS_AND_OPTIONS_COMMANDS"
readonly INITIAL_BASH_FLAGS_AND_OPTIONS_COMMANDS="$( get_bash_flags_and_options_commands )"

# Enable as needed for development of fail-safe code:
# set -ue -o pipefail
# set -xue -o pipefail
# set -xvue -o pipefail
# Cf. the Relax-and-Recover Coding Style
# https://github.com/rear/rear/wiki/Coding-Style
# that reads (excerpt - dated 18. Nov. 2015):
# "TODO Use set -eu to die from unset variables and unhandled errors"

# Versioning
readonly PRODUCT="Relax-and-Recover"
readonly PROGRAM=${0##*/}
readonly VERSION=2.7
readonly RELEASE_DATE=Git

# Where users should report bugs:
readonly BUG_REPORT_SITE="https://github.com/rear/rear/issues"

# Set machine readable and human readable start date and time values.
# START_SECONDS is set to have a single fixed point in time
# where other date and time formats can be derived from in a consistent way.
# One cannot call the 'date' command several times because then there would be
# a small probability that the clock advances by a second between two 'date' calls
# and then START_DATE_TIME_STRING would be a bit later than START_DATE_TIME_NUMBER
# where "a bit later" could be even the next day when midnight passes in between:
readonly START_SECONDS=$( date +%s )
# START_DATE_TIME_NUMBER has the form YYYYmmddHHMMSS
readonly START_DATE_TIME_NUMBER=$( date -d@$START_SECONDS +%Y%m%d%H%M%S )
# START_DATE_TIME_STRING has the form YYYY-mm-dd HH:MM:SS
readonly START_DATE_TIME_STRING=$( date -d@$START_SECONDS +'%F %T' )

# Provide the command line options in an array so that other scripts may read them:
readonly CMD_OPTS=( "$@" )

# Allow workflows to set the exit code to a different value (not "readonly"):
EXIT_CODE=0
# The separated EXIT_FAIL_MESSAGE variable is used to denote a failure exit.
# One cannot use EXIT_CODE for that because there are cases where a non-zero exit code
# is the intended outcome (e.g. in the 'checklayout' workflow, cf. the end of this script).
# Set EXIT_FAIL_MESSAGE to 1 to have an exit failure message by default which is needed
# to get a failure message when it exits at an arbitrary place because of 'set -e'
# cf. https://github.com/rear/rear/issues/700#issuecomment-327755633
# Before a normal exit EXIT_FAIL_MESSAGE is set to 0 (cf. the end of this script):
EXIT_FAIL_MESSAGE=1

# Find out if we're running from checkout
REAR_DIR_PREFIX=""
readonly SCRIPT_FILE="$( readlink -f $( type -p "$0" || echo "$0" ) )"
if test "$SCRIPT_FILE" != "$( readlink -f /usr/sbin/$PROGRAM )" ; then
    REAR_DIR_PREFIX=${SCRIPT_FILE%/usr/sbin/$PROGRAM}
fi
readonly REAR_DIR_PREFIX

# Program directories - they must be set here. Everything else is then dynamic.
# Not yet readonly here because they are set via the /etc/rear/rescue.conf file
# in the recovery system that is sourced by the rear command in recover mode
# and CONFIG_DIR can also be changed via '-c' command line option:
SHARE_DIR="$REAR_DIR_PREFIX/usr/share/rear"
CONFIG_DIR="$REAR_DIR_PREFIX/etc/rear"
# Use REAR_VAR=/tmp/rear rear <workflow> to redirect ReaR VAR_DIR for one invocation,
# primarily used by tools/run-in-docker and development
VAR_DIR="${REAR_VAR:-$REAR_DIR_PREFIX/var}/lib/rear"
LOG_DIR="${REAR_VAR:-$REAR_DIR_PREFIX/var}/log/rear"

# The current working directory when this script (usr/sbin/rear) is launched
# is also the working directory of all the other scripts and config files
# that get sourced via the Source() function in lib/framework-functions.sh
# cf. https://github.com/rear/rear/issues/2461
readonly WORKING_DIR="$( pwd )"

# Generic global variables that are not meant to be configured by the user
# (i.e. that do not belong into usr/share/rear/conf/default.conf),
# see https://github.com/rear/rear/issues/678
# and https://github.com/rear/rear/issues/708
# Location of the disklayout.conf file created by savelayout
# (no readonly DISKLAYOUT_FILE because it is also set in checklayout-workflow.sh and mkbackuponly-workflow.sh):
DISKLAYOUT_FILE="$VAR_DIR/layout/disklayout.conf"
# Location of the root of the filesystem tree of the target system
# in particular the root of the filesystem tree of the to-be-recovered-system in the recovery system
# i.e. the mountpoint in the recovery system where the filesystem tree of the to-be-recovered-system is mounted:
readonly TARGET_FS_ROOT="/mnt/local"

# Initialize defaults: empty value means "false"/"no":
DEBUG=""
DEBUGSCRIPTS=""
DEBUGSCRIPTS_ARGUMENT="x"
DEBUG_OUTPUT_DEV="null"
DISPENSABLE_OUTPUT_DEV="null"
SECRET_OUTPUT_DEV="null"
EXPOSE_SECRETS=""
PORTABLE=""
NON_INTERACTIVE=""
KEEP_BUILD_DIR=""
KERNEL_VERSION=""
RECOVERY_MODE=""
STEPBYSTEP=""
SIMULATE=""
VERBOSE=""
WORKFLOW=""

# Parse options
help_note_text="Use '$PROGRAM --help' or 'man $PROGRAM' for more information."
if ! OPTS="$( getopt -n $PROGRAM -o "c:C:dDhsSvVr:nep" -l "help,version,non-interactive,debugscripts:,expose-secrets,portable" -- "$@" )" ; then
    echo "$help_note_text"
    exit 1
fi
readonly OPTS
eval set -- "$OPTS"
while true ; do
    case "$1" in
        (-h|--help)
            WORKFLOW="help"
            ;;
        (-V|--version)
            echo -e "$PRODUCT $VERSION / $RELEASE_DATE"
            exit 0
            ;;
        (-v)
            VERBOSE=1
            ;;
        (-n|--non-interactive)
            NON_INTERACTIVE=1
            ;;
        (-e|--expose-secrets)
            EXPOSE_SECRETS=1
            ;;
        (-p|--portable)
            PORTABLE=1
            ;;
        (-c)
            if [[ "$2" == -* ]] ; then
                # When the item that follows '-c' starts with a '-'
                # it is considered to be the next option and not an argument for '-c':
                echo "-c requires an argument."
                echo "$help_note_text"
                exit 1
            fi
            CONFIG_DIR="$2"
            shift
            ;;
        (-C)
            if [[ "$2" == -* ]] ; then
                # When the item that follows '-C' starts with a '-'
                # it is considered to be the next option and not an argument for '-C':
                echo "-C requires an argument."
                echo "$help_note_text"
                exit 1
            fi
            CONFIG_APPEND_FILES+="$2 "
            shift
            ;;
        (-d)
            DEBUG=1
            VERBOSE=1
            DEBUG_OUTPUT_DEV="stderr"
            ;;
        (-D)
            DEBUG=1
            VERBOSE=1
            DEBUGSCRIPTS=1
            DEBUG_OUTPUT_DEV="stderr"
            ;;
        (--debugscripts)
            DEBUG=1
            VERBOSE=1
            DEBUGSCRIPTS=1
            DEBUG_OUTPUT_DEV="stderr"
            DISPENSABLE_OUTPUT_DEV="stderr"
            if [[ "$2" == -* ]] ; then
                # When the item that follows '--debugscripts' starts with a '-'
                # it is considered to be the next option and not an argument for '--debugscripts':
                echo "--debugscripts requires an argument."
                echo "$help_note_text"
                exit 1
            fi
            DEBUGSCRIPTS_ARGUMENT="$2"
            shift
            ;;
        (-s)
            SIMULATE=1
            VERBOSE=1
            ;;
        (-S)
            STEPBYSTEP=1
            ;;
        (-r)
            if [[ "$2" == -* ]] ; then
                # When the item that follows '-r' starts with a '-'
                # it is considered to be the next option and not an argument for '-r':
                echo "-r requires an argument."
                echo "$help_note_text"
                exit 1
            fi
            KERNEL_VERSION="$2"
            shift
            ;;
        (--)
            shift
            break
            ;;
        (-*)
            echo "$PROGNAME: unrecognized option '$1'"
            echo "$help_note_text"
            exit 1
            ;;
        (*)
            break
            ;;
    esac
    shift
done

# Set workflow to first command line argument or to "help" as fallback:
if test -z "$WORKFLOW" ; then
    # Usually workflow is now in $1 (after the options and its arguments were shifted above)
    # but when rear is called without a workflow there exists no $1 here so that
    # an empty default value is used to avoid 'set -eu' error exit if $1 is unset:
    if test "${1:-}" ; then
        # Not "$1" to get rid of compound commands:
        WORKFLOW=$1
        shift
    else
        WORKFLOW=help
    fi
fi

# Keep the remaining command line arguments to feed to the workflow:
ARGS=( "$@" )

# The following workflows are always verbose:
case "$WORKFLOW" in
    (validate|shell|recover|mountonly)
        VERBOSE=1
        ;;
esac

# Set the variables v and verbose to be used in called programs
# that support the '-v' or '--verbose' options like
# "cp $v ..." or "rm $verbose ..." and so on.
# Normally stdout and stderr are not redirected to the log file (see below).
# Only in debug modes stdout and stderr are redirected to the log
# so setting that variables only makes sense in debug modes.
# Cf. https://github.com/rear/rear/issues/2416
v=""
verbose=""
if test "$DEBUG" ; then
    v="-v"
    verbose="--verbose"
fi

# By default SECRET_OUTPUT_DEV="null".
# But if '--expose-secrets' is set we set SECRET_OUTPUT_DEV="stderr"
# in particular to get commands with secret arguments of the form
#   { COMMAND $SECRET_ARGUMENT ; } 2>>/dev/$SECRET_OUTPUT_DEV
# in the log file via 'set -x' when DEBUGSCRIPTS is set.
# But also in non-debugscript modes '--expose-secrets'
# could show secrets where /dev/stderr points to
# which is STDOUT_STDERR_FILE in normal modes
# and RUNTIME_LOGFILE in debug modes.
# Cf. https://github.com/rear/rear/issues/2967#issuecomment-1562732484
test "$EXPOSE_SECRETS" && SECRET_OUTPUT_DEV="stderr"

# Mark variables that are not meant to be changed later as constants (i.e. readonly).
# Exceptions here:
# CONFIG_DIR because "rear recover" sets it in /etc/rear/rescue.conf in the recovery system
# WORKFLOW because for the udev workflow WORKFLOW is set to the actual workflow
# KERNEL_VERSION because if empty it is set when sourcing config files below:
readonly ARGS
readonly CONFIG_APPEND_FILES
readonly DEBUG DEBUGSCRIPTS DEBUGSCRIPTS_ARGUMENT DEBUG_OUTPUT_DEV DISPENSABLE_OUTPUT_DEV
readonly EXPOSE_SECRETS SECRET_OUTPUT_DEV
readonly SIMULATE STEPBYSTEP VERBOSE

# The udev workflow is a special case because it is only a wrapper workflow
# that calls an actual workflow that is specified as $UDEV_WORKFLOW
# (e.g. the default UDEV_WORKFLOW is "mkrescue" in default.conf)
# and then WORKFLOW is set to the actual workflow in udev-workflow.sh
# so that WORKFLOW cannot be set readonly for the udev workflow:
test "$WORKFLOW" = "udev" || readonly WORKFLOW

# Make sure we have the necessary paths (eg. in cron), /sbin will be the first path to search.
# some needed binaries are in /lib/udev or /usr/lib/udev
for path in /usr/bin /bin /usr/sbin /sbin ; do
    case ":$PATH:" in
        (*:"$path":*)
            ;;
        (*)
            test -d "$path" && PATH=$path:$PATH
            ;;
    esac
done
# No readonly PATH so that it can be later extended if needed
# (e.g. to have third-party backup/restore tools available via PATH):
PATH=$PATH:/lib/udev:/usr/lib/udev

# ReaR must run as 'root' (i.e. abort if effective user ID is not 0):
if test "$( id --user )" != "0" ; then
    echo "ERROR: $PRODUCT needs 'root' privileges (effective user ID is not 0)" >&2
    exit 1
fi

# Set some bash options:
# With nullglob set when e.g. for foo*bar no file matches are found, then foo*bar is removed
# (e.g. "ls foo*bar" becomes plain "ls" without "foo*bar: No such file or directory" error).
# The extglob shell option enables several extended pattern matching operators.
shopt -s nullglob extglob
# Save the current default bash flags and options in a global readonly variable
# so that exactly the default bash flags and options can be restored if needed via
#   apply_bash_flags_and_options_commands "$DEFAULT_BASH_FLAGS_AND_OPTIONS_COMMANDS"
readonly DEFAULT_BASH_FLAGS_AND_OPTIONS_COMMANDS="$( get_bash_flags_and_options_commands )"

# Forget all remembered full pathnames of commands:
hash -r

# Make sure that we use only English:
export LC_CTYPE=C LC_ALL=C LANG=C

# Remember what TMPDIR was originally set when ReaR was launched
# before TMPDIR may be set (if unset) to /var/tmp in default.conf
# because testing if TMPDIR is '/var/tmp' would be insufficient
# to determine if TMPDIR was set or unset when the user called 'rear'
# because the user may have specified TMPDIR to be '/var/tmp'
# cf. https://github.com/rear/rear/pull/3168#issuecomment-1997057144
readonly TMPDIR_ORIG="${TMPDIR-}"

# Include default config before the RUNTIME_LOGFILE value is set because
# setting the right RUNTIME_LOGFILE value requires values from default.conf
# in particular the default LOGFILE value and the values of the
# LOCKLESS_WORKFLOWS and SIMULTANEOUS_RUNNABLE_WORKFLOWS arrays:
if ! source $SHARE_DIR/conf/default.conf ; then
    echo -e "ERROR: BUG in $PRODUCT\nFailed to source $SHARE_DIR/conf/default.conf\nPlease report it at $BUG_REPORT_SITE" >&2
    exit 1
fi

# Use RUNTIME_LOGFILE for the logfile that is actually used during runtime
# so that the user can specify a different LOGFILE in his local.conf file.
# During runtime use only the actually used RUNTIME_LOGFILE value.
# By default RUNTIME_LOGFILE is the LOGFILE from default.conf:
RUNTIME_LOGFILE="$LOGFILE"
# Set the right RUNTIME_LOGFILE value depending on whether or not
# this currently running instance can run simultaneously with another instance:
can_run_simultaneously=""
# LOCKLESS_WORKFLOWS can run simultaneously with another instance by using LOGFILE.lockless.
# FIXME: Only one lockless workflow can run otherwise the same LOGFILE.lockless
# would be used by more than one simultaneously running lockless workflows:
for lockless_workflow in "${LOCKLESS_WORKFLOWS[@]}" ; do
    if test "$WORKFLOW" = "$lockless_workflow" ; then
        RUNTIME_LOGFILE="$LOGFILE.lockless"
        can_run_simultaneously="yes"
        break
    fi
done
# SIMULTANEOUS_RUNNABLE_WORKFLOWS are allowed to run simultaneously
# but cannot use LOGFILE.lockless instead they get a LOGFILE with PID
# see https://github.com/rear/rear/issues/1102
# When a workflow is both in LOCKLESS_WORKFLOWS and in SIMULTANEOUS_RUNNABLE_WORKFLOWS
# its right LOGFILE value is the one for SIMULTANEOUS_RUNNABLE_WORKFLOWS:
for simultaneous_runnable_workflow in "${SIMULTANEOUS_RUNNABLE_WORKFLOWS[@]}" ; do
    if test "$WORKFLOW" = "$simultaneous_runnable_workflow" ; then
        # Simultaneously runnable workflows require unique logfile names
        # so that the PID is interposed in the LOGFILE value from default.conf
        # which is used as RUNTIME_LOGFILE while the workflow is running
        # and at the end it gets copied to a possibly used-defined LOGFILE.
        # The logfile_suffix also works for logfile names without '.*' suffix
        # (in this case ${LOGFILE##*.} returns the whole $LOGFILE value):
        logfile_suffix=$( test "${LOGFILE##*.}" = "$LOGFILE" && echo 'log' || echo "${LOGFILE##*.}" )
        RUNTIME_LOGFILE="${LOGFILE%.*}.$$.$logfile_suffix"
        can_run_simultaneously="yes"
        break
    fi
done
# The log file that is actually used during runtime must not be changed:
readonly RUNTIME_LOGFILE

# When this currently running instance cannot run simultaneously with another instance
# test that this currently running instance does not run simultaneously with another instance:
if ! test "$can_run_simultaneously" ; then
    # In this case pidof is needed to test what running instances are there:
    if ! type pidof 1>/dev/null ; then
        echo "ERROR: Required program 'pidof' missing" >&2
        exit 1
    fi
    # For unknown reasons '-o %PPID' does not work for pidof at least in SLES11
    # so that a manual test is done to find out if another pid != $$ is running.
    # This test is only some best effort attempt to find what running instances are there
    # because what pidof finds depends how 'rear' is called and what pidof version is used.
    # For example pidof in SLES10 SP4 seems to only consider the basename:
    #   # ps auxw | grep ssh
    #   ... 3132 ... /usr/sbin/sshd -o PidFile=/var/run/sshd.init.pid
    #   ... 3622 ... sshd: root@pts/1
    #   # pidof -x sshd
    #   3622 3132
    #   # pidof -x /usr/sbin/sshd
    #   3622 3132
    #   # pidof -x in/sshd
    #   3622 3132
    # In contrast pidof in openSUSE Leap 15.3 checks exactly what was specified when it contains a path:
    #   # ps auxw | grep ssh
    #   ...  2991 ... /usr/bin/ssh-agent /usr/bin/gpg-agent --sh --daemon --keep-display /etc/X11/xinit/xinitrc
    #   ... 18219 ... /usr/bin/ssh-agent -D -a /run/user/1000/keyring/.ssh
    #   # pidof -x "ssh-agent"
    #   18219 2991
    #   # pidof -x "/usr/bin/ssh-agent"
    #   18219 2991
    #   # pidof -x "usr/bin/ssh-agent"
    #   [no output]
    # We do not want to use only the basename 'rear' to avoid that pidof accidentally also shows
    # other 'rear' programs which are not Relax-and-Recover but some different software.
    # Examples how 'rear' looks in the 'ps' output:
    # When Relax-and-Recover is normally installed (e.g. as RPM package) so the 'rear' program is /usr/sbin/rear:
    #   # type -a rear
    #   rear is /usr/sbin/rear
    #   # rear mkrescue
    #   # ps auxw | grep rear
    #   ... /bin/bash /usr/sbin/rear mkrescue
    #   # cd /
    #   # ./usr/sbin/rear mkrescue
    #   # ps auxw | grep rear
    #   ... /bin/bash ./usr/sbin/rear mkrescue
    #   # cd /usr/sbin
    #   # ./rear mkrescue
    #   # ps auxw | grep rear
    #   ... /bin/bash ./rear mkrescue
    # When a Relax-and-Recover GitHub code checkout/clone is used so the 'rear' program is /path/to/checkout/usr/sbin/rear:
    #   # /path/to/checkout/usr/sbin/rear mkrescue
    #   # ps auxw | grep rear
    #   ... /bin/bash /path/to/checkout/usr/sbin/rear mkrescue
    #   # cd /path/to/checkout
    #   # usr/sbin/rear mkrescue
    #   # ps auxw | grep rear
    #   ... /bin/bash usr/sbin/rear mkrescue
    #   # ./usr/sbin/rear mkrescue
    #   # ps auxw | grep rear
    #   ... /bin/bash ./usr/sbin/rear mkrescue
    #   # cd /path/to/checkout/usr/sbin
    #   # ./rear mkrescue
    #   # ps auxw | grep rear
    #   ... /bin/bash ./rear mkrescue
    # So we check $SCRIPT_FILE which is /usr/sbin/rear or /path/to/checkout/usr/sbin/rear
    # and /usr/sbin/rear usr/sbin/rear ./usr/sbin/rear ./sbin/rear ./rear ($PROGRAM is 'rear').
    # We need to explicitly check /usr/sbin/rear because $SCRIPT_FILE could be /path/to/checkout/usr/sbin/rear
    # and then another simultaneously running instance could be /usr/sbin/rear (e.g. from an RPM package).
    # Things are different when 'rear' is run inside a booted ReaR recovery system:
    # Inside a booted ReaR recovery system 'rear' is '/bin/rear'
    # and /sbin /usr/sbin /usr/bin are symbolic links to /bin
    # and $PATH is only 'bin' so inside a booted ReaR recovery system we have
    #   RESCUE localhost:~ # type -a rear
    #   rear is /bin/rear
    # cf. https://github.com/rear/rear/issues/2826#issuecomment-1172111440
    # When in the ReaR recovery system 'rear' is called normally like
    #   RESCUE localhost:~ # rear recover
    # we get
    #   RESCUE localhost:~ # ps auwx | grep rear
    #   ... /bin/bash /bin/rear recover
    # and $SCRIPT_FILE is /bin/rear so the check below works.
    # The check below fails when in the ReaR recovery system 'rear' is not called normally like
    #   RESCUE localhost:~ # pwd
    #   /root
    #   RESCUE localhost:~ # ../bin/rear recover
    # cf. https://github.com/rear/rear/issues/2826#issuecomment-1172138188
    # This check is not meant to detect any possible way how another rear instance might have been called.
    for pid in $( pidof -x "$SCRIPT_FILE" "/usr/sbin/$PROGRAM" "usr/sbin/$PROGRAM" "./usr/sbin/$PROGRAM" "./sbin/$PROGRAM" "./$PROGRAM" ) ; do
        if test "$pid" != $$ ; then
            echo "ERROR: $PROGRAM is already running with PID $pid, not starting again" >&2
            exit 1
        fi
    done
fi

# Source usr/share/rear/lib/_input-output-functions.sh because therein
# the original STDIN STDOUT and STDERR file descriptors are saved as fd6 fd7 and fd8
# so that ReaR functions for actually intended user messages can use fd7 and fd8
# to show messages to the user regardless where STDOUT and STDERR are redirected
# and fd6 to get input from the user regardless where STDIN is redirected:
if ! source $SHARE_DIR/lib/_input-output-functions.sh ; then
    echo -e "ERROR: BUG in $PRODUCT\nFailed to source $SHARE_DIR/lib/_input-output-functions.sh\nPlease report it at $BUG_REPORT_SITE" >&2
    exit 1
fi

# Via source usr/share/rear/lib/_input-output-functions.sh
# those final exit tasks are now set to be executed (via bash EXIT trap) in the following order:
# "(( EXIT_FAIL_MESSAGE )) && echo '${MESSAGE_PREFIX}$PROGRAM $WORKFLOW failed, check $RUNTIME_LOGFILE for details' 1>&8"
# "exec 8>&-"
# "exec 7>&-"
# "exec 6<&-"

# Set RECOVERY_MODE when we are running inside the rescue/recovery system.
# RECOVERY_MODE is a boolean variable (cf. conf/default.conf) because we need it below
# to set TMPDIR to ReaR's TMP_DIR only when we are not running inside the recovery system
# but we do not yet have lib/global-functions.sh (which contains is_false) sourced here.
# In the recovery system /etc/rear-release is unique (it does not exist otherwise)
# see build/default/970_add_rear_release.sh that adds it to identify the rescue environment:
test -e "/etc/rear-release" && RECOVERY_MODE='y' || RECOVERY_MODE=''
readonly RECOVERY_MODE

# Create a sufficiently safe temporary working area BUILD_DIR for ReaR and
# additionally set TMPDIR to TMP_DIR within BUILD_DIR for programs that are called by ReaR
# (the latter only when ReaR runs on the original system i.e. when we are not in RECOVERY_MODE):
if test "$WORKFLOW" != "help" ; then
    # ReaR fails when TMPDIR is a relative directory, see
    # https://github.com/rear/rear/pull/3168#issuecomment-1999996162
    # so we set TMPDIR to its resolved absolute path.
    # Ignore when 'readlink -e' fails (e.g. it won't fail when TMPDIR is a regular file)
    # and verify afterwards that the 'readlink -e' result is a directory:
    export TMPDIR="$( readlink -e "$TMPDIR" || true )"
    # TMPDIR must be a directory to make 'mktemp' work below.
    # When TMPDIR was unset by the user it is set to '/var/tmp' in default.conf
    # but it is an error when TMPDIR is set by the user to an empty or blank value:
    if ! test -d "$TMPDIR" ; then
        # Don't use Error() unless "QuietAddExitTask cleanup_build_area_and_end_program" was done:
        echo "ERROR: TMPDIR '$TMPDIR' is not a directory" >&2
        exit 1
    fi
    # We rely on sufficiently safe 'mktemp -d' behaviour
    # that the BUILD_DIR directory will have read, write, and search permissions
    # for the current user (i.e. for root), but no permissions for the group or others
    # (i.e. BUILD_DIR is "drwx------ root root /var/tmp/rear.XXXXXXXXXXXXXXX"), cf.
    # https://www.gnu.org/software/coreutils/manual/html_node/mktemp-invocation.html#mktemp-invocation
    # Create ReaR's working area BUILD_DIR:
    if ! BUILD_DIR="$( mktemp -d -t rear.XXXXXXXXXXXXXXX )" ; then
        # Don't use Error() unless "QuietAddExitTask cleanup_build_area_and_end_program" was done:
        echo "ERROR: Could not create working area ('mktemp -d -t rear.XXXXXXXXXXXXXXX' failed)" >&2
        exit 1
    fi
    # Prepend working area (BUILD_DIR) removal exit task
    # so it gets executed last directly before the above listed final exit tasks.
    # On the one hand this must be done after the build area was successfully created,
    # otherwise cleanup_build_area_and_end_program would try to remove a nonexistent build area
    # by calling plain 'rm -Rf --one-file-system' where nothing is specified to be removed
    # where (fortunately) 'rm' does not remove anything when nothing is specified to be removed,
    # see https://github.com/rear/rear/issues/3180
    # On the other hand this must be done before the first possible call of the 'Error' function,
    # otherwise we may error out but leave the build area behind,
    # see https://github.com/rear/rear/pull/2633
    QuietAddExitTask cleanup_build_area_and_end_program
    # Create ROOTFS_DIR in ReaR's working area:
    ROOTFS_DIR=$BUILD_DIR/rootfs
    mkdir -p $ROOTFS_DIR || Error "Could not create $ROOTFS_DIR"
    # Create TMP_DIR in ReaR's working area:
    TMP_DIR=$BUILD_DIR/tmp
    mkdir -p $TMP_DIR || Error "Could not create $TMP_DIR"
    # Normally set TMPDIR to ReaR's TMP_DIR which is /var/tmp/rear.XXXXXXXXXXXXXXX/tmp
    # to ensure that temporary files from programs that are called by ReaR
    # are sufficiently safe against unwanted access by non-root users
    # (only 'root' has permissions to access ReaR's working area)
    # and that those temporary files will get cleaned up "by the way"
    # via the cleanup_build_area_and_end_program() function,
    # see https://github.com/rear/rear/issues/3167
    if ! [[ "$RECOVERY_MODE" || "$PORTABLE" ]] ; then
        # We set TMPDIR to ReaR's TMP_DIR only when we are not in RECOVERY_MODE and also not in PORTABLE mode
        # because TMP_DIR does not exist in the recreated/restored system or in the foreign rescue system where
        # the portable mode should be able to work without the need to have a TMP_DIR in the rescue system.
        # i.e. there is no /mnt/local/var/tmp/rear.XXXXXXXXXXXXXXX/tmp so that
        #   chroot /mnt/local COMMAND
        # fails when COMMAND uses TMPDIR - in particular "chroot /mnt/local dracut" fails
        # see https://github.com/rear/rear/pull/3168#issuecomment-1983377528
        if test "$TMPDIR_ORIG" ; then
            tmpdir_debug_info="Changing TMPDIR to '$TMP_DIR' (was '$TMPDIR_ORIG' when ReaR was launched)"
        else
            tmpdir_debug_info="Setting TMPDIR to '$TMP_DIR' (was unset when ReaR was launched)"
        fi
        export TMPDIR="$TMP_DIR"
    else
        # Also in RECOVERY_MODE provide debug info what TMPDIR is:
        if test "$TMPDIR_ORIG" ; then
            tmpdir_debug_info="Using TMPDIR '$TMPDIR' (was '$TMPDIR_ORIG' when ReaR was launched)"
        else
            tmpdir_debug_info="Setting TMPDIR to '$TMPDIR' (was unset when ReaR was launched)"
        fi
    fi
    # We create TMPDIR inside ROOTFS_DIR i.e. we create $ROOTFS_DIR$TMPDIR which is
    # usually /var/tmp/rear.XXXXXXXXXXXXXXX/rootfs/var/tmp/rear.XXXXXXXXXXXXXXX/tmp
    # (with same 'XXXXXXXXXXXXXXX' characters for both parts)
    # or when TMPDIR was set via "export TMPDIR=/path/to/tmpdir" before ReaR was launched
    # then $ROOTFS_DIR$TMPDIR is /path/to/tmpdir/rear.XXXXXXXXXXXXXXX/rootfs/path/to/tmpdir
    # so that "chroot $ROOTFS_DIR COMMAND" will not fail when COMMAND uses TMPDIR.
    # For example build/default/490_fix_broken_links.sh
    # and build/default/990_verify_rootfs.sh run "chroot $ROOTFS_DIR COMMAND".
    # See https://github.com/rear/rear/pull/3168#issuecomment-1991356186
    # We create with default mode because all in BUILD_DIR is sufficiently safe:
    mkdir -p $ROOTFS_DIR$TMPDIR || Error "Could not create $ROOTFS_DIR$TMPDIR"
    # Since the above BUILD_DIR="$( mktemp ... )" does not always return a path under /tmp
    # the build directory must be excluded from the backup to be on the safe side:
    BACKUP_PROG_EXCLUDE+=( "$BUILD_DIR" )
fi

# Used to determine below whether TMPDIR has been changed in user config.
# After the 'mktemp' above, changing TMPDIR does not have the intended effect.
# Save the current value to detect TMPDIR changes regardless if 'mktemp' above was called
# so even the help workflow would report that error, see https://github.com/rear/rear/pull/3163
saved_tmpdir="$TMPDIR"

# Keep old log file:
test -r "$RUNTIME_LOGFILE" && mv -f "$RUNTIME_LOGFILE" "$RUNTIME_LOGFILE".old 2>/dev/null

# Start logging:
mkdir -p $LOG_DIR || Error "Could not create $LOG_DIR"
rm -f "$RUNTIME_LOGFILE"
cat /dev/null >"$RUNTIME_LOGFILE"
# The log file may contain sensitive information so only root should be able to read it:
chmod u=rw,go=- "$RUNTIME_LOGFILE"
# In debug modes stdout and stderr are redirected to the log
# cf. https://github.com/rear/rear/issues/2416
# In non-debug modes (in particular also in verbose mode)
# stdout and stderr are redirected to a temporary file if possible
# i.e. when TMP_DIR exists - it does not exist for the 'help' workflow
# so use /dev/null as fallback because TMP_DIR does not exists for the 'help' workflow:
STDOUT_STDERR_FILE="/dev/null"
# In non-debug modes the log cannot contain possibly false alarm messages
# cf. https://github.com/rear/rear/issues/2416
# but in non-debug modes stdout and stderr of all programs is still available
# for the Error function to extract some latest messages
# cf. https://github.com/rear/rear/issues/2623
if test "$DEBUG" ; then
    # To be on the safe side append to the log file '>>' instead of plain writing to it '>'
    # because when a program (bash in this case) is plain writing to the log file it can overwrite
    # output of a possibly simultaneously running process that likes to append to the log file
    # (e.g. when a background process runs that also uses the ReaR log file for logging).
    exec 2>>"$RUNTIME_LOGFILE"
    # Make stdout the same what stderr already is because
    # it is a bad idea to handle stderr and stdout differently
    # cf. https://github.com/rear/rear/issues/2416#issuecomment-702159687
    # Here we keep strict ordering of stdout and stderr outputs
    # because both stdout and stderr use one same file descriptor:
    exec 1>&2
else
    # Quoting is mandatory otherwise 'test -d' falsely succeeds if $TMP_DIR is empty, cf.
    # https://github.com/rear/rear/issues/2966#issuecomment-1497825493
    if test -d "$TMP_DIR" ; then
        STDOUT_STDERR_FILE="$TMP_DIR/$PROGRAM.$WORKFLOW.stdout_stderr"
        # The stdout and stderr file that is used during runtime must not be changed:
        readonly STDOUT_STDERR_FILE
        rm -f "$STDOUT_STDERR_FILE"
        cat /dev/null >"$STDOUT_STDERR_FILE"
        # Programs stdout and stderr may show sensitive information so only root should be able to read it:
        chmod u=rw,go=- "$STDOUT_STDERR_FILE"
    fi
    # To be on the safe side append stderr to the file (see above):
    exec 2>>"$STDOUT_STDERR_FILE"
    # Make stdout the same what stderr already is (see above).
    # The stdout redirection is intentionally two times there (both for debug modes and non-debug modes)
    # to make it easier to change the stdout redirection if needed different for both kind of modes:
    exec 1>&2
fi

# Include functions after RUNTIME_LOGFILE is set and readonly
# so that functions can use a fixed RUNTIME_LOGFILE value:
for script in $SHARE_DIR/lib/[a-z]*.sh ; do
    source $script || BugError "Failed to source $script"
done

# Show initial startup messages:
if test "$WORKFLOW" != "help" ; then
    LogPrint "$PRODUCT $VERSION / $RELEASE_DATE"
    LogPrint "Running $PROGRAM $WORKFLOW (PID $MASTER_PID date $START_DATE_TIME_STRING)"
    DebugPrint "Command line options: $0 ${CMD_OPTS[*]}"
    LogPrint "Using log file: $RUNTIME_LOGFILE"
    DebugPrint "Using build area: $BUILD_DIR"
    DebugPrint "$tmpdir_debug_info"
fi

# In DEBUG mode keep by default the build area but that can be overridden in user config files
# therefore no readonly KEEP_BUILD_DIR (it is also set to 1 in build/default/990_verify_rootfs.sh):
test "$DEBUG" && KEEP_BUILD_DIR=1 || true

test "$SIMULATE" && LogPrint "Simulation mode activated, Relax-and-Recover base directory: $SHARE_DIR" || true

if test "$DEBUGSCRIPTS_ARGUMENT" ; then
    Debug "Current set of flags is '$-'"
    Debug "The debugscripts flags are '$DEBUGSCRIPTS_ARGUMENT'"
fi

# All workflows need to read the configurations first.
# Combine configuration files:
Debug "Combining configuration files"
# Use this file to manually override the OS detection:
test -d "$CONFIG_DIR" || Error "Configuration directory $CONFIG_DIR is not a directory"
if test -r "$CONFIG_DIR/os.conf" ; then
    Source "$CONFIG_DIR/os.conf" || Error "Failed to Source $CONFIG_DIR/os.conf"
fi
if test -r "$CONFIG_DIR/$WORKFLOW.conf" ; then
    Source "$CONFIG_DIR/$WORKFLOW.conf" || Error "Failed to Source $CONFIG_DIR/$WORKFLOW.conf"
fi
SetOSVendorAndVersion
# Distribution configuration files:
for config in "$ARCH" "$OS" \
        "$OS_MASTER_VENDOR" "$OS_MASTER_VENDOR_ARCH" "$OS_MASTER_VENDOR_VERSION" "$OS_MASTER_VENDOR_VERSION_ARCH" \
        "$OS_VENDOR" "$OS_VENDOR_ARCH" "$OS_VENDOR_VERSION" "$OS_VENDOR_VERSION_ARCH" ; do
    if test -r "$SHARE_DIR/conf/$config.conf" ; then
        Source "$SHARE_DIR/conf/$config.conf" || BugError "Failed to Source $SHARE_DIR/conf/$config.conf"
    fi
done
# User configuration files, last thing is to overwrite variables if we are in the rescue system:
for config in site local rescue ; do
    if test -r "$CONFIG_DIR/$config.conf" ; then
        # Delete all characters except '\r' and error out if the resulting string is not empty:
        test "$( tr -d -c '\r' < $CONFIG_DIR/$config.conf )" && Error "Carriage return character in $CONFIG_DIR/$config.conf (perhaps DOS or Mac format)"
        Source "$CONFIG_DIR/$config.conf" || Error "Failed to Source $CONFIG_DIR/$config.conf"
    fi
done
# Finally source additional configuration files if specified on the command line:
CONFIG_APPEND_FILES_PATHS=()
if test "$CONFIG_APPEND_FILES" ; then
    # Treat missing additional config files (almost) same as other missing config files
    # which means that missing additional config files do not let ReaR abort with an Error
    # but in contrast to other missing config files missing additional config files
    # are reported to the user via 'LogPrintError' so that the user is at least informed
    # if what he requested via the '-C' command line option cannot be fulfilled.
    # An intended positive side-effect when missing additional config files are no Error
    # is that then it also works (not relly cleanly but it works) for DRLM_MANAGED=y
    # which requires that missing local config files must not let ReaR abort with an Error
    # because the needed config files get later downloaded from the DRLM server and applied
    # (cf. the drlm_import_runtime_config function in lib/drlm-functions.sh)
    # for details see https://github.com/rear/rear/issues/1229
    for config_append_file in $CONFIG_APPEND_FILES ; do
        # If what is specified on the command line starts with '/' an absolute path is meant
        # otherwise what is specified on the command line means a file in CONFIG_DIR.
        # Files in CONFIG_DIR get automatically copied into the recovery system but
        # other files are added to COPY_AS_IS to get them copied into the recovery system:
        case "$config_append_file" in
            (/*)
                config_append_file_path="$config_append_file"
                # If "-C foo" was specified on the command line but 'foo' does not exist
                # try if 'foo.conf' exists and if yes, use that:
                if test -r "$config_append_file_path" ; then
                    COPY_AS_IS+=( "$config_append_file_path" )
                elif test -r "$config_append_file_path.conf" ; then
                    COPY_AS_IS+=( "$config_append_file_path.conf" )
                else
                    LogPrintError "There is '-C $config_append_file' but neither '$config_append_file_path' nor '$config_append_file_path.conf' can be read."
                fi
                ;;
            (*)
                config_append_file_path="$CONFIG_DIR/$config_append_file"
                ;;
        esac
        # If "-C foo" was specified on the command line but 'foo' does not exist
        # try if 'foo.conf' exists and if yes, use that:
        if test -r "$config_append_file_path" ; then
            LogPrint "Sourcing additional configuration file '$config_append_file_path'"
            Source "$config_append_file_path" || Error "Failed to Source $config_append_file_path"
            CONFIG_APPEND_FILES_PATHS+=( "$config_append_file_path" )
        elif test -r "$config_append_file_path.conf" ; then
            LogPrint "Sourcing additional configuration file '$config_append_file_path.conf'"
            Source "$config_append_file_path.conf" || Error "Failed to Source $config_append_file_path.conf"
            CONFIG_APPEND_FILES_PATHS+=( "$config_append_file_path.conf" )
        else
            LogPrintError "There is '-C $config_append_file' but neither '$config_append_file_path' nor '$config_append_file_path.conf' can be sourced."
        fi
    done
fi

if [ "$saved_tmpdir" != "$TMPDIR" ] ; then
    Error "Setting TMPDIR in a configuration file is not supported. To specify a working area directory prefix, export TMPDIR before executing '$PROGRAM'"
fi

readonly CONFIG_APPEND_FILES_PATHS # nothing else should be able to add more configs
COPY_AS_IS+=("${CONFIG_APPEND_FILES_PATHS[@]}") # copy also to rescue system

# Now SHARE_DIR CONFIG_DIR VAR_DIR LOG_DIR and KERNEL_VERSION should be set to a fixed value:
readonly SHARE_DIR CONFIG_DIR VAR_DIR LOG_DIR KERNEL_VERSION

# Enable progress subsystem only in verbose mode, set some stuff that others can use:
if test "$VERBOSE" ; then
    source $SHARE_DIR/lib/progresssubsystem.nosh || BugError "Failed to source $SHARE_DIR/lib/progresssubsystem.nosh"
fi

SourceStage "init"

readonly VERSION_INFO="
$PRODUCT $VERSION / $RELEASE_DATE

$PRODUCT comes with ABSOLUTELY NO WARRANTY; for details see
the GNU General Public License at: http://www.gnu.org/licenses/gpl.html

Host $( uname -n ) using Backup $BACKUP and Output $OUTPUT
Build date: $( date -R )
"

# Check for and run the requested workflow:
if has_binary WORKFLOW_$WORKFLOW ; then
    Log "Running $WORKFLOW workflow"
    # There could be no ARGS[@] which means it would be an unbound variable so that
    # an empty default is used here to avoid an error exit if 'set -eu' is used:
    WORKFLOW_$WORKFLOW "${ARGS[@]:-}"
    Log "Finished running $WORKFLOW workflow"
else
    LogPrintError "ERROR: The specified command '$WORKFLOW' does not exist!"
    EXIT_CODE=1
fi

# Usually RUNTIME_LOGFILE=/var/log/rear/rear-$HOSTNAME.log
# The RUNTIME_LOGFILE name is set above from LOGFILE in default.conf
# but later user config files are sourced where LOGFILE can be set different
# so that the user config LOGFILE is used as final logfile name:
if test "$RUNTIME_LOGFILE" != "$LOGFILE" ; then
    # Do this only when LOGFILE is a regular file (e.g. do not do this if LOGFILE is /dev/null):
    if test -f "$LOGFILE" ; then
        rm -f "$LOGFILE"
        cat /dev/null >"$LOGFILE"
        # The log file may contain sensitive information so only root should be able to read it:
        chmod u=rw,go=- "$LOGFILE"
        test "$WORKFLOW" != "help" && LogPrint "Saving $RUNTIME_LOGFILE as $LOGFILE"
        cat "$RUNTIME_LOGFILE" >"$LOGFILE"
    fi
fi

# When this point is reached, the workflow is done without a real error because
# for real errors one of the Error functions should be called that kills rear and
# then the exit code is usually 143 = 128 + 15 (15 = SIGTERM from the USR1 trap)
# and the Error function results an "ERROR: ..." syslog message.
# Set EXIT_FAIL_MESSAGE to 0 to avoid a false exit failure message:
EXIT_FAIL_MESSAGE=0
# There should be no syslog message for the help workflow:
if test "$WORKFLOW" != "help" ; then
    if test $EXIT_CODE -eq 0 ; then
        LogToSyslog "$PROGRAM $WORKFLOW finished with zero exit code"
    else
        if test "checklayout" = "$WORKFLOW" -a $EXIT_CODE -eq 1 ; then
            # The checklayout workflow is special because it sets EXIT_CODE=1
            # when the disk layout has changed or when configuration files have changed
            # but that exit code 1 is no error but meant as a signal that things have changed
            # which require a new ISO image so that users can run "rear checklayout || rear mkrescue"
            # see https://github.com/rear/rear/issues/564
            LogToSyslog "$PROGRAM checklayout finished with exit code 1 (layout or config changed)"
        else
            LogToSyslog "$PROGRAM $WORKFLOW failed with exit code $EXIT_CODE"
        fi
    fi
fi

exit $EXIT_CODE

# vim: set et ts=4 sw=4: