#!/bin/sh
VERSION=1.1.5
#
#       nw-watchdog is a higly configurable network watchdog written
#       in POSIX shell script for use in Linux.
#
#       Get the latest version from https://github.com/fraxflax/nw-watchdog
#
#       nw-watchdog is free software written by Fredrik Ax <nw-watchdog@axnet.nu>.
#       Feel free to modify and/or (re)distribute it in any way you like.
#       (It's always nice to be mentioned though ;-) )
#
#       nw-watchdog comes with ABSOLUTELY NO WARRANTY.
#
#       If you expirence any problems with nw-watchdog, are lacking
#       any functionality or just want to voice your opions about it,
#       feel free to contact me via e-mail: nw-watchdog@axnet.nu
#
#       POSIX IEEE Std 1003.1-2017 (Revision of IEEE Std 1003.1-2008):
#       https://pubs.opengroup.org/onlinepubs/9699919799/utilities/contents.html
#
###########################################################################################
# DEFAULTS                                    # Corresponding Command Line Options
###########################################################################################
TARGET=''                                     # TARGET (destination)

HELP='';                               HELPRE='(-h|--help)';                                 HELPV=y
PAGERUSE=y;                        PAGERUSERE='(-M|--no-pager|--no-less|--no-more)';         PAGEUSEV=''
PINGNEXTHOP=y;                  PINGNEXTHOPRE='(-N|-G|--no-ping-nexthop|--no-ping-gateway)'; PINGNEXTHOPV=''
PINGTARGET=y;                    PINGTARGETRE='(-P|--no-ping-target)';                       PINGTARGETV=''
IPADDRALRT=y;                    IPADDRALRTRE='(-A|--no-ipaddr-alert)';                      IPADDRALRTV=''
IFCUPDOWN=y;                      IFCUPDOWNRE='(-R|--no-interface-reset)';                   IFCUPDOWNV=''
CONTINUOUSTOPOLOGYDETECT=y;
                   CONTINUOUSTOPOLOGYDETECTRE='(-T|--no-continuous-topology-detect)';        CONTINUOUSTOPOLOGYDETECTV=''
FORK=y;                                FORKRE='(-f|-D|--foreground|--no-daemonize)';         FORKV=''
LISTSYSTEMD='';                 LISTSYSTEMDRE='--list-systemd';                              LISTSYSTEMDV=y
VERBOSE=''                          VERBOSERE='(-v|--verbose)';                              VERBOSEV=y
DEBUG=''                              DEBUGRE='(-d|--debug)';                                DEBUGV=y
VERSHOW='';                         VERSHOWRE='--version';                                   VERSHOWV=y

V=4;                                      VRE='(--verbosity-level|-V)';  VARE='[0-9]+'                  # integer >= 0
IFC='';                                 IFCRE='(--interface|-i)';        IFCARE='[^[:space:]/]+'        # non-empty string without whitespaces and slashes
STATICIFC='';                     STATICIFCRE='(--force-interface|-I)';  STATICIFCARE=$IFCARE           # same as IFCARE
LOGFILE='/var/log/nw-watchdog.log'; LOGFILERE='(--logfile|-l)';          LOGFILEARE='.+'                # non-emty string
LOGSIZE=0;                          LOGSIZERE='(--logsize|-z)';          LOGSIZEARE='[0-9]+[kKmMgG]?'   # integer >= 0,  optional [kKmMgG]
PIDFILE=/run/nw-watchdog.pid;       PIDFILERE='(--pidfile|-p)';          PIDFILEARE='.+'                # non-emty string
SLOWUPTIMEOUT=3;              SLOWUPTIMEOUTRE='(--slow-up-timeout|-t)';  SLOWUPTIMEOUTARE='[1-9][0-9]*' # integer >= 1
SLEEP=10;                             SLEEPRE='(--interval|--sleep|-s)'; SLEEPARE='[1-9][0-9]*'         # integer >= 1
GRACE=20;                             GRACERE='(--ifup-grace|-g)';       GRACEARE='[1-9][0-9]*'         # integer >= 1
MAXNOLINK=1;                      MAXNOLINKRE='(--max-nolink|-n)';       MAXNOLINKARE='[0-9]+'          # integer >= 0
IFCUPT='ip link set up %{IFC}';      IFCUPTRE='(--ifcup|-u)';            IFCUPTARE='.+'                 # non-emty string
IFCDOWNT='ip link set down %{IFC}'; IFCDOWNTRE='(--ifcdown|-U)';         IFCDOWNTARE='.+'               # non-emty string
ALERTCMDT='if which wall >/dev/null; then exec wall; else cat 1>&2; fi';
                                   ALERTCMDTRE='(--alert|-a)';           ALERTCMDTARE='.+'              # non-emty string
SYSTEMDSERVICENAME='';    SYSTEMDSERVICENAMERE='--install-systemd';      SYSTEMDSERVICENAMEARE='.+'     # non-emty string
RMSYSTEMDSERVICE='';        RMSYSTEMDSERVICERE='--remove-systemd';       RMSYSTEMDSERVICEARE='.+'       # non-emty string

##############################################################################
SOPTIONS='HELP PAGERUSE PINGNEXTHOP PINGTARGET IPADDRALRT IFCUPDOWN CONTINUOUSTOPOLOGYDETECT FORK LISTSYSTEMD VERBOSE DEBUG VERSHOW'
SOPTSRE='[hMNGPARTfDvd]' # Short options without arguments
AOPTIONS='V IFC STATICIFC LOGFILE LOGSIZE PIDFILE SLOWUPTIMEOUT SLEEP GRACE MAXNOLINK IFCUPT IFCDOWNT ALERTCMDT SYSTEMDSERVICENAME RMSYSTEMDSERVICE'

# Ensure we have a POSIX (enough) shell
set >/dev/null 2>&1 && export PATH >/dev/null 2>&1 && umask 022 || {
        printf '\nThis script must be run in a POSIX compliant shell!\n\n' 1>&2 \
            || echo 'This script must be run in a POSIX compliant shell!' 1>&2
        exit 1
    }

# Let's put the "standard" paths first in PATH to ensure we get the standard utilities we need
# and remove duplicates in PATH before exporting it
NEWPATH='/sbin:/bin:/usr/sbin:/usr/bin'
OIFS=$IFS
IFS=:
for p in $PATH; do
    case ":$NEWPATH:" in
        *:"$p":*) ;;
        *) NEWPATH="$NEWPATH:$p" ;;
    esac
done
IFS=$OIFS
PATH=$NEWPATH
export PATH

DEPENDENCIES='cat cut date getent grep head id ip ping readlink sed sleep stat tail touch wc'
lacking=''
if printf '' 2>/dev/null; then
    deperr() {
        printf '\n(PATH=%s)\n\n~~~~~~~~~~~~~~~~\nDEPENDENCY ERROR\n~~~~~~~~~~~~~~~~\nThe following executable(s) are lacking in your PATH:\n   %s\n... aborting!\n\n' "$PATH" "$*" 1>&2
        exit 1
    }
else
    lacking='printf'
    deperr() {
        echo "
(PATH=$PATH)

~~~~~~~~~~~~~~~~
DEPENDENCY ERROR
~~~~~~~~~~~~~~~~
The following executable(s) are lacking in your PATH:
   $*
... aborting!'
" 1>&2
        exit 1
    }
fi
which sh >/dev/null 2>&1 || deperr which sh $lacking
for x in $DEPENDENCIES; do which "$x" >/dev/null || lacking="$lacking $x"; done
[ "$lacking" ] && deperr $lacking

ERRMSG=''
COLS=72; _b_=''; ___=''; __='';
_show_errmsg() {
    [ "$ERRMSG" ] || [ "$1" ] || return
    div=''; i=0;
    while [ $i -lt $COLS ]; do div="~$div"; i=$((i+1)); done
    printf '\n%s\n%s: ' "$_b_$div" "${_b_}ERROR$__"
    [ "$1" ] && {
        PTN=$1; shift
        printf "$PTN\n" "$@"
    }
    lines=$(printf %s "$ERRMSG" | sed -E 's/%/%%/g')
    printf "$lines" | while read l; do
        [ "$l" ] || break
        l=$(printf %s "$l" | sed -E 's/\t/\n   /g')
        printf '%s %s\n' "$_b_*$__" "$l"
    done
    printf '%s\n\n' "$_b_$div"
}
nwwatchdog=${0##*/}
_USAGE() {
    TPUTOK=1
    which tput>/dev/null 2>&1 && {
        TPUTOK=0
        COLS=$(tput cols) 2>/dev/null || TPUTOK=$((TPUTOK+1))
        _b_=$(tput bold)  2>/dev/null || TPUTOK=$((TPUTOK+1))
        ___=$(tput smul)  2>/dev/null || TPUTOK=$((TPUTOK+1))
        #__=$(tput sgr0)  2>/dev/null || TPUTOK=$((TPUTOK+1)) 'tput sgr0' in combination with 'less -R' does not work on all terminal
        __=$(printf '\033[0m')                              #  types, so always use the universal ANSI reset escape sequence instead
    }                                                       #  ( '\033[Om' instead of '\033[m^O' )
    [ $TPUTOK -eq 0 ] || { COLS=72; _b_=''; ___=''; __=''; }
    FMT=cat; which fmt >/dev/null && FMT="fmt -s -w $COLS"
    _show_errmsg "$@"
    $FMT<<EOU
$__
${_b_}NAME $__
    $_b_$nwwatchdog$__ - Network Watchdog

${_b_}VERSION $__
EOU
    if printf %s "$VERSION" | grep -q -e-; then $FMT<<EOU
    $VERSION

    This is an unreleased version. The version format for all unreleased versions is:
    LATESTRELESE-BRANCH-CHANGEDATE-RUNNINGNUMBER
    where LATESTRELESE is the release this version is based on. BRANCH is the git branch, CHANGEDATE is the date (YYYYMMDD) the latest change was commited and RUNNINGNUMBER indicates how many changes was commited that day.
EOU
    else printf '    %s\n' "$VERSION"; fi
    $FMT<<EOU

${_b_}SYNOPSIS $__

    $_b_$nwwatchdog$__ ${___}TARGET$__ [ OPTIONS ]

    $_b_$nwwatchdog --list-systemd$__

    $_b_$nwwatchdog --remove-systemd$__ ${___}SERVICENAME$__ | ${___}UNITNAME$__

${_b_}DESCRIPTION $__
    $_b_$nwwatchdog$__ is a higly configurable network watchdog written in POSIX shell script for use in Linux, depending only on Linux most standard tools that are normally installed by default in all distributions (also see the ${_b_}DEPENDENCIES$__ section).

    It monitors the network connectivity to a specified ${___}TARGET${__} and/or the next hop towards that ${___}TARGET${__}, alerting upon lost connectivity explaining what is wrong. It can reset the source interface and will detect topology changes and, if allowed, reconfigure itself accordingly. It's intended to run as a daemon and has an option to install itself as a systemd service.  If you want to monitor the connectivity to several ${___}TARGET${__}s, you can run several instances of $_b_$nwwatchdog$__ using different $_b_--pidfile$__ option arguments.

    $_b_$nwwatchdog$__ is free software written by ${_b_}Fredrik Ax$__ <${_b_}nw-watchdog@axnet.nu$__>.
    Feel free to modify and/or (re)distribute it in any way you like.
    ... it's always nice to be mentioned though ;-)

    $_b_$nwwatchdog$__ comes with ABSOLUTELY NO WARRANTY.

    If you expirence any problems with ${_b_}nw-watchdog$__, are lacking any functionality or just want to voice your opions about it, feel free to contact me via e-mail.

    Get the latest version from https://github.com/fraxflax/nw-watchdog

${_b_}${___}TARGET$__
    The mandatory (unless $_b_--list-systemd$__ or $_b_--remove-systemd$__ is specified) argument ${___}TARGET$__ is the target (destination) for which the connection is monitored. ${___}TARGET$__ can be an IP address or a resolvable hostname / FQDN. If it's a hostname / FQDN, it will be resolved to an IPv4 address (first one found by getent). The resolved IP address will be used for the monitoring. The name is continuously resolved and if the resolved ip address changes the new IP address will be used for the monitoring from there on.
    Use $_b_--no-continuous-topology-detect$__ to resolve the ${___}TARGET${__} only at startup and failed connectivity checks.

    ${___}TARGET$__ can be placed before, after or between valid OPTIONS.

${_b_}OPTIONS $__(no arguments)
    These options take no arguments, and may be specified in any order. They can be grouped (e.g. -vAP) in their short form, also having one of the OPTIONS that requires an argument last in the group.

    $_b_--help | -h $__
        Shows this help, using \$PAGER if set to an executable, otherwise 'less' or 'more' if available in "/sbin:/bin:/usr/sbin:/usr/bin:\$PATH"

        All other options are ignored, apart from --no-pager which can be used to avoid using a pager.

    $_b_--no-pager | --no-less | --no-more | -M$__
        Do NOT use a pager for help and error messages.

        $_b_--install-systemd$__ will enforce $_b_--no-pager$__ for the installed systemd service.

    $_b_--no-ping-target | -P $__
        If the ${___}TARGET${__} is the next hop (on the same subnet), reachability of the ${___}TARGET${__} is checked by arp cache status and ping.
        If the ${___}TARGET${__} is not on the same subnet as the source, the reachability of the ${___}TARGET${__} is checked by pinging it in a certain pattern (see $_b_--slow-up-timeout$__ for details).

        $_b_--no-ping-target$__ disables the ping-checks for the ${___}TARGET${__}. Only connectivity to the NEXTHOP for the ${___}TARGET${__} is checked.
        This can be useful if ${___}TARGET${__} does not reply to ping, or if it desirable to only alert if there is no route to the ${___}TARGET${__} or NEXTHOP is unreachable.

        $_b_--no-ping-target$__ cannot be used in combination with $_b_--no-ping-nexthop$__.

    $_b_--no-ping-nexthop | -N | --no-ping-gateway | -G $__
        By default, if the connectivity to the ${___}TARGET${__} cannot be verified, and the next hop (NEXTHOP) is not the ${___}TARGET${__} itself, the reachability of the NEXTHOP (usually a gateway) is checked, firstly by checking it's status in the arp cache and then by pinging it, rechecking the arp cache status upon failed ping.

        $_b_--no-ping-nexthop$__ disbles the reachaility check for the NEXTHOP so only connectivity to ${___}TARGET${__} itself is checked. Useful if the NEXTHOP is a peer-to-peer address and not setup to reply to ping.

        $_b_--no-ping-nexthop$__ cannot be used in combination with $_b_--no-ping-target$__.

    $_b_--no-ipaddr-alert | -A $__
        Do not alert for not finding any global scope ip addresses on the source interface.

    $_b_--no-interface-reset | -R $__
        Do not try to bring down and up interface after failed connectivity checks.
        (Do not try to "repair" the connection, just monitor it.)

    $_b_--no-continuous-topology-detect | -T $__
        Normaly the topology (resolving the ip address of the ${___}TARGET${__}, detecting which source interface to use and the ip address of the NEXTHOP towards the ${___}TARGET${__}) is detected at startup and continuously monitored for changes.

        $_b_--no-continuous-topology-detect$__ disables the topology detection for as long as the ${___}TARGET${__} replies (or in combination with $_b_--no-ping-target$__, for as long as the NEXTHOP is reachable). The topology will only be detected at startup and if the ${___}TARGET${__} does not reply or if the NEXTHOP cannot be reached, meaning that routing changes making the ${___}TARGET${__} or NEXTHOP unreachable will not be detected as long as the ${___}TARGET${__} / NEXTHOP can be reached using the old topology.

        $_b_--force-interface$__ implies $_b_--no-continuous-topology-detect$__.

    $_b_--foreground | -f | --no-daemonize | -D $__
        Run in foreground, do not fork / daemonize.

    $_b_--list-systemd$__
        If systemd is installed, a brief status of all installed nw-watchdog systemd-services are listed to stdout.

        Cannot be combined with any other options.

    $_b_--verbose | -v $__
        Shortcut for $_b_--verbosity-level=5$__
        If used in combination with $_b_--verbosity-level$__, the specified $_b_--verbosity-level$__ level will take precedence.

    $_b_--debug | -d $__
        Shortcut for: $_b_--verbosity-level=6 $_b_--logfile=- $_b_--logsize=0 $_b_--pidfile=/dev/null $_b_--slow-up-timeout=1 $_b_--sleep=3 $_b_--ifup-grace=5 $_b_--alert='cat' $_b_--foreground$__

        If it's combined with any of the options it provides shortcuts for, the specified option will take precedence over the $_b_--debug$__ shortcut.

        This option cannot be combined with $_b_--install-systemd$__ (but it would be wise to test the configuration with $_b_--debug$__ before installing as a systemd service).

    $_b_--version$__
        Prints the version of nw-watchdog to stdout and immediately exits. All other options are ignored.

${_b_}OPTIONS$__ (with ARGUMENT)
        These opions takes a single argument each and may be specified in any order. Specify with equalsign or space or no space between option and argument. They can only be grouped together with the shortform of the NO-ARGUMENT-OPTIONS above, and must be last in such groupings (e.g. -PAV5).

EOU
    $FMT<<EOU
    $_b_--verbosity-level | -V$__ ${___}level$__
        Default: $V
        ${___}level$__ must be an integer greater than or equal to zero.
        Determines how much info is logged to the logfile and which alerts are triggered.

        ${_b_}0$__ - none
        No output to logfile and no alerts triggered at all ... pretty useless unless you just want to keep traffic going keeping a connecting alive without any alerts.

        ${_b_}1$__ - error
        Only logs and alerts on errors causing the $_b_$nwwatchdog$__ not to function as intended.

        ${_b_}2$__ - warning
        Also logs and alerts on warnings about configuration, etc.

        ${_b_}3$__ - alert
        Log and alert on irreparable connectivity failures, interface down, and other things disrupting the monitored connection, as well as on warnings and errors.
        Also see the $_b_--alert$__ option.

        ${_b_}4$__ - info (default)
        Same as level 3, but also logs some useful information on what's going on, such as topology changes, some test failures forcing more testing, interface resets, etc

        ${_b_}5$__ - trace
        Logs even more info about which action is currently performed, including, all secondary tests that are run, when sleeping longer than usual, etc.

        ${_b_}6$__ - debug
        Logs even more internal details, e.g. SETTINGS used, all tests performed, all sleeps, and other debug info.

    ${_b_}--interface | -i$__ ${___}interface$__
        Default: none
        ${___}interface$__ is the name of the source interface to initially use.

        The interface may dynamically change due to topology detection. If you want to force the use of a specific interface, use $_b_--force-interface$__ instead.

        If neither ${_b_}--interface$__ nor $_b_--force-interface$__ is specified the source interface will be determined from the FIB by looking at the route to the ${___}TARGET${__}. The reason to specify it even so, would be to have $_b_$nwwatchdog$__ bring it up if it's down when starting.

        $_b_--interface$__ cannot be combined with $_b_--force-interface$__.

    $_b_--force-interface | -I$__ ${___}interface$__
        Default: none
        ${___}interface$__ is the name of the source interface to always use.

        Packets will always be sent from this interface. The forwadring table will be ignored as well as conflicting topology changes.
        This is useful for monitoring the preferred path and making sure it's up. It does not check if you have connectivity to the ${___}TARGET${__} via any other path.

        Implies $_b_--no-continuous-topology-detect$__.

        $_b_--force-interface$__ cannot be combined with $_b_--interface$__.

    $_b_--logfile | -l$__ ${___}logfile$__
        Default: '$LOGFILE'
        Logfile to use. If specified as '-' logs are written to stdout.

    $_b_--logsize | -z$__ ${___}size$__
        Default: $LOGSIZE

        If the logfile grows beyond this size, the oldest entries will be removed.

        You can use suffixes K, M, G for kilo / mega / giga bytes. (No suffix is same as K).
        Set to 0 for unlimited logfile size (which you would want if you do log rotation).

        If the logfile is set to '-' (stdout) this option is ignored.

        If ${_b_}flock$__ is available, the logfile will be locked before written to or shrinked, otherwise there is a slight risk of log entries being lost if two or more instances of $_b_$nwwatchdog$__ are concurently running using the same logfile and at least one of them have $_b_--logsize$__ set to a value larger than 0.

    $_b_--pidfile | -p$__ ${___}pidfile$__
        Default: '$PIDFILE'
        Pidfile to use.

    $_b_--slow-up-timeout | -t$__ ${___}seconds$__
        Default: $SLOWUPTIMEOUT
        ${___}seconds$__ must be an integer greater than zero.

        The check whether the ${___}TARGET${__} is up or not, is performed in several steps.
        First a "quick-up" test sends one single ICMP echo packet waiting for the reply for no more than 1 second. If that fails, a more thourough "slow-up" test sends 5 ICMP echos.

        $_b_--slow-up-timeout$__ controls the TIMEOUT for waiting on each packet in the slow-up test.
        5 packets are always sent in the slow-up test.
        The packets are sent adaptively, meaning that as soon as a reply is received the next packet is sent without delay, giving slow-up a total time of 5 * RTT to the ${___}TARGET${__} if the connection is up.
        The DEADLINE = TIMEOUT * 5 is the maximum time the slow-up test will take if the ${___}TARGET${__} is down.

        $_b_--slow-up-timeout=7$__ is useful for monitoring VPN connections via interfaces that need a long wake-up time if idle (due to regotiation of encryption, exchanging keys, reauthentication, etc).

        $_b_--slow-up-timeout=3$__ is useful for monitoring connections to ${___}TARGET${__}s with low to medium latency via interfaces that does not need a long wake-up time (e.g. ethernet interfaces).

        $_b_--slow-up-timeout=1$__ is suitable to use for monitoring local ${___}TARGET${__}s (e.g. NEXTHOP) on ethernet carried subnets.

    $_b_--sleep | -s | --interval$__ ${___}seconds$__
        Default: $SLEEP
        ${___}seconds$__ must be an integer greater than zero.

        How many seconds to sleep after sucessful ping check.

    $_b_--ifup-grace | -g$__ ${___}seconds$__
        Default: $GRACE
        ${___}seconds$__ must be an integer greater than zero.

        How many seconds to sleep before next check after interface has been reset.

    $_b_--max-nolink | -n$__ ${___}number$__
        Default: $MAXNOLINK
        ${___}number$__ must be an integer greater than or equal to zero.

        Maximum number of consecutive failed link checks in which the interface have been reset (brought down and up again) before doing new topology check.

        A word of warning: If set to 0 and interface is not up / goes down, infinite retries to bring the interface up will be made before checking topology. Only set it to 0 if you are sure that the specified interface should always be used and you want to make sure it's up before starting to monitor the connection.
        Typically, you would want to also use $_b_--force-interface$__ when using $_b_--max-nolink=0$__.

    $_b_--ifcup | -u $__ ${___}STRING$__
        Default: '$IFCUPT'
        ${___}STRING$__ will be passed to 'sh -c' to bring the interface up.
        $_b_%{IFC}$__ will be dynmaically replaced with the interface name currently in use as source interface.

        Examples:
            ifupdown:
              --ifcup='ifup %{IFC}'

            ifupdown, non privilege user running $_b_$nwwatchdog$__:
              --ifcup='sudo ifup %{IFC}'

            NetworkManager device:
              --ifcup='nmcli device up %{IFC}'

            NetworkManager connection:
              --ifcup='nmcli connection up connection-name'

            iproute2 + isc-dhcp-client:
              --ifcup='ip link set %{IFC} up && dhclient -pf /run/dhclient-%{IFC}.pid %{IFC}'

            strongSwan IPSec (setup for IPSec policy routing):
              --ifcup='ipsec up connection-name'
              (see ${_b_}EXAMPLES$__  below for a more extensive IPSec example using vti tunnel interface)

    $_b_--ifcdown | -U $__ ${___}STRING$__
        Default: '$IFCDOWNT'
        ${___}STRING$__ will be passed to 'sh -c' to bring the interface down.
        $_b_%{IFC}$__ will be dynmaically replaced with the interface name currently in use as source interface.

        Examples:
            ifupdown:
              --ifcdown='ifdown %{IFC}'

            ifupdown, non privilege user running $_b_$nwwatchdog$__:
              --ifcdown='sudo ifdown %{IFC}'

            NetworkManager device:
              --ifcdown='nmcli device down %{IFC}'

            NetworkManager connection:
              --ifcdown='nmcli connection down %{IFC}-connection-name'

            iproute2 + isc-dhcp-client:
              --ifcdown='kill \$(cat /run/dhclient-%{IFC}.pid) ; ip link set down %{IFC}'

            strongSwan IPSec (setup for IPSec policy routing):
              --ifcup='ipsec down connection-name'
              (see ${_b_}EXAMPLES$__  below for a more extensive IPSec example using vti tunnel interface)

    $_b_--alert | -a $__ ${___}STRING$__
        Default: '$ALERTCMDT'

        Errors, warnings and alerts regarding change of state will be piped to: sh -c '${___}STRING$__'
        Within ${___}STRING$__ the following placeholders can be used:

        $_b_%{IFC}$__ - the interface name

        $_b_%{TARGET}$__ - the ${___}TARGET${__} for which the connection is monitored

        $_b_%{TADDR}$__ - the IP address of the ${___}TARGET${__}

        $_b_%{NEXTHOP}$__  - the IP address of the NEXTHOP towards ${___}TARGET${__}

        $_b_%{STATE}$__ - the state of the alert:
        - ${_b_}ERROR$__ for permanent errors
        - ${_b_}WARNING$__ for things that might need reconfiguration
        - ${_b_}INITIAL$__ for intital problems that needs to be resolved before proceeding
        - ${_b_}DOWN$__ for lost conenctivity to ${___}TARGET${__}
            (will not be fired if already in DOWN, UNREACHABLE or LINKDOWN state)
        - ${_b_}UNREACHABLE$__ for lost conenctivity to NEXTHOP (implies DOWN)
            (will not be fired if already in UNREACHABLE or LINKDOWN state)
        - ${_b_}LINKDOWN$__ no link on source interface after $_b_--max-nolink$__ reset attempts (implies UNREACHABLE + DOWN)
            (will not be fired if already in LINKDOWN state)
        - ${_b_}LINKUP$__ link up on source interface
            (will not be fired if already in LINKUP, REACHABLE or UP state)
        - ${_b_}REACHABLE$__ for restored connectivity to NEXTHOP (implies LINKUP)
            (will not be fired if already in REACHABLE or UP state)
        - ${_b_}UP$__ for restored connectivity to ${___}TARGET${__} (implies REACHABLE and LINKUP)
            (will not be fired if already in UP state)

        The alert command will be launched for every ERROR and WARNING, even repeated ones.
        For the other states the alert command will be launched only upon state change.
        E.g. if the state goes from UP to DOWN the DOWN alert is triggered, but if the next check is also DOWN, no further alert will be sent (until the state changes again).

        EXAMPLE of how to email the alert messages using mailx (mailutils) with a custom from address:
            --alert='mailx -a "From: nw-watchdog@this.hst" -s "nw-watchdog %{STATE} alert for %{TARGET} via %{IFC}" my@email.adr'

    $_b_--install-systemd$__ ${___}SERVICENAME$__
        Default: none

        If systemd is installed, a systemd service file is written to /etc/systemd/system/nw-watchdog-${___}SERVICENAME$__.service launching $_b_$nwwatchdog$__ as a daemon with
        $_b_--pidfile$__=/run/nw-watchdog/${___}SERVICENAME$__.pid$__
        $_b_--logfile$__=/var/log/nw-watchdog/${___}SERVICENAME$__.log$__
        and otherwise the exact same options as run (apart from the $_b_--install-systemd$__ option itself of course).

        If /etc/systemd/system/nw-watchdog-${___}SERVICENAME$__.service already exists, the service will be stopped and the file overwritten.

        It will then enable, start and show status of the newly created nw-watchdog-${___}SERVICENAME$__.service.

        The ${___}SERVICENAME$__ must consist of at least 1 valid character ( 'a-z', 'A-Z', '0-9', '-' and '_' ) and be no longer than 236 characters.

        This option requires root privileges.

    $_b_--remove-systemd$__ ${___}SERVICENAME$__ | ${___}UNITNAME$__
        Default: none
        Specify either the short ${___}SERVICENAME$__ or the full ${___}UNITNAME$__ (as shown by $_b_--list-systemd$__) to stop and completely remove the nw-watchdog-${___}SERVICENAME$__.service from the system.

        This option requires root privileges and cannot be combined with any other options.

        To manually (instead of using $_b_--remove-systemd$__) remove the nw-watchdog-${___}SERVICENAME$__.service systemd service completely do:

            sudo systemctl stop nw-watchdog-SERVICENAME.service
            sudo systemctl disable nw-watchdog-SERVICENAME.service
            sudo rm /etc/systemd/system/nw-watchdog-SERVICENAME.service
            sudo systemctl daemon-reload

        Note:
        The logfile, /var/log/nw-watchdog/${___}SERVICENAME$__.log will NOT be removed by $_b_--remove-systemd$__.
        To manually remove the logfile do:
            sudo rm /var/log/nw-watchdog/SERVICENAME.log

${_b_}EXAMPLES$__

    ${___}ISP gateway monitoring:$__

        $_b_$nwwatchdog$__ 1.2.3.4 $_b_\\
EOU
    printf '        %s\n        %s\n        %s\n        %s\n        %s\n' \
           "  $_b_--interval=${__}60 $_b_\\" \
           "  $_b_--no-ping-target $_b_\\" \
           "  $_b_--interface=${__}eth0 $_b_\\" \
           "  $_b_--slow-up-timeout=${__}1 $_b_\\" \
           "  $_b_--ifup-grace=${__}300"

    $FMT<<EOU

    Checking once a minute ($_b_--interval=${__}60) that we have connectivity to the Internet Service Provider's gateway without actually pinging anything on the Internet ($_b_--no-ping-target$__ 1.2.3.4 ... any Intenet address will do) allowing interface and topology detection (not using --force-interface or --max-no-link=0) but still, if down, bring the supposed initial interface towards the ISP up on startup ($_b_--interface=${__}eth0) expecting the ISP gateway to have an RTT below 1 second ($_b_--slow-up-timeout=${__}1) and allowing the interface to be down for up to 5 minutes before considering it a permanent error rechecking topology ($_b_--ifup-grace=${__}300).


    ${___}ISP gateway monitoring with forced interface:$__

        $_b_$nwwatchdog$__ 1.2.3.4 $_b_\\
EOU
    printf '        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n' \
           "  $_b_--no-ping-target \\" \
           "  $_b_--force-interface=${__}eth0 $_b_\\" \
           "  $_b_--slow-up-timeout=${__}1 $_b_\\" \
           "  $_b_--ifup-grace=${__}30 $_b_\\" \
           "  $_b_--max-no-link=${__}0 $_b_\\" \
           "  $_b_--interval=${__}10$__"
    $FMT<<EOU

    Same as above but enforcing the use of the eth0 interface as we know that the ISP gateway should always be reachable via that interface and that is the only Internet facing interface we have. It must be brought up on startup if not already up and there is no use rechecking the topology if it's not up ($_b_--force-interface=${__}eth0), so if down, we retry to reset it every 30 seconds ($_b_--ifup-grace=${__}30) forever ($_b_--max-no-link=${__}0), checking the connectivity every 10 seconds ($_b_--interval=${__}10).


    ${___}Management of Strongswan IPSec with VTI tunnel interface:$__

    Firstly, we start a $_b_$nwwatchdog$__ for monitoring the connectivity to the IPSec peers public address (1.2.3.4 in this example), emailing alerts to the admin.

        $_b_$nwwatchdog$__ 1.2.3.4 $_b_\\
EOU
    printf '        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n' \
           "  $_b_--verbosity-level=${__}3 $_b_\\" \
           "  $_b_--alert=$__'mailx -a \"From: nwwatchdog@\$(hostname -f)\" -s \"IPSec Peer 1.2.3.4 %{STATE} via %{IFC}\" admin@\$(cat /etc/mailname)' $_b_\\" \
           "  $_b_--slow-up-timeout=${__}2 $_b_\\" \
           "  $_b_--ifup-grace=${__}10 $_b_\\" \
           "  $_b_--interval=${__}10$__" \
           "  $_b_--pidfile=/run/nw-watchdog-ipsecserver.pid$__"

    $FMT<<EOU

    Then we setup the monitor for the IPSec VTI tunnel which will also be created, configured and brought up.
    Note that we are using different $_b_--pidfile$__ for the two watchdogs allowing them to run simultaneously.
    It would be smoother to use a script and $_b_--ifcup=${__}/path/script, but it can also be done like this:

        $_b_$nwwatchdog$__ 169.254.0.1 $_b_\\
EOU
    printf '        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n' \
           "  $_b_--alert=$__'mailx -a \"From: nwwatchdog@\$(hostname -f)\" -s \"IPSec connection-name %{STATE} via %{IFC}\" admin@\$(cat /etc/mailname)' $_b_\\" \
           "  $_b_--force-interface=${__}ipsec0 $_b_\\" \
           "  $_b_--slow-up-timeout=${__}5 $_b_\\" \
           "  $_b_--ifup-grace=${__}25 $_b_\\" \
           "  $_b_--ifcup=$__'ipsec up connection-name ; " \
           "           ip tunnel add %{IFC} mode vti local 4.3.2.1 remote 1.2.3.4 ttl 255 key 111 ; " \
           "           ip addr add dev %{IFC} 169.254.0.2 remote 169.254.0.1 ; " \
           "           ip link set %{IFC} up multicast on mtu 1424 state UP ;" \
           "           ip route add 10.10.0.0/20 via 169.254.0.1 dev %{IFC}'$__ $_b_\\" \
           "  $_b_--ifcdown=$__'ipsec down connection-name ; ip tunnel del %{IFC}' $_b_" \
           "  $_b_--pidfile=/run/nw-watchdog-ipsectunnel.pid$__"
    $FMT<<EOU

   $___${_b_}OBSERVE$__ that the entire $_b_--ifcup=$__'...' command needs to be on a single line without line breaks (linebreaks added above for readability):

EOU
    printf '        %s\n' "$_b_--ifcup=$__'ipsec up connection-name ; ip tunnel add %{IFC} mode vti local 4.3.2.1 remote 1.2.3.4 ttl 255 key 111 ; ip addr add dev %{IFC} 169.254.0.2 remote 169.254.0.1 ; ip link set %{IFC} up multicast on mtu 1424 state UP ; ip route add 10.10.0.0/20 via 169.254.0.1 dev %{IFC}'"
    $FMT<<EOU

    We use $_b_--ifup-grace=25$__ to allow enough time for IPSec to establish the connection upon start and interface reset.
    $_b_--slow-up-timeout=5$__ (making the deadline 25 seconds for the slow-up ping test) should be enough for an idle IPSec connection to wakeup and start forwarding packets when monitored.

    In the above example we monitor the connectivity to the peer address inside the VTI tunnel, which is also the NEXTHOP.
    If we are interested in the connectivity to something in the remote network routed via the tunnel we could use that as a ${___}TARGET${__} instead of the peer address:

        $_b_$nwwatchdog$__ 10.10.1.1 $_b_ ...
    and the rest of the options exactly the same.

    This would give alerts if 10.10.1.1 is down.
    In all other ways the effect would be the same as using 169.254.0.1 as ${___}TARGET${__}.
    Even with ${___}TARGET${__} 10.10.1.1, the NEXTHOP (169.254.0.1) will also be monitored (we are NOT using --no-ping-nexthop) and the interface will NOT be reset as long as the NEXTHOP is reachable.


    ${___}Wireguard full tunnel management:$__

    This is an example of how one can use $_b_$nwwatchdog$__ to setup and monitor a wireguard full tunnel, also monitoring the connectivity to the wireguard server, running both whatchdogs as systemd services getting alerts via e-mail:

    Firstly, we setup the $_b_$nwwatchdog$__ systemd service for the wireguard server which we reach via the default route:

        $_b_$nwwatchdog$__ wgserver.domain.dom $_b_\\
EOU
    printf '        %s\n        %s\n        %s\n        %s\n        %s\n' \
           "  $_b_--verbosity-level=${__}3 $_b_\\" \
           "  $_b_--alert=$__'mailx -a \"From: nwwatchdog@\$(hostname -f)\" -s \"wgserver %{STATE} via %{IFC}\" admin@\$(cat /etc/mailname)' $_b_\\" \
           "  $_b_--slow-up-timeout=${__}3 $_b_\\" \
           "  $_b_--ifup-grace=${__}20 $_b_\\" \
           "  $_b_--install-systemd=${__}wgserver"
    $FMT<<EOU

    Then we setup the monitor for the wireguard full tunnel which will also create and configure and bring the tunnel up (if not already) upon start of the sytemd service, making sure we have a /32 routes to all of the wireguard server's ip addresses that the hostname resolves to.
    Note: It would be smoother to use a script and $_b_--ifcup=${__}/path/script, but it can also be done like this:

        $_b_$nwwatchdog$__ 10.0.0.1 $_b_\\
EOU
    printf '        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n        %s\n' \
           "  $_b_--no-ping-nexthop \\" \
           "  $_b_--verbosity-level=${__}3 $_b_\\" \
           "  $_b_--alert=$__'mailx -a \"From: nwwatchdog@\$(hostname -f)\" -s \"wireguard wg0\" admin@\$(cat /etc/mailname)' $_b_\\" \
           "  $_b_--force-interface=${__}wg0 $_b_\\" \
           "  $_b_--ifcup=$__'ip link add wg0 type wireguard ;" \
           "           ip link set wg0 up ;" \
           "           wg setconf wg0 /etc/wireguard/wg0.conf ;" \
           "           ip address add 10.0.0.2 peer 10.0.0.1 dev wg0 ;" \
           "           getent ahostsv4 wgserver.my.dom | grep -oE \"^[0-9.]+\" | uniq" \
           "           | while read addr; do" \
           "               ip route add \$addr/32 via \$(ip route show default | head -1 | while read -r _ _ i _; do echo \$i; done) ;" \
           "             done ;" \
           "           ip route add 0.0.0.0/1 via 10.0.0.1 dev wg0 src 10.0.0.2 ;" \
           "           ip route add 128.0.0.0/1 via 10.0.0.1 dev wg0 src 10.0.0.2'$__ $_b_\\" \
           "  $_b_--ifcdown=$__'ip link del wg0' $_b_\\" \
           "  $_b_--install-systemd=${__}wg0"

    $FMT<<EOU

    $___${_b_}OBSERVE$__ that the entire $_b_--ifcup=$__'...' command needs to be on a single line without line breaks (linebreaks added above for readability):

EOU
    printf '        %s\n' "$_b_--ifcup=$__'ip link add wg0 type wireguard ; ip link set wg0 up ; wg setconf wg0 /etc/wireguard/wg0.conf ; ip address add 10.0.0.2 peer 10.0.0.1 dev wg0 ; getent ahostsv4 wgserver.my.dom | grep -oE \"^[0-9.]+\" | uniq | while read addr; do ip route add \$addr/32 via \$(ip route show default | head -1 | while read -r _ _ i _; do echo \$i; done) ; done ; ip route add 0.0.0.0/1 via 10.0.0.1 dev wg0 src 10.0.0.2 ; ip route add 128.0.0.0/1 via 10.0.0.1 dev wg0 src 10.0.0.2'"
    $FMT<<EOU

    We use $_b_--no-ping-nexthop$__ as the NEXTHOP is the same as the ${___}TARGET${__} peer-to-peer address we monitor the connection for (in reality we don't need to specify it as it is the default behaviour if the ${___}TARGET${__} address is the same as the NEXTHOP address).


    ${___}Several VPN paths with one preferred interface:$__

    This is a (real life) cornor case but worth explaining to get an understanding of the capabilities of $_b_$nwwatchdog$__.

    We have setup ifupdown to handle three different vpn-interfaces: vpnL vpnP and vpnF
    They all use the same VPN server but have different routes via the server depending on which interface is up.
    Only one of the interfaces can be up at any given time.

    If vpnL is up we route just to the peer local network 10.0.10.0/24 via it.
    If vpnP is up we route to all private addresses via it, including 10.0.0.0/8
    If vpnF is up we use it as a full tunnel, routing all traffic (apart from the VPN-connection itself) via it.

    Our preferred interface is vpnL and if we can't get any traffic through the vpn-server, vpnL is the interface we want to reset.
    BUT as long as we can get traffic through using any of the three interfaces, we don't want to get alerted and have any interface reset, so we can't use $_b_--force-interface$__. The soloution will be to allow continuous topology detection but hardcode the preferred interface in $_b_--ifcup$__ and $_b_--ifcdown$_b_.

        $_b_$nwwatchdog$__ vpn.inside.dom \\
          $_b_--interface=${__}vpnL \\
          $_b_--no-ping-nexthop$__ \\
          $_b_--slow-up-timeout=${__}7 \\
          $_b_--ifup-grace=${__}35 \\
          $_b_--ifcdown=${__}'ifdown vnpL ; ifdown vpnP  ; ifdown vpnF' \\
          $_b_--ifcup=${__}'ifup vpnL'

    Starting the watchdog like the above with any of the three vpn-interfaces up, the vpn-interface that is up will be detected and used as source interface. The continuous topology detection will ensure switching to which ever interface currently is up. If all interfaces are down the topology detection will think the interface for the default gw is the one to monitor (in our example eth0) but since we have $_b_--no-ping-nexthop$__ and hardcoded the preferred vpn-interface in $_b_--ifcup$__ the watchdog will bring up vpnL. If any of the vpn interfaces are up, but have problems the watchdog will bring them all down and then bring up vpnL.

    We add $_b_--verbosity-level=${__}5 to the above, allowing us to get a trace of what is happening in the logfile in the following scenario:
    vpnL is up when $_b_$nwwatchdog$__ starts,
    after a while we see that vpnF is replacing vpnL (as somebody brought up the full tunnel for a while)
    then vpnF is brought down (as somebody did not need it anymore) and
    $_b_$nwwatchdog$__ detetcs that the connectivity via the VPN-server is lost and brings up vpnL:

EOU
    cat <<EOU
    00:00:01   INFO: Target (vpn.inside.dom) resolved to 10.0.10.1 (instead of '').
    00:00:01   INFO: Detected topology: IFC='vpnL' -> 'vpnL' ; NEXTHOP='' -> '10.0.10.1'
$_b_    ^^^^^^^^^^^^^^^^ $__
$_b_    This shows that vpnL is the interface that is up at start $__

    00:00:01  TRACE: Continuously checking if target is up...
    00:00:01  TRACE: ... and for changes in topology
    00:00:26   INFO: Detected topology: IFC='vpnL' -> 'vpnF' ; NEXTHOP='10.0.10.1' -> '10.0.10.1'
$_b_    ^^^^^^^^^^^^^^^^ $__
$_b_    Here somebody replaced vpnL with the full tunnel vpnF $__

    00:00:31   INFO: Detected topology: IFC='vpnF' -> 'eth0' ; NEXTHOP='10.0.10.1' -> '192.168.0.1'
$_b_    ^^^^^^^^^^^^^^^^ $__
$_b_    and here the full tunnel was brought down again (without replacing it with any of the other vpn-interfaces) $__

    00:00:32  TRACE: quick-up failed ... trying slow-up ...
    00:00:32  TRACE: slow-up failed or ambigious result ... verifying ...
    00:00:34  TRACE: No reply from target 'vpn.inside.dom' (10.0.10.1), checking link and topology.
    00:00:35  TRACE: quick-up failed ... trying slow-up ...
    00:00:35  TRACE: slow-up failed or ambigious result ... verifying ...
$_b_    ^^^^^^^^^^^^^^^^ $__
$_b_    Here the $nwwatchdog gives up and concludes that the ${___}TARGET${__}$_b_ is down. $__

    00:00:37  ALERT: DOWN - Not getting replies from target 'vpn.inside.dom' (10.0.10.1) on interface 'eth0'.
                     Resetting interface:
                     ifdown vpnL ; ifdown vpnP ; ifdown vpnF
                     ifup vpnL
    00:00:37   INFO: Resetting interface (eth0).
$_b_    ^^^^^^^^^^^^^^^^                      ^^^^ $__
$_b_    This can be confusing since it's actually not eth0 that is reset (but it's the current source interface) $__
$_b_    Below we see that the watchdog actually brings down all the vpn-interfaces and brings up vpnL $__

    00:00:37  TRACE: sh -c 'ifdown vpnL ; ifdown vpnP ; ifdown vpnF'
    00:00:38  TRACE: sh -c 'ifup vpnL'
    00:00:38  TRACE: Sleeping for 35 seconds.
$_b_    ^^^^^^^^^^^^^^^^ $__
$_b_    This is the grace period we have set in --ifup-grace $__
    00:01:13   INFO: Detected topology: IFC='eth0' -> 'vpnL' ; NEXTHOP='192.168.0.1' -> '10.0.10.1'
    00:01:13  TRACE: Continuously checking if target is up...
    00:01:13  TRACE: ... and for changes in topology
    00:01:13  ALERT: UP - Target 'vpn.inside.dom' (10.0.10.1) is up.
EOU
    $FMT <<EOU

${_b_}DEPENDENCIES$__
    $_b_$nwwatchdog$__ depends on the below executables being available in "/sbin:/bin:/usr/sbin:/usr/bin:\$PATH" or being shell-builtin. A check is done at startup and if any of these tools are missing, $_b_$nwwatchdog$__ will exit with an error telling which are lacking.

EOU
    for dep in sh printf which $DEPENDENCIES; do
        printf "    - ${_b_}%s$__\n" "$dep"
    done
    $FMT<<EOU

    $_b_$nwwatchdog$__ will function without the below listed utilities, but will use them to enhance its functionality if available.

    - ${_b_}flock$__
        If available the logfile will be locked before truncated or written to.
        If not available, there is a slight risk of log entries being lost if two or more instances of $_b_$nwwatchdog$__ are concurently running using the same logfile and at least one of them have $_b_--logsize$__ set to a value larger than 0.

    - ${_b_}fmt$__
        Is used to format the help message if available.

    - ${_b_}less$__ (or ${_b_}more$__)
        Is used to page this help unless \$PAGER is set to something else.

    - ${_b_}tput$__
        Is used to determine the terminal width and output bold and underlined text in this help page.

    - ${_b_}wall$__
        If available it will be used as default $_b_--alert$__ command. Otherwise, alerting will be done to stderr by default.

    - Installed and running ${_b_}systemd$__  with ${_b_}systemctl$__ (and ${_b_}mkdir$__, ${_b_}rm$__)
        ${_b_}systemd$__ is (obviously) required for the ${_b_}--install-systemd$__, ${_b_}--list-systemd$__ and ${_b_}--remove-systemd$__ options to be functional.
        The folder ${_b_}/etc/systemd/system/$__ must exist, ${_b_}systemctl$__ must be in the PATH and ${_b_}systemd$__ must be running.
        Unless all required directories are already existing, ${_b_}mkdir$__ is used by ${_b_}--install-systemd$__.
        ${_b_}rm$__ is used by ${_b_}--remove-systemd$__ to remove the systemd unit file for the service.

EOU
    _show_errmsg "$@"
}
USAGE() {
    [ "$PAGERUSE" ] || {
        _USAGE "$@"
        [ "$ERRMSG" ] || [ "$1" ] || exit 0
        exit 1
    }
    if which "$PAGER" >/dev/null 2>&1; then
        LESS="$LESS -R"; export LESS # make sure less, if used, passes ANSI color escape sequences to terminal
        _USAGE "$@" | "$PAGER"
    elif which less >/dev/null 2>&1; then
        _USAGE "$@" | less -R
    elif which more >/dev/null 2>&1; then
        _USAGE "$@" | more
    else
        _USAGE "$@"
    fi
    [ "$ERRMSG" ] || [ "$1" ] || exit 0
    exit 1
}
SOPTRE=''
for o in $SOPTIONS; do
    [ "$SOPTRE" ] && SOPTRE="$SOPTRE|"
    eval "r='\${${o}RE'}"; eval "SOPTRE=\"$SOPTRE$r\"" # should be \" or SOPTRE will contain variabel names instead of content
done
SOPTRE="($SOPTRE)"
AOPTRE=''
for o in $AOPTIONS; do
    [ "$AOPTRE" ] && AOPTRE="$AOPTRE|"
    eval "r='\${${o}RE'}"; eval "AOPTRE=\"$AOPTRE$r\"" # should be \" or AOPTRE will contain variabel names instead of content
done
AOPTRE="($AOPTRE)"
NDOPTRE=$(printf '%s|%s' "$SOPTRE" "$AOPTRE" | sed -E 's/\(-*|\)//g' | sed -E 's/\|-*/\|/g')

xOPTIONS=:
while [ $# -gt 0 ]; do
    ARG=$1; shift
    if printf %s "$ARG" | grep -qE "^${SOPTRE}$"; then                      # No Argument Option
        for o in $SOPTIONS; do
            eval "R=\"\${${o}RE}\"" # should be \" or R will contain variabel names instead of content
            if printf %s "$ARG" | grep -qE "^$R$"; then
                eval "$o=\"\${${o}V}\"" # should be \" or o will contain variabel names instead of content
                xOPTIONS="${xOPTIONS}${o}:"
                break
            fi
        done
    elif printf %s "$ARG" | grep -qE "^-${SOPTSRE}(${NDOPTRE})"; then       # Grouped Options
        A=$(printf %s "$ARG" | grep -oE '^..')
        B=$(printf %s "$ARG" | cut -c3-)
        set -- "$A" "-$B" "$@"
    elif printf %s "$ARG" | grep -qE "^${AOPTRE}"; then                     # Option with Argument
        if printf %s "$ARG" | grep -qE "^${AOPTRE}$"; then                  # Option [space] Argument
            OPT=$ARG
            ARG=$1
            [ $# -gt 0 ] && shift
        elif printf %s "$ARG" | grep -qE  "^${AOPTRE}=[^[:space:]]"; then # Option [equals] Argument
            OPT=${ARG%%=*}
            ARG=${ARG#*=}
        else                                                                # Option [no-space] Argument
            OPT=$(printf %s "$ARG" | grep -oE "^${AOPTRE}")
            ARG=$(printf %s "$ARG" | sed -E "s/^${AOPTRE}//")
            [ "$ARG" = '=' ] && {
                ERRMSG="'$OPT=' seem to lack an arugment.\tIf the argument really should be '=', use the syntax '$OPT=='.\n$ERRMSG"
                ARG=''
            }
        fi
        [ "$ARG" ] || ERRMSG="Option '$OPT' requires an argument.\n$ERRMSG"
        for o in $AOPTIONS; do
            eval "R=\"\${${o}RE}\"" # should be \" or AR will contain variabel names instead of content
            if printf %s "$OPT" | grep -qE "^$R$"; then
                if printf %s "$xOPTIONS" | grep -qE ":$o:"; then
                    eval "oARG=\"\$${o}\""
                    [ "$oARG" = "$ARG" ] || {
                        ERRMSG="Option '$OPT' $R specified multiple times with different values.\n$ERRMSG"
                        break
                    }
                else
                    xOPTIONS="${xOPTIONS}${o}:"
                fi
                eval "AR=\"\${${o}ARE}\"" # should be \" or AR will contain variabel names instead of content
                printf %s "$ARG" | grep -qE "^$AR$" || {
                    ERRMSG="Invalid argument '$ARG' for option '$OPT' $R.\n$ERRMSG"
                    break
                }
                eval "$o='$ARG'"
                break
            fi
        done
    else                                                   # TARGET
        if printf %s "$ARG" | grep -qE '^-'; then
            ERRMSG="Invalid OPTION: '$ARG'\n$ERRMSG"
        else
            [ "$TARGET" ] && ERRMSG="Multiple TARGETs specified ('$TARGET' + '$ARG').\n$ERRMSG"
            TARGET=$ARG
            printf %s "$TARGET" \
                | grep -qE '^((0*[0-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))\.){3}(0*[0-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))$' \
                || printf %s "$TARGET" \
                    | grep -qE '^[a-zA-Z0-9_][a-zA-Z0-9_.-]*$' || {
                    ERRMSG="Invalid TARGET '$TARGET'. Must be an IP address or a valid hostname.\n$ERRMSG"
                }
        fi
    fi
done

[ "$VERSHOW" ] && { printf '%s\n' "$VERSION"; exit 0; }
[ "$HELP" ] && ERRMSG='' USAGE

[ "$LISTSYSTEMD" ] || printf %s "$xOPTIONS" | grep -qE ':RMSYSTEMDSERVICE:' && {
        printf %s "$xOPTIONS" | grep -qE ':.*:.*:' || [ "$TARGET" ] && \
            ERRMSG='' USAGE %s "--list-systemd and --remove-systemd cannot be combined with any other option"
        [ -d /etc/systemd/system/. ] || USAGE "'/etc/systemd/system/' does not exists, is systemd installed and running?"
        [  -n "$LISTSYSTEMD" ] && {
            printf '\n\n--- Listing all loaded nw-watchdog systemd units: ---\n'
            systemctl list-units --all nw-watchdog-\* 2>/dev/null | grep -oE 'nw-watchdog-.*\.service.*'
            printf '\n--- Listing all installed nw-watchdog systemd units: ---\n'
            systemctl list-unit-files --all nw-watchdog-\* 2>/dev/null
            exit 0
        }
        [ "$RMSYSTEMDSERVICE" ] || ERRMSG='' USAGE %s "--remove-systemd requires a service or unit name to remove"
        [ $(id -u) -eq 0 ] || USAGE %s "$nwwatchdog --remove-systemd must be run as root"
        systemctl daemon-reload || USAGE %s "Failed to reload systemd. Is systemd installed and running?"
        UNITNAME=$RMSYSTEMDSERVICE
        systemctl list-unit-files "$UNITNAME" >/dev/null 2>&1 || {
            UNITNAME="nw-watchdog-$RMSYSTEMDSERVICE.service"
            systemctl list-unit-files "$UNITNAME" >/dev/null 2>&1 \
                || USAGE 'Neither "%s" or "%s" is a valid systemd service unit name.\n       %s' \
                         "$RMSYSTEMDSERVICE" "$UNITNAME" \
                         'Use --list-systemd to list all nw-watchdog systemd units.'
        }
        systemctl stop "$UNITNAME" || printf '\nWARNING: failed to stop "%s"\n' "$UNITNAME" 1>&2
        systemctl disable "$UNITNAME" || printf '\nWARNING: failed to disable "%s"\n' "$UNITNAME" 1>&2
        rm -vf "/etc/systemd/system/$UNITNAME" || printf '\nWARNING: failed to remove "/etc/systemd/system/%s"\n' "$UNITNAME" 1>&2
        systemctl daemon-reload || printf '\nWARNING: Failed to reload systemd.\n' 1>&2
        exit 0
    }

[ "$PINGTARGET" ] || [ "$PINGNEXTHOP" ] || \
    ERRMSG="You must allow pinging of target and/or nexthop!\t(Can't combine --no-ping-target and --no-ping-nexthop!)\n$ERRMSG"

[ "$INSTALLSYSTEMD" ] && [ "$DEBUG" ] && ERRMSG="--install-systemd cannot be combined with --debug\n$ERRMSG"

[ "$TARGET" ] || ERRMSG="TARGET not specified! Specify IP address or valid hostname.\n$ERRMSG"

### Die showing the USAGE if found errors in options and arguments above ###
[ "$ERRMSG" ] && USAGE %s 'Option Errors'

[ "$DEBUG" ] && {
    printf %s "$xOPTIONS" | grep -qE ':V:'             || V=6
    printf %s "$xOPTIONS" | grep -qE ':LOGFILE:'       || LOGFILE=-
    printf %s "$xOPTIONS" | grep -qE ':PIDFILE:'       || PIDFILE=-
    printf %s "$xOPTIONS" | grep -qE ':SLOWUPTIMEOUT:' || SLOWUPTIMEOUT=1
    printf %s "$xOPTIONS" | grep -qE ':SLEEP:'         || SLEEP=3
    printf %s "$xOPTIONS" | grep -qE ':GRACE:'         || GRACE=5
    printf %s "$xOPTIONS" | grep -qE ':ALERTCMDT:'     || ALERTCMDT='cat'
    printf %s "$xOPTIONS" | grep -qE ':ALERTCMD:'      || ALERTCMD='cat'
    FORK=''
}
[ "$LOGFILE" = '-' ] && { LOGSIZE=0 ; LOGSIZEb=0 ; LOGFILE='' ;}
[ "$PIDFILE" = '-' ] && PIDFILE=/dev/null

LOGSIZEb=$LOGSIZE
case "$LOGSIZEb" in
    *[Kk]) LOGSIZEb=$(printf %s "$LOGSIZEb" | sed -E 's/[Kk]$//') ;;
    *[Mm]) LOGSIZEb=$(printf %s "$LOGSIZEb" | sed -E 's/[Mm]$//'); LOGSIZEb=$((LOGSIZEb*1024)) ;;
    *[Gg]) LOGSIZEb=$(printf %s "$LOGSIZEb" | sed -E 's/[Gg]$//'); LOGSIZEb=$((LOGSIZEb*1048576)) ;;
esac
LOGSIZEb=$((LOGSIZEb*1024))
dienl() {
    diePATN="$1 ... aborting!" ; shift
    _show_errmsg "$diePATN" "$@"
    exit 1
}
[ "$LOGFILE" ] && {
    [ "$SYSTEMDSERVICENAME" ] || {
        LOGDIR=${LOGFILE%/*}
        if [ -d "$LOGDIR/." ] || mkdir -p "$LOGDIR"; then
            touch "$LOGFILE" 2>/dev/null || dienl '%s\n%s' "Cannot write to logfile '$LOGFILE'." \
                                                  'Fix permissions, run as user with access or specify another --logfile.'
        else
            dienl 'Cannot create non-exiting directory for --logfile="%s"\n%s' "$LOGFILE" \
                  'Fix permissions, run as user with access or specify another --logfile.'
        fi
    }
}

[ "$STATICIFC" ] && { CONTINUOUSTOPOLOGYDETECT='' ; IFC=$STATICIFC ;}

log() {
    logPATN="%s  $1"; shift
    logentry=$(printf "$logPATN" "$(date '+%Y-%m-%d %H:%M:%S %z')" "$@")
    if [ "$LOGFILE" ]; then
        [ "$LOGSIZEb" -gt 0 ] && {
            # Remove top 10 lines until log file is smaller than LOGSIZEb,
            # silently skip if we can't lock the file
            if which flock >/dev/null 2>&1; then
                ok=y
                while [ "$ok" ] && [ $(stat --printf=%s "$LOGFILE") -gt $LOGSIZEb ]; do
                    flock -w1 "$LOGFILE" -c "cp '$LOGFILE' '$LOGFILE.$$' ; tail -n +11 '$LOGFILE.$$' >'$LOGFILE'; rm '$LOGFILE.$$'" || ok=''
                done
            else
                while [ $(stat --printf=%s $LOGFILE) -gt $LOGSIZEb ]; do
                    cp "$LOGFILE" "$LOGFILE.$$" ; tail -n +11 "$LOGFILE.$$" > "$LOGFILE" ; rm "$LOGFILE.$$"
                done
            fi
        }
        if which flock >/dev/null 2>&1; then
            flock -w1 "$LOGFILE" -c "printf '%s\n' \"$logentry\" >>'$LOGFILE'" || {
                [ $V -ge 2 ] && {
                    printf 'WARNING: Could not lock logfile temporarily logging to "%s"\n' "$LOGFILE.$$" 1>&2
                }
                TLOGFILE=$LOGFILE
                LOGFILE="$LOGFILE.$$"
                printf '%s\n' "$logentry" >>"$LOGFILE"
                LOGFILE=$TLOGFILE
            }
        else
            printf '%s\n' "$logentry" >>"$LOGFILE"
        fi
    else
        printf '%s\n' "$logentry"
    fi
}
ALERTSTATE='INITIAL'
alert() {
    [ $V -ge 3 ] || return
    STATE=$1; shift
    [ "$STATE" = ERROR ] || [ "$STATE" = WARNING ] || [ "$STATE" = INITIAL ] || { # always alert on ERROR, WARNING and INITIAL
        [ "$ALERTSTATE" = INITIAL ] && {                                          # Do not alert if we are up after INITIAL
            [ "$STATE" = UP ] || [ "$STATE" = REACHABLE ] || [ "$STATE" = LINKUP ] && return
        }
        [ "$ALERTSTATE" = "$STATE" ] && return                                # Same state
        [ "$ALERTSTATE" = LINKDOWN ] && [ "$STATE" = DOWN ] && return         # LINKDOWN implies DOWN
        [ "$ALERTSTATE" = LINKDOWN ] && [ "$STATE" = UNREACHABLE ] && return  # LINKDOWN implies UNREACHABLE
        [ "$ALERTSTATE" = UNREACHABLE ] && [ "$STATE" = DOWN ] && return      # UNREACHABLE implies DOWN
        [ "$ALERTSTATE" = UP ] && [ "$STATE" = REACHABLE ] && return          # UP implies REACHABLE
        [ "$ALERTSTATE" = UP ] && [ "$STATE" = LINKUP ] && return             # UP implies LINKUP
        [ "$REACHABLE"  = REACHABLE ] && [ "$STATE" = LINKUP ] && return      # REACHABLE implies LINKUP
    }
    ALERTCMD=$(printf %s "$ALERTCMDT" | sed -E "s/\%\{STATE\}/$STATE/g" | sed -E "s/\%\{LASTSTATE\}/$ALERTSTATE/g")
    if [ "$TARGET" ]; then
        ALERTCMD=$(printf %s "$ALERTCMD" | sed -E "s/\%\{TARGET\}/$TARGET/g")
    else
        ALERTCMD=$(printf %s "$ALERTCMD" | sed -E "s/\%\{TARGET\}/unspecified-target/g")
    fi

    if [ "$IFC" ]; then
        ALERTCMD=$(printf %s "$ALERTCMD" | sed -E "s/\%\{IFC\}/$IFC/g")
    else
        ALERTCMD=$(printf %s "$ALERTCMD" | sed -E "s/\%\{IFC\}/topology-not-yet-detected/g")
    fi

    if [ "$TADDR" ]; then
        ALERTCMD=$(printf %s "$ALERTCMD" | sed -E "s/\%\{TADDR\}/$TADDR/g")
    else
        ALERTCMD=$(printf %s "$ALERTCMD" | sed -E "s/\%\{TADDR\}/unresolved-target/g")
    fi

    if [ "$NEXTHOP" ]; then
        ALERTCMD=$(printf %s "$ALERTCMD" | sed -E "s/\%\{NEXTHOP\}/$NEXTHOP/g")
    else
        ALERTCMD=$(printf %s "$ALERTCMD" | sed -E "s/\%\{NEXTHOP\}/topology-not-yet-detected/g")
    fi

    alertPATN=$1 ; shift
    printf "\n\n$nwwatchdog $STATE ALERT!!!\n$alertPATN\n\n\n" "$@" | sh -c "$ALERTCMD"

    case "$STATE" in
        "ERROR") LSTATE='  ERROR:' ;;
        "WARNING") LSTATE='WARNING:' ;;
        *)
            ALERTSTATE=$STATE
            LSTATE="  ALERT: $STATE -"
            ;;
    esac
    log "$LSTATE $alertPATN" "$@"
}
error() {
    [ $V -ge 1 ] || return
    alert ERROR "$@"
}
die() {
    diePATN="$1 ... aborting!" ; shift
    error "$diePATN" "$@"
    _show_errmsg "$diePATN" "$@"
    exit 1
}
warn() {
    [ $V -ge 2 ] || return
    alert WARNING "$@"
}
info() {
    [ $V -ge 4 ] || return
    infoPATN=$1 ; shift
    log "   INFO: $infoPATN" "$@"
}
trace() {
    [ $V -ge 5 ] || return
    tracePATN=$1 ; shift
    log "  TRACE: $tracePATN" "$@"
}
debug() {
    [ $V -ge 6 ] || return
    debugPATN=$1 ; shift
    log "  DEBUG: $debugPATN" "$@"
}

ifreset() {
    [ "$IFCUPDOWN" ] || {
        info 'Not resetting interface due to --no-interface-reset.'
        return 0
    }
    [ "$IFC" ] || die 'Internal ERROR: No interface to reset!'
    info 'Resetting interface (%s).' "$IFC"
    trace 'sh -c "%s"' "$IFCDOWN"
    IFCDD=$(sh -c "$IFCDOWN" 2>&1)
    EXITCODE=$?

    debug 'ifcdown: "%s"\n\texit code %d and output:\n\t%s' "$IFCDOWN" $EXITCODE "$IFCDD"
    sleep 1
    trace 'sh -c "%s"' "$IFCUP"
    EXITCODE=0
    IFCUD=$(sh -c "$IFCUP" 2>&1) || {
        EXITCODE=$?
        debug 'ifcup: "%s"\n\texit code %d and output:\n\t%s' "$IFCUP" $EXITCODE "$IFCUD"
        warn '%s\n\t%s' \
             "Could not reset interface '$IFC'." \
             "Check your --if-up and --if-down commands and/or privilegdes of effective user (userid=$(id -u))."
        [ "$1" ] && return 1
    }
    debug '"%s" exited with %d and output: "%s"' "$IFCUP" $EXITCODE "$IFCUD"
    trace 'Sleeping for %d seconds.' $GRACE
    sleep $GRACE
    return $EXITCODE
}

linkcheck() {
    debug 'Checking "%s" link state' "$IFC"
    [ "$IFC" ] || die 'Internal ERROR: No interface to check link for!'
    NOLINK=0
    while :; do
        ip link show $IFC>/dev/null 2>&1 && {
            ip link show $IFC | grep -oq 'state DOWN' || {
                IPADDRS=''; ip addr show dev $IFC | grep -qE 'inet6? .* scope global' && IPADDRS=y
                [ "$IPADDRS" ] && [ "$IPADDRALRT" ] \
                    || warn '%s\n\t%s' \
                            "There is no IPv4 or IPv6 global scope ip address on interface '$IFC'!" \
                            "If that is ok, you can disable these alerts with --no-ipaddr-alert."
                alert LINKUP 'Link up on interface "%s"!' "$IFC"
                return 0
            }
        }
        if [ "$IFCUPDOWN" ]; then
            NOLINK=$((NOLINK+1))
            [ $MAXNOLINK -gt 0 -a $NOLINK -gt $MAXNOLINK ] && {
                ip link show $IFC>/dev/null 2>&1 || {
                    alert LINKDOWN 'Interface "%s" ... device does not exist!' "$IFC"
                    return 2
                }
                alert LINKDOWN 'Link down on interface "%s" after %d reset attempts (with %d seconds in between)!' "$IFC" $MAXNOLINK $GRACE
                return 1
            }
            info 'No link on "%s"... reset attempt %d out of %d (0=infinite)' "$IFC" $NOLINK $MAXNOLINK
            ifreset "$1"
        else
            alert LINKDOWN 'Link down on interface "%s"!' "$IFC"
            return 1
        fi
    done
}

nexthop() {
    NIFC=$(ip route get $TADDR | grep -o 'dev [^ ]*' | while read -r _ h _; do echo $h; done)
    if [ "$NIFC" ]; then
        NNEXTHOP=$(ip route get $TADDR | grep -oE 'via [^ ]*' | while read -r _ h _; do echo $h; done)
        [ "$NNEXTHOP" ] || {
            NNEXTHOP=$TADDR
            debug 'No nexthop detected (ok if same subnet (incl. host-local addr) or peer-to-peer addr), using target as nexthop... '
            # If it's a host-local address 'ip route get' will say lo is source interface even if addr is on different interface
            # which will cause 'ping -I$IFC' to fail, so let's figure which interface holds the address is instead
            [ "$NIFC" = lo ] && {
                nexthopADDR=$(printf %s "$TADDR" | sed -E 's/\./\\./g')
                ip addr | grep -qE "inet $nexthopADDR( |\/)" && {
                    LIFC=$(ip addr | grep -B9999 -E "inet $nexthopADDR( |\/)" | grep -oE '^[0-9]+:\s+[^@:]+' | tail -1 | sed -E 's/.*\s//')
                    [ "$LIFC" ] && {
                        debug 'Found %s on local interface "%s" (overriding "%s")' "$TADDR" "$LIFC" "$NIFC"
                        NIFC=$LIFC
                    }
                }
            }
        }
        if [ "$IFC" = "$NIFC" ] && [ "$NEXTHOP" = "$NNEXTHOP" ]; then
            debug 'No topology changes detected. IFC="%s" NEXTHOP="%s"' "$IFC" "$NEXTHOP"
            return 0
        elif [ -z "$STATICIFC" ]; then
            info 'Detected topology: IFC="%s" -> "%s" ; NEXTHOP="%s" -> "%s"' "$IFC" "$NIFC" "$NEXTHOP" "$NNEXTHOP"
            IFC=$NIFC
            NEXTHOP=$NNEXTHOP
            IFCUP=$(printf %s "$IFCUPT" | sed -E "s/\%\{IFC\}/$IFC/g")
            IFCDOWN=$(printf %s "$IFCDOWNT" | sed -E "s/\%\{IFC\}/$IFC/g")
            return 0
        elif [ "$STATICIFC" = "$NIFC" ]; then
            info 'Detected topology on forced interface "%s": NEXTHOP="%s" -> "%s"' "$IFC" "$NEXTHOP" "$NNEXTHOP"
            debug 'STATICIFC="%s" IFC="%s" NIFC="%s" NEXTHOP="%s" NNEXTHOP="%s"' \
                  "$STATICIFC" "$IFC" "$NIFC" "$NEXTHOP" "$NNEXTHOP"
            NEXTHOP=$NNEXTHOP
            return 0
        else
            info 'Ignoring conflicting topology: Route to target "%s" (%s) is on interface "%s", but --force-interface="%s"' \
                 "$TARGET" "$TADDR" "$NIFC" "$STATICIFC"
            return 1
        fi
    else
        info 'No route to target "%s" (%s) available.' "$TARGET" "$TADDR"
        debug 'Hanging on to interface "%s".' "$IFC"
        return 1
    fi
}

[ "$FORK" ] || [ "$SYSTEMDSERVICENAME" ] && {
    debug 'MAINPID=%d BASENAME=%s COMM=%s' $$ "$nwwatchdog" "$(cat /proc/$$/comm)"

    NOPTS='M' # don't use a pager for output from background / systemd processes
    [ "$PINGNEXTHOP" ] || NOPTS="${NOPTS}N"
    [ "$PINGTARGET" ] || NOPTS="${NOPTS}P"
    [ "$IPADDRALRT" ] || NOPTS="${NOPTS}A"
    [ "$IFCUPDOWN" ] || NOPTS="${NOPTS}R"
    [ "$CONTINUOUSTOPOLOGYDETECT" ] || NOPTS="${NOPTS}T"

    AOPTS="V$V -t$SLOWUPTIMEOUT -s$SLEEP -g$GRACE -n$MAXNOLINK -u'$IFCUPT' -U'$IFCDOWNT' -a'$ALERTCMDT' -z$LOGSIZE"
    if [ "$STATICIFC" ]; then AOPTS="$AOPTS -I$IFC"
    elif [ "$IFC" ]; then AOPTS="$AOPTS -i$IFC"; fi

    debug 'NOPTS=%s\nAOPTS=%s\n' "$NOPTS" "$AOPTS"
}
### Systemd service creation
[ "$SYSTEMDSERVICENAME" ] && {
    LOGFILE=''
    printf %s "$SYSTEMDSERVICENAME" | grep -qEe '^[a-zA-Z0-9_-]+$' || \
        USAGE 'Invalid systemd service name "%s"\n       %s\n       %s' "$SYSTEMDSERVICENAME" \
              'The SYSTEMDSERVICENAME must contain only alphanumeric characters, hyphens and underscores' \
              'and be no longer than 236 characters.'
    [ $(printf %s "$SYSTEMDSERVICENAME" | wc -c) -gt 236 ] && \
        USAGE 'Systemd service name\n       "%s"\n       %s' \
              "$SYSTEMDSERVICENAME" "is too long (max 236 characters)"
    [ $(id -u) -eq 0 ] || USAGE %s "$nwwatchdog --install-systemd must be run as root"
    [ -d /etc/systemd/system/. ] || USAGE "'/etc/systemd/system/' does not exists,\n       systemd '%s.service' NOT installed!" \
                                          "$SYSTEMDSERVICENAME"
    cat>/etc/systemd/system/nw-watchdog-$SYSTEMDSERVICENAME.service<<EOF
[Unit]
Description=$nwwatchdog for connectivity to $TARGET
Wants=network.target

[Service]
KillMode=process
Type=forking
PIDFile=/run/nw-watchdog/$SYSTEMDSERVICENAME.pid
ExecStart=/bin/sh '$(readlink -e "$0")' '$TARGET' -$NOPTS$AOPTS -p/run/nw-watchdog/$SYSTEMDSERVICENAME.pid -l/var/log/nw-watchdog/$SYSTEMDSERVICENAME.log

[Install]
WantedBy=sysinit.target
EOF
    systemctl daemon-reload || die 'failed to reload systemd, service not enabled, nor started'
    systemctl enable nw-watchdog-$SYSTEMDSERVICENAME.service || die 'failed to enable the nw-watchdog-%s.service, service not started' "$SYSTEMDSERVICENAME"
    systemctl restart nw-watchdog-$SYSTEMDSERVICENAME.service || die 'failed to start the nw-watchdog-%s.service' "$SYSTEMDSERVICENAME"
    systemctl status --no-pager nw-watchdog-$SYSTEMDSERVICENAME.service
    exit 0

}

resolve_target() {
    [ "$TARGET" = "$TADDR" ] && return 0 # TARGET is an IP address, no need to resolve
    rtOADDR=$TADDR
    TADDR=$(getent ahostsv4 "$TARGET" | head -1 | sed -E 's/^\s*([0-9.]+)\s.*/\1/')
    [ "$TADDR" ] || {
        TADDR=$rtOADDR
        info 'Failed to resolve target "%s", keeping "%s"' "$TARGET" "$TADDR"
        return 1
    }
    if [ "$rtOADDR" = "$TADDR" ]; then
        debug 'Resolved target "%s" to "%s", already in cache.' "$TARGET" "$TADDR"
    else
        info 'Target (%s) resolved to %s (instead of "%s").' "$TARGET" "$TADDR" "$rtOADDR"
    fi
    return 0
}

isup() {
    isupADDR=$1

    [ "$isupADDR" = "$NEXTHOP" ] && {
        # addr is on local subnet (incl. host-local addrs) or peer-to-peer address
        ip neigh get dev $IFC $isupADDR 2>&1 | grep -q 'REACHABLE' && { debug '%s is arp1-up on %s' "$isupADDR" "$IFC"; return 0 ;}
        ping -q -w2 -c1 -I$IFC $isupADDR >/dev/null 2>&1 && { debug '%s is arp-ping-up on %s' "$isupADDR" "$IFC"; return 0 ;} # also refreshes arp-cache
        ip neigh get dev $IFC $isupADDR 2>&1 | grep -q 'REACHABLE' && { debug '%s is arp2-up on %s' "$isupADDR" "$IFC"; return 0 ;}
    }

    # quick check if line is already up
    ping -q -w1 -c1 -I$IFC $isupADDR >/dev/null 2>&1 && {
        debug '%s is quick-up on %s' "$isupADDR" "$IFC"
        return 0
    }
    trace 'quick-up failed ... trying slow-up ...'

    ping -qA -w$SUDEADLINE -W$SLOWUPTIMEOUT -c$SUCOUNT -I$IFC $isupADDR >/dev/null 2>&1 && {
        trace '%s is slow-up via %s' "$isupADDR" "$IFC"
        return 0
    }

    # If ping does not receive any reply packets at all it will exit
    # with code 1. If a packet count and deadline are both specified,
    # and fewer than count packets are received by the time the
    # deadline has arrived, it will also exit with code 1. On other
    # error it exits with code 2. Otherwise it exits with code 0. This
    # makes it possible to use the exit code to see if a host is alive
    # or not.

    # double check that we're up (the above could potentially
    # (but unlikely due to -A) return 1 or 2 even if we're up)
    trace 'slow-up failed or ambigious result ... verifying ...'
    ping -q -w2 -c1 -I$IFC $isupADDR >/dev/null  2>&1 && {
        trace '$isupADDR is verified-up via $IFC'
        return 0
    }
    # We don't get traffic through
    debug '%s is not-up on %s' "$isupADDR" "$IFC"

    [ "$2" = noresolve ] && return 1
    isupOADDR=$TADDR
    if resolve_target; then
        [ "$isupOADDR" = "$TADDR" ] || {
            info 'Target "%s" now resolves to "%s", instead of "%s".' "$TARGET" "$TADDR" "$isupOADDR"
            nexthop && {
                if [ "$isupADDR" = "$isupOADDR" ]; then # it was target that failed, test again with new TADDR
                    isupADDR=$TADDR
                else # it was nexthop that failed, test again with new NEXTHOP
                    isupADDR=$NEXTHOP
                fi
                isup "$isupADDR" noresolve && return 0
            }
        }
    else
        TADDR=$isupOADDR
        warn 'Target "%s" no longer resolves to an ip address, will keep using "%s".' "$TARGET" "$TADDR"
    fi
    return 1
}

[ "$PIDFILE" ] || PIDFILE=/run/nw-watchdog.pid
PIDDIR=${PIDFILE%/*}
if [ -d "$PIDDIR/." ] || mkdir -p "$PIDDIR"; then
    touch "$PIDFILE" 2>/dev/null || die '%s\n%s' "Cannot write to pidfile '$PIDFILE'." \
                                        'Fix permissions, run as user with access or specify another --pidfile.'
else
    die 'Cannot create directory for --pidfile="%s"\n%s' "$PIDFILE" \
        'Fix permissions, run as user with access or specify another --pidfile.'
fi
PID=$(cat "$PIDFILE")
[ "$PID" -gt 0 ] 2>/dev/null && [ -d /proc/$PID ] && die "Process already running with pid $PID"

### Forking
[ "$FORK" ] && {
    if [ "$LOGFILE" ]; then AOPTS="$AOPTS -l'$LOGFILE'"
    else AOPTS="$AOPTS -l-"; fi
    AOPTS="$AOPTS -p'$PIDFILE'"
    if [ -x "$0" ]; then
        trace 'forking: %s' "'$0' '$TARGET' -D$NOPTS$AOPTS"
        { sleep 0.25 ; eval "exec '$0' '$TARGET' -D$NOPTS$AOPTS" ;} &
    else
        trace 'forking: %s' "sh '$0' '$TARGET' -D$NOPTS$AOPTS"
        { sleep 0.25 ;eval "exec sh '$0' '$TARGET' -D$NOPTS$AOPTS" ;} &
    fi
    exit 0
}

printf $$ > "$PIDFILE"
debug 'PID=%d BASENAME=%s COMM=%s' $$ "$nwwatchdog" "$(cat /proc/$$/comm)"

debug '\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s' \
      '~~~ SETTINGS ~~~' \
      "TARGET='$TARGET'" \
      "PINGNEXTHOP='$PINGNEXTHOP'" \
      "PINGTARGET='$PINGTARGET'" \
      "IPADDRALRT='$IPADDRALRT'" \
      "CONTINUOUSTOPOLOGYDETECT='$CONTINUOUSTOPOLOGYDETECT'" \
      "IFCUPDOWN='$IFCUPDOWN'" \
      "FORK='$FORK'" \
      "V='$V'" \
      "IFC='$IFC'" \
      "STATICIFC='$STATICIFC'" \
      "LOGFILE='$LOGFILE'" \
      "LOGSIZE='$LOGSIZE' ($LOGSIZEb bytes)" \
      "PIDFILE='$PIDFILE'" \
      "SLOWUPTIMEOUT='$SLOWUPTIMEOUT'" \
      "SLEEP='$SLEEP'" \
      "GRACE='$GRACE'" \
      "MAXNOLINK='$MAXNOLINK'" \
      "IFCUPT='$IFCUPT'" \
      "IFCDOWNT='$IFCDOWNT'" \
      "ALERTCMDT='$ALERTCMDT'"

TADDR=''
printf %s "$TARGET" | grep -qE '^((0*[0-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))\.){3}(0*[0-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))$' && {
    TADDR=$(printf %s "$TARGET" | sed -E 's/(^|\.)0+([1-9]+)/\1\2/g') # Clean up potential oktet zero-padding
    TARGET=$TADDR
    debug 'TARGET is an IP address: "%s"' "$TARGET"
}
[ "$IFC" ] && {
    IFCUP=$(printf %s "$IFCUPT" | sed -E "s/\%\{IFC\}/$IFC/g")
    IFCDOWN=$(printf %s "$IFCDOWNT" | sed -E "s/\%\{IFC\}/$IFC/g")
    ip link show $IFC>/dev/null 2>&1 || { # device does not exist
        if [ "$IFCUPDOWN" ]; then
            info 'Interface "%s" is not available (no such device), trying to bring it up.' "$IFC"
            sh -c "$IFCUP" >/dev/null 2>&1
            trace 'Sleeping for %d seconds.' $GRACE
            sleep $GRACE
            if ip link show $IFC>/dev/null 2>&1; then
                [ "$STATICIFC" ] || {
                    ip link show $IFC | grep -oq 'state DOWN' && \
                        alert INITIAL 'Initial interface "%s" brought up, but has no link, will continue detecting topology ...' "$IFC"
                }
            else
                if [ "$STATICIFC" ]; then
                    warn '\n\t--force-interface "%s" is not available (no such device),\n\t%s\n\t%s\n\t%s\n\t%s' "$IFC" \
                         'Will keep trying to bring it up!' \
                         '... or you might want to check the nw-watchdog options and ' \
                         'restart this instance with correct ones if wrong.'
                else
                    alert INITIAL \
                          'Initial interface "%s" could not be brought up (no such device),\n\t%s' "$IFC" \
                          'will continue detecting topology without it ...'
                fi
            fi
        elif [ "$STATICIFC" ]; then
            warn '\n\t--force-interface "%s" is not available (no such device),\n\t%s\n\t%s\n\t%s\n\t%s' "$IFC" \
                 'and ---no-interface-reset prevents bringing it up!' \
                 "The status will be checked every $GRACE seconds until it comes up." \
                 'Bring the interface up, or check the nw-watchdog options and ' \
                 'restart this instance with correct ones if wrong.'
        else
            alert INITIAL 'Initial interface "%s" is not available (no such device),\n\t%s\n\t%s' "$IFC" \
                  'and ---no-interface-reset prevents bringing it up!' \
                  'Detecting topology without initial interface.'
        fi
    }
    if [ "$STATICIFC" ]; then
        while ! linkcheck; do sleep $GRACE; done
    elif ip link show $IFC | grep -oq 'state DOWN'; then
        alert INITIAL 'Initial interface "%s" has no link, will continue detecting topology ...' "$IFC"
    fi
}

while [ -z "$TADDR" ]; do
    resolve_target || {
        warn 'Target "%s" does not resolve to an IPv4 address. Make sure name resoloution funtions on the system\n\t%s\n\t%s' \
             "$TARGET" \
             '... or stop this instance and start it again using a valid hostname / FQDN or IP address' \
             "... sleeping for $GRACE seconds."
        sleep $GRACE
    }
done

if [ "$IFC" ]; then
    nexthop || { linkcheck && { nexthop || { ifreset && nexthop ;};};}
else
    while ! nexthop; do
        warn 'No interface specified and no route to target "%s" (%s) available, make sure interface comes up,\n\t%s\n\t%s' \
             "$TARGET" "$TADDR" \
             'or stop this instance and start it again using --interface or --force-interface to bring it up automatically.' \
             "... sleeping for $GRACE seconds."
        sleep $GRACE
    done
fi

while :; do
    if [ "$PINGTARGET" ]; then
        trace 'Continuously checking if target is up...'
        [ "$CONTINUOUSTOPOLOGYDETECT" ] && trace '... and for changes in topology'
        while isup $TADDR; do
            alert UP 'Target "%s" (%s) is up.' "$TARGET" "$TADDR"
            debug 'Target "%s" (%s) is up. Sleeping for %d seconds.' "$TARGET" "$TADDR" $SLEEP
            sleep $SLEEP
            [ "$CONTINUOUSTOPOLOGYDETECT" ] && { resolve_target; nexthop ;}
        done
        trace 'No reply from target "%s" (%s), checking link and topology.' "$TARGET" "$TADDR"
        if nexthop || { linkcheck && nexthop ;}; then
            isup "$TADDR" || {
                if [ "$PINGNEXTHOP" ]; then
                    if isup "$NEXTHOP" noresolve; then
                        alert DOWN \
                              'Not getting replies from target "%s" (%s) on interface "%s", but nexthop "%s" is reachable... sleeping for %d seconds.' \
                              "$TARGET" "$TADDR" "$IFC" "$NEXTHOP" $GRACE
                        sleep $GRACE
                    else
                        alert UNREACHABLE \
                              'Not getting replies from target "%s" (%s) on interface "%s", and nexthop "%s" is not reachable.\n\t%s\n\t%s\n\t%s' \
                              "$TARGET" "$TADDR" "$IFC" "$NEXTHOP" 'Resetting interface:' "$IFCDOWN" "$IFCUP"
                        ifreset && nexthop
                    fi
                else
                    alert DOWN \
                          'Not getting replies from target "%s" (%s) on interface "%s".\n\t%s\n\t%s\n\t%s' \
                          "$TARGET" "$TADDR" "$IFC" 'Resetting interface:' "$IFCDOWN" "$IFCUP"
                    ifreset && nexthop
                fi
            }
        else
            alert UNREACHABLE \
                  'Not getting replies from target "%s" (%s) and no route to target available, resetting interface (%s):\n\t%s\n\t%s' \
                  "$TARGET" "$TADDR" "$IFC" "$IFCDOWN" "$IFCUP"
            ifreset && nexthop
        fi

    else
        trace 'Continuously checking if nexthop is reachable...'
        [ "$CONTINUOUSTOPOLOGYDETECT" ] && trace '... and for changes in topology'
        while isup $NEXTHOP; do
            alert REACHABLE 'Nexthop "%s" for target "%s" (%s) is reachable.' "$NEXTHOP" "$TARGET" "$TADDR"
            debug 'Nexthop "%s" for target "%s" (%s) is reachable. Sleeping for %d seconds.' "$NEXTHOP" "$TARGET" "$TADDR" $SLEEP
            sleep $SLEEP
            [ "$CONTINUOUSTOPOLOGYDETECT" ] && { resolve_target; nexthop ;}
        done
        trace 'nexthop "%s" (for --no-ping-target "%s" (%s)) on interface "%s", not reachable, checking link and nexthop.' \
              "$NEXTHOP" "$TARGET" "$TADDR" "$IFC"
        if { resolve_target && nexthop ;} || { linkcheck && resolve_target && nexthop ;}; then
            isup "$NEXTHOP" noresolve || {
                alert UNREACHABLE \
                      'nexthop "%s" (for --no-ping-target "%s" (%s)), not reachable, resetting interface ("%s"):\n\t%s\n\tsleep 1\n\t%s' \
                      "$NEXTHOP" "$TARGET" "$TADDR" "$IFC" "$IFCDOWN" "$IFCUP"
                ifreset && nexthop
            }
        else
            alert UNREACHBLE \
                  'No route to --no-ping-target "%s" (%s). Resetting interface (%s):\n\t%s\n\tsleep 1\n\t%s' \
                  "$TARGET" "$TADDR" "$IFC" "$IFCDOWN" "$IFCUP"
            ifreset && nexthop
        fi
    fi
done
die 'Internal error: EOF'