#!/bin/bash -eu

if [ "${BASH_SOURCE[0]}" != "$0" ]
then
  echo "${BASH_SOURCE[0]} must be executed, not sourced"
  return 1 # shouldn't use exit when sourced
fi

export LOG_FILE=/mutable/archiveloop.log

function log () {
  echo -n "$( date ): " >> "$LOG_FILE"
  echo "$@" >> "$LOG_FILE"
}

function log_errors_on_exit {
  log "archiveloop exited with code $1. Recent errors follow"
  journalctl -n 15 -u teslausb >> "$LOG_FILE"
  log "end of errors."
  exit "$1"
}

if [ "${FLOCKED:-}" != "$0" ]
then
  if FLOCKED="$0" flock -en -E 99 "$0" "$0" "$@" || case "$?" in
  99) echo already running
      exit 99
      ;;
  *)  log_errors_on_exit $?
      ;;
  esac
  then
    # success
    log_errors_on_exit 0
  fi
fi

# turning off hdmi saves a little bit of power
/usr/bin/tvservice -o

export CAM_MOUNT=/mnt/cam
export MUSIC_MOUNT=/mnt/music
export ARCHIVE_MOUNT=/mnt/archive
export MUSIC_ARCHIVE_MOUNT=/mnt/musicarchive

# when sourced, setup-teslausb sets the config variables in the environment
source /root/bin/setup-teslausb

if [ -z "${ARCHIVE_SERVER+x}" ]
then
  case "${ARCHIVE_SYSTEM:-none}" in
    rsync)
      export ARCHIVE_SERVER="$RSYNC_SERVER"
      ;;
    rclone)
      export ARCHIVE_SERVER="8.8.8.8"
      ;;
    none)
      export ARCHIVE_SERVER=localhost
      ;;
    *)
      log "ARCHIVE_SERVER not set"
      exit 1
      ;;
  esac
fi

function isPi4 {
  grep -q "Pi 4" /sys/firmware/devicetree/base/model
}

function isKernel5 {
  grep -q '^5\.' <<< "$(uname -r)"
}

function timestamp () {
  local prefix=${1:-}
  while IFS= read -r line
  do
    echo "$(date): $prefix$line"
  done
}

function fix_errors_in_image () {
  local image="$1"
  log "Running fsck on $image..."
  loopback=$(losetup --show -f -P "$image")
  /sbin/fsck "${loopback}p1" -- -a |& timestamp '| ' >> "$LOG_FILE" || echo ""
  losetup -d "$loopback"
  log "Finished fsck on $image."
}

function archive_is_reachable () {
  local reachable=true

  /root/bin/archive-is-reachable.sh "$ARCHIVE_SERVER" || reachable=false

  if [ "$reachable" = false ]
  then
    false
    return
  fi
  true
}

function connect_usb_drives_to_host() {
  log "Connecting usb to host..."
  modprobe g_mass_storage
  log "Connected usb to host."
}

function wait_for_archive_to_be_reachable () {
  log "Waiting for archive to be reachable..."
  while true
  do
    if archive_is_reachable
    then
      log "Archive is reachable."
      break
    fi
    if [ -e /tmp/archive_is_reachable ]
    then
      log "Simulating archive is reachable"
      rm /tmp/archive_is_reachable
      break
    fi
    sleep 1
  done
}

function retry () {
  local attempts=0
  while true
  do
    if eval "$@"
    then
      true
      return
    fi
    if [ "$attempts" -ge 10 ]
    then
      log "Attempts exhausted."
      false
      return
    fi
    log "Sleeping before retry..."
    /bin/sleep 1
    attempts=$((attempts + 1))
    log "Retrying..."
  done
  false
  return
}

function mount_mountpoint () {
  local mount_point="$1"
  log "Mounting $mount_point..."

  local mounted=true
  mount "$mount_point" >> "$LOG_FILE" 2>&1 || mounted=false
  if [ "$mounted" = true ]
  then
    log "Mounted $mount_point."
    true
    return
  else
    log "Failed to mount $mount_point."
    false
    return
  fi
}

function ensure_mountpoint_is_mounted () {
  local mount_point="$1"
  local mount_exists=true

  findmnt --mountpoint "$mount_point" > /dev/null || mount_exists=false

  if [ "$mount_exists" = true ]
  then
    log "$mount_point is already mounted."
  else
    mount_mountpoint "$mount_point"
  fi
}

function ensure_mountpoint_is_mounted_with_retry () {
  retry ensure_mountpoint_is_mounted "$1"
}

function ensure_cam_file_is_mounted () {
  log "Ensuring cam file is mounted..."
  ensure_mountpoint_is_mounted_with_retry "$CAM_MOUNT"
  log "Ensured cam file is mounted."
}

function ensure_music_file_is_mounted () {
  log "Ensuring music backing file is mounted..."
  ensure_mountpoint_is_mounted_with_retry "$MUSIC_MOUNT"
  log "Ensured music drive is mounted."
}

function unmount_mount_point () {
  local mount_point="$1"
  log "Unmounting $mount_point..."
  if umount "$mount_point" >> "$LOG_FILE" 2>&1
  then
    log "Unmounted $mount_point."
  else
    log "Failed to unmount $mount_point, trying lazy unmount."
    if umount -l "$mount_point" >> "$LOG_FILE" 2>&1
    then
      log "lazily unmounted $mount_point"
    else
      log "lazy unmount failed"
    fi
  fi
}

function unmount_cam_file () {
  unmount_mount_point "$CAM_MOUNT"
}

function unmount_music_file () {
  unmount_mount_point "$MUSIC_MOUNT"
}

function wait_for_archive_to_be_unreachable () {
  log "Waiting for archive to be unreachable..."
  while true
    do
      if ! retry archive_is_reachable
      then
        log "Archive is unreachable."
        break
      fi
      if [ -e /tmp/archive_is_unreachable ]
      then
        log "Simulating archive being unreachable."
        rm /tmp/archive_is_unreachable
        break
      fi
      sleep 1
  done
}

function check_if_usb_gadget_is_mounted () {
    local found="false"
    for lun in /sys/devices/platform/soc/??980000.usb/gadget/lun0
    do
      if [ -d "$lun" ]
      then
        found="true"
        break
      fi
    done

    if [ "$found" = "false" ]
    then
        log "USB Gadget not mounted. Fixing files and remounting..."
        disconnect_usb_drives_from_host
        fix_errors_in_images
        connect_usb_drives_to_host
    fi
}

function trim_free_space() {
  local mount_point="$1"

  # Make sure the partition is mounted.
  if found=$(findmnt -n --mountpoint "$mount_point")
  then
    loop=$(echo "$found" | awk '{print $2}')
    image=$(losetup -l -n --output=BACK-FILE "$loop")
    log "Trimming free space in $mount_point, which has $(filefrag "$image" | awk '{print $2}') extents"
    if fstrim "$mount_point" >> "$LOG_FILE" 2>&1
    then
      log "Trim complete, image now has $(filefrag "$image" | awk '{print $2}') extents"
    else
      log "Trimming free space in $mount_point failed."
    fi
  else
    log "Could not trim free space in $mount_point. Not Mounted."
  fi
}

function fix_errors_in_images () {
  fix_errors_in_image /backingfiles/cam_disk.bin
  if [ -e /backingfiles/music_disk.bin ]
  then
    fix_errors_in_image /backingfiles/music_disk.bin
  fi
}

function disconnect_usb_drives_from_host () {
  log "Disconnecting usb from host..."
  modprobe -r g_mass_storage
  log "Disconnected usb from host."
}

function convert_seconds_to_nice_time () {
  local -r h=$(($1/3600))
  local -r m=$((($1%3600)/60))
  local -r s=$(($1%60))

  if [ $h -gt 0 ]
  then
    printf "%dh%dm%ds" $h $m $s
  elif [ $m -gt 0 ]
  then
    printf "%dm%ds" $m $s
  else
    printf "%ds" $s
  fi
}

function sortfile {
  touch "$1"
  sort "$1" > /tmp/sort_tmp.$$
  mv /tmp/sort_tmp.$$ "$1"
}

function prunefile {
  sortfile "$1"
  sortfile "$2"
  comm -2 -3 "$1" "$2" > /tmp/prune_tmp.$$
  mv /tmp/prune_tmp.$$ "$1"
}

function intersect {
  sortfile "$1"
  sortfile "$2"
  comm -1 -2 "$1" "$2" > /tmp/prune_tmp.$$
  mv /tmp/prune_tmp.$$ "$1"
}

function clean_cam_mount {
  log "cleaning cam mount"
  # Delete the files that fsck "recovered". These are generally files
  # that were deleted previously by the car or teslausb
  find "${CAM_MOUNT}" \( -type f -name FSCK\*.REC \) -o \( -type d -name \*.M \) -print0 | xargs -0 rm -rf

  # Remove files, oldest first, until there is at least 20GB of free space

  (cd "${CAM_MOUNT}"; find . -type f -a \( -path './TeslaCam/*' -o -path './TeslaTrackMode/*' \) -printf '%P\0') > /tmp/rmcandidates.txt
  if [ -s /tmp/rmcandidates.txt ]
  then
    (cd "${CAM_MOUNT}"; xargs -a /tmp/rmcandidates.txt -0 ls -cr1) > /tmp/rmcandidatesbytime.txt
  fi

  local freespace
  while read -r line
  do
    freespace=$(eval "$(stat --file-system --format="echo \$((%f*%S))" "${CAM_MOUNT}/.")")
    if ((freespace > 20000000000))
    then
      break;
    fi
    rm "${CAM_MOUNT}/$line"
  done < /tmp/rmcandidatesbytime.txt

  # delete directories that are now empty
  find "$CAM_MOUNT/TeslaCam/RecentClips" "$CAM_MOUNT/TeslaCam/SavedClips" \
    "$CAM_MOUNT/TeslaCam/SentryClips" "$CAM_MOUNT/TeslaTrackMode" -depth -type d -empty -delete || true

  log "done cleaning cam mount"
}

# Directory structure car uses:
# TeslaCam/
#   RecentClips/
#      videos.mp4
#   SavedClips/
#     datetime/
#       videos.mp4
#       event.json
#       thumb.png
#   SentryClips/
#     datetime/
#       videos.mp4
#       event.json
#       thumb.png
# TeslaTrackMode/
#   lapvideo.mp4
#   laptelemetry.csv
function archive_teslacam_clips () {
  log "Checking saved folder count..."

  ensure_cam_file_is_mounted

  local -r overlaylower=/mutable/TeslaCam
  local -r overlayupper=/tmp/cam/upper
  local -r overlaywork=/tmp/cam/work
  local -r overlaymerged=/tmp/cam/merged

  mkdir -p "$overlayupper" "$overlaywork" "$overlaymerged"

  mount -t overlay overlay -o "lowerdir=$overlaylower,upperdir=$overlayupper,workdir=$overlaywork" "$overlaymerged"

  # Build list of the files to be archived.
  local -r sentrylist=/tmp/sentry_files
  local -r sentrylist_archived=/tmp/sentry_files_archived
  local -r sentrylist_previously_archived=/mutable/sentry_files_archived
  local -r ignorelist=/tmp/ignore_files

  # Find files by name only. Don't follow the symlinks yet, since that
  # would cause all snapshots to be mounted.
  local -a savedclipsopt
  local -a sentryclipsopt
  local -a trackmodeclipsopt
  local -a recentclipsopt
  if [ "${ARCHIVE_SAVEDCLIPS:-true}" = "true" ]
  then
    savedclipsopt=("-path" "./SavedClips/*")
  else
    savedclipsopt=("-false")
  fi
  if [ "${ARCHIVE_SENTRYCLIPS:-true}" = "true" ]
  then
    sentryclipsopt=("-o" "-path" "./SentryClips/*")
  fi
  if [ "${ARCHIVE_TRACKMODECLIPS:-true}" = "true" ]
  then
    trackmodeclipsopt=("-o" "-path" "./TeslaTrackMode/*")
  fi
  if [ "${ARCHIVE_RECENTCLIPS:-false}" = "true" ]
  then
    recentclipsopt=("-o" "-path" "./RecentClips/*")
  fi
  (cd "$overlaymerged"; find . \( \( "${savedclipsopt[@]}" "${sentryclipsopt[@]}" "${trackmodeclipsopt[@]}" "${recentclipsopt[@]}" \) -type l \) \
        -a -fprintf "$sentrylist" '%P\n')

  # Copy lists of previously-archived files from /mutable
  if [ -r "${sentrylist_previously_archived}" ]
  then
    cp "${sentrylist_previously_archived}" "${sentrylist_archived}"
  fi

  # Remove files that no longer exist in snapshots from the
  # "previously archived" list, so the list doesn't keep growing.
  intersect "${sentrylist_archived}" "${sentrylist}"

  # Remove previously-archived files from the archive candidate list
  prunefile "${sentrylist}" "${sentrylist_archived}"

  # ${sentrylist} is now a list of files that haven't been archived yet.
  # Remove short recordings from this list. This requires following the
  # links and looking at the files themselves.
  if [ -s "${sentrylist}" ]
  then
    (cd "$overlaymerged"; xargs -a "${sentrylist}" echo "find -L" | while read -r line
        do
          eval "$line \( -name \*.mp4 -a -size -100000c -fprintf ${ignorelist} '%p\n' \) -o -print"
        done)
  else
    touch "${ignorelist}"
  fi

  # remove the short files from the list of files to be archived
  prunefile "${sentrylist}" "${ignorelist}"

  local -r sentry_count=$(grep -c -v TeslaTrackMode "$sentrylist")
  local -r trackmode_count=$(grep -c TeslaTrackMode "$sentrylist")
  local -r ignore_count=$(wc -l < "$ignorelist")
  local -r total_count=$((sentry_count + trackmode_count))

  # extract some noteworthy info from the file list
  local -r saved_event_count=$(grep 'SavedClips/' < "$sentrylist" | sed 's/[^\/]\+$//' | sort -u  | wc -l)
  local -r sentry_event_count=$(grep 'SentryClips/' < "$sentrylist" | sed 's/[^\/]\+$//' | sort -u  | wc -l)

  local -r event_count=$((saved_event_count + sentry_event_count))

  log "There are $event_count event folder(s) with $sentry_count file(s) and $trackmode_count track mode file(s) to move." \
      " $ignore_count short recording(s) will be skipped."

  if [[ "$total_count" -gt 0 ]]
  then
    log "Starting recording archiving"
    local -r start_ts=$(date --utc --date "now" +%s)

    local message="Archiving "
    if [[ $sentry_count -gt 0 && $trackmode_count -gt 0 ]]
    then
      message+="$event_count event folder(s) with $sentry_count file(s) and $trackmode_count track mode file(s)"
    elif [[ $sentry_count -gt 0 ]]
    then
      message+="$event_count event folder(s) with $sentry_count file(s)"
    else
      message+="$trackmode_count track mode file(s)"
    fi
    message+=" starting at $(date)"
    /root/bin/send-push-message "$TESLAUSB_HOSTNAME:" "$message"

    # Ensure Sentry Mode is enabled before archiving
    declare is_sentry_mode_enabled
    if [ -x /root/bin/tesla_api.py ]
    then
        is_sentry_mode_enabled=$(/root/bin/tesla_api.py is_sentry_mode_enabled | tr '[:upper:]' '[:lower:]')
        if [ "false" = "${is_sentry_mode_enabled}" ]
        then
            log "Temporarily enabling Sentry Mode to power the RPi while archive job completes..."
            /root/bin/tesla_api.py enable_sentry_mode &>> ${LOG_FILE}
        fi
    fi

    # setup trigger files
    local -r triggerdir=/tmp/triggers
    local -r triggerlist=/tmp/triggers.txt
    rm -rf "${triggerdir}"
    mkdir -p "${triggerdir}/SentryClips"
    mkdir -p "${triggerdir}/SavedClips"
    mkdir -p "${triggerdir}/RecentClips"
    true > "${triggerlist}"
    if [ -n "${TRIGGER_FILE_SAVED+x}" ]
    then
      touch "${triggerdir}/SavedClips/${TRIGGER_FILE_SAVED}"
      echo "SavedClips/${TRIGGER_FILE_SAVED}" >> "${triggerlist}"
    fi
    if [ -n "${TRIGGER_FILE_SENTRY+x}" ]
    then
      touch "${triggerdir}/SentryClips/${TRIGGER_FILE_SENTRY}"
      echo "SentryClips/${TRIGGER_FILE_SENTRY}" >> "${triggerlist}"
    fi
    if [ -n "${TRIGGER_FILE_RECENT+x}" ]
    then
      touch "${triggerdir}/RecentClips/${TRIGGER_FILE_RECENT}"
      echo "SentryClips/${TRIGGER_FILE_RECENT}" >> "${triggerlist}"
    fi
    if [ -n "${TRIGGER_FILE_ANY+x}" ]
    then
      touch "${triggerdir}/${TRIGGER_FILE_ANY}"
      echo "${TRIGGER_FILE_ANY}" >> "${triggerlist}"
    fi

    if /root/bin/archive-clips.sh "$overlaymerged" "${sentrylist}" "${triggerdir}" "${triggerlist}"
    then
      message="Archiving completed successfully. "
    else
      message="Error during archiving. "
    fi

    local -i sentry_archived=0
    local -i trackmode_archived=0

    while read -r line
    do
      if [[ ! -e "$overlaymerged/$line" ]]
      then
        case $line in
          TeslaTrackMode*)
            trackmode_archived=$((trackmode_archived + 1))
            ;;
          *)
            sentry_archived=$((sentry_archived + 1))
            ;;
        esac
        echo "$line" >> "${sentrylist_archived}"
      fi
    done < ${sentrylist}


    # copy the list of archived files back to /mutable if it has changed
    if ! diff -q -N "${sentrylist_archived}" "${sentrylist_previously_archived}" > /dev/null
    then
        cp "${sentrylist_archived}" "${sentrylist_previously_archived}.new"
        mv "${sentrylist_previously_archived}.new" "${sentrylist_previously_archived}"
    fi

    message+="Archived "
    if [[ $sentry_count -gt 0 && $trackmode_count -gt 0 ]]
    then
      message+="${sentry_archived} event files and ${trackmode_archived} trackmode files"
    elif [[ $sentry_count -gt 0 ]]
    then
      message+="${sentry_archived} event files"
    else
      message+="${trackmode_archived} trackmode files"
    fi
    local -r end_ts=$(date --utc --date "now" +%s)
    local -r delta=$((end_ts - start_ts))
    message+=" in $(convert_seconds_to_nice_time $delta)"
    /root/bin/send-push-message "$TESLAUSB_HOSTNAME:" "${message}"

    # If Sentry Mode was previously disabled, restore it to that state
    if [ -x /root/bin/tesla_api.py ]
    then
      if [ "false" = "${is_sentry_mode_enabled}" ]
      then
        log "Restoring Sentry Mode to its previous state (disabled)..."
        /root/bin/tesla_api.py disable_sentry_mode &>> ${LOG_FILE}
      fi
    fi

  fi

  # overlayfs behavior is undefined if the lower is changed while the overlay is active, so umount first
  umount "$overlaymerged"

  if [[ "$ignore_count" -gt 0 ]]
  then
    (
      cd "$CAM_MOUNT/TeslaCam"
      grep -v TeslaTrackMode "$ignorelist" | xargs rm -f
      cd "$CAM_MOUNT"
      grep TeslaTrackMode "$ignorelist" | xargs rm -f
    )
  fi

  clean_cam_mount

  # Trim the camera archive to reduce the number of blocks in the snapshot.
  trim_free_space "$CAM_MOUNT"

  unmount_cam_file
}

function copy_music_files () {
  log "Starting music sync..."

  ensure_music_file_is_mounted

  /root/bin/copy-music.sh

  # Trim the empty space from the music archive.
  trim_free_space "$MUSIC_MOUNT"

  unmount_music_file
}

function archive_clips () {
  log "Archiving..."

  if ! /root/bin/connect-archive.sh
  then
    log "Couldn't connect archive, skipping archive step"
    return
  fi

  if archive_teslacam_clips
  then
    log "Finished archiving."
  else
    log "Archiving failed."
  fi

  if timeout 5 [ -d "$MUSIC_ARCHIVE_MOUNT" -a -d "$MUSIC_MOUNT" ]
  then
    log "Copying music..."
    if copy_music_files
    then
      log "Finished copying music."
    else
      log "Copying music failed."
    fi
  else
    log "Music archive not configured or unreachable"
  fi

  /root/bin/disconnect-archive.sh
}

function truncate_log () {
  local log_length
  log_length=$( wc -l "$LOG_FILE" | cut -d' ' -f 1 )
  if [ "$log_length" -gt 10000 ]
  then
    log "Truncating log..."
    local log_file2="${LOG_FILE}.2"
    tail -n 10000 "$LOG_FILE" > "${LOG_FILE}.2"
    mv "$log_file2" "$LOG_FILE"
  fi
}

function slowblink () {
  echo timer > /sys/class/leds/led0/trigger
  local ON=on
  local OFF=off
  if isPi4 || isKernel5
  then
    ON=off
    OFF=on
  fi
  echo 900 > /sys/class/leds/led0/delay_$ON
  echo 100 > /sys/class/leds/led0/delay_$OFF
}

function fastblink () {
  echo timer > /sys/class/leds/led0/trigger
  local ON=on
  local OFF=off
  if isPi4 || isKernel5
  then
    ON=off
    OFF=on
  fi
  echo 150 > /sys/class/leds/led0/delay_$ON
  echo 50 > /sys/class/leds/led0/delay_$OFF
}


function doubleblink () {
  echo heartbeat > /sys/class/leds/led0/trigger
  if isPi4 || isKernel5
  then
    echo 0 > /sys/class/leds/led0/invert
  else
    echo 1 > /sys/class/leds/led0/invert
  fi
}

function set_time () {
  log "Trying to set time..."
  local -r uptime_start=$(awk '{print $1}' /proc/uptime)
  local -r clocktime_start=$(date +%s.%N)
  for _ in {1..5}
  do
    if sntp -S time.google.com
    then
      local -r uptime_end=$(awk '{print $1}' /proc/uptime)
      local -r clocktime_end=$(date +%s.%N)
      log "$(awk "BEGIN {printf \"Time adjusted by %f seconds after %f seconds\", $clocktime_end-$clocktime_start, $uptime_end-$uptime_start}")"
      return
    fi
    log "sntp failed, retrying..."
    sleep 2
  done
  log "Failed to set time"
}

function snapshotloop {
  while true
  do
    sleep 3480
    /root/bin/waitforidle || true
    /root/bin/make_snapshot.sh
  done
}

function wifichecker {
  dmesg -w | {
    while TMOUT=1 read -r line
    do
      true
    done
    wifi=working
    while read -r line
    do
      case $line in
        *"failed to enable fw supplicant")
          if [ "$wifi" = "working" ]
          then
            wifi="notworking"
          else
            log "restarting wifi because of: $line"
            modprobe -r brcmfmac cfg80211 brcmutil || true
            modprobe brcmfmac || true
            while TMOUT=1 read -r line
            do
              true
            done
            wifi="working"
          fi
          ;;
        *)
          wifi=working
          ;;
      esac
    done
  }
}

function set_sys_param() {
  local -r parampath="$1"
  local -r var="$2"
  local -r defaultval="$3"

  local -r val=${!var:-$defaultval}
  if [[ ${val} = "default" ]]
  then
    log "not setting $parampath"
  else
    echo "$val" > "$parampath" || true
  fi
}

export -f mount_mountpoint
export -f ensure_mountpoint_is_mounted
export -f retry
export -f ensure_mountpoint_is_mounted_with_retry
export -f log

echo "==============================================" >> "$LOG_FILE"
log "Starting archiveloop at $(awk '{print $1}' /proc/uptime) seconds uptime..."

set_sys_param /proc/sys/vm/dirty_background_bytes DIRTY_BACKGROUND_BYTES 65536
set_sys_param /proc/sys/vm/dirty_ratio DIRTY_RATIO 80
set_sys_param /sys/devices/system/cpu/cpufreq/policy0/scaling_governor CPU_GOVERNOR conservative

/root/bin/make_snapshot.sh
snapshotloop &

wifichecker &

fix_errors_in_images

if archive_is_reachable
then
  fastblink

  set_time

  archive_clips

  doubleblink

  connect_usb_drives_to_host

  wait_for_archive_to_be_unreachable
else
  slowblink

  connect_usb_drives_to_host
fi

while true
do
  slowblink

  wait_for_archive_to_be_reachable

  fastblink

  set_time

  sleep "${ARCHIVE_DELAY:-20}"

  # take a snapshot before archive_clips starts deleting files
  /root/bin/make_snapshot.sh

  disconnect_usb_drives_from_host

  fix_errors_in_images

  archive_clips

  truncate_log

  doubleblink

  connect_usb_drives_to_host

  wait_for_archive_to_be_unreachable

  check_if_usb_gadget_is_mounted
done