#!/bin/bash -eu if [ "${BASH_SOURCE[0]}" != "$0" ] then echo "${BASH_SOURCE[0]} must be executed, not sourced" return 1 # shouldn't use exit when sourced fi export LOG_FILE=/mutable/archiveloop.log function log () { echo -n "$( date ): " >> "$LOG_FILE" echo "$1" >> "$LOG_FILE" } function log_errors_on_exit { log "archiveloop exited with code $1. Recent errors follow" journalctl -n 15 -u teslausb >> "$LOG_FILE" log "end of errors." exit "$1" } if [ "${FLOCKED:-}" != "$0" ] then if FLOCKED="$0" flock -en -E 99 "$0" "$0" "$@" || case "$?" in 99) echo already running exit 99 ;; *) log_errors_on_exit $? ;; esac then # success log_errors_on_exit 0 fi fi # turning off hdmi saves a little bit of power /usr/bin/tvservice -o export CAM_MOUNT=/mnt/cam export MUSIC_MOUNT=/mnt/music export ARCHIVE_MOUNT=/mnt/archive export MUSIC_ARCHIVE_MOUNT=/mnt/musicarchive # when sourced, setup-teslausb sets the config variables in the environment source /root/bin/setup-teslausb if [ -z "${ARCHIVE_SERVER+x}" ] then case "${ARCHIVE_SYSTEM:-none}" in rsync) export ARCHIVE_SERVER="$RSYNC_SERVER" ;; rclone) export ARCHIVE_SERVER="8.8.8.8" ;; none) export ARCHIVE_SERVER=localhost ;; *) log "ARCHIVE_SERVER not set" exit 1 ;; esac fi function isPi4 { grep -q "Pi 4" /sys/firmware/devicetree/base/model } function isKernel5 { grep -q '^5\.' <<< "$(uname -r)" } function timestamp () { local prefix=${1:-} while IFS= read -r line do echo "$(date): $prefix$line" done } function fix_errors_in_image () { local image="$1" log "Running fsck on $image..." loopback=$(losetup --show -f -P "$image") /sbin/fsck "${loopback}p1" -- -a |& timestamp '| ' >> "$LOG_FILE" || echo "" losetup -d "$loopback" log "Finished fsck on $image." } function archive_is_reachable () { local reachable=true /root/bin/archive-is-reachable.sh "$ARCHIVE_SERVER" || reachable=false if [ "$reachable" = false ] then false return fi true } function connect_usb_drives_to_host() { log "Connecting usb to host..." modprobe g_mass_storage log "Connected usb to host." } function wait_for_archive_to_be_reachable () { log "Waiting for archive to be reachable..." while true do if archive_is_reachable then log "Archive is reachable." break fi if [ -e /tmp/archive_is_reachable ] then log "Simulating archive is reachable" rm /tmp/archive_is_reachable break fi sleep 1 done } function retry () { local attempts=0 while true do if eval "$@" then true return fi if [ "$attempts" -ge 10 ] then log "Attempts exhausted." false return fi log "Sleeping before retry..." /bin/sleep 1 attempts=$((attempts + 1)) log "Retrying..." done false return } function mount_mountpoint () { local mount_point="$1" log "Mounting $mount_point..." local mounted=true mount "$mount_point" >> "$LOG_FILE" 2>&1 || mounted=false if [ "$mounted" = true ] then log "Mounted $mount_point." true return else log "Failed to mount $mount_point." false return fi } function ensure_mountpoint_is_mounted () { local mount_point="$1" local mount_exists=true findmnt --mountpoint "$mount_point" > /dev/null || mount_exists=false if [ "$mount_exists" = true ] then log "$mount_point is already mounted." else mount_mountpoint "$mount_point" fi } function ensure_mountpoint_is_mounted_with_retry () { retry ensure_mountpoint_is_mounted "$1" } function ensure_cam_file_is_mounted () { log "Ensuring cam file is mounted..." ensure_mountpoint_is_mounted_with_retry "$CAM_MOUNT" log "Ensured cam file is mounted." } function ensure_music_file_is_mounted () { log "Ensuring music backing file is mounted..." ensure_mountpoint_is_mounted_with_retry "$MUSIC_MOUNT" log "Ensured music drive is mounted." } function unmount_mount_point () { local mount_point="$1" log "Unmounting $mount_point..." if umount "$mount_point" >> "$LOG_FILE" 2>&1 then log "Unmounted $mount_point." else log "Failed to unmount $mount_point, trying lazy unmount." if umount -l "$mount_point" >> "$LOG_FILE" 2>&1 then log "lazily unmounted $mount_point" else log "lazy unmount failed" fi fi } function unmount_cam_file () { unmount_mount_point "$CAM_MOUNT" } function unmount_music_file () { unmount_mount_point "$MUSIC_MOUNT" } function wait_for_archive_to_be_unreachable () { log "Waiting for archive to be unreachable..." while true do if ! retry archive_is_reachable then log "Archive is unreachable." break fi if [ -e /tmp/archive_is_unreachable ] then log "Simulating archive being unreachable." rm /tmp/archive_is_unreachable break fi sleep 1 done } function check_if_usb_gadget_is_mounted () { local found="false" for lun in /sys/devices/platform/soc/??980000.usb/gadget/lun0 do if [ -d "$lun" ] then found="true" break fi done if [ "$found" = "false" ] then log "USB Gadget not mounted. Fixing files and remounting..." disconnect_usb_drives_from_host fix_errors_in_images connect_usb_drives_to_host fi } function trim_free_space() { local mount_point="$1" # Make sure the partition is mounted. if found=$(findmnt -n --mountpoint "$mount_point") then loop=$(echo "$found" | awk '{print $2}') image=$(losetup -l -n --output=BACK-FILE "$loop") log "Trimming free space in $mount_point, which has $(filefrag "$image" | awk '{print $2}') extents" if fstrim "$mount_point" >> "$LOG_FILE" 2>&1 then log "Trim complete, image now has $(filefrag "$image" | awk '{print $2}') extents" else log "Trimming free space in $mount_point failed." fi else log "Could not trim free space in $mount_point. Not Mounted." fi } function fix_errors_in_images () { fix_errors_in_image /backingfiles/cam_disk.bin if [ -e /backingfiles/music_disk.bin ] then fix_errors_in_image /backingfiles/music_disk.bin fi } function disconnect_usb_drives_from_host () { log "Disconnecting usb from host..." modprobe -r g_mass_storage log "Disconnected usb from host." } function convert_seconds_to_nice_time () { local -r h=$(($1/3600)) local -r m=$((($1%3600)/60)) local -r s=$(($1%60)) if [ $h -gt 0 ] then printf "%dh%dm%ds" $h $m $s elif [ $m -gt 0 ] then printf "%dm%ds" $m $s else printf "%ds" $s fi } # Directory structure car uses: # TeslaCam/ # RecentClips/ # videos.mp4 # SavedClips/ # datetime/ # videos.mp4 # event.json # thumb.png # SentryClips/ # datetime/ # videos.mp4 # event.json # thumb.png # TeslaTrackMode/ # lapvideo.mp4 # laptelemetry.csv function archive_teslacam_clips () { log "Checking saved folder count..." ensure_cam_file_is_mounted # Build lists of the files to be archived. # Two lists are built: one for the folders under TeslaCam, and one for the TeslaTrackMode folder. local -r sentrylist=/tmp/sentry_files local -r ignorelist1=/tmp/ignore_files1 (cd "$CAM_MOUNT/TeslaCam"; find . \( \( -path './SavedClips/*' -o -path './SentryClips/*' \) -type f \) \ -a \( \( -name '*.mp4' -size -100000c -fprintf "$ignorelist1" '%P\n' \) -o -fprintf "$sentrylist" '%P\n' \) ) local -r trackmodelist=/tmp/trackmode_files local -r ignorelist2=/tmp/ignore_files2 (cd "$CAM_MOUNT"; find . \( -path './TeslaTrackMode/*' -type f \) \ -a \( \( -name '*.mp4' -size -100000c -fprintf /dev/stdout '%P\n' \) -o -fprintf "$trackmodelist" '%P\n' \) > "$ignorelist2" ) local -r sentry_count=$(wc -l < "$sentrylist") local -r trackmode_count=$(wc -l < "$trackmodelist") local -r ignore_count1=$(wc -l < "$ignorelist1") local -r ignore_count2=$(wc -l < "$ignorelist2") local -r total_count=$((sentry_count + trackmode_count)) # extract some noteworthy info from the file list local -r saved_event_count=$(grep 'SavedClips/' < "$sentrylist" | sed 's/[^\/]\+$//' | sort -u | wc -l) local -r sentry_event_count=$(grep 'SentryClips/' < "$sentrylist" | sed 's/[^\/]\+$//' | sort -u | wc -l) local -r event_count=$((saved_event_count + sentry_event_count)) log "There are $event_count event folder(s) with $sentry_count file(s) and $trackmode_count track mode file(s) to move. $((ignore_count1+ignore_count2)) short recording(s) will be skipped." if [[ "$total_count" -gt 0 ]] then log "Starting recording archiving" local -r start_ts=$(date --utc --date "now" +%s) /root/bin/send-push-message "$TESLAUSB_HOSTNAME:" "Archiving $event_count event folder(s) with $sentry_count file(s) and $trackmode_count track mode file(s) starting at $(date)" # Ensure Sentry Mode is enabled before archiving declare is_sentry_mode_enabled if [ -x /root/bin/tesla_api.py ] then is_sentry_mode_enabled=$(/root/bin/tesla_api.py is_sentry_mode_enabled | tr '[:upper:]' '[:lower:]') if [ "false" = "${is_sentry_mode_enabled}" ] then log "Temporarily enabling Sentry Mode to power the RPi while archive job completes..." /root/bin/tesla_api.py enable_sentry_mode &>> ${LOG_FILE} fi fi # setup trigger files local -r triggerdir=/tmp/triggers local -r triggerlist=/tmp/triggers.txt rm -rf "${triggerdir}" mkdir -p "${triggerdir}/SentryClips" mkdir -p "${triggerdir}/SavedClips" true > "${triggerlist}" if [ -n "${TRIGGER_FILE_SAVED+x}" ] then touch "${triggerdir}/SavedClips/${TRIGGER_FILE_SAVED}" echo "SavedClips/${TRIGGER_FILE_SAVED}" >> "${triggerlist}" fi if [ -n "${TRIGGER_FILE_SENTRY+x}" ] then touch "${triggerdir}/SentryClips/${TRIGGER_FILE_SENTRY}" echo "SentryClips/${TRIGGER_FILE_SENTRY}" >> "${triggerlist}" fi if [ -n "${TRIGGER_FILE_ANY+x}" ] then touch "${triggerdir}/${TRIGGER_FILE_ANY}" echo "${TRIGGER_FILE_ANY}" >> "${triggerlist}" fi local message if /root/bin/archive-clips.sh "$CAM_MOUNT/TeslaCam" "${sentrylist}" "$CAM_MOUNT" "${trackmodelist}" "${triggerdir}" "${triggerlist}" then message="Archiving completed successfully. " else message+="Error during archiving. " fi local -i sentry_archived=0 local -i trackmode_archived=0 while read -r line do if [[ ! -e "$CAM_MOUNT/TeslaCam/$line" ]] then sentry_archived=$((sentry_archived + 1)) fi done < ${sentrylist} while read -r line do if [[ ! -e "$CAM_MOUNT/$line" ]] then trackmode_archived=$((trackmode_archived + 1)) fi done < ${trackmodelist} message+="Archived " if [[ $sentry_count -gt 0 && $trackmode_count -gt 0 ]] then message+="${sentry_archived} event files and ${trackmode_archived} trackmode files" elif [[ $sentry_count -gt 0 ]] then message+="${sentry_archived} event files" else message+="${trackmode_archived} trackmode files" fi local -r end_ts=$(date --utc --date "now" +%s) local -r delta=$((end_ts - start_ts)) message+=" in $(convert_seconds_to_nice_time $delta)" /root/bin/send-push-message "$TESLAUSB_HOSTNAME:" "${message}" # delete empty directories under SavedClips, SentryClips and TeslaTrackMode rmdir --ignore-fail-on-non-empty "$CAM_MOUNT/TeslaCam/SavedClips"/* "$CAM_MOUNT/TeslaCam/SentryClips"/* "$CAM_MOUNT/TeslaTrackMode"/* || true # If Sentry Mode was previously disabled, restore it to that state if [ -x /root/bin/tesla_api.py ] then if [ "false" = "${is_sentry_mode_enabled}" ] then log "Restoring Sentry Mode to its previous state (disabled)..." /root/bin/tesla_api.py disable_sentry_mode &>> ${LOG_FILE} fi fi fi if [[ "$ignore_count1" -gt 0 ]] then ( cd "$CAM_MOUNT/TeslaCam" xargs rm -f < "$ignorelist1" ) fi if [[ "$ignore_count2" -gt 0 ]] then ( cd "$CAM_MOUNT" xargs rm -f < "$ignorelist2" ) fi # Delete the files that fsck "recovered". These are generally files # that were deleted previously by the car or teslausb rm -f "${CAM_MOUNT}"/FSCK????.REC # Trim the camera archive to reduce the number of blocks in the snapshot. trim_free_space "$CAM_MOUNT" unmount_cam_file } function copy_music_files () { log "Starting music sync..." ensure_music_file_is_mounted /root/bin/copy-music.sh # Trim the empty space from the music archive. trim_free_space "$MUSIC_MOUNT" unmount_music_file } function archive_clips () { log "Archiving..." if ! /root/bin/connect-archive.sh then log "Couldn't connect archive, skipping archive step" return fi if archive_teslacam_clips then log "Finished archiving." else log "Archiving failed." fi if timeout 5 [ -d "$MUSIC_ARCHIVE_MOUNT" -a -d "$MUSIC_MOUNT" ] then log "Copying music..." if copy_music_files then log "Finished copying music." else log "Copying music failed." fi else log "Music archive not configured or unreachable" fi /root/bin/disconnect-archive.sh } function truncate_log () { local log_length log_length=$( wc -l "$LOG_FILE" | cut -d' ' -f 1 ) if [ "$log_length" -gt 10000 ] then log "Truncating log..." local log_file2="${LOG_FILE}.2" tail -n 10000 "$LOG_FILE" > "${LOG_FILE}.2" mv "$log_file2" "$LOG_FILE" fi } function slowblink () { echo timer > /sys/class/leds/led0/trigger local ON=on local OFF=off if isPi4 || isKernel5 then ON=off OFF=on fi echo 900 > /sys/class/leds/led0/delay_$ON echo 100 > /sys/class/leds/led0/delay_$OFF } function fastblink () { echo timer > /sys/class/leds/led0/trigger local ON=on local OFF=off if isPi4 || isKernel5 then ON=off OFF=on fi echo 150 > /sys/class/leds/led0/delay_$ON echo 50 > /sys/class/leds/led0/delay_$OFF } function doubleblink () { echo heartbeat > /sys/class/leds/led0/trigger if isPi4 || isKernel5 then echo 0 > /sys/class/leds/led0/invert else echo 1 > /sys/class/leds/led0/invert fi } function set_time () { log "Trying to set time..." local -r uptime_start=$(awk '{print $1}' /proc/uptime) local -r clocktime_start=$(date +%s.%N) for _ in {1..5} do if sntp -S time.google.com then local -r uptime_end=$(awk '{print $1}' /proc/uptime) local -r clocktime_end=$(date +%s.%N) log "$(awk "BEGIN {printf \"Time adjusted by %f seconds after %f seconds\", $clocktime_end-$clocktime_start, $uptime_end-$uptime_start}")" return fi log "sntp failed, retrying..." sleep 2 done log "Failed to set time" } function snapshotloop { while true do sleep 3480 /root/bin/waitforidle || true /root/bin/make_snapshot.sh done } function wifichecker { dmesg -w | { while TMOUT=1 read -r line do true done wifi=working while read -r line do case $line in *"failed to enable fw supplicant") if [ "$wifi" = "working" ] then wifi="notworking" else log "restarting wifi because of: $line" modprobe -r brcmfmac cfg80211 brcmutil || true modprobe brcmfmac || true while TMOUT=1 read -r line do true done wifi="working" fi ;; *) wifi=working ;; esac done } } function set_sys_param() { local -r parampath="$1" local -r var="$2" local -r defaultval="$3" local -r val=${!var:-$defaultval} if [[ ${val} = "default" ]] then log "not setting $parampath" else echo "$val" > "$parampath" || true fi } export -f mount_mountpoint export -f ensure_mountpoint_is_mounted export -f retry export -f ensure_mountpoint_is_mounted_with_retry export -f log echo "==============================================" >> "$LOG_FILE" log "Starting archiveloop at $(awk '{print $1}' /proc/uptime) seconds uptime..." set_sys_param /proc/sys/vm/dirty_background_bytes DIRTY_BACKGROUND_BYTES 65536 set_sys_param /proc/sys/vm/dirty_ratio DIRTY_RATIO 80 set_sys_param /sys/devices/system/cpu/cpufreq/policy0/scaling_governor CPU_GOVERNOR conservative /root/bin/make_snapshot.sh snapshotloop & wifichecker & fix_errors_in_images if archive_is_reachable then fastblink set_time archive_clips doubleblink connect_usb_drives_to_host wait_for_archive_to_be_unreachable else slowblink connect_usb_drives_to_host fi while true do slowblink wait_for_archive_to_be_reachable fastblink set_time sleep "${archivedelay:-20}" # take a snapshot before archive_clips starts deleting files /root/bin/make_snapshot.sh disconnect_usb_drives_from_host fix_errors_in_images archive_clips truncate_log doubleblink connect_usb_drives_to_host wait_for_archive_to_be_unreachable check_if_usb_gadget_is_mounted done