# Restores a stored HELIOS backup in a detached `docker:cli` container (see # DetachedRunner for the shared launch / in-progress machinery). It clears # the database directories, starts empty database services from the # restored configuration and imports PostgreSQL plus InfluxDB. # # End-to-end flow (three actors, like BackupRunner, but reversed: the tar # is the input and the running stack is the output): # # 1. `start(filename)` (this class, Ruby/HELIOS side) # - validate_locks! — no concurrent backup, no concurrent restore # - pause_updates! — freeze Watchtower for the whole run, the S3 # download and the config swap below included # (see Orchestration::UpdatePause) # - pull_image_if_needed! (docker:cli) # - clear_errors! — wipe last run's error markers # - mark_pending! — UI shows the target filename # # Then the S3 vs. local/external paths diverge so the tar is fetched # at most once: # - S3: Downloader thread fetches the tar into local staging; # the rest of the restore runs in its on-complete block # because the restored config will overwrite the current # S3 credentials. See `s3_download_in_progress` for how # the UI keeps reporting "in progress" during the fetch. # - other: tar already lives at the destination — run inline. # # 2. run_restore! (still HELIOS-side, but only once the tar is local) # - validate_archive! — config.yaml + PostgreSQL + InfluxDB # entries must all be present in the # archive (read via tar index, no # full extraction) # - prepare_restored_stack! — atomic swap of `config.yaml` to the # archive's copy, regenerate # `compose.yaml` + `.env` from it # (Export::Builder), then re-read # INFLUX_ADMIN_TOKEN from the new env # - run_container! — `docker run -d` against restore.sh # # 3. restore.sh (inside the detached sidecar; positional argv) # Phases written atomically to /runtime/restore-phase.txt: # :extracting → safe-path check on the tar index, then # extract the influx subdir straight into # the shared /influx-backup-staging mount # and the rest into local /runtime work # :stopping_services → `compose down -v --remove-orphans` for # every service except `helios` (stopping # our own container would kill the user's # UI mid-restore). `-v` drops the InfluxDB # image's anonymous /etc/influxdb2 volume, # which otherwise traps a stale CLI config # and breaks the next setup run. # :starting_databases → fresh `compose up -d postgresql influxdb` # on empty data dirs, then poll TCP # readiness (the compose healthcheck flips # "healthy" against the bootstrap server # that only listens on the Unix socket, # so `--wait` returns too early), then # create the `solectrus_production` DB # that SOLECTRUS would normally create on # its own first boot. # :restoring_postgres → `gunzip | psql` the dump # :restoring_influx → `influx restore --full` against the # shared staging mount; # `--operator-token` is required when the # source instance had hashed tokens # (InfluxDB OSS 2.9+ default), otherwise # auth breaks after --full overwrites the # bolt. Passed only when the influx CLI # knows the flag # :starting_services → only if every non-HELIOS service was # running before the restore; otherwise # leave manually-stopped services alone # Failures land in /runtime/restore-error.txt (local, never # /output — the destination may be the cause). # # Mutual exclusion: same CONTAINER_NAME lock as BackupRunner; the # cross-runner check (`BackupRunner.running?`) reads docker live, not # the IN_PROGRESS_CACHE_TTL cache, so a restore cannot launch against a # database that a still-running backup is reading. class RestoreRunner < DetachedRunner CONTAINER_NAME = 'helios-restore-runner'.freeze IMAGE = BackupRunner::IMAGE POSTGRES_SERVICE = BackupRunner::POSTGRES_SERVICE INFLUXDB_SERVICE = BackupRunner::INFLUXDB_SERVICE ERROR_FILENAME = 'restore-error.txt'.freeze PHASE_FILENAME = 'restore-phase.txt'.freeze I18N_SCOPE = 'backups.restorer'.freeze UPDATE_PAUSE_REASON = :restore # Phase names the restore script writes into PHASE_FILENAME. Anything # outside this allowlist falls back to the generic "Restoring…" label. KNOWN_PHASES = %i[extracting stopping_services starting_databases restoring_postgres restoring_influx starting_services].freeze SCRIPT = ::File.read(::File.join(__dir__, 'restore_runner', 'restore.sh')).freeze class << self def clear_error! BackupRepository.clear_error!(ERROR_FILENAME) end # Extends the inherited in_progress with the S3 download phase, so the # UI keeps showing "in progress" while HELIOS fetches the tar before # the detached restore container runs. The script-phase enrichment # itself happens in DetachedRunner. def in_progress super || s3_download_in_progress end def s3_download_in_progress return nil unless BackupRepository.s3? BackupRepository::S3::Downloader.current end end def start(filename) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength @backup = BackupRepository.find!(filename) logger.info("start filename=#{filename} s3=#{BackupRepository.s3?}") validate_locks! pause_updates! pull_image_if_needed! clear_errors! RunnerLog.record_started!(:restore) BackupRepository.mark_pending! if BackupRepository.s3? # Validation, config switch and container start all run after the # download so the tar is only fetched once. The restored config # would overwrite the current S3 credentials anyway, so the # download must precede it. Failures surface via restore-error.txt # in detect_completion!. backup.bytes feeds the progress callback # without a HEAD round-trip (some S3-compatible endpoints forbid # HEAD while allowing GET). logger.info("handing off to Downloader (bytes=#{backup.bytes})") BackupRepository::S3::Downloader.start_async(backup.filename, total: backup.bytes) do logger.info('download complete, entering run_restore!') run_restore! logger.info('run_restore! returned cleanly') end else run_restore! end self.class.invalidate_in_progress_cache! rescue BackupRepository::NotFound raise Error, error(:backup_not_found) end private attr_reader :backup def validate_locks! raise Error, error(:backup_in_progress) if BackupRunner.running? raise Error, error(:csv_import_in_progress) if CsvImportRunner.in_progress? raise Error, error(:already_in_progress) if self.class.in_progress end def validate_archive! raise Error, error(:missing_config) if restored_configuration_data.blank? raise Error, error(:missing_postgres) unless postgresql_entry? raise Error, error(:missing_influx) unless influxdb_entry? end def run_restore! validate_archive! prepare_restored_stack! run_container! end def prepare_restored_stack! restored = restored_configuration_data.deep_dup preserve_target_volume_paths!(restored) preserve_target_backup_destination!(restored) write_restored_configuration!(restored) Export::Builder.new(Configuration.current).write! raise Error, error(:missing_token) if influx_admin_token.blank? end # `volume_path` is host infrastructure — the target's storage layout, not # backup payload. Cross-host restores (e.g. Pi → Proxmox/LXC) would # otherwise replay the source's absolute paths onto a system where they # don't exist or don't belong. Force the target's value (or its absence, # which means "use the default `/` mount") for # every persistent service before writing the restored config to disk. # See issue #145. def preserve_target_volume_paths!(restored) target = Configuration.current Export::Compose.persistent_services.each do |service_class| setting = service_class.config_keys.first target_path = target.public_send(setting).volume_path if target_path.present? (restored[setting] ||= {})['volume_path'] = target_path else restored[setting]&.delete('volume_path') end end end # The backup destination is host infrastructure as well: an external path # names a mount of the source host that may not exist on the target. The # target's destination is known to work, the restore has just read the # tar from it, so it wins over the archive's. def preserve_target_backup_destination!(restored) target = Configuration.load_file(Configuration.path)['backup'] if target.present? restored['backup'] = target else restored.delete('backup') end end def write_restored_configuration!(data) tmp_path = "#{Configuration.path}.tmp" ::File.write(tmp_path, Configuration.dump(data)) ::File.rename(tmp_path, Configuration.path) Current.configuration = nil end def clear_errors! BackupRepository.clear_error! self.class.clear_error! end def docker_run_command [ 'docker', 'run', '--rm', '-d', '--name', CONTAINER_NAME, '-v', '/var/run/docker.sock:/var/run/docker.sock', # See BackupRunner#docker_run_command for why the destination uses # `--mount type=bind` instead of `-v` — refusing the start beats # restoring from a phantom directory. '--mount', "type=bind,source=#{BackupRepository.host_directory},target=/output", '-v', "#{self.class.host_runtime_directory}:#{RUNTIME_MOUNT}", '-v', "#{self.class.host_influx_staging_directory}:#{INFLUX_STAGING_MOUNT}", *data_mount_args, '--entrypoint', 'sh', IMAGE, '-c', SCRIPT, '_', *positional_args ] end def positional_args [ influx_admin_token, backup.filename, host_data_path, postgresql_data_path, influxdb_data_path, redis_data_path, restart_after_flag, services_except_self.join(' '), ::Compose.filename, cleanup_tar_after_flag ] end # "1" iff /output is the S3 staging dir — the tar there is a throwaway # copy the Downloader fetched and should be removed once the restore # consumed it. For local/external destinations /output is the user's # actual backups dir; the tar must survive the restore. def cleanup_tar_after_flag BackupRepository.s3? ? '1' : '0' end def data_mount_args args = ['-v', "#{host_data_path}:/data"] args.push('-v', "#{host_data_path}:#{host_data_path}") unless host_data_path == '/data' data_paths.each do |path| args.push('-v', "#{path}:#{path}") if mount_data_path?(path) end args end def mount_data_path?(path) Pathname.new(path).absolute? && !path.start_with?("#{host_data_path}/") end def host_data_path @host_data_path ||= Orchestration::Runner.host_data_path end # "1" iff every configured service (minus HELIOS itself) is currently # running. The restore script restarts the stack only in that case; # otherwise it leaves manually-stopped services alone after the restore. def restart_after_flag running = Orchestration::Container.all.select(&:running?).filter_map(&:service_name) (services_except_self - running).empty? ? '1' : '0' end # The HELIOS service must never appear in compose down/up calls issued by # the restore script — stopping our own container would kill the user's # UI mid-restore. Mirrors `Orchestration::Runner.services_except_self`. def services_except_self @services_except_self ||= ::Compose.load.services.names - [Orchestration::Runner::SELF_SERVICE] end def influx_admin_token @influx_admin_token ||= Env.load&.[]('INFLUX_ADMIN_TOKEN') end def data_paths @data_paths ||= [postgresql_data_path, influxdb_data_path, redis_data_path].uniq end def restored_configuration_data archive_contents.config end def postgresql_entry? archive_contents.entries.any? { |entry| entry.name.match?(BackupRepository::POSTGRESQL_ENTRY_PATTERN) } end def influxdb_entry? archive_contents.entries.any? { |entry| entry.name.match?(BackupRepository::INFLUXDB_ENTRY_PATTERN) } end # For S3, validation runs after Downloader has fetched the tar into # staging — parsing the local copy avoids a second remote read and # the 400 MB-class StringIO that the in-memory variant would buffer. def archive_contents @archive_contents ||= if BackupRepository.s3? BackupRepository.read_archive(BackupRepository::S3.staging_path(backup.filename)) else BackupRepository.read_archive_for(backup.filename) end end def postgresql_data_path storage_path_for('postgresql', POSTGRES_SERVICE) end def influxdb_data_path storage_path_for('influxdb', INFLUXDB_SERVICE) end def redis_data_path storage_path_for('redis', 'redis') end def storage_path_for(section, default_dir) configured = Configuration.current.public_send(section).volume_path.presence path = configured || ::File.join(host_data_path, default_dir) return path if Pathname.new(path).absolute? ::File.expand_path(path, host_data_path) end end