server->uuid; } /** * The job that starts an rclone container creates its execution or activity before it, except a database * import, which creates its activity right after it. Younger containers are kept. */ private const RCLONE_MIN_AGE_SECONDS = 900; private static function helperContainersCommand(): string { return 'docker container ps --format \'{{json .}}\' | jq -s \'map(select(.Image|test("(^|/)coollabsio/coolify-helper(:|@)")))\''; } /** * Removes helper containers of finished deployments and the S3 upload containers (`backup-of-`) * of database backups that are not running. Imports and file browsers also run in helper containers, so * other containers stay. */ public function handle(): void { try { $activeTransfers = $this->activeTransferOwners(); $this->removeOrphanedHelperContainers($activeTransfers); $this->removeOrphanedRcloneContainers($activeTransfers); } catch (\Throwable $e) { send_internal_notification('CleanupHelperContainersJob failed with error: '.$e->getMessage()); } } /** * @param Collection $activeTransfers */ private function removeOrphanedHelperContainers(Collection $activeTransfers): void { $containers = instant_remote_process_with_timeout([self::helperContainersCommand()], $this->server, false); $helperContainers = collect(json_decode((string) $containers, true) ?: []) ->filter(fn (mixed $container): bool => is_array($container) && filled(data_get($container, 'ID')) && filled(data_get($container, 'Names'))); if ($helperContainers->isEmpty()) { return; } $finishedDeploymentUuids = ApplicationDeploymentQueue::query() ->whereIn('deployment_uuid', $helperContainers->pluck('Names')->all()) ->whereNotIn('status', [ ApplicationDeploymentStatus::IN_PROGRESS->value, ApplicationDeploymentStatus::QUEUED->value, ]) ->pluck('deployment_uuid') ->all(); foreach ($helperContainers as $container) { $isFinishedDeployment = in_array($container['Names'], $finishedDeploymentUuids, true); // A worker that is killed during the upload never removes its upload container. $isOrphanedBackupUpload = str_starts_with($container['Names'], 'backup-of-') && ! $activeTransfers->contains(Str::after($container['Names'], 'backup-of-')); if (! $isFinishedDeployment && ! $isOrphanedBackupUpload) { continue; } \Log::info('CleanupHelperContainersJob - Removing orphaned helper container', [ 'container' => $container['Names'], 'id' => $container['ID'], ]); instant_remote_process_with_timeout(['docker container rm -f '.escapeshellarg($container['ID'])], $this->server, false); } } /** * An rclone container has the storage config with its password or SSH key. It stays until the end of its sleep * when a worker timeout kills the backup or import job before the job removes it. * * @param Collection $activeTransfers */ private function removeOrphanedRcloneContainers(Collection $activeTransfers): void { $output = instant_remote_process_with_timeout([ 'docker container ps --filter label='.RcloneTransfer::LABEL." --format '{{json .}}'", ], $this->server, false); $containers = collect(explode("\n", (string) $output)) ->map(fn (string $line): mixed => json_decode(trim($line), true)) ->filter(fn (mixed $container): bool => is_array($container) && filled($container['ID'] ?? null)); if ($containers->isEmpty()) { return; } foreach ($containers as $container) { $name = (string) ($container['Names'] ?? ''); $createdAt = Carbon::createFromFormat('Y-m-d H:i:s O', substr((string) ($container['CreatedAt'] ?? ''), 0, 25)); if (! $createdAt || $createdAt->gt(now()->subSeconds(self::RCLONE_MIN_AGE_SECONDS))) { continue; } if (self::belongsToActiveTransfer($name, $activeTransfers)) { continue; } \Log::info('CleanupHelperContainersJob - Removing orphaned rclone container', ['container' => $name]); instant_remote_process_with_timeout(['docker container rm -f '.escapeshellarg((string) $container['ID'])], $this->server, false); } } /** * @param Collection $activeTransfers */ private static function belongsToActiveTransfer(string $containerName, Collection $activeTransfers): bool { return preg_match('/^(backup-of|volume-backup|volume-upload|s3-restore|backup-decompress)-(.+)$/', $containerName, $matches) === 1 && $activeTransfers->contains($matches[2]); } /** * UUIDs of the running backups and database imports. Their upload and restore containers (helper or rclone) are * named "backup-of-", "volume-backup-", "volume-upload-", "s3-restore-", or "backup-decompress-" and the UUID. * * @return Collection */ private function activeTransferOwners(): Collection { $imports = Activity::query() ->where('properties->operation', ResourceStartActivity::DATABASE_IMPORT_OPERATION) ->whereIn('properties->status', [ProcessStatus::QUEUED->value, ProcessStatus::IN_PROGRESS->value]) ->get() ->map(fn (Activity $activity): string => (string) data_get($activity, 'properties.operation_uuid')); return ScheduledDatabaseBackupExecution::query()->where('status', 'running')->pluck('uuid') ->merge(ScheduledVolumeBackupExecution::query()->where('status', 'running')->pluck('uuid')) ->merge($imports) ->filter() ->values(); } }