<?php

namespace App\Models;

use App\Enums\BackupStorageType;
use App\Helpers\SshMultiplexingHelper;
use App\Helpers\SshRetryHandler;
use App\Rules\SafeWebhookUrl;
use App\Rules\ValidS3BucketName;
use App\Support\FtpsFilesystemAdapter;
use App\Support\PinnedSftpConnectionProvider;
use App\Support\SftpFilesystemAdapter;
use App\Support\WebdavFilesystemAdapter;
use App\Traits\Auditable;
use App\Traits\HasSafeStringAttribute;
use GuzzleHttp\Client;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
use Illuminate\Filesystem\FilesystemAdapter;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Facades\Validator;
use Illuminate\Support\Str;
use Illuminate\Validation\Rule;
use League\Flysystem\Filesystem;
use League\Flysystem\FilesystemAdapter as FlysystemAdapter;
use phpseclib3\Exception\UnableToConnectException;

class S3Storage extends BaseModel
{
    use Auditable, HasFactory, HasSafeStringAttribute;

    private const CONNECTION_TIMEOUT_SECONDS = 15;

    private const REQUEST_TIMEOUT_SECONDS = 15;

    /**
     * Endpoint host suffixes of providers that reject path-style URLs: Alibaba Cloud OSS, Tencent Cloud COS
     * (buckets created after 2024-01-01), and Volcengine / BytePlus TOS.
     */
    private const VIRTUAL_HOSTED_STYLE_HOST_SUFFIXES = ['aliyuncs.com', 'myqcloud.com', 'volces.com', 'bytepluses.com'];

    /**
     * Parts of the errors that these providers return for a path-style request.
     */
    private const PATH_STYLE_REJECTION_ERRORS = ['secondleveldomainforbidden', 'pathstyledomainforbidden', 'virtual hosted style', 'virtual-hosted style', 'virtual-styled'];

    /**
     * Waits before the second and the third attempt of an upload with the MinIO client.
     */
    private const MINIO_CLIENT_UPLOAD_RETRY_DELAYS_MS = [10_000, 30_000];

    /**
     * The most parts that an upload with the MinIO client uses. Most providers accept 10000 parts, but some, like
     * Scaleway, accept only 1000. The MinIO client sizes the parts for 10000 parts (16 MiB for files up to about
     * 156 GiB), so without a part size, files larger than about 15.6 GiB fail on these providers.
     */
    private const MINIO_CLIENT_MAX_UPLOAD_PARTS = 1000;

    /**
     * The smallest part size of an upload with the MinIO client. It is also the default part size of the MinIO client.
     */
    private const MINIO_CLIENT_MIN_PART_SIZE_BYTES = 16 * 1024 * 1024;

    /**
     * How long the presigned URL of a curl transfer is valid. S3 checks it when the request starts, so it must
     * cover the wait for the earlier transfers and the retries of the backup job.
     */
    private const CURL_TRANSFER_URL_TTL_HOURS = 24;

    private const CURL_TRANSFER_CONFIG_PATH = '/tmp/s3-transfer.curl';

    /**
     * The exception code of runSshProcess() when a remote command times out.
     */
    private const REMOTE_COMMAND_TIMEOUT_EXIT_CODE = 124;

    /**
     * The name of the rclone remote in rcloneConfig().
     */
    public const RCLONE_REMOTE = 'storage';

    public const SFTP_HOST_KEY_MISSING_MESSAGE = 'The SFTP host key of this storage is not saved yet. Validate the connection in the storage settings to save it.';

    /**
     * The key of the rclone password obscuring (fs/config/obscure in rclone). It only hides the password from a casual
     * look at the config file; it is not encryption.
     */
    private const RCLONE_OBSCURE_KEY = '9c935b48730a554d6bfd7c63c886a92bd390198eb8128afbf4de162b8b95f638';

    /**
     * How long the helper container waits for a TCP connection to one IP of the endpoint.
     */
    private const RESOLVE_PROBE_TIMEOUT_SECONDS = 5;

    protected $fillable = [
        'team_id',
        'type',
        'name',
        'description',
        'region',
        'key',
        'secret',
        'bucket',
        'endpoint',
        'use_path_style_endpoint',
        'private_key_id',
        'skip_tls_verification',
        'is_usable',
        'unusable_email_sent',
    ];

    protected $hidden = [
        'key',
        'secret',
    ];

    protected $attributes = [
        'type' => 's3',
    ];

    protected $casts = [
        'type' => BackupStorageType::class,
        'is_usable' => 'boolean',
        'use_path_style_endpoint' => 'boolean',
        'skip_tls_verification' => 'boolean',
        'key' => 'encrypted',
        'secret' => 'encrypted',
    ];

    /**
     * Boot the model and register event listeners.
     */
    protected static function boot(): void
    {
        parent::boot();

        // Trim whitespace from credentials before saving to prevent
        // "Malformed Access Key Id" errors from accidental whitespace in pasted values.
        // Note: We use the saving event instead of Attribute mutators because key/secret
        // use Laravel's 'encrypted' cast. Attribute mutators fire before casts, which
        // would cause issues with the encryption/decryption cycle.
        static::saving(function (S3Storage $storage) {
            if ($storage->key !== null) {
                $storage->key = trim($storage->key);
            }
            // An SFTP storage that logs in with a private key has no password. The column is not nullable.
            $storage->secret = trim((string) $storage->secret);
        });

        static::deleting(function (S3Storage $storage) {
            $storage->detachFromBackups();
        });
    }

    /**
     * Removes this storage from every backup schedule and marks the copies in it as deleted. A schedule moves its
     * primary destination to a remaining storage, or stops S3 backups when none remains.
     */
    public function detachFromBackups(): void
    {
        DB::transaction(function (): void {
            foreach ([ScheduledDatabaseBackup::class, ScheduledVolumeBackup::class] as $backupModel) {
                $backupModel::query()
                    ->usingS3Storage($this->id)
                    ->lazyById()
                    ->each(function (ScheduledDatabaseBackup|ScheduledVolumeBackup $backup): void {
                        $backup->syncS3Storages($backup->s3Storages()->whereKeyNot($this->id)->pluck('s3_storages.id')->all());
                        if ($backup->s3_storage_id === null && $backup->save_s3) {
                            $backup->update(['save_s3' => false]);
                        }
                    });
            }

            foreach ([DatabaseBackupS3Replica::class, VolumeBackupS3Replica::class] as $replicaModel) {
                $replicaModel::query()
                    ->with('execution')
                    ->where('s3_storage_id', $this->id)
                    ->where('s3_storage_deleted', false)
                    ->lazyById()
                    ->each(function (DatabaseBackupS3Replica|VolumeBackupS3Replica $replica): void {
                        $replica->update(['s3_storage_deleted' => true]);
                        $replica->execution?->refreshS3Summary();
                    });
            }

            ScheduledVolumeBackupExecution::query()
                ->where('s3_cleanup_pending', true)
                ->whereHas('s3Replicas', fn (Builder $query) => $query->where('s3_storage_id', $this->id))
                ->whereDoesntHave('s3Replicas', fn (Builder $query) => $query
                    ->where('s3_storage_deleted', false)
                    ->where(fn (Builder $query) => $query->whereNull('s3_uploaded')->orWhere('s3_uploaded', false)))
                ->update(['s3_cleanup_pending' => false]);
        });
    }

    public static function ownedByCurrentTeam(array $select = ['*'])
    {
        $selectArray = collect($select)->concat(['id']);

        return S3Storage::whereTeamId(currentTeam()->id)->select($selectArray->all())->orderBy('name');
    }

    public static function ownedByCurrentTeamAPI(int $teamId, array $select = ['*'])
    {
        $selectArray = collect($select)->concat(['id']);

        return S3Storage::whereTeamId($teamId)->select($selectArray->all())->orderBy('name');
    }

    public function isUsable()
    {
        return $this->is_usable;
    }

    /**
     * Storages that serve presigned URLs: file downloads and avatars need them, so they only use S3 storages.
     */
    public function scopeS3Only(Builder $query): Builder
    {
        return $query->where('type', BackupStorageType::S3->value);
    }

    public function isS3(): bool
    {
        return $this->storageType() === BackupStorageType::S3;
    }

    /**
     * The type of the storage. A model without the `type` attribute (a partial select, or raw attributes) is an S3
     * storage, as every storage was before other types existed.
     */
    public function storageType(): BackupStorageType
    {
        return $this->type ?? BackupStorageType::S3;
    }

    /**
     * Validation rules for the connection fields of a storage of this type. `$partial` makes every field optional, for
     * updates that only send some fields.
     *
     * `$savedEndpoint` is the endpoint that is saved now: it may keep a username or password from before this check.
     *
     * @param  array<int, string>  $trustedInternalHosts
     * @return array<string, array<int, mixed>>
     */
    public static function connectionRules(BackupStorageType $type, bool $partial = false, array $trustedInternalHosts = [], ?int $teamId = null, ?string $savedEndpoint = null): array
    {
        $required = $partial ? 'sometimes' : 'required';
        $endpoint = [
            $required,
            'string',
            'max:255',
            // Team members and read-only API tokens see the endpoint, so it must not hold credentials.
            function (string $attribute, mixed $value, \Closure $fail) use ($savedEndpoint): void {
                $parts = is_string($value) ? (parse_url(trim($value)) ?: []) : [];
                if ((isset($parts['user']) || isset($parts['pass'])) && trim((string) $value) !== trim((string) $savedEndpoint)) {
                    $fail('The :attribute must not contain a username or password. Enter the credentials in their own fields.');
                }
                if (self::hasControlCharacters(rawurldecode((string) ($parts['path'] ?? '')))) {
                    $fail('The :attribute path must not contain control characters.');
                }
            },
            new SafeWebhookUrl(trustedInternalHosts: $trustedInternalHosts, schemes: $type->endpointSchemes()),
        ];

        if ($type === BackupStorageType::S3) {
            return [
                'endpoint' => $endpoint,
                'bucket' => [$required, new ValidS3BucketName],
                'region' => [$required, 'string', 'max:255'],
                'key' => [$required, 'string', 'max:255'],
                'secret' => [$required, 'string', 'max:255'],
            ];
        }

        $rules = [
            'endpoint' => $endpoint,
            // rclone reads the user from a config file line, so it must not contain line breaks or quotes.
            'key' => [$required, 'string', 'max:255', 'regex:/^[^\x00-\x1f\x7f"]+$/'],
            'secret' => [$required, 'string', 'max:255'],
        ];

        if ($type === BackupStorageType::Sftp) {
            // SFTP logs in with a private key of the storage team or with a password.
            $rules['private_key_id'] = ['nullable', 'integer', Rule::exists('private_keys', 'id')->where('team_id', $teamId ?? -1)->where(fn ($query) => $query->where('is_git_related', false))];
            $rules['secret'] = [$partial ? 'sometimes' : 'required_without:private_key_id', 'nullable', 'string', 'max:255'];
        }

        return $rules;
    }

    /**
     * The endpoint URL of an SFTP or FTPS storage, from the form fields.
     */
    public static function serverEndpoint(BackupStorageType $type, string $host, ?string $port = '', string $path = ''): string
    {
        $host = trim($host);
        if (str_contains($host, ':') && ! str_starts_with($host, '[')) {
            $host = '['.$host.']';
        }
        $port = trim((string) $port);
        $path = implode('/', array_map('rawurlencode', array_filter(explode('/', trim($path)), fn (string $segment): bool => $segment !== '')));

        return $type->endpointSchemes()[0].'://'.$host.($port !== '' ? ':'.$port : '').($path !== '' ? '/'.$path : '');
    }

    /**
     * The form fields of an SFTP or FTPS endpoint URL.
     *
     * @return array{host: string, port: string, path: string}
     */
    public static function splitServerEndpoint(?string $endpoint): array
    {
        $parts = parse_url((string) $endpoint) ?: [];

        return [
            'host' => trim((string) ($parts['host'] ?? ''), '[]'),
            'port' => isset($parts['port']) ? (string) $parts['port'] : '',
            'path' => trim(rawurldecode((string) ($parts['path'] ?? '')), '/'),
        ];
    }

    /**
     * The directory on an SFTP or FTPS server that holds the backups: the path of the endpoint URL. Empty means the
     * login directory of the user.
     */
    public function basePath(): string
    {
        $path = rtrim(rawurldecode((string) parse_url((string) $this->endpoint, PHP_URL_PATH)), '/');
        if (self::hasControlCharacters($path)) {
            throw new \RuntimeException('The storage path must not contain control characters.');
        }

        return $path;
    }

    private static function hasControlCharacters(string $value): bool
    {
        return preg_match('/[\x00-\x1f\x7f]/', $value) === 1;
    }

    public function endpointPort(): int
    {
        return (int) (parse_url((string) $this->endpoint, PHP_URL_PORT) ?: $this->storageType()->defaultPort());
    }

    /**
     * The path of a backup file in the storage, as rclone reads it. The file keeps its full local path, as the S3 object
     * key does, below the base path (SFTP, FTPS) or the base URL (WebDAV).
     */
    public function rcloneRemotePath(string $file): string
    {
        $file = ltrim($file, '/');
        $base = $this->storageType() === BackupStorageType::Webdav ? '' : $this->basePath();

        return self::RCLONE_REMOTE.':'.($base === '' ? $file : $base.'/'.$file);
    }

    /**
     * The rclone config of this storage. The server gets it through stdin, so the password is never on a command line.
     */
    public function rcloneConfig(): string
    {
        $host = trim((string) parse_url((string) $this->endpoint, PHP_URL_HOST), '[]');
        $options = match ($this->storageType()) {
            // host_keys makes rclone refuse another host key, and offer only the algorithm of the saved key.
            BackupStorageType::Sftp => ['type' => 'sftp', 'host' => $host, 'port' => $this->endpointPort(), 'shell_type' => 'none', 'host_keys' => $this->requireSftpHostKey()],
            BackupStorageType::Ftps => ['type' => 'ftp', 'host' => $host, 'port' => $this->endpointPort(), 'explicit_tls' => 'true'],
            BackupStorageType::Webdav => ['type' => 'webdav', 'url' => $this->endpoint, 'vendor' => 'other'],
            BackupStorageType::S3 => throw new \LogicException('S3 storages use the MinIO client.'),
        };
        $options['user'] = $this->key;
        $privateKey = $this->sftpPrivateKey();
        if ($privateKey) {
            // rclone reads key_pem from one line, with \n for each line break.
            $options['key_pem'] = str_replace("\n", '\n', trim(str_replace("\r\n", "\n", (string) $privateKey->private_key)));
        } else {
            $options['pass'] = self::rcloneObscure((string) $this->secret);
        }

        $lines = ['['.self::RCLONE_REMOTE.']'];
        foreach ($options as $name => $value) {
            if (preg_match('/[\x00-\x1f\x7f]/', (string) $value)) {
                throw new \RuntimeException("The storage {$name} must not contain control characters.");
            }
            $lines[] = "{$name} = {$value}";
        }

        return implode("\n", $lines)."\n";
    }

    /**
     * Obscures a password as `rclone obscure` does: AES-CTR with the fixed rclone key and a random IV, in URL-safe
     * base64 without padding.
     */
    public static function rcloneObscure(string $value): string
    {
        $iv = random_bytes(16);
        $ciphertext = openssl_encrypt($value, 'aes-256-ctr', hex2bin(self::RCLONE_OBSCURE_KEY), OPENSSL_RAW_DATA, $iv);

        return rtrim(strtr(base64_encode($iv.$ciphertext), '+/', '-_'), '=');
    }

    public function team()
    {
        return $this->belongsTo(Team::class);
    }

    /**
     * The private key that an SFTP storage logs in with. Null: it logs in with the password.
     */
    public function privateKey(): BelongsTo
    {
        return $this->belongsTo(PrivateKey::class);
    }

    /**
     * The private key of an SFTP storage, only when it belongs to the team of the storage.
     */
    public function sftpPrivateKey(): ?PrivateKey
    {
        if ($this->storageType() !== BackupStorageType::Sftp || $this->private_key_id === null) {
            return null;
        }

        $privateKey = $this->privateKey;
        if (! $privateKey || (int) $privateKey->team_id !== (int) $this->team_id || $privateKey->is_git_related) {
            throw new \RuntimeException('The private key of this storage does not exist, belongs to another team, or is a Git deploy key.');
        }

        return $privateKey;
    }

    public function scheduledBackups()
    {
        return $this->hasMany(ScheduledDatabaseBackup::class, 's3_storage_id');
    }

    public function scheduledVolumeBackups()
    {
        return $this->hasMany(ScheduledVolumeBackup::class, 's3_storage_id');
    }

    public function awsUrl()
    {
        return "{$this->endpoint}/{$this->bucket}";
    }

    protected function path(): Attribute
    {
        return Attribute::make(
            set: function (?string $value) {
                if ($value === null || $value === '') {
                    return null;
                }

                return str($value)->trim()->start('/')->value();
            }
        );
    }

    /**
     * Trim whitespace from endpoint to prevent malformed URLs. Another host or port is another server, so the saved SFTP
     * host key no longer applies.
     */
    protected function endpoint(): Attribute
    {
        return Attribute::make(
            set: function (?string $value, array $attributes): array {
                $value = $value ? trim($value) : null;
                $previous = $attributes['endpoint'] ?? null;
                $server = fn (?string $endpoint): array => [strtolower(trim((string) parse_url((string) $endpoint, PHP_URL_HOST), '[]')), parse_url((string) $endpoint, PHP_URL_PORT)];

                if ($previous === null || $server($previous) === $server($value)) {
                    return ['endpoint' => $value];
                }

                // Without the host key, the storage needs a new connection test before Coolify uses it.
                return ['endpoint' => $value, 'sftp_host_key' => null, ...(($attributes['sftp_host_key'] ?? null) !== null ? ['is_usable' => false] : [])];
            },
        );
    }

    /**
     * The SHA256 fingerprint of the saved SFTP host key, as `ssh-keygen -l` prints it.
     */
    public function sftpHostKeyFingerprint(): ?string
    {
        return $this->sftp_host_key ? PinnedSftpConnectionProvider::fingerprint($this->sftp_host_key) : null;
    }

    /**
     * Trim whitespace from bucket name to prevent connection errors.
     */
    protected function bucket(): Attribute
    {
        return Attribute::make(
            set: fn (?string $value) => $value ? trim($value) : null,
        );
    }

    /**
     * Trim whitespace from region to prevent connection errors.
     */
    protected function region(): Attribute
    {
        return Attribute::make(
            set: fn (?string $value) => $value ? trim($value) : null,
        );
    }

    /**
     * @param  bool  $acceptNewHostKey  Only when an admin validates the storage: an SFTP storage without a saved host
     *                                  key then saves the presented key. Background tests refuse such a storage, so a
     *                                  reset or changed key is never trusted without the admin.
     */
    public function testConnection(bool $shouldSave = false, bool $acceptNewHostKey = false)
    {
        try {
            if ($this->isS3()) {
                $validator = Validator::make(
                    [
                        'endpoint' => $this['endpoint'],
                        'bucket' => $this['bucket'],
                    ],
                    [
                        'endpoint' => ['required', new SafeWebhookUrl(trustedInternalHosts: $this->trustedInternalHosts())],
                        'bucket' => ['required', new ValidS3BucketName],
                    ],
                );
                $validator->fails();
                if ($validator->errors()->has('endpoint')) {
                    throw new \RuntimeException('S3 endpoint is not allowed: '.$validator->errors()->first('endpoint'));
                }
                if ($validator->errors()->has('bucket')) {
                    throw new \RuntimeException('S3 bucket name is not allowed: '.$validator->errors()->first('bucket'));
                }

                $this->listFilesWithAddressingStyleFallback();
            } else {
                $validator = Validator::make(
                    $this->only(['endpoint', 'key', 'secret', 'private_key_id']),
                    self::connectionRules($this->storageType(), trustedInternalHosts: $this->trustedInternalHosts(), teamId: $this->team_id, savedEndpoint: $this->endpoint),
                );
                if ($validator->fails()) {
                    throw new \RuntimeException('Storage settings are not allowed: '.$validator->errors()->first());
                }

                if ($this->storageType() === BackupStorageType::Sftp && ! $acceptNewHostKey) {
                    $this->requireSftpHostKey();
                }

                $disk = $this->storageType() === BackupStorageType::Sftp ? $this->sftpFilesystem() : $this->filesystem();
                $disk->files();
                $adapter = $disk->getAdapter();
                if ($adapter instanceof SftpFilesystemAdapter && $this->sftp_host_key === null) {
                    $this->pinSftpHostKey((string) $adapter->presentedHostKey(), $shouldSave);
                }
                $this->assertWritable($disk);
            }

            $this->unusable_email_sent = false;
            $this->is_usable = true;
        } catch (\Throwable $e) {
            $exception = $this->toUserFriendlyConnectionException($e);
            $this->is_usable = false;
            if ($this->unusable_email_sent === false && is_transactional_emails_enabled()) {
                try {
                    $mail = new MailMessage;
                    $mail->subject('Coolify: Backup Storage Connection Error');
                    $mail->view('emails.s3-connection-error', ['name' => $this->name, 'reason' => $e->getMessage(), 'url' => base_url().'/storages/'.$this->uuid]);

                    // Load the team with its members and their roles explicitly
                    $team = $this->team()->with(['members' => function ($query) {
                        $query->withPivot('role');
                    }])->first();

                    // Get admins directly from the pivot relationship for this specific team
                    $users = $team->members()->wherePivotIn('role', ['admin', 'owner'])->get(['users.id', 'users.email']);
                    foreach ($users as $user) {
                        send_user_an_email($mail, $user->email, asNotification: true);
                    }
                    $this->unusable_email_sent = true;
                } catch (\Throwable $emailException) {
                    \Log::warning('Failed to send S3 connection error notification: '.$emailException->getMessage());
                }
            }

            throw $exception;
        } finally {
            if ($shouldSave) {
                $this->save();
            }
        }
    }

    /**
     * Creates the base directory, then writes and deletes a small file, as a backup upload does. Listing alone passes
     * on a directory that the user cannot create or write, for example "/backups" on a Hetzner Storage Box.
     */
    private function assertWritable(FilesystemAdapter $disk): void
    {
        $adapter = $disk->getAdapter();
        $file = '.coolify-write-test-'.Str::lower(Str::random(12));

        try {
            if ($adapter instanceof SftpFilesystemAdapter) {
                $adapter->createRootDirectory($this->basePath());
            }
            $disk->getDriver()->write($file, 'coolify');
        } catch (\Throwable $exception) {
            $directory = $this->basePath() === '' ? 'the login directory' : '"'.$this->basePath().'"';

            throw new \RuntimeException('Coolify cannot write to '.$directory.' on the storage: '.$exception->getMessage(), previous: $exception);
        }

        try {
            $disk->getDriver()->delete($file);
        } catch (\Throwable) {
            // Some servers allow uploads but not deletes. Backups still work there.
        }
    }

    /**
     * The files of the storage, as Coolify reads them (connection tests, restores, downloads, retention). SFTP and FTPS
     * connect to the validated IP of the host, and WebDAV pins it, so a DNS change cannot send them to another address.
     * SFTP also checks the saved host key, and refuses to connect before testConnection() saved it.
     */
    public function filesystem(): FilesystemAdapter
    {
        if ($this->storageType() === BackupStorageType::Sftp) {
            $this->requireSftpHostKey();
        }

        return match ($this->storageType()) {
            BackupStorageType::S3 => $this->s3Filesystem(),
            BackupStorageType::Sftp => $this->sftpFilesystem(),
            BackupStorageType::Ftps => $this->adapterFilesystem(new FtpsFilesystemAdapter(
                host: trim((string) parse_url((string) $this->endpoint, PHP_URL_HOST), '[]'),
                port: $this->endpointPort(),
                username: (string) $this->key,
                password: (string) $this->secret,
                root: $this->basePath(),
                pinnedIp: $this->pinnedHost(),
            )),
            BackupStorageType::Webdav => $this->webdavFilesystem(),
        };
    }

    /**
     * Without a saved host key, it accepts any key. Only testConnection() uses it so, to save the presented key.
     */
    private function sftpFilesystem(): FilesystemAdapter
    {
        return $this->adapterFilesystem(new SftpFilesystemAdapter(PinnedSftpConnectionProvider::create(
            host: trim((string) parse_url((string) $this->endpoint, PHP_URL_HOST), '[]'),
            address: $this->pinnedHost(),
            port: $this->endpointPort(),
            username: (string) $this->key,
            password: $this->sftpPrivateKey() ? null : (string) $this->secret,
            privateKey: $this->sftpPrivateKey()?->private_key,
            pinnedHostKey: $this->sftp_host_key,
            timeout: self::CONNECTION_TIMEOUT_SECONDS,
        ), $this->basePath()));
    }

    private function requireSftpHostKey(): string
    {
        if (blank($this->sftp_host_key)) {
            throw new \RuntimeException(self::SFTP_HOST_KEY_MISSING_MESSAGE);
        }

        return $this->sftp_host_key;
    }

    /**
     * Saves the host key of the first connection only while the saved storage still has the tested endpoint and no
     * key, so a test that started before an endpoint change cannot save the key of the old server.
     */
    private function pinSftpHostKey(string $hostKey, bool $shouldSave): void
    {
        if ($hostKey === '') {
            throw new \RuntimeException('The SFTP server did not present a host key.');
        }
        if ($shouldSave && $this->exists) {
            $saved = static::query()->whereKey($this->getKey())
                ->where('endpoint', $this->getAttributes()['endpoint'] ?? null)
                ->whereNull('sftp_host_key')
                ->update(['sftp_host_key' => $hostKey]);
            if ($saved === 0) {
                throw new \RuntimeException('The storage settings changed during the connection test. Validate the connection again.');
            }
            $this->sftp_host_key = $hostKey;
            $this->syncOriginalAttribute('sftp_host_key');

            return;
        }

        $this->sftp_host_key = $hostKey;
    }

    private function pinnedHost(): string
    {
        return SafeWebhookUrl::pinnedAddress((string) $this->endpoint, $this->storageType()->endpointSchemes(), $this->trustedInternalHosts());
    }

    private function adapterFilesystem(FlysystemAdapter $adapter): FilesystemAdapter
    {
        return new FilesystemAdapter(new Filesystem($adapter), $adapter);
    }

    private function webdavFilesystem(): FilesystemAdapter
    {
        $baseUrl = rtrim((string) $this->endpoint, '/').'/';
        $client = new Client(array_merge(SafeWebhookUrl::httpClientOptions($baseUrl, $this->trustedInternalHosts()), [
            'base_uri' => $baseUrl,
            'auth' => [$this->key, $this->secret],
            'http_errors' => false,
            'connect_timeout' => self::CONNECTION_TIMEOUT_SECONDS,
        ]));

        return $this->adapterFilesystem(new WebdavFilesystemAdapter($client, $baseUrl));
    }

    private function s3Filesystem(): FilesystemAdapter
    {
        return Storage::build([
            'driver' => 's3',
            'region' => $this['region'],
            'key' => $this['key'],
            'secret' => $this['secret'],
            'bucket' => $this['bucket'],
            'endpoint' => $this['endpoint'],
            'use_path_style_endpoint' => $this->usesPathStyleEndpoint(),
            'http' => array_merge($this->httpClientOptions(), [
                'connect_timeout' => self::CONNECTION_TIMEOUT_SECONDS,
                'timeout' => self::REQUEST_TIMEOUT_SECONDS,
            ]),
        ]);
    }

    /**
     * The Guzzle options of every request from Coolify to the endpoint: the DNS pin of SafeWebhookUrl, and the TLS
     * certificate check, which a storage with a self-signed or private CA certificate can turn off.
     *
     * @return array<string, mixed>
     */
    public function httpClientOptions(): array
    {
        return array_merge(SafeWebhookUrl::httpClientOptions((string) $this->endpoint, $this->trustedInternalHosts()), [
            'verify' => ! $this->skip_tls_verification,
        ]);
    }

    /**
     * The `docker run` options of the MinIO client helper. `MC_REGION` makes mc sign requests for the storage region,
     * as filesystem() does. Without it, mc signs for us-east-1, and providers that enforce the region reject the
     * requests. `MC_INSECURE` turns off the TLS certificate check, as httpClientOptions() does.
     */
    public function minioClientEnvOptions(): string
    {
        return '-e '.escapeshellarg('MC_REGION='.$this->region)
            .($this->skip_tls_verification ? ' -e MC_INSECURE=true' : '');
    }

    /**
     * The `docker exec` option that makes `mc cp` upload a file of $fileSize bytes in at most 1000 parts.
     * `mc cp` has no part size flag, but it reads the part size from MC_UPLOAD_MULTIPART_SIZE.
     */
    public static function minioClientPartSizeOption(int $fileSize): string
    {
        $mebibyte = 1024 * 1024;
        $partSize = max(
            self::MINIO_CLIENT_MIN_PART_SIZE_BYTES,
            (int) ceil($fileSize / self::MINIO_CLIENT_MAX_UPLOAD_PARTS / $mebibyte) * $mebibyte,
        );

        return '-e MC_UPLOAD_MULTIPART_SIZE='.$partSize;
    }

    /**
     * Path-style URLs (`endpoint/bucket`) are the default. Providers like Alibaba Cloud OSS only accept
     * virtual-hosted-style URLs (`bucket.endpoint`). A storage that is not saved yet has no value, so it uses the default.
     */
    public function usesPathStyleEndpoint(): bool
    {
        return $this->use_path_style_endpoint ?? true;
    }

    /**
     * The URL style for a new storage: virtual-hosted style for providers that reject path-style URLs, else path-style.
     */
    public static function detectPathStyleEndpoint(?string $endpoint): bool
    {
        $host = strtolower((string) parse_url((string) $endpoint, PHP_URL_HOST));

        return ! collect(self::VIRTUAL_HOSTED_STYLE_HOST_SUFFIXES)
            ->contains(fn (string $suffix): bool => $host === $suffix || str_ends_with($host, '.'.$suffix));
    }

    /**
     * Tests the connection with ListObjectsV2. When the provider rejects path-style URLs, it tests again with
     * virtual-hosted-style URLs and keeps that style when it works.
     */
    private function listFilesWithAddressingStyleFallback(): void
    {
        try {
            $this->filesystem()->files();
        } catch (\Throwable $exception) {
            if (! $this->usesPathStyleEndpoint() || ! self::isPathStyleRejection($exception)) {
                throw $exception;
            }

            $this->use_path_style_endpoint = false;
            try {
                $this->filesystem()->files();
            } catch (\Throwable) {
                $this->use_path_style_endpoint = true;

                throw $exception;
            }
        }
    }

    private static function isPathStyleRejection(\Throwable $exception): bool
    {
        for ($current = $exception; $current !== null; $current = $current->getPrevious()) {
            if (str($current->getMessage())->lower()->contains(self::PATH_STYLE_REJECTION_ERRORS)) {
                return true;
            }
        }

        return false;
    }

    /**
     * The `mc alias set` options. `--api S3v4` signs as filesystem() does. Without it, mc probes the signature
     * version on every run, and a network error during the probe makes it try S3v2, which most providers reject.
     * `--path off` uses virtual-hosted-style URLs, as filesystem() does. With path-style URLs, mc keeps its own
     * `auto` lookup, as before.
     */
    public function minioClientAliasOptions(): string
    {
        return ' --api S3v4'.($this->usesPathStyleEndpoint() ? '' : ' --path off');
    }

    /**
     * The MinIO client accepts only `scheme://host[:port]` endpoints. Providers like Supabase Storage need a path
     * (`/storage/v1/s3`), so transfers to these endpoints use curl with a presigned URL instead.
     */
    public function usesCurlTransfer(): bool
    {
        return $this->isS3() && trim((string) parse_url((string) $this->endpoint, PHP_URL_PATH), '/') !== '';
    }

    /**
     * Prepares the upload of $localPath in $container to $key and returns the command that runs it.
     */
    public function prepareCurlUpload(Server $server, string $container, string $localPath, string $key): string
    {
        $url = $this->filesystem()->temporaryUploadUrl($key, now()->addHours(self::CURL_TRANSFER_URL_TTL_HOURS))['url'];

        // An S3 error is an XML body. Write it to stderr, so it becomes the error message of the backup.
        return $this->prepareCurlTransfer($server, $container, $url, ['upload-file = '.self::curlConfigValue($localPath), 'fail-with-body', 'output = "/dev/stderr"']);
    }

    /**
     * Prepares the download of $key to $localPath in $container and returns the command that runs it.
     */
    public function prepareCurlDownload(Server $server, string $container, string $key, string $localPath): string
    {
        $url = $this->filesystem()->temporaryUrl($key, now()->addHours(self::CURL_TRANSFER_URL_TTL_HOURS));

        return $this->prepareCurlTransfer($server, $container, $url, ['output = '.self::curlConfigValue($localPath), 'fail']);
    }

    /**
     * Writes the curl config of a transfer into $container through the stdin of the SSH process. The config holds
     * the presigned URL, so the URL is not in the arguments of any process, and the S3 keys never reach the server.
     * The `resolve` line pins the endpoint host to its validated IPs, as minioClientResolveOptions() does for mc.
     *
     * @param  array<int, string>  $options
     */
    private function prepareCurlTransfer(Server $server, string $container, string $url, array $options): string
    {
        $entries = SafeWebhookUrl::minioClientResolveOptions((string) $this->endpoint, $this->trustedInternalHosts());
        $config = collect(['url = '.self::curlConfigValue($url), 'silent', 'show-error', ...$options])
            ->when($entries !== [], fn ($lines) => $lines->push('resolve = '.self::curlConfigValue(
                str($entries[0])->beforeLast('=')->toString().':'.collect($entries)
                    ->map(fn (string $entry): string => str($entry)->afterLast('=')->toString())
                    ->map(fn (string $ip): string => str_contains($ip, ':') ? "[{$ip}]" : $ip)
                    ->implode(',')
            )))
            ->when($this->skip_tls_verification, fn ($lines) => $lines->push('insecure'))
            ->implode("\n")."\n";

        $remoteCommand = ($server->isNonRoot() ? 'sudo ' : '').'docker exec -i '.escapeshellarg($container)
            .' sh -c '.escapeshellarg('umask 077 && cat > '.self::CURL_TRANSFER_CONFIG_PATH);
        $timeout = (int) config('constants.ssh.command_timeout');

        SshRetryHandler::retry(
            function () use ($server, $remoteCommand, $config, $timeout) {
                $sshCommand = SshMultiplexingHelper::generateSshStdinCommand($server, $remoteCommand, disableMultiplexing: true, commandTimeout: $timeout);
                $process = runSshProcess($sshCommand, $timeout, $config);

                if ($process->exitCode() !== 0) {
                    excludeCertainErrors($process->errorOutput(), $process->exitCode());
                }
            },
            ['server' => $server->ip, 'function' => 'prepareCurlTransfer'],
        );

        return 'docker exec '.escapeshellarg($container).' curl --config '.self::CURL_TRANSFER_CONFIG_PATH;
    }

    private static function curlConfigValue(string $value): string
    {
        return '"'.addcslashes($value, '"\\').'"';
    }

    /**
     * The mc `--resolve` option that pins the endpoint host to one of its validated IPs. mc keeps one IP per host,
     * so when the host has more than one IP, a helper container on the upload network connects to each IP, IPv4
     * first, and the first IP that connects is used. A server without IPv4 then uses an IPv6 address of a
     * dual-stack endpoint. When no IP connects, the first IP is used, so mc reports the real connection error.
     */
    public function minioClientResolveOptions(Server $server, string $image, ?string $network = null): string
    {
        $entries = SafeWebhookUrl::minioClientResolveOptions($this->endpoint, $this->trustedInternalHosts());
        if ($entries === []) {
            return '';
        }

        $entry = $entries[0];
        if (count($entries) > 1) {
            $ips = array_map(fn (string $entry): string => str($entry)->afterLast('=')->toString(), $entries);
            $port = (int) str($entries[0])->beforeLast('=')->afterLast(':')->toString();
            $probe = 'for ip in '.implode(' ', array_map('escapeshellarg', $ips)).'; do'
                .' if timeout '.self::RESOLVE_PROBE_TIMEOUT_SECONDS.' bash -c "exec 3<>/dev/tcp/$ip/'.$port.'" 2>/dev/null; then echo "$ip"; exit 0; fi;'
                .' done';
            $reachableIp = trim((string) instant_remote_process(
                ['docker run --rm'.($network === null ? '' : ' --network '.escapeshellarg($network))
                    .' '.escapeshellarg($image).' bash -c '.escapeshellarg($probe)],
                $server,
                throwError: false,
                timeout: count($ips) * self::RESOLVE_PROBE_TIMEOUT_SECONDS + 60,
                disableMultiplexing: true,
            ));
            $reachableIndex = array_search($reachableIp, $ips, true);
            if ($reachableIndex !== false) {
                $entry = $entries[$reachableIndex];
            }
        }

        return ' --resolve '.escapeshellarg($entry);
    }

    /**
     * Runs an upload of a local file with the MinIO client again when it fails, for example after a short network
     * outage. A timeout is not retried, because another full attempt can pass the timeout of the backup job.
     */
    public static function retryMinioClientUpload(callable $upload): mixed
    {
        return retry(
            self::MINIO_CLIENT_UPLOAD_RETRY_DELAYS_MS,
            $upload,
            when: fn (\Throwable $exception): bool => $exception->getCode() !== self::REMOTE_COMMAND_TIMEOUT_EXIT_CODE,
        );
    }

    /**
     * The bundled MinIO container is a trusted internal S3 target, not a user-supplied webhook destination.
     *
     * @return array<int, string>
     */
    public function trustedInternalHosts(): array
    {
        return $this->uuid === 'minio' && parse_url($this->endpoint, PHP_URL_HOST) === 'coolify-minio'
            ? ['coolify-minio']
            : [];
    }

    private function toUserFriendlyConnectionException(\Throwable $exception): \Throwable
    {
        $rootCause = $exception;
        while ($rootCause->getPrevious() !== null) {
            $rootCause = $rootCause->getPrevious();
        }
        if ($rootCause instanceof UnableToConnectException) {
            // Flysystem shows only "Unable to connect to host". The network error (refused, timed out) is in phpseclib.
            return new \RuntimeException('Could not connect to the '.$this->storageType()->label().' server: '.$rootCause->getMessage(), previous: $exception);
        }

        $message = str($exception->getMessage())->lower();

        if ($message->contains(['timed out', 'timeout', 'connection refused', 'could not resolve', 'curl error 28'])) {
            return new \RuntimeException(
                $this->isS3()
                    ? 'Could not connect to the S3 endpoint within 15 seconds. Please verify the endpoint, bucket, credentials, region, and network/firewall settings.'
                    : 'Could not connect to the '.$this->storageType()->label().' server within 15 seconds. Please verify the host, port, credentials, and network/firewall settings.',
                previous: $exception,
            );
        }

        return $exception;
    }
}