using System.Security.Cryptography;
using FluentAssertions;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Nostos.Backend.Configuration;
using Nostos.Backend.Data;
using Nostos.Backend.Data.Models;
using Nostos.Backend.Services;
using Nostos.Backend.Services.Library;
using Nostos.Backend.Services.Portability;
using Nostos.Backend.Services.Portability.Activation;
using Nostos.Backend.Services.Portability.Migration;
using Nostos.Backend.Services.Portability.Transfers;
namespace Nostos.Backend.Tests.Portability;
///
/// Immutable, process-wide activation source shared by every test bed copy: a
/// migrated host database seeded with the activation job and a real prepared
/// import staged on disk. Building it once keeps the crash matrix (one bed per
/// boundary) and the 10x loop cheap; every bed copies the database bytes so
/// cases cannot contaminate each other.
///
internal sealed class ActivationCoordinatorTemplate : IDisposable
{
internal const string OriginalRevision = "41";
internal const long AdvancedRevision = 42;
private static readonly Lazy PopulatedTemplate =
new(() => Create(populated: true), LazyThreadSafetyMode.ExecutionAndPublication);
private static readonly Lazy EmptyTemplate =
new(() => Create(populated: false), LazyThreadSafetyMode.ExecutionAndPublication);
static ActivationCoordinatorTemplate()
{
AppDomain.CurrentDomain.ProcessExit += (_, _) =>
{
if (PopulatedTemplate.IsValueCreated) PopulatedTemplate.Value.Dispose();
if (EmptyTemplate.IsValueCreated) EmptyTemplate.Value.Dispose();
};
}
internal static ActivationCoordinatorTemplate For(bool populated) =>
populated ? PopulatedTemplate.Value : EmptyTemplate.Value;
internal string Root { get; }
internal bool Populated { get; }
internal string ArchivePath { get; }
internal Guid JobId { get; } = Guid.Parse("6a3490c2d6fd4a63a2f9b67ad56b31c1");
internal Guid ReservationId { get; private set; }
internal string TemplateDatabase { get; }
internal string TemplateMedia { get; }
internal string StagingRoot { get; }
internal LocalPortableImportStaging Staging { get; }
internal IPreparedPortableImport Prepared { get; private set; } = null!;
internal PortablePreparedImportVerification ExpectedHandle { get; private set; } = null!;
internal PortableFixtureIds SourceIds { get; private set; } = null!;
internal string RevisionToken { get; private set; } = string.Empty;
internal Dictionary> OriginalAllTables { get; private set; } = null!;
internal string OriginalDatabaseSha256 { get; private set; } = string.Empty;
internal Dictionary> OriginalPortable { get; private set; } = null!;
internal Dictionary OriginalMedia { get; private set; } = null!;
internal Dictionary ExpectedImportedMedia { get; private set; } = null!;
internal DateTimeOffset BaseTime { get; } = new(2026, 10, 4, 12, 0, 0, TimeSpan.Zero);
internal ManualTimeProvider Clock { get; } = new(new DateTimeOffset(2026, 10, 4, 12, 0, 0, TimeSpan.Zero));
private ActivationCoordinatorTemplate(bool populated)
{
Populated = populated;
Root = Path.Combine(Path.GetTempPath(), $"nostos-681-template-{Guid.NewGuid():N}");
ArchivePath = Path.Combine(Root, "source.nostos");
TemplateDatabase = Path.Combine(Root, "template.db");
TemplateMedia = Path.Combine(Root, "template-media");
StagingRoot = Path.Combine(Root, "staging");
Staging = new LocalPortableImportStaging(StagingRoot);
Directory.CreateDirectory(Path.Combine(Root, "db-volume"));
Directory.CreateDirectory(TemplateMedia);
}
private static ActivationCoordinatorTemplate Create(bool populated)
{
var template = new ActivationCoordinatorTemplate(populated);
template.Initialize();
return template;
}
private void Initialize()
{
var liveDatabase = Path.Combine(Root, "db-volume", "live.db");
ActivationBuildFixture.BootstrapAsync(liveDatabase).GetAwaiter().GetResult();
if (Populated)
{
CreateLiveMedia();
}
var source = LocalPortableTestLibrary.CreateAsync().GetAwaiter().GetResult();
try
{
SourceIds = PortableArchiveTestSupport.PopulateRepresentativeAsync(source.Db, source.Storage)
.GetAwaiter().GetResult();
var archive = SelfHostedActivationTestSupport.ExportArchiveAsync(source).GetAwaiter().GetResult();
File.WriteAllBytes(ArchivePath, archive);
var archiveSource = new FilePortableArchiveSource(ArchivePath);
try
{
Prepared = new PortableArchiveReader()
.PrepareImportAsync(archiveSource, Staging).GetAwaiter().GetResult();
}
finally
{
archiveSource.DisposeAsync().AsTask().GetAwaiter().GetResult();
}
}
finally
{
source.DisposeAsync().AsTask().GetAwaiter().GetResult();
}
ExpectedHandle = new PortableLibraryVerifier()
.VerifyPreparedImportAsync(Staging, Prepared, default).GetAwaiter().GetResult();
ExpectedHandle.Passed.Should().BeTrue();
ExpectedImportedMedia = ReadStagedMediaSnapshotAsync().GetAwaiter().GetResult();
SeedHostState(liveDatabase);
OriginalAllTables = ActivationBuildFixture.DumpAllTables(liveDatabase);
OriginalPortable = DumpPortable(liveDatabase);
OriginalMedia = ActivationBuildFixture.MediaSnapshot(TemplateMedia);
// Publish a self-contained template database; the live file may carry a
// WAL while the source context is open.
using (var connection = SelfHostedSqliteFile.Open(liveDatabase, readOnly: false))
{
SelfHostedSqliteFile.CheckpointTruncate(connection);
}
SelfHostedSqliteFile.ClearPools();
File.Copy(liveDatabase, TemplateDatabase, overwrite: true);
OriginalDatabaseSha256 = Convert.ToHexString(
SHA256.HashData(File.ReadAllBytes(TemplateDatabase)));
File.Delete(liveDatabase);
File.Delete(liveDatabase + "-wal");
File.Delete(liveDatabase + "-shm");
}
private void CreateLiveMedia()
{
var bookId = Guid.NewGuid();
var folder = Path.Combine(TemplateMedia, bookId.ToString());
Directory.CreateDirectory(folder);
File.WriteAllText(Path.Combine(folder, "book.epub"), "LIVE-ONLY-BOOK");
File.WriteAllText(Path.Combine(folder, "cover.jpg"), "LIVE-ONLY-COVER");
File.WriteAllText(Path.Combine(folder, "cover-thumb-96.webp"), "LIVE-ONLY-THUMB");
}
private void SeedHostState(string liveDatabase)
{
using var db = new NostosDbContext(new DbContextOptionsBuilder()
.UseSqlite($"Data Source={liveDatabase};Pooling=False")
.Options);
var state = db.LibraryStates.Single();
db.BackupRecords.Add(new BackupRecord
{
Id = Guid.NewGuid(),
CreatedAt = new DateTime(2026, 9, 29, 8, 0, 0, DateTimeKind.Utc),
SizeBytes = 4096,
Status = BackupStatus.Completed,
Provider = BackupProvider.Local,
LocalArchivePath = "Storage/backups/live.nostos",
ManifestJson = "{\"version\":1}",
IncludeBookFiles = true,
});
if (Populated)
{
db.Collections.Add(new CollectionModel
{
Id = Guid.NewGuid(),
Name = "LIVE-ONLY-COLLECTION",
});
}
db.SaveChanges();
// The host state above may contain portable rows, so it advanced the
// revision through the save pipeline. Stamp the fixture's authoritative
// revision in its own save: a save that only changes LibraryState is not
// a portable mutation, so the value persists (issue #679 Slice 11).
state.StateVersion = OriginalRevision;
state.UpdatedAt = new DateTime(2026, 9, 30, 8, 0, 0, DateTimeKind.Utc);
db.SaveChanges();
// The job's stored destination revision is the same opaque provider token
// preflight hands to the client; activation reads the current token only
// through the provider.
RevisionToken = new LibraryStateDestinationRevisionProvider(db)
.GetCurrentAsync(default).GetAwaiter().GetResult();
var options = new TransferStorageOptions { DiskSafetyMarginBytes = 0, DiskSafetyMarginPercent = 0 };
var volume = new FakeTransferVolume
{
AvailableFreeSpaceBytes = 20L * 1024 * 1024 * 1024,
TotalSizeBytes = 20L * 1024 * 1024 * 1024,
};
var capacity = new TransferStorageCapacity(db, volume, Options.Create(options), Clock);
var reserved = capacity.TryReserveAsync(
Prepared.Metadata.ArchiveBytes + Prepared.Metadata.MediaBytes + 1_000_000,
MigrationSessionPurpose.Import,
TimeSpan.FromHours(24),
default).GetAwaiter().GetResult();
reserved.IsAdmitted.Should().BeTrue();
ReservationId = reserved.ReservationId!.Value;
capacity.ClaimAsync(ReservationId, JobId, default).GetAwaiter().GetResult();
db.MigrationJobRecords.Add(new MigrationJobRecord
{
Id = JobId,
Direction = (int)MigrationDirection.Import,
State = (int)MigrationJobState.ReadyToActivate,
RecoveryStatus = (int)MigrationRecoveryStatus.Pending,
ProgressPhase = (int)MigrationProgressPhase.PreparingActivation,
CreatedAtUtc = new DateTime(2026, 10, 3, 8, 0, 0, DateTimeKind.Utc),
UpdatedAtUtc = new DateTime(2026, 10, 3, 8, 30, 0, DateTimeKind.Utc),
IdempotencyKey = "activation-coordinator-job",
CreationPayloadHash = new string('a', 64),
ExpiresAtUtc = new DateTime(2026, 10, 10, 8, 0, 0, DateTimeKind.Utc),
AttemptNumber = 1,
DestinationRevision = RevisionToken,
PreparedStagingId = Prepared.Metadata.StagingId.Value,
PreparedImportMetadataJson = "{}",
ReservationId = ReservationId,
ReservedStorageBytes = Prepared.Metadata.ArchiveBytes + Prepared.Metadata.MediaBytes + 1_000_000,
Version = 3,
});
db.SaveChanges();
SeedUnrelatedOperationalRows(db);
db.SaveChanges();
}
///
/// Seeds unrelated migration jobs with their own session, chunk receipt,
/// artifact and reservation rows. The restored-original comparison must
/// keep every one of these byte-identical, so the row-scoped exclusion for
/// the activating job is genuinely proven.
///
private static void SeedUnrelatedOperationalRows(NostosDbContext db)
{
var pendingExport = new MigrationJobRecord
{
Id = Guid.NewGuid(),
Direction = (int)MigrationDirection.Export,
State = (int)MigrationJobState.Pending,
RecoveryStatus = (int)MigrationRecoveryStatus.NotRequired,
CreatedAtUtc = new DateTime(2026, 10, 2, 9, 0, 0, DateTimeKind.Utc),
UpdatedAtUtc = new DateTime(2026, 10, 2, 9, 30, 0, DateTimeKind.Utc),
IdempotencyKey = "unrelated-pending-export",
CreationPayloadHash = new string('b', 64),
ExpiresAtUtc = new DateTime(2026, 10, 12, 9, 0, 0, DateTimeKind.Utc),
AttemptNumber = 1,
Version = 2,
};
var completedExport = new MigrationJobRecord
{
Id = Guid.NewGuid(),
Direction = (int)MigrationDirection.Export,
State = (int)MigrationJobState.Completed,
RecoveryStatus = (int)MigrationRecoveryStatus.NotRequired,
CreatedAtUtc = new DateTime(2026, 10, 1, 7, 0, 0, DateTimeKind.Utc),
UpdatedAtUtc = new DateTime(2026, 10, 1, 7, 45, 0, DateTimeKind.Utc),
CompletedAtUtc = new DateTime(2026, 10, 1, 7, 45, 0, DateTimeKind.Utc),
IdempotencyKey = "unrelated-completed-export",
CreationPayloadHash = new string('c', 64),
ExpiresAtUtc = new DateTime(2026, 10, 11, 7, 0, 0, DateTimeKind.Utc),
AttemptNumber = 1,
Version = 5,
};
db.MigrationJobRecords.AddRange(pendingExport, completedExport);
var session = new MigrationSessionRecord
{
Id = Guid.NewGuid(),
JobId = pendingExport.Id,
Purpose = (int)MigrationSessionPurpose.Export,
State = (int)MigrationSessionState.Complete,
TotalBytes = 2048,
ChunkSize = MigrationContractLimits.DefaultChunkBytes,
TotalChunks = 1,
FileIdentitySizeBytes = 2048,
FileIdentitySha256 = new string('d', 64),
ClientFingerprint = "unrelated-client",
IdempotencyKey = "unrelated-session",
CreationPayloadHash = new string('e', 64),
ReceivedBytes = 2048,
CreatedAtUtc = new DateTime(2026, 10, 2, 9, 1, 0, DateTimeKind.Utc),
UpdatedAtUtc = new DateTime(2026, 10, 2, 9, 2, 0, DateTimeKind.Utc),
ExpiresAtUtc = new DateTime(2026, 10, 12, 9, 0, 0, DateTimeKind.Utc),
CompletedAtUtc = new DateTime(2026, 10, 2, 9, 2, 0, DateTimeKind.Utc),
StorageKey = "migration/sessions/unrelated-session",
Version = 1,
};
db.MigrationSessionRecords.Add(session);
db.MigrationChunkReceiptRecords.Add(new MigrationChunkReceiptRecord
{
SessionId = session.Id,
ChunkIndex = 0,
OffsetBytes = 0,
LengthBytes = 2048,
Sha256 = new string('f', 64),
ReceivedAtUtc = new DateTime(2026, 10, 2, 9, 2, 0, DateTimeKind.Utc),
});
db.MigrationStorageReservations.Add(new MigrationStorageReservationRecord
{
Id = Guid.NewGuid(),
Purpose = (int)MigrationSessionPurpose.Export,
ReservedBytes = 4096,
MaterializedBytes = 2048,
CreatedAtUtc = new DateTime(2026, 10, 2, 9, 0, 30, DateTimeKind.Utc),
ExpiresAtUtc = new DateTime(2026, 10, 12, 9, 0, 0, DateTimeKind.Utc),
ClaimedJobId = pendingExport.Id,
Version = 1,
});
db.MigrationExportArtifactRecords.Add(new MigrationExportArtifactRecord
{
JobId = completedExport.Id,
State = (int)MigrationExportArtifactState.Available,
StorageKey = "migration/exports/unrelated.nostos",
FileName = "unrelated.nostos",
ContentType = "application/zip",
SizeBytes = 8192,
Sha256 = new string('1', 64),
CreatedAtUtc = new DateTime(2026, 10, 1, 7, 1, 0, DateTimeKind.Utc),
AvailableAtUtc = new DateTime(2026, 10, 1, 7, 2, 0, DateTimeKind.Utc),
ExpiresAtUtc = new DateTime(2026, 10, 8, 7, 0, 0, DateTimeKind.Utc),
Version = 1,
});
}
private async Task> ReadStagedMediaSnapshotAsync()
{
var result = new Dictionary(StringComparer.Ordinal);
foreach (var item in Prepared.Media)
{
var descriptor = item.Descriptor;
var extension = Path.GetExtension(descriptor.FileName).ToLowerInvariant();
var relative = Path.Combine(descriptor.BookId.ToString(), descriptor.Kind + extension);
await using var stream = await Staging.OpenMediaReadAsync(
Prepared.Metadata.StagingId, item.Reference, default);
using var buffer = new MemoryStream();
await stream.CopyToAsync(buffer);
result[relative] = Convert.ToHexString(SHA256.HashData(buffer.ToArray()));
}
return result;
}
internal static Dictionary> DumpPortable(string path)
{
var all = ActivationBuildFixture.DumpAllTables(path);
return ActivationCoordinatorTestBed.PortableTableNames.ToDictionary(name => name, name => all[name]);
}
public void Dispose()
{
try
{
if (Directory.Exists(Root)) Directory.Delete(Root, recursive: true);
}
catch (IOException)
{
}
catch (UnauthorizedAccessException)
{
}
}
}
///
/// One isolated host copy: real live database bytes copied from the template, a
/// real live media root, real DI services, and a fresh maintenance generation
/// over the identical files.
///
internal sealed class ActivationCoordinatorTestBed : IAsyncDisposable
{
private readonly ActivationCoordinatorTemplate _template;
private readonly TimeSpan _drainTimeout;
private readonly List _providers = [];
internal string Root { get; }
internal Guid JobId => _template.JobId;
internal bool Populated => _template.Populated;
internal SelfHostedActivationPaths Paths { get; }
internal ManualTimeProvider Clock { get; }
internal TransferStorageOptions StorageOptions { get; } = new()
{
DiskSafetyMarginBytes = 0,
DiskSafetyMarginPercent = 0,
};
internal FakeTransferVolume TransferVolume { get; } = new()
{
AvailableFreeSpaceBytes = 20L * 1024 * 1024 * 1024,
TotalSizeBytes = 20L * 1024 * 1024 * 1024,
};
internal FakeVolumeSpaceProbe Probe { get; } = new();
internal IPreparedPortableImport Prepared => _prepared ?? _template.Prepared;
internal PortablePreparedImportVerification ExpectedHandle => _expected ?? _template.ExpectedHandle;
internal PortableFixtureIds SourceIds => _template.SourceIds;
internal Guid ReservationId => _template.ReservationId;
internal string RevisionToken => _template.RevisionToken;
internal Dictionary> OriginalAllTables => _template.OriginalAllTables;
internal Dictionary> OriginalPortable => _template.OriginalPortable;
internal Dictionary OriginalMedia => _template.OriginalMedia;
internal Dictionary ExpectedImportedMedia => _template.ExpectedImportedMedia;
internal string OriginalDatabaseSha256 => _template.OriginalDatabaseSha256;
internal string TransferRoot { get; }
internal string StagingRoot => _ownStagingRoot ?? _template.StagingRoot;
/// Test seam: overrides the current destination revision token.
internal Func>? RevisionOverride { get; set; }
/// Test seam: throws during the media component's rollback rename.
internal Action? MediaRecoveryRenameFailure { get; set; }
/// Test seam: throws during the database component's rollback rename.
internal Action? DatabaseRecoveryRenameFailure { get; set; }
///
/// Test seam: runs after the real working-copy migration. Used to model a
/// migration that succeeds but loses or alters portable state.
///
internal Func? RecoveryMigrationOverride { get; set; }
private IPreparedPortableImport? _prepared;
private PortablePreparedImportVerification? _expected;
private LocalPortableImportStaging? _ownStaging;
private string? _ownStagingRoot;
internal LibraryMaintenanceCoordinator Maintenance { get; private set; } = null!;
internal SelfHostedActivationJournalStore Journals { get; private set; } = null!;
internal SelfHostedRecoveryManifestStore Manifests { get; private set; } = null!;
internal ServiceProvider Host { get; private set; } = null!;
private ActivationCoordinatorTestBed(ActivationCoordinatorTemplate template, TimeSpan drainTimeout)
{
_template = template;
_drainTimeout = drainTimeout;
Clock = new ManualTimeProvider(template.BaseTime);
Root = Path.Combine(Path.GetTempPath(), $"nostos-681-activation-{Guid.NewGuid():N}");
TransferRoot = Path.Combine(Root, "transfer-volume", "transfers");
Directory.CreateDirectory(Path.Combine(Root, "db-volume"));
Directory.CreateDirectory(Path.Combine(Root, "media-volume"));
Directory.CreateDirectory(TransferRoot);
Paths = new SelfHostedActivationPaths(
Path.Combine(Root, "db-volume", "nostos.db"),
Path.Combine(Root, "media-volume", "library"));
}
internal static async Task CreateAsync(
bool populated,
TimeSpan? drainTimeout = null,
bool freshStaging = false)
{
var bed = new ActivationCoordinatorTestBed(
ActivationCoordinatorTemplate.For(populated),
drainTimeout ?? TimeSpan.FromSeconds(5));
await bed.InitializeAsync(freshStaging);
return bed;
}
private async Task InitializeAsync(bool freshStaging)
{
File.Copy(_template.TemplateDatabase, Paths.LiveDatabase, overwrite: true);
PortableArchiveTestSupport.CopyDirectory(_template.TemplateMedia, Paths.LiveMedia);
if (freshStaging)
{
// The corruption tests need a staging area they may damage without
// invalidating the process-wide template every other test shares.
_ownStagingRoot = Path.Combine(Root, "staging");
_ownStaging = new LocalPortableImportStaging(_ownStagingRoot);
var archiveSource = new FilePortableArchiveSource(_template.ArchivePath);
try
{
_prepared = await new PortableArchiveReader().PrepareImportAsync(archiveSource, _ownStaging);
}
finally
{
await archiveSource.DisposeAsync();
}
_expected = await new PortableLibraryVerifier()
.VerifyPreparedImportAsync(_ownStaging, _prepared, default);
_expected.Passed.Should().BeTrue();
}
Probe.Set(Paths.LiveDatabase, 10L * 1024 * 1024 * 1024, 10L * 1024 * 1024 * 1024, "db");
Probe.Set(Paths.LiveMedia, 10L * 1024 * 1024 * 1024, 10L * 1024 * 1024 * 1024, "media");
Probe.Set(TransferRoot, 10L * 1024 * 1024 * 1024, 10L * 1024 * 1024 * 1024, "transfer");
await RestartHostAsync();
}
internal NostosDbContext OpenDatabase() =>
new(new DbContextOptionsBuilder()
.UseSqlite($"Data Source={Paths.LiveDatabase};Pooling=False")
.Options);
/// Rebuilds the host services as a fresh process over the same files.
internal async Task RestartHostAsync()
{
if (Host is not null)
{
await StopDispatcherAsync(Host);
await Host.DisposeAsync();
_providers.Remove(Host);
}
BuildMaintenance();
BuildHost();
await StartDispatcherAsync(Host);
}
private static async Task StartDispatcherAsync(ServiceProvider provider)
{
await provider.GetRequiredService().StartAsync(default);
}
private static async Task StopDispatcherAsync(ServiceProvider provider)
{
try
{
await provider.GetRequiredService().StopAsync(default);
}
catch (Exception)
{
// Test cleanup only.
}
}
private void BuildMaintenance()
{
// Production always constructs the coordinator with the durable marker,
// so a sticky fail-closed state can survive a runtime rollback failure.
var marker = new LibraryMaintenanceMarker(Paths.LiveDatabase);
Maintenance = new LibraryMaintenanceCoordinator(
new LibraryMaintenanceOptions { DrainTimeout = _drainTimeout }, clock: Clock, marker: marker);
Maintenance.InitializeAfterRecovery();
Journals = new SelfHostedActivationJournalStore(Paths, Maintenance);
Manifests = new SelfHostedRecoveryManifestStore(Paths);
}
private void BuildHost()
{
var services = new ServiceCollection();
services.AddLogging();
services.AddSingleton(Clock);
services.AddSingleton(Clock);
services.AddSingleton(Maintenance);
services.AddSingleton(Maintenance);
services.AddSingleton();
services.AddDbContextFactory(options =>
options.UseSqlite($"Data Source={Paths.LiveDatabase};Pooling=False"));
services.AddSingleton(Options.Create(StorageOptions));
services.AddSingleton(TransferVolume);
services.AddScoped();
services.AddSingleton(new TransferPathResolver(TransferRoot));
services.AddSingleton(_ownStaging is null
? _template.Staging
: _ownStaging);
services.AddSingleton(Paths);
services.AddSingleton(Journals);
services.AddSingleton(Manifests);
services.AddSingleton(
new SelfHostedActivationComponentStep(Paths, database: false,
afterRenameForTesting: () => MediaRecoveryRenameFailure?.Invoke()));
services.AddSingleton(
new SelfHostedActivationComponentStep(Paths, database: true,
afterRenameForTesting: () => DatabaseRecoveryRenameFailure?.Invoke()));
services.AddSingleton(
new SelfHostedActivationEmptyRetentionStep(Paths, Manifests));
services.AddSingleton();
services.AddSingleton(Probe);
services.AddScoped();
services.AddScoped(sp =>
new SelfHostedActivationCandidateMediaBuilder(
Paths, sp.GetRequiredService()));
services.AddScoped(sp =>
new SelfHostedActivationDatabaseBuilder(
Paths, sp.GetRequiredService(), Maintenance, Clock));
services.AddSingleton();
services.AddSingleton();
services.AddSingleton(new FileStorageService(
new PortableTestWebHostEnvironment(Root),
Options.Create(new FileStorageOptions { BooksRoot = Paths.LiveMedia }),
NullLogger.Instance));
services.AddScoped(sp => new SelfHostedMigrationRecoveryService(
Paths,
Manifests,
Journals,
sp.GetRequiredService(),
sp.GetRequiredService(),
sp.GetRequiredService(),
Maintenance,
sp.GetRequiredService>(),
Clock));
services.AddScoped(sp =>
sp.GetRequiredService());
services.AddSingleton();
services.AddScoped();
services.AddScoped();
services.AddScoped(sp =>
new SwitchableRevisionProvider(
new LibraryStateDestinationRevisionProvider(sp.GetRequiredService()),
this));
services.AddScoped();
services.AddScoped(sp =>
sp.GetRequiredService());
services.AddScoped(sp =>
sp.GetRequiredService());
services.AddSingleton();
services.AddSingleton(sp =>
sp.GetRequiredService());
services.AddSingleton(new SwitchableRecoveryMigrator(this));
services.AddScoped();
services.AddScoped();
services.AddScoped(sp =>
sp.GetRequiredService());
Host = services.BuildServiceProvider(new ServiceProviderOptions
{
ValidateScopes = false,
ValidateOnBuild = false,
});
_providers.Add(Host);
}
internal async Task ActivateAsync(
bool confirm = true,
Action? observer = null,
CancellationToken ct = default)
{
await using var scope = Host.CreateAsyncScope();
var coordinator = scope.ServiceProvider.GetRequiredService();
coordinator.StepObserverForTesting = observer;
return await coordinator.ActivateAsync(
JobId,
new MigrationActivateRequest(RevisionToken, confirm),
ct);
}
/// The production interface entry point, with no explicit request object.
internal async Task ActivatePublicAsync(CancellationToken ct = default)
{
await using var scope = Host.CreateAsyncScope();
await scope.ServiceProvider.GetRequiredService().ActivateAsync(JobId, ct);
}
/// Runs the production startup reconciler with a fresh host generation.
internal Task ReconcileAsync() => ReconcileWithStepsAsync(mediaStep: true, databaseStep: true, emptyDiscardStep: true);
///
/// Runs the reconciler with a chosen subset of steps. Used by the
/// crash-matrix sensitivity checks that prove each component step is what
/// prevents a mixed generation from being admitted.
///
internal async Task ReconcileWithStepsAsync(bool mediaStep, bool databaseStep, bool emptyDiscardStep)
{
var marker = new LibraryMaintenanceMarker(Paths.LiveDatabase);
var gate = new LibraryMaintenanceCoordinator(marker: marker);
var journals = new SelfHostedActivationJournalStore(Paths, gate);
var manifests = new SelfHostedRecoveryManifestStore(Paths);
var steps = new List();
if (mediaStep) steps.Add(new SelfHostedActivationComponentStep(Paths, database: false));
if (databaseStep) steps.Add(new SelfHostedActivationComponentStep(Paths, database: true));
if (emptyDiscardStep) steps.Add(new SelfHostedActivationEmptyRetentionStep(Paths, manifests));
var engine = new SelfHostedActivationRecoveryStartupService(Paths, journals, gate, steps);
await engine.ReconcileIncompleteAsync();
gate.IsMaintenanceActive.Should().BeFalse();
File.Exists(marker.MarkerPath).Should().BeFalse();
}
/// Disposes the crashed host, reconciles, and builds a fresh host.
internal async Task RecoverHostAsync()
{
if (Host is not null)
{
await StopDispatcherAsync(Host);
await Host.DisposeAsync();
_providers.Remove(Host);
}
await ReconcileAsync();
BuildMaintenance();
BuildHost();
await StartDispatcherAsync(Host);
}
internal static IReadOnlyList PortableTableNames { get; } = BuildPortableTableNames();
private static string[] BuildPortableTableNames()
{
using var db = new NostosDbContext(new DbContextOptionsBuilder()
.UseSqlite("Data Source=:memory:").Options);
var types = new[]
{
typeof(WorkModel), typeof(BookModel), typeof(CollectionModel), typeof(BookCollectionModel),
typeof(NoteModel), typeof(TopicModel), typeof(NoteTopicModel), typeof(WritingModel),
typeof(WritingNoteModel), typeof(BookAcquisitionModel), typeof(NoteImportBookLink),
typeof(AssistantSettingsModel), typeof(ReadingListEntryModel),
};
return types.Select(type => db.Model.FindEntityType(type)!.GetTableName()!).ToArray();
}
///
/// Compares the restored original generation with the pre-activation logical
/// dump: every table, every row and every column, plus the media SHA-256 and
/// path set. Exactly ONE row is excluded by key: the tested activation job's
/// own MigrationJobRecords row, whose State/lease/UpdatedAt/Version
/// the protocol legitimately changes (callers assert its expected state
/// explicitly). Every other row must be identical, including the unrelated
/// migration jobs, sessions, chunk receipts, artifacts and reservations the
/// fixture seeds. There is no whole-table exclusion. No SQLite-file hash is
/// asserted: the job's own lease/state writes mean the main database file is
/// not untouched.
///
internal void AssertOriginalGeneration()
{
var actual = ActivationBuildFixture.DumpAllTables(Paths.LiveDatabase);
var expected = OriginalAllTables;
actual.Keys.Should().BeEquivalentTo(expected.Keys, "no table may appear or disappear");
var expectedJobs = expected["MigrationJobRecords"];
var actualJobs = actual["MigrationJobRecords"];
expectedJobs.Count(IsActivationJobRow).Should().Be(1, "the fixture seeds exactly one activation job");
actualJobs.Count(IsActivationJobRow).Should().Be(1, "the activation job row must still exist");
(expectedJobs.Count - 1).Should().BeGreaterThan(
0, "the fixture must seed at least one unrelated migration job for the row-scoped exclusion to be meaningful");
foreach (var table in expected.Keys)
{
var expectedRows = table == "MigrationJobRecords"
? expectedJobs.Where(row => !IsActivationJobRow(row)).ToList()
: expected[table];
var actualRows = table == "MigrationJobRecords"
? actualJobs.Where(row => !IsActivationJobRow(row)).ToList()
: actual[table];
actualRows.Should().BeEquivalentTo(expectedRows,
$"table {table} must be logically identical to the original except the activation job's own row");
}
ActivationBuildFixture.MediaSnapshot(Paths.LiveMedia)
.Should().BeEquivalentTo(OriginalMedia, "the media root must be identical to the original");
}
private bool IsActivationJobRow(string row) =>
string.Equals(row.Split('|', 2)[0], JobId.ToString(), StringComparison.OrdinalIgnoreCase);
internal async Task AssertImportedGenerationAsync()
{
await using var scope = Host.CreateAsyncScope();
var db = scope.ServiceProvider.GetRequiredService();
var report = await new PortableLibraryVerifier()
.VerifyCandidateAsync(db, Paths.LiveMedia, Prepared, ExpectedHandle, default);
report.Passed.Should().BeTrue(
"the activated generation must be exactly the verified import: {0}",
string.Join(", ", report.Failures.Select(failure => failure.Code)));
ActivationBuildFixture.MediaSnapshot(Paths.LiveMedia)
.Should().BeEquivalentTo(ExpectedImportedMedia, "only the imported primary media may remain active");
}
internal async Task ReadJobAsync()
{
await using var scope = Host.CreateAsyncScope();
return await scope.ServiceProvider.GetRequiredService()
.MigrationJobRecords.AsNoTracking().SingleAsync(job => job.Id == JobId);
}
internal async Task CurrentRevisionAsync()
{
await using var scope = Host.CreateAsyncScope();
var value = await scope.ServiceProvider.GetRequiredService()
.LibraryStates.AsNoTracking().Select(state => state.StateVersion).SingleAsync();
return long.Parse(value);
}
internal async Task WritePortableRevisionBumpAsync()
{
await using var db = OpenDatabase();
db.Collections.Add(new CollectionModel { Id = Guid.NewGuid(), Name = "WRITE-DURING-ACTIVATION" });
var state = await db.LibraryStates.SingleAsync();
state.StateVersion = (long.Parse(state.StateVersion) + 1).ToString();
await db.SaveChangesAsync();
}
internal async Task WriteHostRecordAsync()
{
await using var db = OpenDatabase();
db.BackupRecords.Add(new BackupRecord
{
Id = Guid.NewGuid(),
CreatedAt = DateTime.UtcNow,
SizeBytes = 128,
Status = BackupStatus.Completed,
Provider = BackupProvider.Local,
LocalArchivePath = "Storage/backups/during-activation.nostos",
ManifestJson = "{}",
IncludeBookFiles = true,
});
await db.SaveChangesAsync();
}
/// Tampering seam: flips one byte of the first file under a candidate root.
internal static void CorruptFirstFile(string root)
{
var file = Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories).First();
var bytes = File.ReadAllBytes(file);
bytes[0] ^= 0xFF;
File.WriteAllBytes(file, bytes);
}
/// Tampering seam: corrupts the SQLite database bytes so integrity checks fail.
internal static void CorruptDatabaseByte(string path)
{
var bytes = File.ReadAllBytes(path);
bytes[0] ^= 0xFF; // the "SQLite format 3" magic: any open or integrity check fails
File.WriteAllBytes(path, bytes);
}
/// SHA-256 of every file under the bed root, for freeze assertions.
internal Dictionary SnapshotActivationTree() =>
Directory.EnumerateFiles(Root, "*", SearchOption.AllDirectories)
.ToDictionary(
path => Path.GetRelativePath(Root, path),
path => Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(path))),
StringComparer.Ordinal);
///
/// Adds structurally valid but database-unindexed media (a GUID folder with a
/// book file) to the live root, modelling orphan user files.
///
internal void WriteOrphanMedia()
{
var folder = Path.Combine(Paths.LiveMedia, Guid.NewGuid().ToString());
Directory.CreateDirectory(folder);
File.WriteAllText(Path.Combine(folder, "book.epub"), "ORPHAN-USER-MEDIA");
}
// ---- Slice 9: recovery restore ("restore previous library") ----
/// The recovery copy retained by the activation under test.
internal Guid RecoveryId => JobId;
internal async Task CurrentRevisionTokenAsync()
{
await using var scope = Host.CreateAsyncScope();
return await scope.ServiceProvider.GetRequiredService()
.GetCurrentAsync(default);
}
/// Claims and runs a restore through the shared library-switch dispatcher.
internal async Task RestoreAsync(
bool confirm = true,
Action? observer = null,
string? revision = null,
Guid? recoveryId = null)
{
var id = recoveryId ?? RecoveryId;
var dispatcher = Host.GetRequiredService();
dispatcher.RestoreStepObserverForTesting = observer;
var token = revision ?? await CurrentRevisionTokenAsync();
await dispatcher.RequestRestoreAsync(
id, new MigrationRecoveryRestoreRequest(token, confirm), default);
await dispatcher.AwaitRestoreFinishedAsync(id, default);
return MapRestoreResult(await dispatcher.GetRestoreStatusAsync(id, default));
}
///
/// Runs the restore coordinator directly with a crash-matrix observer. The
/// sentinel propagates exactly like process death; production dispatch runs
/// through .
///
internal async Task CrashRestoreAsync(
Action observer,
string? revision = null,
Guid? recoveryId = null)
{
var id = recoveryId ?? RecoveryId;
var token = revision ?? await CurrentRevisionTokenAsync();
await using var scope = Host.CreateAsyncScope();
var coordinator = scope.ServiceProvider.GetRequiredService();
coordinator.StepObserverForTesting = observer;
await coordinator.RequestRestoreAsync(
id, new MigrationRecoveryRestoreRequest(token, true), default);
return await coordinator.RestorePreviousLibraryAsync(id, default);
}
/// Resumes a durably claimed restore on the current host generation.
internal async Task ResumeRestoreAsync(
Action? observer = null,
Guid? recoveryId = null)
{
var id = recoveryId ?? RecoveryId;
var dispatcher = Host.GetRequiredService();
dispatcher.RestoreStepObserverForTesting = observer;
await dispatcher.ScanPendingRestoresAsync(default);
await dispatcher.AwaitRestoreFinishedAsync(id, default);
return MapRestoreResult(await dispatcher.GetRestoreStatusAsync(id, default));
}
private static SelfHostedRecoveryRestoreResult MapRestoreResult(
MigrationRecoveryRestoreStatusResponse status) =>
new(
status.RecoveryId,
status.Outcome switch
{
MigrationActivationOutcome.Completed => SelfHostedRecoveryRestoreOutcome.Restored,
MigrationActivationOutcome.Running or MigrationActivationOutcome.Accepted =>
SelfHostedRecoveryRestoreOutcome.Busy,
_ => SelfHostedRecoveryRestoreOutcome.Interrupted,
},
status.Status,
status.ErrorCode);
/// Claims only (no run), for concurrency and status tests.
internal async Task RequestRestoreOnlyAsync(
bool confirm = true,
string? revision = null,
Guid? recoveryId = null)
{
var id = recoveryId ?? RecoveryId;
var token = revision ?? await CurrentRevisionTokenAsync();
await using var scope = Host.CreateAsyncScope();
var coordinator = scope.ServiceProvider.GetRequiredService();
return await coordinator.RequestRestoreAsync(
id, new MigrationRecoveryRestoreRequest(token, confirm), default);
}
internal SelfHostedRecoveryManifest? ReadRecoveryManifest(Guid? recoveryId = null) =>
Manifests.Read(recoveryId ?? RecoveryId);
///
/// Seeds a second ReadyToActivate import job over the same prepared staging,
/// for dispatcher mutual-exclusion tests (activation versus restore).
///
internal async Task SeedReadyToActivateJobAsync(string destinationRevision)
{
var id = Guid.NewGuid();
await using var db = OpenDatabase();
db.MigrationJobRecords.Add(new MigrationJobRecord
{
Id = id,
Direction = (int)MigrationDirection.Import,
State = (int)MigrationJobState.ReadyToActivate,
RecoveryStatus = (int)MigrationRecoveryStatus.Pending,
ProgressPhase = (int)MigrationProgressPhase.PreparingActivation,
CreatedAtUtc = DateTime.UtcNow.AddMinutes(-10),
UpdatedAtUtc = DateTime.UtcNow.AddMinutes(-5),
IdempotencyKey = $"queued-activation-{id:N}",
CreationPayloadHash = new string('a', 64),
ExpiresAtUtc = DateTime.UtcNow.AddDays(1),
AttemptNumber = 1,
DestinationRevision = destinationRevision,
PreparedStagingId = Prepared.Metadata.StagingId.Value,
PreparedImportMetadataJson = "{}",
Version = 1,
});
await db.SaveChangesAsync();
return id;
}
internal (Dictionary> Tables, Dictionary Media)
SnapshotLiveGeneration() =>
(ActivationBuildFixture.DumpAllTables(Paths.LiveDatabase),
ActivationBuildFixture.MediaSnapshot(Paths.LiveMedia));
/// Every table, every row and every media byte must be identical.
internal void AssertGenerationEquals(
(Dictionary> Tables, Dictionary Media) expected,
string because)
{
var actual = SnapshotLiveGeneration();
actual.Tables.Keys.Should().BeEquivalentTo(expected.Tables.Keys, because);
foreach (var table in expected.Tables.Keys)
{
actual.Tables[table].Should().BeEquivalentTo(
expected.Tables[table], $"{because} (table {table})");
}
actual.Media.Should().BeEquivalentTo(expected.Media, because);
}
/// Portable tables of the live database must equal a captured generation.
internal void AssertPortableEquals(Dictionary> allTables, string because)
{
var actual = ActivationCoordinatorTemplate.DumpPortable(Paths.LiveDatabase);
foreach (var table in PortableTableNames)
{
actual[table].Should().BeEquivalentTo(allTables[table], $"{because} (table {table})");
}
}
/// The restored portable state and media must be the retained original library.
internal Task AssertRestoredPortableGenerationAsync()
{
ActivationCoordinatorTemplate.DumpPortable(Paths.LiveDatabase)
.Should().BeEquivalentTo(OriginalPortable,
"the restored portable state must be the retained previous library");
ActivationBuildFixture.MediaSnapshot(Paths.LiveMedia)
.Should().BeEquivalentTo(OriginalMedia,
"the restored media must be the retained previous generation");
return Task.CompletedTask;
}
internal void TamperRecoveryMediaByte()
{
var root = Paths.PreviousMedia(RecoveryId);
var file = Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories).First();
var bytes = File.ReadAllBytes(file);
bytes[0] ^= 0xFF;
File.WriteAllBytes(file, bytes);
}
internal void TamperRecoveryDatabaseByte()
{
var path = Paths.PreviousDatabase(RecoveryId);
var bytes = File.ReadAllBytes(path);
bytes[0] ^= 0xFF;
File.WriteAllBytes(path, bytes);
}
internal void CorruptRecoveryManifest() =>
File.WriteAllText(Paths.RecoveryManifest(RecoveryId), "{not-json");
internal void ExpireRecoveryCopy()
{
var manifest = Manifests.Read(RecoveryId)!;
var delta = TimeSpan.FromDays(MigrationContractLimits.RecoveryRetentionDays + 1);
Manifests.Write(manifest with
{
CreatedAtUtc = manifest.CreatedAtUtc - delta,
ExpiresAtUtc = manifest.ExpiresAtUtc - delta,
});
}
///
/// Rolls the retained database back exactly one migration (a real older
/// schema and history) and refreshes the manifest facts, modelling a copy
/// retained by an earlier release.
///
internal async Task DowngradeRecoveryDatabaseOneMigrationAsync()
{
var path = Paths.PreviousDatabase(RecoveryId);
string target;
await using (var db = OpenRecoveryDatabase(path))
{
var applied = (await db.Database.GetAppliedMigrationsAsync()).ToArray();
applied.Length.Should().BeGreaterThan(1);
target = applied[^2];
}
await DowngradeRecoveryDatabaseToAsync(target);
}
/// Rolls the retained database back to an exact migration level.
internal async Task DowngradeRecoveryDatabaseToAsync(string migrationId)
{
var path = Paths.PreviousDatabase(RecoveryId);
await using (var db = OpenRecoveryDatabase(path))
{
(await db.Database.GetAppliedMigrationsAsync()).Should().Contain(migrationId);
await db.GetService().MigrateAsync(migrationId);
}
RefreshRecoveryManifestFacts(path);
}
internal string SourceRecoveryDatabaseSha256()
{
var path = Paths.PreviousDatabase(RecoveryId);
return Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(path))).ToLowerInvariant();
}
/// Creates a second recovery directory whose manifest is unreadable.
internal Guid CreateCorruptRecoveryCopy()
{
var id = Guid.NewGuid();
Paths.PrepareRecovery(id);
File.WriteAllText(Paths.RecoveryManifest(id), "{not-json");
return id;
}
///
/// Seeds one work/book/membership into the retained recovery copy (the
/// template's live library only has a collection) and refreshes the
/// manifest, so a lossy migration can drop a real relationship row.
///
internal async Task SeedRecoveryMembershipAsync()
{
var path = Paths.PreviousDatabase(RecoveryId);
await using (var db = OpenRecoveryDatabase(path))
{
var collection = await db.Collections.FirstAsync();
var work = new WorkModel
{
Id = Guid.NewGuid(),
Title = "RECOVERY-WORK",
NormalizedTitle = "RECOVERY-WORK",
NormalizedAuthor = string.Empty,
};
var book = new PhysicalBookModel
{
Id = Guid.NewGuid(),
Work = work,
Title = "RECOVERY-BOOK",
};
db.Works.Add(work);
db.Books.Add(book);
db.BookCollections.Add(new BookCollectionModel
{
BookId = book.Id,
CollectionId = collection.Id,
AddedAt = DateTime.UtcNow,
});
await db.SaveChangesAsync();
}
RefreshRecoveryManifestFacts(path);
}
private static NostosDbContext OpenRecoveryDatabase(string path) =>
new(new DbContextOptionsBuilder()
.UseSqlite($"Data Source={path};Pooling=False", sqlite =>
sqlite.MigrationsAssembly(typeof(Program).Assembly.FullName))
.Options);
internal void AddUnknownMigrationToRecovery()
{
var path = Paths.PreviousDatabase(RecoveryId);
using (var connection = new SqliteConnection($"Data Source={path};Pooling=False"))
{
connection.Open();
using var command = connection.CreateCommand();
command.CommandText =
"INSERT INTO __EFMigrationsHistory (\"MigrationId\", \"ProductVersion\") "
+ "VALUES ('99999999999999_FutureMigration', '99.0.0');";
command.ExecuteNonQuery();
}
RefreshRecoveryManifestFacts(path);
}
private void RefreshRecoveryManifestFacts(string databasePath)
{
foreach (var suffix in new[] { "-wal", "-shm" })
{
if (File.Exists(databasePath + suffix)) File.Delete(databasePath + suffix);
}
var manifest = Manifests.Read(RecoveryId)!;
var info = new FileInfo(databasePath);
var sha = Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(databasePath))).ToLowerInvariant();
Manifests.Write(manifest with
{
DatabaseBytes = info.Length,
DatabaseSha256 = sha,
Counts = ReadRecoveryCounts(databasePath),
});
}
/// Counts the retained copy's portable rows exactly as the activation capture did.
private static MigrationExistingCounts ReadRecoveryCounts(string databasePath)
{
using var connection = new SqliteConnection($"Data Source={databasePath};Pooling=False");
connection.Open();
var tables = new HashSet(StringComparer.Ordinal);
using (var command = connection.CreateCommand())
{
command.CommandText = "SELECT name FROM sqlite_master WHERE type = 'table';";
using var reader = command.ExecuteReader();
while (reader.Read()) tables.Add(reader.GetString(0));
}
long Count(string table, string? where = null)
{
if (!tables.Contains(table)) return 0;
using var command = connection.CreateCommand();
command.CommandText = $"SELECT COUNT(*) FROM \"{table}\"" + (where is null ? ";" : $" WHERE {where};");
return Convert.ToInt64(command.ExecuteScalar());
}
return new MigrationExistingCounts(
Works: Count("Works"),
Books: Count("Books"),
Notes: Count("Notes"),
Topics: Count("Topics"),
NoteTopics: Count("NoteTopics"),
Writings: Count("Writings"),
WritingNotes: Count("WritingNotes"),
Collections: Count("Collections"),
BookCollections: Count("BookCollections"),
Acquisitions: Count("BookAcquisitions"),
NoteImportBookLinks: Count("NoteImportBookLinks"),
AssistantSettings: Count("AssistantSettings", "\"CaptureProcessingMode\" IS NOT NULL"),
ReadingListEntries: Count("ReadingListEntries"));
}
public async ValueTask DisposeAsync()
{
foreach (var provider in _providers.ToArray())
{
try
{
await StopDispatcherAsync(provider);
await provider.DisposeAsync();
}
catch
{
// Test cleanup only.
}
}
try
{
if (Directory.Exists(Root)) Directory.Delete(Root, recursive: true);
}
catch (IOException)
{
}
catch (UnauthorizedAccessException)
{
}
}
/// Delegates to the real provider unless a test overrides the token.
private sealed class SwitchableRevisionProvider(
ILibraryDestinationRevisionProvider inner,
ActivationCoordinatorTestBed bed) : ILibraryDestinationRevisionProvider
{
public Task GetCurrentAsync(CancellationToken ct) =>
bed.RevisionOverride is { } overridden
? overridden(ct)
: inner.GetCurrentAsync(ct);
}
///
/// Runs the real working-copy migration and then an optional test hook that
/// models a successful but lossy migration.
///
private sealed class SwitchableRecoveryMigrator(ActivationCoordinatorTestBed bed)
: ISelfHostedRecoverySchemaMigrator
{
public async Task MigrateAsync(NostosDbContext database, CancellationToken ct)
{
await new DatabaseBootstrapService(database).EnsureReadyAsync(ct);
if (bed.RecoveryMigrationOverride is { } overridden)
{
await overridden(database);
}
}
}
}