using FluentAssertions;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Nostos.Backend.Data;
using Nostos.Backend.Data.Models;
using Nostos.Backend.Services;
using Nostos.Backend.Services.BookText;
using Nostos.Backend.Services.Portability;
using Nostos.Backend.Services.Portability.Activation;
using Nostos.Backend.Services.Portability.Migration;
using Nostos.Backend.Services.Portability.Transfers;
using Nostos.Product.BookText;
namespace Nostos.Backend.Tests.Portability;
///
/// Real temp directories, real SQLite files and a deterministic clock for the
/// Slice 10 maintenance workers: recovery expiry cleanup, activation orphan
/// sweep and the post-activation derived rebuild.
///
internal sealed class ActivationMaintenanceTestBed : IDisposable
{
private static readonly Guid FirstBook = Guid.Parse("11111111-1111-1111-1111-111111111111");
internal string Root { get; } = Path.Combine(Path.GetTempPath(), $"nostos-681-maintenance-{Guid.NewGuid():N}");
internal ManualTimeProvider Clock { get; } = new(new DateTimeOffset(2026, 10, 4, 12, 0, 0, TimeSpan.Zero));
internal SelfHostedActivationPaths Paths { get; }
internal LibraryMaintenanceCoordinator Gate { get; } =
new(new LibraryMaintenanceOptions { DrainTimeout = TimeSpan.FromSeconds(5) });
internal SelfHostedActivationJournalStore Journals { get; }
internal SelfHostedRecoveryManifestStore Manifests { get; }
internal TransferStorageOptions StorageOptions { get; } = new()
{
DiskSafetyMarginBytes = 0,
DiskSafetyMarginPercent = 0,
};
internal SelfHostedActivationMaintenanceOptions MaintenanceOptions { get; } = new();
internal FakeTransferVolume TransferVolume { get; } = new()
{
AvailableFreeSpaceBytes = 1_000_000_000,
TotalSizeBytes = 1_000_000_000,
};
internal FakeVolumeSpaceProbe Probe { get; } = new();
internal DerivedSchedulerProbe SchedulerProbe { get; } = new();
internal FailableDerivedArtifactStorage ArtifactStorage { get; } = new();
internal ServiceProvider Services { get; }
internal ActivationMaintenanceTestBed()
{
Directory.CreateDirectory(Path.Combine(Root, "db-volume"));
Directory.CreateDirectory(Path.Combine(Root, "media-volume"));
Paths = new SelfHostedActivationPaths(
Path.Combine(Root, "db-volume", "nostos.db"),
Path.Combine(Root, "media-volume", "library"));
Journals = new SelfHostedActivationJournalStore(Paths, Gate);
Manifests = new SelfHostedRecoveryManifestStore(Paths);
Probe.Set(Paths.LiveDatabase, 1_000_000_000, 1_000_000_000, "db");
Probe.Set(Paths.LiveMedia, 1_000_000_000, 1_000_000_000, "media");
RecoveryTestBed.CreateDatabaseAsync(Paths.LiveDatabase, "active").GetAwaiter().GetResult();
var services = new ServiceCollection();
services.AddLogging();
services.AddSingleton(Clock);
services.AddSingleton(Clock);
services.AddSingleton(Gate);
services.AddSingleton(Gate);
services.AddSingleton(Paths);
services.AddSingleton(Journals);
services.AddSingleton(Manifests);
services.AddSingleton(Options.Create(StorageOptions));
services.AddSingleton(Options.Create(MaintenanceOptions));
services.AddSingleton(TransferVolume);
services.AddSingleton(Probe);
services.AddDbContextFactory(options =>
options.UseSqlite($"Data Source={Paths.LiveDatabase};Pooling=False"));
services.AddScoped();
services.AddScoped();
services.AddScoped(sp => new SelfHostedMigrationRecoveryService(
Paths,
Manifests,
Journals,
sp.GetRequiredService(),
sp.GetRequiredService(),
sp.GetRequiredService(),
Gate,
sp.GetRequiredService>(),
Clock));
services.AddScoped(sp =>
sp.GetRequiredService());
services.AddScoped();
services.AddScoped();
services.AddScoped(sp => sp.GetRequiredService());
services.AddScoped();
services.AddSingleton(ArtifactStorage);
services.AddScoped();
services.AddScoped(sp => new RecordingDerivedScheduler(
new StrictBookTextDerivedScheduler(sp.GetRequiredService(), ArtifactStorage),
SchedulerProbe));
services.AddSingleton();
Services = services.BuildServiceProvider();
}
internal NostosDbContext OpenDatabase() =>
new(new DbContextOptionsBuilder()
.UseSqlite($"Data Source={Paths.LiveDatabase};Pooling=False")
.Options);
internal SelfHostedRecoveryCleanupWorker CreateCleanupWorker() =>
new(Services.GetRequiredService(), Options.Create(MaintenanceOptions), Clock,
Gate, Manifests, NullLogger.Instance);
internal SelfHostedActivationOrphanSweep CreateSweep() =>
new(Paths, Journals, Manifests, OpenDatabase(), Options.Create(MaintenanceOptions), Clock,
NullLogger.Instance);
internal SelfHostedDerivedRebuildService CreateRebuildService() =>
new(Paths, Journals, Gate, Services.GetRequiredService(), Clock,
NullLogger.Instance);
internal SelfHostedDerivedRebuildWorker CreateRebuildWorker(SelfHostedDerivedRebuildService service) =>
new(service, Options.Create(MaintenanceOptions), Clock,
NullLogger.Instance);
internal async Task SeedJobAsync(
Guid jobId,
MigrationJobState state,
MigrationRecoveryStatus recoveryStatus = MigrationRecoveryStatus.NotRequired,
DateTime? leaseExpiresAtUtc = null,
string? leaseToken = null)
{
await using var db = OpenDatabase();
db.MigrationJobRecords.Add(new MigrationJobRecord
{
Id = jobId,
Direction = (int)MigrationDirection.Import,
State = (int)state,
RecoveryStatus = (int)recoveryStatus,
ExpiresAtUtc = Clock.UtcNow.UtcDateTime.AddDays(1),
LeaseExpiresAtUtc = leaseExpiresAtUtc,
MigrationLeaseToken = leaseToken,
IdempotencyKey = jobId.ToString("N"),
});
await db.SaveChangesAsync();
}
internal async Task ReadJobAsync(Guid jobId)
{
await using var db = OpenDatabase();
return await db.MigrationJobRecords.AsNoTracking().SingleAsync(job => job.Id == jobId);
}
internal void SeedJournal(Guid jobId, Guid? operationId = null, SelfHostedActivationPhase phase = SelfHostedActivationPhase.CandidatePrepared)
{
Journals.Write(new SelfHostedActivationJournal(
jobId, operationId ?? Guid.NewGuid(), phase, "revision-1", true, Clock.GetUtcNow()));
}
internal void SeedCommittedResolvedJournal(Guid jobId, Guid? operationId = null) =>
SeedResolvedJournal(jobId, SelfHostedActivationPhase.Committed, operationId);
///
/// A resolved rollback journal. The cutover clears the live derived/
/// caches before retention, so the preserved original generation needs the
/// same wipe-and-reschedule pass a committed one gets.
///
internal void SeedRolledBackResolvedJournal(Guid jobId, Guid? operationId = null) =>
SeedResolvedJournal(jobId, SelfHostedActivationPhase.RolledBack, operationId);
private void SeedResolvedJournal(Guid jobId, SelfHostedActivationPhase phase, Guid? operationId)
{
var directory = Path.GetDirectoryName(Paths.ResolvedJournal(jobId))!;
Directory.CreateDirectory(directory);
var journal = new SelfHostedActivationJournal(
jobId, operationId ?? Guid.NewGuid(), phase,
"revision-1", true, Clock.GetUtcNow());
File.WriteAllText(Paths.ResolvedJournal(jobId), SelfHostedActivationDocument.Encode(journal));
}
internal (Guid JobId, Guid ReservationId) SeedExpiredCopy(
DateTimeOffset createdAt,
MigrationRecoveryStatus status = MigrationRecoveryStatus.Available,
Guid? jobId = null)
{
jobId ??= Guid.NewGuid();
CreatePreviousMaterial(jobId.Value);
using var db = OpenDatabase();
var capacity = new TransferStorageCapacity(db, TransferVolume, Options.Create(StorageOptions), Clock);
var reserved = capacity
.TryReserveAsync(1_234, MigrationSessionPurpose.Import, TimeSpan.FromMinutes(15), default)
.GetAwaiter().GetResult();
capacity.ClaimAsync(reserved.ReservationId!.Value, jobId.Value, default).GetAwaiter().GetResult();
var manifest = new SelfHostedRecoveryManifest(
jobId.Value,
Guid.NewGuid(),
createdAt,
SelfHostedRecoveryManifest.Expiry(createdAt),
status,
"revision-1",
new MigrationExistingCounts(Books: 1),
100,
200,
new string('a', 64),
[new RecoveryMediaDescriptor(FirstBook, "book", ".epub", 100, new string('b', 64))],
DatabaseSchemaVersion: "20260101000000_Initial",
DatabaseMigrationCount: 1,
DatabaseRetained: true,
MediaRetained: true,
RetentionReservationId: reserved.ReservationId);
Manifests.Write(manifest);
return (jobId.Value, reserved.ReservationId.Value);
}
internal void CreatePreviousMaterial(Guid jobId)
{
Paths.PrepareRecovery(jobId);
RecoveryTestBed.CreateRawDatabase(Paths.PreviousDatabase(jobId), "previous");
var media = Paths.PreviousMedia(jobId);
Directory.CreateDirectory(Path.Combine(media, FirstBook.ToString("N")));
File.WriteAllText(Path.Combine(media, FirstBook.ToString("N"), "book.epub"), "previous-book");
}
internal void SeedCandidateDatabase(Guid jobId, DateTime writeTimeUtc)
{
var path = Paths.CandidateDatabase(jobId);
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
File.WriteAllText(path, "candidate");
File.SetLastWriteTimeUtc(path, writeTimeUtc);
}
internal void SeedCandidateMedia(Guid jobId, DateTime writeTimeUtc)
{
var root = Paths.CandidateMedia(jobId);
Directory.CreateDirectory(Path.Combine(root, FirstBook.ToString("N")));
File.WriteAllText(Path.Combine(root, FirstBook.ToString("N"), "book.epub"), "candidate-book");
SetTreeWriteTimeUtc(root, writeTimeUtc);
}
internal static void SetTreeWriteTimeUtc(string path, DateTime writeTimeUtc)
{
foreach (var file in Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories))
{
File.SetLastWriteTimeUtc(file, writeTimeUtc);
}
foreach (var directory in Directory.EnumerateDirectories(path, "*", SearchOption.AllDirectories)
.OrderByDescending(directory => directory.Length))
{
Directory.SetLastWriteTimeUtc(directory, writeTimeUtc);
}
Directory.SetLastWriteTimeUtc(path, writeTimeUtc);
}
internal async Task SeedBookAsync(Guid bookId, string fileName)
{
await using var db = OpenDatabase();
db.Books.Add(new EBookModel
{
Id = bookId,
Title = "Imported Book",
FileDetails = new FileInfoDetails { HasFile = true, FileName = fileName },
});
await db.SaveChangesAsync();
}
internal async Task CountDerivedRowsAsync(string table)
{
await using var connection = new SqliteConnection(
new SqliteConnectionStringBuilder { DataSource = Paths.LiveDatabase, Pooling = false }.ToString());
await connection.OpenAsync();
await using var command = connection.CreateCommand();
command.CommandText = "SELECT COUNT(*) FROM " + table + ";";
return (long)(await command.ExecuteScalarAsync())!;
}
internal async Task EnsureBookTextSchemaAsync()
{
await using var scope = Services.CreateAsyncScope();
await scope.ServiceProvider.GetRequiredService().EnsureSchemaAsync();
}
internal string DerivedRebuildMarkerPath(Guid jobId) =>
Path.Combine(Path.GetDirectoryName(Paths.Journal(jobId))!, "derived.rebuilt.json");
internal string DerivedResetRecordPath(Guid jobId) =>
Path.Combine(Path.GetDirectoryName(Paths.Journal(jobId))!, "derived.reset.json");
/// Runs the real guarded retry transition; returns null on success.
internal async Task TryRetryAsync(Guid jobId)
{
await using var db = OpenDatabase();
var store = new EfMigrationJobStore(db, Clock);
try
{
await store.RetryAsync(jobId, new MigrationRetryRequest(), CancellationToken.None);
return null;
}
catch (MigrationJobStoreException exception)
{
return exception;
}
}
///
/// Marks a book as fully ingested with a preserved chunk, modelling progress
/// made after the rebuild wiped the derived state.
///
internal async Task MarkDerivedStateReadyAsync(Guid bookId)
{
var version = BookTextArtifactSchema.CurrentExtractorVersion;
await using var connection = new SqliteConnection(
new SqliteConnectionStringBuilder { DataSource = Paths.LiveDatabase, Pooling = false }.ToString());
await connection.OpenAsync();
await using var command = connection.CreateCommand();
command.CommandText = """
INSERT INTO BookTextIngestionStates
(BookId, Status, SourceFileName, Format, SourceSha256, ExtractorVersion,
ErrorCode, ErrorMessage, Attempts, ChunkCount, CharacterCount, UpdatedAtUtc)
VALUES
($book, 'Ready', 'book.epub', 'Epub', $hash, $version,
NULL, NULL, 1, 1, 10, '2026-01-01T00:00:00.0000000+00:00')
ON CONFLICT(BookId) DO UPDATE SET
Status='Ready',
SourceSha256=excluded.SourceSha256,
ExtractorVersion=excluded.ExtractorVersion,
ChunkCount=1;
INSERT INTO BookTextChunks
(Id, BookId, SourceSha256, ExtractorVersion, Format, Ordinal,
Text, HeadingPathJson, SourceSegmentsJson)
VALUES
($chunk, $book, $hash, $version, 'Epub', 0, 'preserved text', '[]', '[]');
""";
command.Parameters.AddWithValue("$book", bookId.ToString("D"));
command.Parameters.AddWithValue("$chunk", Guid.NewGuid().ToString("D"));
command.Parameters.AddWithValue("$hash", new string('c', 64));
command.Parameters.AddWithValue("$version", version);
await command.ExecuteNonQueryAsync();
}
internal async Task CountChunksForBookAsync(Guid bookId)
{
await using var connection = new SqliteConnection(
new SqliteConnectionStringBuilder { DataSource = Paths.LiveDatabase, Pooling = false }.ToString());
await connection.OpenAsync();
await using var command = connection.CreateCommand();
command.CommandText = "SELECT COUNT(*) FROM BookTextChunks WHERE BookId=$book;";
command.Parameters.AddWithValue("$book", bookId.ToString("D"));
return (long)(await command.ExecuteScalarAsync())!;
}
internal async Task ReadDerivedStateStatusAsync(Guid bookId)
{
await using var connection = new SqliteConnection(
new SqliteConnectionStringBuilder { DataSource = Paths.LiveDatabase, Pooling = false }.ToString());
await connection.OpenAsync();
await using var command = connection.CreateCommand();
command.CommandText = "SELECT Status FROM BookTextIngestionStates WHERE BookId=$book;";
command.Parameters.AddWithValue("$book", bookId.ToString("D"));
var value = await command.ExecuteScalarAsync();
return value is null or DBNull ? null : (string)value;
}
internal SelfHostedRecoveryManifest BuildManifest(
Guid jobId,
DateTimeOffset createdAt,
MigrationRecoveryStatus status = MigrationRecoveryStatus.Available,
Guid? reservationId = null) =>
new(
jobId,
Guid.NewGuid(),
createdAt,
SelfHostedRecoveryManifest.Expiry(createdAt),
status,
"revision-1",
new MigrationExistingCounts(Books: 1),
100,
200,
new string('a', 64),
[new RecoveryMediaDescriptor(FirstBook, "book", ".epub", 100, new string('b', 64))],
DatabaseSchemaVersion: "20260101000000_Initial",
DatabaseMigrationCount: 1,
DatabaseRetained: true,
MediaRetained: true,
RetentionReservationId: reservationId);
internal void WriteManifestBytes(Guid jobId, byte[] bytes)
{
Paths.PrepareRecovery(jobId);
File.WriteAllBytes(Paths.RecoveryManifest(jobId), bytes);
}
public void Dispose()
{
Services.Dispose();
try
{
Directory.Delete(Root, recursive: true);
}
catch (IOException)
{
}
catch (UnauthorizedAccessException)
{
}
}
}
/// Records every strict scheduling attempt across rebuild passes.
internal sealed class DerivedSchedulerProbe
{
internal List Calls { get; } = [];
}
internal sealed class RecordingDerivedScheduler(
IBookTextDerivedScheduler inner,
DerivedSchedulerProbe probe) : IBookTextDerivedScheduler
{
public Task ScheduleAsync(
Guid bookId,
string fileName,
CancellationToken ct = default)
{
probe.Calls.Add(bookId);
return inner.ScheduleAsync(bookId, fileName, ct);
}
}
/// Derived-artifact storage whose cleanup can be made to fail for chosen books.
internal sealed class FailableDerivedArtifactStorage : IBookDerivedArtifactStorage
{
internal HashSet FailFor { get; } = [];
internal List Deleted { get; } = [];
public Task WriteAsync(
BookTextSourceRevision revision,
Func writer,
CancellationToken ct = default) => Task.CompletedTask;
public Task DeleteBookArtifactsAsync(Guid bookId, CancellationToken ct = default)
{
Deleted.Add(bookId);
if (FailFor.Contains(bookId))
{
throw new IOException("simulated derived-artifact cleanup failure");
}
return Task.CompletedTask;
}
public Task PruneBookArtifactsAsync(
Guid bookId,
BookTextSourceRevision keep,
CancellationToken ct = default) => Task.CompletedTask;
}