using System.IO.Compression; using System.Security.Cryptography; using System.Text; using System.Text.Json.Nodes; using Microsoft.AspNetCore.Hosting; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Nostos.Backend.Configuration; using Nostos.Backend.Data; using Nostos.Backend.Data.Models; using Nostos.Backend.Services; using Nostos.Backend.Services.Portability; namespace Nostos.Backend.Tests.Portability; internal sealed record TestArchiveEntry(string Name, byte[] Bytes); internal sealed record PortableFixtureIds( Guid EpubBookId, Guid PdfBookId, Guid AudioBookId, Guid PhysicalBookId, Guid ReadingCollectionId, Guid NestedCollectionId, Guid NoteId, Guid TopicId, Guid WritingDocumentId); internal sealed class LocalPortableTestLibrary : IAsyncDisposable { private LocalPortableTestLibrary( string root, NostosDbContext db, FileStorageService storage) { Root = root; Db = db; Storage = storage; } public string Root { get; } public NostosDbContext Db { get; } public FileStorageService Storage { get; } public static async Task CreateAsync( Action>? configureOptions = null) { var root = Path.Combine( Path.GetTempPath(), $"nostos-portability-test-{Guid.NewGuid():N}"); Directory.CreateDirectory(root); var dbPath = Path.Combine(root, "nostos.db"); var optionsBuilder = new DbContextOptionsBuilder() .UseSqlite($"Data Source={dbPath}"); configureOptions?.Invoke(optionsBuilder); var db = new NostosDbContext(optionsBuilder.Options); await db.Database.EnsureCreatedAsync(); var env = new PortableTestWebHostEnvironment(root); var storage = new FileStorageService( env, Options.Create(new FileStorageOptions { BooksRoot = Path.Combine(root, "books"), }), NullLogger.Instance); return new LocalPortableTestLibrary(root, db, storage); } public PortableArchiveService Portability(TimeProvider? timeProvider = null) => new( Db, Storage, NullLogger.Instance, bookTextScheduler: null, timeProvider: timeProvider, trackStorage: Storage); public async ValueTask DisposeAsync() { await Db.DisposeAsync(); try { if (Directory.Exists(Root)) Directory.Delete(Root, recursive: true); } catch { // Test cleanup only. } } } internal static class PortableArchiveTestSupport { /// /// Rewrites a current relational payload into the shape archives had before /// rating, favourite and review moved onto the work (data version 5 and /// older): each edition carries the book's values as its own. An edition's /// own rating or review stays; one without takes the work's. Call it after /// setting the older version; it does nothing for a current payload. /// internal static void AsPerEditionBookValues(JsonObject data) { if (data["version"]!.GetValue() >= 6) return; var works = data["works"]!.AsArray().ToDictionary( w => w!["id"]!.GetValue(), w => w!.AsObject()); foreach (var book in data["books"]!.AsArray()) { var progress = book!["progress"]!.AsObject(); var work = works[book["workId"]!.GetValue()]; if ((progress["rating"]?.GetValue() ?? 0) == 0) progress["rating"] = work["rating"]?.GetValue() ?? 0; if (string.IsNullOrEmpty(progress["personalReview"]?.GetValue())) progress["personalReview"] = work["personalReview"]?.GetValue(); progress["isFavorite"] = work["isFavorite"]?.GetValue() ?? false; } foreach (var work in works.Values) { work.Remove("rating"); work.Remove("isFavorite"); work.Remove("personalReview"); } } public const string SecretMarker = "THIS-SECRET-MUST-NEVER-BE-EXPORTED"; public static async Task PopulateRepresentativeAsync( NostosDbContext db, IBookAssetStorage storage) { var now = DateTime.UtcNow.AddDays(-10); var workA = new WorkModel { Id = Guid.NewGuid(), Title = "Portable Work A", Author = "Example Author", NormalizedTitle = "PORTABLE WORK A", NormalizedAuthor = "EXAMPLE AUTHOR", CreatedAt = now, Rating = 5, IsFavorite = true, PersonalReview = "Important review", }; var workB = new WorkModel { Id = Guid.NewGuid(), Title = "Portable Work B", Author = "Second Author", NormalizedTitle = "PORTABLE WORK B", NormalizedAuthor = "SECOND AUTHOR", CreatedAt = now.AddMinutes(1), }; var workC = new WorkModel { Id = Guid.NewGuid(), Title = "Portable Work C", Author = null, NormalizedTitle = "PORTABLE WORK C", NormalizedAuthor = string.Empty, CreatedAt = now.AddMinutes(2), }; var epub = new EBookModel { Id = Guid.NewGuid(), WorkId = workA.Id, Work = workA, Title = "Portable EPUB", Author = "Example Author", CreatedAt = now.AddHours(1), PageCount = 321, Metadata = new BookMetadata { Subtitle = "A portable edition", Translator = "Translator", Publisher = "Portable Press", Language = "en", Edition = "2", }, Progress = new ReadingProgress { LastLocation = "epubcfi(/6/4)", ProgressPercent = 42, LastReadAt = now.AddDays(3), }, FileDetails = new FileInfoDetails { HasFile = true, FileName = @"C:\source-machine\private\reader.epub", CoverFileName = "/srv/private/cover.jpg", ChaptersJson = "[{\"generated\":true}]", LocationsJson = "{\"cache\":\"must-not-cross\"}", }, }; var pdf = new EBookModel { Id = Guid.NewGuid(), WorkId = workA.Id, Work = workA, Title = "Portable PDF Edition", Author = "Example Author", CreatedAt = now.AddHours(2), PageCount = 280, Progress = new ReadingProgress { LastLocation = "page:73", ProgressPercent = 26, Rating = 4, }, FileDetails = new FileInfoDetails { HasFile = true, FileName = "/mnt/library/original.pdf", }, }; var audio = new AudioBookModel { Id = Guid.NewGuid(), WorkId = workB.Id, Work = workB, Title = "Portable Audio", Author = "Second Author", CreatedAt = now.AddHours(3), Asin = "B000000001", Duration = "10:11:12", Narrator = "Narrator", Progress = new ReadingProgress { LastLocation = "3721.5", ProgressPercent = 37, LastReadAt = now.AddDays(4), }, FileDetails = new FileInfoDetails { HasFile = true, FileName = "/another-machine/audio.m4b", CoverFileName = "/another-machine/audio-cover.png", }, }; var physical = new PhysicalBookModel { Id = Guid.NewGuid(), WorkId = workC.Id, Work = workC, Title = "Portable Physical", CreatedAt = now.AddHours(4), PageCount = 199, Progress = new ReadingProgress { LastLocation = "page:12", ProgressPercent = 6, }, }; var reading = new CollectionModel { Id = Guid.NewGuid(), Name = "Reading", }; var philosophy = new CollectionModel { Id = Guid.NewGuid(), Name = "Philosophy", ParentId = reading.Id, Parent = reading, }; var note = new NoteModel { Id = Guid.NewGuid(), BookId = epub.Id, Book = epub, Content = "Processed thought", RawContent = "raw thought", SelectedText = "selected passage", CfiRange = "epubcfi(/6/4!/4/2)", HighlightColour = "sage", // Passage fields (issue #1014), so a round trip proves they travel. Kind = "passage", PassageTitle = "The Grand Inquisitor", PassageEnd = "epubcfi(/6/8!/4/20/1:12)", PassageHtml = "

selected passage

", CaptureSource = "voice", ProcessingMode = "light_polish", SourceAnchorKind = "epub_cfi", SourceAnchorValue = "epubcfi(/6/4!/4/2)", AnchorVerified = true, CreatedAt = now.AddDays(2), }; var topic = new TopicModel { Id = Guid.NewGuid(), Topic = "Portability", // The reader's own words, so a round trip proves they travel. Description = "Moving a library between hosts without losing anything.", }; var studioFolder = new WritingModel { Id = Guid.NewGuid(), Name = "Essays", Type = WritingType.Folder, CreatedAt = now, UpdatedAt = now.AddDays(1), }; var studioDocument = new WritingModel { Id = Guid.NewGuid(), Name = "Portable draft", Type = WritingType.Document, Content = "

User-owned studio prose.

", // How it was last exported, so a round trip proves it travels. PageSetup = "{\"paper\":\"letter\",\"marginMm\":25.4,\"font\":\"newsreader\",\"lineSpacing\":1.5,\"citationStyle\":\"apa\"}", ParentId = studioFolder.Id, Parent = studioFolder, CreatedAt = now.AddHours(1), UpdatedAt = now.AddDays(5), }; db.Works.AddRange(workA, workB, workC); db.Books.AddRange(epub, pdf, audio, physical); db.Collections.AddRange(reading, philosophy); db.BookCollections.AddRange( new BookCollectionModel { BookId = epub.Id, Book = epub, CollectionId = reading.Id, Collection = reading, AddedAt = now.AddDays(1), }, new BookCollectionModel { BookId = epub.Id, Book = epub, CollectionId = philosophy.Id, Collection = philosophy, AddedAt = now.AddDays(2), }, new BookCollectionModel { BookId = audio.Id, Book = audio, CollectionId = reading.Id, Collection = reading, AddedAt = now.AddDays(3), }); db.Notes.Add(note); db.Topics.Add(topic); db.NoteTopics.Add(new NoteTopicModel { NoteId = note.Id, Note = note, TopicId = topic.Id, Topic = topic, }); db.Writings.AddRange(studioFolder, studioDocument); db.WritingNotes.Add(new WritingNoteModel { WritingId = studioDocument.Id, Writing = studioDocument, NoteId = note.Id, Note = note, AddedAt = now.AddDays(3), }); db.BookAcquisitions.Add(new BookAcquisitionModel { Id = Guid.NewGuid(), BookId = epub.Id, Book = epub, ProviderId = "gutenberg", ProviderDisplayName = "Project Gutenberg", ExternalId = "12345", AssetId = "epub3-images", AssetFormat = "epub3-images", ImportedExtension = ".epub", SourceUrl = "https://www.gutenberg.org/ebooks/12345", RightsStatement = "Public domain in the USA.", AcquiredAt = now.AddDays(1), }); db.AssistantSettings.Add(new AssistantSettingsModel { CaptureProcessingMode = "light_polish", UpdatedAtUtc = now.AddDays(4), }); db.AiProviderSettings.Add(new AiProviderSettingsModel { LlmEnabled = true, LlmBaseUrl = "https://provider.example.test", LlmModel = "private-model", LlmApiKeyEncrypted = SecretMarker, SttApiKeyEncrypted = SecretMarker + "-STT", UpdatedAtUtc = now, }); // Two wishes, one on the ordered shelf and one in the long tail, so // every round trip carries the reading list in both of its states. db.ReadingListEntries.AddRange( new ReadingListEntryModel { Id = Guid.NewGuid(), Title = "Wanted Next", Author = "Wished Author", Isbn = "9781590171998", PublishedDate = "1965", Why = "A reason worth keeping.", RecommendedBy = "A friend", NextPosition = 0, CreatedAt = now.AddDays(1), }, new ReadingListEntryModel { Id = Guid.NewGuid(), Title = "Wanted Someday", CreatedAt = now.AddDays(2), }); await db.SaveChangesAsync(); await SaveAsync(storage, epub.Id, "source.epub", "EPUB-CONTENT-PORTABLE"); await SaveAsync(storage, pdf.Id, "source.pdf", "PDF-CONTENT-PORTABLE"); await SaveAsync(storage, audio.Id, "source.m4b", "AUDIO-CONTENT-PORTABLE"); await SaveCoverAsync(storage, epub.Id, "source.jpg", "EPUB-COVER-PORTABLE"); await SaveCoverAsync(storage, audio.Id, "source.png", "AUDIO-COVER-PORTABLE"); return new PortableFixtureIds( epub.Id, pdf.Id, audio.Id, physical.Id, reading.Id, philosophy.Id, note.Id, topic.Id, studioDocument.Id); } public static async Task ReadBookAsync( IBookAssetStorage storage, Guid bookId) { await using var opened = await storage.OpenBookFileAsync(bookId); if (opened is null) throw new InvalidOperationException($"Book {bookId} has no stored file."); using var output = new MemoryStream(); await opened.Content.CopyToAsync(output); return output.ToArray(); } internal static string Sha256Hex(byte[] bytes) => Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); internal static string Sha256HexFile(string path) { using var stream = File.OpenRead(path); return Convert.ToHexString(SHA256.HashData(stream)).ToLowerInvariant(); } internal static List ReadEntries(byte[] archiveBytes) { using var source = new MemoryStream(archiveBytes, writable: false); using var archive = new ZipArchive(source, ZipArchiveMode.Read, leaveOpen: false); var entries = new List(); foreach (var entry in archive.Entries) { using var input = entry.Open(); using var output = new MemoryStream(); input.CopyTo(output); entries.Add(new TestArchiveEntry(entry.FullName, output.ToArray())); } return entries; } internal static async Task> ReadEntriesAsync(Stream source) { source.Position = 0; var entries = new List(); using var archive = new ZipArchive(source, ZipArchiveMode.Read, leaveOpen: true); foreach (var entry in archive.Entries) { await using var input = entry.Open(); using var output = new MemoryStream(); await input.CopyToAsync(output); entries.Add(new TestArchiveEntry(entry.FullName, output.ToArray())); } source.Position = 0; return entries; } internal static byte[] BuildArchive(IEnumerable entries) { using var output = new MemoryStream(); using (var archive = new ZipArchive(output, ZipArchiveMode.Create, leaveOpen: true)) { foreach (var item in entries) { var entry = archive.CreateEntry(item.Name, CompressionLevel.NoCompression); using var target = entry.Open(); target.Write(item.Bytes); } } return output.ToArray(); } internal static Task BuildArchiveAsync(IEnumerable entries) => Task.FromResult(new MemoryStream(BuildArchive(entries))); internal static void MutateJsonEntry( List entries, string name, Action mutate) { var index = entries.FindIndex(entry => string.Equals(entry.Name, name, StringComparison.Ordinal)); if (index < 0) throw new InvalidDataException($"Archive entry '{name}' is missing."); var root = JsonNode.Parse(entries[index].Bytes)?.AsObject() ?? throw new InvalidDataException($"Archive entry '{name}' is not a JSON object."); mutate(root); entries[index] = new TestArchiveEntry( name, Encoding.UTF8.GetBytes(root.ToJsonString())); } internal static void RehashDataDescriptor(List entries) { var data = entries .Single(entry => string.Equals( entry.Name, PortableArchiveFormat.DataPath, StringComparison.Ordinal)) .Bytes; MutateJsonEntry(entries, PortableArchiveFormat.ManifestPath, root => { var descriptor = root["data"]!.AsObject(); descriptor["length"] = data.LongLength; descriptor["sha256"] = Sha256Hex(data); }); } internal static void CopyDirectory(string source, string target) { Directory.CreateDirectory(target); foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories)) { var destination = Path.Combine(target, Path.GetRelativePath(source, file)); Directory.CreateDirectory(Path.GetDirectoryName(destination)!); File.Copy(file, destination, overwrite: true); } } private static async Task SaveAsync( IBookAssetStorage storage, Guid bookId, string fileName, string content) { await using var stream = new MemoryStream( System.Text.Encoding.UTF8.GetBytes(content)); await storage.SaveBookFileAsync(bookId, stream, fileName); } private static async Task SaveCoverAsync( IBookAssetStorage storage, Guid bookId, string fileName, string content) { await using var stream = new MemoryStream( System.Text.Encoding.UTF8.GetBytes(content)); await storage.SaveBookCoverAsync(bookId, stream, fileName); } } internal sealed class PortableTestWebHostEnvironment(string root) : IWebHostEnvironment { public string ApplicationName { get; set; } = "Nostos.Backend.Tests"; public IFileProvider WebRootFileProvider { get; set; } = new NullFileProvider(); public string WebRootPath { get; set; } = root; public string EnvironmentName { get; set; } = "Testing"; public string ContentRootPath { get; set; } = root; public IFileProvider ContentRootFileProvider { get; set; } = new NullFileProvider(); }