using System; using System.Collections.Generic; using System.Collections.Concurrent; using System.Diagnostics; using System.Linq; using System.Text.RegularExpressions; using System.Threading; using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Core.Datastore; using NzbDrone.Core.Configuration; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.MetadataSource; namespace NzbDrone.Core.Books { public interface IRefreshSeriesService { bool RefreshSeriesInfo(int authorId, List remoteBooks, Author remoteData, bool forceBookRefresh, bool forceUpdateFileTags, DateTime? lastUpdate); } public class RefreshSeriesService : RefreshEntityServiceBase, IRefreshSeriesService { private readonly IBookService _bookService; private readonly ISeriesService _seriesService; private readonly ISeriesBookLinkService _linkService; private readonly IRefreshSeriesBookLinkService _refreshLinkService; private readonly Logger _logger; private readonly IConfigService _configService; private readonly IEditionService _editionService; private readonly IAudioTagService _audioTagService; private readonly ConcurrentDictionary _seriesRefreshLocks = new(StringComparer.OrdinalIgnoreCase); private sealed class AuthorScopedSeries : Series { public Action> CaptureBeforeWrite { get; set; } public List AllLinks { get; set; } public AuthorScopedSeries(Series source, List links) { UseMetadataFrom(source); UseDbFieldsFrom(source); LinkItems = links; } } public RefreshSeriesService(IBookService bookService, ISeriesService seriesService, ISeriesBookLinkService linkService, IRefreshSeriesBookLinkService refreshLinkService, Logger logger, IConfigService configService, IEditionService editionService, IAudioTagService audioTagService) : base(logger) { _bookService = bookService; _seriesService = seriesService; _linkService = linkService; _refreshLinkService = refreshLinkService; _logger = logger; _configService = configService; _editionService = editionService; _audioTagService = audioTagService; } private static bool HasMatchingGoodreadsSeriesId(Series local, Series remote) { // Series identity is Goodreads-backed only. Do not fall back to Amazon or title matching. return local?.GoodreadsSeriesId.IsNotNullOrWhiteSpace() == true && remote?.GoodreadsSeriesId.IsNotNullOrWhiteSpace() == true && local.GoodreadsSeriesId.Equals(remote.GoodreadsSeriesId, StringComparison.OrdinalIgnoreCase); } private static bool MatchesRemoteSeries(Series local, Series remote) { return local != null && remote != null && local.MediaType == remote.MediaType && local.IsOriginal && HasMatchingGoodreadsSeriesId(local, remote); } private static string BuildBookLookupKey(BookMediaType mediaType, string providerId) { if (providerId.IsNullOrWhiteSpace()) { return null; } return $"{(int)mediaType}|{providerId.Trim()}"; } private static string NormalizeSeriesBookProviderId(string providerId) { if (providerId.IsNullOrWhiteSpace()) { return null; } return ProviderIdHelper.Normalize(providerId.Trim(), defaultPrefix: null); } private static void AddBookLookup(Dictionary> lookup, Book book, string providerId) { var key = BuildBookLookupKey(book.MediaType, providerId); if (key.IsNullOrWhiteSpace()) { return; } if (!lookup.TryGetValue(key, out var books)) { books = new List(); lookup[key] = books; } if (books.All(existing => existing.Id != book.Id)) { books.Add(book); } } private Dictionary> BuildLocalBookProviderLookup(int authorId, List localBooks) { var lookup = new Dictionary>(StringComparer.OrdinalIgnoreCase); foreach (var book in localBooks.Where(b => b != null)) { var providerIds = BookIdentity.GetProviderIdentityTokens(book) .Where(id => !id.IsNullOrWhiteSpace()) .Distinct(StringComparer.OrdinalIgnoreCase); foreach (var providerId in providerIds) { AddBookLookup(lookup, book, providerId); } } _logger.Debug("[SERIES] Built local series book lookup for authorId {0}: {1} books, {2} provider keys", authorId, localBooks.Count, lookup.Count); return lookup; } private List BuildLocalSeriesLinks(Series remote, Dictionary> localBookLookup) { var result = new List(); var seriesInstanceType = remote?.IsNarratorVariant == true ? "narrator_variant" : "original"; if (remote?.SeriesBooks == null || !remote.SeriesBooks.Any()) { return result; } foreach (var seriesBook in remote.SeriesBooks) { string normalizedForeignBookId; try { normalizedForeignBookId = NormalizeSeriesBookProviderId(seriesBook.BookId); } catch (Exception ex) { _logger.Debug(ex, "[SERIES] Ignoring invalid series book provider ID '{0}' for series '{1}'", seriesBook.BookId, remote.Title); continue; } var key = BuildBookLookupKey(remote.MediaType, normalizedForeignBookId); if (key.IsNullOrWhiteSpace() || !localBookLookup.TryGetValue(key, out var matchingBooks)) { continue; } foreach (var matchingBook in matchingBooks.Where(b => b != null).DistinctBy(b => b.Id)) { result.Add(new SeriesBookLink { Position = seriesBook.Position, SeriesPosition = int.TryParse(seriesBook.Position, out var pos) ? pos : 0, Book = matchingBook, BookId = matchingBook.Id, // Primary status is metadata; an unknown flag does not remove the membership. IsPrimary = seriesBook.IsPrimary, SeriesInstanceType = seriesInstanceType }); } } return result; } private List FilterRemoteSeriesToLocalBooks(int authorId, List remoteSeries, List localBooks) { var localBookLookup = BuildLocalBookProviderLookup(authorId, localBooks); if (localBookLookup.Count == 0) { _logger.Debug("[SERIES] No local provider IDs available for authorId {0}; no remote series can be linked", authorId); return new List(); } var filtered = new List(); var linkedBookCount = 0; foreach (var remote in remoteSeries) { if (remote?.SeriesBooks?.Any() != true && remote?.LinkItems?.Any() == true) { filtered.Add(remote); linkedBookCount += remote.LinkItems.Count; continue; } var linkItems = BuildLocalSeriesLinks(remote, localBookLookup); if (!linkItems.Any()) { continue; } remote.LinkItems = linkItems; filtered.Add(remote); linkedBookCount += linkItems.Count; } _logger.Debug("[SERIES] Filtered {0} remote series to {1} series linked to local books for authorId {2} ({3} links)", remoteSeries.Count, filtered.Count, authorId, linkedBookCount); return filtered; } protected override RemoteData GetRemoteData(Series local, List remote, Author data, RefreshContext context) { // FIX: Handle potential duplicates by using FirstOrDefault instead of SingleOrDefault // This can happen when multiple series have the same provider IDs var matches = remote.Where(x => MatchesRemoteSeries(local, x)).ToList(); if (matches.Count > 1) { _logger.Debug("[SERIES-MATCH] Found {0} matches for series '{1}' (GR: {2}). Using first match.", matches.Count, local.Title, local.GoodreadsSeriesId); } return new RemoteData { Entity = matches.FirstOrDefault(), // Metadata is now integrated into Author }; } protected override bool ShouldDelete(Series local) { return local?.IsNarratorVariant != true; } protected override bool IsMerge(Series local, Series remote) { // Detect upstream splits: only compare when BOTH sides have the provider ID. // Null/empty means "data not available", not "different series". return !string.IsNullOrEmpty(local.GoodreadsSeriesId) && !string.IsNullOrEmpty(remote.GoodreadsSeriesId) && !local.GoodreadsSeriesId.Equals(remote.GoodreadsSeriesId, StringComparison.OrdinalIgnoreCase); } protected override UpdateResult UpdateEntity(Series local, Series remote) { // Compare what the metadata writer will store, including retained optional fields, // so an unchanged snapshot does not report another update on every refresh. var remoteForUpdate = new Series(); remoteForUpdate.UseMetadataFrom(local); remoteForUpdate.UseMetadataFrom(remote); remoteForUpdate.UseDbFieldsFrom(local); if (local.Equals(remoteForUpdate)) { return UpdateResult.None; } if (local.Title != remoteForUpdate.Title && local is AuthorScopedSeries scoped) { scoped.CaptureBeforeWrite(scoped.AllLinks.Select(link => link.BookId)); } local.UseMetadataFrom(remoteForUpdate); return UpdateResult.UpdateTags; } protected override Series GetEntityByForeignId(Series local) { if (!string.IsNullOrEmpty(local.GoodreadsSeriesId)) { var series = _seriesService.FindById(local.GoodreadsSeriesId, local.MediaType); if (series != null) return series; } return null; } protected override void SaveEntity(Series local) { // Use UpdateMany to avoid firing the book edited event _seriesService.UpdateMany(new List { local }); } protected override void DeleteEntity(Series local, bool deleteFiles) { _logger.Trace($"Removing links for series {local}"); var children = GetLocalChildren(local, null, null); if (local is AuthorScopedSeries scoped) scoped.CaptureBeforeWrite(children.Select(link => link.BookId)); _linkService.DeleteMany(children); if (_seriesService.DeleteIfUnlinked(local.Id)) { _logger.Trace($"Series {local} has no links remaining, removing"); } } protected override List GetRemoteChildren(Series local, Series remote, RefreshContext context) { var result = new List(); var seriesInstanceType = local?.IsNarratorVariant == true ? "narrator_variant" : "original"; var isOriginalSeries = local?.IsNarratorVariant != true; var claimedBookIds = isOriginalSeries ? _linkService.GetClaimedBookIdsForSeriesIdentity(local.MediaType, local.GoodreadsSeriesId) : new HashSet(); if (remote.LinkItems != null) { _logger.Trace("[SERIES-DEBUG] Using prefiltered LinkItems for series {0}", remote.Title); result = remote.LinkItems; foreach (var link in result) { link.SeriesInstanceType = seriesInstanceType; } if (isOriginalSeries && claimedBookIds.Count > 0) { result = result.Where(l => l != null && !claimedBookIds.Contains(l.BookId)).ToList(); } } // Fallback for direct callers that have not prefiltered V5 SeriesBooks. else if (remote.SeriesBooks != null && remote.SeriesBooks.Any()) { _logger.Trace("[SERIES-DEBUG] Converting {0} SeriesBooks to SeriesBookLink objects for series {1}", remote.SeriesBooks.Count, remote.Title); foreach (var seriesBook in remote.SeriesBooks) { var foreignBookId = seriesBook.BookId?.Trim(); if (foreignBookId.IsNullOrWhiteSpace()) { continue; } var normalizedForeignBookId = NormalizeSeriesBookProviderId(foreignBookId); var idx = normalizedForeignBookId.IndexOf(':'); var provider = normalizedForeignBookId.Substring(0, idx).Trim(); // Look up ALL local book copies by provider ID, scoped to the series media type. var matchingBooks = _bookService.FindAllByProviderId(provider, normalizedForeignBookId, local.MediaType); if (matchingBooks.Count == 0) { _logger.Trace("[SERIES-DEBUG] Could not match series book '{0}' ({1}) for series {2} ({3})", seriesBook.Title, foreignBookId, remote.Title, local.MediaType); continue; } foreach (var matchingBook in matchingBooks.DistinctBy(b => b.Id)) { if (matchingBook == null) { continue; } // Clean separation: original series excludes any copies that are already claimed by narrator variants. if (isOriginalSeries && claimedBookIds.Contains(matchingBook.Id)) { continue; } var link = new SeriesBookLink { Position = seriesBook.Position, SeriesPosition = int.TryParse(seriesBook.Position, out var pos) ? pos : 0, Book = matchingBook, BookId = matchingBook.Id, // Primary status is metadata; an unknown flag does not remove the membership. IsPrimary = seriesBook.IsPrimary, SeriesInstanceType = seriesInstanceType }; result.Add(link); _logger.Trace("[SERIES-DEBUG] Created SeriesBookLink for book {0} at position {1}", matchingBook.Id, seriesBook.Position); } } } else { _logger.Trace("[SERIES-DEBUG] No book links found for series {0}", remote.Title); } // Defensive: dedupe remote children. Some upstream payloads contain duplicate series members // (or multiple provider IDs that resolve to the same local book), which would violate the // unique constraint on (BookId, SeriesId, SeriesInstanceType) during bulk insert. if (result.Count > 1) { static int GetBookId(SeriesBookLink link) { if (link == null) { return 0; } if (link.BookId != 0) { return link.BookId; } return link.Book?.Value?.Id ?? 0; } var deduped = result .Where(l => l != null) .GroupBy(l => new { BookId = GetBookId(l), InstanceType = (l.SeriesInstanceType ?? seriesInstanceType).Trim().ToLowerInvariant() }) .Select(g => { // Keep provider positions intact; integer parsing loses decimal/range ordering. return g .OrderBy(l => string.IsNullOrWhiteSpace(l.Position)) .ThenBy(l => BookSeries.PositionOrder(l.Position)) .ThenBy(l => l.Position, StringComparer.Ordinal) .First(); }) .ToList(); if (deduped.Count != result.Count) { _logger.Warn("[SERIES] Remote series '{0}' ({1}) contained {2} duplicate book links; deduped to {3}", remote.Title, local?.GoodreadsSeriesId ?? local?.AmazonSeriesAsin ?? local?.HardcoverSeriesId ?? local?.OpenLibrarySeriesId ?? local?.Id.ToString(), result.Count - deduped.Count, deduped.Count); } result = deduped; } return result; } protected override List GetLocalChildren(Series entity, List remoteChildren, RefreshContext context) { return entity is AuthorScopedSeries ? entity.LinkItems : throw new InvalidOperationException("Series reconciliation requires an author-scoped link snapshot."); } protected override Tuple> GetMatchingExistingChildren(List existingChildren, SeriesBookLink remote, RefreshContext context) { var existingChild = existingChildren.SingleOrDefault(x => x.BookId == remote.Book.Value.Id); var mergeChildren = new List(); return Tuple.Create(existingChild, mergeChildren); } protected override void PrepareNewChild(SeriesBookLink child, Series entity) { child.Series = entity; child.SeriesId = entity.Id; // Ensure BookId is set from the Book object if (child.Book?.Value != null) { child.BookId = child.Book.Value.Id; } _logger.Trace("[SERIES-DEBUG] PrepareNewChild: SeriesId={0}, BookId={1}, Position={2}", child.SeriesId, child.BookId, child.Position); } protected override void PrepareExistingChild(SeriesBookLink local, SeriesBookLink remote, Series entity) { local.Series = entity; local.SeriesId = entity.Id; } protected override bool AreChildrenUpToDate(SeriesBookLink local, SeriesBookLink remote) { if (local == null || remote == null) { return false; } var remoteForCompare = RefreshEntityCopy.CloneSeriesBookLink(remote); remoteForCompare.UseDbFieldsFrom(local); return local.Equals(remoteForCompare); } protected override SeriesBookLink CreateChildForAdd(SeriesBookLink remoteChild, Series entity) { return RefreshEntityCopy.CloneSeriesBookLink(remoteChild); } protected override void ProcessChildren(Series entity, SortedChildren children) { if (entity is AuthorScopedSeries scoped) { scoped.CaptureBeforeWrite(children.Added.Concat(children.Updated).Concat(children.Deleted) .Concat(children.Merged.Select(pair => pair.Item1)).Select(link => link.BookId)); } } protected override void AddChildren(List children) { _logger.Trace("[SERIES-DEBUG] AddChildren called with {0} SeriesBookLink records to insert", children.Count); if (children.Any()) { foreach (var link in children.Take(5)) { _logger.Trace("[SERIES-DEBUG] Inserting link: SeriesId={0}, BookId={1}, Position={2}", link.SeriesId, link.BookId, link.Position); } _linkService.InsertMany(children); _logger.Trace("[SERIES-DEBUG] Successfully inserted {0} SeriesBookLink records", children.Count); } } protected override bool RefreshChildren(SortedChildren localChildren, List remoteChildren, Author remoteData, bool forceChildRefresh, bool forceUpdateFileTags, DateTime? lastUpdate, RefreshContext context) { return _refreshLinkService.RefreshSeriesBookLinkInfo(localChildren.Added, localChildren.Updated, localChildren.Merged, localChildren.Deleted, localChildren.UpToDate, remoteChildren, forceUpdateFileTags); } public bool RefreshSeriesInfo(int authorId, List remoteSeries, Author remoteData, bool forceBookRefresh, bool forceUpdateFileTags, DateTime? lastUpdate) { lock (BookService.GetWantedEditionLock(authorId)) { return RefreshAuthorSeries(authorId, remoteSeries, remoteData, forceBookRefresh, forceUpdateFileTags, lastUpdate); } } private bool RefreshAuthorSeries(int authorId, List remoteSeries, Author remoteData, bool forceBookRefresh, bool forceUpdateFileTags, DateTime? lastUpdate) { remoteSeries ??= new List(); _logger.Trace("[SERIES-DEBUG] RefreshSeriesInfo called for authorId {0} with {1} remote series", authorId, remoteSeries.Count); // We only persist Goodreads-backed series. Amazon-only series are considered invalid and are not saved. var goodreadsBackedRemoteSeries = remoteSeries .Where(s => s != null && !s.GoodreadsSeriesId.IsNullOrWhiteSpace()) .ToList(); if (goodreadsBackedRemoteSeries.Count != remoteSeries.Count) { _logger.Debug("[SERIES] Filtered {0} non-Goodreads remote series for authorId {1}", remoteSeries.Count - goodreadsBackedRemoteSeries.Count, authorId); } // Guardrail: if upstream returned zero Goodreads-backed series, do NOT prune existing Goodreads-backed series. // This avoids nuking user data on transient upstream issues. // We still allow the migration (and future refresh passes) to clean up any invalid local rows. var remoteMediaTypes = goodreadsBackedRemoteSeries .Select(s => s.MediaType) .Distinct() .ToHashSet(); if (remoteMediaTypes.Count == 0) { _logger.Debug("[SERIES] No Goodreads-backed series returned for authorId {0}; skipping refresh", authorId); return false; } // Defensive: prevent duplicate inserts if the metadata payload contains duplicates. var providerBackedRemoteSeries = goodreadsBackedRemoteSeries .GroupBy(s => $"{(int)s.MediaType}|{s.GoodreadsSeriesId?.Trim()}", StringComparer.OrdinalIgnoreCase) .Select(g => RefreshEntityCopy.CloneSeries(g.First())) .ToList(); if (providerBackedRemoteSeries.Count != goodreadsBackedRemoteSeries.Count) { _logger.Warn("[SERIES] Metadata payload contained {0} duplicate Goodreads series entries for authorId {1}; ignoring duplicates", goodreadsBackedRemoteSeries.Count - providerBackedRemoteSeries.Count, authorId); } var localBooks = _bookService.GetBooksByAuthor(authorId) ?? new List(); var localBookIds = localBooks.Select(b => b.Id).ToHashSet(); providerBackedRemoteSeries = FilterRemoteSeriesToLocalBooks(authorId, providerBackedRemoteSeries, localBooks); if (providerBackedRemoteSeries.Count == 0) { _logger.Debug("[SERIES] No Goodreads-backed series could be linked to local books for authorId {0}; skipping refresh", authorId); return false; } var sync = _configService.WriteAudioTags == WriteAudioTagsType.Sync; var booksById = localBooks.ToDictionary(book => book.Id); var seriesBefore = new Dictionary(); // Snapshot only books whose links or shared series are about to be written. // The final read checks persisted values, including partially completed refreshes. void CaptureBeforeWrite(int seriesId, BookMediaType mediaType, IEnumerable ids) { if (!sync || mediaType != BookMediaType.Audiobook) return; var pending = ids.Distinct().Where(id => !seriesBefore.ContainsKey(id)).ToList(); if (pending.Count == 0) return; var timer = Stopwatch.StartNew(); var missing = pending.Where(id => !booksById.ContainsKey(id)).ToList(); if (missing.Count > 0) { foreach (var book in _bookService.GetExistingBooks(missing)) booksById[book.Id] = book; } var audio = pending.Select(id => booksById.TryGetValue(id, out var book) ? book : null) .Where(book => book?.MediaType == BookMediaType.Audiobook).ToList(); // BookService already hydrates these links; do not repeat that read. foreach (var book in audio) seriesBefore[book.Id] = BookSeries.AudioTagPair(book); var unavailable = missing.Count(id => !booksById.ContainsKey(id)); if (unavailable > 0) { _logger.Warn("Series {0}: skipped {1} missing book records while preparing audio tag sync", seriesId, unavailable); } _logger.Debug("Series tag snapshot: {0} affected books, {1} additional books loaded, {2} missing books skipped, {3} ms", audio.Count, missing.Count - unavailable, unavailable, timer.ElapsedMilliseconds); } var updated = false; // Discover existing series via: // 1) author-linked series (SeriesBookLink -> Book -> AuthorId) // 2) provider ID lookup (so linkless-but-persisted series can still be repaired) var existingByAuthor = _seriesService.GetByAuthorId(authorId); // Shared identities must stay stable from discovery through link persistence/deletion. // Ordered acquisition preserves batching without serializing unrelated authors' series. var seriesLocks = existingByAuthor.Concat(providerBackedRemoteSeries) .Select(series => $"{(int)series.MediaType}|{series.GoodreadsSeriesId ?? $"local:{series.Id}"}") .Distinct(StringComparer.OrdinalIgnoreCase).OrderBy(key => key, StringComparer.OrdinalIgnoreCase) .Select(key => _seriesRefreshLocks.GetOrAdd(key, _ => new object())).ToList(); var acquired = 0; try { foreach (var seriesLock in seriesLocks) { Monitor.Enter(seriesLock); acquired++; } var existingBySeries = providerBackedRemoteSeries .GroupBy(s => s.MediaType) .SelectMany(g => _seriesService.FindById( g.Select(s => s.GoodreadsSeriesId) .Where(id => !id.IsNullOrWhiteSpace()) .Distinct(StringComparer.OrdinalIgnoreCase) .ToList(), g.Key)) .ToList(); var existing = existingByAuthor .Concat(existingBySeries) .Where(s => s != null && s.IsOriginal) .GroupBy(s => s.Id) .Select(g => g.First()) .ToList(); var toAdd = providerBackedRemoteSeries .Where(remote => !existing.Any(local => MatchesRemoteSeries(local, remote))) .ToList(); if (toAdd.Any()) { _seriesService.InsertMany(toAdd); } // Refresh ALL existing + newly added series through the base-class path. // - Matched originals get metadata + link reconciliation // - Unmatched narrator variants are preserved via ShouldDelete=false // - Legacy provider-id-less originals (if any slip through) are deleted via ShouldDelete=true var all = existing .Concat(toAdd) .Where(s => s != null && s.IsOriginal && remoteMediaTypes.Contains(s.MediaType)) .GroupBy(s => s.Id) .Select(g => g.First()) .ToList(); foreach (var item in all) { // Scope reconciliation to the refreshing author so shared series retain other authors' links. // A per-call copy also scopes removal when this author leaves a shared series. var timer = Stopwatch.StartNew(); var allLinks = _linkService.GetLinksBySeries(item.Id); _logger.Debug("Series refresh read {0} links for series {1} in {2} ms", allLinks.Count, item.Id, timer.ElapsedMilliseconds); var scoped = new AuthorScopedSeries(item, allLinks.Where(link => localBookIds.Contains(link.BookId)).ToList()) { AllLinks = allLinks, CaptureBeforeWrite = ids => CaptureBeforeWrite(item.Id, item.MediaType, ids) }; updated |= RefreshEntityInfo(scoped, providerBackedRemoteSeries, remoteData, forceBookRefresh, forceUpdateFileTags, lastUpdate); } } finally { var changed = new List(); try { if (seriesBefore.Count > 0) { var timer = Stopwatch.StartNew(); var affected = seriesBefore.Keys.Select(id => booksById[id]).ToList(); BookSeries.LoadLinks(affected, _linkService.GetLinksByBook, reload: true); _logger.Debug("Series tag reload: {0} books, {1} memberships, {2} ms", affected.Count, affected.Sum(book => book.SeriesLinks.Count), timer.ElapsedMilliseconds); changed = affected.Where(book => BookSeries.AudioTagPairChanged(seriesBefore[book.Id], BookSeries.AudioTagPair(book))) .Select(book => book.Id).ToList(); } } catch (Exception ex) { // Do not replace a refresh failure with a secondary tag-sync failure. _logger.Warn(ex, "Unable to reload series tags for {0} affected books", seriesBefore.Count); } finally { for (var i = acquired - 1; i >= 0; i--) Monitor.Exit(seriesLocks[i]); } if (changed.Count > 0) { try { var timer = Stopwatch.StartNew(); var editions = _editionService.GetEditionsByBook(changed); _logger.Debug("Series tag editions: {0} books, {1} editions, {2} ms", changed.Count, editions.Count, timer.ElapsedMilliseconds); _audioTagService.SyncTags(editions); } catch (Exception ex) { _logger.Warn(ex, "Unable to sync series tags for {0} affected books", changed.Count); } } } return updated; } } }