using System; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.Linq; using System.Net.Http; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; using Flurl.Http; using Kavita.API.Database; using Kavita.API.Repositories; using Kavita.API.Services; using Kavita.API.Services.Plus; using Kavita.Common; using Kavita.Common.EnvironmentInfo; using Kavita.Common.Extensions; using Kavita.Common.Helpers; using Kavita.Database; using Kavita.Models.DTOs.KavitaPlus.License; using Kavita.Models.DTOs.Stats; using Kavita.Models.DTOs.Stats.V3; using Kavita.Models.Entities; using Kavita.Models.Entities.Enums; using Kavita.Models.Entities.User; using Kavita.Services.Scanner; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Identity; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace Kavita.Services; /// /// This is for reporting to the stat server /// public class StatsService : IStatsService { private readonly ILogger _logger; private readonly IUnitOfWork _unitOfWork; private readonly DataContext _context; private readonly ILicenseService _licenseService; private readonly UserManager _userManager; private readonly IEmailService _emailService; private readonly ICacheService _cacheService; private readonly string _apiUrl; private const string ApiKey = "MsnvA2DfQqxSK5jh"; // It's not important this is public, just a way to keep bots from hitting the API willy-nilly public StatsService(ILogger logger, IUnitOfWork unitOfWork, DataContext context, ILicenseService licenseService, UserManager userManager, IEmailService emailService, ICacheService cacheService, IHostEnvironment environment) { _logger = logger; _unitOfWork = unitOfWork; _context = context; _licenseService = licenseService; _userManager = userManager; _emailService = emailService; _cacheService = cacheService; FlurlConfiguration.ConfigureClientForUrl(Configuration.StatsApiUrl); _apiUrl = environment.IsDevelopment() ? "http://localhost:5001" : Configuration.StatsApiUrl; } /// /// Due to all instances firing this at the same time, we can DDOS our server. This task when fired will schedule the task to be run /// randomly over a six-hour spread /// /// public async Task Send(CancellationToken ct = default) { var allowStatCollection = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync(ct)).AllowStatCollection; if (!allowStatCollection) { return; } await SendData(); } /// /// This must be public for Hangfire. Do not call this directly. /// // ReSharper disable once MemberCanBePrivate.Global public async Task SendData(CancellationToken ct = default) { var sw = Stopwatch.StartNew(); var data = await GetStatV3Payload(); _logger.LogDebug("Collecting stats took {Time} ms", sw.ElapsedMilliseconds); sw.Stop(); await SendDataToStatsServer(data, ct); } private async Task SendDataToStatsServer(ServerInfoV3Dto data, CancellationToken ct = default) { var responseContent = string.Empty; try { var response = await (_apiUrl + "/api/v3/stats") .WithBasicHeaders(ApiKey) .PostJsonAsync(data, cancellationToken: ct); if (response.StatusCode != StatusCodes.Status200OK) { _logger.LogError("KavitaStats did not respond successfully. {Content}", response); } // Increment stats api hits await _context.Database.ExecuteSqlAsync( $""" UPDATE ServerSetting SET Value = CAST(CAST(Value AS INTEGER) + 1 AS TEXT) WHERE Key = {ServerSettingKey.StatsApiHits} """, cancellationToken: ct); } catch (HttpRequestException e) { var info = new { dataSent = data, response = responseContent }; _logger.LogError(e, "KavitaStats did not respond successfully. {Content}", info); } catch (Exception e) { _logger.LogError(e, "An error happened during the request to KavitaStats"); } } public async Task GetServerInfoSlim(CancellationToken ct = default) { var serverSettings = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync(ct); return new ServerInfoSlimDto() { InstallId = serverSettings.InstallId, KavitaVersion = serverSettings.InstallVersion, IsDocker = OsInfo.IsDocker, FirstInstallDate = serverSettings.FirstInstallDate, FirstInstallVersion = serverSettings.FirstInstallVersion }; } public async Task SendCancellation(CancellationToken ct = default) { _logger.LogInformation("Informing KavitaStats that this instance is no longer sending stats"); var installId = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync(ct)).InstallId; var responseContent = string.Empty; try { var response = await (_apiUrl + "/api/v2/stats/opt-out?installId=" + installId) .WithBasicHeaders(ApiKey) .WithTimeout(TimeSpan.FromSeconds(30)) .PostAsync(cancellationToken: ct); if (response.StatusCode != StatusCodes.Status200OK) { _logger.LogError("KavitaStats did not respond successfully. {Content}", response); } } catch (HttpRequestException e) { _logger.LogError(e, "KavitaStats did not respond successfully. {Response}", responseContent); } catch (Exception e) { _logger.LogError(e, "An error happened during the request to KavitaStats"); } } private static async Task PingStatsApi() { try { var sw = Stopwatch.StartNew(); var response = await (Configuration.StatsApiUrl + "/api/health/") .WithBasicHeaders(ApiKey) .GetAsync(); if (response.StatusCode == StatusCodes.Status200OK) { sw.Stop(); return sw.ElapsedMilliseconds; } } catch (Exception) { /* Swallow */ } return 0; } private async Task MaxSeriesInAnyLibrary() { // If first time flow, return 0 if (!await _context.Series.AnyAsync()) return 0; return await _context.Series .Select(s => _context.Library.Where(l => l.Id == s.LibraryId).SelectMany(l => l.Series!).Count()) .MaxAsync(); } private async Task MaxVolumesInASeries() { // If first time flow, just return 0 if (!await _context.Volume.AnyAsync()) return 0; return await _context.Volume .Select(v => new { v.SeriesId, Count = _context.Series.Where(s => s.Id == v.SeriesId).SelectMany(s => s.Volumes!).Count() }) .AsNoTracking() .AsSplitQuery() .MaxAsync(d => d.Count); } private async Task MaxChaptersInASeries() { // If first time flow, just return 0 if (!await _context.Chapter.AnyAsync()) return 0; return await _context.Series .AsNoTracking() .AsSplitQuery() .MaxAsync(s => s.Volumes! .Where(v => v.MinNumber == Parser.LooseLeafVolumeNumber) .SelectMany(v => v.Chapters!) .Count()); } private async Task> CountMultiFileEpubChaptersByLibrary() { var sw = Stopwatch.StartNew(); var libraryIds = await _context.MangaFile .Where(f => f.Format == MangaFormat.Epub) .GroupBy(f => new { f.ChapterId, f.Chapter.Volume.Series.LibraryId }) .Where(g => g.Count() > 1) .Select(g => g.Key.LibraryId) .ToListAsync(); _logger.LogDebug("[StatsService] Counted {Count} multi-file epub chapters in {ElapsedMs} ms", libraryIds.Count, sw.ElapsedMilliseconds); return libraryIds .GroupBy(id => id) .ToDictionary(g => g.Key, g => g.Count()); } private async Task SetKavitaPlusState(ServerInfoV3Dto dto) { var licenseKey = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value; if (string.IsNullOrEmpty(licenseKey)) { dto.KavitaPlusState = KavitaPlusState.None; return; } LicenseInfoDto? info; try { info = await _licenseService.GetLicenseInfo(); } catch (Exception) { info = null; } if (info == null) { dto.KavitaPlusState = KavitaPlusState.Unreachable; return; } dto.KavitaPlusState = ToStatsKavitaPlusState(info.State); dto.KavitaPlusIsFree = info.PriceAmount == null ? null : info.PriceAmount == 0; } internal static KavitaPlusState ToStatsKavitaPlusState(KavitaPlusSubscriptionState state) { return state switch { KavitaPlusSubscriptionState.Active => KavitaPlusState.Active, KavitaPlusSubscriptionState.Cancelling => KavitaPlusState.Cancelling, KavitaPlusSubscriptionState.Paused => KavitaPlusState.Paused, KavitaPlusSubscriptionState.Expired => KavitaPlusState.Expired, KavitaPlusSubscriptionState.Cancelled => KavitaPlusState.Expired, _ => throw new ArgumentOutOfRangeException(nameof(state), state, null) }; } private async Task GetStatV3Payload() { var serverSettings = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync(); var mediaSettings = await _unitOfWork.SettingsRepository.GetMetadataSettings(); var dto = new ServerInfoV3Dto() { InstallId = serverSettings.InstallId, KavitaVersion = serverSettings.InstallVersion, InitialKavitaVersion = serverSettings.FirstInstallVersion, InitialInstallDate = (DateTime)serverSettings.FirstInstallDate!, IsDocker = OsInfo.IsDocker, Os = RuntimeInformation.OSDescription, NumOfCores = Math.Max(Environment.ProcessorCount, 1), DotnetVersion = Environment.Version.ToString(), OpdsEnabled = serverSettings.EnableOpds, EnableFolderWatching = serverSettings.EnableFolderWatching, EncodeMediaAs = serverSettings.EncodeMediaAs, MatchedMetadataEnabled = mediaSettings.Enabled, OidcEnabled = !string.IsNullOrEmpty(serverSettings.OidcConfig.Authority), }; dto.OsLocale = CultureInfo.CurrentCulture.EnglishName; dto.LastReadTime = await _unitOfWork.AppUserProgressRepository.GetLatestProgress(); dto.MaxSeriesInALibrary = await MaxSeriesInAnyLibrary(); dto.MaxVolumesInASeries = await MaxVolumesInASeries(); dto.MaxChaptersInASeries = await MaxChaptersInASeries(); dto.TotalFiles = await _context.MangaFile.CountAsync(); dto.TotalGenres = await _context.Genre.CountAsync(); dto.TotalPeople = await _context.Person.CountAsync(); dto.TotalSeries = await _context.Series.CountAsync(); dto.TotalLibraries = await _context.Library.CountAsync(); dto.NumberOfCollections = await _context.AppUserCollection.CountAsync(); dto.NumberOfReadingLists = await _context.ReadingList.CountAsync(); try { dto.ActiveKavitaPlusSubscription = await _licenseService.HasActiveLicense(); } catch (Exception) { dto.ActiveKavitaPlusSubscription = false; } await SetKavitaPlusState(dto); // Find a random cbz/zip file and open it for reading await OpenRandomFile(dto); dto.TimeToPingKavitaStatsApi = await PingStatsApi(); #region Relationships dto.Relationships = await _context.SeriesRelation .GroupBy(sr => sr.RelationKind) .Select(g => new RelationshipStatV3 { Relationship = g.Key, Count = g.Count() }) .ToListAsync(); #endregion dto.MediaErrors = await _context.MediaError .GroupBy(e => e.Reason) .Select(g => new MediaErrorStatV3 { Reason = g.Key, Count = g.Count(), DismissedCount = g.Count(e => e.IsDismissed) }) .ToListAsync(); var multiFileEpubChapters = await CountMultiFileEpubChaptersByLibrary(); #region Libraries var allLibraries = (await _unitOfWork.LibraryRepository.GetLibrariesAsync(LibraryIncludes.Folders | LibraryIncludes.FileTypes | LibraryIncludes.ExcludePatterns | LibraryIncludes.AppUser)).ToList(); dto.Libraries ??= []; foreach (var library in allLibraries) { var libDto = new LibraryStatV3(); libDto.IncludeInDashboard = library.IncludeInDashboard; libDto.IncludeInSearch = library.IncludeInSearch; libDto.LastScanned = library.LastScanned; libDto.NumberOfFolders = library.Folders.Count; libDto.FileTypes = library.LibraryFileTypes.Select(s => s.FileTypeGroup).Distinct().ToList(); libDto.UsingExcludePatterns = library.LibraryExcludePatterns.Any(p => !string.IsNullOrEmpty(p.Pattern)); libDto.UsingFolderWatching = library.FolderWatchMode != FolderWatchMode.Off; // TODO: Remove this once we finish the File Watcher rework libDto.FolderWatchMode = library.FolderWatchMode; libDto.FolderWatchMethod = serverSettings.EnableFolderWatching ? library.FolderWatchState.WatchMethod(library.FolderWatchMode) : FolderWatchMethod.Off; libDto.CreateCollectionsFromMetadata = library.ManageCollections; libDto.CreateReadingListsFromMetadata = library.ManageReadingLists; libDto.EnabledMetadata = library.EnableMetadata; libDto.LibraryType = library.Type; libDto.LastScanDurationMs = library.ScanInfo.Normal?.DurationMs; libDto.LastScanFolderCount = library.ScanInfo.Normal?.WalkedFolderCount; libDto.LastScanListMs = library.ScanInfo.Normal?.ListMs; libDto.LastScanWalkMs = library.ScanInfo.Normal?.WalkMs; libDto.LastScanParseMs = library.ScanInfo.Normal?.ParseMs; libDto.MultiFileEpubChapterCount = multiFileEpubChapters.GetValueOrDefault(library.Id); dto.Libraries.Add(libDto); } #endregion #region Users // Create a dictionary mapping user IDs to the libraries they have access to var userLibraryAccess = allLibraries .SelectMany(l => l.AppUsers.Select(appUser => new { l, appUser.Id })) .GroupBy(x => x.Id) .ToDictionary(g => g.Key, g => g.Select(x => x.l).ToList()); dto.Users ??= []; var allUsers = await _unitOfWork.UserRepository.GetAllUsersAsync(AppUserIncludes.UserPreferences | AppUserIncludes.ReadingLists | AppUserIncludes.Bookmarks | AppUserIncludes.Collections | AppUserIncludes.Devices | AppUserIncludes.Ratings | AppUserIncludes.SmartFilters | AppUserIncludes.WantToRead, false); var lastReadTimes = await _context.AppUserProgresses .GroupBy(p => p.AppUserId) .Select(g => new { AppUserId = g.Key, LastReadTime = g.Max(p => p.LastModifiedUtc) }) .ToDictionaryAsync(x => x.AppUserId, x => x.LastReadTime); foreach (var user in allUsers) { var userDto = new UserStatV3 { HasMALToken = !string.IsNullOrEmpty(user.ScrobbleProviders[ScrobbleProvider.Mal].AuthenticationToken), HasAniListToken = !string.IsNullOrEmpty(user.ScrobbleProviders[ScrobbleProvider.AniList].AuthenticationToken), AgeRestriction = new AgeRestriction() { AgeRating = user.AgeRestriction, IncludeUnknowns = user.AgeRestrictionIncludeUnknowns }, Locale = user.UserPreferences.Locale, Roles = [.. _userManager.GetRolesAsync(user).Result], LastLogin = user.LastActiveUtc, HasValidEmail = user.Email != null && _emailService.IsValidEmail(user.Email), IsEmailConfirmed = user.EmailConfirmed, ActiveTheme = user.UserPreferences.Theme.Name, CollectionsCreatedCount = user.Collections.Count, ReadingListsCreatedCount = user.ReadingLists.Count, LastReadTime = lastReadTimes.GetValueOrDefault(user.Id), DevicePlatforms = user.Devices.Select(d => d.Platform).ToList(), SeriesBookmarksCreatedCount = user.Bookmarks.Count, SmartFilterCreatedCount = user.SmartFilters.Count, IsSharingReviews = user.UserPreferences.SocialPreferences.ShareReviews, IsSharingProfile = user.UserPreferences.SocialPreferences.ShareProfile, IsSharingAnnotations = user.UserPreferences.SocialPreferences.ShareAnnotations, WantToReadSeriesCount = user.WantToRead.Count, IdentityProvider = user.IdentityProvider }; // Social Profile var activityData = await _context.AppUserReadingSession .Where(s => s.AppUserId == user.Id) .SelectMany(s => s.ActivityData) .Select(ad => new { ad.PagesRead, ad.WordsRead, ad.StartTime, ad.EndTime }) .ToListAsync(); userDto.TotalPagesRead = activityData.Sum(ad => ad.PagesRead); userDto.TotalWordsRead = activityData.Sum(ad => ad.WordsRead); userDto.TotalSecondsRead = activityData .Where(ad => ad.EndTime.HasValue) .Sum(ad => (long)(ad.EndTime!.Value - ad.StartTime).TotalSeconds); // Since social profiles require some sort of way to identify the user, let's use userId + installId. These can be disassociated, but should be stable enough userDto.UserId = $"{serverSettings.InstallId}_{user.Id}"; if (allLibraries.Count > 0 && userLibraryAccess.TryGetValue(user.Id, out var accessibleLibraries)) { userDto.PercentageOfLibrariesHasAccess = (1f * accessibleLibraries.Count) / allLibraries.Count; } else { userDto.PercentageOfLibrariesHasAccess = 0; } dto.Users.Add(userDto); } #endregion return dto; } private async Task OpenRandomFile(ServerInfoV3Dto dto) { // Skip this if we've sent enough var samplesTaken = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).StatsApiHits; if (samplesTaken > 2) { dto.TimeToOpeCbzMs = null; dto.TimeToOpenCbzPages = null; return; } var random = new Random(); List extensions = [".cbz", ".zip"]; // Count the total number of files that match the criteria var count = await _context.MangaFile.AsNoTracking() .Where(r => r.Extension != null && extensions.Contains(r.Extension)) .CountAsync(); if (count == 0) { dto.TimeToOpeCbzMs = 0; dto.TimeToOpenCbzPages = 0; return; } // Generate a random skip value var skip = random.Next(count); // Fetch the random file var randomFile = await _context.MangaFile.AsNoTracking() .Where(r => r.Extension != null && extensions.Contains(r.Extension)) .Skip(skip) .Take(1) .FirstAsync(); var sw = Stopwatch.StartNew(); await _cacheService.Ensure(randomFile.ChapterId); var time = sw.ElapsedMilliseconds; sw.Stop(); dto.TimeToOpeCbzMs = time; dto.TimeToOpenCbzPages = randomFile.Pages; } }