diff --git a/README.md b/README.md index ca0dfd4..6815a4e 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,17 @@ languages without changing Jellyfin's stored metadata. The plugin is built for the Jellyfin 10.11.x stable series on Linux x64 and is currently early beta software. +## Upgrading to 0.2.5 or 0.3.4 + +Existing plugin data is upgraded automatically. Make fresh database exports after +upgrading: older translation-database exports cannot be imported by these versions. +User-settings-only exports keep their existing format. Obsolete text-based rules +are no longer supported; current classification rules are unaffected. + +The first missing-data scan records the new per-language and per-provider fetch +state, so it may fetch more than subsequent scans. Later scans retry only missing +data when due and repair downloaded artwork without refetching its metadata. + ## What It Does Multilang can fetch and display metadata and artwork in chosen languages for: @@ -87,9 +98,12 @@ the UI. Some requests can be slow because Multilang may need to ask Jellyfin for a larger list than the UI will finally display. This is needed so the plugin can translate, filter, sort, and slice the list using the user's own rules. -Multilang caches slow transformed responses for a configurable time and also -uses a very short burst cache for repeated requests during navigation. Cache -entries are invalidated when affected items are refreshed. +Multilang caches Jellyfin's responses for a configurable time, sharing the fetched +list across pages and sort orders. Current translations and user rules are applied +each time that list is used. Simultaneous requests share an in-progress fetch, and +a short burst cache also handles fast repeated requests. Library, permission and +watched-state changes invalidate affected caches. Movie and show libraries can +optionally be fetched in advance when a user becomes active. ## Original-Language Support diff --git a/src/Jellyfin.Plugin.Multilang/Api/MultilangController.cs b/src/Jellyfin.Plugin.Multilang/Api/MultilangController.cs index e844412..35f02fa 100644 --- a/src/Jellyfin.Plugin.Multilang/Api/MultilangController.cs +++ b/src/Jellyfin.Plugin.Multilang/Api/MultilangController.cs @@ -1,4 +1,6 @@ using System.Reflection; +using Jellyfin.Data; +using Jellyfin.Database.Implementations.Enums; using System.Diagnostics; using System.Globalization; using System.Text.Json; @@ -27,15 +29,6 @@ namespace Jellyfin.Plugin.Multilang.Api; [Route("Multilang")] public sealed class MultilangController : ControllerBase { - private static readonly string[] RequirementFieldOrder = - [ - "original_language", - "spoken_languages", - "audio_language", - "origin_countries", - "production_countries" - ]; - private readonly TranslationStore _store; private readonly ISessionManager _sessionManager; private readonly IAuthorizationContext _authorizationContext; @@ -108,10 +101,18 @@ public sealed class MultilangController : ControllerBase => ServeEmbedded("Jellyfin.Plugin.Multilang.Configuration.shared.css", "text/css"); [HttpGet("Assets/{**path}")] - public IActionResult Asset(string path) - => _assetStorage.TryResolveLocalAsset(path, out var fullPath, out var contentType) - ? PhysicalFile(fullPath, contentType) - : NotFound(); + public async Task Asset(string path) + { + var userId = await GetUserIdAsync().ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(userId)) + return Unauthorized(new { Error = "NotAuthenticated" }); + if (!Guid.TryParse(path.Split('/')[0], out var itemId) || !CanAccessItem(itemId, userId) || + !_store.IsAssetReferenced(itemId.ToString("N"), TranslationStore.LocalAssetUrlPrefix + path) || + !_assetStorage.TryResolveLocalAsset(path, out var fullPath, out var contentType)) + return NotFound(); + Response.Headers.CacheControl = "private, no-cache"; + return PhysicalFile(fullPath, contentType); + } [HttpGet("Providers")] public IActionResult Providers() @@ -214,10 +215,9 @@ public sealed class MultilangController : ControllerBase return Unauthorized(new { Error = "NotAuthenticated" }); var allowed = (Plugin.Instance?.Configuration?.Languages ?? []).ToHashSet(StringComparer.OrdinalIgnoreCase); - request.Categories = NormalizeCategories(request.Categories, allowed); - request.FallbackFieldActions = NormalizeActionLists(request.FallbackFieldActions, allowed); + UserRulesNormalizer.Normalize(request, allowed); _store.SaveUserRules(userId, request); - _itemsProxyCache.ClearAll(); + _itemsProxyCache.ClearUser(userId); return Ok(request); } @@ -265,7 +265,11 @@ public sealed class MultilangController : ControllerBase if (!TryParseItemId(itemId, out _, out var itemGuid)) return BadRequest(new { Error = "InvalidItemId" }); - await _refreshService.RefreshItemAsync(itemGuid, includeChildren, cancellationToken).ConfigureAwait(false); + var user = _userManager.GetUserById(Guid.Parse(userId)); + if (user is null || !CanAccessItem(itemGuid, userId)) + return NotFound(); + + await _refreshService.RefreshItemAsync(itemGuid, includeChildren, cancellationToken, user).ConfigureAwait(false); return Ok(new { ItemId = itemGuid.ToString("N"), IncludeChildren = includeChildren }); } @@ -283,8 +287,13 @@ public sealed class MultilangController : ControllerBase if (string.IsNullOrWhiteSpace(userId)) return Unauthorized(new { Error = "NotAuthenticated" }); + var user = _userManager.GetUserById(Guid.Parse(userId)); + if (user is null || user.HasPermission(PermissionKind.IsDisabled)) + return Unauthorized(new { Error = "NotAuthenticated" }); + // Jellyfin's UpdatePolicyAsync does not publish a user-update event. + var generation = _itemsProxyCache.ObserveUserPolicy(userId, JsonSerializer.Serialize(_userManager.GetUserDto(user).Policy)); var rules = GetRulesOrDefault(userId); - var proxyRequest = ItemsProxyRequestBuilder.Build(Request, url, token, userId, rules.Enabled); + var proxyRequest = ItemsProxyRequestBuilder.Build(Request, url, userId, rules.Enabled); if (proxyRequest is null) return BadRequest(new { Error = "InvalidUrl" }); @@ -304,6 +313,7 @@ public sealed class MultilangController : ControllerBase if (cacheAllowed && _itemsProxyCache.TryGet(proxyRequest.CacheKey, cacheTtl, out var cached)) { + var transformed = _itemsProxyTransformer.TransformItemsResponse(cached.Body, rules, proxyRequest.Controls); totalSw.Stop(); _itemsProxyCache.RecordRequest( proxyRequest.NormalizedUrlForCache, @@ -313,9 +323,9 @@ public sealed class MultilangController : ControllerBase itemCount: cached.ItemCount, totalMs: totalSw.ElapsedMilliseconds, upstreamMs: 0, - transformMs: 0, + transformMs: totalSw.ElapsedMilliseconds, sizeBytes: cached.SizeBytes); - return Content(cached.Body, cached.ContentType); + return Content(transformed.Body, cached.ContentType); } if (rules.Enabled && proxyRequest.IsGenresRequest && _itemsProxyTransformer.TryBuildGenresResponse(proxyRequest.GenreMedia, proxyRequest.Controls.ClientLocale, out var genresBody)) @@ -337,11 +347,13 @@ public sealed class MultilangController : ControllerBase var upstreamSw = Stopwatch.StartNew(); var upstream = await _itemsProxyRequestCoalescer.GetOrFetchAsync( - proxyRequest.NormalizedUrlForCache, + generation + "|" + proxyRequest.NormalizedUrlForCache, async sharedCancellationToken => { - var http = _httpClientFactory.CreateClient(); - using var request = new HttpRequestMessage(HttpMethod.Get, proxyRequest.Upstream); + var http = _httpClientFactory.CreateClient("Multilang.Jellyfin"); + var localUri = new Uri(_itemsProxyPrecacheService.LocalApiUri, proxyRequest.Upstream.PathAndQuery); + using var request = new HttpRequestMessage(HttpMethod.Get, localUri); + request.Headers.TryAddWithoutValidation("Authorization", $"MediaBrowser Token=\"{token}\""); request.Headers.TryAddWithoutValidation("X-Emby-Token", token); request.Headers.TryAddWithoutValidation("X-MediaBrowser-Token", token); using var response = await http.SendAsync(request, sharedCancellationToken).ConfigureAwait(false); @@ -379,12 +391,13 @@ public sealed class MultilangController : ControllerBase proxyRequest.CacheKey, userId, proxyRequest.NormalizedUrlForCache, - responseItemIds, - body, + responseItemIds.Length, + upstream.Response.Body, contentType, totalSw.ElapsedMilliseconds, cfg.ItemsProxyCacheMaxMiB * 1024L * 1024L, - cacheTtl); + cacheTtl, + generation); } if (cacheAllowed && @@ -396,11 +409,13 @@ public sealed class MultilangController : ControllerBase proxyRequest.CacheKey, userId, proxyRequest.NormalizedUrlForCache, - responseItemIds, - body, + responseItemIds.Length, + upstream.Response.Body, contentType, totalSw.ElapsedMilliseconds, - cfg.ItemsProxyCacheMaxMiB * 1024L * 1024L); + cfg.ItemsProxyCacheMaxMiB * 1024L * 1024L, + generation, + cacheTtl); } _itemsProxyCache.RecordRequest( @@ -423,19 +438,10 @@ public sealed class MultilangController : ControllerBase }; } - private sealed record DebugResolutionAttempt( - string Action, - string Source, - string LookupKey, - bool HasValue, - bool Chosen, - string? Value, - string Reason); - private sealed record DebugTextResolution( string Field, string[] Actions, - DebugResolutionAttempt[] Attempts, + ItemsProxyResolutionAttempt[] Attempts, ItemsProxyResolvedText Result, string SelectedAction, string Reason); @@ -443,7 +449,7 @@ public sealed class MultilangController : ControllerBase private sealed record DebugAssetResolution( string Kind, string[] Actions, - DebugResolutionAttempt[] Attempts, + ItemsProxyResolutionAttempt[] Attempts, ItemsProxyResolvedAsset Result, string SelectedAction, string Reason); @@ -514,6 +520,7 @@ public sealed class MultilangController : ControllerBase if (downloadedAssets && !translationsDatabase) return BadRequest(new { Error = "AssetsRequireDatabase" }); + using var maintenance = await _store.EnterMaintenanceAsync(HttpContext.RequestAborted).ConfigureAwait(false); var bytes = _backupService.Export( new BackupExportOptions(pluginSettings, userSettings, translationsDatabase, downloadedAssets), Plugin.Instance?.Configuration ?? new PluginConfiguration()); @@ -543,6 +550,7 @@ public sealed class MultilangController : ControllerBase try { await using var stream = file.OpenReadStream(); + using var maintenance = await _store.EnterMaintenanceAsync(HttpContext.RequestAborted).ConfigureAwait(false); var result = _backupService.ImportAdmin( stream, options, @@ -566,6 +574,7 @@ public sealed class MultilangController : ControllerBase if (request.Confirm != true) return BadRequest(new { Error = "ConfirmationRequired" }); + using var maintenance = await _store.EnterMaintenanceAsync(HttpContext.RequestAborted).ConfigureAwait(false); Plugin.Instance?.UpdateConfiguration(new PluginConfiguration()); _store.ResetAll(); _itemsProxyCache.ClearAll(); @@ -624,7 +633,7 @@ public sealed class MultilangController : ControllerBase await using var stream = file.OpenReadStream(); var result = _backupService.ImportUser(stream, userId, form.TryGetValue("sourceUserId", out var sourceUserId) ? sourceUserId.ToString() : null); if (result.UserSettingsImported > 0) - _itemsProxyCache.ClearAll(); + _itemsProxyCache.ClearUser(userId); return Ok(result); } catch (InvalidOperationException ex) @@ -640,9 +649,12 @@ public sealed class MultilangController : ControllerBase if (string.IsNullOrWhiteSpace(userId)) return Unauthorized(new { Error = "NotAuthenticated" }); - if (!TryParseItemId(itemId, out var itemId32, out _)) + if (!TryParseItemId(itemId, out var itemId32, out var itemGuid)) return BadRequest(new { Error = "InvalidItemId" }); + if (!CanAccessItem(itemGuid, userId)) + return NotFound(); + return Ok(new { ItemId = itemId32, @@ -672,7 +684,7 @@ public sealed class MultilangController : ControllerBase if (normalizedUserId is null) return BadRequest(new { Error = "InvalidUserId" }); - return DebugResolveForUser(itemId, normalizedUserId); + return DebugResolveForUser(itemId, normalizedUserId, administrator: true); } [HttpGet("DebugResolve/{itemId}")] @@ -685,11 +697,14 @@ public sealed class MultilangController : ControllerBase return DebugResolveForUser(itemId, userId); } - private IActionResult DebugResolveForUser(string itemId, string userId) + private IActionResult DebugResolveForUser(string itemId, string userId, bool administrator = false) { - if (!TryParseItemId(itemId, out var itemId32, out _)) + if (!TryParseItemId(itemId, out var itemId32, out var itemGuid)) return BadRequest(new { Error = "InvalidItemId" }); + if (!administrator && !CanAccessItem(itemGuid, userId)) + return NotFound(); + var facts = _store.GetFacts(itemId32); if (facts is null) return NotFound(new { Error = "FactsMissing" }); @@ -808,128 +823,27 @@ public sealed class MultilangController : ControllerBase } private static DebugTextResolution TraceResolveField( - string field, - IReadOnlyList actions, - FactsData facts, + string field, IReadOnlyList actions, FactsData facts, Dictionary>? byLang) { - var attempts = new List(); - foreach (var action in actions) - { - if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)) - { - var result = new ItemsProxyResolvedText(false, null); - attempts.Add(new DebugResolutionAttempt(action, JellyfinAction, string.Empty, true, true, null, "Use Jellyfin value")); - return new DebugTextResolution(field, actions.ToArray(), attempts.ToArray(), result, JellyfinAction, "Jellyfin action reached"); - } - - if (field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) && - action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase)) - { - var hasOriginalTitle = !string.IsNullOrWhiteSpace(facts.OriginalTitle); - attempts.Add(new DebugResolutionAttempt(action, "OriginalTitle", "facts.original_title", hasOriginalTitle, hasOriginalTitle, facts.OriginalTitle, hasOriginalTitle ? "Original title found" : "Original title missing")); - if (hasOriginalTitle) - { - var result = new ItemsProxyResolvedText(true, facts.OriginalTitle); - return new DebugTextResolution(field, actions.ToArray(), attempts.ToArray(), result, action, "Original title found"); - } - - continue; - } - - if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase)) - { - var originalValue = ItemsProxyTransformer.GetTranslatedField(byLang, OriginalAction, field); - var hasOriginalValue = !string.IsNullOrWhiteSpace(originalValue); - attempts.Add(new DebugResolutionAttempt(action, "Translation", "Original/" + field, hasOriginalValue, hasOriginalValue, originalValue, hasOriginalValue ? "Original-language translation found" : "Original-language translation missing")); - if (hasOriginalValue) - { - var result = new ItemsProxyResolvedText(true, originalValue); - return new DebugTextResolution(field, actions.ToArray(), attempts.ToArray(), result, action, "Original-language translation found"); - } - - continue; - } - - if (!action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)) - { - attempts.Add(new DebugResolutionAttempt(action, "Unknown", string.Empty, false, false, null, "Unsupported action token")); - continue; - } - - var lang = action[LanguagePrefix.Length..]; - var value = ItemsProxyTransformer.GetTranslatedField(byLang, lang, field); - var hasValue = !string.IsNullOrWhiteSpace(value); - attempts.Add(new DebugResolutionAttempt(action, "Translation", lang + "/" + field, hasValue, hasValue, value, hasValue ? "Translation found" : "Translation missing or empty")); - if (hasValue) - { - var result = new ItemsProxyResolvedText(true, value); - return new DebugTextResolution(field, actions.ToArray(), attempts.ToArray(), result, action, "Translation found"); - } - } - - var fallback = field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) - ? new ItemsProxyResolvedText(false, null) - : new ItemsProxyResolvedText(true, string.Empty); - var reason = field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) - ? "No action had data; title falls back to Jellyfin" - : "No action had data; field is cleared"; - return new DebugTextResolution(field, actions.ToArray(), attempts.ToArray(), fallback, field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) ? JellyfinAction : ClearAction, reason); + var attempts = new List(); + var result = ItemsProxyTransformer.ResolveField(field, actions, facts, byLang, attempts.Add); + var chosen = attempts.FirstOrDefault(attempt => attempt.Chosen); + return new(field, actions.ToArray(), attempts.ToArray(), result, + chosen?.Action ?? (result.Change ? ClearAction : JellyfinAction), + chosen?.Reason ?? (result.Change ? "No action had data; field is cleared" : "No action had data; use Jellyfin")); } private static DebugAssetResolution TraceResolveAsset( - string kind, - IReadOnlyList actions, + string kind, IReadOnlyList actions, Dictionary>? byKind) { - var attempts = new List(); - foreach (var action in actions) - { - if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)) - { - var result = new ItemsProxyResolvedAsset(false, null); - attempts.Add(new DebugResolutionAttempt(action, JellyfinAction, string.Empty, true, true, null, "Use Jellyfin image")); - return new DebugAssetResolution(kind, actions.ToArray(), attempts.ToArray(), result, JellyfinAction, "Jellyfin action reached"); - } - - if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase)) - { - var originalValue = ItemsProxyTransformer.GetAsset(byKind, kind, OriginalAction); - var hasOriginalValue = !string.IsNullOrWhiteSpace(originalValue); - attempts.Add(new DebugResolutionAttempt(action, "Asset", kind + "/Original", hasOriginalValue, hasOriginalValue, originalValue, hasOriginalValue ? "Original-language asset found" : "Original-language asset missing")); - if (hasOriginalValue) - { - var result = new ItemsProxyResolvedAsset(true, originalValue); - return new DebugAssetResolution(kind, actions.ToArray(), attempts.ToArray(), result, action, "Original-language asset found"); - } - - continue; - } - - if (!action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)) - { - attempts.Add(new DebugResolutionAttempt(action, "Unknown", string.Empty, false, false, null, "Unsupported action token")); - continue; - } - - var lang = action[LanguagePrefix.Length..]; - var value = ItemsProxyTransformer.GetAsset(byKind, kind, lang); - var hasValue = !string.IsNullOrWhiteSpace(value); - attempts.Add(new DebugResolutionAttempt(action, "Asset", kind + "/" + lang, hasValue, hasValue, value, hasValue ? "Asset found" : "Asset missing")); - if (hasValue) - { - var result = new ItemsProxyResolvedAsset(true, value); - return new DebugAssetResolution(kind, actions.ToArray(), attempts.ToArray(), result, action, "Asset found"); - } - } - - var fallback = kind.Equals(PosterKind, StringComparison.OrdinalIgnoreCase) - ? new ItemsProxyResolvedAsset(false, null) - : new ItemsProxyResolvedAsset(true, string.Empty); - var reason = kind.Equals(PosterKind, StringComparison.OrdinalIgnoreCase) - ? "No action had data; poster falls back to Jellyfin" - : "No action had data; image is cleared"; - return new DebugAssetResolution(kind, actions.ToArray(), attempts.ToArray(), fallback, kind.Equals(PosterKind, StringComparison.OrdinalIgnoreCase) ? JellyfinAction : ClearAction, reason); + var attempts = new List(); + var result = ItemsProxyTransformer.ResolveAsset(kind, actions, byKind, attempts.Add); + var chosen = attempts.FirstOrDefault(attempt => attempt.Chosen); + return new(kind, actions.ToArray(), attempts.ToArray(), result, + chosen?.Action ?? (result.Change ? ClearAction : JellyfinAction), + chosen?.Reason ?? (result.Change ? "No action had data; image is cleared" : "No action had data; use Jellyfin")); } private static object[] BuildMissingDataReport( @@ -1024,238 +938,7 @@ public sealed class MultilangController : ControllerBase } private UserRulesDocument GetRulesOrDefault(string userId) - { - var existing = _store.GetUserRules(userId); - if (existing is not null) - { - existing.Categories ??= []; - existing.FallbackFieldActions = NormalizeActionLists(existing.FallbackFieldActions, (Plugin.Instance?.Configuration?.Languages ?? []).ToHashSet(StringComparer.OrdinalIgnoreCase)); - existing.Categories = NormalizeCategories(existing.Categories, (Plugin.Instance?.Configuration?.Languages ?? []).ToHashSet(StringComparer.OrdinalIgnoreCase)); - return existing; - } - - return new UserRulesDocument - { - Enabled = false, - SortLocale = "Auto", - TrustTmdbCollections = true, - FallbackFieldActions = DefaultActionLists(), - Categories = [] - }; - } - - private static UserCategoryRule[] NormalizeCategories(IEnumerable? categories, IReadOnlySet allowedLanguages) - => (categories ?? []) - .Select(category => new UserCategoryRule - { - Id = string.IsNullOrWhiteSpace(category.Id) ? Guid.NewGuid().ToString("N") : category.Id.Trim(), - Label = string.IsNullOrWhiteSpace(category.Label) ? "Category" : category.Label.Trim(), - CriteriaText = category.CriteriaText?.Trim() ?? string.Empty, - Requirements = NormalizeRequirements(category.Requirements), - MatchAllConditions = category.MatchAllConditions, - Scopes = (category.Scopes ?? []) - .Where(scope => scope is "M" or "S" or "C") - .Distinct(StringComparer.OrdinalIgnoreCase) - .DefaultIfEmpty("M") - .ToArray(), - FieldActions = NormalizeFieldActions(category.FieldActions, allowedLanguages), - FieldActionLists = NormalizeActionLists(category.FieldActionLists, allowedLanguages, category.FieldActions) - }) - .ToArray(); - - private static UserCategoryRequirement[] NormalizeRequirements(IEnumerable? requirements) - => (requirements ?? []) - .Select(requirement => - { - var fields = NormalizeRequirementFields(requirement.Fields, requirement.Field); - return new UserCategoryRequirement - { - Fields = fields, - Field = fields.FirstOrDefault() ?? string.Empty, - UseFieldOr = requirement.UseFieldOr, - Relation = NormalizeRequirementRelation(requirement.Relation), - UseOr = requirement.UseOr, - Values = (requirement.Values ?? []) - .Where(v => !string.IsNullOrWhiteSpace(v)) - .Select(v => v.Trim()) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray() - }; - }) - .Where(requirement => - requirement.Fields.Length > 0 && - requirement.Relation.Length > 0 && - requirement.Values.Length > 0) - .ToArray(); - - private static string[] NormalizeRequirementFields(IEnumerable? fields, string? legacyField) - { - var requested = (fields ?? []) - .Append(legacyField ?? string.Empty) - .Select(NormalizeRequirementField) - .Where(field => field.Length > 0) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToHashSet(StringComparer.OrdinalIgnoreCase); - - var result = new List(); - string? kind = null; - foreach (var field in RequirementFieldOrder) - { - if (!requested.Contains(field)) - continue; - - var currentKind = IsCountryRequirementField(field) ? "country" : "language"; - kind ??= currentKind; - if (currentKind == kind) - result.Add(field); - } - - return result.ToArray(); - } - - private static string NormalizeRequirementField(string? field) - { - var normalized = (field ?? string.Empty).Trim().ToLowerInvariant(); - return normalized is "original_language" or "spoken_languages" or "audio_language" or "origin_countries" or "production_countries" - ? normalized - : string.Empty; - } - - private static bool IsCountryRequirementField(string field) - => field.Equals("origin_countries", StringComparison.OrdinalIgnoreCase) || - field.Equals("production_countries", StringComparison.OrdinalIgnoreCase); - - private static string NormalizeRequirementRelation(string? relation) - { - var normalized = (relation ?? string.Empty).Trim().ToLowerInvariant(); - return normalized is "is" or "is_not" or "contains" or "not_contains" - ? normalized - : string.Empty; - } - - private static Dictionary NormalizeActionLists( - IReadOnlyDictionary? actions, - IReadOnlySet allowedLanguages, - IReadOnlyDictionary? legacyActions = null) - { - if (legacyActions is not null && LooksLikeDefaultActionLists(actions) && HasMeaningfulLegacyActions(legacyActions)) - actions = null; - - var result = DefaultActionLists(); - foreach (var field in result.Keys.ToArray()) - { - var raw = actions is not null && actions.TryGetValue(field, out var configured) - ? configured - : LegacyActionToList(field, legacyActions); - result[field] = NormalizeActionList(field, raw, allowedLanguages); - } - - return result; - } - - private static bool LooksLikeDefaultActionLists(IReadOnlyDictionary? actions) - => actions is null || - DefaultActionLists().Keys.All(field => - actions.TryGetValue(field, out var list) && - list.Length == 1 && - list[0].Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)); - - private static bool HasMeaningfulLegacyActions(IReadOnlyDictionary legacyActions) - => legacyActions.Values.Any(action => - !string.IsNullOrWhiteSpace(action) && - !action.Equals(FallbackAction, StringComparison.OrdinalIgnoreCase) && - !action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)); - - private static string[] LegacyActionToList(string field, IReadOnlyDictionary? legacyActions) - { - if (legacyActions is null || !legacyActions.TryGetValue(field, out var action)) - return [JellyfinAction]; - if (string.IsNullOrWhiteSpace(action) || action.Equals(FallbackAction, StringComparison.OrdinalIgnoreCase)) - return [JellyfinAction]; - return [action]; - } - - private static Dictionary NormalizeFieldActions( - IReadOnlyDictionary? actions, - IReadOnlySet allowedLanguages) - { - var result = DefaultFieldActions(); - if (actions is null) - return result; - - foreach (var field in result.Keys.ToArray()) - { - if (!actions.TryGetValue(field, out var raw)) - continue; - - result[field] = NormalizeAction(field, raw, allowedLanguages); - } - - return result; - } - - private static string NormalizeAction(string field, string? raw, IReadOnlySet allowedLanguages) - { - var action = (raw ?? string.Empty).Trim(); - if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase) || - action.Equals(FallbackAction, StringComparison.OrdinalIgnoreCase)) - return action; - if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase)) - return OriginalAction; - if ((field is OverviewField or TaglineField or LogoKind) && - action.Equals(ClearAction, StringComparison.OrdinalIgnoreCase)) - return ClearAction; - if (action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)) - { - var lang = action[LanguagePrefix.Length..].Trim(); - var canonical = allowedLanguages.FirstOrDefault(l => l.Equals(lang, StringComparison.OrdinalIgnoreCase)); - if (!string.IsNullOrWhiteSpace(canonical)) - return LanguagePrefix + canonical; - } - - return FallbackAction; - } - - private static string[] NormalizeActionList(string field, IEnumerable? raw, IReadOnlySet allowedLanguages) - { - var result = new List(); - foreach (var action in raw ?? []) - { - var normalized = NormalizeActionToken(field, action, allowedLanguages); - if (normalized.Length == 0) - continue; - if (result.Contains(normalized, StringComparer.OrdinalIgnoreCase)) - continue; - result.Add(normalized); - } - - if (RequiresJellyfinFallback(field)) - { - if (!result.Contains(JellyfinAction, StringComparer.OrdinalIgnoreCase)) - result.Add(JellyfinAction); - } - - return result.ToArray(); - } - - private static string NormalizeActionToken(string field, string? raw, IReadOnlySet allowedLanguages) - { - var action = (raw ?? string.Empty).Trim(); - if (action.Equals("J", StringComparison.OrdinalIgnoreCase)) - return JellyfinAction; - if (action.Equals("O", StringComparison.OrdinalIgnoreCase)) - return OriginalAction; - if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)) - return JellyfinAction; - if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase)) - return OriginalAction; - - var lang = action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase) - ? action[LanguagePrefix.Length..].Trim() - : action; - var canonical = allowedLanguages.FirstOrDefault(l => l.Equals(lang, StringComparison.OrdinalIgnoreCase)); - return string.IsNullOrWhiteSpace(canonical) ? string.Empty : LanguagePrefix + canonical; - } + => _store.GetUserRules(userId) ?? new UserRulesDocument { Enabled = false }; private IActionResult ServeEmbedded(string resource, string contentType) { @@ -1291,10 +974,6 @@ public sealed class MultilangController : ControllerBase if (string.IsNullOrWhiteSpace(token)) return null; - var tokenUserId = await GetUserIdFromJellyfinApiAsync(token).ConfigureAwait(false); - if (!string.IsNullOrWhiteSpace(tokenUserId)) - return tokenUserId; - var remoteEndpoint = HttpContext.Connection.RemoteIpAddress?.ToString() ?? string.Empty; var deviceId = info?.DeviceId ?? string.Empty; var session = await _sessionManager.GetSessionByAuthenticationToken(token, deviceId, remoteEndpoint).ConfigureAwait(false); @@ -1315,54 +994,18 @@ public sealed class MultilangController : ControllerBase : null; } - private async Task GetUserIdFromJellyfinApiAsync(string token) - { - var baseUri = $"{Request.Scheme}://{Request.Host}{Request.PathBase}"; - var usersMe = new UriBuilder(new Uri(new Uri(baseUri), "/Users/Me")); - usersMe.Query = Microsoft.AspNetCore.WebUtilities.QueryHelpers.AddQueryString( - string.Empty, - "api_key", - token).TrimStart('?'); - - var http = _httpClientFactory.CreateClient(); - using var request = new HttpRequestMessage(HttpMethod.Get, usersMe.Uri); - request.Headers.TryAddWithoutValidation("X-Emby-Token", token); - request.Headers.TryAddWithoutValidation("X-MediaBrowser-Token", token); - using var response = await http.SendAsync(request, HttpContext.RequestAborted).ConfigureAwait(false); - if (!response.IsSuccessStatusCode) - return null; - - await using var stream = await response.Content.ReadAsStreamAsync(HttpContext.RequestAborted).ConfigureAwait(false); - using var doc = await JsonDocument.ParseAsync(stream, cancellationToken: HttpContext.RequestAborted).ConfigureAwait(false); - if (!doc.RootElement.TryGetProperty("Id", out var idProperty)) - return null; - - var raw = idProperty.GetString(); - return Guid.TryParse(raw, out var guid) - ? guid.ToString("N") - : raw?.Length == 32 && raw.All(Uri.IsHexDigit) ? raw.ToLowerInvariant() : null; - } - private async Task IsRequesterAdminAsync() { - var info = await _authorizationContext.GetAuthorizationInfo(HttpContext).ConfigureAwait(false); - if (info?.User is not null && TryGetIsAdmin(info.User, out var authInfoAdmin)) - return authInfoAdmin; - var userId = await GetUserIdAsync().ConfigureAwait(false); - if (string.IsNullOrWhiteSpace(userId) || !Guid.TryParseExact(userId, "N", out var guid)) - return false; + return Guid.TryParse(userId, out var guid) && + _userManager.GetUserById(guid)?.HasPermission(PermissionKind.IsAdministrator) == true; + } - var user = _userManager.GetUserById(guid); - if (user is null) - return false; - - if (TryGetIsAdmin(user, out var isAdmin)) - return isAdmin; - - var remoteEndpoint = HttpContext.Connection.RemoteIpAddress?.ToString() ?? string.Empty; - var dto = _userManager.GetUserDto(user, remoteEndpoint); - return dto.Policy?.IsAdministrator == true; + private bool CanAccessItem(Guid itemId, string userId) + { + var user = _userManager.GetUserById(Guid.Parse(userId)); + return user is not null && !user.HasPermission(PermissionKind.IsDisabled) && + _libraryManager.GetItemById(itemId)?.IsVisibleStandalone(user) == true; } private static bool FormBool(IFormCollection form, string key, bool fallback) @@ -1375,28 +1018,6 @@ public sealed class MultilangController : ControllerBase value.Equals("on", StringComparison.OrdinalIgnoreCase); } - private static bool TryGetIsAdmin(object user, out bool isAdmin) - { - isAdmin = false; - - var policy = user.GetType().GetProperty("Policy")?.GetValue(user); - var policyAdmin = policy?.GetType().GetProperty("IsAdministrator")?.GetValue(policy); - if (policyAdmin is bool isPolicyAdmin) - { - isAdmin = isPolicyAdmin; - return true; - } - - var directAdmin = user.GetType().GetProperty("IsAdministrator")?.GetValue(user); - if (directAdmin is bool isDirectAdmin) - { - isAdmin = isDirectAdmin; - return true; - } - - return false; - } - private string? GetTokenFromRequest() { var req = HttpContext?.Request; diff --git a/src/Jellyfin.Plugin.Multilang/Configuration/configPage.html b/src/Jellyfin.Plugin.Multilang/Configuration/configPage.html index e9c73cd..d5d100d 100644 --- a/src/Jellyfin.Plugin.Multilang/Configuration/configPage.html +++ b/src/Jellyfin.Plugin.Multilang/Configuration/configPage.html @@ -276,7 +276,7 @@ function renderJellyfinTitleLanguages() { var select = document.getElementById("ml-jellyfin-title-language"); var selected = M.prop(state.config, "JellyfinTitleLanguageFallback", "jellyfinTitleLanguageFallback", ""); - select.replaceChildren(); + M.replaceChildren(select); var defaultOption = make("option", "", "Jellyfin's default"); defaultOption.value = ""; select.appendChild(defaultOption); @@ -317,7 +317,7 @@ } function renderProviderInputs() { - providerInputs.replaceChildren(); + M.replaceChildren(providerInputs); state.providers.forEach(function (provider) { var id = providerId(provider); var keyField = providerKeyField(provider); @@ -371,7 +371,7 @@ .map(providerId) .filter(function (id) { return providerSupports(providerById(id), bucket) && orderFor(id, bucket) >= 0; }) .sort(function (a, b) { return orderFor(a, bucket) - orderFor(b, bucket); }); - list.replaceChildren(); + M.replaceChildren(list); ids.forEach(function (id) { list.appendChild(makeProviderChip(id)); }); } @@ -412,7 +412,7 @@ } function renderArticles() { - articleList.replaceChildren(); + M.replaceChildren(articleList); var configured = new Map(); (state.config.ArticleEntries || []).forEach(function (entry) { var lang = normalizeLang(entry.Language || entry.language); @@ -541,7 +541,7 @@ function renderCacheEntries(entries, error) { var root = document.getElementById("ml-cache-entries"); - root.replaceChildren(); + M.replaceChildren(root); if (error) { setDiagnosticsSummary("ml-cache-entries-summary", "Cached responses"); root.appendChild(make("div", "ml-error", error)); @@ -576,7 +576,7 @@ function renderProxyRequests(entries, error) { var root = document.getElementById("ml-proxy-requests"); - root.replaceChildren(); + M.replaceChildren(root); if (error) { setDiagnosticsSummary("ml-proxy-requests-summary", "Recent proxy requests"); root.appendChild(make("div", "ml-error", error)); @@ -674,9 +674,9 @@ var scanRoot = document.getElementById("ml-refresh-scan"); var queueRoot = document.getElementById("ml-refresh-queue"); var httpRoot = document.getElementById("ml-provider-http"); - scanRoot.replaceChildren(); - queueRoot.replaceChildren(); - httpRoot.replaceChildren(); + M.replaceChildren(scanRoot); + M.replaceChildren(queueRoot); + M.replaceChildren(httpRoot); if (error) { setDiagnosticsSummary("ml-refresh-scan-summary", "Latest scan"); @@ -728,18 +728,18 @@ var scanError = String(prop(scan, "Error", "error", "") || ""); if (scanError) scanRoot.appendChild(make("div", "ml-error", scanError)); - var active = prop(queue, "Active", "active", null); + var activeItems = queue.ActiveItems || []; var queued = prop(queue, "Queued", "queued", []) || []; var recent = prop(queue, "Recent", "recent", []) || []; setDiagnosticsSummary("ml-refresh-queue-summary", "Queue (" + Number(prop(queue, "QueuedCount", "queuedCount", queued.length) || 0) + " pending)"); - if (active) { + activeItems.forEach(function (active) { queueRoot.appendChild(makeDiagnosticRow("Active item: " + String(prop(active, "ItemId", "itemId", "")), [ "running now", "source: " + String(prop(active, "SourceTier", "sourceTier", "")), "type: " + String(prop(active, "WorkClass", "workClass", "")), "refresh: " + String(prop(active, "JobType", "jobType", "")) ])); - } + }); if (queued.length === 0) { queueRoot.appendChild(make("div", "ml-muted", "No pending refresh items.")); } else { diff --git a/src/Jellyfin.Plugin.Multilang/Configuration/debugPage.html b/src/Jellyfin.Plugin.Multilang/Configuration/debugPage.html index f2b5e29..620b452 100644 --- a/src/Jellyfin.Plugin.Multilang/Configuration/debugPage.html +++ b/src/Jellyfin.Plugin.Multilang/Configuration/debugPage.html @@ -185,7 +185,7 @@ } function renderSummary(data) { - summary.replaceChildren(); + M.replaceChildren(summary); if (mode.value !== "resolved") return; var top = make("div", "ml-diagnostic-meta"); top.append( @@ -207,7 +207,7 @@ M.setStatus(status, "Loaded."); } catch (err) { lastJson = ""; - summary.replaceChildren(); + M.replaceChildren(summary); output.textContent = authHint(err); M.setStatus(status, err.message, true); } diff --git a/src/Jellyfin.Plugin.Multilang/Configuration/shared.js b/src/Jellyfin.Plugin.Multilang/Configuration/shared.js index 4f34c11..f8958ab 100644 --- a/src/Jellyfin.Plugin.Multilang/Configuration/shared.js +++ b/src/Jellyfin.Plugin.Multilang/Configuration/shared.js @@ -1,11 +1,4 @@ (function () { - if (!Element.prototype.replaceChildren) { - Element.prototype.replaceChildren = function () { - this.textContent = ""; - for (let index = 0; index < arguments.length; index++) this.appendChild(arguments[index]); - }; - } - const basePath = () => { const path = window.location.pathname || ""; const marker = "/web/"; @@ -21,23 +14,13 @@ } } - function headers() { - const qs = new URLSearchParams(location.search); - const t = token() || qs.get("token") || qs.get("api_key") || ""; - const h = { "Content-Type": "application/json" }; - if (t) { - h["X-Emby-Token"] = t; - h["X-MediaBrowser-Token"] = t; - } - return h; - } - function authHeaders(contentType) { const qs = new URLSearchParams(location.search); const t = token() || qs.get("token") || qs.get("api_key") || ""; const h = {}; if (contentType) h["Content-Type"] = contentType; if (t) { + h.Authorization = `MediaBrowser Token="${t}"`; h["X-Emby-Token"] = t; h["X-MediaBrowser-Token"] = t; } @@ -56,7 +39,7 @@ async function request(path, options) { const response = await fetch(`${basePath()}${path}`, { cache: "no-store", - headers: headers(), + headers: authHeaders("application/json"), ...options }); if (!response.ok) throw new Error(`HTTP ${response.status}`); @@ -111,41 +94,9 @@ URL.revokeObjectURL(url); } - function chip(label) { - const el = document.createElement("span"); - el.className = "ml-chip"; - el.textContent = label; - el.draggable = true; - el.dataset.value = label; - el.addEventListener("dragstart", (event) => { - event.dataTransfer.setData("text/plain", label); - event.dataTransfer.effectAllowed = "move"; - el.classList.add("ml-dragging"); - }); - el.addEventListener("dragend", () => el.classList.remove("ml-dragging")); - return el; - } - - function bucketValues(bucket) { - return [...bucket.querySelectorAll(".ml-chip")].map((el) => el.dataset.value || el.textContent.trim()).filter(Boolean); - } - - function renderBucket(bucket, values) { - bucket.replaceChildren(...values.map(chip)); - } - - function wireBucket(bucket, onDrop) { - bucket.addEventListener("dragover", (event) => { - event.preventDefault(); - bucket.classList.add("ml-over"); - }); - bucket.addEventListener("dragleave", () => bucket.classList.remove("ml-over")); - bucket.addEventListener("drop", (event) => { - event.preventDefault(); - bucket.classList.remove("ml-over"); - const value = event.dataTransfer.getData("text/plain"); - if (value) onDrop(value, bucket); - }); + function replaceChildren(parent, ...children) { + parent.textContent = ""; + parent.append(...children); } const staticData = { @@ -167,10 +118,7 @@ showModal, hideModals, downloadBlob, - chip, - bucketValues, - renderBucket, - wireBucket, + replaceChildren, staticData }; })(); diff --git a/src/Jellyfin.Plugin.Multilang/Configuration/userRulesPage.html b/src/Jellyfin.Plugin.Multilang/Configuration/userRulesPage.html index 7a5b590..84a12b4 100644 --- a/src/Jellyfin.Plugin.Multilang/Configuration/userRulesPage.html +++ b/src/Jellyfin.Plugin.Multilang/Configuration/userRulesPage.html @@ -109,8 +109,19 @@ var ISO3166 = M.staticData.iso3166; var LANG_REGION_COMBOS = M.staticData.langRegionCombos; + var listeners = []; + function listen(target, type, handler, capture) { + target.addEventListener(type, handler, capture); + listeners.push(() => target.removeEventListener(type, handler, capture)); + } + page._mlDispose = function () { + listeners.forEach(remove => remove()); + listeners = []; + cleanupActionDrag(); + }; + function $(id) { - return document.getElementById(id); + return page.querySelector("#" + id); } function make(tag, className, text) { @@ -144,18 +155,18 @@ } function categoryId(category) { - return category.Id || category.id || ""; + return category.Id; } function categoryLabel(category, index) { - var label = String(category.Label || category.label || "").trim(); + var label = String(category.Label || "").trim(); return label || categoryPlaceholder(index); } function categoryPlaceholder(index) { var used = {}; categories().forEach(function (category) { - var label = String(category.Label || category.label || "").trim().toLowerCase(); + var label = String(category.Label || "").trim().toLowerCase(); if (label) used[label] = true; }); var n = Math.max(1, index + 1); @@ -163,11 +174,10 @@ return "Category " + n; } - function newCategory(label, criteria) { + function newCategory(label) { return { Id: randomId(), Label: label || "", - CriteriaText: criteria || "", Requirements: [emptyRequirement()], MatchAllConditions: true, Scopes: ["M", "S", "C"], @@ -176,7 +186,7 @@ } function emptyRequirement() { - return { Field: "", Fields: [], UseFieldOr: false, Relation: "", UseOr: false, Values: [] }; + return { Fields: [], UseFieldOr: false, Relation: "", UseOr: false, Values: [] }; } function normalizeActionValue(fieldId, value) { @@ -189,13 +199,6 @@ return canonical ? "Language:" + canonical : ""; } - function legacyActionToList(fieldId, legacy) { - if (!legacy || legacy.toLowerCase() === "fallback") { - return ["Jellyfin"]; - } - return [legacy]; - } - function normalizeActionList(fieldId, values) { var field = fields.find(function (f) { return f.id === fieldId; }); var result = []; @@ -211,64 +214,31 @@ return result.length || !(field && field.lockedJ) ? result : ["Jellyfin"]; } - function normalizeActionMap(source, legacy) { - if (legacy && looksLikeDefaultActionMap(source) && hasMeaningfulLegacyActions(legacy)) { - source = null; - } + function normalizeActionMap(source) { var result = {}; fields.forEach(function (field) { - var values = source && (source[field.id] || source[field.label]); - if (!values && legacy) values = legacyActionToList(field.id, legacy[field.id] || legacy[field.label]); - result[field.id] = normalizeActionList(field.id, Array.isArray(values) ? values : values ? [values] : ["Jellyfin"]); + result[field.id] = normalizeActionList(field.id, source[field.id]); }); return result; } - function looksLikeDefaultActionMap(source) { - if (!source) return true; - return fields.every(function (field) { - var values = source[field.id] || source[field.label]; - return Array.isArray(values) && values.length === 1 && String(values[0]).toLowerCase() === "jellyfin"; - }); - } - - function hasMeaningfulLegacyActions(legacy) { - return Object.keys(legacy || {}).some(function (key) { - var action = String(legacy[key] || "").toLowerCase(); - return action && action !== "fallback" && action !== "jellyfin"; - }); - } - function normalizeRules() { - state.rules = state.rules || {}; - state.rules.Categories = (state.rules.Categories || []).filter(function (category) { - var id = categoryId(category).toLowerCase(); - var label = String(category.Label || category.label || "").trim().toLowerCase(); - var criteria = String(category.CriteriaText || category.criteriaText || "").trim(); - return !(id === "fallback" && label === "fallback" && criteria.length === 0); - }).map(function (category, index) { - category.Id = categoryId(category) || randomId(); - category.Label = String(category.Label || category.label || "").trim(); - category.CriteriaText = category.CriteriaText || category.criteriaText || ""; - category.Requirements = normalizeRequirements(category.Requirements || category.requirements); - category.MatchAllConditions = category.MatchAllConditions !== false && category.matchAllConditions !== false; - category.Scopes = category.Scopes || category.scopes || ["M", "S", "C"]; - category.FieldActionLists = normalizeActionMap(category.FieldActionLists || category.fieldActionLists, category.FieldActions || category.fieldActions); - return category; + state.rules.Categories.forEach(function (category) { + category.Requirements = normalizeRequirements(category.Requirements); + category.FieldActionLists = normalizeActionMap(category.FieldActionLists); }); - state.rules.FallbackFieldActions = normalizeActionMap(state.rules.FallbackFieldActions || state.rules.fallbackFieldActions, null); + state.rules.FallbackFieldActions = normalizeActionMap(state.rules.FallbackFieldActions); } function normalizeRequirements(requirements) { var normalized = (requirements || []).map(function (requirement) { - var fields = normalizeRequirementFields(requirement.Fields || requirement.fields, requirement.Field || requirement.field); + var fields = normalizeRequirementFields(requirement.Fields); return { - Field: fields[0] || "", Fields: fields, - UseFieldOr: requirement.UseFieldOr === true || requirement.useFieldOr === true, - Relation: String(requirement.Relation || requirement.relation || "").trim(), - UseOr: requirement.UseOr === true || requirement.useOr === true, - Values: orderedRequirementValues(fields[0] || "", requirement.Values || requirement.values || []) + UseFieldOr: requirement.UseFieldOr === true, + Relation: String(requirement.Relation || "").trim(), + UseOr: requirement.UseOr === true, + Values: orderedRequirementValues(fields[0] || "", requirement.Values || []) }; }); return normalized.length ? normalized : [emptyRequirement()]; @@ -294,14 +264,12 @@ return criteriaFields.find(function (item) { return item.id === id; }); } - function normalizeRequirementFields(values, legacy) { + function normalizeRequirementFields(values) { var raw = []; (values || []).forEach(function (value) { value = String(value || "").trim(); if (value) raw.push(value); }); - legacy = String(legacy || "").trim(); - if (legacy) raw.push(legacy); var wanted = {}; raw.forEach(function (value) { var id = value.toLowerCase(); @@ -319,7 +287,7 @@ } function requirementFields(requirement) { - return normalizeRequirementFields(requirement && (requirement.Fields || requirement.fields), requirement && (requirement.Field || requirement.field)); + return normalizeRequirementFields(requirement.Fields); } function criteriaLabel(id) { @@ -329,7 +297,7 @@ function renderSortLocales() { var select = $("ml-sort-locale"); - select.replaceChildren(option("Auto", "Auto", false)); + M.replaceChildren(select, option("Auto", "Auto", false)); state.allowed.forEach(function (lang) { select.appendChild(option(lang, lang, false)); }); @@ -363,7 +331,6 @@ function syncCategoryFromRow(row, category) { category.Label = row.querySelector("[data-category-label]").value; category.MatchAllConditions = row.querySelector("[data-category-match-all]").checked; - category.CriteriaText = categoryCriteriaText(category); category.Scopes = Array.prototype.slice.call(row.querySelectorAll("[data-category-scope]:checked")) .map(function (input) { return input.value; }); } @@ -376,7 +343,7 @@ } function addSelectOptions(select, items, selectedValue) { - select.replaceChildren(); + M.replaceChildren(select); items.forEach(function (item) { select.appendChild(option(item.label, item.id, item.id === selectedValue)); }); @@ -462,23 +429,6 @@ return values.join(joiner); } - function requirementText(requirement) { - if (!completeRequirement(requirement)) return ""; - var values = orderedRequirementValues(requirementFields(requirement)[0] || "", requirement.Values || []); - var right = values.length === 1 ? values[0] : "(" + values.join(requirement.UseOr ? " or " : " and ") + ")"; - var fields = requirementFields(requirement); - var left = fields.length === 1 ? fields[0] : "(" + fields.join(requirement.UseFieldOr ? " or " : " and ") + ")"; - return left + " " + requirement.Relation + " " + right; - } - - function categoryCriteriaText(category) { - var joiner = category.MatchAllConditions === false ? " or " : " and "; - return (category.Requirements || []) - .filter(completeRequirement) - .map(requirementText) - .join(joiner); - } - function makeRequirementRow(category, requirement, index) { var row = make("div", "ml-requirement-row"); var selectedFields = requirementFields(requirement); @@ -524,7 +474,7 @@ var current = requirement.Relation || ""; var fields = requirementFields(requirement); var multiple = fields.length > 1; - select.replaceChildren(option("Choose relation", "", !current)); + M.replaceChildren(select, option("Choose relation", "", !current)); if (fields.length) { relations.forEach(function (item) { select.appendChild(option(multiple ? item.pluralLabel : item.label, item.id, item.id === current)); @@ -569,7 +519,6 @@ fields = normalizeRequirementFields(fields, ""); var nextKind = fields.length ? fieldKind(fields[0]) : ""; requirement.Fields = fields; - requirement.Field = fields[0] || ""; button.textContent = fieldSummary(requirement); refreshFieldPanelOptions(panel, requirement); if (onChange) onChange(previousKind, nextKind); @@ -674,7 +623,6 @@ function cloneRequirements(requirements) { return normalizeRequirements(requirements).map(function (requirement) { return { - Field: requirement.Field, Fields: (requirement.Fields || []).slice(), UseFieldOr: requirement.UseFieldOr === true, Relation: requirement.Relation, @@ -688,7 +636,6 @@ return { Id: randomId(), Label: "", - CriteriaText: categoryCriteriaText(category), Requirements: cloneRequirements(category.Requirements), MatchAllConditions: category.MatchAllConditions !== false, Scopes: (category.Scopes || ["M", "S", "C"]).slice(), @@ -698,7 +645,7 @@ function renderCategories() { var root = $("ml-categories"); - root.replaceChildren(); + M.replaceChildren(root); categories().forEach(function (category, index) { var row = make("div", "ml-classification-rule"); row.dataset.categoryId = categoryId(category); @@ -809,7 +756,7 @@ function renderActionSource() { var root = $("ml-action-source"); - root.replaceChildren(); + M.replaceChildren(root); actionSourceValues().forEach(function (value) { root.appendChild(makeActionToken(value, false, true)); }); @@ -831,7 +778,7 @@ function renderActionMatrix() { renderActionSource(); var root = $("ml-action-matrix"); - root.replaceChildren(); + M.replaceChildren(root); actionRows().forEach(function (row) { var line = make("div", "ml-action-category-row"); if (row.fallback) line.classList.add("ml-action-category-row-fallback"); @@ -1068,7 +1015,7 @@ function clearCategoryActions(categoryId) { boxesForCategory(categoryId).forEach(function (box) { - box.replaceChildren(); + M.replaceChildren(box); if (actionFieldLocksJ(box.dataset.fieldId)) box.appendChild(makeActionToken("Jellyfin", true, false)); }); } @@ -1085,7 +1032,7 @@ return token.dataset.value !== "Jellyfin"; }); var token = makeActionToken(value, actionFieldLocksJ(box.dataset.fieldId) && value === "Jellyfin", false); - box.replaceChildren(); + M.replaceChildren(box); if (prepend) box.appendChild(token); existing.forEach(function (item) { box.appendChild(item); }); if (!prepend) box.appendChild(token); @@ -1128,7 +1075,6 @@ categories().forEach(function (category) { var complete = (category.Requirements || []).filter(completeRequirement); category.Requirements = complete.length ? complete : [emptyRequirement()]; - category.CriteriaText = complete.map(requirementText).join(" and "); }); return { Enabled: $("ml-enabled").checked, @@ -1140,12 +1086,10 @@ return { Id: category.Id, Label: categoryLabel(category, index), - CriteriaText: complete.map(requirementText).join(category.MatchAllConditions === false ? " or " : " and "), Requirements: complete, MatchAllConditions: category.MatchAllConditions !== false, Scopes: category.Scopes, - FieldActionLists: category.FieldActionLists, - FieldActions: {} + FieldActionLists: category.FieldActionLists }; }) }; @@ -1183,7 +1127,7 @@ $("ml-user-import-modal").hidden = !show; if (!show) { $("ml-user-import-choice").hidden = true; - $("ml-user-import-source").replaceChildren(); + M.replaceChildren($("ml-user-import-source")); } } @@ -1251,7 +1195,7 @@ } var select = $("ml-user-import-source"); - select.replaceChildren(); + M.replaceChildren(select); ids.forEach(function (id) { select.appendChild(option(id, id, false)); }); choice.hidden = false; M.setStatus(status, "No matching user ID was found. Choose which exported user settings to import.", true); @@ -1295,26 +1239,26 @@ $("ml-user-import-run").addEventListener("click", function () { runUserImport().catch(function (err) { M.setStatus(status, String(err), true); }); }); - document.addEventListener("click", function () { closeValuePanels(null); }); + listen(document, "click", function () { closeValuePanels(null); }); $("ml-add-category").addEventListener("click", function () { syncCategoriesFromDom(); updateActionStateFromBoxes(); - categories().push(newCategory("", "")); + categories().push(newCategory("")); renderAll(); }); - document.addEventListener("dragstart", function (event) { + listen(document, "dragstart", function (event) { if (event.target.closest && event.target.closest(".ml-action-token")) event.preventDefault(); }, true); page.addEventListener("pointerdown", beginActionDrag, true); - document.addEventListener("pointermove", moveActionDrag, true); - document.addEventListener("pointerup", endActionDrag, true); - document.addEventListener("pointercancel", function (event) { + listen(document, "pointermove", moveActionDrag, true); + listen(document, "pointerup", endActionDrag, true); + listen(document, "pointercancel", function (event) { if (!actionDrag || event.pointerId !== actionDrag.pointerId) return; restoreActionDrag(); cleanupActionDrag(); updateActionStateFromBoxes(); }, true); - document.addEventListener("viewshow", function (event) { + listen(document, "viewshow", function (event) { if (event.target && event.target.id === "ml-user") load(); }); } diff --git a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Cleanup.cs b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Cleanup.cs index 20c98ac..84f178d 100644 --- a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Cleanup.cs +++ b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Cleanup.cs @@ -42,13 +42,13 @@ public sealed partial class TranslationStore delete.CommandText = @" DELETE FROM facts WHERE item_id = $item_id; DELETE FROM translations WHERE item_id = $item_id; -DELETE FROM assets WHERE item_id = $item_id;"; +DELETE FROM assets WHERE item_id = $item_id; +DELETE FROM fetch_state WHERE item_id = $item_id;"; delete.Parameters.AddWithValue("$item_id", itemId); delete.ExecuteNonQuery(); } tx.Commit(); - CleanupUnreferencedAssetFiles(); } public LocalCleanupResult CleanupForConfiguration(IReadOnlySet liveItemIds, IEnumerable allowedLanguages, bool localAssetStorage) @@ -62,10 +62,18 @@ DELETE FROM assets WHERE item_id = $item_id;"; .Distinct(StringComparer.OrdinalIgnoreCase) .ToArray(); + using (var con = Open()) + { + con.Open(); + using var cmd = con.CreateCommand(); + var parameters = AddParams(cmd, allowed, "$lang"); + cmd.CommandText = $"DELETE FROM fetch_state WHERE source LIKE 'metadata:%' AND substr(source, 10) NOT IN ({parameters});"; + cmd.ExecuteNonQuery(); + } var translationsDeleted = DeleteRowsOutsideLanguages("translations", allowed); var genresDeleted = DeleteRowsOutsideLanguages("genres", allowed); var assetsDeleted = DeleteRowsOutsideLanguages("assets", allowed); - var modeRowsDeleted = localAssetStorage ? 0 : DeleteLocalAssetRows(resetMissingCheckedAt: true); + var modeRowsDeleted = localAssetStorage ? 0 : SwitchLocalAssetsToUrls(); var assetFilesDeleted = CleanupUnreferencedAssetFiles(); return new LocalCleanupResult(translationsDeleted, assetsDeleted + modeRowsDeleted, genresDeleted, assetFilesDeleted); } @@ -80,38 +88,22 @@ DELETE FROM assets WHERE item_id = $item_id;"; return cmd.ExecuteNonQuery(); } - private int DeleteLocalAssetRows(bool resetMissingCheckedAt) + private int SwitchLocalAssetsToUrls() { using var con = Open(); con.Open(); using var tx = con.BeginTransaction(); - var affected = new List(); - using (var select = con.CreateCommand()) - { - select.Transaction = tx; - select.CommandText = "SELECT DISTINCT item_id FROM assets WHERE path_low LIKE '/multilang/assets/%';"; - using var reader = select.ExecuteReader(); - while (reader.Read()) - affected.Add(reader.GetString(0)); - } - - int deleted; - using (var delete = con.CreateCommand()) - { - delete.Transaction = tx; - delete.CommandText = "DELETE FROM assets WHERE path_low LIKE '/multilang/assets/%';"; - deleted = delete.ExecuteNonQuery(); - } - - if (resetMissingCheckedAt && affected.Count > 0) - { - using var update = con.CreateCommand(); - update.Transaction = tx; - var idParams = AddParams(update, affected, "$item"); - update.CommandText = $"UPDATE facts SET missing_checked_at = 0 WHERE item_id IN ({idParams});"; - update.ExecuteNonQuery(); - } - + using var cmd = con.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = @"UPDATE assets SET path = source_url, path_low = lower(source_url) +WHERE path_low LIKE '/multilang/assets/%' AND source_url <> ''; +DELETE FROM fetch_state WHERE source LIKE 'artwork:%' AND item_id IN +(SELECT item_id FROM assets WHERE path_low LIKE '/multilang/assets/%'); +UPDATE facts SET missing_checked_at = 0 WHERE item_id IN +(SELECT item_id FROM assets WHERE path_low LIKE '/multilang/assets/%');"; + cmd.ExecuteNonQuery(); + cmd.CommandText = "DELETE FROM assets WHERE path_low LIKE '/multilang/assets/%';"; + var deleted = cmd.ExecuteNonQuery(); tx.Commit(); return deleted; } diff --git a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.ImportExport.cs b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.ImportExport.cs index 851de78..11f91bf 100644 --- a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.ImportExport.cs +++ b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.ImportExport.cs @@ -15,21 +15,34 @@ public sealed partial class TranslationStore destination.Open(); using (var schema = destination.CreateCommand()) { - schema.CommandText = SchemaSql; + schema.CommandText = SchemaSql + "\nPRAGMA user_version = 1;"; schema.ExecuteNonQuery(); } using var source = Open(); source.Open(); - CopyTable(source, destination, "facts", "item_id, tmdb_id, kind, original_title, original_language, original_language_all, origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, genre_tmdb_ids_json, missing_checked_at, full_checked_at"); - CopyTable(source, destination, "translations", "item_id, lang, field, text"); - CopyTable(source, destination, "assets", "item_id, lang, kind, path, path_low, updated_at"); - CopyTable(source, destination, "genres", "tmdb_id, media, lang, name, name_norm"); - CopyTable(source, destination, "scan_state", "id, last_scan_started"); + using var snapshot = source.BeginTransaction(); + CopyTable(snapshot, destination, "facts", "item_id, tmdb_id, kind, original_title, original_language, original_language_all, origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, genre_tmdb_ids_json, missing_checked_at, full_checked_at"); + CopyTable(snapshot, destination, "translations", "item_id, lang, field, text"); + CopyTable(snapshot, destination, "assets", "item_id, lang, kind, path, path_low, updated_at, source_url, provider"); + CopyTable(snapshot, destination, "fetch_state", "item_id, source, scope, checked_at, complete"); + CopyTable(snapshot, destination, "genres", "tmdb_id, media, lang, name, name_norm"); + CopyTable(snapshot, destination, "scan_state", "id, last_scan_started"); + } + + public static void ValidateTranslationBackup(string path) + { + using var connection = new SqliteConnection($"Data Source={path};Mode=ReadOnly"); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = "PRAGMA user_version;"; + if (Convert.ToInt32(command.ExecuteScalar()) != 1) + throw new InvalidOperationException("Unsupported translations backup version. Export a new backup using the current plugin."); } public ImportDatabaseResult ImportTranslationDatabase(string sourcePath, IReadOnlySet liveItemIds, IReadOnlySet allowedLanguages) { + ValidateTranslationBackup(sourcePath); var allowed = allowedLanguages.ToHashSet(StringComparer.OrdinalIgnoreCase); allowed.Add(MultilangConstants.OriginalAction); @@ -39,187 +52,62 @@ public sealed partial class TranslationStore source.Open(); using var tx = destination.BeginTransaction(); - var facts = ImportFacts(source, destination, tx, liveItemIds); - var translations = ImportTranslations(source, destination, tx, liveItemIds, allowed); - var assets = ImportAssets(source, destination, tx, liveItemIds, allowed); - var genres = ImportGenres(source, destination, tx, allowed); + bool Live(SqliteDataReader row) => liveItemIds.Contains(row.GetString(0)); + bool Translation(SqliteDataReader row) => Live(row) && allowed.Contains(row.GetString(1)); + var facts = ImportTable(source, tx, "facts", "item_id, tmdb_id, kind, original_title, original_language, original_language_all, origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, genre_tmdb_ids_json, missing_checked_at, full_checked_at", Live); + var translations = ImportTable(source, tx, "translations", "item_id, lang, field, text", Translation); + var assets = ImportTable(source, tx, "assets", "item_id, lang, kind, path, path_low, updated_at, source_url, provider", Translation); + var genres = ImportTable(source, tx, "genres", "tmdb_id, media, lang, name, name_norm", row => allowed.Contains(row.GetString(2))); + ImportTable(source, tx, "fetch_state", "item_id, source, scope, checked_at, complete", row => + Live(row) && (row.GetString(1).StartsWith("artwork:") || (row.GetString(1).StartsWith("metadata:", StringComparison.Ordinal) && allowed.Contains(row.GetString(1)["metadata:".Length..])))); tx.Commit(); return new ImportDatabaseResult(facts, translations, assets, genres); } - private static void CopyTable(SqliteConnection source, SqliteConnection destination, string table, string columns) + private static void CopyTable(SqliteTransaction snapshot, SqliteConnection destination, string table, string columns) { - using var read = source.CreateCommand(); + using var read = snapshot.Connection!.CreateCommand(); + read.Transaction = snapshot; read.CommandText = $"SELECT {columns} FROM {table};"; using var reader = read.ExecuteReader(); using var tx = destination.BeginTransaction(); + var names = columns.Split(',', StringSplitOptions.TrimEntries); + using var insert = destination.CreateCommand(); + insert.Transaction = tx; + insert.CommandText = $"INSERT OR REPLACE INTO {table}({columns}) VALUES({string.Join(",", names.Select((_, i) => "$v" + i.ToString(CultureInfo.InvariantCulture)))});"; + for (var i = 0; i < names.Length; i++) + insert.Parameters.Add(new SqliteParameter("$v" + i.ToString(CultureInfo.InvariantCulture), DBNull.Value)); while (reader.Read()) { - using var insert = destination.CreateCommand(); - insert.Transaction = tx; - var names = columns.Split(',', StringSplitOptions.TrimEntries); - insert.CommandText = $"INSERT OR REPLACE INTO {table}({columns}) VALUES({string.Join(",", names.Select((_, i) => "$v" + i.ToString(CultureInfo.InvariantCulture)))});"; for (var i = 0; i < names.Length; i++) - insert.Parameters.AddWithValue("$v" + i.ToString(CultureInfo.InvariantCulture), reader.GetValue(i)); + insert.Parameters[i].Value = reader.GetValue(i); insert.ExecuteNonQuery(); } tx.Commit(); } - private static int ImportFacts(SqliteConnection source, SqliteConnection destination, SqliteTransaction tx, IReadOnlySet liveItemIds) + private static int ImportTable(SqliteConnection source, SqliteTransaction tx, string table, string columns, Func include) { using var read = source.CreateCommand(); - read.CommandText = @" -SELECT item_id, tmdb_id, kind, original_title, original_language, original_language_all, - origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, - genre_tmdb_ids_json, missing_checked_at, full_checked_at -FROM facts;"; + read.CommandText = $"SELECT {columns} FROM {table};"; using var reader = read.ExecuteReader(); + using var insert = tx.Connection!.CreateCommand(); + insert.Transaction = tx; + var names = columns.Split(',', StringSplitOptions.TrimEntries); + insert.CommandText = $"INSERT OR REPLACE INTO {table}({columns}) VALUES({string.Join(",", names.Select((_, i) => "$v" + i))});"; + for (var i = 0; i < names.Length; i++) + insert.Parameters.Add(new SqliteParameter("$v" + i, DBNull.Value)); var count = 0; while (reader.Read()) { - var itemId = reader.GetString(0); - if (!liveItemIds.Contains(itemId)) + if (!include(reader)) continue; - using var insert = destination.CreateCommand(); - insert.Transaction = tx; - insert.CommandText = @" -INSERT INTO facts(item_id, tmdb_id, kind, original_title, original_language, original_language_all, - origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, - genre_tmdb_ids_json, missing_checked_at, full_checked_at) -VALUES($item_id, $tmdb_id, $kind, $original_title, $original_language, $original_language_all, - $origin_countries_json, $production_countries_json, $spoken_languages_json, $audio_track_language, - $genre_tmdb_ids_json, $missing_checked_at, $full_checked_at) -ON CONFLICT(item_id) DO UPDATE SET - tmdb_id = excluded.tmdb_id, - kind = excluded.kind, - original_title = excluded.original_title, - original_language = excluded.original_language, - original_language_all = excluded.original_language_all, - origin_countries_json = excluded.origin_countries_json, - production_countries_json = excluded.production_countries_json, - spoken_languages_json = excluded.spoken_languages_json, - audio_track_language = excluded.audio_track_language, - genre_tmdb_ids_json = excluded.genre_tmdb_ids_json, - missing_checked_at = excluded.missing_checked_at, - full_checked_at = excluded.full_checked_at;"; - var names = new[] - { - "$item_id", - "$tmdb_id", - "$kind", - "$original_title", - "$original_language", - "$original_language_all", - "$origin_countries_json", - "$production_countries_json", - "$spoken_languages_json", - "$audio_track_language", - "$genre_tmdb_ids_json", - "$missing_checked_at", - "$full_checked_at" - }; for (var i = 0; i < names.Length; i++) - insert.Parameters.AddWithValue(names[i], reader.GetValue(i)); + insert.Parameters[i].Value = reader.GetValue(i); insert.ExecuteNonQuery(); count++; } - - return count; - } - - private static int ImportTranslations(SqliteConnection source, SqliteConnection destination, SqliteTransaction tx, IReadOnlySet liveItemIds, IReadOnlySet allowedLanguages) - { - using var read = source.CreateCommand(); - read.CommandText = "SELECT item_id, lang, field, text FROM translations;"; - using var reader = read.ExecuteReader(); - var count = 0; - while (reader.Read()) - { - var itemId = reader.GetString(0); - var lang = reader.GetString(1); - if (!liveItemIds.Contains(itemId) || !allowedLanguages.Contains(lang)) - continue; - using var insert = destination.CreateCommand(); - insert.Transaction = tx; - insert.CommandText = @" -INSERT INTO translations(item_id, lang, field, text) -VALUES($item_id, $lang, $field, $text) -ON CONFLICT(item_id, lang, field) DO UPDATE SET text = excluded.text;"; - insert.Parameters.AddWithValue("$item_id", itemId); - insert.Parameters.AddWithValue("$lang", lang); - insert.Parameters.AddWithValue("$field", reader.GetString(2)); - insert.Parameters.AddWithValue("$text", reader.GetString(3)); - insert.ExecuteNonQuery(); - count++; - } - - return count; - } - - private static int ImportAssets(SqliteConnection source, SqliteConnection destination, SqliteTransaction tx, IReadOnlySet liveItemIds, IReadOnlySet allowedLanguages) - { - using var read = source.CreateCommand(); - read.CommandText = "SELECT item_id, lang, kind, path, path_low, updated_at FROM assets;"; - using var reader = read.ExecuteReader(); - var count = 0; - while (reader.Read()) - { - var itemId = reader.GetString(0); - var lang = reader.GetString(1); - if (!liveItemIds.Contains(itemId) || !allowedLanguages.Contains(lang)) - continue; - using var insert = destination.CreateCommand(); - insert.Transaction = tx; - insert.CommandText = @" -INSERT INTO assets(item_id, lang, kind, path, path_low, updated_at) -VALUES($item_id, $lang, $kind, $path, $path_low, $updated_at) -ON CONFLICT(item_id, lang, kind) DO UPDATE SET - path = excluded.path, - path_low = excluded.path_low, - updated_at = excluded.updated_at;"; - insert.Parameters.AddWithValue("$item_id", itemId); - insert.Parameters.AddWithValue("$lang", lang); - insert.Parameters.AddWithValue("$kind", reader.GetString(2)); - insert.Parameters.AddWithValue("$path", reader.GetString(3)); - insert.Parameters.AddWithValue("$path_low", reader.GetString(4)); - insert.Parameters.AddWithValue("$updated_at", reader.GetInt64(5)); - insert.ExecuteNonQuery(); - count++; - } - - return count; - } - - private static int ImportGenres(SqliteConnection source, SqliteConnection destination, SqliteTransaction tx, IReadOnlySet allowedLanguages) - { - using var read = source.CreateCommand(); - read.CommandText = "SELECT tmdb_id, media, lang, name, name_norm FROM genres;"; - using var reader = read.ExecuteReader(); - var count = 0; - while (reader.Read()) - { - var lang = reader.GetString(2); - if (!allowedLanguages.Contains(lang)) - continue; - using var insert = destination.CreateCommand(); - insert.Transaction = tx; - insert.CommandText = @" -INSERT INTO genres(tmdb_id, media, lang, name, name_norm) -VALUES($tmdb_id, $media, $lang, $name, $name_norm) -ON CONFLICT(tmdb_id, media, lang) DO UPDATE SET - name = excluded.name, - name_norm = excluded.name_norm;"; - insert.Parameters.AddWithValue("$tmdb_id", reader.GetInt32(0)); - insert.Parameters.AddWithValue("$media", reader.GetString(1)); - insert.Parameters.AddWithValue("$lang", lang); - insert.Parameters.AddWithValue("$name", reader.GetString(3)); - insert.Parameters.AddWithValue("$name_norm", reader.GetString(4)); - insert.ExecuteNonQuery(); - count++; - } - return count; } } diff --git a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Models.cs b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Models.cs index 6bba5ba..faa2746 100644 --- a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Models.cs +++ b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Models.cs @@ -4,8 +4,6 @@ namespace Jellyfin.Plugin.Multilang.Data; public readonly record struct FactsStatus(bool Exists, long MissingCheckedAt, long FullCheckedAt); -public readonly record struct AssetPresence(bool Any, bool AnyRemote); - public readonly record struct LocalCleanupResult(int TranslationsDeleted, int AssetsDeleted, int GenresDeleted, int AssetFilesDeleted); public readonly record struct ImportDatabaseResult(int Facts, int Translations, int Assets, int Genres); @@ -46,23 +44,17 @@ public sealed class UserCategoryRule public string Label { get; set; } = string.Empty; - public string CriteriaText { get; set; } = string.Empty; - public UserCategoryRequirement[] Requirements { get; set; } = []; public bool MatchAllConditions { get; set; } = true; public string[] Scopes { get; set; } = ["M", "S", "C"]; - public Dictionary FieldActions { get; set; } = MultilangConstants.DefaultFieldActions(); - public Dictionary FieldActionLists { get; set; } = MultilangConstants.DefaultActionLists(); } public sealed class UserCategoryRequirement { - public string Field { get; set; } = string.Empty; - public string[] Fields { get; set; } = []; public bool UseFieldOr { get; set; } diff --git a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Refresh.cs b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Refresh.cs new file mode 100644 index 0000000..ea22f9e --- /dev/null +++ b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Refresh.cs @@ -0,0 +1,125 @@ +using Jellyfin.Plugin.Multilang.Services.Providers; +using Jellyfin.Plugin.Multilang.Services.Refresh; +using Microsoft.Data.Sqlite; +using static Jellyfin.Plugin.Multilang.MultilangConstants; + +namespace Jellyfin.Plugin.Multilang.Data; + +public sealed record FetchState(string Scope, long CheckedAt, bool Complete); +public sealed record StoredAsset(string Language, string Kind, string Path, string SourceUrl, string Provider); + +public sealed partial class TranslationStore +{ + private void Write(SqliteTransaction? transaction, Action action) + { + if (transaction is not null) + { + action(transaction); + return; + } + using var connection = Open(); + connection.Open(); + using var tx = connection.BeginTransaction(); + action(tx); + tx.Commit(); + } + + public Dictionary GetFetchStates(string itemId) + { + using var connection = Open(); + connection.Open(); + using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT source, scope, checked_at, complete FROM fetch_state WHERE item_id = $id;"; + cmd.Parameters.AddWithValue("$id", itemId); + using var reader = cmd.ExecuteReader(); + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + while (reader.Read()) + result.Add(reader.GetString(0), new(reader.GetString(1), reader.GetInt64(2), reader.GetBoolean(3))); + return result; + } + + private static void SaveFetchState(SqliteTransaction tx, string itemId, string source, FetchState state) + { + using var cmd = tx.Connection!.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = "INSERT OR REPLACE INTO fetch_state(item_id, source, scope, checked_at, complete) VALUES($id, $source, $scope, $time, $complete);"; + cmd.Parameters.AddWithValue("$id", itemId); + cmd.Parameters.AddWithValue("$source", source); + cmd.Parameters.AddWithValue("$scope", state.Scope); + cmd.Parameters.AddWithValue("$time", state.CheckedAt); + cmd.Parameters.AddWithValue("$complete", state.Complete); + cmd.ExecuteNonQuery(); + } + + public void SaveMetadata(RefreshItemInfo item, string language, TmdbMetadata? metadata, FetchState state, bool full, bool updateFacts = true) + => Write(null, tx => + { + if (metadata is not null) + { + if (updateFacts) + UpsertFacts(item, metadata, state.CheckedAt, full ? state.CheckedAt : 0, tx); + UpsertTranslation(item.ItemId, language, TitleField, metadata.Title, tx); + UpsertTranslation(item.ItemId, language, OverviewField, metadata.Overview, tx); + UpsertTranslation(item.ItemId, language, TaglineField, metadata.Tagline, tx); + foreach (var genre in metadata.Genres) + UpsertGenre(genre.Id, item.Kind is "movie" or "collection" ? "movie" : "tv", language, genre.Name, tx); + } + else + MarkFactsChecked(item, state.CheckedAt, full ? state.CheckedAt : 0, tx); + SaveFetchState(tx, item.ItemId, "metadata:" + language, state); + }); + + public void CopyOriginalTranslations(string itemId, string language) + => Write(null, tx => + { + using var cmd = tx.Connection!.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = @"DELETE FROM translations WHERE item_id = $id AND lang = 'Original'; +INSERT INTO translations(item_id, lang, field, text) +SELECT item_id, 'Original', field, text FROM translations WHERE item_id = $id AND lang = $lang;"; + cmd.Parameters.AddWithValue("$id", itemId); + cmd.Parameters.AddWithValue("$lang", language); + cmd.ExecuteNonQuery(); + }); + + public List GetStoredAssets(string itemId) + { + using var connection = Open(); + connection.Open(); + using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT lang, kind, path, source_url, provider FROM assets WHERE item_id = $id;"; + cmd.Parameters.AddWithValue("$id", itemId); + using var reader = cmd.ExecuteReader(); + var result = new List(); + while (reader.Read()) + result.Add(new(reader.GetString(0), reader.GetString(1), reader.GetString(2), reader.GetString(3), reader.GetString(4))); + return result; + } + + public void SaveArtwork(string itemId, IEnumerable assets, IReadOnlyDictionary states) + => Write(null, tx => + { + using var cmd = tx.Connection!.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = "DELETE FROM assets WHERE item_id = $id;"; + cmd.Parameters.AddWithValue("$id", itemId); + cmd.ExecuteNonQuery(); + cmd.CommandText = @"INSERT INTO assets(item_id, lang, kind, path, path_low, source_url, provider, updated_at) +VALUES($id, $lang, $kind, $path, $low, $url, $provider, $time);"; + foreach (var name in new[] { "$lang", "$kind", "$path", "$low", "$url", "$provider", "$time" }) + cmd.Parameters.Add(new SqliteParameter(name, DBNull.Value)); + foreach (var asset in assets) + { + cmd.Parameters["$lang"].Value = asset.Language; + cmd.Parameters["$kind"].Value = asset.Kind; + cmd.Parameters["$path"].Value = asset.Path; + cmd.Parameters["$low"].Value = asset.Path.ToLowerInvariant(); + cmd.Parameters["$url"].Value = asset.SourceUrl; + cmd.Parameters["$provider"].Value = asset.Provider; + cmd.Parameters["$time"].Value = NowUnixUtc(); + cmd.ExecuteNonQuery(); + } + foreach (var (provider, state) in states) + SaveFetchState(tx, itemId, provider, state); + }); +} diff --git a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Schema.cs b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Schema.cs index 10b6315..60b75fa 100644 --- a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Schema.cs +++ b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Schema.cs @@ -38,11 +38,22 @@ CREATE TABLE IF NOT EXISTS assets ( kind TEXT NOT NULL, path TEXT NOT NULL, path_low TEXT NOT NULL DEFAULT '', + source_url TEXT NOT NULL DEFAULT '', + provider TEXT NOT NULL DEFAULT '', updated_at INTEGER NOT NULL, PRIMARY KEY (item_id, lang, kind) ); CREATE INDEX IF NOT EXISTS idx_assets_item ON assets(item_id); +CREATE TABLE IF NOT EXISTS fetch_state ( + item_id TEXT NOT NULL, + source TEXT NOT NULL, + scope TEXT NOT NULL, + checked_at INTEGER NOT NULL, + complete INTEGER NOT NULL, + PRIMARY KEY (item_id, source) +); + CREATE TABLE IF NOT EXISTS genres ( tmdb_id INTEGER NOT NULL, media TEXT NOT NULL, diff --git a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.cs b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.cs index 7c2115b..a8052eb 100644 --- a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.cs +++ b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.cs @@ -15,6 +15,34 @@ public sealed partial class TranslationStore private readonly string _dataDir; private readonly string _assetsDir; private readonly string _dbPath; + private readonly SemaphoreSlim _maintenance = new(1, 1); + private readonly SemaphoreSlim _refreshSlots = new(MultilangConstants.RefreshConcurrency, MultilangConstants.RefreshConcurrency); + + public Task EnterRefreshAsync(CancellationToken ct) => EnterAsync(1, ct); + public Task EnterMaintenanceAsync(CancellationToken ct) => EnterAsync(MultilangConstants.RefreshConcurrency, ct); + + private async Task EnterAsync(int slots, CancellationToken ct) + { + await _maintenance.WaitAsync(ct).ConfigureAwait(false); + var acquired = 0; + try + { + for (; acquired < slots; acquired++) + await _refreshSlots.WaitAsync(ct).ConfigureAwait(false); + return new MaintenanceLease(_refreshSlots, slots); + } + catch + { + if (acquired > 0) _refreshSlots.Release(acquired); + throw; + } + finally { _maintenance.Release(); } + } + + private sealed class MaintenanceLease(SemaphoreSlim semaphore, int slots) : IDisposable + { + public void Dispose() => semaphore.Release(slots); + } public TranslationStore(IApplicationPaths appPaths) { @@ -24,6 +52,8 @@ public sealed partial class TranslationStore Directory.CreateDirectory(_assetsDir); _dbPath = Path.Combine(_dataDir, "translations.sqlite"); Initialize(); + if (Plugin.Instance is { } plugin) + plugin.Store = this; } public string DataDirectory => _dataDir; @@ -42,27 +72,38 @@ public sealed partial class TranslationStore => JsonSerializer.Serialize(values.Where(v => !string.IsNullOrWhiteSpace(v)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray()); public SqliteConnection Open() - => new($"Data Source={_dbPath};Cache=Shared;Pooling=False"); + => new($"Data Source={_dbPath};Pooling=False"); private void Initialize() { using var con = Open(); con.Open(); using var cmd = con.CreateCommand(); + cmd.CommandText = "PRAGMA journal_mode=WAL;"; + cmd.ExecuteNonQuery(); cmd.CommandText = SchemaSql; cmd.ExecuteNonQuery(); EnsureColumn(con, "assets", "path_low", "TEXT NOT NULL DEFAULT ''"); + EnsureColumn(con, "assets", "source_url", "TEXT NOT NULL DEFAULT ''"); + EnsureColumn(con, "assets", "provider", "TEXT NOT NULL DEFAULT ''"); + cmd.CommandText = "UPDATE assets SET source_url = path WHERE source_url = '' AND (path_low LIKE 'https://%' OR path_low LIKE 'http://%');"; + cmd.ExecuteNonQuery(); } public void ResetAll() { - SqliteConnection.ClearAllPools(); - if (Directory.Exists(_dataDir)) - Directory.Delete(_dataDir, recursive: true); - Directory.CreateDirectory(_dataDir); + using var con = Open(); + con.Open(); + using var tx = con.BeginTransaction(); + using var cmd = con.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = "DELETE FROM facts; DELETE FROM translations; DELETE FROM assets; DELETE FROM fetch_state; DELETE FROM genres; " + + "DELETE FROM user_rules; DELETE FROM user_display_langs; UPDATE scan_state SET last_scan_started = 0;"; + cmd.ExecuteNonQuery(); + tx.Commit(); + if (Directory.Exists(_assetsDir)) + Directory.Delete(_assetsDir, recursive: true); Directory.CreateDirectory(_assetsDir); - Initialize(); - SqliteConnection.ClearAllPools(); } public (long DatabaseBytes, long AssetsBytes) GetStorageUsage() @@ -147,12 +188,13 @@ ON CONFLICT(id) DO UPDATE SET last_scan_started = excluded.last_scan_started;"; return result; } - public void UpsertFacts(RefreshItemInfo item, TmdbMetadata metadata, long missingCheckedAt, long fullCheckedAt) + public void UpsertFacts(RefreshItemInfo item, TmdbMetadata metadata, long missingCheckedAt, long fullCheckedAt, SqliteTransaction? transaction = null) { - using var con = Open(); - con.Open(); - using var cmd = con.CreateCommand(); - cmd.CommandText = @" + Write(transaction, tx => + { + using var cmd = tx.Connection!.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = @" INSERT INTO facts( item_id, tmdb_id, kind, original_title, original_language, original_language_all, origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, @@ -174,28 +216,30 @@ ON CONFLICT(item_id) DO UPDATE SET genre_tmdb_ids_json = excluded.genre_tmdb_ids_json, missing_checked_at = excluded.missing_checked_at, full_checked_at = CASE WHEN excluded.full_checked_at > 0 THEN excluded.full_checked_at ELSE facts.full_checked_at END;"; - cmd.Parameters.AddWithValue("$item_id", item.ItemId); - cmd.Parameters.AddWithValue("$tmdb_id", item.TmdbId); - cmd.Parameters.AddWithValue("$kind", item.Kind); - cmd.Parameters.AddWithValue("$original_title", metadata.OriginalTitle); - cmd.Parameters.AddWithValue("$original_language", metadata.OriginalLanguage); - cmd.Parameters.AddWithValue("$original_language_all", metadata.OriginalLanguage); - cmd.Parameters.AddWithValue("$origin_countries_json", JsonArray(metadata.OriginCountries)); - cmd.Parameters.AddWithValue("$production_countries_json", JsonArray(metadata.ProductionCountries)); - cmd.Parameters.AddWithValue("$spoken_languages_json", JsonArray(metadata.SpokenLanguages)); - cmd.Parameters.AddWithValue("$audio_track_language", item.AudioLanguage ?? string.Empty); - cmd.Parameters.AddWithValue("$genre_tmdb_ids_json", JsonSerializer.Serialize(metadata.GenreIds)); - cmd.Parameters.AddWithValue("$missing_checked_at", missingCheckedAt); - cmd.Parameters.AddWithValue("$full_checked_at", fullCheckedAt); - cmd.ExecuteNonQuery(); + cmd.Parameters.AddWithValue("$item_id", item.ItemId); + cmd.Parameters.AddWithValue("$tmdb_id", item.TmdbId); + cmd.Parameters.AddWithValue("$kind", item.Kind); + cmd.Parameters.AddWithValue("$original_title", metadata.OriginalTitle); + cmd.Parameters.AddWithValue("$original_language", metadata.OriginalLanguage); + cmd.Parameters.AddWithValue("$original_language_all", metadata.OriginalLanguage); + cmd.Parameters.AddWithValue("$origin_countries_json", JsonArray(metadata.OriginCountries)); + cmd.Parameters.AddWithValue("$production_countries_json", JsonArray(metadata.ProductionCountries)); + cmd.Parameters.AddWithValue("$spoken_languages_json", JsonArray(metadata.SpokenLanguages)); + cmd.Parameters.AddWithValue("$audio_track_language", item.AudioLanguage ?? string.Empty); + cmd.Parameters.AddWithValue("$genre_tmdb_ids_json", JsonSerializer.Serialize(metadata.GenreIds)); + cmd.Parameters.AddWithValue("$missing_checked_at", missingCheckedAt); + cmd.Parameters.AddWithValue("$full_checked_at", fullCheckedAt); + cmd.ExecuteNonQuery(); + }); } - public void MarkFactsChecked(RefreshItemInfo item, long missingCheckedAt, long fullCheckedAt) + public void MarkFactsChecked(RefreshItemInfo item, long missingCheckedAt, long fullCheckedAt, SqliteTransaction? transaction = null) { - using var con = Open(); - con.Open(); - using var cmd = con.CreateCommand(); - cmd.CommandText = @" + Write(transaction, tx => + { + using var cmd = tx.Connection!.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = @" INSERT INTO facts( item_id, tmdb_id, kind, original_title, original_language, original_language_all, origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, @@ -209,12 +253,13 @@ ON CONFLICT(item_id) DO UPDATE SET kind = excluded.kind, missing_checked_at = excluded.missing_checked_at, full_checked_at = CASE WHEN excluded.full_checked_at > 0 THEN excluded.full_checked_at ELSE facts.full_checked_at END;"; - cmd.Parameters.AddWithValue("$item_id", item.ItemId); - cmd.Parameters.AddWithValue("$tmdb_id", item.TmdbId); - cmd.Parameters.AddWithValue("$kind", item.Kind); - cmd.Parameters.AddWithValue("$missing_checked_at", missingCheckedAt); - cmd.Parameters.AddWithValue("$full_checked_at", fullCheckedAt); - cmd.ExecuteNonQuery(); + cmd.Parameters.AddWithValue("$item_id", item.ItemId); + cmd.Parameters.AddWithValue("$tmdb_id", item.TmdbId); + cmd.Parameters.AddWithValue("$kind", item.Kind); + cmd.Parameters.AddWithValue("$missing_checked_at", missingCheckedAt); + cmd.Parameters.AddWithValue("$full_checked_at", fullCheckedAt); + cmd.ExecuteNonQuery(); + }); } public FactsData? GetFacts(string itemId) @@ -290,41 +335,47 @@ WHERE item_id IN ({idParams});"; return rows.ToArray(); } - public void UpsertTranslation(string itemId, string lang, string field, string text) + public void UpsertTranslation(string itemId, string lang, string field, string text, SqliteTransaction? transaction = null) { - using var con = Open(); - con.Open(); - using var cmd = con.CreateCommand(); - cmd.CommandText = @" + Write(transaction, tx => + { + using var cmd = tx.Connection!.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = @" INSERT INTO translations(item_id, lang, field, text) VALUES($item_id, $lang, $field, $text) ON CONFLICT(item_id, lang, field) DO UPDATE SET text = excluded.text;"; - cmd.Parameters.AddWithValue("$item_id", itemId); - cmd.Parameters.AddWithValue("$lang", lang); - cmd.Parameters.AddWithValue("$field", field); - cmd.Parameters.AddWithValue("$text", text); - cmd.ExecuteNonQuery(); + cmd.Parameters.AddWithValue("$item_id", itemId); + cmd.Parameters.AddWithValue("$lang", lang); + cmd.Parameters.AddWithValue("$field", field); + cmd.Parameters.AddWithValue("$text", text); + cmd.ExecuteNonQuery(); + }); } - public void UpsertAsset(string itemId, string lang, string kind, string path, long updatedAt) + public void UpsertAsset(string itemId, string lang, string kind, string path, long updatedAt, SqliteTransaction? transaction = null) { - using var con = Open(); - con.Open(); - using var cmd = con.CreateCommand(); - cmd.CommandText = @" -INSERT INTO assets(item_id, lang, kind, path, path_low, updated_at) -VALUES($item_id, $lang, $kind, $path, $path_low, $updated_at) + Write(transaction, tx => + { + using var cmd = tx.Connection!.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = @" +INSERT INTO assets(item_id, lang, kind, path, path_low, updated_at, source_url) +VALUES($item_id, $lang, $kind, $path, $path_low, $updated_at, $source_url) ON CONFLICT(item_id, lang, kind) DO UPDATE SET path = excluded.path, path_low = excluded.path_low, + source_url = excluded.source_url, updated_at = excluded.updated_at;"; - cmd.Parameters.AddWithValue("$item_id", itemId); - cmd.Parameters.AddWithValue("$lang", lang); - cmd.Parameters.AddWithValue("$kind", kind); - cmd.Parameters.AddWithValue("$path", path); - cmd.Parameters.AddWithValue("$path_low", path.ToLowerInvariant()); - cmd.Parameters.AddWithValue("$updated_at", updatedAt); - cmd.ExecuteNonQuery(); + cmd.Parameters.AddWithValue("$item_id", itemId); + cmd.Parameters.AddWithValue("$lang", lang); + cmd.Parameters.AddWithValue("$kind", kind); + cmd.Parameters.AddWithValue("$path", path); + cmd.Parameters.AddWithValue("$path_low", path.ToLowerInvariant()); + cmd.Parameters.AddWithValue("$source_url", Uri.TryCreate(path, UriKind.Absolute, out var url) && url.Scheme is "https" or "http" ? path : ""); + cmd.Parameters.AddWithValue("$updated_at", updatedAt); + cmd.ExecuteNonQuery(); + }); } public string? GetAssetPath(string itemId, string lang, string kind) @@ -339,76 +390,39 @@ ON CONFLICT(item_id, lang, kind) DO UPDATE SET return cmd.ExecuteScalar() as string; } - public AssetPresence GetAssetPresence(string itemId, IEnumerable languages) + public bool IsAssetReferenced(string itemId, string path) { - var langs = languages.Where(l => !string.IsNullOrWhiteSpace(l)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); - if (langs.Length == 0) - return new AssetPresence(false, false); - using var con = Open(); con.Open(); using var cmd = con.CreateCommand(); - var langParams = AddParams(cmd, langs, "$lang"); - cmd.CommandText = $"SELECT path_low FROM assets WHERE item_id = $item_id AND lang IN ({langParams});"; - cmd.Parameters.AddWithValue("$item_id", itemId); - using var reader = cmd.ExecuteReader(); - var any = false; - var anyRemote = false; - while (reader.Read()) - { - any = true; - var path = reader.GetString(0); - if (path.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || path.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) - anyRemote = true; - } - - return new AssetPresence(any, anyRemote); + cmd.CommandText = "SELECT 1 FROM assets WHERE item_id = $item AND path = $path LIMIT 1;"; + cmd.Parameters.AddWithValue("$item", itemId); + cmd.Parameters.AddWithValue("$path", path); + return cmd.ExecuteScalar() is not null; } - public bool HasMissingConfiguredTranslations(string itemId, IEnumerable languages) - { - var langs = languages.Where(l => !string.IsNullOrWhiteSpace(l)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); - if (langs.Length == 0) - return false; - - using var con = Open(); - con.Open(); - using var cmd = con.CreateCommand(); - var langParams = AddParams(cmd, langs, "$lang"); - cmd.CommandText = $@" -SELECT lang, COUNT(DISTINCT field) -FROM translations -WHERE item_id = $item_id AND lang IN ({langParams}) AND field IN ('title', 'overview', 'tagline') -GROUP BY lang;"; - cmd.Parameters.AddWithValue("$item_id", itemId); - using var reader = cmd.ExecuteReader(); - var counts = new Dictionary(StringComparer.OrdinalIgnoreCase); - while (reader.Read()) - counts[reader.GetString(0)] = reader.GetInt32(1); - - return langs.Any(lang => !counts.TryGetValue(lang, out var count) || count < 3); - } - - public void UpsertGenre(int tmdbId, string media, string lang, string name) + public void UpsertGenre(int tmdbId, string media, string lang, string name, SqliteTransaction? transaction = null) { if (tmdbId <= 0 || string.IsNullOrWhiteSpace(media) || string.IsNullOrWhiteSpace(lang) || string.IsNullOrWhiteSpace(name)) return; - using var con = Open(); - con.Open(); - using var cmd = con.CreateCommand(); - cmd.CommandText = @" + Write(transaction, tx => + { + using var cmd = tx.Connection!.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = @" INSERT INTO genres(tmdb_id, media, lang, name, name_norm) VALUES($tmdb_id, $media, $lang, $name, $name_norm) ON CONFLICT(tmdb_id, media, lang) DO UPDATE SET name = excluded.name, name_norm = excluded.name_norm;"; - cmd.Parameters.AddWithValue("$tmdb_id", tmdbId); - cmd.Parameters.AddWithValue("$media", media.Trim().ToLowerInvariant()); - cmd.Parameters.AddWithValue("$lang", lang.Trim()); - cmd.Parameters.AddWithValue("$name", name.Trim()); - cmd.Parameters.AddWithValue("$name_norm", name.Trim().ToLowerInvariant()); - cmd.ExecuteNonQuery(); + cmd.Parameters.AddWithValue("$tmdb_id", tmdbId); + cmd.Parameters.AddWithValue("$media", media.Trim().ToLowerInvariant()); + cmd.Parameters.AddWithValue("$lang", lang.Trim()); + cmd.Parameters.AddWithValue("$name", name.Trim()); + cmd.Parameters.AddWithValue("$name_norm", name.Trim().ToLowerInvariant()); + cmd.ExecuteNonQuery(); + }); } public GenreRow[] GetGenres(string media) diff --git a/src/Jellyfin.Plugin.Multilang/Jellyfin.Plugin.Multilang.csproj b/src/Jellyfin.Plugin.Multilang/Jellyfin.Plugin.Multilang.csproj index 80f8017..8483c9b 100644 --- a/src/Jellyfin.Plugin.Multilang/Jellyfin.Plugin.Multilang.csproj +++ b/src/Jellyfin.Plugin.Multilang/Jellyfin.Plugin.Multilang.csproj @@ -7,9 +7,9 @@ enable enable false - 0.2.4 - 0.2.4.0 - 0.2.4.0 + 0.2.5 + 0.2.5.0 + 0.2.5.0 ajp_anton diff --git a/src/Jellyfin.Plugin.Multilang/MultilangConstants.cs b/src/Jellyfin.Plugin.Multilang/MultilangConstants.cs index 8b60285..0c04ad0 100644 --- a/src/Jellyfin.Plugin.Multilang/MultilangConstants.cs +++ b/src/Jellyfin.Plugin.Multilang/MultilangConstants.cs @@ -2,6 +2,7 @@ namespace Jellyfin.Plugin.Multilang; internal static class MultilangConstants { + public const int RefreshConcurrency = 4; public const string JellyfinAction = "Jellyfin"; public const string OriginalAction = "Original"; public const string FallbackAction = "Fallback"; @@ -25,12 +26,6 @@ internal static class MultilangConstants => field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) || field.Equals(PosterKind, StringComparison.OrdinalIgnoreCase); - public static Dictionary DefaultFieldActions() - => ActionFields.ToDictionary( - field => field, - _ => FallbackAction, - StringComparer.OrdinalIgnoreCase); - public static Dictionary DefaultActionLists() => ActionFields.ToDictionary( field => field, diff --git a/src/Jellyfin.Plugin.Multilang/Plugin.cs b/src/Jellyfin.Plugin.Multilang/Plugin.cs index 1fb5706..a766455 100644 --- a/src/Jellyfin.Plugin.Multilang/Plugin.cs +++ b/src/Jellyfin.Plugin.Multilang/Plugin.cs @@ -10,6 +10,8 @@ namespace Jellyfin.Plugin.Multilang; public sealed class Plugin : BasePlugin, IHasWebPages { + internal Data.TranslationStore? Store { get; set; } + internal bool Uninstalling { get; private set; } public static Plugin? Instance { get; private set; } public Plugin(IApplicationPaths applicationPaths, IXmlSerializer xmlSerializer) @@ -24,6 +26,8 @@ public sealed class Plugin : BasePlugin, IHasWebPages public override void OnUninstalling() { + Uninstalling = true; + using var maintenance = Store?.EnterMaintenanceAsync(CancellationToken.None).GetAwaiter().GetResult(); WebScriptInjector.RemoveInjected(ApplicationPaths.WebPath); if (Configuration.CleanupDataOnUninstall) diff --git a/src/Jellyfin.Plugin.Multilang/PluginServiceRegistrator.cs b/src/Jellyfin.Plugin.Multilang/PluginServiceRegistrator.cs index a7f97f4..75eaca6 100644 --- a/src/Jellyfin.Plugin.Multilang/PluginServiceRegistrator.cs +++ b/src/Jellyfin.Plugin.Multilang/PluginServiceRegistrator.cs @@ -17,6 +17,8 @@ public sealed class PluginServiceRegistrator : IPluginServiceRegistrator public void RegisterServices(IServiceCollection services, IServerApplicationHost applicationHost) { services.AddMvc().AddApplicationPart(typeof(Plugin).Assembly); + services.AddHttpClient("Multilang.Jellyfin") + .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler { AllowAutoRedirect = false }); services.AddSingleton(); services.AddSingleton(); @@ -25,7 +27,9 @@ public sealed class PluginServiceRegistrator : IPluginServiceRegistrator services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); + services.AddHostedService(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Jellyfin.Plugin.Multilang/Rules/CategoryRuleEvaluator.cs b/src/Jellyfin.Plugin.Multilang/Rules/CategoryRuleEvaluator.cs index f21670e..09a4ea0 100644 --- a/src/Jellyfin.Plugin.Multilang/Rules/CategoryRuleEvaluator.cs +++ b/src/Jellyfin.Plugin.Multilang/Rules/CategoryRuleEvaluator.cs @@ -10,10 +10,8 @@ public sealed record CategoryRuleTrace( string[] Scopes, bool ScopeMatches, bool MatchAllConditions, - bool UsesStructuredRequirements, bool Result, - RequirementTrace[] Requirements, - string CriteriaText); + RequirementTrace[] Requirements); public sealed record RequirementTrace( string[] Fields, @@ -33,59 +31,40 @@ public static class CategoryRuleEvaluator { public static bool Matches(UserCategoryRule category, FactsData facts) { - return Trace(category, facts).Result; + if (!ScopeMatches(category, facts.Kind)) + return false; + if (category.Requirements is not { Length: > 0 }) + return false; + + return category.MatchAllConditions + ? category.Requirements.All(requirement => MatchesRequirement(requirement, facts)) + : category.Requirements.Any(requirement => MatchesRequirement(requirement, facts)); } + private static bool MatchesRequirement(UserCategoryRequirement requirement, FactsData facts) + { + var fields = RequirementFields(requirement); + var expected = NormalizeValues(requirement.Values ?? []); + if (fields.Length == 0 || expected.Length == 0 || string.IsNullOrWhiteSpace(requirement.Relation)) + return false; + bool Match(string field) => CompareRequirement(NormalizeValues(FieldValues(field, facts)), + expected, requirement.Relation.Trim(), requirement.UseOr); + return requirement.UseFieldOr ? fields.Any(Match) : fields.All(Match); + } + + private static string[] NormalizeValues(IEnumerable values) + => values.Where(v => !string.IsNullOrWhiteSpace(v)).Select(v => v.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); + public static CategoryRuleTrace Trace(UserCategoryRule category, FactsData facts) { var scopeMatches = ScopeMatches(category, facts.Kind); - if (!scopeMatches) - { - return new CategoryRuleTrace( - category.Id, - category.Label, - facts.Kind, - category.Scopes ?? [], - false, - category.MatchAllConditions, - category.Requirements is { Length: > 0 }, - false, - [], - category.CriteriaText ?? string.Empty); - } - - if (category.Requirements is { Length: > 0 }) - { - var requirements = category.Requirements.Select(requirement => TraceRequirement(requirement, facts)).ToArray(); - var result = category.MatchAllConditions - ? requirements.All(requirement => requirement.Result) - : requirements.Any(requirement => requirement.Result); - return new CategoryRuleTrace( - category.Id, - category.Label, - facts.Kind, - category.Scopes ?? [], - true, - category.MatchAllConditions, - true, - result, - requirements, - category.CriteriaText ?? string.Empty); - } - - var criteria = (category.CriteriaText ?? string.Empty).Trim(); - var criteriaResult = criteria.Length > 0 && EvaluateCriteria(criteria, facts); - return new CategoryRuleTrace( - category.Id, - category.Label, - facts.Kind, - category.Scopes ?? [], - true, - category.MatchAllConditions, - false, - criteriaResult, - [], - criteria); + var requirements = category.Requirements.Select(requirement => TraceRequirement(requirement, facts)).ToArray(); + var result = scopeMatches && requirements.Length > 0 && (category.MatchAllConditions + ? requirements.All(requirement => requirement.Result) + : requirements.Any(requirement => requirement.Result)); + return new(category.Id, category.Label, facts.Kind, category.Scopes, + scopeMatches, category.MatchAllConditions, result, requirements); } public static string[] ParseJsonStringArray(string json) @@ -120,42 +99,6 @@ public static class CategoryRuleEvaluator return scopes.Contains(wanted, StringComparer.OrdinalIgnoreCase); } - private static bool EvaluateCriteria(string criteria, FactsData facts) - { - var orGroups = SplitByOperator(criteria, "or"); - foreach (var group in orGroups) - { - var terms = SplitByOperator(group, "and"); - if (terms.All(term => EvaluateTerm(term, facts))) - return true; - } - - return false; - } - - private static bool EvaluateTerm(string term, FactsData facts) - { - term = term.Trim(); - if (term.StartsWith("not ", StringComparison.OrdinalIgnoreCase)) - return !EvaluateTerm(term[4..], facts); - - var relation = FindRelation(term); - if (relation is null) - return false; - - var field = term[..relation.Value.Index].Trim(); - var rawValue = term[(relation.Value.Index + relation.Value.Token.Length)..].Trim().Trim('"', '\''); - var result = relation.Value.Token switch - { - "contains" => FieldValues(field, facts).Any(v => ValueMatches(v, rawValue, IsLanguageField(field))), - "not_contains" => !FieldValues(field, facts).Any(v => ValueMatches(v, rawValue, IsLanguageField(field))), - "is" => FieldValues(field, facts) is { Length: 1 } values && ValueMatches(values[0], rawValue, IsLanguageField(field)), - "is_not" => !(FieldValues(field, facts) is { Length: 1 } values && ValueMatches(values[0], rawValue, IsLanguageField(field))), - _ => false - }; - return result; - } - private static RequirementTrace TraceRequirement(UserCategoryRequirement requirement, FactsData facts) { var fields = RequirementFields(requirement); @@ -201,7 +144,11 @@ public static class CategoryRuleEvaluator .Distinct(StringComparer.OrdinalIgnoreCase) .ToArray(); - var result = relation switch + return new RequirementFieldTrace(field, actual, CompareRequirement(actual, expected, relation, useOr)); + } + + private static bool CompareRequirement(string[] actual, string[] expected, string relation, bool useOr) + => relation switch { "is" => RequirementIs(actual, expected, useOr), "is_not" => !RequirementIs(actual, expected, useOr), @@ -209,23 +156,9 @@ public static class CategoryRuleEvaluator "not_contains" => !RequirementContains(actual, expected, useOr), _ => false }; - return new RequirementFieldTrace(field, actual, result); - } private static string[] RequirementFields(UserCategoryRequirement requirement) - { - var fields = (requirement.Fields ?? []) - .Where(v => !string.IsNullOrWhiteSpace(v)) - .Select(v => v.Trim()) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); - - if (fields.Length > 0) - return fields; - - var field = (requirement.Field ?? string.Empty).Trim(); - return field.Length == 0 ? [] : [field]; - } + => NormalizeValues(requirement.Fields); private static bool RequirementIs(string[] actual, string[] expected, bool useOr) { @@ -252,19 +185,6 @@ public static class CategoryRuleEvaluator private static bool EqualsValue(string actual, string expected) => string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase); - private static (string Token, int Index)? FindRelation(string term) - { - foreach (var token in new[] { "not_contains", "contains", "is_not", "is" }) - { - var needle = " " + token + " "; - var index = term.IndexOf(needle, StringComparison.OrdinalIgnoreCase); - if (index >= 0) - return (token, index + 1); - } - - return null; - } - private static string[] FieldValues(string field, FactsData facts) { field = field.Trim().ToLowerInvariant(); @@ -279,44 +199,4 @@ public static class CategoryRuleEvaluator }; } - private static bool IsLanguageField(string field) - => field.Equals("spoken_languages", StringComparison.OrdinalIgnoreCase) || - field.Equals("original_language", StringComparison.OrdinalIgnoreCase) || - field.Equals("audio_language", StringComparison.OrdinalIgnoreCase); - - private static bool ValueMatches(string actual, string expected, bool language) - { - if (expected.Equals("empty", StringComparison.OrdinalIgnoreCase)) - return string.IsNullOrWhiteSpace(actual); - if (!language) - return string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase); - - if (expected.Contains('-', StringComparison.Ordinal)) - return string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase); - return string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase) || - actual.StartsWith(expected + "-", StringComparison.OrdinalIgnoreCase); - } - - private static string[] SplitByOperator(string text, string op) - { - var parts = new List(); - var tokens = text.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - var current = new List(); - foreach (var token in tokens) - { - if (token.Equals(op, StringComparison.OrdinalIgnoreCase)) - { - parts.Add(string.Join(' ', current)); - current.Clear(); - } - else - { - current.Add(token); - } - } - - if (current.Count > 0) - parts.Add(string.Join(' ', current)); - return parts.Where(p => p.Trim().Length > 0).ToArray(); - } } diff --git a/src/Jellyfin.Plugin.Multilang/Rules/UserRulesNormalizer.cs b/src/Jellyfin.Plugin.Multilang/Rules/UserRulesNormalizer.cs new file mode 100644 index 0000000..318a57c --- /dev/null +++ b/src/Jellyfin.Plugin.Multilang/Rules/UserRulesNormalizer.cs @@ -0,0 +1,166 @@ +using Jellyfin.Plugin.Multilang.Data; +using static Jellyfin.Plugin.Multilang.MultilangConstants; + +namespace Jellyfin.Plugin.Multilang.Rules; + +public static class UserRulesNormalizer +{ + private static readonly string[] RequirementFieldOrder = + [ + "original_language", + "spoken_languages", + "audio_language", + "origin_countries", + "production_countries" + ]; + + public static UserRulesDocument Normalize(UserRulesDocument rules, IEnumerable? languages = null) + { + var allowed = (languages ?? (rules.Categories ?? []).SelectMany(c => (c.FieldActionLists ?? []).Values) + .Concat((rules.FallbackFieldActions ?? []).Values).SelectMany(actions => actions ?? []) + .Where(action => action is not null && action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)).Select(action => action[LanguagePrefix.Length..])) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + rules.Categories = NormalizeCategories(rules.Categories, allowed); + rules.FallbackFieldActions = NormalizeActionLists(rules.FallbackFieldActions, allowed); + return rules; + } + + private static UserCategoryRule[] NormalizeCategories(IEnumerable? categories, IReadOnlySet allowedLanguages) + => (categories ?? []) + .Select(category => new UserCategoryRule + { + Id = string.IsNullOrWhiteSpace(category.Id) ? Guid.NewGuid().ToString("N") : category.Id.Trim(), + Label = string.IsNullOrWhiteSpace(category.Label) ? "Category" : category.Label.Trim(), + Requirements = NormalizeRequirements(category.Requirements), + MatchAllConditions = category.MatchAllConditions, + Scopes = (category.Scopes ?? []) + .Where(scope => scope is "M" or "S" or "C") + .Distinct(StringComparer.OrdinalIgnoreCase) + .DefaultIfEmpty("M") + .ToArray(), + FieldActionLists = NormalizeActionLists(category.FieldActionLists, allowedLanguages) + }) + .ToArray(); + + private static UserCategoryRequirement[] NormalizeRequirements(IEnumerable? requirements) + => (requirements ?? []) + .Select(requirement => + { + var fields = NormalizeRequirementFields(requirement.Fields); + return new UserCategoryRequirement + { + Fields = fields, + UseFieldOr = requirement.UseFieldOr, + Relation = NormalizeRequirementRelation(requirement.Relation), + UseOr = requirement.UseOr, + Values = (requirement.Values ?? []) + .Where(v => !string.IsNullOrWhiteSpace(v)) + .Select(v => v.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() + }; + }) + .Where(requirement => + requirement.Fields.Length > 0 && + requirement.Relation.Length > 0 && + requirement.Values.Length > 0) + .ToArray(); + + private static string[] NormalizeRequirementFields(IEnumerable? fields) + { + var requested = (fields ?? []) + .Select(NormalizeRequirementField) + .Where(field => field.Length > 0) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + var result = new List(); + string? kind = null; + foreach (var field in RequirementFieldOrder) + { + if (!requested.Contains(field)) + continue; + + var currentKind = IsCountryRequirementField(field) ? "country" : "language"; + kind ??= currentKind; + if (currentKind == kind) + result.Add(field); + } + + return result.ToArray(); + } + + private static string NormalizeRequirementField(string? field) + { + var normalized = (field ?? string.Empty).Trim().ToLowerInvariant(); + return normalized is "original_language" or "spoken_languages" or "audio_language" or "origin_countries" or "production_countries" + ? normalized + : string.Empty; + } + + private static bool IsCountryRequirementField(string field) + => field.Equals("origin_countries", StringComparison.OrdinalIgnoreCase) || + field.Equals("production_countries", StringComparison.OrdinalIgnoreCase); + + private static string NormalizeRequirementRelation(string? relation) + { + var normalized = (relation ?? string.Empty).Trim().ToLowerInvariant(); + return normalized is "is" or "is_not" or "contains" or "not_contains" + ? normalized + : string.Empty; + } + + private static Dictionary NormalizeActionLists( + IReadOnlyDictionary? actions, + IReadOnlySet allowedLanguages) + { + var result = DefaultActionLists(); + foreach (var field in result.Keys.ToArray()) + { + var raw = actions is not null && actions.TryGetValue(field, out var configured) + ? configured + : [JellyfinAction]; + result[field] = NormalizeActionList(field, raw, allowedLanguages); + } + + return result; + } + + private static string[] NormalizeActionList(string field, IEnumerable? raw, IReadOnlySet allowedLanguages) + { + var result = new List(); + foreach (var action in raw ?? []) + { + var normalized = NormalizeActionToken(action, allowedLanguages); + if (normalized.Length == 0) + continue; + if (result.Contains(normalized, StringComparer.OrdinalIgnoreCase)) + continue; + result.Add(normalized); + } + + if (RequiresJellyfinFallback(field)) + { + if (!result.Contains(JellyfinAction, StringComparer.OrdinalIgnoreCase)) + result.Add(JellyfinAction); + } + + return result.ToArray(); + } + + private static string NormalizeActionToken(string? raw, IReadOnlySet allowedLanguages) + { + var action = (raw ?? string.Empty).Trim(); + if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)) + return JellyfinAction; + if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase)) + return OriginalAction; + + var lang = action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase) + ? action[LanguagePrefix.Length..].Trim() + : string.Empty; + var canonical = allowedLanguages.FirstOrDefault(l => l.Equals(lang, StringComparison.OrdinalIgnoreCase)); + return string.IsNullOrWhiteSpace(canonical) ? string.Empty : LanguagePrefix + canonical; + } + +} diff --git a/src/Jellyfin.Plugin.Multilang/Services/Assets/AssetStorageService.cs b/src/Jellyfin.Plugin.Multilang/Services/Assets/AssetStorageService.cs index 799c3cb..d099426 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Assets/AssetStorageService.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Assets/AssetStorageService.cs @@ -1,4 +1,3 @@ -using System.Net.Http.Headers; using System.Security.Cryptography; using Jellyfin.Plugin.Multilang.Data; @@ -18,8 +17,6 @@ public sealed class AssetStorageService public async Task StoreAsync( string itemId, - string lang, - string kind, string sourceUrl, bool localStorage, CancellationToken cancellationToken) @@ -30,22 +27,17 @@ public sealed class AssetStorageService if (!localStorage) return sourceUrl; - var relative = BuildRelativePath(itemId, lang, kind, sourceUrl); + var relative = BuildRelativePath(itemId, sourceUrl); var expectedUrl = TranslationStore.LocalAssetUrlPrefix + relative.Replace('\\', '/'); - if (_store.GetAssetPath(itemId, lang, kind) is { } existing && - existing.Equals(expectedUrl, StringComparison.OrdinalIgnoreCase) && - _store.TryNormalizeLocalAssetPath(existing, out var existingPath) && - File.Exists(existingPath)) - { - return existing; - } - var destination = Path.GetFullPath(Path.Combine(_store.AssetsDirectory, relative)); var root = Path.GetFullPath(_store.AssetsDirectory) + Path.DirectorySeparatorChar; if (!destination.StartsWith(root, StringComparison.OrdinalIgnoreCase)) throw new InvalidOperationException("Resolved asset path escaped the Multilang assets directory."); - Directory.CreateDirectory(Path.GetDirectoryName(destination) ?? _store.AssetsDirectory); + if (File.Exists(destination)) + return expectedUrl; + + Directory.CreateDirectory(Path.GetDirectoryName(destination)!); await DownloadAsync(sourceUrl, destination, cancellationToken).ConfigureAwait(false); return expectedUrl; } @@ -101,11 +93,11 @@ public sealed class AssetStorageService File.Move(temp, destination, overwrite: true); } - private static string BuildRelativePath(string itemId, string lang, string kind, string sourceUrl) + private static string BuildRelativePath(string itemId, string sourceUrl) { var extension = ExtensionFromUrl(sourceUrl); var hash = Convert.ToHexString(SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(sourceUrl))).ToLowerInvariant()[..16]; - return Path.Combine(SafeSegment(itemId), SafeSegment(lang), SafeSegment(kind) + "-" + hash + extension); + return Path.Combine(SafeSegment(itemId), hash + extension); } private static string SafeSegment(string value) @@ -133,6 +125,6 @@ public sealed class AssetStorageService ".png" => "image/png", ".webp" => "image/webp", ".gif" => "image/gif", - _ => MediaTypeHeaderValue.Parse("application/octet-stream").MediaType ?? "application/octet-stream" + _ => "application/octet-stream" }; } diff --git a/src/Jellyfin.Plugin.Multilang/Services/Backup/MultilangBackupService.cs b/src/Jellyfin.Plugin.Multilang/Services/Backup/MultilangBackupService.cs index 8a6ed50..0ff1914 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Backup/MultilangBackupService.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Backup/MultilangBackupService.cs @@ -1,3 +1,4 @@ +using Jellyfin.Plugin.Multilang.Rules; using System.Globalization; using System.IO.Compression; using System.Text.Json; @@ -142,6 +143,14 @@ public sealed class MultilangBackupService using var zip = ZipFile.OpenRead(zipPath); ValidateManifest(zip); + var dbPath = Path.Combine(tempDir, DatabaseName); + var dbEntry = options.TranslationsDatabase ? zip.GetEntry(DatabaseName) : null; + if (dbEntry is not null) + { + dbEntry.ExtractToFile(dbPath); + TranslationStore.ValidateTranslationBackup(dbPath); + } + var importedConfig = false; var importedUsers = 0; var ignoredUsers = 0; @@ -173,21 +182,13 @@ public sealed class MultilangBackupService continue; } - _store.SaveUserRules(userId, payload.Rules); + _store.SaveUserRules(userId, UserRulesNormalizer.Normalize(payload.Rules, currentConfiguration.Languages)); importedUsers++; } } - if (options.TranslationsDatabase) - { - var dbEntry = zip.GetEntry(DatabaseName); - if (dbEntry is not null) - { - var dbPath = Path.Combine(tempDir, DatabaseName); - dbEntry.ExtractToFile(dbPath, overwrite: true); - database = _store.ImportTranslationDatabase(dbPath, GetLiveItemIds(), GetAllowedLanguages(currentConfiguration)); - } - } + if (dbEntry is not null) + database = _store.ImportTranslationDatabase(dbPath, GetLiveItemIds(), GetAllowedLanguages(currentConfiguration)); if (options.DownloadedAssets) { @@ -268,7 +269,7 @@ public sealed class MultilangBackupService if (payload?.Rules is null) return new BackupImportResult(false, 0, 0, 0, 0, 0, 0, 0, 0, [], false); - _store.SaveUserRules(current, payload.Rules); + _store.SaveUserRules(current, UserRulesNormalizer.Normalize(payload.Rules, Plugin.Instance?.Configuration.Languages)); return new BackupImportResult(false, 1, 0, 0, 0, 0, 0, 0, 0, [], false); } finally diff --git a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyCache.cs b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyCache.cs index abf1877..380cd74 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyCache.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyCache.cs @@ -10,7 +10,7 @@ public sealed class ItemsProxyCache public required string Url { get; init; } - public required HashSet ItemIds { get; init; } + public required int ItemCount { get; init; } public required string Body { get; init; } @@ -21,6 +21,7 @@ public sealed class ItemsProxyCache public required long DurationMs { get; init; } public required long SizeBytes { get; init; } + public bool Micro { get; init; } } private sealed class RecentRequest @@ -50,49 +51,43 @@ public sealed class ItemsProxyCache private readonly object _lock = new(); private readonly Dictionary _entries = new(StringComparer.Ordinal); - private readonly Dictionary _microEntries = new(StringComparer.Ordinal); + private readonly Dictionary _userPolicies = new(StringComparer.Ordinal); private readonly Queue _recentRequests = new(); private long _totalBytes; + private long _generation; + public long Generation { get { lock (_lock) return _generation; } } private const int MaxRecentRequests = 100; private const int MaxMicroEntries = 50; private static readonly TimeSpan MicroTtl = TimeSpan.FromSeconds(3); + public long ObserveUserPolicy(string userId, string policy) + { + lock (_lock) + { + if (_userPolicies.TryGetValue(userId, out var previous) && previous != policy) + ClearUser(userId); + _userPolicies[userId] = policy; + return _generation; + } + } + public void ClearAll() { lock (_lock) { _entries.Clear(); - _microEntries.Clear(); _totalBytes = 0; + _generation++; } } - public void InvalidateItems(IEnumerable itemIds) + public void ClearUser(string userId) { - var ids = itemIds - .Where(id => !string.IsNullOrWhiteSpace(id)) - .Select(id => id.Trim().ToLowerInvariant()) - .ToHashSet(StringComparer.OrdinalIgnoreCase); - if (ids.Count == 0) - return; - lock (_lock) { - foreach (var key in _entries.Values - .Where(e => e.ItemIds.Overlaps(ids)) - .Select(e => e.Key) - .ToArray()) - { + _generation++; + foreach (var key in _entries.Values.Where(e => e.UserId == userId).Select(e => e.Key).ToArray()) Remove(key); - } - - foreach (var key in _microEntries.Values - .Where(e => e.ItemIds.Overlaps(ids)) - .Select(e => e.Key) - .ToArray()) - { - _microEntries.Remove(key); - } } } @@ -103,23 +98,16 @@ public sealed class ItemsProxyCache PurgeExpired(ttl); if (!_entries.TryGetValue(key, out var entry)) { - PurgeExpired(_microEntries, MicroTtl); - if (!_microEntries.TryGetValue(key, out var microEntry)) - { - response = default; - return false; - } - - response = new CachedItemsProxyResponse(microEntry.Body, microEntry.ContentType, microEntry.ItemIds.Count, microEntry.SizeBytes); - return true; + response = default; + return false; } - response = new CachedItemsProxyResponse(entry.Body, entry.ContentType, entry.ItemIds.Count, entry.SizeBytes); + response = new CachedItemsProxyResponse(entry.Body, entry.ContentType, entry.ItemCount, entry.SizeBytes); return true; } } - public bool Store(string key, string userId, string url, IEnumerable itemIds, string body, string contentType, long durationMs, long maxBytes, TimeSpan ttl) + public bool Store(string key, string userId, string url, int itemCount, string body, string contentType, long durationMs, long maxBytes, TimeSpan ttl, long? generation = null, bool micro = false) { if (maxBytes <= 0 || ttl <= TimeSpan.Zero) return false; @@ -130,76 +118,40 @@ public sealed class ItemsProxyCache lock (_lock) { + if (generation.HasValue && generation != _generation) + return false; PurgeExpired(ttl); Remove(key); + if (micro && _entries.Values.Count(e => e.Micro) >= MaxMicroEntries) + Remove(_entries.Values.Where(e => e.Micro).MinBy(e => e.CreatedUtc)!.Key); + while (_totalBytes + sizeBytes > maxBytes && _entries.Count > 0) { var victim = _entries.Values.OrderBy(e => e.CreatedUtc).First(); Remove(victim.Key); } - if (_totalBytes + sizeBytes > maxBytes) - return false; - _entries[key] = new Entry { Key = key, UserId = userId, Url = url, - ItemIds = itemIds - .Where(id => !string.IsNullOrWhiteSpace(id)) - .Select(id => id.Trim().ToLowerInvariant()) - .ToHashSet(StringComparer.OrdinalIgnoreCase), + ItemCount = itemCount, Body = body, ContentType = contentType, CreatedUtc = DateTimeOffset.UtcNow, DurationMs = durationMs, - SizeBytes = sizeBytes + SizeBytes = sizeBytes, + Micro = micro }; _totalBytes += sizeBytes; return true; } } - public bool StoreMicro(string key, string userId, string url, IEnumerable itemIds, string body, string contentType, long durationMs, long maxBytes) - { - if (maxBytes <= 0) - return false; - - var sizeBytes = System.Text.Encoding.UTF8.GetByteCount(body); - if (sizeBytes > maxBytes) - return false; - - lock (_lock) - { - PurgeExpired(_microEntries, MicroTtl); - _microEntries.Remove(key); - - while (_microEntries.Count >= MaxMicroEntries) - { - var victim = _microEntries.Values.OrderBy(e => e.CreatedUtc).First(); - _microEntries.Remove(victim.Key); - } - - _microEntries[key] = new Entry - { - Key = key, - UserId = userId, - Url = url, - ItemIds = itemIds - .Where(id => !string.IsNullOrWhiteSpace(id)) - .Select(id => id.Trim().ToLowerInvariant()) - .ToHashSet(StringComparer.OrdinalIgnoreCase), - Body = body, - ContentType = contentType, - CreatedUtc = DateTimeOffset.UtcNow, - DurationMs = durationMs, - SizeBytes = sizeBytes - }; - return true; - } - } + public bool StoreMicro(string key, string userId, string url, int itemCount, string body, string contentType, long durationMs, long maxBytes, long? generation = null, TimeSpan? ttl = null) + => Store(key, userId, url, itemCount, body, contentType, durationMs, maxBytes, ttl ?? TimeSpan.FromMinutes(1), generation, micro: true); public object[] GetEntries() { @@ -207,6 +159,7 @@ public sealed class ItemsProxyCache { var now = DateTimeOffset.UtcNow; return _entries.Values + .Where(e => !e.Micro) .OrderByDescending(e => e.CreatedUtc) .Select(e => new { @@ -215,7 +168,7 @@ public sealed class ItemsProxyCache AgeSeconds = (long)(now - e.CreatedUtc).TotalSeconds, e.DurationMs, e.SizeBytes, - ItemCount = e.ItemIds.Count + ItemCount = e.ItemCount }) .Cast() .ToArray(); @@ -288,26 +241,16 @@ public sealed class ItemsProxyCache return; var now = DateTimeOffset.UtcNow; - foreach (var key in _entries.Values.Where(e => now - e.CreatedUtc > ttl).Select(e => e.Key).ToArray()) + foreach (var key in _entries.Values.Where(e => now - e.CreatedUtc > (e.Micro ? MicroTtl : ttl)).Select(e => e.Key).ToArray()) Remove(key); } - private static void PurgeExpired(Dictionary entries, TimeSpan ttl) - { - if (ttl <= TimeSpan.Zero || entries.Count == 0) - return; - - var now = DateTimeOffset.UtcNow; - foreach (var key in entries.Values.Where(e => now - e.CreatedUtc > ttl).Select(e => e.Key).ToArray()) - entries.Remove(key); - } - private void Remove(string key) { if (!_entries.Remove(key, out var existing)) return; - _totalBytes = Math.Max(0, _totalBytes - existing.SizeBytes); + _totalBytes -= existing.SizeBytes; } } diff --git a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyInvalidationService.cs b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyInvalidationService.cs new file mode 100644 index 0000000..0133412 --- /dev/null +++ b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyInvalidationService.cs @@ -0,0 +1,35 @@ +using Jellyfin.Data.Events; +using Jellyfin.Database.Implementations.Entities; +using MediaBrowser.Controller.Library; +using Microsoft.Extensions.Hosting; + +namespace Jellyfin.Plugin.Multilang.Services; + +public sealed class ItemsProxyInvalidationService( + ILibraryManager library, IUserManager users, IUserDataManager userData, ItemsProxyCache cache) : IHostedService +{ + public Task StartAsync(CancellationToken cancellationToken) + { + library.ItemAdded += LibraryChanged; + library.ItemUpdated += LibraryChanged; + library.ItemRemoved += LibraryChanged; + users.OnUserUpdated += UserChanged; + userData.UserDataSaved += UserDataChanged; + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + library.ItemAdded -= LibraryChanged; + library.ItemUpdated -= LibraryChanged; + library.ItemRemoved -= LibraryChanged; + users.OnUserUpdated -= UserChanged; + userData.UserDataSaved -= UserDataChanged; + return Task.CompletedTask; + } + + // Membership and native filters can change even when an item wasn't in a cached result. + private void LibraryChanged(object? sender, ItemChangeEventArgs args) => cache.ClearAll(); + private void UserChanged(object? sender, GenericEventArgs args) => cache.ClearUser(args.Argument.Id.ToString("N")); + private void UserDataChanged(object? sender, UserDataSaveEventArgs args) => cache.ClearUser(args.UserId.ToString("N")); +} diff --git a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyPrecacheService.cs b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyPrecacheService.cs index 74a7eae..2194a36 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyPrecacheService.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyPrecacheService.cs @@ -60,8 +60,7 @@ public sealed class ItemsProxyPrecacheService { try { - var port = _configurationManager.GetNetworkConfiguration().InternalHttpPort; - var baseUri = new Uri(_applicationHost.GetLocalApiUrl("127.0.0.1", "http", port).TrimEnd('/') + "/"); + var baseUri = LocalApiUri; var views = await GetJsonAsync(new Uri(baseUri, $"Users/{userId}/Views"), token).ConfigureAwait(false); var items = views["Items"]?.AsArray() ?? []; var targets = items @@ -86,7 +85,7 @@ public sealed class ItemsProxyPrecacheService using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(baseUri, endpoint)); AddTokenHeaders(request, token); - using var response = await _httpClientFactory.CreateClient().SendAsync(request, CancellationToken.None).ConfigureAwait(false); + using var response = await _httpClientFactory.CreateClient("Multilang.Jellyfin").SendAsync(request, CancellationToken.None).ConfigureAwait(false); if (!response.IsSuccessStatusCode) _logger.LogWarning("Multilang pre-cache failed user={UserId} library={LibraryId} status={StatusCode}", userId, target.Id, (int)response.StatusCode); } @@ -106,7 +105,7 @@ public sealed class ItemsProxyPrecacheService { using var request = new HttpRequestMessage(HttpMethod.Get, uri); AddTokenHeaders(request, token); - using var response = await _httpClientFactory.CreateClient().SendAsync(request, CancellationToken.None).ConfigureAwait(false); + using var response = await _httpClientFactory.CreateClient("Multilang.Jellyfin").SendAsync(request, CancellationToken.None).ConfigureAwait(false); response.EnsureSuccessStatusCode(); var body = await response.Content.ReadAsStringAsync(CancellationToken.None).ConfigureAwait(false); return JsonNode.Parse(body)?.AsObject() ?? throw new InvalidOperationException("Jellyfin views response was not a JSON object."); @@ -118,4 +117,7 @@ public sealed class ItemsProxyPrecacheService request.Headers.TryAddWithoutValidation("X-Emby-Token", token); request.Headers.TryAddWithoutValidation("X-MediaBrowser-Token", token); } + + public Uri LocalApiUri => new(_applicationHost.GetLocalApiUrl("127.0.0.1", "http", + _configurationManager.GetNetworkConfiguration().InternalHttpPort).TrimEnd('/') + "/"); } diff --git a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyRequestBuilder.cs b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyRequestBuilder.cs index 021bad5..4dc1ed3 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyRequestBuilder.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyRequestBuilder.cs @@ -15,28 +15,23 @@ public sealed record ItemsProxyRequest( public static class ItemsProxyRequestBuilder { - public static ItemsProxyRequest? Build(HttpRequest request, string rawUrl, string token, string userId, bool multilangEnabled) + private const string BrowseFields = "PrimaryImageAspectRatio,MediaSourceCount,SortName,Overview,Genres,GenreItems"; + public static ItemsProxyRequest? Build(HttpRequest request, string rawUrl, string userId, bool multilangEnabled) { - var baseUri = $"{request.Scheme}://{request.Host}{request.PathBase}"; - var baseAddress = new Uri(baseUri); - var uri = new Uri(baseAddress, rawUrl.StartsWith('/') ? rawUrl : "/" + rawUrl); - if ((rawUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || - rawUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) && - Uri.TryCreate(rawUrl, UriKind.Absolute, out var absolute)) - { - if (!absolute.Scheme.Equals(Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) && - !absolute.Scheme.Equals(Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) - return null; + var baseAddress = new Uri($"{request.Scheme}://{request.Host}{request.PathBase}/"); + if (rawUrl.Contains('\\') || !Uri.TryCreate(baseAddress, rawUrl, out var uri) || + uri.Scheme != baseAddress.Scheme || uri.Authority != baseAddress.Authority || + uri.UserInfo.Length > 0 || uri.Fragment.Length > 0) + return null; - if (!absolute.Host.Equals(baseAddress.Host, StringComparison.OrdinalIgnoreCase) || - absolute.Port != baseAddress.Port) - return null; - - uri = absolute; - } - - var path = NormalizeUpstreamPath(uri.AbsolutePath, userId); - var builder = new UriBuilder(uri) { Path = path }; + var pathBase = request.PathBase.Value ?? string.Empty; + if (!uri.AbsolutePath.StartsWith(pathBase + "/", StringComparison.OrdinalIgnoreCase)) + return null; + var route = uri.AbsolutePath[pathBase.Length..]; + if (!IsSupportedRoute(route)) + return null; + var path = NormalizeUpstreamPath(route, userId); + var builder = new UriBuilder(uri) { Path = pathBase + path }; var query = QueryHelpers.ParseQuery(builder.Query); var rawControls = new ItemsProxyControls( GetQueryString(query, "SortBy"), @@ -48,36 +43,25 @@ public static class ItemsProxyRequestBuilder GetQueryString(query, "GenreIds")); var hasLocalGenreFilter = multilangEnabled && ParseLocalGenreIds(rawControls.GenreIds).Length > 0; var useLocalControls = multilangEnabled && ShouldUseLocalControls(path, query, hasLocalGenreFilter); + if (useLocalControls) + query["Fields"] = string.Join(',', (BrowseFields + "," + GetQueryString(query, "Fields")) + .Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries) + .Distinct(StringComparer.OrdinalIgnoreCase).OrderBy(field => field, StringComparer.OrdinalIgnoreCase)); var controls = useLocalControls ? rawControls - : new ItemsProxyControls(string.Empty, string.Empty, string.Empty, 0, 0, rawControls.ClientLocale, rawControls.GenreIds); - var localCacheKey = string.Join('|', - controls.SortBy, - controls.SortOrder, - controls.NameStartsWith, - controls.StartIndex.ToString(CultureInfo.InvariantCulture), - controls.Limit.ToString(CultureInfo.InvariantCulture), - controls.ClientLocale, - controls.GenreIds); + : new ItemsProxyControls(string.Empty, string.Empty, string.Empty, 0, 0, rawControls.ClientLocale, rawControls.GenreIds, ApplyLocalPaging: false); var pairs = query .Where(kv => !IsLocalProxyQuery(kv.Key, hasLocalGenreFilter, useLocalControls)) .SelectMany(kv => kv.Value.Select(v => KeyValuePair.Create(kv.Key, v ?? string.Empty))) .ToList(); pairs.Add(KeyValuePair.Create("userId", userId)); - if (hasLocalGenreFilter) - { - pairs.Add(KeyValuePair.Create("EnableImageTypes", "Primary,Backdrop,Banner,Thumb")); - pairs.Add(KeyValuePair.Create("Fields", "Genres,GenreItems")); - } - - var cacheQuery = QueryHelpers.AddQueryString(string.Empty, pairs).TrimStart('?'); - pairs.Add(KeyValuePair.Create("api_key", token)); - builder.Query = QueryHelpers.AddQueryString(string.Empty, pairs).TrimStart('?'); + var cacheQuery = QueryHelpers.AddQueryString(string.Empty, pairs.OrderBy(p => p.Key, StringComparer.OrdinalIgnoreCase)).TrimStart('?'); + builder.Query = cacheQuery; var normalizedForCache = builder.Path + (cacheQuery.Length > 0 ? "?" + cacheQuery : string.Empty); return new ItemsProxyRequest( builder.Uri, controls, - $"{userId}|{normalizedForCache}|{localCacheKey}", + $"{userId}|{normalizedForCache}", normalizedForCache, IsGenresPath(path), GenreMediaFromQuery(query)); @@ -92,6 +76,22 @@ public static class ItemsProxyRequestBuilder .Distinct() .ToArray(); + private static bool IsSupportedRoute(string path) + { + var parts = path.ToLowerInvariant().Split('/', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 3 && parts[0] == "users" && Guid.TryParse(parts[1], out _)) + parts = parts[2..]; + return parts switch + { + ["items"] or ["genres"] => true, + ["items", "latest" or "resume" or "nextup"] => true, + ["items" or "genres", var id] => Guid.TryParse(id, out _), + ["shows", "nextup"] => true, + ["shows", var id, "seasons" or "episodes"] => Guid.TryParse(id, out _), + _ => false + }; + } + private static bool ShouldUseLocalControls( string normalizedPath, IReadOnlyDictionary query, @@ -136,17 +136,14 @@ public static class ItemsProxyRequestBuilder key.Equals("Limit", StringComparison.OrdinalIgnoreCase) || key.Equals("NameStartsWith", StringComparison.OrdinalIgnoreCase))) || (hasGenreFilter && (key.Equals("GenreIds", StringComparison.OrdinalIgnoreCase) || - key.Equals("EnableTotalRecordCount", StringComparison.OrdinalIgnoreCase) || - key.Equals("EnableImageTypes", StringComparison.OrdinalIgnoreCase) || - key.Equals("Fields", StringComparison.OrdinalIgnoreCase))); + key.Equals("EnableTotalRecordCount", StringComparison.OrdinalIgnoreCase))); private static string NormalizeUpstreamPath(string path, string userId) { var segments = path.Split('/', StringSplitOptions.RemoveEmptyEntries).ToList(); if (segments.Count >= 2 && segments[0].Equals("Users", StringComparison.OrdinalIgnoreCase) && - segments[1].Length == 32 && - segments[1].All(Uri.IsHexDigit)) + Guid.TryParse(segments[1], out _)) { segments[1] = userId; return "/" + string.Join('/', segments); diff --git a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxySorting.cs b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxySorting.cs index c71ddae..25324dd 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxySorting.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxySorting.cs @@ -12,7 +12,8 @@ public sealed record ItemsProxyControls( int StartIndex, int Limit, string ClientLocale, - string GenreIds); + string GenreIds, + bool ApplyLocalPaging = true); public static class ItemsProxySorting { @@ -33,6 +34,8 @@ public static class ItemsProxySorting public static void Apply(JsonNode root, ItemsProxyControls controls, CultureInfo culture, Func titleArticlesForItem) { + if (!controls.ApplyLocalPaging) + return; if (!TryGetItemsArray(root, out var itemsArray)) return; diff --git a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyTransformer.cs b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyTransformer.cs index 73063ea..a15da40 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyTransformer.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyTransformer.cs @@ -22,6 +22,9 @@ public readonly record struct ItemsProxyCategoryMatch( UserCategoryRule? Effective, bool DuplicateLabelResolved); +public sealed record ItemsProxyResolutionAttempt( + string Action, string Source, string LookupKey, bool HasValue, bool Chosen, string? Value, string Reason); + public sealed class ItemsProxyTransformer { private readonly TranslationStore _store; @@ -157,8 +160,6 @@ public sealed class ItemsProxyTransformer { foreach (var action in GetAllActionTokens(category.FieldActionLists)) AddLanguageAction(langs, action); - foreach (var action in (category.FieldActions ?? []).Values) - AddLanguageAction(langs, action); } foreach (var action in GetAllActionTokens(rules.FallbackFieldActions)) @@ -184,29 +185,28 @@ public sealed class ItemsProxyTransformer } public FactsData GetClassificationFacts(UserRulesDocument rules, FactsData facts) + => GetClassificationFacts(rules, new Dictionary { [facts.ItemId] = facts })[facts.ItemId]; + + private IReadOnlyDictionary GetClassificationFacts(UserRulesDocument rules, IReadOnlyDictionary facts) { - if (rules.TrustTmdbCollections || - !facts.Kind.Equals("collection", StringComparison.OrdinalIgnoreCase) || - !Guid.TryParseExact(facts.ItemId, "N", out var itemGuid)) - { + if (rules.TrustTmdbCollections) return facts; - } + var children = facts.Values.Where(fact => fact.Kind == "collection") + .ToDictionary(fact => fact.ItemId, fact => + (_libraryManager.GetItemById(Guid.Parse(fact.ItemId)) as Folder)?.GetLinkedChildren() + .OfType() + .Select(child => TranslationStore.ToItemId32(child.Id)).ToArray() ?? []); + var childFacts = _store.GetFactsForItems(children.Values.SelectMany(ids => ids)); + var result = new Dictionary(facts, StringComparer.OrdinalIgnoreCase); + foreach (var (id, ids) in children) + result[id] = AggregateCollectionFacts(facts[id], ids.Where(childFacts.ContainsKey).Select(id => childFacts[id]).ToArray()); + return result; + } - var item = _libraryManager.GetItemById(itemGuid); - var children = (item as Folder)?.GetLinkedChildren() - .Where(child => child.GetType().Name.Equals("Movie", StringComparison.OrdinalIgnoreCase)) - .ToArray() ?? []; - - var childIds = children.Select(child => TranslationStore.ToItemId32(child.Id)).ToArray(); - if (childIds.Length == 0) - return facts; - - var childFacts = _store.GetFactsForItems(childIds).Values - .Where(child => child.Kind.Equals("movie", StringComparison.OrdinalIgnoreCase)) - .ToArray(); + public static FactsData AggregateCollectionFacts(FactsData facts, FactsData[] childFacts) + { if (childFacts.Length == 0) return facts; - return facts with { OriginalLanguage = SharedSingle(childFacts.Select(child => child.OriginalLanguage)), @@ -228,74 +228,61 @@ public sealed class ItemsProxyTransformer if (actions is not null && actions.TryGetValue(field, out var list)) return list; - return category is not null && category.FieldActions is not null - ? LegacyActionToList(field, category.FieldActions) - : [JellyfinAction]; + return [JellyfinAction]; } public static ItemsProxyResolvedText ResolveField( - string field, - IReadOnlyList actions, - FactsData facts, - Dictionary>? byLang) + string field, IReadOnlyList actions, FactsData facts, + Dictionary>? byLang, + Action? trace = null) { foreach (var action in actions) { if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)) - return new ItemsProxyResolvedText(false, null); - if (field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) && - action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase) && - !string.IsNullOrWhiteSpace(facts.OriginalTitle)) - return new ItemsProxyResolvedText(true, facts.OriginalTitle, facts.OriginalLanguage); - if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase)) { - var originalValue = GetTranslatedField(byLang, OriginalAction, field); - if (!string.IsNullOrWhiteSpace(originalValue)) - return new ItemsProxyResolvedText(true, originalValue, facts.OriginalLanguage); - continue; + trace?.Invoke(new(action, JellyfinAction, "", true, true, null, "Use Jellyfin value")); + return new(false, null); } - - if (!action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)) + var original = action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase); + if (!original && !action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)) continue; - - var value = GetTranslatedField(byLang, action[LanguagePrefix.Length..], field); - if (!string.IsNullOrWhiteSpace(value)) - return new ItemsProxyResolvedText(true, value, action[LanguagePrefix.Length..]); + var lang = original ? OriginalAction : action[LanguagePrefix.Length..]; + var originalTitle = original && field == TitleField && !string.IsNullOrWhiteSpace(facts.OriginalTitle); + var value = originalTitle ? facts.OriginalTitle : GetTranslatedField(byLang, lang, field); + var found = !string.IsNullOrWhiteSpace(value); + trace?.Invoke(new(action, originalTitle ? "OriginalTitle" : "Translation", + originalTitle ? "facts.original_title" : lang + "/" + field, + found, found, value, found ? "Value found" : "Value missing or empty")); + if (found) + return new(true, value, original ? facts.OriginalLanguage : lang); } - - return field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) - ? new ItemsProxyResolvedText(false, null) - : new ItemsProxyResolvedText(true, string.Empty); + return field == TitleField ? new(false, null) : new(true, string.Empty); } public static ItemsProxyResolvedAsset ResolveAsset( - string kind, - IReadOnlyList actions, - Dictionary>? byKind) + string kind, IReadOnlyList actions, + Dictionary>? byKind, + Action? trace = null) { foreach (var action in actions) { if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)) - return new ItemsProxyResolvedAsset(false, null); - if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase)) { - var originalValue = GetAsset(byKind, kind, OriginalAction); - if (!string.IsNullOrWhiteSpace(originalValue)) - return new ItemsProxyResolvedAsset(true, originalValue); - continue; + trace?.Invoke(new(action, JellyfinAction, "", true, true, null, "Use Jellyfin image")); + return new(false, null); } - - if (!action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)) + var original = action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase); + if (!original && !action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)) continue; - - var value = GetAsset(byKind, kind, action[LanguagePrefix.Length..]); - if (!string.IsNullOrWhiteSpace(value)) - return new ItemsProxyResolvedAsset(true, value); + var lang = original ? OriginalAction : action[LanguagePrefix.Length..]; + var value = GetAsset(byKind, kind, lang); + var found = !string.IsNullOrWhiteSpace(value); + trace?.Invoke(new(action, "Asset", kind + "/" + lang, found, found, value, + found ? "Asset found" : "Asset missing")); + if (found) + return new(true, value); } - - return kind.Equals(PosterKind, StringComparison.OrdinalIgnoreCase) - ? new ItemsProxyResolvedAsset(false, null) - : new ItemsProxyResolvedAsset(true, string.Empty); + return kind == PosterKind ? new(false, null) : new(true, string.Empty); } public static string? GetTranslatedField( @@ -373,6 +360,7 @@ public sealed class ItemsProxyTransformer var langs = GetNeededLanguages(rules); var translations = _store.GetTranslations(ids, langs); var assets = _store.GetAssets(ids, langs); + var classification = GetClassificationFacts(rules, factsByItem); var titleLanguages = new Dictionary(StringComparer.OrdinalIgnoreCase); var movieGenreNames = _store.GetGenreNames("movie", clientLocale); var tvGenreNames = _store.GetGenreNames("tv", clientLocale); @@ -384,7 +372,7 @@ public sealed class ItemsProxyTransformer continue; translations.TryGetValue(itemId, out var byLang); - var classificationFacts = GetClassificationFacts(rules, facts); + var classificationFacts = classification[itemId]; var category = PickCategoryMatch(rules, classificationFacts).Effective; var title = ResolveField(TitleField, GetActionList(rules, category, TitleField), facts, byLang); @@ -583,15 +571,6 @@ public sealed class ItemsProxyTransformer langs.Add(action[LanguagePrefix.Length..]); } - private static string[] LegacyActionToList(string field, IReadOnlyDictionary? legacyActions) - { - if (legacyActions is null || !legacyActions.TryGetValue(field, out var action)) - return [JellyfinAction]; - if (string.IsNullOrWhiteSpace(action) || action.Equals(FallbackAction, StringComparison.OrdinalIgnoreCase)) - return [JellyfinAction]; - return [action]; - } - private static string SharedSingle(IEnumerable values) { var unique = values diff --git a/src/Jellyfin.Plugin.Multilang/Services/Providers/FanartClient.cs b/src/Jellyfin.Plugin.Multilang/Services/Providers/FanartClient.cs index 9aefb69..de69743 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Providers/FanartClient.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Providers/FanartClient.cs @@ -32,8 +32,9 @@ public sealed class FanartClient var url = $"https://webservice.fanart.tv/v3/movies/{Uri.EscapeDataString(tmdbId)}?api_key={Uri.EscapeDataString(apiKey)}"; using var request = new HttpRequestMessage(HttpMethod.Get, url); using var response = await ProviderHttp.SendWithRetryAsync(http, request, cancellationToken).ConfigureAwait(false); - if (!response.IsSuccessStatusCode) + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) return null; + response.EnsureSuccessStatusCode(); await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); using var doc = await System.Text.Json.JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); diff --git a/src/Jellyfin.Plugin.Multilang/Services/Providers/FetchRateLimiter.cs b/src/Jellyfin.Plugin.Multilang/Services/Providers/FetchRateLimiter.cs index 4070464..7d79a65 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Providers/FetchRateLimiter.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Providers/FetchRateLimiter.cs @@ -6,6 +6,7 @@ public sealed class FetchRateLimiter { private readonly TimeSpan _minimumSpacing; private long _lastStartTicks; + private readonly SemaphoreSlim _admission = new(1, 1); public FetchRateLimiter(TimeSpan minimumSpacing) { @@ -14,15 +15,18 @@ public sealed class FetchRateLimiter public async Task WaitAsync(CancellationToken cancellationToken) { - var last = Interlocked.Read(ref _lastStartTicks); - if (last != 0) + await _admission.WaitAsync(cancellationToken).ConfigureAwait(false); + try { - var elapsed = TimeSpan.FromSeconds((Stopwatch.GetTimestamp() - last) / (double)Stopwatch.Frequency); - var remaining = _minimumSpacing - elapsed; - if (remaining > TimeSpan.Zero) - await Task.Delay(remaining, cancellationToken).ConfigureAwait(false); - } + if (_lastStartTicks != 0) + { + var remaining = _minimumSpacing - Stopwatch.GetElapsedTime(_lastStartTicks); + if (remaining > TimeSpan.Zero) + await Task.Delay(remaining, cancellationToken).ConfigureAwait(false); + } - Interlocked.Exchange(ref _lastStartTicks, Stopwatch.GetTimestamp()); + _lastStartTicks = Stopwatch.GetTimestamp(); + } + finally { _admission.Release(); } } } diff --git a/src/Jellyfin.Plugin.Multilang/Services/Providers/ProviderHttp.cs b/src/Jellyfin.Plugin.Multilang/Services/Providers/ProviderHttp.cs index 6ce9fca..4bb9878 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Providers/ProviderHttp.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Providers/ProviderHttp.cs @@ -39,29 +39,33 @@ public static class ProviderHttp public static async Task SendWithRetryAsync( HttpClient http, HttpRequestMessage request, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + FetchRateLimiter? rateLimiter = null) { const int maxAttempts = 3; for (var attempt = 1; attempt <= maxAttempts; attempt++) { try { - var response = await http.SendAsync(Clone(request), cancellationToken).ConfigureAwait(false); + if (rateLimiter is not null) + await rateLimiter.WaitAsync(cancellationToken).ConfigureAwait(false); + using var attemptRequest = Clone(request); + var response = await http.SendAsync(attemptRequest, cancellationToken).ConfigureAwait(false); + Record(request, "HTTP " + (int)response.StatusCode, attempt); if (response.IsSuccessStatusCode || Classify(response.StatusCode) == ProviderFailureKind.Terminal || attempt == maxAttempts) { - Record(request, "HTTP " + (int)response.StatusCode, attempt); return response; } var delay = GetRetryDelay(response, attempt); response.Dispose(); - await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + await Task.Delay(delay < TimeSpan.Zero ? TimeSpan.Zero : delay, cancellationToken).ConfigureAwait(false); } - catch (Exception ex) + catch (Exception ex) when (!cancellationToken.IsCancellationRequested && ex is HttpRequestException or OperationCanceledException) { + Record(request, ex.GetType().Name, attempt); if (attempt >= maxAttempts) { - Record(request, ex.GetType().Name, attempt); throw; } diff --git a/src/Jellyfin.Plugin.Multilang/Services/Providers/TmdbClient.cs b/src/Jellyfin.Plugin.Multilang/Services/Providers/TmdbClient.cs index 3c59bdf..ab315de 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Providers/TmdbClient.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Providers/TmdbClient.cs @@ -23,11 +23,10 @@ public sealed class TmdbClient if (string.IsNullOrWhiteSpace(apiKey)) throw new InvalidOperationException("TMDb API key is required."); - await rateLimiter.WaitAsync(cancellationToken).ConfigureAwait(false); var http = _httpClientFactory.CreateClient(); var url = $"https://api.themoviedb.org/3/{BuildPath(item)}?api_key={Uri.EscapeDataString(apiKey)}&language={Uri.EscapeDataString(language)}"; using var request = new HttpRequestMessage(HttpMethod.Get, url); - using var response = await ProviderHttp.SendWithRetryAsync(http, request, cancellationToken).ConfigureAwait(false); + using var response = await ProviderHttp.SendWithRetryAsync(http, request, cancellationToken, rateLimiter).ConfigureAwait(false); if (response.StatusCode == HttpStatusCode.NotFound) return null; response.EnsureSuccessStatusCode(); @@ -58,11 +57,10 @@ public sealed class TmdbClient if (item.Kind is "tvepisode") return new TmdbImages([], [], []); - await rateLimiter.WaitAsync(cancellationToken).ConfigureAwait(false); var http = _httpClientFactory.CreateClient(); var url = $"https://api.themoviedb.org/3/{BuildPath(item)}/images?api_key={Uri.EscapeDataString(apiKey)}"; using var request = new HttpRequestMessage(HttpMethod.Get, url); - using var response = await ProviderHttp.SendWithRetryAsync(http, request, cancellationToken).ConfigureAwait(false); + using var response = await ProviderHttp.SendWithRetryAsync(http, request, cancellationToken, rateLimiter).ConfigureAwait(false); if (response.StatusCode == HttpStatusCode.NotFound) return null; response.EnsureSuccessStatusCode(); diff --git a/src/Jellyfin.Plugin.Multilang/Services/Refresh/ItemFetcher.cs b/src/Jellyfin.Plugin.Multilang/Services/Refresh/ItemFetcher.cs new file mode 100644 index 0000000..2b946e1 --- /dev/null +++ b/src/Jellyfin.Plugin.Multilang/Services/Refresh/ItemFetcher.cs @@ -0,0 +1,220 @@ +using Jellyfin.Plugin.Multilang.Configuration; +using Jellyfin.Plugin.Multilang.Data; +using Jellyfin.Plugin.Multilang.Services.Assets; +using Jellyfin.Plugin.Multilang.Services.Providers; +using Microsoft.Extensions.Logging; +using static Jellyfin.Plugin.Multilang.MultilangConstants; + +namespace Jellyfin.Plugin.Multilang.Services.Refresh; + +public sealed class ItemFetcher( + TranslationStore store, TmdbClient tmdb, FanartClient fanart, + AssetStorageService storage, ILogger logger) +{ + private readonly FetchRateLimiter _limiter = new(TimeSpan.FromMilliseconds(250)); + + private static string Scope(RefreshItemInfo item) + => $"{item.Kind}:{item.TmdbId}:{item.SeasonNumber}:{item.EpisodeNumber}"; + + public static bool NeedsFetch(FetchState? state, string scope, bool dataMissing, bool full, long cutoff) + => full || state is null || state.Scope != scope || + (dataMissing && (state.Complete || state.CheckedAt <= cutoff)) || + (!state.Complete && state.CheckedAt <= cutoff); + + public bool NeedsLocalWork(string itemId, PluginConfiguration cfg) + => cfg.AssetStorageMode == "local" && store.GetStoredAssets(itemId).Any(asset => asset.SourceUrl.Length > 0 && + (!store.TryNormalizeLocalAssetPath(asset.Path, out var path) || !File.Exists(path))); + + public bool NeedsRefresh(RefreshItemInfo item, PluginConfiguration cfg) + { + var states = store.GetFetchStates(item.ItemId); + var languages = Languages(cfg); + var texts = store.GetTranslations([item.ItemId], languages.Append(OriginalAction)).GetValueOrDefault(item.ItemId) ?? []; + var cutoff = MissingCutoff(cfg); + var original = store.GetFacts(item.ItemId)?.OriginalLanguage ?? ""; + foreach (var lang in languages) + if (NeedsMetadata(item, lang, lang, texts, states, false, cutoff)) + return true; + if (original.Length > 0 && !languages.Any(lang => SameLanguage(lang, original)) && + NeedsMetadata(item, OriginalAction, original, texts, states, false, cutoff)) + return true; + + if (original.Length > 0 && languages.Any(lang => SameLanguage(lang, original)) && !texts.ContainsKey(OriginalAction)) + return true; + var assets = store.GetStoredAssets(item.ItemId); + if (cfg.AssetStorageMode == "local" && assets.Any(asset => asset.SourceUrl.Length > 0 && + (!store.TryNormalizeLocalAssetPath(asset.Path, out var path) || !File.Exists(path)))) + return true; + var providers = Providers(item, cfg); + var scope = ArtworkScope(item, languages, original, providers); + return providers.Any(provider => NeedsFetch(states.GetValueOrDefault("artwork:" + provider), scope, + ArtworkMissing(assets, languages.Append(OriginalAction).Where(lang => lang != OriginalAction || original.Length > 0), providers, provider), false, cutoff)); + } + + public async Task RefreshAsync(RefreshItemInfo item, PluginConfiguration cfg, RefreshJobType job, CancellationToken ct) + { + var full = job == RefreshJobType.Full; + var localOnly = job == RefreshJobType.Aggregate; + var states = store.GetFetchStates(item.ItemId); + var languages = Languages(cfg); + var texts = store.GetTranslations([item.ItemId], languages.Append(OriginalAction)).GetValueOrDefault(item.ItemId) ?? []; + var cutoff = MissingCutoff(cfg); + var fetched = 0; + async Task FetchMetadata(string storedLanguage, string requestedLanguage) + { + if (localOnly || !NeedsMetadata(item, storedLanguage, requestedLanguage, texts, states, full, cutoff)) + return; + var metadata = await tmdb.FetchMetadataAsync(item, requestedLanguage, cfg.TmdbApiKey, _limiter, ct).ConfigureAwait(false); + var complete = metadata is not null && metadata.Title.Length > 0 && metadata.Overview.Length > 0 && + (item.Kind != "movie" || metadata.Tagline.Length > 0); + store.SaveMetadata(item, storedLanguage, metadata, + new(Scope(item) + ":" + requestedLanguage, TranslationStore.NowUnixUtc(), complete), full, + item.Kind is "movie" or "tv" || storedLanguage == languages[0]); + if (metadata is not null) + Interlocked.Increment(ref fetched); + if (cfg.EnableLogging) + logger.LogInformation("TMDb metadata {Result} item={ItemId} lang={Language} storedAs={StoredLanguage}", + metadata is null ? "404" : "fetched", item.ItemId, requestedLanguage, storedLanguage); + } + + await Parallel.ForEachAsync(languages, new ParallelOptions { MaxDegreeOfParallelism = 4, CancellationToken = ct }, + async (lang, _) => await FetchMetadata(lang, lang).ConfigureAwait(false)).ConfigureAwait(false); + var original = store.GetFacts(item.ItemId)?.OriginalLanguage ?? ""; + if (original.Length > 0) + { + var configured = languages.FirstOrDefault(lang => SameLanguage(lang, original)); + if (configured is not null) + store.CopyOriginalTranslations(item.ItemId, configured); + else + await FetchMetadata(OriginalAction, original).ConfigureAwait(false); + } + + var artworkFetched = await RefreshArtwork(item, cfg, languages, original, states, full, localOnly, cutoff, ct).ConfigureAwait(false); + if (!localOnly) + store.MarkFactsChecked(item, TranslationStore.NowUnixUtc(), full ? TranslationStore.NowUnixUtc() : 0); + return fetched > 0 || artworkFetched; + } + + private static bool NeedsMetadata(RefreshItemInfo item, string storedLanguage, string requestedLanguage, + Dictionary> texts, Dictionary states, bool full, long cutoff) + { + var fields = texts.GetValueOrDefault(storedLanguage); + var missing = fields is null || TextFields.Any(field => !fields.ContainsKey(field) || + ((field != TaglineField || item.Kind == "movie") && string.IsNullOrWhiteSpace(fields[field]))); + return NeedsFetch(states.GetValueOrDefault("metadata:" + storedLanguage), + Scope(item) + ":" + requestedLanguage, missing, full, cutoff); + } + + private async Task RefreshArtwork(RefreshItemInfo item, PluginConfiguration cfg, string[] languages, + string original, Dictionary states, bool full, bool localOnly, long cutoff, CancellationToken ct) + { + if (localOnly) + { + if (cfg.AssetStorageMode != "local") + return false; + var assets = store.GetStoredAssets(item.ItemId); + var repaired = new List(); + foreach (var asset in assets) + { + var path = asset.SourceUrl.Length == 0 ? asset.Path + : await storage.StoreAsync(item.ItemId, asset.SourceUrl, true, ct).ConfigureAwait(false); + repaired.Add(asset with { Path = path! }); + } + store.SaveArtwork(item.ItemId, repaired, new Dictionary()); + return true; + } + var providers = Providers(item, cfg); + var priority = providers.Select((provider, index) => (provider, index)).ToDictionary(p => p.provider, p => p.index); + var wanted = languages.ToDictionary(lang => lang, BaseLanguage, StringComparer.OrdinalIgnoreCase); + if (original.Length > 0) + wanted[OriginalAction] = BaseLanguage(original); + var previous = store.GetStoredAssets(item.ItemId); + var selected = previous.Where(asset => wanted.ContainsKey(asset.Language) && priority.ContainsKey(asset.Provider)) + .ToDictionary(asset => (asset.Language, asset.Kind)); + var updates = new Dictionary(); + var scope = ArtworkScope(item, languages, original, providers); + foreach (var provider in providers) + { + var kinds = ArtworkKinds(provider); + var keys = wanted.Keys.SelectMany(lang => kinds.Select(kind => (Language: lang, Kind: kind))).ToArray(); + var missing = keys.Any(key => !selected.TryGetValue(key, out var asset) || priority[asset.Provider] > priority[provider]); + if (localOnly || !NeedsFetch(states.GetValueOrDefault("artwork:" + provider), scope, missing, full, cutoff)) + continue; + + Dictionary> images; + if (provider == "tmdb") + { + var response = await tmdb.FetchImagesAsync(item, cfg.TmdbApiKey, _limiter, ct).ConfigureAwait(false); + images = new() { [PosterKind] = response?.Posters ?? [], [LogoKind] = response?.Logos ?? [], [BackdropKind] = response?.Backdrops ?? [] }; + } + else + { + var response = await fanart.FetchMovieImagesAsync(item.TmdbId, cfg.FanartApiKey, ct).ConfigureAwait(false); + images = new() { [PosterKind] = response?.Posters ?? [], [LogoKind] = response?.Logos ?? [], [BannerKind] = response?.Banners ?? [], [ThumbKind] = response?.Thumbs ?? [] }; + } + + var complete = true; + foreach (var key in keys) + { + selected.TryGetValue(key, out var existing); + var url = images[key.Kind].GetValueOrDefault(wanted[key.Language]); + if (string.IsNullOrWhiteSpace(url)) + { + complete = false; + if (existing?.Provider == provider) + selected.Remove(key); + } + else if (existing is null || priority[provider] <= priority[existing.Provider]) + { + // Keep an existing local path until the scheduled storage-mode cleanup. + var path = existing?.SourceUrl == url ? existing.Path : url; + selected[key] = new(key.Language, key.Kind, path, url, provider); + } + } + updates["artwork:" + provider] = new(scope, TranslationStore.NowUnixUtc(), complete); + if (cfg.EnableLogging && cfg.VerboseLogging) + logger.LogInformation("Artwork fetched item={ItemId} provider={Provider} languages={Languages}", + item.ItemId, provider, string.Join(",", wanted.Keys)); + } + + var changed = updates.Count > 0; + if (cfg.AssetStorageMode == "local") + { + foreach (var key in selected.Keys.ToArray()) + { + var asset = selected[key]; + var path = await storage.StoreAsync(item.ItemId, asset.SourceUrl, true, ct).ConfigureAwait(false); + selected[key] = asset with { Path = path! }; + changed |= path != asset.Path; + } + } + var result = selected.Values.ToArray(); + if (changed || !previous.OrderBy(a => a.Language).ThenBy(a => a.Kind).SequenceEqual(result.OrderBy(a => a.Language).ThenBy(a => a.Kind))) + store.SaveArtwork(item.ItemId, result, updates); + return changed; + } + + private static string[] ArtworkKinds(string provider) + => provider == "tmdb" ? [PosterKind, LogoKind, BackdropKind] : [PosterKind, LogoKind, BannerKind, ThumbKind]; + + private static bool ArtworkMissing(IEnumerable assets, IEnumerable languages, string[] providers, string provider) + { + var available = assets.Where(asset => Array.IndexOf(providers, asset.Provider) is var rank && rank >= 0 && rank <= Array.IndexOf(providers, provider)) + .Select(asset => (asset.Language, asset.Kind)).ToHashSet(); + return languages.Any(lang => ArtworkKinds(provider).Any(kind => !available.Contains((lang, kind)))); + } + + private static string[] Providers(RefreshItemInfo item, PluginConfiguration cfg) + => item.Kind == "tvepisode" ? [] : cfg.Providers.Where(p => p.ArtworkOrder >= 0) + .OrderBy(p => p.ArtworkOrder).Select(p => p.Id.Trim().ToLowerInvariant()) + .Where(p => p == "tmdb" || (p == "fanart" && item.Kind == "movie" && !string.IsNullOrWhiteSpace(cfg.FanartApiKey))).ToArray(); + + private static string ArtworkScope(RefreshItemInfo item, string[] languages, string original, string[] providers) + => Scope(item) + "|" + string.Join(",", languages.Order(StringComparer.OrdinalIgnoreCase)) + "|" + original + "|" + string.Join(",", providers); + private static long MissingCutoff(PluginConfiguration cfg) + => TranslationStore.NowUnixUtc() - (long)TimeSpan.FromDays(Math.Max(0, cfg.WaitDaysForMissingData)).TotalSeconds; + private static string[] Languages(PluginConfiguration cfg) + => cfg.Languages.Select(lang => lang.Trim()).Where(lang => lang.Length > 0).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); + private static string BaseLanguage(string lang) => lang.Split('-')[0].ToLowerInvariant(); + private static bool SameLanguage(string left, string right) => BaseLanguage(left) == BaseLanguage(right); +} diff --git a/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshCoordinator.cs b/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshCoordinator.cs index 24261fa..9569d4e 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshCoordinator.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshCoordinator.cs @@ -3,7 +3,7 @@ using System.Diagnostics; namespace Jellyfin.Plugin.Multilang.Services.Refresh; -public sealed class RefreshCoordinator +public sealed class RefreshCoordinator : IDisposable { private sealed class QueueEntry { @@ -20,22 +20,23 @@ public sealed class RefreshCoordinator public required TaskCompletionSource Completion { get; init; } public long Sequence { get; init; } + public bool Active { get; set; } } private readonly object _lock = new(); private readonly Dictionary _entries = new(StringComparer.OrdinalIgnoreCase); private readonly Queue _recent = new(); private readonly SemaphoreSlim _signal = new(0, int.MaxValue); - private readonly ItemsProxyCache _itemsProxyCache; private long _sequence; - private QueueEntry? _active; + private readonly CancellationTokenSource _stop = new(); public SemaphoreSlim ScanMutex { get; } = new(1, 1); - public RefreshCoordinator(ItemsProxyCache itemsProxyCache) + public RefreshCoordinator(int concurrency = MultilangConstants.RefreshConcurrency) { - _itemsProxyCache = itemsProxyCache; - _ = Task.Run(WorkerLoop); + ArgumentOutOfRangeException.ThrowIfLessThan(concurrency, 1); + for (var i = 0; i < concurrency; i++) + _ = Task.Run(WorkerLoop); } public Task EnqueueAsync( @@ -53,6 +54,7 @@ public sealed class RefreshCoordinator lock (_lock) { + ObjectDisposedException.ThrowIf(_stop.IsCancellationRequested, this); if (_entries.TryGetValue(itemId, out var existing)) { if (jobType > existing.JobType) @@ -64,7 +66,7 @@ public sealed class RefreshCoordinator if (sourceTier > existing.SourceTier) existing.SourceTier = sourceTier; - return existing.Completion.Task; + return cancellationToken.CanBeCanceled ? existing.Completion.Task.WaitAsync(cancellationToken) : existing.Completion.Task; } var entry = new QueueEntry @@ -80,7 +82,7 @@ public sealed class RefreshCoordinator _entries.Add(itemId, entry); _signal.Release(); - return entry.Completion.Task; + return cancellationToken.CanBeCanceled ? entry.Completion.Task.WaitAsync(cancellationToken) : entry.Completion.Task; } } @@ -88,7 +90,8 @@ public sealed class RefreshCoordinator { while (true) { - await _signal.WaitAsync().ConfigureAwait(false); + try { await _signal.WaitAsync(_stop.Token).ConfigureAwait(false); } + catch (OperationCanceledException) { return; } QueueEntry? entry; lock (_lock) @@ -102,12 +105,24 @@ public sealed class RefreshCoordinator var error = string.Empty; try { - lock (_lock) - _active = entry; - - ok = await entry.Work(entry.JobType, CancellationToken.None).ConfigureAwait(false); - if (ok) - _itemsProxyCache.InvalidateItems([entry.ItemId]); + while (true) + { + RefreshJobType executing; + Func> work; + lock (_lock) + { + executing = entry.JobType; + work = entry.Work; + } + ok |= await work(executing, _stop.Token).ConfigureAwait(false); + lock (_lock) + { + if (entry.JobType > executing) + continue; + _entries.Remove(entry.ItemId); + break; + } + } entry.Completion.TrySetResult(ok); } catch (Exception ex) @@ -120,9 +135,8 @@ public sealed class RefreshCoordinator sw.Stop(); lock (_lock) { - if (ReferenceEquals(_active, entry)) - _active = null; - _entries.Remove(entry.ItemId); + if (_entries.TryGetValue(entry.ItemId, out var current) && ReferenceEquals(current, entry)) + _entries.Remove(entry.ItemId); _recent.Enqueue(new RefreshRecentItem( entry.ItemId, entry.SourceTier.ToString(), @@ -144,11 +158,14 @@ public sealed class RefreshCoordinator if (_entries.Count == 0) return null; - return _entries.Values + var entry = _entries.Values + .Where(e => !e.Active) .OrderByDescending(e => e.SourceTier) .ThenBy(e => e.WorkClass) .ThenBy(e => e.Sequence) - .First(); + .FirstOrDefault(); + if (entry is not null) entry.Active = true; + return entry; } public RefreshQueueDiagnostics GetDiagnostics() @@ -156,7 +173,7 @@ public sealed class RefreshCoordinator lock (_lock) { var queued = _entries.Values - .Where(e => !ReferenceEquals(e, _active)) + .Where(e => !e.Active) .OrderByDescending(e => e.SourceTier) .ThenBy(e => e.WorkClass) .ThenBy(e => e.Sequence) @@ -166,7 +183,7 @@ public sealed class RefreshCoordinator return new RefreshQueueDiagnostics( queued.Length, queued, - _active is null ? null : ToDiagnostics(_active), + _entries.Values.Where(e => e.Active).Select(ToDiagnostics).ToArray(), _recent.Reverse().ToArray()); } } @@ -178,4 +195,15 @@ public sealed class RefreshCoordinator entry.WorkClass.ToString(), entry.JobType.ToString(), entry.Sequence); + + public void Dispose() + { + _stop.Cancel(); + lock (_lock) + { + foreach (var entry in _entries.Values) + entry.Completion.TrySetCanceled(); + _entries.Clear(); + } + } } diff --git a/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshModels.cs b/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshModels.cs index 7c76f97..115f2a6 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshModels.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshModels.cs @@ -53,7 +53,7 @@ public sealed record RefreshRecentItem( public sealed record RefreshQueueDiagnostics( int QueuedCount, RefreshQueueEntry[] Queued, - RefreshQueueEntry? Active, + RefreshQueueEntry[] ActiveItems, RefreshRecentItem[] Recent); public sealed record RefreshScanDiagnostics( diff --git a/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshService.cs b/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshService.cs index 3eed2e1..364b8cc 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshService.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshService.cs @@ -2,7 +2,6 @@ using System.Diagnostics; using Jellyfin.Data.Enums; using Jellyfin.Plugin.Multilang.Configuration; using Jellyfin.Plugin.Multilang.Data; -using Jellyfin.Plugin.Multilang.Services.Assets; using Jellyfin.Plugin.Multilang.Services.Providers; using static Jellyfin.Plugin.Multilang.MultilangConstants; using MediaBrowser.Controller.Entities; @@ -16,9 +15,7 @@ public sealed class RefreshService { private readonly ILibraryManager _libraryManager; private readonly TranslationStore _store; - private readonly TmdbClient _tmdbClient; - private readonly FanartClient _fanartClient; - private readonly AssetStorageService _assetStorage; + private readonly ItemFetcher _fetcher; private readonly RefreshCoordinator _coordinator; private readonly ILogger _logger; private readonly object _diagnosticsLock = new(); @@ -27,17 +24,13 @@ public sealed class RefreshService public RefreshService( ILibraryManager libraryManager, TranslationStore store, - TmdbClient tmdbClient, - FanartClient fanartClient, - AssetStorageService assetStorage, + ItemFetcher fetcher, RefreshCoordinator coordinator, ILogger logger) { _libraryManager = libraryManager; _store = store; - _tmdbClient = tmdbClient; - _fanartClient = fanartClient; - _assetStorage = assetStorage; + _fetcher = fetcher; _coordinator = coordinator; _logger = logger; } @@ -151,7 +144,6 @@ public sealed class RefreshService UpdateRunningScan(); var items = GetAllItemsWithTmdbId(); var now = TranslationStore.NowUnixUtc(); - var missingCutoff = now - (long)TimeSpan.FromDays(Math.Max(0, cfg.WaitDaysForMissingData)).TotalSeconds; var fullCutoff = now - (long)TimeSpan.FromDays(Math.Max(0, cfg.WaitDaysForExistingData)).TotalSeconds; var ordered = items.OrderBy(i => i.WorkClass).ToList(); itemCount = ordered.Count; @@ -162,7 +154,14 @@ public sealed class RefreshService // Run local housekeeping only from full-library scans. ItemsProxy browsing may enqueue // missing items, but it must not delete local assets just because the admin briefly // changed artwork storage mode. - var localCleanup = _store.CleanupForConfiguration(liveIds, NormalizeLanguages(cfg.Languages), IsLocalAssetStorage(cfg)); + LocalCleanupResult localCleanup; + using (await _store.EnterMaintenanceAsync(cancellationToken).ConfigureAwait(false)) + { + cfg = Plugin.Instance?.Configuration ?? throw new InvalidOperationException("Plugin unavailable."); + if (Plugin.Instance.Uninstalling) + throw new OperationCanceledException("Plugin is being uninstalled."); + localCleanup = _store.CleanupForConfiguration(liveIds, NormalizeLanguages(cfg.Languages), IsLocalAssetStorage(cfg)); + } if (localCleanup.TranslationsDeleted > 0 || localCleanup.AssetsDeleted > 0 || localCleanup.GenresDeleted > 0 || localCleanup.AssetFilesDeleted > 0) { _logger.LogInformation( @@ -176,72 +175,90 @@ public sealed class RefreshService _logger.LogInformation("Multilang refresh started mode={Mode} source={Source} items={Items} new={New} deleted={Deleted}", jobType, runSource, ordered.Count, newCount, deletedCount); var index = 0; UpdateRunningScan(); - foreach (var item in ordered) - { - cancellationToken.ThrowIfCancellationRequested(); - index++; - SetCurrent(item, index, "checking"); - var status = _store.GetFactsStatus(item.ItemId); - var isNew = !status.Exists; - var missingConfiguredData = jobType == RefreshJobType.Missing && status.Exists && ItemNeedsMissingRefresh(item, cfg, langs: null); - var due = jobType == RefreshJobType.Full - ? !status.Exists || status.FullCheckedAt < fullCutoff - : !status.Exists || status.MissingCheckedAt < missingCutoff || missingConfiguredData; - - if (!due) + var completed = 0; + var scanLock = new object(); + foreach (var group in ordered.GroupBy(item => item.WorkClass)) + await Parallel.ForEachAsync(group, + new ParallelOptions { MaxDegreeOfParallelism = RefreshConcurrency, CancellationToken = cancellationToken }, + async (item, ct) => { - skippedNotDueCount++; - if (cfg.VerboseLogging && cfg.EnableLogging) + ct.ThrowIfCancellationRequested(); + var scanIndex = Interlocked.Increment(ref index); + var status = _store.GetFactsStatus(item.ItemId); + var isNew = !status.Exists; + var fullDue = jobType == RefreshJobType.Full && (!status.Exists || status.FullCheckedAt < fullCutoff); + var due = fullDue || (jobType == RefreshJobType.Missing + ? !status.Exists || _fetcher.NeedsRefresh(item, cfg) + : _fetcher.NeedsLocalWork(item.ItemId, cfg)); + + if (!due) { - _logger.LogInformation( - "Multilang refresh skip [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind} reason=not-due", - index, - ordered.Count, - item.ItemId, - item.DisplayName, - item.Kind); + Interlocked.Increment(ref skippedNotDueCount); + if (cfg.VerboseLogging && cfg.EnableLogging) + { + _logger.LogInformation( + "Multilang refresh skip [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind} reason=not-due", + scanIndex, + ordered.Count, + item.ItemId, + item.DisplayName, + item.Kind); + } + + lock (scanLock) + { + SetCurrent(item, completed + 1, "skipped-not-due"); + progress.Report(++completed * 100.0 / Math.Max(1, ordered.Count)); + UpdateRunningScan(); + } + return; } - progress.Report(index * 100.0 / Math.Max(1, ordered.Count)); - UpdateRunningScan("skipped-not-due"); - continue; - } + var actualJobType = fullDue || !status.Exists ? RefreshJobType.Full + : jobType == RefreshJobType.Full ? RefreshJobType.Aggregate : RefreshJobType.Missing; + Interlocked.Increment(ref dueCount); + Log( + cfg.EnableLogging, + LogLevel.Information, + "Multilang refresh fetch [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind} tmdb={TmdbId} mode={Mode} new={New}", + scanIndex, + ordered.Count, + item.ItemId, + item.DisplayName, + item.Kind, + item.TmdbId, + actualJobType, + isNew); - var scanIndex = index; - var actualJobType = jobType == RefreshJobType.Full || !status.Exists ? RefreshJobType.Full : RefreshJobType.Missing; - dueCount++; - SetCurrent(item, scanIndex, "fetching"); - Log( - cfg.EnableLogging, - LogLevel.Information, - "Multilang refresh fetch [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind} tmdb={TmdbId} mode={Mode} new={New}", - scanIndex, - ordered.Count, - item.ItemId, - item.DisplayName, - item.Kind, - item.TmdbId, - actualJobType, - isNew); + var ok = await _coordinator.EnqueueAsync( + item.ItemId, + RefreshSourceTier.Background, + item.WorkClass, + actualJobType, + async (actualJobType, ct) => await RefreshItemCoreAsync(item, actualJobType, ct).ConfigureAwait(false), + ct).ConfigureAwait(false); + lock (scanLock) + { + if (ok) + refreshedCount++; + else + skippedNoDataCount++; - var ok = await _coordinator.EnqueueAsync( - item.ItemId, - RefreshSourceTier.Background, - item.WorkClass, - actualJobType, - async (actualJobType, ct) => await RefreshItemCoreAsync(item, cfg, actualJobType, scanIndex, ordered.Count, ct).ConfigureAwait(false), - cancellationToken).ConfigureAwait(false); - if (ok) - refreshedCount++; - else - skippedNoDataCount++; + progress.Report(++completed * 100.0 / Math.Max(1, ordered.Count)); + SetCurrent(item, completed, ok ? "refreshed" : "skipped-no-data"); + UpdateRunningScan(); + } + }).ConfigureAwait(false); - progress.Report(index * 100.0 / Math.Max(1, ordered.Count)); - UpdateRunningScan(ok ? "refreshed" : "skipped-no-data"); + using (await _store.EnterMaintenanceAsync(cancellationToken).ConfigureAwait(false)) + { + if (Plugin.Instance!.Uninstalling) + throw new OperationCanceledException("Plugin is being uninstalled."); + _store.CleanupUnreferencedAssetFiles(); + _store.SetLastScanStarted(now); } sw.Stop(); - _store.SetLastScanStarted(now); CompleteScan(true, string.Empty); _logger.LogInformation( "Multilang refresh complete mode={Mode} source={Source} elapsed={ElapsedMs}ms items={Items} due={Due} refreshed={Refreshed} skippedNotDue={SkippedNotDue} skippedNoData={SkippedNoData} new={New} deleted={Deleted}", @@ -268,7 +285,8 @@ public sealed class RefreshService } } - public async Task RefreshItemAsync(Guid itemId, bool includeChildren, CancellationToken cancellationToken) + public async Task RefreshItemAsync(Guid itemId, bool includeChildren, CancellationToken cancellationToken, + Jellyfin.Database.Implementations.Entities.User user) { var cfg = Plugin.Instance?.Configuration ?? throw new InvalidOperationException("Plugin configuration unavailable."); ValidateRefreshConfiguration(cfg); @@ -279,43 +297,46 @@ public sealed class RefreshService return; } - var items = BuildItemInfos(GetItemAndChildren(root, includeChildren)).ToList(); + var items = BuildItemInfos(GetItemAndChildren(root, includeChildren) + .Where(item => item.IsVisibleStandalone(user))).ToList(); var total = items.Count; var index = 0; _logger.LogInformation("Manual Multilang refresh started item={ItemId} name={Name} items={Items} includeChildren={IncludeChildren}", itemId, root.Name ?? string.Empty, total, includeChildren); - foreach (var item in items) - { - index++; - var scanIndex = index; - Log( - cfg.EnableLogging, - LogLevel.Information, - "Manual Multilang refresh fetch [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind} tmdb={TmdbId}", - scanIndex, - total, - item.ItemId, - item.DisplayName, - item.Kind, - item.TmdbId); + foreach (var group in items.OrderBy(item => item.WorkClass).GroupBy(item => item.WorkClass)) + await Parallel.ForEachAsync(group, + new ParallelOptions { MaxDegreeOfParallelism = RefreshConcurrency, CancellationToken = cancellationToken }, + async (item, ct) => + { + var scanIndex = Interlocked.Increment(ref index); + Log( + cfg.EnableLogging, + LogLevel.Information, + "Manual Multilang refresh fetch [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind} tmdb={TmdbId}", + scanIndex, + total, + item.ItemId, + item.DisplayName, + item.Kind, + item.TmdbId); - var ok = await _coordinator.EnqueueAsync( - item.ItemId, - RefreshSourceTier.Manual, - item.WorkClass, - RefreshJobType.Full, - async (actualJobType, ct) => await RefreshItemCoreAsync(item, cfg, actualJobType, scanIndex, total, ct).ConfigureAwait(false), - cancellationToken).ConfigureAwait(false); - Log( - cfg.EnableLogging, - ok ? LogLevel.Information : LogLevel.Warning, - "Manual Multilang refresh item {Status} [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind}", - ok ? "complete" : "skipped", - scanIndex, - total, - item.ItemId, - item.DisplayName, - item.Kind); - } + var ok = await _coordinator.EnqueueAsync( + item.ItemId, + RefreshSourceTier.Manual, + item.WorkClass, + RefreshJobType.Full, + async (actualJobType, ct) => await RefreshItemCoreAsync(item, actualJobType, ct).ConfigureAwait(false), + ct).ConfigureAwait(false); + Log( + cfg.EnableLogging, + ok ? LogLevel.Information : LogLevel.Warning, + "Manual Multilang refresh item {Status} [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind}", + ok ? "complete" : "skipped", + scanIndex, + total, + item.ItemId, + item.DisplayName, + item.Kind); + }).ConfigureAwait(false); } public bool EnqueueOnTheFlyMissing(IEnumerable itemIds) @@ -365,7 +386,7 @@ public sealed class RefreshService sourceTier, info.WorkClass, RefreshJobType.Full, - async (actualJobType, ct) => await RefreshItemCoreAsync(info, cfg, actualJobType, 0, 0, ct).ConfigureAwait(false), + async (actualJobType, ct) => await RefreshItemCoreAsync(info, actualJobType, ct).ConfigureAwait(false), CancellationToken.None) .ContinueWith( task => @@ -388,245 +409,16 @@ public sealed class RefreshService private async Task RefreshItemCoreAsync( RefreshItemInfo item, - PluginConfiguration cfg, RefreshJobType jobType, - int scanIndex, - int totalItems, CancellationToken cancellationToken) { - var langs = NormalizeLanguages(cfg.Languages); - if (langs.Length == 0) - throw new InvalidOperationException("No Multilang languages are configured."); - - if (cfg.VerboseLogging && cfg.EnableLogging) - _logger.LogInformation("[{Index}/{Total}] Fetching item={ItemId} name=\"{Name}\" kind={Kind} tmdb={TmdbId}", scanIndex, totalItems, item.ItemId, item.DisplayName, item.Kind, item.TmdbId); - - var now = TranslationStore.NowUnixUtc(); - var limiter = new FetchRateLimiter(TimeSpan.FromMilliseconds(250)); - var wroteAny = false; - TmdbMetadata? firstMeta = null; - var fetchedMetadata = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (var lang in langs) - { - var meta = await _tmdbClient.FetchMetadataAsync(item, lang, cfg.TmdbApiKey, limiter, cancellationToken).ConfigureAwait(false); - if (meta is null) - { - Log(cfg.EnableLogging, LogLevel.Information, "[{Index}/{Total}] TMDb 404, skipping item={ItemId} name=\"{Name}\" tmdb={TmdbId} kind={Kind}", scanIndex, totalItems, item.ItemId, item.DisplayName, item.TmdbId, item.Kind); - _store.MarkFactsChecked(item, now, jobType == RefreshJobType.Full ? now : 0); - return false; - } - - Log(cfg.VerboseLogging && cfg.EnableLogging, LogLevel.Information, "[{Index}/{Total}] TMDb metadata ok item={ItemId} lang={Lang}", scanIndex, totalItems, item.ItemId, lang); - _store.UpsertTranslation(item.ItemId, lang, TitleField, meta.Title); - _store.UpsertTranslation(item.ItemId, lang, OverviewField, meta.Overview); - _store.UpsertTranslation(item.ItemId, lang, TaglineField, meta.Tagline); - StoreGenres(item, lang, meta); - fetchedMetadata[lang] = meta; - firstMeta ??= meta; - if (!wroteAny) - { - _store.UpsertFacts(item, meta, now, jobType == RefreshJobType.Full ? now : 0); - wroteAny = true; - } - } - - var artworkLanguages = langs; - if (firstMeta is not null && !string.IsNullOrWhiteSpace(firstMeta.OriginalLanguage)) - { - var originalLang = firstMeta.OriginalLanguage.Trim(); - var existingLang = langs.FirstOrDefault(lang => SameLanguageBase(lang, originalLang)); - Log( - cfg.EnableLogging, - LogLevel.Information, - "[{Index}/{Total}] Original language item={ItemId} lang={OriginalLang} metadataSource={Source}", - scanIndex, - totalItems, - item.ItemId, - originalLang, - existingLang is not null ? "configured-language:" + existingLang : "extra-fetch"); - var originalMeta = existingLang is not null - ? fetchedMetadata[existingLang] - : await _tmdbClient.FetchMetadataAsync(item, originalLang, cfg.TmdbApiKey, limiter, cancellationToken).ConfigureAwait(false); - - if (originalMeta is not null) - { - _store.UpsertTranslation(item.ItemId, OriginalAction, TitleField, originalMeta.Title); - _store.UpsertTranslation(item.ItemId, OriginalAction, OverviewField, originalMeta.Overview); - _store.UpsertTranslation(item.ItemId, OriginalAction, TaglineField, originalMeta.Tagline); - StoreGenres(item, OriginalAction, originalMeta); - Log( - cfg.VerboseLogging && cfg.EnableLogging, - LogLevel.Information, - "[{Index}/{Total}] Original metadata stored item={ItemId} lang={OriginalLang} titlePresent={TitlePresent} overviewPresent={OverviewPresent} taglinePresent={TaglinePresent}", - scanIndex, - totalItems, - item.ItemId, - originalLang, - !string.IsNullOrWhiteSpace(originalMeta.Title), - !string.IsNullOrWhiteSpace(originalMeta.Overview), - !string.IsNullOrWhiteSpace(originalMeta.Tagline)); - } - else - { - Log( - cfg.EnableLogging, - LogLevel.Warning, - "[{Index}/{Total}] Original metadata unavailable item={ItemId} lang={OriginalLang}", - scanIndex, - totalItems, - item.ItemId, - originalLang); - } - - artworkLanguages = AppendOriginalLanguage(langs, originalLang); - Log( - cfg.VerboseLogging && cfg.EnableLogging, - LogLevel.Information, - "[{Index}/{Total}] Artwork language set item={ItemId} langs={Languages}", - scanIndex, - totalItems, - item.ItemId, - string.Join(",", artworkLanguages)); - } - - await StoreArtworkAsync(item, cfg, artworkLanguages, limiter, now, cancellationToken).ConfigureAwait(false); - - Log(cfg.EnableLogging, LogLevel.Information, "Multilang refresh item complete item={ItemId} kind={Kind} tmdb={TmdbId} mode={Mode}", item.ItemId, item.Kind, item.TmdbId, jobType); - return true; - } - - private void StoreGenres(RefreshItemInfo item, string lang, TmdbMetadata metadata) - { - var media = GenreMediaFor(item.Kind); - if (media is null) - return; - - foreach (var genre in metadata.Genres) - _store.UpsertGenre(genre.Id, media, lang, genre.Name); - } - - private static string? GenreMediaFor(string kind) - => kind.Equals("movie", StringComparison.OrdinalIgnoreCase) || - kind.Equals("collection", StringComparison.OrdinalIgnoreCase) - ? "movie" - : kind.Equals("tv", StringComparison.OrdinalIgnoreCase) || - kind.Equals("tvseason", StringComparison.OrdinalIgnoreCase) || - kind.Equals("tvepisode", StringComparison.OrdinalIgnoreCase) - ? "tv" - : null; - - private async Task StoreArtworkAsync( - RefreshItemInfo item, - PluginConfiguration cfg, - string[] languages, - FetchRateLimiter limiter, - long updatedAt, - CancellationToken cancellationToken) - { - if (item.Kind is "tvepisode") - return; - - var providers = cfg.Providers - .Where(p => p.ArtworkOrder >= 0) - .OrderBy(p => p.ArtworkOrder) - .Select(p => p.Id.Trim().ToLowerInvariant()) - .ToArray(); - if (providers.Length == 0) - return; - - TmdbImages? tmdbImages = null; - FanartImages? fanartImages = null; - var written = new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (var provider in providers) - { - cancellationToken.ThrowIfCancellationRequested(); - - if (provider.Equals("tmdb", StringComparison.OrdinalIgnoreCase)) - { - tmdbImages ??= await _tmdbClient.FetchImagesAsync(item, cfg.TmdbApiKey, limiter, cancellationToken).ConfigureAwait(false); - if (tmdbImages is null) - continue; - - await StoreImageMapAsync(item.ItemId, languages, PosterKind, tmdbImages.Posters, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false); - await StoreImageMapAsync(item.ItemId, languages, LogoKind, tmdbImages.Logos, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false); - await StoreImageMapAsync(item.ItemId, languages, BackdropKind, tmdbImages.Backdrops, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false); - Log(cfg.VerboseLogging && cfg.EnableLogging, LogLevel.Information, "TMDb artwork stored item={ItemId} langs={Languages}", item.ItemId, string.Join(",", languages)); - continue; - } - - if (provider.Equals("fanart", StringComparison.OrdinalIgnoreCase) && item.Kind.Equals("movie", StringComparison.OrdinalIgnoreCase)) - { - fanartImages ??= await _fanartClient.FetchMovieImagesAsync(item.TmdbId, cfg.FanartApiKey, cancellationToken).ConfigureAwait(false); - if (fanartImages is null) - continue; - - await StoreImageMapAsync(item.ItemId, languages, PosterKind, fanartImages.Posters, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false); - await StoreImageMapAsync(item.ItemId, languages, LogoKind, fanartImages.Logos, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false); - await StoreImageMapAsync(item.ItemId, languages, BannerKind, fanartImages.Banners, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false); - await StoreImageMapAsync(item.ItemId, languages, ThumbKind, fanartImages.Thumbs, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false); - Log(cfg.VerboseLogging && cfg.EnableLogging, LogLevel.Information, "Fanart artwork stored item={ItemId} langs={Languages}", item.ItemId, string.Join(",", languages)); - } - } - } - - private async Task StoreImageMapAsync( - string itemId, - IEnumerable languages, - string kind, - IReadOnlyDictionary images, - PluginConfiguration cfg, - long updatedAt, - HashSet written, - CancellationToken cancellationToken) - { - foreach (var lang in languages) - { - var storeLang = lang.StartsWith(OriginalAction + ":", StringComparison.OrdinalIgnoreCase) - ? OriginalAction - : lang; - var key = kind + ":" + storeLang; - if (written.Contains(key)) - continue; - - var iso = ToIso639(lang); - if (!images.TryGetValue(iso, out var url) || string.IsNullOrWhiteSpace(url)) - continue; - - var storedPath = await _assetStorage.StoreAsync(itemId, storeLang, kind, url, IsLocalAssetStorage(cfg), cancellationToken).ConfigureAwait(false); - if (string.IsNullOrWhiteSpace(storedPath)) - continue; - - _store.UpsertAsset(itemId, storeLang, kind, storedPath, updatedAt); - written.Add(key); - } - } - - private bool ItemNeedsMissingRefresh(RefreshItemInfo item, PluginConfiguration cfg, string[]? langs) - { - langs ??= NormalizeLanguages(cfg.Languages); - if (_store.HasMissingConfiguredTranslations(item.ItemId, langs)) - return true; - - var assetPresence = _store.GetAssetPresence(item.ItemId, langs.Append(OriginalAction)); - return IsLocalAssetStorage(cfg) && assetPresence.AnyRemote; - } - - private static string[] AppendOriginalLanguage(string[] languages, string originalLanguage) - => languages - .Concat([OriginalAction + ":" + originalLanguage]) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); - - private static bool SameLanguageBase(string left, string right) - => ToIso639(left).Equals(ToIso639(right), StringComparison.OrdinalIgnoreCase); - - private static string ToIso639(string language) - { - if (language.StartsWith(OriginalAction + ":", StringComparison.OrdinalIgnoreCase)) - language = language[(OriginalAction.Length + 1)..]; - var index = language.IndexOf('-', StringComparison.Ordinal); - return (index > 0 ? language[..index] : language).Trim().ToLowerInvariant(); + using var lease = await _store.EnterRefreshAsync(cancellationToken).ConfigureAwait(false); + var plugin = Plugin.Instance ?? throw new InvalidOperationException("Plugin unavailable."); + if (plugin.Uninstalling) + throw new OperationCanceledException("Plugin is being uninstalled."); + var cfg = plugin.Configuration; + ValidateRefreshConfiguration(cfg); + return await _fetcher.RefreshAsync(item, cfg, jobType, cancellationToken).ConfigureAwait(false); } private List GetAllItemsWithTmdbId() diff --git a/src/Jellyfin.Plugin.Multilang/wwwroot/inject.js b/src/Jellyfin.Plugin.Multilang/wwwroot/inject.js index 6aaa1bb..7a9cde1 100644 --- a/src/Jellyfin.Plugin.Multilang/wwwroot/inject.js +++ b/src/Jellyfin.Plugin.Multilang/wwwroot/inject.js @@ -1,16 +1,8 @@ (() => { - if (!Element.prototype.replaceChildren) { - Element.prototype.replaceChildren = function () { - this.textContent = ""; - for (let index = 0; index < arguments.length; index++) this.appendChild(arguments[index]); - }; - } - const BASE_PATH = window.location.pathname.split("/web/")[0] || ""; const MENU_ID = "mlUserRulesMenuItem"; const USER_RULES_ROUTE = "multilang/user-rules"; let lastMenuItemId = ""; - let userRulesRenderRun = 0; function getClientToken() { try { @@ -227,6 +219,7 @@ function closeUserRulesShellIfInactive() { if (isUserRulesRoute()) return; + document.querySelector("#ml-user-shell #ml-user")?._mlDispose?.(); document.getElementById("ml-user-shell")?.remove(); } @@ -234,7 +227,6 @@ event?.preventDefault?.(); event?.stopPropagation?.(); event?.stopImmediatePropagation?.(); - document.querySelector("#app-user-menu .MuiBackdrop-root")?.click(); history.pushState({}, "", userRulesHref()); scheduleUserRulesPageRender(); } @@ -250,32 +242,25 @@ const next = document.createElement("script"); [...script.attributes].forEach((attr) => next.setAttribute(attr.name, attr.value)); if (script.src) { - next.onload = () => resolve(); - next.onerror = () => reject(new Error(`Failed to load ${script.src}`)); + next.onload = () => { next.remove(); resolve(); }; + next.onerror = () => { next.remove(); reject(new Error(`Failed to load ${script.src}`)); }; next.src = script.src; } else { next.text = script.textContent || ""; } document.body.appendChild(next); - if (!script.src) resolve(); + if (!script.src) { next.remove(); resolve(); } }); } - async function renderUserRulesPage(run) { - if (!isUserRulesRoute() || run !== userRulesRenderRun) return; - - const host = userRulesHost(); - if (!host) return; - if (host.dataset.mlUserRulesRendered === "1" && document.getElementById("ml-user")) return; - - host.dataset.mlUserRulesRendered = "1"; + async function renderUserRulesPage(host) { host.innerHTML = "
Loading Multilang settings...
"; const response = await fetch(`${BASE_PATH}/web/configurationpage?name=MultilangUser`, { cache: "no-store" }); if (!response.ok) throw new Error(`HTTP ${response.status}`); const fragment = await response.text(); - if (run !== userRulesRenderRun || !isUserRulesRoute()) return; + if (!host.isConnected || !isUserRulesRoute()) return; const parsed = new DOMParser().parseFromString(fragment, "text/html"); const sourcePage = parsed.getElementById("ml-user"); @@ -289,26 +274,26 @@ page.id = "ml-user"; page.className = "ml-root"; while (sourceContent.firstChild) page.appendChild(sourceContent.firstChild); - host.replaceChildren(page); + host.textContent = ""; + host.appendChild(page); for (const script of scripts) { - if (run !== userRulesRenderRun || !isUserRulesRoute()) return; + if (!host.isConnected || !isUserRulesRoute()) return; await executeScript(script); } } function scheduleUserRulesPageRender() { - userRulesRenderRun++; if (!isUserRulesRoute()) { closeUserRulesShellIfInactive(); return; } - const run = userRulesRenderRun; - [0, 100, 350].forEach((delay) => { - setTimeout(() => { - renderUserRulesPage(run).catch((err) => console.error("[Multilang] failed to render user rules page", err)); - }, delay); + const host = userRulesHost(); + if (!host || host._mlLoad) return; + host._mlLoad = renderUserRulesPage(host).catch((err) => { + console.error("[Multilang] failed to render user rules page", err); + if (host.isConnected) host.textContent = `Unable to load Multilang settings: ${err.message}`; }); } @@ -325,7 +310,9 @@ function refreshItems(itemIds) { const token = getClientToken(); - const headers = token ? { "X-Emby-Token": token, "X-MediaBrowser-Token": token } : {}; + const headers = token + ? { Authorization: `MediaBrowser Token="${token}"`, "X-Emby-Token": token, "X-MediaBrowser-Token": token } + : {}; itemIds.forEach((itemId) => { fetch(`${BASE_PATH}/Multilang/RefreshItem/${itemId}?includeChildren=true${token ? `&token=${encodeURIComponent(token)}` : ""}`, { method: "POST", headers }); }); @@ -390,6 +377,11 @@ if (!url.startsWith("/")) url = "/" + url; if (BASE_PATH && !url.toLowerCase().startsWith(BASE_PATH.toLowerCase() + "/")) url = BASE_PATH + url; } + const local = new URL(url, window.location.origin); + if (local.origin === window.location.origin && local.pathname.startsWith(BASE_PATH + "/Multilang/Assets/")) { + local.searchParams.set("api_key", getClientToken()); + return local.href; + } return url; } diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/AssetStorageServiceTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/AssetStorageServiceTests.cs index fa487a6..8691957 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/AssetStorageServiceTests.cs +++ b/tests/Jellyfin.Plugin.Multilang.Tests/AssetStorageServiceTests.cs @@ -19,7 +19,7 @@ public sealed class AssetStorageServiceTests }); var service = new AssetStorageService(factory, store); - var path = await service.StoreAsync("item", "fi", "poster", "https://images.example/poster.jpg", true, CancellationToken.None); + var path = await service.StoreAsync("item", "https://images.example/poster.jpg", true, CancellationToken.None); Assert.NotNull(path); Assert.StartsWith(TranslationStore.LocalAssetUrlPrefix, path, StringComparison.Ordinal); @@ -28,6 +28,19 @@ public sealed class AssetStorageServiceTests Assert.True(service.TryResolveLocalAsset(path[TranslationStore.LocalAssetUrlPrefix.Length..], out var resolved, out var contentType)); Assert.Equal(fullPath, resolved); Assert.Equal("image/jpeg", contentType); + + store.UpsertAsset("item", "fi", "poster", path, 1); + var original = await service.StoreAsync("item", "https://images.example/poster.jpg", true, CancellationToken.None); + store.UpsertAsset("item", "Original", "poster", original!, 1); + Assert.Equal(path, original); + Assert.Single(factory.Requests); + Assert.True(store.IsAssetReferenced("item", path)); + Assert.True(File.Exists(fullPath)); + File.Delete(fullPath); + Assert.False(File.Exists(fullPath)); + await service.StoreAsync("item", "https://images.example/poster.jpg", true, CancellationToken.None); + Assert.True(File.Exists(fullPath)); + Assert.Equal(2, factory.Requests.Count); } finally { @@ -46,7 +59,7 @@ public sealed class AssetStorageServiceTests var service = new AssetStorageService(factory, store); const string url = "https://images.example/poster.webp"; - var result = await service.StoreAsync("item", "fi", "poster", url, false, CancellationToken.None); + var result = await service.StoreAsync("item", url, false, CancellationToken.None); Assert.Equal(url, result); Assert.Empty(factory.Requests); @@ -73,7 +86,7 @@ public sealed class AssetStorageServiceTests }); var service = new AssetStorageService(factory, store); - await Assert.ThrowsAsync(() => service.StoreAsync("item", "fi", "poster", "https://images.example/poster.jpg", true, CancellationToken.None)); + await Assert.ThrowsAsync(() => service.StoreAsync("item", "https://images.example/poster.jpg", true, CancellationToken.None)); Assert.Empty(Directory.EnumerateFiles(store.AssetsDirectory, "*", SearchOption.AllDirectories)); } diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/CategoryRuleEvaluatorTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/CategoryRuleEvaluatorTests.cs index 7032db7..110230b 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/CategoryRuleEvaluatorTests.cs +++ b/tests/Jellyfin.Plugin.Multilang.Tests/CategoryRuleEvaluatorTests.cs @@ -6,6 +6,24 @@ namespace Jellyfin.Plugin.Multilang.Tests; public sealed class CategoryRuleEvaluatorTests { + [Fact] + public void FastMatchingAndDiagnosticTraceAgreeAcrossAllRuleModes() + { + foreach (var relation in new[] { "is", "is_not", "contains", "not_contains" }) + foreach (var fieldOr in new[] { false, true }) + foreach (var valueOr in new[] { false, true }) + foreach (var matchAll in new[] { false, true }) + foreach (var facts in new[] { Facts(originalLanguage: "sv"), Facts(spokenLanguages: ["fi", "en"]), Facts(kind: "tv") }) + { + var rule = Rule("original_language", relation, valueOr, "sv", "fi"); + rule.Requirements[0].Fields = ["original_language", "spoken_languages"]; + rule.Requirements[0].UseFieldOr = fieldOr; + rule.Requirements = [rule.Requirements[0], Requirement("production_countries", "contains", false, "FI")]; + rule.MatchAllConditions = matchAll; + Assert.Equal(CategoryRuleEvaluator.Trace(rule, facts).Result, CategoryRuleEvaluator.Matches(rule, facts)); + } + } + [Fact] public void IsWithAndListRequiresExactSet() { @@ -95,11 +113,9 @@ public sealed class CategoryRuleEvaluatorTests { Id = rule.Id, Label = rule.Label, - CriteriaText = rule.CriteriaText, Requirements = rule.Requirements, MatchAllConditions = matchAll, Scopes = rule.Scopes, - FieldActions = rule.FieldActions, FieldActionLists = rule.FieldActionLists }; diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/ItemFetcherTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/ItemFetcherTests.cs new file mode 100644 index 0000000..d8f0356 --- /dev/null +++ b/tests/Jellyfin.Plugin.Multilang.Tests/ItemFetcherTests.cs @@ -0,0 +1,232 @@ +using System.Net; +using System.Text.Json; +using Jellyfin.Plugin.Multilang.Configuration; +using Jellyfin.Plugin.Multilang.Data; +using Jellyfin.Plugin.Multilang.Services.Assets; +using Jellyfin.Plugin.Multilang.Services.Providers; +using Jellyfin.Plugin.Multilang.Services.Refresh; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Jellyfin.Plugin.Multilang.Tests; + +public sealed class ItemFetcherTests : IDisposable +{ + private readonly string _root = TestPaths.CreateRoot(); + private readonly TranslationStore _store; + private readonly TestHttpClientFactory _http; + private readonly ItemFetcher _fetcher; + private readonly RefreshItemInfo _item = new("11111111111111111111111111111111", "1", "movie", RefreshWorkClass.Movies, 0, 0, null, "Movie"); + private readonly PluginConfiguration _config = new() + { + Languages = ["en", "fi"], + TmdbApiKey = "test", + Providers = [new() { Id = "tmdb", MetadataOrder = 0 }] + }; + private string _original = "en"; + private string _overview = "Overview"; + private bool _missingFinnishTagline; + private bool _notFound; + private bool _missingTmdbPoster; + private bool _fanartUnauthorized; + + public ItemFetcherTests() + { + _store = new(new TestApplicationPaths(_root)); + _http = new(request => + { + if (_notFound) return new(HttpStatusCode.NotFound); + var url = request.RequestUri!; + if (url.Host == "webservice.fanart.tv") + return _fanartUnauthorized ? new(HttpStatusCode.Unauthorized) + : Json(new { movieposter = new[] { new { lang = "en", url = "https://assets.fanart.tv/poster.jpg", likes = "5" } } }); + if (url.Host == "image.tmdb.org") + return new(HttpStatusCode.OK) { Content = new ByteArrayContent([1, 2, 3]) }; + if (url.AbsolutePath.EndsWith("/images")) + return Json(new { posters = _missingTmdbPoster ? [] : new[] { new { iso_639_1 = "en", file_path = "/poster.jpg" } }, logos = Array.Empty(), backdrops = Array.Empty() }); + var language = Microsoft.AspNetCore.WebUtilities.QueryHelpers.ParseQuery(url.Query)["language"].ToString(); + return Json(new + { + title = language + " title", + overview = _overview, + tagline = language == "fi" && _missingFinnishTagline ? "" : "Tagline", + original_language = _original, + original_title = "Original title", + genres = Array.Empty() + }); + }); + _fetcher = new(_store, new(_http), new(_http), new(_http, _store), NullLogger.Instance); + } + + private static HttpResponseMessage Json(object value) + => new(HttpStatusCode.OK) { Content = new StringContent(JsonSerializer.Serialize(value)) }; + private Task Fetch(RefreshJobType job = RefreshJobType.Missing) => _fetcher.RefreshAsync(_item, _config, job, CancellationToken.None); + private string Text(string language, string field) => _store.GetTranslations([_item.ItemId], [language])[_item.ItemId][language][field]; + + [Fact] + public async Task CompleteMetadataIsNotRefetchedAndOriginalReusesConfiguredLanguage() + { + Assert.True(await Fetch()); + Assert.Equal(2, _http.Requests.Count); + Assert.Equal(Text("en", "title"), Text("Original", "title")); + Assert.False(_fetcher.NeedsRefresh(_item, _config)); + Assert.False(await Fetch()); + Assert.Equal(2, _http.Requests.Count); + await Fetch(RefreshJobType.Full); + Assert.Equal(4, _http.Requests.Count); + } + + [Fact] + public async Task MissingRetryFetchesOnlyIncompleteLanguageAndUpdatesWholeResponse() + { + _missingFinnishTagline = true; + await Fetch(); + Assert.False(_fetcher.NeedsRefresh(_item, _config)); + await Fetch(); + Assert.Equal(2, _http.Requests.Count); + _config.WaitDaysForMissingData = 0; + Assert.True(_fetcher.NeedsRefresh(_item, _config)); + _missingFinnishTagline = false; + _overview = "Updated overview"; + await Fetch(); + Assert.Equal(3, _http.Requests.Count); + Assert.Contains("language=fi", _http.Requests.Last()); + Assert.Equal("Updated overview", Text("fi", "overview")); + Assert.Equal("Overview", Text("en", "overview")); + Assert.False(_fetcher.NeedsRefresh(_item, _config)); + } + + [Fact] + public async Task AdditionalOriginalLanguageIsFetchedOnceAndNewConfiguredLanguageDoesNotRefetchOthers() + { + _original = "sv"; + await Fetch(); + Assert.Equal(3, _http.Requests.Count); + Assert.Equal("sv title", Text("Original", "title")); + await Fetch(); + Assert.Equal(3, _http.Requests.Count); + _config.Languages = ["en", "fi", "fr"]; + await Fetch(); + Assert.Equal(4, _http.Requests.Count); + Assert.Contains("language=fr", _http.Requests.Last()); + } + + [Fact] + public async Task NotFoundIsRememberedUntilRetryCutoff() + { + _notFound = true; + Assert.False(await Fetch()); + Assert.False(_fetcher.NeedsRefresh(_item, _config)); + await Fetch(); + Assert.Equal(2, _http.Requests.Count); + _config.WaitDaysForMissingData = 0; + Assert.True(_fetcher.NeedsRefresh(_item, _config)); + await Fetch(); + Assert.Equal(4, _http.Requests.Count); + } + + [Fact] + public async Task LocalRepairDoesNotFetchMetadataAndUrlCleanupOnlyHappensInScheduledHousekeeping() + { + _config.Providers[0].ArtworkOrder = 0; + await Fetch(); + Assert.Equal(3, _http.Requests.Count); + _config.AssetStorageMode = "local"; + Assert.True(_fetcher.NeedsLocalWork(_item.ItemId, _config)); + await Fetch(RefreshJobType.Aggregate); + Assert.Equal(4, _http.Requests.Count); + Assert.Contains("image.tmdb.org", _http.Requests.Last()); + var path = _store.GetAssetPath(_item.ItemId, "en", "poster")!; + Assert.Equal(path, _store.GetAssetPath(_item.ItemId, "Original", "poster")); + Assert.True(_store.TryNormalizeLocalAssetPath(path, out var file)); + File.Delete(file); + await Fetch(); + Assert.Equal(5, _http.Requests.Count); + Assert.True(File.Exists(file)); + _config.AssetStorageMode = "url"; + await Fetch(); + Assert.Equal(path, _store.GetAssetPath(_item.ItemId, "en", "poster")); + Assert.True(File.Exists(file)); + _store.CleanupForConfiguration(new HashSet { _item.ItemId }, _config.Languages, false); + Assert.StartsWith("https://", _store.GetAssetPath(_item.ItemId, "en", "poster")); + Assert.False(File.Exists(file)); + Assert.Equal(5, _http.Requests.Count); + } + + [Fact] + public async Task ArtworkUsesProviderPriorityAndFallsBackWhenHigherPriorityImageDisappears() + { + _config.Providers = [new() { Id = "tmdb", MetadataOrder = 0, ArtworkOrder = 0 }, new() { Id = "fanart", ArtworkOrder = 1 }]; + _config.FanartApiKey = "test"; + await Fetch(); + Assert.Contains("image.tmdb.org", _store.GetAssetPath(_item.ItemId, "en", "poster")); + _missingTmdbPoster = true; + await Fetch(RefreshJobType.Full); + Assert.Contains("assets.fanart.tv", _store.GetAssetPath(_item.ItemId, "en", "poster")); + _fanartUnauthorized = true; + await Assert.ThrowsAsync(() => Fetch(RefreshJobType.Full)); + Assert.Contains("assets.fanart.tv", _store.GetAssetPath(_item.ItemId, "en", "poster")); + } + + [Fact] + public async Task BackupPreservesSourceUrlsAndFetchStateAndFiltersUnwantedLanguages() + { + _config.Providers[0].ArtworkOrder = 0; + await Fetch(); + var backup = Path.Combine(_root, "export.sqlite"); + _store.ExportTranslationDatabase(backup); + _store.ResetAll(); + _store.ImportTranslationDatabase(backup, new HashSet { _item.ItemId }, new HashSet { "en" }); + Assert.Equal(2, _store.GetFetchStates(_item.ItemId).Count); + Assert.DoesNotContain("metadata:fi", _store.GetFetchStates(_item.ItemId).Keys); + Assert.Equal("https://image.tmdb.org/t/p/original/poster.jpg", _store.GetStoredAssets(_item.ItemId).First().SourceUrl); + Assert.Equal("tmdb", _store.GetStoredAssets(_item.ItemId).First().Provider); + } + + [Theory] + [InlineData(true, false, false)] + [InlineData(true, true, true)] + [InlineData(false, false, false)] + [InlineData(false, true, false)] + public void YoungFetchStateHonorsKnownMissingVersusDeletedData(bool complete, bool missing, bool expected) + => Assert.Equal(expected, ItemFetcher.NeedsFetch(new("scope", 100, complete), "scope", missing, false, 50)); + + [Fact] + public void FailedArtworkPacketRollsBackDeletionAndNewRowsTogether() + { + var original = new StoredAsset("en", "poster", "https://example.test/old.jpg", "https://example.test/old.jpg", "tmdb"); + _store.SaveArtwork(_item.ItemId, [original], new Dictionary()); + Assert.Throws(() => _store.SaveArtwork(_item.ItemId, + [original with { Path = "new" }, original], new Dictionary())); + Assert.Equal(original, Assert.Single(_store.GetStoredAssets(_item.ItemId))); + } + + [Fact] + public async Task ParallelMetadataPacketsDoNotLoseRows() + { + await Task.WhenAll(Enumerable.Range(0, 12).Select(index => Task.Run(() => + _store.SaveMetadata(_item with { ItemId = index.ToString() }, "en", + new("Title", "Overview", "Tagline", "Title", "en", [], [], [], []), + new("scope", 1, true), true)))); + foreach (var index in Enumerable.Range(0, 12)) + { + Assert.NotNull(_store.GetFacts(index.ToString())); + Assert.Equal(3, _store.GetTranslations([index.ToString()], ["en"])[index.ToString()]["en"].Count); + Assert.True(_store.GetFetchStates(index.ToString())["metadata:en"].Complete); + } + } + + [Fact] + public void CollectionAggregationKeepsOnlySharedSingleLanguageAndUnionsCountrySets() + { + var collection = new FactsData("collection", "1", "collection", "Collection", "en", "en", "[]", "[]", "[]", "", "[]", 1, 1); + var finnish = collection with { Kind = "movie", OriginalLanguage = "fi", ProductionCountriesJson = "[\"FI\"]" }; + var swedish = finnish with { OriginalLanguage = "sv", ProductionCountriesJson = "[\"SE\",\"FI\"]" }; + var mixed = Services.ItemsProxyTransformer.AggregateCollectionFacts(collection, [finnish, swedish]); + Assert.Equal("", mixed.OriginalLanguage); + Assert.Equal("fi,sv", mixed.OriginalLanguageAll); + Assert.Equal(new HashSet { "FI", "SE" }, JsonSerializer.Deserialize>(mixed.ProductionCountriesJson)); + Assert.Equal("fi", Services.ItemsProxyTransformer.AggregateCollectionFacts(collection, [finnish, finnish]).OriginalLanguage); + } + + public void Dispose() => TestPaths.DeleteRoot(_root); +} diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyCacheTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyCacheTests.cs index 665b279..0f62adc 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyCacheTests.cs +++ b/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyCacheTests.cs @@ -5,16 +5,29 @@ namespace Jellyfin.Plugin.Multilang.Tests; public sealed class ItemsProxyCacheTests { [Fact] - public void StoreRetrievesAndInvalidatesByItemId() + public void PolicyChangesInvalidateCachedDataAndOldInFlightWrites() + { + var cache = new ItemsProxyCache(); + var generation = cache.ObserveUserPolicy("user", "all libraries"); + Store(cache, "key"); + Assert.Equal(generation, cache.ObserveUserPolicy("user", "all libraries")); + Assert.True(cache.TryGet("key", TimeSpan.FromMinutes(1), out _)); + Assert.True(cache.ObserveUserPolicy("user", "no libraries") > generation); + Assert.False(cache.TryGet("key", TimeSpan.FromMinutes(1), out _)); + Assert.False(cache.Store("key", "user", "/Items", 1, "body", "application/json", 1, 100, TimeSpan.FromMinutes(1), generation)); + } + + [Fact] + public void StoreRetrievesAndClearsCachedResponses() { var cache = new ItemsProxyCache(); - Assert.True(Store(cache, "key", ["A"])); + Assert.True(Store(cache, "key")); Assert.True(cache.TryGet("key", TimeSpan.FromMinutes(1), out var response)); Assert.Equal("body-key", response.Body); Assert.Equal(1, response.ItemCount); - cache.InvalidateItems(["a"]); + cache.ClearAll(); Assert.False(cache.TryGet("key", TimeSpan.FromMinutes(1), out _)); } @@ -24,11 +37,11 @@ public sealed class ItemsProxyCacheTests { var cache = new ItemsProxyCache(); - Assert.True(cache.StoreMicro("key", "user", "/Items", ["A"], "micro", "application/json", 1, 1024)); + Assert.True(cache.StoreMicro("key", "user", "/Items", 1, "micro", "application/json", 1, 1024)); Assert.True(cache.TryGet("key", TimeSpan.FromMinutes(1), out var response)); Assert.Equal("micro", response.Body); - cache.InvalidateItems(["A"]); + cache.ClearAll(); Assert.False(cache.TryGet("key", TimeSpan.FromMinutes(1), out _)); } @@ -38,9 +51,9 @@ public sealed class ItemsProxyCacheTests { var cache = new ItemsProxyCache(); - Assert.True(Store(cache, "first", ["A"], body: "1111", maxBytes: 6)); + Assert.True(Store(cache, "first", body: "1111", maxBytes: 6)); Thread.Sleep(10); - Assert.True(Store(cache, "second", ["B"], body: "2222", maxBytes: 6)); + Assert.True(Store(cache, "second", body: "2222", maxBytes: 6)); Assert.False(cache.TryGet("first", TimeSpan.FromMinutes(1), out _)); Assert.True(cache.TryGet("second", TimeSpan.FromMinutes(1), out _)); @@ -51,7 +64,7 @@ public sealed class ItemsProxyCacheTests { var cache = new ItemsProxyCache(); - Assert.True(Store(cache, "key", ["A"], ttl: TimeSpan.FromMilliseconds(1))); + Assert.True(Store(cache, "key", ttl: TimeSpan.FromMilliseconds(1))); Thread.Sleep(20); Assert.False(cache.TryGet("key", TimeSpan.FromMilliseconds(1), out _)); @@ -62,14 +75,14 @@ public sealed class ItemsProxyCacheTests { var cache = new ItemsProxyCache(); - Assert.False(Store(cache, "key", ["A"], body: "12345", maxBytes: 4)); + Assert.False(Store(cache, "key", body: "12345", maxBytes: 4)); Assert.False(cache.TryGet("key", TimeSpan.FromMinutes(1), out _)); } private static bool Store( ItemsProxyCache cache, string key, - string[] itemIds, + int itemCount = 1, string? body = null, long maxBytes = 1024, TimeSpan? ttl = null) @@ -77,10 +90,43 @@ public sealed class ItemsProxyCacheTests key, "user", "/Items", - itemIds, + itemCount, body ?? "body-" + key, "application/json", 100, maxBytes, ttl ?? TimeSpan.FromMinutes(1)); + + [Fact] + public void MicroAndMainEntriesShareOneBudget() + { + var cache = new ItemsProxyCache(); + Assert.True(Store(cache, "main", body: "1111", maxBytes: 6)); + Assert.True(cache.StoreMicro("micro1", "user", "/Items", 1, "2222", "application/json", 1, 6)); + Assert.False(cache.TryGet("main", TimeSpan.FromMinutes(1), out _)); + Assert.True(cache.StoreMicro("micro2", "user", "/Items", 1, "3333", "application/json", 1, 6)); + Assert.False(cache.TryGet("micro1", TimeSpan.FromMinutes(1), out _)); + Assert.True(cache.TryGet("micro2", TimeSpan.FromMinutes(1), out _)); + } + + [Fact] + public void InvalidationRejectsStaleInFlightWrites() + { + var cache = new ItemsProxyCache(); + var generation = cache.Generation; + cache.ClearAll(); + Assert.False(cache.Store("key", "user", "/Items", 1, "body", "application/json", 1, 100, TimeSpan.FromMinutes(1), generation)); + Assert.False(cache.StoreMicro("key", "user", "/Items", 1, "body", "application/json", 1, 100, generation)); + } + + [Fact] + public void ClearingOneUserPreservesOtherUsersCachedResponses() + { + var cache = new ItemsProxyCache(); + Store(cache, "one"); + cache.Store("two", "other", "/Items", 1, "body", "application/json", 1, 100, TimeSpan.FromMinutes(1)); + cache.ClearUser("user"); + Assert.False(cache.TryGet("one", TimeSpan.FromMinutes(1), out _)); + Assert.True(cache.TryGet("two", TimeSpan.FromMinutes(1), out _)); + } } diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyRequestBuilderTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyRequestBuilderTests.cs index 9c2d661..24a537e 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyRequestBuilderTests.cs +++ b/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyRequestBuilderTests.cs @@ -16,14 +16,13 @@ public sealed class ItemsProxyRequestBuilderTests var result = ItemsProxyRequestBuilder.Build( request, "/Items?Recursive=true&api_key=old", - "token", UserId, multilangEnabled: true); Assert.NotNull(result); Assert.Equal($"/Users/{UserId}/Items", result.Upstream.AbsolutePath); - Assert.Contains("api_key=token", result.Upstream.Query); Assert.DoesNotContain("api_key=old", result.Upstream.Query); + Assert.DoesNotContain("api_key", result.Upstream.Query, StringComparison.OrdinalIgnoreCase); Assert.DoesNotContain("api_key", result.NormalizedUrlForCache, StringComparison.OrdinalIgnoreCase); Assert.Contains("userId=" + UserId, result.NormalizedUrlForCache); } @@ -36,7 +35,6 @@ public sealed class ItemsProxyRequestBuilderTests var result = ItemsProxyRequestBuilder.Build( request, $"/Users/{OtherUserId}/Items?Limit=5", - "token", UserId, multilangEnabled: true); @@ -52,7 +50,6 @@ public sealed class ItemsProxyRequestBuilderTests var result = ItemsProxyRequestBuilder.Build( request, "/Items?SortBy=SortName&SortOrder=Descending&StartIndex=20&Limit=10&NameStartsWith=L", - "token", UserId, multilangEnabled: true); @@ -65,7 +62,7 @@ public sealed class ItemsProxyRequestBuilderTests Assert.Equal("fi-FI", result.Controls.ClientLocale); Assert.DoesNotContain("SortBy=", result.Upstream.Query); Assert.DoesNotContain("Limit=", result.Upstream.Query); - Assert.Contains("|SortName|Descending|L|20|10|fi-FI|", result.CacheKey); + Assert.DoesNotContain("SortBy=", result.CacheKey); } [Fact] @@ -76,20 +73,18 @@ public sealed class ItemsProxyRequestBuilderTests var first = ItemsProxyRequestBuilder.Build( request, "/Items?SortBy=SortName&StartIndex=0&Limit=100", - "token", UserId, multilangEnabled: true); var second = ItemsProxyRequestBuilder.Build( request, "/Items?SortBy=SortName&StartIndex=100&Limit=100", - "token", UserId, multilangEnabled: true); Assert.NotNull(first); Assert.NotNull(second); Assert.Equal(first.NormalizedUrlForCache, second.NormalizedUrlForCache); - Assert.NotEqual(first.CacheKey, second.CacheKey); + Assert.Equal(first.CacheKey, second.CacheKey); } [Fact] @@ -100,7 +95,6 @@ public sealed class ItemsProxyRequestBuilderTests var result = ItemsProxyRequestBuilder.Build( request, "/Items?SortBy=SortName&Limit=10", - "token", UserId, multilangEnabled: false); @@ -119,7 +113,6 @@ public sealed class ItemsProxyRequestBuilderTests var result = ItemsProxyRequestBuilder.Build( request, "http://other-host:8096/Items", - "token", UserId, multilangEnabled: true); @@ -132,6 +125,18 @@ public sealed class ItemsProxyRequestBuilderTests Assert.Equal([1, 2], ItemsProxyRequestBuilder.ParseLocalGenreIds("tmdb-1,abc,2,tmdb-2,tmdb-x,1")); } + [Fact] + public void GenreFilteringRetainsTheFieldsNeededForLocalFiltering() + { + var result = ItemsProxyRequestBuilder.Build(HttpRequest(), "/Items?GenreIds=tmdb-18&Fields=Overview", UserId, true); + Assert.NotNull(result); + var query = Microsoft.AspNetCore.WebUtilities.QueryHelpers.ParseQuery(result.Upstream.Query); + Assert.False(query.ContainsKey("GenreIds")); + Assert.Equal(1, query["Fields"].Count); + Assert.Contains("GenreItems", query["Fields"].ToString()); + Assert.Contains("Overview", query["Fields"].ToString()); + } + private static HttpRequest HttpRequest(string query = "") { var context = new DefaultHttpContext(); @@ -140,4 +145,40 @@ public sealed class ItemsProxyRequestBuilderTests context.Request.QueryString = new QueryString(query); return context.Request; } + + [Theory] + [InlineData("//other-host:8096/Items")] + [InlineData("/\\other-host/Items")] + [InlineData("https://server.local:8096/Items")] + [InlineData("/Multilang/ItemsProxy")] + [InlineData("/System/Configuration")] + [InlineData("/Items/Filters")] + [InlineData("/Items/../System/Info")] + public void RejectsUnsupportedDestinations(string url) + => Assert.Null(ItemsProxyRequestBuilder.Build(HttpRequest(), url, UserId, true)); + + [Fact] + public void EquivalentQueryOrderSharesCacheAndInFlightKeys() + { + var first = ItemsProxyRequestBuilder.Build(HttpRequest(), "/Items?Fields=Overview&Recursive=true", UserId, true)!; + var second = ItemsProxyRequestBuilder.Build(HttpRequest(), "/Items?Recursive=true&Fields=Overview", UserId, true)!; + Assert.Equal(first.CacheKey, second.CacheKey); + Assert.Equal(first.NormalizedUrlForCache, second.NormalizedUrlForCache); + } + + [Fact] + public void BasePathAndDashedUserIdsAreNormalized() + { + var request = HttpRequest(); + request.PathBase = "/jellyfin"; + var result = ItemsProxyRequestBuilder.Build(request, "/jellyfin/Users/22222222-2222-2222-2222-222222222222/Items", UserId, true)!; + Assert.Equal($"/jellyfin/Users/{UserId}/Items", result.Upstream.AbsolutePath); + Assert.Null(ItemsProxyRequestBuilder.Build(request, "/Items", UserId, true)); + } + + [Theory] + [InlineData("/Items/Resume?StartIndex=100&Limit=20")] + [InlineData("/Shows/NextUp?StartIndex=100&Limit=20")] + public void BoundedListsKeepUpstreamPaging(string url) + => Assert.False(ItemsProxyRequestBuilder.Build(HttpRequest(), url, UserId, true)!.Controls.ApplyLocalPaging); } diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxySortingTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxySortingTests.cs index b6cd88d..a3a4997 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxySortingTests.cs +++ b/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxySortingTests.cs @@ -7,6 +7,14 @@ namespace Jellyfin.Plugin.Multilang.Tests; public sealed class ItemsProxySortingTests { + [Fact] + public void TranslationOnlyDoesNotRewriteUpstreamPaging() + { + var root = JsonNode.Parse("""{"Items":[{"Name":"One"},{"Name":"Two"}],"TotalRecordCount":200,"StartIndex":100}""")!; + var original = root.ToJsonString(); + ItemsProxySorting.Apply(root, new ItemsProxyControls("", "", "", 0, 0, "", "", false), CultureInfo.InvariantCulture, []); + Assert.Equal(original, root.ToJsonString()); + } [Fact] public void GetSortArticlesUsesBuiltInLanguageFallback() { diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyTransformerResolutionTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyTransformerResolutionTests.cs index 691a6aa..4ae8417 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyTransformerResolutionTests.cs +++ b/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyTransformerResolutionTests.cs @@ -13,6 +13,22 @@ public sealed class ItemsProxyTransformerResolutionTests private const string OriginalAction = "Original"; private const string LanguagePrefix = "Language:"; + [Fact] + public void TraceUsesStoredOriginalTranslationWhenOriginalTitleIsEmpty() + { + var facts = Facts(originalTitle: "", originalLanguage: "fi"); + var translations = TextTranslations((OriginalAction, TitleField, "Alkuperainen nimi")); + var attempts = new List(); + var normal = ItemsProxyTransformer.ResolveField(TitleField, [OriginalAction], facts, translations); + var traced = ItemsProxyTransformer.ResolveField(TitleField, [OriginalAction], facts, translations, attempts.Add); + Assert.Equal(normal, traced); + Assert.Equal("fi", traced.Language); + var chosen = Assert.Single(attempts); + Assert.True(chosen.Chosen); + Assert.Equal("Original/title", chosen.LookupKey); + Assert.Equal(traced.Value, chosen.Value); + } + [Fact] public void ResolveFieldUsesFirstLanguageWithData() { diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/MaintenanceAndProviderTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/MaintenanceAndProviderTests.cs new file mode 100644 index 0000000..570473f --- /dev/null +++ b/tests/Jellyfin.Plugin.Multilang.Tests/MaintenanceAndProviderTests.cs @@ -0,0 +1,99 @@ +using System.Diagnostics; +using System.Net; +using Jellyfin.Plugin.Multilang.Data; +using Jellyfin.Plugin.Multilang.Services.Providers; + +namespace Jellyfin.Plugin.Multilang.Tests; + +public sealed class MaintenanceAndProviderTests +{ + [Fact] + public async Task ConcurrentRefreshesCanOverlapButMaintenanceWaitsForAllOfThem() + { + var root = TestPaths.CreateRoot(); + try + { + var store = new TranslationStore(new TestApplicationPaths(root)); + var first = await store.EnterRefreshAsync(CancellationToken.None); + var second = await store.EnterRefreshAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5)); + using var cancel = new CancellationTokenSource(); + var maintenance = store.EnterMaintenanceAsync(cancel.Token); + Assert.False(maintenance.IsCompleted); + first.Dispose(); + Assert.False(maintenance.IsCompleted); + cancel.Cancel(); + await Assert.ThrowsAnyAsync(() => maintenance); + second.Dispose(); + using var allSlots = await store.EnterMaintenanceAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5)); + } + finally { TestPaths.DeleteRoot(root); } + } + + [Fact] + public async Task MaintenanceWaitsForWriterAndCanceledWaiterDoesNotLeakLease() + { + var root = TestPaths.CreateRoot(); + try + { + var store = new TranslationStore(new TestApplicationPaths(root)); + var writer = await store.EnterMaintenanceAsync(CancellationToken.None); + using var canceled = new CancellationTokenSource(); + var waiting = store.EnterMaintenanceAsync(canceled.Token); + Assert.False(waiting.IsCompleted); + canceled.Cancel(); + await Assert.ThrowsAnyAsync(() => waiting); + var cleanup = store.EnterMaintenanceAsync(CancellationToken.None); + Assert.False(cleanup.IsCompleted); + writer.Dispose(); + using var lease = await cleanup.WaitAsync(TimeSpan.FromSeconds(5)); + } + finally { TestPaths.DeleteRoot(root); } + } + + [Fact] + public void ResetKeepsExistingConnectionsOnTheSameDatabase() + { + var root = TestPaths.CreateRoot(); + try + { + var store = new TranslationStore(new TestApplicationPaths(root)); + store.SaveUserRules("user", new UserRulesDocument()); + using var connection = store.Open(); + connection.Open(); + using var query = connection.CreateCommand(); + query.CommandText = "SELECT COUNT(*) FROM user_rules;"; + Assert.Equal(1L, query.ExecuteScalar()); + store.ResetAll(); + Assert.Equal(0L, query.ExecuteScalar()); + } + finally { TestPaths.DeleteRoot(root); } + } + + [Fact] + public async Task ProviderDoesNotRetryProgrammingErrors() + { + var attempts = 0; + using var http = new HttpClient(new TestHttpMessageHandler(_ => + { + attempts++; + throw new InvalidOperationException("Programming error"); + })); + using var request = new HttpRequestMessage(HttpMethod.Get, "https://example.test/"); + await Assert.ThrowsAsync(() => ProviderHttp.SendWithRetryAsync(http, request, CancellationToken.None)); + Assert.Equal(1, attempts); + } + + [Fact] + public async Task ConcurrentProviderAdmissionsKeepTheirSpacing() + { + var limiter = new FetchRateLimiter(TimeSpan.FromMilliseconds(50)); + var starts = await Task.WhenAll(Enumerable.Range(0, 4).Select(async _ => + { + await limiter.WaitAsync(CancellationToken.None); + return Stopwatch.GetTimestamp(); + })); + Array.Sort(starts); + Assert.All(starts.Zip(starts.Skip(1)), pair => + Assert.True(Stopwatch.GetElapsedTime(pair.First, pair.Second) >= TimeSpan.FromMilliseconds(40))); + } +} diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/RefreshCoordinatorTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/RefreshCoordinatorTests.cs index 4b7fb8c..6bfcbf7 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/RefreshCoordinatorTests.cs +++ b/tests/Jellyfin.Plugin.Multilang.Tests/RefreshCoordinatorTests.cs @@ -5,10 +5,76 @@ namespace Jellyfin.Plugin.Multilang.Tests; public sealed class RefreshCoordinatorTests { + [Fact] + public async Task IndependentItemsOverlapWithinWorkerLimitAndDuplicateWorkIsShared() + { + using var coordinator = new RefreshCoordinator(concurrency: 2); + var twoStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var count = 0; + Task Queue(string id) => coordinator.EnqueueAsync(id, RefreshSourceTier.Manual, RefreshWorkClass.Movies, + RefreshJobType.Full, async (_, _) => + { + if (Interlocked.Increment(ref count) == 2) twoStarted.SetResult(); + await release.Task; + return true; + }, CancellationToken.None); + var first = Queue("first"); + var second = Queue("second"); + await twoStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Same(first, Queue("first")); + var third = Queue("third"); + Assert.Equal(2, count); + Assert.Equal(2, coordinator.GetDiagnostics().ActiveItems.Length); + release.SetResult(); + await Task.WhenAll(first, second, third).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(3, count); + } + + [Fact] + public async Task ActiveMissingRefreshRunsFullUpgradeBeforeCompleting() + { + using var coordinator = new RefreshCoordinator(concurrency: 1); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var fullRan = false; + var first = coordinator.EnqueueAsync("item", RefreshSourceTier.Background, RefreshWorkClass.Movies, RefreshJobType.Missing, async (_, _) => + { + started.SetResult(); + await release.Task; + return false; + }, CancellationToken.None); + await started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var full = coordinator.EnqueueAsync("item", RefreshSourceTier.Manual, RefreshWorkClass.Movies, RefreshJobType.Full, (_, _) => + { + fullRan = true; + return Task.FromResult(true); + }, CancellationToken.None); + release.SetResult(); + Assert.True(await full.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.True(fullRan); + Assert.True(await first); + } + + [Fact] + public async Task CanceledWaiterDoesNotCancelAnotherWaiter() + { + using var coordinator = new RefreshCoordinator(concurrency: 1); + using var cancel = new CancellationTokenSource(); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var canceled = coordinator.EnqueueAsync("item", RefreshSourceTier.Background, RefreshWorkClass.Movies, RefreshJobType.Full, + async (_, _) => { await release.Task; return true; }, cancel.Token); + var survivor = coordinator.EnqueueAsync("item", RefreshSourceTier.Background, RefreshWorkClass.Movies, RefreshJobType.Full, + (_, _) => Task.FromResult(false), CancellationToken.None); + cancel.Cancel(); + await Assert.ThrowsAnyAsync(() => canceled); + release.SetResult(); + Assert.True(await survivor.WaitAsync(TimeSpan.FromSeconds(5))); + } [Fact] public async Task QueuedItemsUseSourceTierThenWorkClassPriority() { - var coordinator = new RefreshCoordinator(new ItemsProxyCache()); + using var coordinator = new RefreshCoordinator(concurrency: 1); var activeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseActive = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var order = new List(); @@ -44,11 +110,11 @@ public sealed class RefreshCoordinatorTests } [Fact] - public async Task DuplicateQueuedItemUsesTheHighestJobAndTierAndInvalidatesCache() + public async Task DuplicateQueuedItemUsesHighestJobAndTierWithoutInvalidatingUpstreamData() { var cache = new ItemsProxyCache(); - Assert.True(cache.Store("target", "user", "/Items", ["target"], "body", "application/json", 1, 1024, TimeSpan.FromMinutes(1))); - var coordinator = new RefreshCoordinator(cache); + Assert.True(cache.Store("target", "user", "/Items", 1, "body", "application/json", 1, 1024, TimeSpan.FromMinutes(1))); + using var coordinator = new RefreshCoordinator(concurrency: 1); var activeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseActive = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var runs = new List(); @@ -77,7 +143,7 @@ public sealed class RefreshCoordinatorTests await Task.WhenAll(active, full); Assert.Equal([RefreshJobType.Full], runs); - Assert.False(cache.TryGet("target", TimeSpan.FromMinutes(1), out _)); + Assert.True(cache.TryGet("target", TimeSpan.FromMinutes(1), out _)); var recent = coordinator.GetDiagnostics().Recent.Single(item => item.ItemId == "target"); Assert.Equal("Manual", recent.SourceTier); Assert.Equal("Full", recent.JobType); diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/TranslationStoreBatchTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/TranslationStoreBatchTests.cs new file mode 100644 index 0000000..594a601 --- /dev/null +++ b/tests/Jellyfin.Plugin.Multilang.Tests/TranslationStoreBatchTests.cs @@ -0,0 +1,33 @@ +using System.Diagnostics; +using Jellyfin.Plugin.Multilang.Data; +using Jellyfin.Plugin.Multilang.Services.Refresh; +using Xunit.Abstractions; + +namespace Jellyfin.Plugin.Multilang.Tests; + +public sealed class TranslationStoreBatchTests(ITestOutputHelper output) +{ + [Fact] + public void BatchFactsLookupMatchesIndividualReads() + { + var root = TestPaths.CreateRoot(); + try + { + var store = new TranslationStore(new TestApplicationPaths(root)); + var ids = Enumerable.Range(0, 100).Select(index => index.ToString()).ToArray(); + foreach (var id in ids) + store.SaveMetadata(new(id, id, "movie", RefreshWorkClass.Movies, 0, 0, null, "Movie"), "fi", + new("Title", "Overview", "Tagline", "Title", "fi", [], [], [], []), new("scope", 1, true), true); + var timer = Stopwatch.StartNew(); + var individual = ids.Select(id => store.GetFacts(id)!).ToArray(); + var separateMs = timer.Elapsed.TotalMilliseconds; + timer.Restart(); + var batch = store.GetFactsForItems(ids.Concat(ids)); + var batchMs = timer.Elapsed.TotalMilliseconds; + Assert.Equal(individual, ids.Select(id => batch[id])); + Assert.Equal(ids.Length, batch.Count); + output.WriteLine($"100 individual fact reads: {separateMs:F2} ms; one batched read: {batchMs:F2} ms. Timing is diagnostic, not a pass condition."); + } + finally { TestPaths.DeleteRoot(root); } + } +} diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/TranslationStoreCleanupTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/TranslationStoreCleanupTests.cs index b4b4c1d..d7ee5fb 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/TranslationStoreCleanupTests.cs +++ b/tests/Jellyfin.Plugin.Multilang.Tests/TranslationStoreCleanupTests.cs @@ -34,7 +34,7 @@ public sealed class TranslationStoreCleanupTests ["en"], localAssetStorage: false); - Assert.Equal(new LocalCleanupResult(TranslationsDeleted: 1, AssetsDeleted: 2, GenresDeleted: 1, AssetFilesDeleted: 2), result); + Assert.Equal(new LocalCleanupResult(TranslationsDeleted: 1, AssetsDeleted: 2, GenresDeleted: 1, AssetFilesDeleted: 4), result); Assert.NotNull(store.GetFacts(live)); Assert.Null(store.GetFacts(stale)); Assert.Equal("English", Translation(store, live, "en", "title")); diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/UserRulesNormalizerTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/UserRulesNormalizerTests.cs new file mode 100644 index 0000000..648fcd4 --- /dev/null +++ b/tests/Jellyfin.Plugin.Multilang.Tests/UserRulesNormalizerTests.cs @@ -0,0 +1,61 @@ +using System.Text.Json; +using Jellyfin.Plugin.Multilang.Data; +using Jellyfin.Plugin.Multilang.Rules; + +namespace Jellyfin.Plugin.Multilang.Tests; + +public sealed class UserRulesNormalizerTests +{ + [Fact] + public void LegacyFieldsAreNotConvertedOrWrittenBack() + { + var rules = JsonSerializer.Deserialize(""" + {"Categories":[{"CriteriaText":"original_language is fi", + "FieldActions":{"title":"Language:fi"}, + "Requirements":[{"Field":"original_language","Relation":"is","Values":["fi"]}]}]} + """)!; + UserRulesNormalizer.Normalize(rules, ["fi"]); + var category = Assert.Single(rules.Categories); + Assert.Empty(category.Requirements); + Assert.Equal(["Jellyfin"], category.FieldActionLists["title"]); + var saved = JsonSerializer.Serialize(rules); + Assert.DoesNotContain("CriteriaText", saved); + Assert.DoesNotContain("\"Field\":", saved); + Assert.DoesNotContain("\"FieldActions\":", saved); + } + + [Fact] + public void CurrentRulesPreserveBooleanModesAndNormalizeOnlyCompleteConditions() + { + var rules = new UserRulesDocument + { + Categories = [new() + { + Label = "Finnish", MatchAllConditions = false, Scopes = ["M", "C"], + Requirements = [new() { Fields = ["audio_language", "original_language", "origin_countries"], + UseFieldOr = true, Relation = "contains", UseOr = true, Values = ["fi", "sv"] }, new()], + FieldActionLists = new() { ["title"] = ["Language:FI", "Language:de", "Original"], ["overview"] = [] } + }] + }; + UserRulesNormalizer.Normalize(rules, ["fi"]); + var category = Assert.Single(rules.Categories); + Assert.False(category.MatchAllConditions); + Assert.Equal(["M", "C"], category.Scopes); + var condition = Assert.Single(category.Requirements); + Assert.Equal(["original_language", "audio_language"], condition.Fields); + Assert.True(condition.UseFieldOr); + Assert.True(condition.UseOr); + Assert.Equal(["Language:fi", "Original", "Jellyfin"], category.FieldActionLists["title"]); + Assert.Empty(category.FieldActionLists["overview"]); + Assert.Equal(JsonSerializer.Serialize(rules), JsonSerializer.Serialize(UserRulesNormalizer.Normalize(rules, ["fi"]))); + } + + [Fact] + public void NullExternalCollectionsNormalizeToDefaults() + { + var rules = JsonSerializer.Deserialize("""{"Categories":null,"FallbackFieldActions":null}""")!; + UserRulesNormalizer.Normalize(rules); + Assert.Empty(rules.Categories); + Assert.Equal(["Jellyfin"], rules.FallbackFieldActions["poster"]); + } +} diff --git a/tests/browser/multilang.spec.js b/tests/browser/multilang.spec.js index b89397f..6c0a38c 100644 --- a/tests/browser/multilang.spec.js +++ b/tests/browser/multilang.spec.js @@ -101,9 +101,15 @@ test('pre-caches the fixture movie library after user activity', async ({ reques const entries = await apiGet(request, '/Multilang/CacheEntries'); return entries.some(entry => entry.Url.includes(`ParentId=${fixture.libraryId}`)); }, { timeout: 60000 }).toBeTruthy(); + await getFixtureItems(request); + const recent = await apiGet(request, '/Multilang/ItemsProxyRequests'); + expect(recent[0].CacheHit).toBeTruthy(); }); test('shows configured translations and uses the injected ItemsProxy route', async ({ page, request }) => { + const rules = await apiGet(request, '/Multilang/UserRules/self'); + rules.FallbackFieldActions.poster = ['Language:fi', 'Jellyfin']; + await apiPost(request, '/Multilang/UserRules/self', rules); const proxiedRequests = []; page.on('request', request => { if (new URL(request.url()).pathname === '/Multilang/ItemsProxy') { @@ -112,7 +118,10 @@ test('shows configured translations and uses the injected ItemsProxy route', asy }); await signIn(page); + const artworkResponse = page.waitForResponse(response => + new URL(response.url()).pathname === `/Multilang/Assets/${fixture.orderedItemIds[0]}/fixture.png`); await openFixtureLibrary(page, fixture.translatedTitles[0]); + expect((await artworkResponse).status()).toBe(200); for (const title of fixture.translatedTitles) { await expect(page.getByText(title, { exact: true })).toBeVisible(); @@ -151,7 +160,6 @@ test('applies a structured category rule before fallback translation actions', a Id: 'browser-fixture-swedish', Label: 'Swedish originals', Requirements: [{ - Field: 'original_language', Fields: ['original_language'], Relation: 'is', Values: ['sv'], @@ -161,7 +169,6 @@ test('applies a structured category rule before fallback translation actions', a MatchAllConditions: true, Scopes: ['M'], FieldActionLists: categoryActions, - FieldActions: {} }]; await apiPost(request, '/Multilang/UserRules/self', rules); @@ -257,3 +264,81 @@ test('saves a structured classification rule through the user settings page', as Values: ['sv'] })]); }); + +test('rejects unsafe proxy targets and enforces restricted-user item access', async ({ request }) => { + for (const url of ['//outside.invalid/Items', '/System/Info', '/Multilang/ItemsProxy']) { + const response = await request.get('/Multilang/ItemsProxy', { headers: authHeaders(), params: { url } }); + expect(response.status()).toBe(400); + } + const name = 'multilang-access-test-' + Date.now(); + const user = await apiPost(request, '/Users/New', { Name: name }); + const policy = { ...user.Policy, IsAdministrator: false, EnableAllFolders: true }; + let response = await request.post(`/Users/${user.Id}/Policy`, { headers: authHeaders(), data: policy }); + expect(response.ok()).toBeTruthy(); + response = await request.post('/Users/AuthenticateByName', { + headers: { Authorization: 'MediaBrowser Client="Multilang tests", Device="test", DeviceId="multilang-access-test", Version="1"' }, + data: { Username: name, Pw: '' } + }); + expect(response.ok()).toBeTruthy(); + const login = await response.json(); + const headers = { Authorization: `MediaBrowser Token="${login.AccessToken}"` }; + const asset = `/Multilang/Assets/${fixture.orderedItemIds[0]}/fixture.png`; + expect((await request.get(asset)).status()).toBe(401); + expect((await request.get(asset, { headers })).status()).toBe(200); + response = await request.post('/Multilang/UserRules/self', { headers, data: { Enabled: true, FallbackFieldActions: jellyfinActions() } }); + expect(response.ok()).toBeTruthy(); + const params = { url: `/Items?ParentId=${fixture.libraryId}&IncludeItemTypes=Movie&Recursive=true&SortBy=SortName` }; + response = await request.get('/Multilang/ItemsProxy', { headers, params }); + expect(response.ok()).toBeTruthy(); + expect((await response.json()).Items).toHaveLength(3); + let entries = await apiGet(request, '/Multilang/CacheEntries'); + expect(entries.some(entry => entry.UserId === user.Id)).toBeTruthy(); + + policy.EnableAllFolders = false; + policy.EnabledFolders = []; + response = await request.post(`/Users/${user.Id}/Policy`, { headers: authHeaders(), data: policy }); + expect(response.ok()).toBeTruthy(); + response = await request.get('/Multilang/ItemsProxy', { headers, params }); + const native = await request.get(`/Users/${user.Id}/Items?ParentId=${fixture.libraryId}`, { headers }); + expect(native.status()).toBe(401); + expect(response.status()).toBe(native.status()); + const recent = await apiGet(request, '/Multilang/ItemsProxyRequests'); + expect(recent[0].CacheHit).toBeFalsy(); + expect((await request.get(asset, { headers })).status()).toBe(404); + entries = await apiGet(request, '/Multilang/CacheEntries'); + expect(entries.some(entry => entry.UserId === user.Id && entry.ItemCount > 0)).toBeFalsy(); + for (const path of [`Debug/${fixture.orderedItemIds[0]}`, `Debug/${fixture.orderedItemIds[0]}/self`]) { + expect((await request.get(`/Multilang/${path}`, { headers })).status()).toBe(404); + } + expect((await request.post(`/Multilang/RefreshItem/${fixture.orderedItemIds[0]}?includeChildren=true`, { headers })).status()).toBe(404); + response = await request.get('/Multilang/ItemsProxy', { headers, params }); + expect(response.status()).toBe(401); +}); + +test('reuses a library fetch across different pages and sorting controls', async ({ request }) => { + const base = `/Items?ParentId=${fixture.libraryId}&IncludeItemTypes=Movie&Recursive=true`; + let response = await request.get('/Multilang/ItemsProxy', { headers: authHeaders(), params: { url: base + '&SortBy=SortName&Limit=1', mlLocale: 'fi-FI' } }); + expect((await response.json()).Items.map(item => item.Name)).toEqual(['Ankkuri']); + response = await request.get('/Multilang/ItemsProxy', { headers: authHeaders(), params: { url: base + '&SortBy=SortName&StartIndex=1&Limit=1', mlLocale: 'fi-FI' } }); + expect((await response.json()).Items.map(item => item.Name)).toEqual(['The Clock']); + const recent = await apiGet(request, '/Multilang/ItemsProxyRequests'); + expect(recent[0].CacheHit).toBeTruthy(); +}); + +test('reopens user settings without duplicate loads or a native replaceChildren dependency', async ({ page }) => { + await page.addInitScript(() => { + Object.defineProperty(Element.prototype, 'replaceChildren', { configurable: true, value: undefined }); + }); + await signIn(page); + await page.goto(`/web/#/mypreferencesmenu?userId=${fixture.userId}`); + let loads = 0; + page.on('request', request => { if (new URL(request.url()).pathname === '/Multilang/UserRules/self') loads++; }); + for (let i = 0; i < 3; i++) { + await expect(page.locator('#mlUserRulesMenuItem')).toBeVisible(); + await page.locator('#mlUserRulesMenuItem').click(); + await expect(page.locator('#ml-sort-locale option[value="en"]')).toHaveCount(1); + expect(loads).toBe(i + 1); + await page.evaluate(userId => { location.hash = '#/mypreferencesmenu?userId=' + userId; }, fixture.userId); + await expect(page.locator('#ml-user-shell')).toHaveCount(0); + } +}); diff --git a/tools/run_jellyfin12_browser_tests.py b/tools/run_jellyfin12_browser_tests.py index e97e0af..d9ab845 100644 --- a/tools/run_jellyfin12_browser_tests.py +++ b/tools/run_jellyfin12_browser_tests.py @@ -144,6 +144,14 @@ for row in rows: INSERT INTO genres(tmdb_id, media, lang, name, name_norm) VALUES (?, 'movie', 'fi', ?, ?) ON CONFLICT(tmdb_id, media, lang) DO UPDATE SET name=excluded.name, name_norm=excluded.name_norm """, (row["genre_id"], row["genre"], row["genre"].lower())) + relative = row["item_id"] + "/fixture.png" + destination = os.path.join(os.path.dirname(database), "assets", relative) + os.makedirs(os.path.dirname(destination), exist_ok=True) + with open(destination, "wb") as image: + image.write(base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jZ1kAAAAASUVORK5CYII=")) + path = "/Multilang/Assets/" + relative + connection.execute("INSERT OR REPLACE INTO assets(item_id,lang,kind,path,path_low,updated_at) VALUES (?, 'fi', 'poster', ?, ?, ?)", + (row["item_id"], path, path.lower(), now)) connection.commit() ''' remote.stop()