From 096c6be8c793504595f8accf80ee8506eaa22c1f Mon Sep 17 00:00:00 2001 From: ajp_anton Date: Wed, 9 Sep 2026 01:18:43 +0000 Subject: [PATCH] Release 0.3.4: Jellyfin 12 stable support and audit fixes Harden proxy permissions and cache behavior, streamline metadata fetching, remove obsolete rules, and validate the stable SDK with expanded regression coverage. Document backup format changes. --- README.md | 23 +- .../Api/MultilangController.cs | 512 +++--------------- .../Configuration/configPage.html | 24 +- .../Configuration/debugPage.html | 4 +- .../Configuration/shared.js | 64 +-- .../Configuration/userRulesPage.html | 152 ++---- .../Data/TranslationStore.Cleanup.cs | 54 +- .../Data/TranslationStore.ImportExport.cs | 206 ++----- .../Data/TranslationStore.Models.cs | 8 - .../Data/TranslationStore.Refresh.cs | 125 +++++ .../Data/TranslationStore.Schema.cs | 11 + .../Data/TranslationStore.cs | 246 +++++---- .../Jellyfin.Plugin.Multilang.csproj | 10 +- .../MultilangConstants.cs | 7 +- src/Jellyfin.Plugin.Multilang/Plugin.cs | 4 + .../PluginServiceRegistrator.cs | 4 + .../Rules/CategoryRuleEvaluator.cs | 192 ++----- .../Rules/UserRulesNormalizer.cs | 166 ++++++ .../Services/Assets/AssetStorageService.cs | 24 +- .../Services/Backup/MultilangBackupService.cs | 25 +- .../Services/ItemsProxyCache.cs | 133 ++--- .../Services/ItemsProxyInvalidationService.cs | 35 ++ .../Services/ItemsProxyPrecacheService.cs | 10 +- .../Services/ItemsProxyRequestBuilder.cs | 84 ++- .../Services/ItemsProxySorting.cs | 5 +- .../Services/ItemsProxyTransformer.cs | 131 ++--- .../Services/Providers/FanartClient.cs | 3 +- .../Services/Providers/FetchRateLimiter.cs | 20 +- .../Services/Providers/ProviderHttp.cs | 16 +- .../Services/Providers/TmdbClient.cs | 6 +- .../Services/Refresh/ItemFetcher.cs | 220 ++++++++ .../Services/Refresh/RefreshCoordinator.cs | 72 ++- .../Services/Refresh/RefreshModels.cs | 2 +- .../Services/Refresh/RefreshService.cs | 470 +++++----------- .../wwwroot/inject.js | 48 +- .../AssetStorageServiceTests.cs | 19 +- .../CategoryRuleEvaluatorTests.cs | 20 +- .../ItemFetcherTests.cs | 232 ++++++++ .../ItemsProxyCacheTests.cs | 68 ++- .../ItemsProxyRequestBuilderTests.cs | 59 +- .../ItemsProxySortingTests.cs | 8 + .../ItemsProxyTransformerResolutionTests.cs | 16 + .../Jellyfin.Plugin.Multilang.Tests.csproj | 4 +- .../MaintenanceAndProviderTests.cs | 99 ++++ .../RefreshCoordinatorTests.cs | 76 ++- .../TranslationStoreBatchTests.cs | 33 ++ .../TranslationStoreCleanupTests.cs | 2 +- .../UserRulesNormalizerTests.cs | 61 +++ tests/browser/multilang.spec.js | 89 ++- tools/run_jellyfin12_browser_tests.py | 8 + 50 files changed, 2126 insertions(+), 1784 deletions(-) create mode 100644 src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Refresh.cs create mode 100644 src/Jellyfin.Plugin.Multilang/Rules/UserRulesNormalizer.cs create mode 100644 src/Jellyfin.Plugin.Multilang/Services/ItemsProxyInvalidationService.cs create mode 100644 src/Jellyfin.Plugin.Multilang/Services/Refresh/ItemFetcher.cs create mode 100644 tests/Jellyfin.Plugin.Multilang.Tests/ItemFetcherTests.cs create mode 100644 tests/Jellyfin.Plugin.Multilang.Tests/MaintenanceAndProviderTests.cs create mode 100644 tests/Jellyfin.Plugin.Multilang.Tests/TranslationStoreBatchTests.cs create mode 100644 tests/Jellyfin.Plugin.Multilang.Tests/UserRulesNormalizerTests.cs diff --git a/README.md b/README.md index e6302cd..ce53f9e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,19 @@ Multilang is a Jellyfin plugin that lets different users see different metadata languages without changing Jellyfin's stored metadata. The `main` branch is built for the Jellyfin 10.11.x stable series. The -`jellyfin12` branch targets Jellyfin 12 and is currently early beta software. +`jellyfin12` branch targets Jellyfin 12.0 stable on Linux x64. The plugin remains +beta software. + +## Upgrading to 0.2.5 or 0.3.4 + +Existing plugin data is upgraded automatically. Make fresh database exports after +upgrading: older translation-database exports cannot be imported by these versions. +User-settings-only exports keep their existing format. Obsolete text-based rules +are no longer supported; current classification rules are unaffected. + +The first missing-data scan records the new per-language and per-provider fetch +state, so it may fetch more than subsequent scans. Later scans retry only missing +data when due and repair downloaded artwork without refetching its metadata. ## What It Does @@ -87,9 +99,12 @@ the UI. Some requests can be slow because Multilang may need to ask Jellyfin for a larger list than the UI will finally display. This is needed so the plugin can translate, filter, sort, and slice the list using the user's own rules. -Multilang caches slow transformed responses for a configurable time and also -uses a very short burst cache for repeated requests during navigation. Cache -entries are invalidated when affected items are refreshed. +Multilang caches Jellyfin's responses for a configurable time, sharing the fetched +list across pages and sort orders. Current translations and user rules are applied +each time that list is used. Simultaneous requests share an in-progress fetch, and +a short burst cache also handles fast repeated requests. Library, permission and +watched-state changes invalidate affected caches. Movie and show libraries can +optionally be fetched in advance when a user becomes active. ## Original-Language Support diff --git a/src/Jellyfin.Plugin.Multilang/Api/MultilangController.cs b/src/Jellyfin.Plugin.Multilang/Api/MultilangController.cs index 9572224..35f02fa 100644 --- a/src/Jellyfin.Plugin.Multilang/Api/MultilangController.cs +++ b/src/Jellyfin.Plugin.Multilang/Api/MultilangController.cs @@ -1,4 +1,6 @@ using System.Reflection; +using Jellyfin.Data; +using Jellyfin.Database.Implementations.Enums; using System.Diagnostics; using System.Globalization; using System.Text.Json; @@ -27,15 +29,6 @@ namespace Jellyfin.Plugin.Multilang.Api; [Route("Multilang")] public sealed class MultilangController : ControllerBase { - private static readonly string[] RequirementFieldOrder = - [ - "original_language", - "spoken_languages", - "audio_language", - "origin_countries", - "production_countries" - ]; - private readonly TranslationStore _store; private readonly ISessionManager _sessionManager; private readonly IAuthorizationContext _authorizationContext; @@ -108,10 +101,18 @@ public sealed class MultilangController : ControllerBase => ServeEmbedded("Jellyfin.Plugin.Multilang.Configuration.shared.css", "text/css"); [HttpGet("Assets/{**path}")] - public IActionResult Asset(string path) - => _assetStorage.TryResolveLocalAsset(path, out var fullPath, out var contentType) - ? PhysicalFile(fullPath, contentType) - : NotFound(); + public async Task Asset(string path) + { + var userId = await GetUserIdAsync().ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(userId)) + return Unauthorized(new { Error = "NotAuthenticated" }); + if (!Guid.TryParse(path.Split('/')[0], out var itemId) || !CanAccessItem(itemId, userId) || + !_store.IsAssetReferenced(itemId.ToString("N"), TranslationStore.LocalAssetUrlPrefix + path) || + !_assetStorage.TryResolveLocalAsset(path, out var fullPath, out var contentType)) + return NotFound(); + Response.Headers.CacheControl = "private, no-cache"; + return PhysicalFile(fullPath, contentType); + } [HttpGet("Providers")] public IActionResult Providers() @@ -214,10 +215,9 @@ public sealed class MultilangController : ControllerBase return Unauthorized(new { Error = "NotAuthenticated" }); var allowed = (Plugin.Instance?.Configuration?.Languages ?? []).ToHashSet(StringComparer.OrdinalIgnoreCase); - request.Categories = NormalizeCategories(request.Categories, allowed); - request.FallbackFieldActions = NormalizeActionLists(request.FallbackFieldActions, allowed); + UserRulesNormalizer.Normalize(request, allowed); _store.SaveUserRules(userId, request); - _itemsProxyCache.ClearAll(); + _itemsProxyCache.ClearUser(userId); return Ok(request); } @@ -265,7 +265,11 @@ public sealed class MultilangController : ControllerBase if (!TryParseItemId(itemId, out _, out var itemGuid)) return BadRequest(new { Error = "InvalidItemId" }); - await _refreshService.RefreshItemAsync(itemGuid, includeChildren, cancellationToken).ConfigureAwait(false); + var user = _userManager.GetUserById(Guid.Parse(userId)); + if (user is null || !CanAccessItem(itemGuid, userId)) + return NotFound(); + + await _refreshService.RefreshItemAsync(itemGuid, includeChildren, cancellationToken, user).ConfigureAwait(false); return Ok(new { ItemId = itemGuid.ToString("N"), IncludeChildren = includeChildren }); } @@ -283,8 +287,13 @@ public sealed class MultilangController : ControllerBase if (string.IsNullOrWhiteSpace(userId)) return Unauthorized(new { Error = "NotAuthenticated" }); + var user = _userManager.GetUserById(Guid.Parse(userId)); + if (user is null || user.HasPermission(PermissionKind.IsDisabled)) + return Unauthorized(new { Error = "NotAuthenticated" }); + // Jellyfin's UpdatePolicyAsync does not publish a user-update event. + var generation = _itemsProxyCache.ObserveUserPolicy(userId, JsonSerializer.Serialize(_userManager.GetUserDto(user).Policy)); var rules = GetRulesOrDefault(userId); - var proxyRequest = ItemsProxyRequestBuilder.Build(Request, url, token, userId, rules.Enabled); + var proxyRequest = ItemsProxyRequestBuilder.Build(Request, url, userId, rules.Enabled); if (proxyRequest is null) return BadRequest(new { Error = "InvalidUrl" }); @@ -304,6 +313,7 @@ public sealed class MultilangController : ControllerBase if (cacheAllowed && _itemsProxyCache.TryGet(proxyRequest.CacheKey, cacheTtl, out var cached)) { + var transformed = _itemsProxyTransformer.TransformItemsResponse(cached.Body, rules, proxyRequest.Controls); totalSw.Stop(); _itemsProxyCache.RecordRequest( proxyRequest.NormalizedUrlForCache, @@ -313,9 +323,9 @@ public sealed class MultilangController : ControllerBase itemCount: cached.ItemCount, totalMs: totalSw.ElapsedMilliseconds, upstreamMs: 0, - transformMs: 0, + transformMs: totalSw.ElapsedMilliseconds, sizeBytes: cached.SizeBytes); - return Content(cached.Body, cached.ContentType); + return Content(transformed.Body, cached.ContentType); } if (rules.Enabled && proxyRequest.IsGenresRequest && _itemsProxyTransformer.TryBuildGenresResponse(proxyRequest.GenreMedia, proxyRequest.Controls.ClientLocale, out var genresBody)) @@ -337,11 +347,12 @@ public sealed class MultilangController : ControllerBase var upstreamSw = Stopwatch.StartNew(); var upstream = await _itemsProxyRequestCoalescer.GetOrFetchAsync( - proxyRequest.NormalizedUrlForCache, + generation + "|" + proxyRequest.NormalizedUrlForCache, async sharedCancellationToken => { - var http = _httpClientFactory.CreateClient(); - using var request = new HttpRequestMessage(HttpMethod.Get, proxyRequest.Upstream); + var http = _httpClientFactory.CreateClient("Multilang.Jellyfin"); + var localUri = new Uri(_itemsProxyPrecacheService.LocalApiUri, proxyRequest.Upstream.PathAndQuery); + using var request = new HttpRequestMessage(HttpMethod.Get, localUri); request.Headers.TryAddWithoutValidation("Authorization", $"MediaBrowser Token=\"{token}\""); request.Headers.TryAddWithoutValidation("X-Emby-Token", token); request.Headers.TryAddWithoutValidation("X-MediaBrowser-Token", token); @@ -380,12 +391,13 @@ public sealed class MultilangController : ControllerBase proxyRequest.CacheKey, userId, proxyRequest.NormalizedUrlForCache, - responseItemIds, - body, + responseItemIds.Length, + upstream.Response.Body, contentType, totalSw.ElapsedMilliseconds, cfg.ItemsProxyCacheMaxMiB * 1024L * 1024L, - cacheTtl); + cacheTtl, + generation); } if (cacheAllowed && @@ -397,11 +409,13 @@ public sealed class MultilangController : ControllerBase proxyRequest.CacheKey, userId, proxyRequest.NormalizedUrlForCache, - responseItemIds, - body, + responseItemIds.Length, + upstream.Response.Body, contentType, totalSw.ElapsedMilliseconds, - cfg.ItemsProxyCacheMaxMiB * 1024L * 1024L); + cfg.ItemsProxyCacheMaxMiB * 1024L * 1024L, + generation, + cacheTtl); } _itemsProxyCache.RecordRequest( @@ -424,19 +438,10 @@ public sealed class MultilangController : ControllerBase }; } - private sealed record DebugResolutionAttempt( - string Action, - string Source, - string LookupKey, - bool HasValue, - bool Chosen, - string? Value, - string Reason); - private sealed record DebugTextResolution( string Field, string[] Actions, - DebugResolutionAttempt[] Attempts, + ItemsProxyResolutionAttempt[] Attempts, ItemsProxyResolvedText Result, string SelectedAction, string Reason); @@ -444,7 +449,7 @@ public sealed class MultilangController : ControllerBase private sealed record DebugAssetResolution( string Kind, string[] Actions, - DebugResolutionAttempt[] Attempts, + ItemsProxyResolutionAttempt[] Attempts, ItemsProxyResolvedAsset Result, string SelectedAction, string Reason); @@ -515,6 +520,7 @@ public sealed class MultilangController : ControllerBase if (downloadedAssets && !translationsDatabase) return BadRequest(new { Error = "AssetsRequireDatabase" }); + using var maintenance = await _store.EnterMaintenanceAsync(HttpContext.RequestAborted).ConfigureAwait(false); var bytes = _backupService.Export( new BackupExportOptions(pluginSettings, userSettings, translationsDatabase, downloadedAssets), Plugin.Instance?.Configuration ?? new PluginConfiguration()); @@ -544,6 +550,7 @@ public sealed class MultilangController : ControllerBase try { await using var stream = file.OpenReadStream(); + using var maintenance = await _store.EnterMaintenanceAsync(HttpContext.RequestAborted).ConfigureAwait(false); var result = _backupService.ImportAdmin( stream, options, @@ -567,6 +574,7 @@ public sealed class MultilangController : ControllerBase if (request.Confirm != true) return BadRequest(new { Error = "ConfirmationRequired" }); + using var maintenance = await _store.EnterMaintenanceAsync(HttpContext.RequestAborted).ConfigureAwait(false); Plugin.Instance?.UpdateConfiguration(new PluginConfiguration()); _store.ResetAll(); _itemsProxyCache.ClearAll(); @@ -625,7 +633,7 @@ public sealed class MultilangController : ControllerBase await using var stream = file.OpenReadStream(); var result = _backupService.ImportUser(stream, userId, form.TryGetValue("sourceUserId", out var sourceUserId) ? sourceUserId.ToString() : null); if (result.UserSettingsImported > 0) - _itemsProxyCache.ClearAll(); + _itemsProxyCache.ClearUser(userId); return Ok(result); } catch (InvalidOperationException ex) @@ -641,9 +649,12 @@ public sealed class MultilangController : ControllerBase if (string.IsNullOrWhiteSpace(userId)) return Unauthorized(new { Error = "NotAuthenticated" }); - if (!TryParseItemId(itemId, out var itemId32, out _)) + if (!TryParseItemId(itemId, out var itemId32, out var itemGuid)) return BadRequest(new { Error = "InvalidItemId" }); + if (!CanAccessItem(itemGuid, userId)) + return NotFound(); + return Ok(new { ItemId = itemId32, @@ -673,7 +684,7 @@ public sealed class MultilangController : ControllerBase if (normalizedUserId is null) return BadRequest(new { Error = "InvalidUserId" }); - return DebugResolveForUser(itemId, normalizedUserId); + return DebugResolveForUser(itemId, normalizedUserId, administrator: true); } [HttpGet("DebugResolve/{itemId}")] @@ -686,11 +697,14 @@ public sealed class MultilangController : ControllerBase return DebugResolveForUser(itemId, userId); } - private IActionResult DebugResolveForUser(string itemId, string userId) + private IActionResult DebugResolveForUser(string itemId, string userId, bool administrator = false) { - if (!TryParseItemId(itemId, out var itemId32, out _)) + if (!TryParseItemId(itemId, out var itemId32, out var itemGuid)) return BadRequest(new { Error = "InvalidItemId" }); + if (!administrator && !CanAccessItem(itemGuid, userId)) + return NotFound(); + var facts = _store.GetFacts(itemId32); if (facts is null) return NotFound(new { Error = "FactsMissing" }); @@ -809,128 +823,27 @@ public sealed class MultilangController : ControllerBase } private static DebugTextResolution TraceResolveField( - string field, - IReadOnlyList actions, - FactsData facts, + string field, IReadOnlyList actions, FactsData facts, Dictionary>? byLang) { - var attempts = new List(); - foreach (var action in actions) - { - if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)) - { - var result = new ItemsProxyResolvedText(false, null); - attempts.Add(new DebugResolutionAttempt(action, JellyfinAction, string.Empty, true, true, null, "Use Jellyfin value")); - return new DebugTextResolution(field, actions.ToArray(), attempts.ToArray(), result, JellyfinAction, "Jellyfin action reached"); - } - - if (field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) && - action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase)) - { - var hasOriginalTitle = !string.IsNullOrWhiteSpace(facts.OriginalTitle); - attempts.Add(new DebugResolutionAttempt(action, "OriginalTitle", "facts.original_title", hasOriginalTitle, hasOriginalTitle, facts.OriginalTitle, hasOriginalTitle ? "Original title found" : "Original title missing")); - if (hasOriginalTitle) - { - var result = new ItemsProxyResolvedText(true, facts.OriginalTitle); - return new DebugTextResolution(field, actions.ToArray(), attempts.ToArray(), result, action, "Original title found"); - } - - continue; - } - - if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase)) - { - var originalValue = ItemsProxyTransformer.GetTranslatedField(byLang, OriginalAction, field); - var hasOriginalValue = !string.IsNullOrWhiteSpace(originalValue); - attempts.Add(new DebugResolutionAttempt(action, "Translation", "Original/" + field, hasOriginalValue, hasOriginalValue, originalValue, hasOriginalValue ? "Original-language translation found" : "Original-language translation missing")); - if (hasOriginalValue) - { - var result = new ItemsProxyResolvedText(true, originalValue); - return new DebugTextResolution(field, actions.ToArray(), attempts.ToArray(), result, action, "Original-language translation found"); - } - - continue; - } - - if (!action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)) - { - attempts.Add(new DebugResolutionAttempt(action, "Unknown", string.Empty, false, false, null, "Unsupported action token")); - continue; - } - - var lang = action[LanguagePrefix.Length..]; - var value = ItemsProxyTransformer.GetTranslatedField(byLang, lang, field); - var hasValue = !string.IsNullOrWhiteSpace(value); - attempts.Add(new DebugResolutionAttempt(action, "Translation", lang + "/" + field, hasValue, hasValue, value, hasValue ? "Translation found" : "Translation missing or empty")); - if (hasValue) - { - var result = new ItemsProxyResolvedText(true, value); - return new DebugTextResolution(field, actions.ToArray(), attempts.ToArray(), result, action, "Translation found"); - } - } - - var fallback = field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) - ? new ItemsProxyResolvedText(false, null) - : new ItemsProxyResolvedText(true, string.Empty); - var reason = field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) - ? "No action had data; title falls back to Jellyfin" - : "No action had data; field is cleared"; - return new DebugTextResolution(field, actions.ToArray(), attempts.ToArray(), fallback, field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) ? JellyfinAction : ClearAction, reason); + var attempts = new List(); + var result = ItemsProxyTransformer.ResolveField(field, actions, facts, byLang, attempts.Add); + var chosen = attempts.FirstOrDefault(attempt => attempt.Chosen); + return new(field, actions.ToArray(), attempts.ToArray(), result, + chosen?.Action ?? (result.Change ? ClearAction : JellyfinAction), + chosen?.Reason ?? (result.Change ? "No action had data; field is cleared" : "No action had data; use Jellyfin")); } private static DebugAssetResolution TraceResolveAsset( - string kind, - IReadOnlyList actions, + string kind, IReadOnlyList actions, Dictionary>? byKind) { - var attempts = new List(); - foreach (var action in actions) - { - if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)) - { - var result = new ItemsProxyResolvedAsset(false, null); - attempts.Add(new DebugResolutionAttempt(action, JellyfinAction, string.Empty, true, true, null, "Use Jellyfin image")); - return new DebugAssetResolution(kind, actions.ToArray(), attempts.ToArray(), result, JellyfinAction, "Jellyfin action reached"); - } - - if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase)) - { - var originalValue = ItemsProxyTransformer.GetAsset(byKind, kind, OriginalAction); - var hasOriginalValue = !string.IsNullOrWhiteSpace(originalValue); - attempts.Add(new DebugResolutionAttempt(action, "Asset", kind + "/Original", hasOriginalValue, hasOriginalValue, originalValue, hasOriginalValue ? "Original-language asset found" : "Original-language asset missing")); - if (hasOriginalValue) - { - var result = new ItemsProxyResolvedAsset(true, originalValue); - return new DebugAssetResolution(kind, actions.ToArray(), attempts.ToArray(), result, action, "Original-language asset found"); - } - - continue; - } - - if (!action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)) - { - attempts.Add(new DebugResolutionAttempt(action, "Unknown", string.Empty, false, false, null, "Unsupported action token")); - continue; - } - - var lang = action[LanguagePrefix.Length..]; - var value = ItemsProxyTransformer.GetAsset(byKind, kind, lang); - var hasValue = !string.IsNullOrWhiteSpace(value); - attempts.Add(new DebugResolutionAttempt(action, "Asset", kind + "/" + lang, hasValue, hasValue, value, hasValue ? "Asset found" : "Asset missing")); - if (hasValue) - { - var result = new ItemsProxyResolvedAsset(true, value); - return new DebugAssetResolution(kind, actions.ToArray(), attempts.ToArray(), result, action, "Asset found"); - } - } - - var fallback = kind.Equals(PosterKind, StringComparison.OrdinalIgnoreCase) - ? new ItemsProxyResolvedAsset(false, null) - : new ItemsProxyResolvedAsset(true, string.Empty); - var reason = kind.Equals(PosterKind, StringComparison.OrdinalIgnoreCase) - ? "No action had data; poster falls back to Jellyfin" - : "No action had data; image is cleared"; - return new DebugAssetResolution(kind, actions.ToArray(), attempts.ToArray(), fallback, kind.Equals(PosterKind, StringComparison.OrdinalIgnoreCase) ? JellyfinAction : ClearAction, reason); + var attempts = new List(); + var result = ItemsProxyTransformer.ResolveAsset(kind, actions, byKind, attempts.Add); + var chosen = attempts.FirstOrDefault(attempt => attempt.Chosen); + return new(kind, actions.ToArray(), attempts.ToArray(), result, + chosen?.Action ?? (result.Change ? ClearAction : JellyfinAction), + chosen?.Reason ?? (result.Change ? "No action had data; image is cleared" : "No action had data; use Jellyfin")); } private static object[] BuildMissingDataReport( @@ -1025,238 +938,7 @@ public sealed class MultilangController : ControllerBase } private UserRulesDocument GetRulesOrDefault(string userId) - { - var existing = _store.GetUserRules(userId); - if (existing is not null) - { - existing.Categories ??= []; - existing.FallbackFieldActions = NormalizeActionLists(existing.FallbackFieldActions, (Plugin.Instance?.Configuration?.Languages ?? []).ToHashSet(StringComparer.OrdinalIgnoreCase)); - existing.Categories = NormalizeCategories(existing.Categories, (Plugin.Instance?.Configuration?.Languages ?? []).ToHashSet(StringComparer.OrdinalIgnoreCase)); - return existing; - } - - return new UserRulesDocument - { - Enabled = false, - SortLocale = "Auto", - TrustTmdbCollections = true, - FallbackFieldActions = DefaultActionLists(), - Categories = [] - }; - } - - private static UserCategoryRule[] NormalizeCategories(IEnumerable? categories, IReadOnlySet allowedLanguages) - => (categories ?? []) - .Select(category => new UserCategoryRule - { - Id = string.IsNullOrWhiteSpace(category.Id) ? Guid.NewGuid().ToString("N") : category.Id.Trim(), - Label = string.IsNullOrWhiteSpace(category.Label) ? "Category" : category.Label.Trim(), - CriteriaText = category.CriteriaText?.Trim() ?? string.Empty, - Requirements = NormalizeRequirements(category.Requirements), - MatchAllConditions = category.MatchAllConditions, - Scopes = (category.Scopes ?? []) - .Where(scope => scope is "M" or "S" or "C") - .Distinct(StringComparer.OrdinalIgnoreCase) - .DefaultIfEmpty("M") - .ToArray(), - FieldActions = NormalizeFieldActions(category.FieldActions, allowedLanguages), - FieldActionLists = NormalizeActionLists(category.FieldActionLists, allowedLanguages, category.FieldActions) - }) - .ToArray(); - - private static UserCategoryRequirement[] NormalizeRequirements(IEnumerable? requirements) - => (requirements ?? []) - .Select(requirement => - { - var fields = NormalizeRequirementFields(requirement.Fields, requirement.Field); - return new UserCategoryRequirement - { - Fields = fields, - Field = fields.FirstOrDefault() ?? string.Empty, - UseFieldOr = requirement.UseFieldOr, - Relation = NormalizeRequirementRelation(requirement.Relation), - UseOr = requirement.UseOr, - Values = (requirement.Values ?? []) - .Where(v => !string.IsNullOrWhiteSpace(v)) - .Select(v => v.Trim()) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray() - }; - }) - .Where(requirement => - requirement.Fields.Length > 0 && - requirement.Relation.Length > 0 && - requirement.Values.Length > 0) - .ToArray(); - - private static string[] NormalizeRequirementFields(IEnumerable? fields, string? legacyField) - { - var requested = (fields ?? []) - .Append(legacyField ?? string.Empty) - .Select(NormalizeRequirementField) - .Where(field => field.Length > 0) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToHashSet(StringComparer.OrdinalIgnoreCase); - - var result = new List(); - string? kind = null; - foreach (var field in RequirementFieldOrder) - { - if (!requested.Contains(field)) - continue; - - var currentKind = IsCountryRequirementField(field) ? "country" : "language"; - kind ??= currentKind; - if (currentKind == kind) - result.Add(field); - } - - return result.ToArray(); - } - - private static string NormalizeRequirementField(string? field) - { - var normalized = (field ?? string.Empty).Trim().ToLowerInvariant(); - return normalized is "original_language" or "spoken_languages" or "audio_language" or "origin_countries" or "production_countries" - ? normalized - : string.Empty; - } - - private static bool IsCountryRequirementField(string field) - => field.Equals("origin_countries", StringComparison.OrdinalIgnoreCase) || - field.Equals("production_countries", StringComparison.OrdinalIgnoreCase); - - private static string NormalizeRequirementRelation(string? relation) - { - var normalized = (relation ?? string.Empty).Trim().ToLowerInvariant(); - return normalized is "is" or "is_not" or "contains" or "not_contains" - ? normalized - : string.Empty; - } - - private static Dictionary NormalizeActionLists( - IReadOnlyDictionary? actions, - IReadOnlySet allowedLanguages, - IReadOnlyDictionary? legacyActions = null) - { - if (legacyActions is not null && LooksLikeDefaultActionLists(actions) && HasMeaningfulLegacyActions(legacyActions)) - actions = null; - - var result = DefaultActionLists(); - foreach (var field in result.Keys.ToArray()) - { - var raw = actions is not null && actions.TryGetValue(field, out var configured) - ? configured - : LegacyActionToList(field, legacyActions); - result[field] = NormalizeActionList(field, raw, allowedLanguages); - } - - return result; - } - - private static bool LooksLikeDefaultActionLists(IReadOnlyDictionary? actions) - => actions is null || - DefaultActionLists().Keys.All(field => - actions.TryGetValue(field, out var list) && - list.Length == 1 && - list[0].Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)); - - private static bool HasMeaningfulLegacyActions(IReadOnlyDictionary legacyActions) - => legacyActions.Values.Any(action => - !string.IsNullOrWhiteSpace(action) && - !action.Equals(FallbackAction, StringComparison.OrdinalIgnoreCase) && - !action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)); - - private static string[] LegacyActionToList(string field, IReadOnlyDictionary? legacyActions) - { - if (legacyActions is null || !legacyActions.TryGetValue(field, out var action)) - return [JellyfinAction]; - if (string.IsNullOrWhiteSpace(action) || action.Equals(FallbackAction, StringComparison.OrdinalIgnoreCase)) - return [JellyfinAction]; - return [action]; - } - - private static Dictionary NormalizeFieldActions( - IReadOnlyDictionary? actions, - IReadOnlySet allowedLanguages) - { - var result = DefaultFieldActions(); - if (actions is null) - return result; - - foreach (var field in result.Keys.ToArray()) - { - if (!actions.TryGetValue(field, out var raw)) - continue; - - result[field] = NormalizeAction(field, raw, allowedLanguages); - } - - return result; - } - - private static string NormalizeAction(string field, string? raw, IReadOnlySet allowedLanguages) - { - var action = (raw ?? string.Empty).Trim(); - if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase) || - action.Equals(FallbackAction, StringComparison.OrdinalIgnoreCase)) - return action; - if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase)) - return OriginalAction; - if ((field is OverviewField or TaglineField or LogoKind) && - action.Equals(ClearAction, StringComparison.OrdinalIgnoreCase)) - return ClearAction; - if (action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)) - { - var lang = action[LanguagePrefix.Length..].Trim(); - var canonical = allowedLanguages.FirstOrDefault(l => l.Equals(lang, StringComparison.OrdinalIgnoreCase)); - if (!string.IsNullOrWhiteSpace(canonical)) - return LanguagePrefix + canonical; - } - - return FallbackAction; - } - - private static string[] NormalizeActionList(string field, IEnumerable? raw, IReadOnlySet allowedLanguages) - { - var result = new List(); - foreach (var action in raw ?? []) - { - var normalized = NormalizeActionToken(field, action, allowedLanguages); - if (normalized.Length == 0) - continue; - if (result.Contains(normalized, StringComparer.OrdinalIgnoreCase)) - continue; - result.Add(normalized); - } - - if (RequiresJellyfinFallback(field)) - { - if (!result.Contains(JellyfinAction, StringComparer.OrdinalIgnoreCase)) - result.Add(JellyfinAction); - } - - return result.ToArray(); - } - - private static string NormalizeActionToken(string field, string? raw, IReadOnlySet allowedLanguages) - { - var action = (raw ?? string.Empty).Trim(); - if (action.Equals("J", StringComparison.OrdinalIgnoreCase)) - return JellyfinAction; - if (action.Equals("O", StringComparison.OrdinalIgnoreCase)) - return OriginalAction; - if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)) - return JellyfinAction; - if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase)) - return OriginalAction; - - var lang = action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase) - ? action[LanguagePrefix.Length..].Trim() - : action; - var canonical = allowedLanguages.FirstOrDefault(l => l.Equals(lang, StringComparison.OrdinalIgnoreCase)); - return string.IsNullOrWhiteSpace(canonical) ? string.Empty : LanguagePrefix + canonical; - } + => _store.GetUserRules(userId) ?? new UserRulesDocument { Enabled = false }; private IActionResult ServeEmbedded(string resource, string contentType) { @@ -1314,24 +996,16 @@ public sealed class MultilangController : ControllerBase private async Task IsRequesterAdminAsync() { - var info = await _authorizationContext.GetAuthorizationInfo(HttpContext).ConfigureAwait(false); - if (info?.User is not null && TryGetIsAdmin(info.User, out var authInfoAdmin)) - return authInfoAdmin; - var userId = await GetUserIdAsync().ConfigureAwait(false); - if (string.IsNullOrWhiteSpace(userId) || !Guid.TryParseExact(userId, "N", out var guid)) - return false; + return Guid.TryParse(userId, out var guid) && + _userManager.GetUserById(guid)?.HasPermission(PermissionKind.IsAdministrator) == true; + } - var user = _userManager.GetUserById(guid); - if (user is null) - return false; - - if (TryGetIsAdmin(user, out var isAdmin)) - return isAdmin; - - var remoteEndpoint = HttpContext.Connection.RemoteIpAddress?.ToString() ?? string.Empty; - var dto = _userManager.GetUserDto(user, remoteEndpoint); - return dto.Policy?.IsAdministrator == true; + private bool CanAccessItem(Guid itemId, string userId) + { + var user = _userManager.GetUserById(Guid.Parse(userId)); + return user is not null && !user.HasPermission(PermissionKind.IsDisabled) && + _libraryManager.GetItemById(itemId)?.IsVisibleStandalone(user) == true; } private static bool FormBool(IFormCollection form, string key, bool fallback) @@ -1344,28 +1018,6 @@ public sealed class MultilangController : ControllerBase value.Equals("on", StringComparison.OrdinalIgnoreCase); } - private static bool TryGetIsAdmin(object user, out bool isAdmin) - { - isAdmin = false; - - var policy = user.GetType().GetProperty("Policy")?.GetValue(user); - var policyAdmin = policy?.GetType().GetProperty("IsAdministrator")?.GetValue(policy); - if (policyAdmin is bool isPolicyAdmin) - { - isAdmin = isPolicyAdmin; - return true; - } - - var directAdmin = user.GetType().GetProperty("IsAdministrator")?.GetValue(user); - if (directAdmin is bool isDirectAdmin) - { - isAdmin = isDirectAdmin; - return true; - } - - return false; - } - private string? GetTokenFromRequest() { var req = HttpContext?.Request; diff --git a/src/Jellyfin.Plugin.Multilang/Configuration/configPage.html b/src/Jellyfin.Plugin.Multilang/Configuration/configPage.html index e9c73cd..d5d100d 100644 --- a/src/Jellyfin.Plugin.Multilang/Configuration/configPage.html +++ b/src/Jellyfin.Plugin.Multilang/Configuration/configPage.html @@ -276,7 +276,7 @@ function renderJellyfinTitleLanguages() { var select = document.getElementById("ml-jellyfin-title-language"); var selected = M.prop(state.config, "JellyfinTitleLanguageFallback", "jellyfinTitleLanguageFallback", ""); - select.replaceChildren(); + M.replaceChildren(select); var defaultOption = make("option", "", "Jellyfin's default"); defaultOption.value = ""; select.appendChild(defaultOption); @@ -317,7 +317,7 @@ } function renderProviderInputs() { - providerInputs.replaceChildren(); + M.replaceChildren(providerInputs); state.providers.forEach(function (provider) { var id = providerId(provider); var keyField = providerKeyField(provider); @@ -371,7 +371,7 @@ .map(providerId) .filter(function (id) { return providerSupports(providerById(id), bucket) && orderFor(id, bucket) >= 0; }) .sort(function (a, b) { return orderFor(a, bucket) - orderFor(b, bucket); }); - list.replaceChildren(); + M.replaceChildren(list); ids.forEach(function (id) { list.appendChild(makeProviderChip(id)); }); } @@ -412,7 +412,7 @@ } function renderArticles() { - articleList.replaceChildren(); + M.replaceChildren(articleList); var configured = new Map(); (state.config.ArticleEntries || []).forEach(function (entry) { var lang = normalizeLang(entry.Language || entry.language); @@ -541,7 +541,7 @@ function renderCacheEntries(entries, error) { var root = document.getElementById("ml-cache-entries"); - root.replaceChildren(); + M.replaceChildren(root); if (error) { setDiagnosticsSummary("ml-cache-entries-summary", "Cached responses"); root.appendChild(make("div", "ml-error", error)); @@ -576,7 +576,7 @@ function renderProxyRequests(entries, error) { var root = document.getElementById("ml-proxy-requests"); - root.replaceChildren(); + M.replaceChildren(root); if (error) { setDiagnosticsSummary("ml-proxy-requests-summary", "Recent proxy requests"); root.appendChild(make("div", "ml-error", error)); @@ -674,9 +674,9 @@ var scanRoot = document.getElementById("ml-refresh-scan"); var queueRoot = document.getElementById("ml-refresh-queue"); var httpRoot = document.getElementById("ml-provider-http"); - scanRoot.replaceChildren(); - queueRoot.replaceChildren(); - httpRoot.replaceChildren(); + M.replaceChildren(scanRoot); + M.replaceChildren(queueRoot); + M.replaceChildren(httpRoot); if (error) { setDiagnosticsSummary("ml-refresh-scan-summary", "Latest scan"); @@ -728,18 +728,18 @@ var scanError = String(prop(scan, "Error", "error", "") || ""); if (scanError) scanRoot.appendChild(make("div", "ml-error", scanError)); - var active = prop(queue, "Active", "active", null); + var activeItems = queue.ActiveItems || []; var queued = prop(queue, "Queued", "queued", []) || []; var recent = prop(queue, "Recent", "recent", []) || []; setDiagnosticsSummary("ml-refresh-queue-summary", "Queue (" + Number(prop(queue, "QueuedCount", "queuedCount", queued.length) || 0) + " pending)"); - if (active) { + activeItems.forEach(function (active) { queueRoot.appendChild(makeDiagnosticRow("Active item: " + String(prop(active, "ItemId", "itemId", "")), [ "running now", "source: " + String(prop(active, "SourceTier", "sourceTier", "")), "type: " + String(prop(active, "WorkClass", "workClass", "")), "refresh: " + String(prop(active, "JobType", "jobType", "")) ])); - } + }); if (queued.length === 0) { queueRoot.appendChild(make("div", "ml-muted", "No pending refresh items.")); } else { diff --git a/src/Jellyfin.Plugin.Multilang/Configuration/debugPage.html b/src/Jellyfin.Plugin.Multilang/Configuration/debugPage.html index f2b5e29..620b452 100644 --- a/src/Jellyfin.Plugin.Multilang/Configuration/debugPage.html +++ b/src/Jellyfin.Plugin.Multilang/Configuration/debugPage.html @@ -185,7 +185,7 @@ } function renderSummary(data) { - summary.replaceChildren(); + M.replaceChildren(summary); if (mode.value !== "resolved") return; var top = make("div", "ml-diagnostic-meta"); top.append( @@ -207,7 +207,7 @@ M.setStatus(status, "Loaded."); } catch (err) { lastJson = ""; - summary.replaceChildren(); + M.replaceChildren(summary); output.textContent = authHint(err); M.setStatus(status, err.message, true); } diff --git a/src/Jellyfin.Plugin.Multilang/Configuration/shared.js b/src/Jellyfin.Plugin.Multilang/Configuration/shared.js index 64737a6..f8958ab 100644 --- a/src/Jellyfin.Plugin.Multilang/Configuration/shared.js +++ b/src/Jellyfin.Plugin.Multilang/Configuration/shared.js @@ -1,11 +1,4 @@ (function () { - if (!Element.prototype.replaceChildren) { - Element.prototype.replaceChildren = function () { - this.textContent = ""; - for (let index = 0; index < arguments.length; index++) this.appendChild(arguments[index]); - }; - } - const basePath = () => { const path = window.location.pathname || ""; const marker = "/web/"; @@ -21,18 +14,6 @@ } } - function headers() { - const qs = new URLSearchParams(location.search); - const t = token() || qs.get("token") || qs.get("api_key") || ""; - const h = { "Content-Type": "application/json" }; - if (t) { - h.Authorization = `MediaBrowser Token="${t}"`; - h["X-Emby-Token"] = t; - h["X-MediaBrowser-Token"] = t; - } - return h; - } - function authHeaders(contentType) { const qs = new URLSearchParams(location.search); const t = token() || qs.get("token") || qs.get("api_key") || ""; @@ -58,7 +39,7 @@ async function request(path, options) { const response = await fetch(`${basePath()}${path}`, { cache: "no-store", - headers: headers(), + headers: authHeaders("application/json"), ...options }); if (!response.ok) throw new Error(`HTTP ${response.status}`); @@ -113,41 +94,9 @@ URL.revokeObjectURL(url); } - function chip(label) { - const el = document.createElement("span"); - el.className = "ml-chip"; - el.textContent = label; - el.draggable = true; - el.dataset.value = label; - el.addEventListener("dragstart", (event) => { - event.dataTransfer.setData("text/plain", label); - event.dataTransfer.effectAllowed = "move"; - el.classList.add("ml-dragging"); - }); - el.addEventListener("dragend", () => el.classList.remove("ml-dragging")); - return el; - } - - function bucketValues(bucket) { - return [...bucket.querySelectorAll(".ml-chip")].map((el) => el.dataset.value || el.textContent.trim()).filter(Boolean); - } - - function renderBucket(bucket, values) { - bucket.replaceChildren(...values.map(chip)); - } - - function wireBucket(bucket, onDrop) { - bucket.addEventListener("dragover", (event) => { - event.preventDefault(); - bucket.classList.add("ml-over"); - }); - bucket.addEventListener("dragleave", () => bucket.classList.remove("ml-over")); - bucket.addEventListener("drop", (event) => { - event.preventDefault(); - bucket.classList.remove("ml-over"); - const value = event.dataTransfer.getData("text/plain"); - if (value) onDrop(value, bucket); - }); + function replaceChildren(parent, ...children) { + parent.textContent = ""; + parent.append(...children); } const staticData = { @@ -169,10 +118,7 @@ showModal, hideModals, downloadBlob, - chip, - bucketValues, - renderBucket, - wireBucket, + replaceChildren, staticData }; })(); diff --git a/src/Jellyfin.Plugin.Multilang/Configuration/userRulesPage.html b/src/Jellyfin.Plugin.Multilang/Configuration/userRulesPage.html index 7a5b590..84a12b4 100644 --- a/src/Jellyfin.Plugin.Multilang/Configuration/userRulesPage.html +++ b/src/Jellyfin.Plugin.Multilang/Configuration/userRulesPage.html @@ -109,8 +109,19 @@ var ISO3166 = M.staticData.iso3166; var LANG_REGION_COMBOS = M.staticData.langRegionCombos; + var listeners = []; + function listen(target, type, handler, capture) { + target.addEventListener(type, handler, capture); + listeners.push(() => target.removeEventListener(type, handler, capture)); + } + page._mlDispose = function () { + listeners.forEach(remove => remove()); + listeners = []; + cleanupActionDrag(); + }; + function $(id) { - return document.getElementById(id); + return page.querySelector("#" + id); } function make(tag, className, text) { @@ -144,18 +155,18 @@ } function categoryId(category) { - return category.Id || category.id || ""; + return category.Id; } function categoryLabel(category, index) { - var label = String(category.Label || category.label || "").trim(); + var label = String(category.Label || "").trim(); return label || categoryPlaceholder(index); } function categoryPlaceholder(index) { var used = {}; categories().forEach(function (category) { - var label = String(category.Label || category.label || "").trim().toLowerCase(); + var label = String(category.Label || "").trim().toLowerCase(); if (label) used[label] = true; }); var n = Math.max(1, index + 1); @@ -163,11 +174,10 @@ return "Category " + n; } - function newCategory(label, criteria) { + function newCategory(label) { return { Id: randomId(), Label: label || "", - CriteriaText: criteria || "", Requirements: [emptyRequirement()], MatchAllConditions: true, Scopes: ["M", "S", "C"], @@ -176,7 +186,7 @@ } function emptyRequirement() { - return { Field: "", Fields: [], UseFieldOr: false, Relation: "", UseOr: false, Values: [] }; + return { Fields: [], UseFieldOr: false, Relation: "", UseOr: false, Values: [] }; } function normalizeActionValue(fieldId, value) { @@ -189,13 +199,6 @@ return canonical ? "Language:" + canonical : ""; } - function legacyActionToList(fieldId, legacy) { - if (!legacy || legacy.toLowerCase() === "fallback") { - return ["Jellyfin"]; - } - return [legacy]; - } - function normalizeActionList(fieldId, values) { var field = fields.find(function (f) { return f.id === fieldId; }); var result = []; @@ -211,64 +214,31 @@ return result.length || !(field && field.lockedJ) ? result : ["Jellyfin"]; } - function normalizeActionMap(source, legacy) { - if (legacy && looksLikeDefaultActionMap(source) && hasMeaningfulLegacyActions(legacy)) { - source = null; - } + function normalizeActionMap(source) { var result = {}; fields.forEach(function (field) { - var values = source && (source[field.id] || source[field.label]); - if (!values && legacy) values = legacyActionToList(field.id, legacy[field.id] || legacy[field.label]); - result[field.id] = normalizeActionList(field.id, Array.isArray(values) ? values : values ? [values] : ["Jellyfin"]); + result[field.id] = normalizeActionList(field.id, source[field.id]); }); return result; } - function looksLikeDefaultActionMap(source) { - if (!source) return true; - return fields.every(function (field) { - var values = source[field.id] || source[field.label]; - return Array.isArray(values) && values.length === 1 && String(values[0]).toLowerCase() === "jellyfin"; - }); - } - - function hasMeaningfulLegacyActions(legacy) { - return Object.keys(legacy || {}).some(function (key) { - var action = String(legacy[key] || "").toLowerCase(); - return action && action !== "fallback" && action !== "jellyfin"; - }); - } - function normalizeRules() { - state.rules = state.rules || {}; - state.rules.Categories = (state.rules.Categories || []).filter(function (category) { - var id = categoryId(category).toLowerCase(); - var label = String(category.Label || category.label || "").trim().toLowerCase(); - var criteria = String(category.CriteriaText || category.criteriaText || "").trim(); - return !(id === "fallback" && label === "fallback" && criteria.length === 0); - }).map(function (category, index) { - category.Id = categoryId(category) || randomId(); - category.Label = String(category.Label || category.label || "").trim(); - category.CriteriaText = category.CriteriaText || category.criteriaText || ""; - category.Requirements = normalizeRequirements(category.Requirements || category.requirements); - category.MatchAllConditions = category.MatchAllConditions !== false && category.matchAllConditions !== false; - category.Scopes = category.Scopes || category.scopes || ["M", "S", "C"]; - category.FieldActionLists = normalizeActionMap(category.FieldActionLists || category.fieldActionLists, category.FieldActions || category.fieldActions); - return category; + state.rules.Categories.forEach(function (category) { + category.Requirements = normalizeRequirements(category.Requirements); + category.FieldActionLists = normalizeActionMap(category.FieldActionLists); }); - state.rules.FallbackFieldActions = normalizeActionMap(state.rules.FallbackFieldActions || state.rules.fallbackFieldActions, null); + state.rules.FallbackFieldActions = normalizeActionMap(state.rules.FallbackFieldActions); } function normalizeRequirements(requirements) { var normalized = (requirements || []).map(function (requirement) { - var fields = normalizeRequirementFields(requirement.Fields || requirement.fields, requirement.Field || requirement.field); + var fields = normalizeRequirementFields(requirement.Fields); return { - Field: fields[0] || "", Fields: fields, - UseFieldOr: requirement.UseFieldOr === true || requirement.useFieldOr === true, - Relation: String(requirement.Relation || requirement.relation || "").trim(), - UseOr: requirement.UseOr === true || requirement.useOr === true, - Values: orderedRequirementValues(fields[0] || "", requirement.Values || requirement.values || []) + UseFieldOr: requirement.UseFieldOr === true, + Relation: String(requirement.Relation || "").trim(), + UseOr: requirement.UseOr === true, + Values: orderedRequirementValues(fields[0] || "", requirement.Values || []) }; }); return normalized.length ? normalized : [emptyRequirement()]; @@ -294,14 +264,12 @@ return criteriaFields.find(function (item) { return item.id === id; }); } - function normalizeRequirementFields(values, legacy) { + function normalizeRequirementFields(values) { var raw = []; (values || []).forEach(function (value) { value = String(value || "").trim(); if (value) raw.push(value); }); - legacy = String(legacy || "").trim(); - if (legacy) raw.push(legacy); var wanted = {}; raw.forEach(function (value) { var id = value.toLowerCase(); @@ -319,7 +287,7 @@ } function requirementFields(requirement) { - return normalizeRequirementFields(requirement && (requirement.Fields || requirement.fields), requirement && (requirement.Field || requirement.field)); + return normalizeRequirementFields(requirement.Fields); } function criteriaLabel(id) { @@ -329,7 +297,7 @@ function renderSortLocales() { var select = $("ml-sort-locale"); - select.replaceChildren(option("Auto", "Auto", false)); + M.replaceChildren(select, option("Auto", "Auto", false)); state.allowed.forEach(function (lang) { select.appendChild(option(lang, lang, false)); }); @@ -363,7 +331,6 @@ function syncCategoryFromRow(row, category) { category.Label = row.querySelector("[data-category-label]").value; category.MatchAllConditions = row.querySelector("[data-category-match-all]").checked; - category.CriteriaText = categoryCriteriaText(category); category.Scopes = Array.prototype.slice.call(row.querySelectorAll("[data-category-scope]:checked")) .map(function (input) { return input.value; }); } @@ -376,7 +343,7 @@ } function addSelectOptions(select, items, selectedValue) { - select.replaceChildren(); + M.replaceChildren(select); items.forEach(function (item) { select.appendChild(option(item.label, item.id, item.id === selectedValue)); }); @@ -462,23 +429,6 @@ return values.join(joiner); } - function requirementText(requirement) { - if (!completeRequirement(requirement)) return ""; - var values = orderedRequirementValues(requirementFields(requirement)[0] || "", requirement.Values || []); - var right = values.length === 1 ? values[0] : "(" + values.join(requirement.UseOr ? " or " : " and ") + ")"; - var fields = requirementFields(requirement); - var left = fields.length === 1 ? fields[0] : "(" + fields.join(requirement.UseFieldOr ? " or " : " and ") + ")"; - return left + " " + requirement.Relation + " " + right; - } - - function categoryCriteriaText(category) { - var joiner = category.MatchAllConditions === false ? " or " : " and "; - return (category.Requirements || []) - .filter(completeRequirement) - .map(requirementText) - .join(joiner); - } - function makeRequirementRow(category, requirement, index) { var row = make("div", "ml-requirement-row"); var selectedFields = requirementFields(requirement); @@ -524,7 +474,7 @@ var current = requirement.Relation || ""; var fields = requirementFields(requirement); var multiple = fields.length > 1; - select.replaceChildren(option("Choose relation", "", !current)); + M.replaceChildren(select, option("Choose relation", "", !current)); if (fields.length) { relations.forEach(function (item) { select.appendChild(option(multiple ? item.pluralLabel : item.label, item.id, item.id === current)); @@ -569,7 +519,6 @@ fields = normalizeRequirementFields(fields, ""); var nextKind = fields.length ? fieldKind(fields[0]) : ""; requirement.Fields = fields; - requirement.Field = fields[0] || ""; button.textContent = fieldSummary(requirement); refreshFieldPanelOptions(panel, requirement); if (onChange) onChange(previousKind, nextKind); @@ -674,7 +623,6 @@ function cloneRequirements(requirements) { return normalizeRequirements(requirements).map(function (requirement) { return { - Field: requirement.Field, Fields: (requirement.Fields || []).slice(), UseFieldOr: requirement.UseFieldOr === true, Relation: requirement.Relation, @@ -688,7 +636,6 @@ return { Id: randomId(), Label: "", - CriteriaText: categoryCriteriaText(category), Requirements: cloneRequirements(category.Requirements), MatchAllConditions: category.MatchAllConditions !== false, Scopes: (category.Scopes || ["M", "S", "C"]).slice(), @@ -698,7 +645,7 @@ function renderCategories() { var root = $("ml-categories"); - root.replaceChildren(); + M.replaceChildren(root); categories().forEach(function (category, index) { var row = make("div", "ml-classification-rule"); row.dataset.categoryId = categoryId(category); @@ -809,7 +756,7 @@ function renderActionSource() { var root = $("ml-action-source"); - root.replaceChildren(); + M.replaceChildren(root); actionSourceValues().forEach(function (value) { root.appendChild(makeActionToken(value, false, true)); }); @@ -831,7 +778,7 @@ function renderActionMatrix() { renderActionSource(); var root = $("ml-action-matrix"); - root.replaceChildren(); + M.replaceChildren(root); actionRows().forEach(function (row) { var line = make("div", "ml-action-category-row"); if (row.fallback) line.classList.add("ml-action-category-row-fallback"); @@ -1068,7 +1015,7 @@ function clearCategoryActions(categoryId) { boxesForCategory(categoryId).forEach(function (box) { - box.replaceChildren(); + M.replaceChildren(box); if (actionFieldLocksJ(box.dataset.fieldId)) box.appendChild(makeActionToken("Jellyfin", true, false)); }); } @@ -1085,7 +1032,7 @@ return token.dataset.value !== "Jellyfin"; }); var token = makeActionToken(value, actionFieldLocksJ(box.dataset.fieldId) && value === "Jellyfin", false); - box.replaceChildren(); + M.replaceChildren(box); if (prepend) box.appendChild(token); existing.forEach(function (item) { box.appendChild(item); }); if (!prepend) box.appendChild(token); @@ -1128,7 +1075,6 @@ categories().forEach(function (category) { var complete = (category.Requirements || []).filter(completeRequirement); category.Requirements = complete.length ? complete : [emptyRequirement()]; - category.CriteriaText = complete.map(requirementText).join(" and "); }); return { Enabled: $("ml-enabled").checked, @@ -1140,12 +1086,10 @@ return { Id: category.Id, Label: categoryLabel(category, index), - CriteriaText: complete.map(requirementText).join(category.MatchAllConditions === false ? " or " : " and "), Requirements: complete, MatchAllConditions: category.MatchAllConditions !== false, Scopes: category.Scopes, - FieldActionLists: category.FieldActionLists, - FieldActions: {} + FieldActionLists: category.FieldActionLists }; }) }; @@ -1183,7 +1127,7 @@ $("ml-user-import-modal").hidden = !show; if (!show) { $("ml-user-import-choice").hidden = true; - $("ml-user-import-source").replaceChildren(); + M.replaceChildren($("ml-user-import-source")); } } @@ -1251,7 +1195,7 @@ } var select = $("ml-user-import-source"); - select.replaceChildren(); + M.replaceChildren(select); ids.forEach(function (id) { select.appendChild(option(id, id, false)); }); choice.hidden = false; M.setStatus(status, "No matching user ID was found. Choose which exported user settings to import.", true); @@ -1295,26 +1239,26 @@ $("ml-user-import-run").addEventListener("click", function () { runUserImport().catch(function (err) { M.setStatus(status, String(err), true); }); }); - document.addEventListener("click", function () { closeValuePanels(null); }); + listen(document, "click", function () { closeValuePanels(null); }); $("ml-add-category").addEventListener("click", function () { syncCategoriesFromDom(); updateActionStateFromBoxes(); - categories().push(newCategory("", "")); + categories().push(newCategory("")); renderAll(); }); - document.addEventListener("dragstart", function (event) { + listen(document, "dragstart", function (event) { if (event.target.closest && event.target.closest(".ml-action-token")) event.preventDefault(); }, true); page.addEventListener("pointerdown", beginActionDrag, true); - document.addEventListener("pointermove", moveActionDrag, true); - document.addEventListener("pointerup", endActionDrag, true); - document.addEventListener("pointercancel", function (event) { + listen(document, "pointermove", moveActionDrag, true); + listen(document, "pointerup", endActionDrag, true); + listen(document, "pointercancel", function (event) { if (!actionDrag || event.pointerId !== actionDrag.pointerId) return; restoreActionDrag(); cleanupActionDrag(); updateActionStateFromBoxes(); }, true); - document.addEventListener("viewshow", function (event) { + listen(document, "viewshow", function (event) { if (event.target && event.target.id === "ml-user") load(); }); } diff --git a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Cleanup.cs b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Cleanup.cs index 20c98ac..84f178d 100644 --- a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Cleanup.cs +++ b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Cleanup.cs @@ -42,13 +42,13 @@ public sealed partial class TranslationStore delete.CommandText = @" DELETE FROM facts WHERE item_id = $item_id; DELETE FROM translations WHERE item_id = $item_id; -DELETE FROM assets WHERE item_id = $item_id;"; +DELETE FROM assets WHERE item_id = $item_id; +DELETE FROM fetch_state WHERE item_id = $item_id;"; delete.Parameters.AddWithValue("$item_id", itemId); delete.ExecuteNonQuery(); } tx.Commit(); - CleanupUnreferencedAssetFiles(); } public LocalCleanupResult CleanupForConfiguration(IReadOnlySet liveItemIds, IEnumerable allowedLanguages, bool localAssetStorage) @@ -62,10 +62,18 @@ DELETE FROM assets WHERE item_id = $item_id;"; .Distinct(StringComparer.OrdinalIgnoreCase) .ToArray(); + using (var con = Open()) + { + con.Open(); + using var cmd = con.CreateCommand(); + var parameters = AddParams(cmd, allowed, "$lang"); + cmd.CommandText = $"DELETE FROM fetch_state WHERE source LIKE 'metadata:%' AND substr(source, 10) NOT IN ({parameters});"; + cmd.ExecuteNonQuery(); + } var translationsDeleted = DeleteRowsOutsideLanguages("translations", allowed); var genresDeleted = DeleteRowsOutsideLanguages("genres", allowed); var assetsDeleted = DeleteRowsOutsideLanguages("assets", allowed); - var modeRowsDeleted = localAssetStorage ? 0 : DeleteLocalAssetRows(resetMissingCheckedAt: true); + var modeRowsDeleted = localAssetStorage ? 0 : SwitchLocalAssetsToUrls(); var assetFilesDeleted = CleanupUnreferencedAssetFiles(); return new LocalCleanupResult(translationsDeleted, assetsDeleted + modeRowsDeleted, genresDeleted, assetFilesDeleted); } @@ -80,38 +88,22 @@ DELETE FROM assets WHERE item_id = $item_id;"; return cmd.ExecuteNonQuery(); } - private int DeleteLocalAssetRows(bool resetMissingCheckedAt) + private int SwitchLocalAssetsToUrls() { using var con = Open(); con.Open(); using var tx = con.BeginTransaction(); - var affected = new List(); - using (var select = con.CreateCommand()) - { - select.Transaction = tx; - select.CommandText = "SELECT DISTINCT item_id FROM assets WHERE path_low LIKE '/multilang/assets/%';"; - using var reader = select.ExecuteReader(); - while (reader.Read()) - affected.Add(reader.GetString(0)); - } - - int deleted; - using (var delete = con.CreateCommand()) - { - delete.Transaction = tx; - delete.CommandText = "DELETE FROM assets WHERE path_low LIKE '/multilang/assets/%';"; - deleted = delete.ExecuteNonQuery(); - } - - if (resetMissingCheckedAt && affected.Count > 0) - { - using var update = con.CreateCommand(); - update.Transaction = tx; - var idParams = AddParams(update, affected, "$item"); - update.CommandText = $"UPDATE facts SET missing_checked_at = 0 WHERE item_id IN ({idParams});"; - update.ExecuteNonQuery(); - } - + using var cmd = con.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = @"UPDATE assets SET path = source_url, path_low = lower(source_url) +WHERE path_low LIKE '/multilang/assets/%' AND source_url <> ''; +DELETE FROM fetch_state WHERE source LIKE 'artwork:%' AND item_id IN +(SELECT item_id FROM assets WHERE path_low LIKE '/multilang/assets/%'); +UPDATE facts SET missing_checked_at = 0 WHERE item_id IN +(SELECT item_id FROM assets WHERE path_low LIKE '/multilang/assets/%');"; + cmd.ExecuteNonQuery(); + cmd.CommandText = "DELETE FROM assets WHERE path_low LIKE '/multilang/assets/%';"; + var deleted = cmd.ExecuteNonQuery(); tx.Commit(); return deleted; } diff --git a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.ImportExport.cs b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.ImportExport.cs index 851de78..11f91bf 100644 --- a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.ImportExport.cs +++ b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.ImportExport.cs @@ -15,21 +15,34 @@ public sealed partial class TranslationStore destination.Open(); using (var schema = destination.CreateCommand()) { - schema.CommandText = SchemaSql; + schema.CommandText = SchemaSql + "\nPRAGMA user_version = 1;"; schema.ExecuteNonQuery(); } using var source = Open(); source.Open(); - CopyTable(source, destination, "facts", "item_id, tmdb_id, kind, original_title, original_language, original_language_all, origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, genre_tmdb_ids_json, missing_checked_at, full_checked_at"); - CopyTable(source, destination, "translations", "item_id, lang, field, text"); - CopyTable(source, destination, "assets", "item_id, lang, kind, path, path_low, updated_at"); - CopyTable(source, destination, "genres", "tmdb_id, media, lang, name, name_norm"); - CopyTable(source, destination, "scan_state", "id, last_scan_started"); + using var snapshot = source.BeginTransaction(); + CopyTable(snapshot, destination, "facts", "item_id, tmdb_id, kind, original_title, original_language, original_language_all, origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, genre_tmdb_ids_json, missing_checked_at, full_checked_at"); + CopyTable(snapshot, destination, "translations", "item_id, lang, field, text"); + CopyTable(snapshot, destination, "assets", "item_id, lang, kind, path, path_low, updated_at, source_url, provider"); + CopyTable(snapshot, destination, "fetch_state", "item_id, source, scope, checked_at, complete"); + CopyTable(snapshot, destination, "genres", "tmdb_id, media, lang, name, name_norm"); + CopyTable(snapshot, destination, "scan_state", "id, last_scan_started"); + } + + public static void ValidateTranslationBackup(string path) + { + using var connection = new SqliteConnection($"Data Source={path};Mode=ReadOnly"); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = "PRAGMA user_version;"; + if (Convert.ToInt32(command.ExecuteScalar()) != 1) + throw new InvalidOperationException("Unsupported translations backup version. Export a new backup using the current plugin."); } public ImportDatabaseResult ImportTranslationDatabase(string sourcePath, IReadOnlySet liveItemIds, IReadOnlySet allowedLanguages) { + ValidateTranslationBackup(sourcePath); var allowed = allowedLanguages.ToHashSet(StringComparer.OrdinalIgnoreCase); allowed.Add(MultilangConstants.OriginalAction); @@ -39,187 +52,62 @@ public sealed partial class TranslationStore source.Open(); using var tx = destination.BeginTransaction(); - var facts = ImportFacts(source, destination, tx, liveItemIds); - var translations = ImportTranslations(source, destination, tx, liveItemIds, allowed); - var assets = ImportAssets(source, destination, tx, liveItemIds, allowed); - var genres = ImportGenres(source, destination, tx, allowed); + bool Live(SqliteDataReader row) => liveItemIds.Contains(row.GetString(0)); + bool Translation(SqliteDataReader row) => Live(row) && allowed.Contains(row.GetString(1)); + var facts = ImportTable(source, tx, "facts", "item_id, tmdb_id, kind, original_title, original_language, original_language_all, origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, genre_tmdb_ids_json, missing_checked_at, full_checked_at", Live); + var translations = ImportTable(source, tx, "translations", "item_id, lang, field, text", Translation); + var assets = ImportTable(source, tx, "assets", "item_id, lang, kind, path, path_low, updated_at, source_url, provider", Translation); + var genres = ImportTable(source, tx, "genres", "tmdb_id, media, lang, name, name_norm", row => allowed.Contains(row.GetString(2))); + ImportTable(source, tx, "fetch_state", "item_id, source, scope, checked_at, complete", row => + Live(row) && (row.GetString(1).StartsWith("artwork:") || (row.GetString(1).StartsWith("metadata:", StringComparison.Ordinal) && allowed.Contains(row.GetString(1)["metadata:".Length..])))); tx.Commit(); return new ImportDatabaseResult(facts, translations, assets, genres); } - private static void CopyTable(SqliteConnection source, SqliteConnection destination, string table, string columns) + private static void CopyTable(SqliteTransaction snapshot, SqliteConnection destination, string table, string columns) { - using var read = source.CreateCommand(); + using var read = snapshot.Connection!.CreateCommand(); + read.Transaction = snapshot; read.CommandText = $"SELECT {columns} FROM {table};"; using var reader = read.ExecuteReader(); using var tx = destination.BeginTransaction(); + var names = columns.Split(',', StringSplitOptions.TrimEntries); + using var insert = destination.CreateCommand(); + insert.Transaction = tx; + insert.CommandText = $"INSERT OR REPLACE INTO {table}({columns}) VALUES({string.Join(",", names.Select((_, i) => "$v" + i.ToString(CultureInfo.InvariantCulture)))});"; + for (var i = 0; i < names.Length; i++) + insert.Parameters.Add(new SqliteParameter("$v" + i.ToString(CultureInfo.InvariantCulture), DBNull.Value)); while (reader.Read()) { - using var insert = destination.CreateCommand(); - insert.Transaction = tx; - var names = columns.Split(',', StringSplitOptions.TrimEntries); - insert.CommandText = $"INSERT OR REPLACE INTO {table}({columns}) VALUES({string.Join(",", names.Select((_, i) => "$v" + i.ToString(CultureInfo.InvariantCulture)))});"; for (var i = 0; i < names.Length; i++) - insert.Parameters.AddWithValue("$v" + i.ToString(CultureInfo.InvariantCulture), reader.GetValue(i)); + insert.Parameters[i].Value = reader.GetValue(i); insert.ExecuteNonQuery(); } tx.Commit(); } - private static int ImportFacts(SqliteConnection source, SqliteConnection destination, SqliteTransaction tx, IReadOnlySet liveItemIds) + private static int ImportTable(SqliteConnection source, SqliteTransaction tx, string table, string columns, Func include) { using var read = source.CreateCommand(); - read.CommandText = @" -SELECT item_id, tmdb_id, kind, original_title, original_language, original_language_all, - origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, - genre_tmdb_ids_json, missing_checked_at, full_checked_at -FROM facts;"; + read.CommandText = $"SELECT {columns} FROM {table};"; using var reader = read.ExecuteReader(); + using var insert = tx.Connection!.CreateCommand(); + insert.Transaction = tx; + var names = columns.Split(',', StringSplitOptions.TrimEntries); + insert.CommandText = $"INSERT OR REPLACE INTO {table}({columns}) VALUES({string.Join(",", names.Select((_, i) => "$v" + i))});"; + for (var i = 0; i < names.Length; i++) + insert.Parameters.Add(new SqliteParameter("$v" + i, DBNull.Value)); var count = 0; while (reader.Read()) { - var itemId = reader.GetString(0); - if (!liveItemIds.Contains(itemId)) + if (!include(reader)) continue; - using var insert = destination.CreateCommand(); - insert.Transaction = tx; - insert.CommandText = @" -INSERT INTO facts(item_id, tmdb_id, kind, original_title, original_language, original_language_all, - origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, - genre_tmdb_ids_json, missing_checked_at, full_checked_at) -VALUES($item_id, $tmdb_id, $kind, $original_title, $original_language, $original_language_all, - $origin_countries_json, $production_countries_json, $spoken_languages_json, $audio_track_language, - $genre_tmdb_ids_json, $missing_checked_at, $full_checked_at) -ON CONFLICT(item_id) DO UPDATE SET - tmdb_id = excluded.tmdb_id, - kind = excluded.kind, - original_title = excluded.original_title, - original_language = excluded.original_language, - original_language_all = excluded.original_language_all, - origin_countries_json = excluded.origin_countries_json, - production_countries_json = excluded.production_countries_json, - spoken_languages_json = excluded.spoken_languages_json, - audio_track_language = excluded.audio_track_language, - genre_tmdb_ids_json = excluded.genre_tmdb_ids_json, - missing_checked_at = excluded.missing_checked_at, - full_checked_at = excluded.full_checked_at;"; - var names = new[] - { - "$item_id", - "$tmdb_id", - "$kind", - "$original_title", - "$original_language", - "$original_language_all", - "$origin_countries_json", - "$production_countries_json", - "$spoken_languages_json", - "$audio_track_language", - "$genre_tmdb_ids_json", - "$missing_checked_at", - "$full_checked_at" - }; for (var i = 0; i < names.Length; i++) - insert.Parameters.AddWithValue(names[i], reader.GetValue(i)); + insert.Parameters[i].Value = reader.GetValue(i); insert.ExecuteNonQuery(); count++; } - - return count; - } - - private static int ImportTranslations(SqliteConnection source, SqliteConnection destination, SqliteTransaction tx, IReadOnlySet liveItemIds, IReadOnlySet allowedLanguages) - { - using var read = source.CreateCommand(); - read.CommandText = "SELECT item_id, lang, field, text FROM translations;"; - using var reader = read.ExecuteReader(); - var count = 0; - while (reader.Read()) - { - var itemId = reader.GetString(0); - var lang = reader.GetString(1); - if (!liveItemIds.Contains(itemId) || !allowedLanguages.Contains(lang)) - continue; - using var insert = destination.CreateCommand(); - insert.Transaction = tx; - insert.CommandText = @" -INSERT INTO translations(item_id, lang, field, text) -VALUES($item_id, $lang, $field, $text) -ON CONFLICT(item_id, lang, field) DO UPDATE SET text = excluded.text;"; - insert.Parameters.AddWithValue("$item_id", itemId); - insert.Parameters.AddWithValue("$lang", lang); - insert.Parameters.AddWithValue("$field", reader.GetString(2)); - insert.Parameters.AddWithValue("$text", reader.GetString(3)); - insert.ExecuteNonQuery(); - count++; - } - - return count; - } - - private static int ImportAssets(SqliteConnection source, SqliteConnection destination, SqliteTransaction tx, IReadOnlySet liveItemIds, IReadOnlySet allowedLanguages) - { - using var read = source.CreateCommand(); - read.CommandText = "SELECT item_id, lang, kind, path, path_low, updated_at FROM assets;"; - using var reader = read.ExecuteReader(); - var count = 0; - while (reader.Read()) - { - var itemId = reader.GetString(0); - var lang = reader.GetString(1); - if (!liveItemIds.Contains(itemId) || !allowedLanguages.Contains(lang)) - continue; - using var insert = destination.CreateCommand(); - insert.Transaction = tx; - insert.CommandText = @" -INSERT INTO assets(item_id, lang, kind, path, path_low, updated_at) -VALUES($item_id, $lang, $kind, $path, $path_low, $updated_at) -ON CONFLICT(item_id, lang, kind) DO UPDATE SET - path = excluded.path, - path_low = excluded.path_low, - updated_at = excluded.updated_at;"; - insert.Parameters.AddWithValue("$item_id", itemId); - insert.Parameters.AddWithValue("$lang", lang); - insert.Parameters.AddWithValue("$kind", reader.GetString(2)); - insert.Parameters.AddWithValue("$path", reader.GetString(3)); - insert.Parameters.AddWithValue("$path_low", reader.GetString(4)); - insert.Parameters.AddWithValue("$updated_at", reader.GetInt64(5)); - insert.ExecuteNonQuery(); - count++; - } - - return count; - } - - private static int ImportGenres(SqliteConnection source, SqliteConnection destination, SqliteTransaction tx, IReadOnlySet allowedLanguages) - { - using var read = source.CreateCommand(); - read.CommandText = "SELECT tmdb_id, media, lang, name, name_norm FROM genres;"; - using var reader = read.ExecuteReader(); - var count = 0; - while (reader.Read()) - { - var lang = reader.GetString(2); - if (!allowedLanguages.Contains(lang)) - continue; - using var insert = destination.CreateCommand(); - insert.Transaction = tx; - insert.CommandText = @" -INSERT INTO genres(tmdb_id, media, lang, name, name_norm) -VALUES($tmdb_id, $media, $lang, $name, $name_norm) -ON CONFLICT(tmdb_id, media, lang) DO UPDATE SET - name = excluded.name, - name_norm = excluded.name_norm;"; - insert.Parameters.AddWithValue("$tmdb_id", reader.GetInt32(0)); - insert.Parameters.AddWithValue("$media", reader.GetString(1)); - insert.Parameters.AddWithValue("$lang", lang); - insert.Parameters.AddWithValue("$name", reader.GetString(3)); - insert.Parameters.AddWithValue("$name_norm", reader.GetString(4)); - insert.ExecuteNonQuery(); - count++; - } - return count; } } diff --git a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Models.cs b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Models.cs index 6bba5ba..faa2746 100644 --- a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Models.cs +++ b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Models.cs @@ -4,8 +4,6 @@ namespace Jellyfin.Plugin.Multilang.Data; public readonly record struct FactsStatus(bool Exists, long MissingCheckedAt, long FullCheckedAt); -public readonly record struct AssetPresence(bool Any, bool AnyRemote); - public readonly record struct LocalCleanupResult(int TranslationsDeleted, int AssetsDeleted, int GenresDeleted, int AssetFilesDeleted); public readonly record struct ImportDatabaseResult(int Facts, int Translations, int Assets, int Genres); @@ -46,23 +44,17 @@ public sealed class UserCategoryRule public string Label { get; set; } = string.Empty; - public string CriteriaText { get; set; } = string.Empty; - public UserCategoryRequirement[] Requirements { get; set; } = []; public bool MatchAllConditions { get; set; } = true; public string[] Scopes { get; set; } = ["M", "S", "C"]; - public Dictionary FieldActions { get; set; } = MultilangConstants.DefaultFieldActions(); - public Dictionary FieldActionLists { get; set; } = MultilangConstants.DefaultActionLists(); } public sealed class UserCategoryRequirement { - public string Field { get; set; } = string.Empty; - public string[] Fields { get; set; } = []; public bool UseFieldOr { get; set; } diff --git a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Refresh.cs b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Refresh.cs new file mode 100644 index 0000000..ea22f9e --- /dev/null +++ b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Refresh.cs @@ -0,0 +1,125 @@ +using Jellyfin.Plugin.Multilang.Services.Providers; +using Jellyfin.Plugin.Multilang.Services.Refresh; +using Microsoft.Data.Sqlite; +using static Jellyfin.Plugin.Multilang.MultilangConstants; + +namespace Jellyfin.Plugin.Multilang.Data; + +public sealed record FetchState(string Scope, long CheckedAt, bool Complete); +public sealed record StoredAsset(string Language, string Kind, string Path, string SourceUrl, string Provider); + +public sealed partial class TranslationStore +{ + private void Write(SqliteTransaction? transaction, Action action) + { + if (transaction is not null) + { + action(transaction); + return; + } + using var connection = Open(); + connection.Open(); + using var tx = connection.BeginTransaction(); + action(tx); + tx.Commit(); + } + + public Dictionary GetFetchStates(string itemId) + { + using var connection = Open(); + connection.Open(); + using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT source, scope, checked_at, complete FROM fetch_state WHERE item_id = $id;"; + cmd.Parameters.AddWithValue("$id", itemId); + using var reader = cmd.ExecuteReader(); + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + while (reader.Read()) + result.Add(reader.GetString(0), new(reader.GetString(1), reader.GetInt64(2), reader.GetBoolean(3))); + return result; + } + + private static void SaveFetchState(SqliteTransaction tx, string itemId, string source, FetchState state) + { + using var cmd = tx.Connection!.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = "INSERT OR REPLACE INTO fetch_state(item_id, source, scope, checked_at, complete) VALUES($id, $source, $scope, $time, $complete);"; + cmd.Parameters.AddWithValue("$id", itemId); + cmd.Parameters.AddWithValue("$source", source); + cmd.Parameters.AddWithValue("$scope", state.Scope); + cmd.Parameters.AddWithValue("$time", state.CheckedAt); + cmd.Parameters.AddWithValue("$complete", state.Complete); + cmd.ExecuteNonQuery(); + } + + public void SaveMetadata(RefreshItemInfo item, string language, TmdbMetadata? metadata, FetchState state, bool full, bool updateFacts = true) + => Write(null, tx => + { + if (metadata is not null) + { + if (updateFacts) + UpsertFacts(item, metadata, state.CheckedAt, full ? state.CheckedAt : 0, tx); + UpsertTranslation(item.ItemId, language, TitleField, metadata.Title, tx); + UpsertTranslation(item.ItemId, language, OverviewField, metadata.Overview, tx); + UpsertTranslation(item.ItemId, language, TaglineField, metadata.Tagline, tx); + foreach (var genre in metadata.Genres) + UpsertGenre(genre.Id, item.Kind is "movie" or "collection" ? "movie" : "tv", language, genre.Name, tx); + } + else + MarkFactsChecked(item, state.CheckedAt, full ? state.CheckedAt : 0, tx); + SaveFetchState(tx, item.ItemId, "metadata:" + language, state); + }); + + public void CopyOriginalTranslations(string itemId, string language) + => Write(null, tx => + { + using var cmd = tx.Connection!.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = @"DELETE FROM translations WHERE item_id = $id AND lang = 'Original'; +INSERT INTO translations(item_id, lang, field, text) +SELECT item_id, 'Original', field, text FROM translations WHERE item_id = $id AND lang = $lang;"; + cmd.Parameters.AddWithValue("$id", itemId); + cmd.Parameters.AddWithValue("$lang", language); + cmd.ExecuteNonQuery(); + }); + + public List GetStoredAssets(string itemId) + { + using var connection = Open(); + connection.Open(); + using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT lang, kind, path, source_url, provider FROM assets WHERE item_id = $id;"; + cmd.Parameters.AddWithValue("$id", itemId); + using var reader = cmd.ExecuteReader(); + var result = new List(); + while (reader.Read()) + result.Add(new(reader.GetString(0), reader.GetString(1), reader.GetString(2), reader.GetString(3), reader.GetString(4))); + return result; + } + + public void SaveArtwork(string itemId, IEnumerable assets, IReadOnlyDictionary states) + => Write(null, tx => + { + using var cmd = tx.Connection!.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = "DELETE FROM assets WHERE item_id = $id;"; + cmd.Parameters.AddWithValue("$id", itemId); + cmd.ExecuteNonQuery(); + cmd.CommandText = @"INSERT INTO assets(item_id, lang, kind, path, path_low, source_url, provider, updated_at) +VALUES($id, $lang, $kind, $path, $low, $url, $provider, $time);"; + foreach (var name in new[] { "$lang", "$kind", "$path", "$low", "$url", "$provider", "$time" }) + cmd.Parameters.Add(new SqliteParameter(name, DBNull.Value)); + foreach (var asset in assets) + { + cmd.Parameters["$lang"].Value = asset.Language; + cmd.Parameters["$kind"].Value = asset.Kind; + cmd.Parameters["$path"].Value = asset.Path; + cmd.Parameters["$low"].Value = asset.Path.ToLowerInvariant(); + cmd.Parameters["$url"].Value = asset.SourceUrl; + cmd.Parameters["$provider"].Value = asset.Provider; + cmd.Parameters["$time"].Value = NowUnixUtc(); + cmd.ExecuteNonQuery(); + } + foreach (var (provider, state) in states) + SaveFetchState(tx, itemId, provider, state); + }); +} diff --git a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Schema.cs b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Schema.cs index 10b6315..60b75fa 100644 --- a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Schema.cs +++ b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.Schema.cs @@ -38,11 +38,22 @@ CREATE TABLE IF NOT EXISTS assets ( kind TEXT NOT NULL, path TEXT NOT NULL, path_low TEXT NOT NULL DEFAULT '', + source_url TEXT NOT NULL DEFAULT '', + provider TEXT NOT NULL DEFAULT '', updated_at INTEGER NOT NULL, PRIMARY KEY (item_id, lang, kind) ); CREATE INDEX IF NOT EXISTS idx_assets_item ON assets(item_id); +CREATE TABLE IF NOT EXISTS fetch_state ( + item_id TEXT NOT NULL, + source TEXT NOT NULL, + scope TEXT NOT NULL, + checked_at INTEGER NOT NULL, + complete INTEGER NOT NULL, + PRIMARY KEY (item_id, source) +); + CREATE TABLE IF NOT EXISTS genres ( tmdb_id INTEGER NOT NULL, media TEXT NOT NULL, diff --git a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.cs b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.cs index 7c2115b..a8052eb 100644 --- a/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.cs +++ b/src/Jellyfin.Plugin.Multilang/Data/TranslationStore.cs @@ -15,6 +15,34 @@ public sealed partial class TranslationStore private readonly string _dataDir; private readonly string _assetsDir; private readonly string _dbPath; + private readonly SemaphoreSlim _maintenance = new(1, 1); + private readonly SemaphoreSlim _refreshSlots = new(MultilangConstants.RefreshConcurrency, MultilangConstants.RefreshConcurrency); + + public Task EnterRefreshAsync(CancellationToken ct) => EnterAsync(1, ct); + public Task EnterMaintenanceAsync(CancellationToken ct) => EnterAsync(MultilangConstants.RefreshConcurrency, ct); + + private async Task EnterAsync(int slots, CancellationToken ct) + { + await _maintenance.WaitAsync(ct).ConfigureAwait(false); + var acquired = 0; + try + { + for (; acquired < slots; acquired++) + await _refreshSlots.WaitAsync(ct).ConfigureAwait(false); + return new MaintenanceLease(_refreshSlots, slots); + } + catch + { + if (acquired > 0) _refreshSlots.Release(acquired); + throw; + } + finally { _maintenance.Release(); } + } + + private sealed class MaintenanceLease(SemaphoreSlim semaphore, int slots) : IDisposable + { + public void Dispose() => semaphore.Release(slots); + } public TranslationStore(IApplicationPaths appPaths) { @@ -24,6 +52,8 @@ public sealed partial class TranslationStore Directory.CreateDirectory(_assetsDir); _dbPath = Path.Combine(_dataDir, "translations.sqlite"); Initialize(); + if (Plugin.Instance is { } plugin) + plugin.Store = this; } public string DataDirectory => _dataDir; @@ -42,27 +72,38 @@ public sealed partial class TranslationStore => JsonSerializer.Serialize(values.Where(v => !string.IsNullOrWhiteSpace(v)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray()); public SqliteConnection Open() - => new($"Data Source={_dbPath};Cache=Shared;Pooling=False"); + => new($"Data Source={_dbPath};Pooling=False"); private void Initialize() { using var con = Open(); con.Open(); using var cmd = con.CreateCommand(); + cmd.CommandText = "PRAGMA journal_mode=WAL;"; + cmd.ExecuteNonQuery(); cmd.CommandText = SchemaSql; cmd.ExecuteNonQuery(); EnsureColumn(con, "assets", "path_low", "TEXT NOT NULL DEFAULT ''"); + EnsureColumn(con, "assets", "source_url", "TEXT NOT NULL DEFAULT ''"); + EnsureColumn(con, "assets", "provider", "TEXT NOT NULL DEFAULT ''"); + cmd.CommandText = "UPDATE assets SET source_url = path WHERE source_url = '' AND (path_low LIKE 'https://%' OR path_low LIKE 'http://%');"; + cmd.ExecuteNonQuery(); } public void ResetAll() { - SqliteConnection.ClearAllPools(); - if (Directory.Exists(_dataDir)) - Directory.Delete(_dataDir, recursive: true); - Directory.CreateDirectory(_dataDir); + using var con = Open(); + con.Open(); + using var tx = con.BeginTransaction(); + using var cmd = con.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = "DELETE FROM facts; DELETE FROM translations; DELETE FROM assets; DELETE FROM fetch_state; DELETE FROM genres; " + + "DELETE FROM user_rules; DELETE FROM user_display_langs; UPDATE scan_state SET last_scan_started = 0;"; + cmd.ExecuteNonQuery(); + tx.Commit(); + if (Directory.Exists(_assetsDir)) + Directory.Delete(_assetsDir, recursive: true); Directory.CreateDirectory(_assetsDir); - Initialize(); - SqliteConnection.ClearAllPools(); } public (long DatabaseBytes, long AssetsBytes) GetStorageUsage() @@ -147,12 +188,13 @@ ON CONFLICT(id) DO UPDATE SET last_scan_started = excluded.last_scan_started;"; return result; } - public void UpsertFacts(RefreshItemInfo item, TmdbMetadata metadata, long missingCheckedAt, long fullCheckedAt) + public void UpsertFacts(RefreshItemInfo item, TmdbMetadata metadata, long missingCheckedAt, long fullCheckedAt, SqliteTransaction? transaction = null) { - using var con = Open(); - con.Open(); - using var cmd = con.CreateCommand(); - cmd.CommandText = @" + Write(transaction, tx => + { + using var cmd = tx.Connection!.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = @" INSERT INTO facts( item_id, tmdb_id, kind, original_title, original_language, original_language_all, origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, @@ -174,28 +216,30 @@ ON CONFLICT(item_id) DO UPDATE SET genre_tmdb_ids_json = excluded.genre_tmdb_ids_json, missing_checked_at = excluded.missing_checked_at, full_checked_at = CASE WHEN excluded.full_checked_at > 0 THEN excluded.full_checked_at ELSE facts.full_checked_at END;"; - cmd.Parameters.AddWithValue("$item_id", item.ItemId); - cmd.Parameters.AddWithValue("$tmdb_id", item.TmdbId); - cmd.Parameters.AddWithValue("$kind", item.Kind); - cmd.Parameters.AddWithValue("$original_title", metadata.OriginalTitle); - cmd.Parameters.AddWithValue("$original_language", metadata.OriginalLanguage); - cmd.Parameters.AddWithValue("$original_language_all", metadata.OriginalLanguage); - cmd.Parameters.AddWithValue("$origin_countries_json", JsonArray(metadata.OriginCountries)); - cmd.Parameters.AddWithValue("$production_countries_json", JsonArray(metadata.ProductionCountries)); - cmd.Parameters.AddWithValue("$spoken_languages_json", JsonArray(metadata.SpokenLanguages)); - cmd.Parameters.AddWithValue("$audio_track_language", item.AudioLanguage ?? string.Empty); - cmd.Parameters.AddWithValue("$genre_tmdb_ids_json", JsonSerializer.Serialize(metadata.GenreIds)); - cmd.Parameters.AddWithValue("$missing_checked_at", missingCheckedAt); - cmd.Parameters.AddWithValue("$full_checked_at", fullCheckedAt); - cmd.ExecuteNonQuery(); + cmd.Parameters.AddWithValue("$item_id", item.ItemId); + cmd.Parameters.AddWithValue("$tmdb_id", item.TmdbId); + cmd.Parameters.AddWithValue("$kind", item.Kind); + cmd.Parameters.AddWithValue("$original_title", metadata.OriginalTitle); + cmd.Parameters.AddWithValue("$original_language", metadata.OriginalLanguage); + cmd.Parameters.AddWithValue("$original_language_all", metadata.OriginalLanguage); + cmd.Parameters.AddWithValue("$origin_countries_json", JsonArray(metadata.OriginCountries)); + cmd.Parameters.AddWithValue("$production_countries_json", JsonArray(metadata.ProductionCountries)); + cmd.Parameters.AddWithValue("$spoken_languages_json", JsonArray(metadata.SpokenLanguages)); + cmd.Parameters.AddWithValue("$audio_track_language", item.AudioLanguage ?? string.Empty); + cmd.Parameters.AddWithValue("$genre_tmdb_ids_json", JsonSerializer.Serialize(metadata.GenreIds)); + cmd.Parameters.AddWithValue("$missing_checked_at", missingCheckedAt); + cmd.Parameters.AddWithValue("$full_checked_at", fullCheckedAt); + cmd.ExecuteNonQuery(); + }); } - public void MarkFactsChecked(RefreshItemInfo item, long missingCheckedAt, long fullCheckedAt) + public void MarkFactsChecked(RefreshItemInfo item, long missingCheckedAt, long fullCheckedAt, SqliteTransaction? transaction = null) { - using var con = Open(); - con.Open(); - using var cmd = con.CreateCommand(); - cmd.CommandText = @" + Write(transaction, tx => + { + using var cmd = tx.Connection!.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = @" INSERT INTO facts( item_id, tmdb_id, kind, original_title, original_language, original_language_all, origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, @@ -209,12 +253,13 @@ ON CONFLICT(item_id) DO UPDATE SET kind = excluded.kind, missing_checked_at = excluded.missing_checked_at, full_checked_at = CASE WHEN excluded.full_checked_at > 0 THEN excluded.full_checked_at ELSE facts.full_checked_at END;"; - cmd.Parameters.AddWithValue("$item_id", item.ItemId); - cmd.Parameters.AddWithValue("$tmdb_id", item.TmdbId); - cmd.Parameters.AddWithValue("$kind", item.Kind); - cmd.Parameters.AddWithValue("$missing_checked_at", missingCheckedAt); - cmd.Parameters.AddWithValue("$full_checked_at", fullCheckedAt); - cmd.ExecuteNonQuery(); + cmd.Parameters.AddWithValue("$item_id", item.ItemId); + cmd.Parameters.AddWithValue("$tmdb_id", item.TmdbId); + cmd.Parameters.AddWithValue("$kind", item.Kind); + cmd.Parameters.AddWithValue("$missing_checked_at", missingCheckedAt); + cmd.Parameters.AddWithValue("$full_checked_at", fullCheckedAt); + cmd.ExecuteNonQuery(); + }); } public FactsData? GetFacts(string itemId) @@ -290,41 +335,47 @@ WHERE item_id IN ({idParams});"; return rows.ToArray(); } - public void UpsertTranslation(string itemId, string lang, string field, string text) + public void UpsertTranslation(string itemId, string lang, string field, string text, SqliteTransaction? transaction = null) { - using var con = Open(); - con.Open(); - using var cmd = con.CreateCommand(); - cmd.CommandText = @" + Write(transaction, tx => + { + using var cmd = tx.Connection!.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = @" INSERT INTO translations(item_id, lang, field, text) VALUES($item_id, $lang, $field, $text) ON CONFLICT(item_id, lang, field) DO UPDATE SET text = excluded.text;"; - cmd.Parameters.AddWithValue("$item_id", itemId); - cmd.Parameters.AddWithValue("$lang", lang); - cmd.Parameters.AddWithValue("$field", field); - cmd.Parameters.AddWithValue("$text", text); - cmd.ExecuteNonQuery(); + cmd.Parameters.AddWithValue("$item_id", itemId); + cmd.Parameters.AddWithValue("$lang", lang); + cmd.Parameters.AddWithValue("$field", field); + cmd.Parameters.AddWithValue("$text", text); + cmd.ExecuteNonQuery(); + }); } - public void UpsertAsset(string itemId, string lang, string kind, string path, long updatedAt) + public void UpsertAsset(string itemId, string lang, string kind, string path, long updatedAt, SqliteTransaction? transaction = null) { - using var con = Open(); - con.Open(); - using var cmd = con.CreateCommand(); - cmd.CommandText = @" -INSERT INTO assets(item_id, lang, kind, path, path_low, updated_at) -VALUES($item_id, $lang, $kind, $path, $path_low, $updated_at) + Write(transaction, tx => + { + using var cmd = tx.Connection!.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = @" +INSERT INTO assets(item_id, lang, kind, path, path_low, updated_at, source_url) +VALUES($item_id, $lang, $kind, $path, $path_low, $updated_at, $source_url) ON CONFLICT(item_id, lang, kind) DO UPDATE SET path = excluded.path, path_low = excluded.path_low, + source_url = excluded.source_url, updated_at = excluded.updated_at;"; - cmd.Parameters.AddWithValue("$item_id", itemId); - cmd.Parameters.AddWithValue("$lang", lang); - cmd.Parameters.AddWithValue("$kind", kind); - cmd.Parameters.AddWithValue("$path", path); - cmd.Parameters.AddWithValue("$path_low", path.ToLowerInvariant()); - cmd.Parameters.AddWithValue("$updated_at", updatedAt); - cmd.ExecuteNonQuery(); + cmd.Parameters.AddWithValue("$item_id", itemId); + cmd.Parameters.AddWithValue("$lang", lang); + cmd.Parameters.AddWithValue("$kind", kind); + cmd.Parameters.AddWithValue("$path", path); + cmd.Parameters.AddWithValue("$path_low", path.ToLowerInvariant()); + cmd.Parameters.AddWithValue("$source_url", Uri.TryCreate(path, UriKind.Absolute, out var url) && url.Scheme is "https" or "http" ? path : ""); + cmd.Parameters.AddWithValue("$updated_at", updatedAt); + cmd.ExecuteNonQuery(); + }); } public string? GetAssetPath(string itemId, string lang, string kind) @@ -339,76 +390,39 @@ ON CONFLICT(item_id, lang, kind) DO UPDATE SET return cmd.ExecuteScalar() as string; } - public AssetPresence GetAssetPresence(string itemId, IEnumerable languages) + public bool IsAssetReferenced(string itemId, string path) { - var langs = languages.Where(l => !string.IsNullOrWhiteSpace(l)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); - if (langs.Length == 0) - return new AssetPresence(false, false); - using var con = Open(); con.Open(); using var cmd = con.CreateCommand(); - var langParams = AddParams(cmd, langs, "$lang"); - cmd.CommandText = $"SELECT path_low FROM assets WHERE item_id = $item_id AND lang IN ({langParams});"; - cmd.Parameters.AddWithValue("$item_id", itemId); - using var reader = cmd.ExecuteReader(); - var any = false; - var anyRemote = false; - while (reader.Read()) - { - any = true; - var path = reader.GetString(0); - if (path.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || path.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) - anyRemote = true; - } - - return new AssetPresence(any, anyRemote); + cmd.CommandText = "SELECT 1 FROM assets WHERE item_id = $item AND path = $path LIMIT 1;"; + cmd.Parameters.AddWithValue("$item", itemId); + cmd.Parameters.AddWithValue("$path", path); + return cmd.ExecuteScalar() is not null; } - public bool HasMissingConfiguredTranslations(string itemId, IEnumerable languages) - { - var langs = languages.Where(l => !string.IsNullOrWhiteSpace(l)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); - if (langs.Length == 0) - return false; - - using var con = Open(); - con.Open(); - using var cmd = con.CreateCommand(); - var langParams = AddParams(cmd, langs, "$lang"); - cmd.CommandText = $@" -SELECT lang, COUNT(DISTINCT field) -FROM translations -WHERE item_id = $item_id AND lang IN ({langParams}) AND field IN ('title', 'overview', 'tagline') -GROUP BY lang;"; - cmd.Parameters.AddWithValue("$item_id", itemId); - using var reader = cmd.ExecuteReader(); - var counts = new Dictionary(StringComparer.OrdinalIgnoreCase); - while (reader.Read()) - counts[reader.GetString(0)] = reader.GetInt32(1); - - return langs.Any(lang => !counts.TryGetValue(lang, out var count) || count < 3); - } - - public void UpsertGenre(int tmdbId, string media, string lang, string name) + public void UpsertGenre(int tmdbId, string media, string lang, string name, SqliteTransaction? transaction = null) { if (tmdbId <= 0 || string.IsNullOrWhiteSpace(media) || string.IsNullOrWhiteSpace(lang) || string.IsNullOrWhiteSpace(name)) return; - using var con = Open(); - con.Open(); - using var cmd = con.CreateCommand(); - cmd.CommandText = @" + Write(transaction, tx => + { + using var cmd = tx.Connection!.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = @" INSERT INTO genres(tmdb_id, media, lang, name, name_norm) VALUES($tmdb_id, $media, $lang, $name, $name_norm) ON CONFLICT(tmdb_id, media, lang) DO UPDATE SET name = excluded.name, name_norm = excluded.name_norm;"; - cmd.Parameters.AddWithValue("$tmdb_id", tmdbId); - cmd.Parameters.AddWithValue("$media", media.Trim().ToLowerInvariant()); - cmd.Parameters.AddWithValue("$lang", lang.Trim()); - cmd.Parameters.AddWithValue("$name", name.Trim()); - cmd.Parameters.AddWithValue("$name_norm", name.Trim().ToLowerInvariant()); - cmd.ExecuteNonQuery(); + cmd.Parameters.AddWithValue("$tmdb_id", tmdbId); + cmd.Parameters.AddWithValue("$media", media.Trim().ToLowerInvariant()); + cmd.Parameters.AddWithValue("$lang", lang.Trim()); + cmd.Parameters.AddWithValue("$name", name.Trim()); + cmd.Parameters.AddWithValue("$name_norm", name.Trim().ToLowerInvariant()); + cmd.ExecuteNonQuery(); + }); } public GenreRow[] GetGenres(string media) diff --git a/src/Jellyfin.Plugin.Multilang/Jellyfin.Plugin.Multilang.csproj b/src/Jellyfin.Plugin.Multilang/Jellyfin.Plugin.Multilang.csproj index d2f80dd..44d9e2f 100644 --- a/src/Jellyfin.Plugin.Multilang/Jellyfin.Plugin.Multilang.csproj +++ b/src/Jellyfin.Plugin.Multilang/Jellyfin.Plugin.Multilang.csproj @@ -7,9 +7,9 @@ enable enable false - 0.3.3 - 0.3.3.0 - 0.3.3.0 + 0.3.4 + 0.3.4.0 + 0.3.4.0 ajp_anton @@ -18,10 +18,10 @@ - + runtime - + runtime diff --git a/src/Jellyfin.Plugin.Multilang/MultilangConstants.cs b/src/Jellyfin.Plugin.Multilang/MultilangConstants.cs index 8b60285..0c04ad0 100644 --- a/src/Jellyfin.Plugin.Multilang/MultilangConstants.cs +++ b/src/Jellyfin.Plugin.Multilang/MultilangConstants.cs @@ -2,6 +2,7 @@ namespace Jellyfin.Plugin.Multilang; internal static class MultilangConstants { + public const int RefreshConcurrency = 4; public const string JellyfinAction = "Jellyfin"; public const string OriginalAction = "Original"; public const string FallbackAction = "Fallback"; @@ -25,12 +26,6 @@ internal static class MultilangConstants => field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) || field.Equals(PosterKind, StringComparison.OrdinalIgnoreCase); - public static Dictionary DefaultFieldActions() - => ActionFields.ToDictionary( - field => field, - _ => FallbackAction, - StringComparer.OrdinalIgnoreCase); - public static Dictionary DefaultActionLists() => ActionFields.ToDictionary( field => field, diff --git a/src/Jellyfin.Plugin.Multilang/Plugin.cs b/src/Jellyfin.Plugin.Multilang/Plugin.cs index 1fb5706..a766455 100644 --- a/src/Jellyfin.Plugin.Multilang/Plugin.cs +++ b/src/Jellyfin.Plugin.Multilang/Plugin.cs @@ -10,6 +10,8 @@ namespace Jellyfin.Plugin.Multilang; public sealed class Plugin : BasePlugin, IHasWebPages { + internal Data.TranslationStore? Store { get; set; } + internal bool Uninstalling { get; private set; } public static Plugin? Instance { get; private set; } public Plugin(IApplicationPaths applicationPaths, IXmlSerializer xmlSerializer) @@ -24,6 +26,8 @@ public sealed class Plugin : BasePlugin, IHasWebPages public override void OnUninstalling() { + Uninstalling = true; + using var maintenance = Store?.EnterMaintenanceAsync(CancellationToken.None).GetAwaiter().GetResult(); WebScriptInjector.RemoveInjected(ApplicationPaths.WebPath); if (Configuration.CleanupDataOnUninstall) diff --git a/src/Jellyfin.Plugin.Multilang/PluginServiceRegistrator.cs b/src/Jellyfin.Plugin.Multilang/PluginServiceRegistrator.cs index a7f97f4..75eaca6 100644 --- a/src/Jellyfin.Plugin.Multilang/PluginServiceRegistrator.cs +++ b/src/Jellyfin.Plugin.Multilang/PluginServiceRegistrator.cs @@ -17,6 +17,8 @@ public sealed class PluginServiceRegistrator : IPluginServiceRegistrator public void RegisterServices(IServiceCollection services, IServerApplicationHost applicationHost) { services.AddMvc().AddApplicationPart(typeof(Plugin).Assembly); + services.AddHttpClient("Multilang.Jellyfin") + .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler { AllowAutoRedirect = false }); services.AddSingleton(); services.AddSingleton(); @@ -25,7 +27,9 @@ public sealed class PluginServiceRegistrator : IPluginServiceRegistrator services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); + services.AddHostedService(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Jellyfin.Plugin.Multilang/Rules/CategoryRuleEvaluator.cs b/src/Jellyfin.Plugin.Multilang/Rules/CategoryRuleEvaluator.cs index f21670e..09a4ea0 100644 --- a/src/Jellyfin.Plugin.Multilang/Rules/CategoryRuleEvaluator.cs +++ b/src/Jellyfin.Plugin.Multilang/Rules/CategoryRuleEvaluator.cs @@ -10,10 +10,8 @@ public sealed record CategoryRuleTrace( string[] Scopes, bool ScopeMatches, bool MatchAllConditions, - bool UsesStructuredRequirements, bool Result, - RequirementTrace[] Requirements, - string CriteriaText); + RequirementTrace[] Requirements); public sealed record RequirementTrace( string[] Fields, @@ -33,59 +31,40 @@ public static class CategoryRuleEvaluator { public static bool Matches(UserCategoryRule category, FactsData facts) { - return Trace(category, facts).Result; + if (!ScopeMatches(category, facts.Kind)) + return false; + if (category.Requirements is not { Length: > 0 }) + return false; + + return category.MatchAllConditions + ? category.Requirements.All(requirement => MatchesRequirement(requirement, facts)) + : category.Requirements.Any(requirement => MatchesRequirement(requirement, facts)); } + private static bool MatchesRequirement(UserCategoryRequirement requirement, FactsData facts) + { + var fields = RequirementFields(requirement); + var expected = NormalizeValues(requirement.Values ?? []); + if (fields.Length == 0 || expected.Length == 0 || string.IsNullOrWhiteSpace(requirement.Relation)) + return false; + bool Match(string field) => CompareRequirement(NormalizeValues(FieldValues(field, facts)), + expected, requirement.Relation.Trim(), requirement.UseOr); + return requirement.UseFieldOr ? fields.Any(Match) : fields.All(Match); + } + + private static string[] NormalizeValues(IEnumerable values) + => values.Where(v => !string.IsNullOrWhiteSpace(v)).Select(v => v.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); + public static CategoryRuleTrace Trace(UserCategoryRule category, FactsData facts) { var scopeMatches = ScopeMatches(category, facts.Kind); - if (!scopeMatches) - { - return new CategoryRuleTrace( - category.Id, - category.Label, - facts.Kind, - category.Scopes ?? [], - false, - category.MatchAllConditions, - category.Requirements is { Length: > 0 }, - false, - [], - category.CriteriaText ?? string.Empty); - } - - if (category.Requirements is { Length: > 0 }) - { - var requirements = category.Requirements.Select(requirement => TraceRequirement(requirement, facts)).ToArray(); - var result = category.MatchAllConditions - ? requirements.All(requirement => requirement.Result) - : requirements.Any(requirement => requirement.Result); - return new CategoryRuleTrace( - category.Id, - category.Label, - facts.Kind, - category.Scopes ?? [], - true, - category.MatchAllConditions, - true, - result, - requirements, - category.CriteriaText ?? string.Empty); - } - - var criteria = (category.CriteriaText ?? string.Empty).Trim(); - var criteriaResult = criteria.Length > 0 && EvaluateCriteria(criteria, facts); - return new CategoryRuleTrace( - category.Id, - category.Label, - facts.Kind, - category.Scopes ?? [], - true, - category.MatchAllConditions, - false, - criteriaResult, - [], - criteria); + var requirements = category.Requirements.Select(requirement => TraceRequirement(requirement, facts)).ToArray(); + var result = scopeMatches && requirements.Length > 0 && (category.MatchAllConditions + ? requirements.All(requirement => requirement.Result) + : requirements.Any(requirement => requirement.Result)); + return new(category.Id, category.Label, facts.Kind, category.Scopes, + scopeMatches, category.MatchAllConditions, result, requirements); } public static string[] ParseJsonStringArray(string json) @@ -120,42 +99,6 @@ public static class CategoryRuleEvaluator return scopes.Contains(wanted, StringComparer.OrdinalIgnoreCase); } - private static bool EvaluateCriteria(string criteria, FactsData facts) - { - var orGroups = SplitByOperator(criteria, "or"); - foreach (var group in orGroups) - { - var terms = SplitByOperator(group, "and"); - if (terms.All(term => EvaluateTerm(term, facts))) - return true; - } - - return false; - } - - private static bool EvaluateTerm(string term, FactsData facts) - { - term = term.Trim(); - if (term.StartsWith("not ", StringComparison.OrdinalIgnoreCase)) - return !EvaluateTerm(term[4..], facts); - - var relation = FindRelation(term); - if (relation is null) - return false; - - var field = term[..relation.Value.Index].Trim(); - var rawValue = term[(relation.Value.Index + relation.Value.Token.Length)..].Trim().Trim('"', '\''); - var result = relation.Value.Token switch - { - "contains" => FieldValues(field, facts).Any(v => ValueMatches(v, rawValue, IsLanguageField(field))), - "not_contains" => !FieldValues(field, facts).Any(v => ValueMatches(v, rawValue, IsLanguageField(field))), - "is" => FieldValues(field, facts) is { Length: 1 } values && ValueMatches(values[0], rawValue, IsLanguageField(field)), - "is_not" => !(FieldValues(field, facts) is { Length: 1 } values && ValueMatches(values[0], rawValue, IsLanguageField(field))), - _ => false - }; - return result; - } - private static RequirementTrace TraceRequirement(UserCategoryRequirement requirement, FactsData facts) { var fields = RequirementFields(requirement); @@ -201,7 +144,11 @@ public static class CategoryRuleEvaluator .Distinct(StringComparer.OrdinalIgnoreCase) .ToArray(); - var result = relation switch + return new RequirementFieldTrace(field, actual, CompareRequirement(actual, expected, relation, useOr)); + } + + private static bool CompareRequirement(string[] actual, string[] expected, string relation, bool useOr) + => relation switch { "is" => RequirementIs(actual, expected, useOr), "is_not" => !RequirementIs(actual, expected, useOr), @@ -209,23 +156,9 @@ public static class CategoryRuleEvaluator "not_contains" => !RequirementContains(actual, expected, useOr), _ => false }; - return new RequirementFieldTrace(field, actual, result); - } private static string[] RequirementFields(UserCategoryRequirement requirement) - { - var fields = (requirement.Fields ?? []) - .Where(v => !string.IsNullOrWhiteSpace(v)) - .Select(v => v.Trim()) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); - - if (fields.Length > 0) - return fields; - - var field = (requirement.Field ?? string.Empty).Trim(); - return field.Length == 0 ? [] : [field]; - } + => NormalizeValues(requirement.Fields); private static bool RequirementIs(string[] actual, string[] expected, bool useOr) { @@ -252,19 +185,6 @@ public static class CategoryRuleEvaluator private static bool EqualsValue(string actual, string expected) => string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase); - private static (string Token, int Index)? FindRelation(string term) - { - foreach (var token in new[] { "not_contains", "contains", "is_not", "is" }) - { - var needle = " " + token + " "; - var index = term.IndexOf(needle, StringComparison.OrdinalIgnoreCase); - if (index >= 0) - return (token, index + 1); - } - - return null; - } - private static string[] FieldValues(string field, FactsData facts) { field = field.Trim().ToLowerInvariant(); @@ -279,44 +199,4 @@ public static class CategoryRuleEvaluator }; } - private static bool IsLanguageField(string field) - => field.Equals("spoken_languages", StringComparison.OrdinalIgnoreCase) || - field.Equals("original_language", StringComparison.OrdinalIgnoreCase) || - field.Equals("audio_language", StringComparison.OrdinalIgnoreCase); - - private static bool ValueMatches(string actual, string expected, bool language) - { - if (expected.Equals("empty", StringComparison.OrdinalIgnoreCase)) - return string.IsNullOrWhiteSpace(actual); - if (!language) - return string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase); - - if (expected.Contains('-', StringComparison.Ordinal)) - return string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase); - return string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase) || - actual.StartsWith(expected + "-", StringComparison.OrdinalIgnoreCase); - } - - private static string[] SplitByOperator(string text, string op) - { - var parts = new List(); - var tokens = text.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - var current = new List(); - foreach (var token in tokens) - { - if (token.Equals(op, StringComparison.OrdinalIgnoreCase)) - { - parts.Add(string.Join(' ', current)); - current.Clear(); - } - else - { - current.Add(token); - } - } - - if (current.Count > 0) - parts.Add(string.Join(' ', current)); - return parts.Where(p => p.Trim().Length > 0).ToArray(); - } } diff --git a/src/Jellyfin.Plugin.Multilang/Rules/UserRulesNormalizer.cs b/src/Jellyfin.Plugin.Multilang/Rules/UserRulesNormalizer.cs new file mode 100644 index 0000000..318a57c --- /dev/null +++ b/src/Jellyfin.Plugin.Multilang/Rules/UserRulesNormalizer.cs @@ -0,0 +1,166 @@ +using Jellyfin.Plugin.Multilang.Data; +using static Jellyfin.Plugin.Multilang.MultilangConstants; + +namespace Jellyfin.Plugin.Multilang.Rules; + +public static class UserRulesNormalizer +{ + private static readonly string[] RequirementFieldOrder = + [ + "original_language", + "spoken_languages", + "audio_language", + "origin_countries", + "production_countries" + ]; + + public static UserRulesDocument Normalize(UserRulesDocument rules, IEnumerable? languages = null) + { + var allowed = (languages ?? (rules.Categories ?? []).SelectMany(c => (c.FieldActionLists ?? []).Values) + .Concat((rules.FallbackFieldActions ?? []).Values).SelectMany(actions => actions ?? []) + .Where(action => action is not null && action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)).Select(action => action[LanguagePrefix.Length..])) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + rules.Categories = NormalizeCategories(rules.Categories, allowed); + rules.FallbackFieldActions = NormalizeActionLists(rules.FallbackFieldActions, allowed); + return rules; + } + + private static UserCategoryRule[] NormalizeCategories(IEnumerable? categories, IReadOnlySet allowedLanguages) + => (categories ?? []) + .Select(category => new UserCategoryRule + { + Id = string.IsNullOrWhiteSpace(category.Id) ? Guid.NewGuid().ToString("N") : category.Id.Trim(), + Label = string.IsNullOrWhiteSpace(category.Label) ? "Category" : category.Label.Trim(), + Requirements = NormalizeRequirements(category.Requirements), + MatchAllConditions = category.MatchAllConditions, + Scopes = (category.Scopes ?? []) + .Where(scope => scope is "M" or "S" or "C") + .Distinct(StringComparer.OrdinalIgnoreCase) + .DefaultIfEmpty("M") + .ToArray(), + FieldActionLists = NormalizeActionLists(category.FieldActionLists, allowedLanguages) + }) + .ToArray(); + + private static UserCategoryRequirement[] NormalizeRequirements(IEnumerable? requirements) + => (requirements ?? []) + .Select(requirement => + { + var fields = NormalizeRequirementFields(requirement.Fields); + return new UserCategoryRequirement + { + Fields = fields, + UseFieldOr = requirement.UseFieldOr, + Relation = NormalizeRequirementRelation(requirement.Relation), + UseOr = requirement.UseOr, + Values = (requirement.Values ?? []) + .Where(v => !string.IsNullOrWhiteSpace(v)) + .Select(v => v.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() + }; + }) + .Where(requirement => + requirement.Fields.Length > 0 && + requirement.Relation.Length > 0 && + requirement.Values.Length > 0) + .ToArray(); + + private static string[] NormalizeRequirementFields(IEnumerable? fields) + { + var requested = (fields ?? []) + .Select(NormalizeRequirementField) + .Where(field => field.Length > 0) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + var result = new List(); + string? kind = null; + foreach (var field in RequirementFieldOrder) + { + if (!requested.Contains(field)) + continue; + + var currentKind = IsCountryRequirementField(field) ? "country" : "language"; + kind ??= currentKind; + if (currentKind == kind) + result.Add(field); + } + + return result.ToArray(); + } + + private static string NormalizeRequirementField(string? field) + { + var normalized = (field ?? string.Empty).Trim().ToLowerInvariant(); + return normalized is "original_language" or "spoken_languages" or "audio_language" or "origin_countries" or "production_countries" + ? normalized + : string.Empty; + } + + private static bool IsCountryRequirementField(string field) + => field.Equals("origin_countries", StringComparison.OrdinalIgnoreCase) || + field.Equals("production_countries", StringComparison.OrdinalIgnoreCase); + + private static string NormalizeRequirementRelation(string? relation) + { + var normalized = (relation ?? string.Empty).Trim().ToLowerInvariant(); + return normalized is "is" or "is_not" or "contains" or "not_contains" + ? normalized + : string.Empty; + } + + private static Dictionary NormalizeActionLists( + IReadOnlyDictionary? actions, + IReadOnlySet allowedLanguages) + { + var result = DefaultActionLists(); + foreach (var field in result.Keys.ToArray()) + { + var raw = actions is not null && actions.TryGetValue(field, out var configured) + ? configured + : [JellyfinAction]; + result[field] = NormalizeActionList(field, raw, allowedLanguages); + } + + return result; + } + + private static string[] NormalizeActionList(string field, IEnumerable? raw, IReadOnlySet allowedLanguages) + { + var result = new List(); + foreach (var action in raw ?? []) + { + var normalized = NormalizeActionToken(action, allowedLanguages); + if (normalized.Length == 0) + continue; + if (result.Contains(normalized, StringComparer.OrdinalIgnoreCase)) + continue; + result.Add(normalized); + } + + if (RequiresJellyfinFallback(field)) + { + if (!result.Contains(JellyfinAction, StringComparer.OrdinalIgnoreCase)) + result.Add(JellyfinAction); + } + + return result.ToArray(); + } + + private static string NormalizeActionToken(string? raw, IReadOnlySet allowedLanguages) + { + var action = (raw ?? string.Empty).Trim(); + if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)) + return JellyfinAction; + if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase)) + return OriginalAction; + + var lang = action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase) + ? action[LanguagePrefix.Length..].Trim() + : string.Empty; + var canonical = allowedLanguages.FirstOrDefault(l => l.Equals(lang, StringComparison.OrdinalIgnoreCase)); + return string.IsNullOrWhiteSpace(canonical) ? string.Empty : LanguagePrefix + canonical; + } + +} diff --git a/src/Jellyfin.Plugin.Multilang/Services/Assets/AssetStorageService.cs b/src/Jellyfin.Plugin.Multilang/Services/Assets/AssetStorageService.cs index 799c3cb..d099426 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Assets/AssetStorageService.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Assets/AssetStorageService.cs @@ -1,4 +1,3 @@ -using System.Net.Http.Headers; using System.Security.Cryptography; using Jellyfin.Plugin.Multilang.Data; @@ -18,8 +17,6 @@ public sealed class AssetStorageService public async Task StoreAsync( string itemId, - string lang, - string kind, string sourceUrl, bool localStorage, CancellationToken cancellationToken) @@ -30,22 +27,17 @@ public sealed class AssetStorageService if (!localStorage) return sourceUrl; - var relative = BuildRelativePath(itemId, lang, kind, sourceUrl); + var relative = BuildRelativePath(itemId, sourceUrl); var expectedUrl = TranslationStore.LocalAssetUrlPrefix + relative.Replace('\\', '/'); - if (_store.GetAssetPath(itemId, lang, kind) is { } existing && - existing.Equals(expectedUrl, StringComparison.OrdinalIgnoreCase) && - _store.TryNormalizeLocalAssetPath(existing, out var existingPath) && - File.Exists(existingPath)) - { - return existing; - } - var destination = Path.GetFullPath(Path.Combine(_store.AssetsDirectory, relative)); var root = Path.GetFullPath(_store.AssetsDirectory) + Path.DirectorySeparatorChar; if (!destination.StartsWith(root, StringComparison.OrdinalIgnoreCase)) throw new InvalidOperationException("Resolved asset path escaped the Multilang assets directory."); - Directory.CreateDirectory(Path.GetDirectoryName(destination) ?? _store.AssetsDirectory); + if (File.Exists(destination)) + return expectedUrl; + + Directory.CreateDirectory(Path.GetDirectoryName(destination)!); await DownloadAsync(sourceUrl, destination, cancellationToken).ConfigureAwait(false); return expectedUrl; } @@ -101,11 +93,11 @@ public sealed class AssetStorageService File.Move(temp, destination, overwrite: true); } - private static string BuildRelativePath(string itemId, string lang, string kind, string sourceUrl) + private static string BuildRelativePath(string itemId, string sourceUrl) { var extension = ExtensionFromUrl(sourceUrl); var hash = Convert.ToHexString(SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(sourceUrl))).ToLowerInvariant()[..16]; - return Path.Combine(SafeSegment(itemId), SafeSegment(lang), SafeSegment(kind) + "-" + hash + extension); + return Path.Combine(SafeSegment(itemId), hash + extension); } private static string SafeSegment(string value) @@ -133,6 +125,6 @@ public sealed class AssetStorageService ".png" => "image/png", ".webp" => "image/webp", ".gif" => "image/gif", - _ => MediaTypeHeaderValue.Parse("application/octet-stream").MediaType ?? "application/octet-stream" + _ => "application/octet-stream" }; } diff --git a/src/Jellyfin.Plugin.Multilang/Services/Backup/MultilangBackupService.cs b/src/Jellyfin.Plugin.Multilang/Services/Backup/MultilangBackupService.cs index 8a6ed50..0ff1914 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Backup/MultilangBackupService.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Backup/MultilangBackupService.cs @@ -1,3 +1,4 @@ +using Jellyfin.Plugin.Multilang.Rules; using System.Globalization; using System.IO.Compression; using System.Text.Json; @@ -142,6 +143,14 @@ public sealed class MultilangBackupService using var zip = ZipFile.OpenRead(zipPath); ValidateManifest(zip); + var dbPath = Path.Combine(tempDir, DatabaseName); + var dbEntry = options.TranslationsDatabase ? zip.GetEntry(DatabaseName) : null; + if (dbEntry is not null) + { + dbEntry.ExtractToFile(dbPath); + TranslationStore.ValidateTranslationBackup(dbPath); + } + var importedConfig = false; var importedUsers = 0; var ignoredUsers = 0; @@ -173,21 +182,13 @@ public sealed class MultilangBackupService continue; } - _store.SaveUserRules(userId, payload.Rules); + _store.SaveUserRules(userId, UserRulesNormalizer.Normalize(payload.Rules, currentConfiguration.Languages)); importedUsers++; } } - if (options.TranslationsDatabase) - { - var dbEntry = zip.GetEntry(DatabaseName); - if (dbEntry is not null) - { - var dbPath = Path.Combine(tempDir, DatabaseName); - dbEntry.ExtractToFile(dbPath, overwrite: true); - database = _store.ImportTranslationDatabase(dbPath, GetLiveItemIds(), GetAllowedLanguages(currentConfiguration)); - } - } + if (dbEntry is not null) + database = _store.ImportTranslationDatabase(dbPath, GetLiveItemIds(), GetAllowedLanguages(currentConfiguration)); if (options.DownloadedAssets) { @@ -268,7 +269,7 @@ public sealed class MultilangBackupService if (payload?.Rules is null) return new BackupImportResult(false, 0, 0, 0, 0, 0, 0, 0, 0, [], false); - _store.SaveUserRules(current, payload.Rules); + _store.SaveUserRules(current, UserRulesNormalizer.Normalize(payload.Rules, Plugin.Instance?.Configuration.Languages)); return new BackupImportResult(false, 1, 0, 0, 0, 0, 0, 0, 0, [], false); } finally diff --git a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyCache.cs b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyCache.cs index abf1877..380cd74 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyCache.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyCache.cs @@ -10,7 +10,7 @@ public sealed class ItemsProxyCache public required string Url { get; init; } - public required HashSet ItemIds { get; init; } + public required int ItemCount { get; init; } public required string Body { get; init; } @@ -21,6 +21,7 @@ public sealed class ItemsProxyCache public required long DurationMs { get; init; } public required long SizeBytes { get; init; } + public bool Micro { get; init; } } private sealed class RecentRequest @@ -50,49 +51,43 @@ public sealed class ItemsProxyCache private readonly object _lock = new(); private readonly Dictionary _entries = new(StringComparer.Ordinal); - private readonly Dictionary _microEntries = new(StringComparer.Ordinal); + private readonly Dictionary _userPolicies = new(StringComparer.Ordinal); private readonly Queue _recentRequests = new(); private long _totalBytes; + private long _generation; + public long Generation { get { lock (_lock) return _generation; } } private const int MaxRecentRequests = 100; private const int MaxMicroEntries = 50; private static readonly TimeSpan MicroTtl = TimeSpan.FromSeconds(3); + public long ObserveUserPolicy(string userId, string policy) + { + lock (_lock) + { + if (_userPolicies.TryGetValue(userId, out var previous) && previous != policy) + ClearUser(userId); + _userPolicies[userId] = policy; + return _generation; + } + } + public void ClearAll() { lock (_lock) { _entries.Clear(); - _microEntries.Clear(); _totalBytes = 0; + _generation++; } } - public void InvalidateItems(IEnumerable itemIds) + public void ClearUser(string userId) { - var ids = itemIds - .Where(id => !string.IsNullOrWhiteSpace(id)) - .Select(id => id.Trim().ToLowerInvariant()) - .ToHashSet(StringComparer.OrdinalIgnoreCase); - if (ids.Count == 0) - return; - lock (_lock) { - foreach (var key in _entries.Values - .Where(e => e.ItemIds.Overlaps(ids)) - .Select(e => e.Key) - .ToArray()) - { + _generation++; + foreach (var key in _entries.Values.Where(e => e.UserId == userId).Select(e => e.Key).ToArray()) Remove(key); - } - - foreach (var key in _microEntries.Values - .Where(e => e.ItemIds.Overlaps(ids)) - .Select(e => e.Key) - .ToArray()) - { - _microEntries.Remove(key); - } } } @@ -103,23 +98,16 @@ public sealed class ItemsProxyCache PurgeExpired(ttl); if (!_entries.TryGetValue(key, out var entry)) { - PurgeExpired(_microEntries, MicroTtl); - if (!_microEntries.TryGetValue(key, out var microEntry)) - { - response = default; - return false; - } - - response = new CachedItemsProxyResponse(microEntry.Body, microEntry.ContentType, microEntry.ItemIds.Count, microEntry.SizeBytes); - return true; + response = default; + return false; } - response = new CachedItemsProxyResponse(entry.Body, entry.ContentType, entry.ItemIds.Count, entry.SizeBytes); + response = new CachedItemsProxyResponse(entry.Body, entry.ContentType, entry.ItemCount, entry.SizeBytes); return true; } } - public bool Store(string key, string userId, string url, IEnumerable itemIds, string body, string contentType, long durationMs, long maxBytes, TimeSpan ttl) + public bool Store(string key, string userId, string url, int itemCount, string body, string contentType, long durationMs, long maxBytes, TimeSpan ttl, long? generation = null, bool micro = false) { if (maxBytes <= 0 || ttl <= TimeSpan.Zero) return false; @@ -130,76 +118,40 @@ public sealed class ItemsProxyCache lock (_lock) { + if (generation.HasValue && generation != _generation) + return false; PurgeExpired(ttl); Remove(key); + if (micro && _entries.Values.Count(e => e.Micro) >= MaxMicroEntries) + Remove(_entries.Values.Where(e => e.Micro).MinBy(e => e.CreatedUtc)!.Key); + while (_totalBytes + sizeBytes > maxBytes && _entries.Count > 0) { var victim = _entries.Values.OrderBy(e => e.CreatedUtc).First(); Remove(victim.Key); } - if (_totalBytes + sizeBytes > maxBytes) - return false; - _entries[key] = new Entry { Key = key, UserId = userId, Url = url, - ItemIds = itemIds - .Where(id => !string.IsNullOrWhiteSpace(id)) - .Select(id => id.Trim().ToLowerInvariant()) - .ToHashSet(StringComparer.OrdinalIgnoreCase), + ItemCount = itemCount, Body = body, ContentType = contentType, CreatedUtc = DateTimeOffset.UtcNow, DurationMs = durationMs, - SizeBytes = sizeBytes + SizeBytes = sizeBytes, + Micro = micro }; _totalBytes += sizeBytes; return true; } } - public bool StoreMicro(string key, string userId, string url, IEnumerable itemIds, string body, string contentType, long durationMs, long maxBytes) - { - if (maxBytes <= 0) - return false; - - var sizeBytes = System.Text.Encoding.UTF8.GetByteCount(body); - if (sizeBytes > maxBytes) - return false; - - lock (_lock) - { - PurgeExpired(_microEntries, MicroTtl); - _microEntries.Remove(key); - - while (_microEntries.Count >= MaxMicroEntries) - { - var victim = _microEntries.Values.OrderBy(e => e.CreatedUtc).First(); - _microEntries.Remove(victim.Key); - } - - _microEntries[key] = new Entry - { - Key = key, - UserId = userId, - Url = url, - ItemIds = itemIds - .Where(id => !string.IsNullOrWhiteSpace(id)) - .Select(id => id.Trim().ToLowerInvariant()) - .ToHashSet(StringComparer.OrdinalIgnoreCase), - Body = body, - ContentType = contentType, - CreatedUtc = DateTimeOffset.UtcNow, - DurationMs = durationMs, - SizeBytes = sizeBytes - }; - return true; - } - } + public bool StoreMicro(string key, string userId, string url, int itemCount, string body, string contentType, long durationMs, long maxBytes, long? generation = null, TimeSpan? ttl = null) + => Store(key, userId, url, itemCount, body, contentType, durationMs, maxBytes, ttl ?? TimeSpan.FromMinutes(1), generation, micro: true); public object[] GetEntries() { @@ -207,6 +159,7 @@ public sealed class ItemsProxyCache { var now = DateTimeOffset.UtcNow; return _entries.Values + .Where(e => !e.Micro) .OrderByDescending(e => e.CreatedUtc) .Select(e => new { @@ -215,7 +168,7 @@ public sealed class ItemsProxyCache AgeSeconds = (long)(now - e.CreatedUtc).TotalSeconds, e.DurationMs, e.SizeBytes, - ItemCount = e.ItemIds.Count + ItemCount = e.ItemCount }) .Cast() .ToArray(); @@ -288,26 +241,16 @@ public sealed class ItemsProxyCache return; var now = DateTimeOffset.UtcNow; - foreach (var key in _entries.Values.Where(e => now - e.CreatedUtc > ttl).Select(e => e.Key).ToArray()) + foreach (var key in _entries.Values.Where(e => now - e.CreatedUtc > (e.Micro ? MicroTtl : ttl)).Select(e => e.Key).ToArray()) Remove(key); } - private static void PurgeExpired(Dictionary entries, TimeSpan ttl) - { - if (ttl <= TimeSpan.Zero || entries.Count == 0) - return; - - var now = DateTimeOffset.UtcNow; - foreach (var key in entries.Values.Where(e => now - e.CreatedUtc > ttl).Select(e => e.Key).ToArray()) - entries.Remove(key); - } - private void Remove(string key) { if (!_entries.Remove(key, out var existing)) return; - _totalBytes = Math.Max(0, _totalBytes - existing.SizeBytes); + _totalBytes -= existing.SizeBytes; } } diff --git a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyInvalidationService.cs b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyInvalidationService.cs new file mode 100644 index 0000000..0133412 --- /dev/null +++ b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyInvalidationService.cs @@ -0,0 +1,35 @@ +using Jellyfin.Data.Events; +using Jellyfin.Database.Implementations.Entities; +using MediaBrowser.Controller.Library; +using Microsoft.Extensions.Hosting; + +namespace Jellyfin.Plugin.Multilang.Services; + +public sealed class ItemsProxyInvalidationService( + ILibraryManager library, IUserManager users, IUserDataManager userData, ItemsProxyCache cache) : IHostedService +{ + public Task StartAsync(CancellationToken cancellationToken) + { + library.ItemAdded += LibraryChanged; + library.ItemUpdated += LibraryChanged; + library.ItemRemoved += LibraryChanged; + users.OnUserUpdated += UserChanged; + userData.UserDataSaved += UserDataChanged; + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + library.ItemAdded -= LibraryChanged; + library.ItemUpdated -= LibraryChanged; + library.ItemRemoved -= LibraryChanged; + users.OnUserUpdated -= UserChanged; + userData.UserDataSaved -= UserDataChanged; + return Task.CompletedTask; + } + + // Membership and native filters can change even when an item wasn't in a cached result. + private void LibraryChanged(object? sender, ItemChangeEventArgs args) => cache.ClearAll(); + private void UserChanged(object? sender, GenericEventArgs args) => cache.ClearUser(args.Argument.Id.ToString("N")); + private void UserDataChanged(object? sender, UserDataSaveEventArgs args) => cache.ClearUser(args.UserId.ToString("N")); +} diff --git a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyPrecacheService.cs b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyPrecacheService.cs index 74a7eae..2194a36 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyPrecacheService.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyPrecacheService.cs @@ -60,8 +60,7 @@ public sealed class ItemsProxyPrecacheService { try { - var port = _configurationManager.GetNetworkConfiguration().InternalHttpPort; - var baseUri = new Uri(_applicationHost.GetLocalApiUrl("127.0.0.1", "http", port).TrimEnd('/') + "/"); + var baseUri = LocalApiUri; var views = await GetJsonAsync(new Uri(baseUri, $"Users/{userId}/Views"), token).ConfigureAwait(false); var items = views["Items"]?.AsArray() ?? []; var targets = items @@ -86,7 +85,7 @@ public sealed class ItemsProxyPrecacheService using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(baseUri, endpoint)); AddTokenHeaders(request, token); - using var response = await _httpClientFactory.CreateClient().SendAsync(request, CancellationToken.None).ConfigureAwait(false); + using var response = await _httpClientFactory.CreateClient("Multilang.Jellyfin").SendAsync(request, CancellationToken.None).ConfigureAwait(false); if (!response.IsSuccessStatusCode) _logger.LogWarning("Multilang pre-cache failed user={UserId} library={LibraryId} status={StatusCode}", userId, target.Id, (int)response.StatusCode); } @@ -106,7 +105,7 @@ public sealed class ItemsProxyPrecacheService { using var request = new HttpRequestMessage(HttpMethod.Get, uri); AddTokenHeaders(request, token); - using var response = await _httpClientFactory.CreateClient().SendAsync(request, CancellationToken.None).ConfigureAwait(false); + using var response = await _httpClientFactory.CreateClient("Multilang.Jellyfin").SendAsync(request, CancellationToken.None).ConfigureAwait(false); response.EnsureSuccessStatusCode(); var body = await response.Content.ReadAsStringAsync(CancellationToken.None).ConfigureAwait(false); return JsonNode.Parse(body)?.AsObject() ?? throw new InvalidOperationException("Jellyfin views response was not a JSON object."); @@ -118,4 +117,7 @@ public sealed class ItemsProxyPrecacheService request.Headers.TryAddWithoutValidation("X-Emby-Token", token); request.Headers.TryAddWithoutValidation("X-MediaBrowser-Token", token); } + + public Uri LocalApiUri => new(_applicationHost.GetLocalApiUrl("127.0.0.1", "http", + _configurationManager.GetNetworkConfiguration().InternalHttpPort).TrimEnd('/') + "/"); } diff --git a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyRequestBuilder.cs b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyRequestBuilder.cs index 143f85e..4dc1ed3 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyRequestBuilder.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyRequestBuilder.cs @@ -15,28 +15,23 @@ public sealed record ItemsProxyRequest( public static class ItemsProxyRequestBuilder { - public static ItemsProxyRequest? Build(HttpRequest request, string rawUrl, string token, string userId, bool multilangEnabled) + private const string BrowseFields = "PrimaryImageAspectRatio,MediaSourceCount,SortName,Overview,Genres,GenreItems"; + public static ItemsProxyRequest? Build(HttpRequest request, string rawUrl, string userId, bool multilangEnabled) { - var baseUri = $"{request.Scheme}://{request.Host}{request.PathBase}"; - var baseAddress = new Uri(baseUri); - var uri = new Uri(baseAddress, rawUrl.StartsWith('/') ? rawUrl : "/" + rawUrl); - if ((rawUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || - rawUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) && - Uri.TryCreate(rawUrl, UriKind.Absolute, out var absolute)) - { - if (!absolute.Scheme.Equals(Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) && - !absolute.Scheme.Equals(Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) - return null; + var baseAddress = new Uri($"{request.Scheme}://{request.Host}{request.PathBase}/"); + if (rawUrl.Contains('\\') || !Uri.TryCreate(baseAddress, rawUrl, out var uri) || + uri.Scheme != baseAddress.Scheme || uri.Authority != baseAddress.Authority || + uri.UserInfo.Length > 0 || uri.Fragment.Length > 0) + return null; - if (!absolute.Host.Equals(baseAddress.Host, StringComparison.OrdinalIgnoreCase) || - absolute.Port != baseAddress.Port) - return null; - - uri = absolute; - } - - var path = NormalizeUpstreamPath(uri.AbsolutePath, userId); - var builder = new UriBuilder(uri) { Path = path }; + var pathBase = request.PathBase.Value ?? string.Empty; + if (!uri.AbsolutePath.StartsWith(pathBase + "/", StringComparison.OrdinalIgnoreCase)) + return null; + var route = uri.AbsolutePath[pathBase.Length..]; + if (!IsSupportedRoute(route)) + return null; + var path = NormalizeUpstreamPath(route, userId); + var builder = new UriBuilder(uri) { Path = pathBase + path }; var query = QueryHelpers.ParseQuery(builder.Query); var rawControls = new ItemsProxyControls( GetQueryString(query, "SortBy"), @@ -48,35 +43,25 @@ public static class ItemsProxyRequestBuilder GetQueryString(query, "GenreIds")); var hasLocalGenreFilter = multilangEnabled && ParseLocalGenreIds(rawControls.GenreIds).Length > 0; var useLocalControls = multilangEnabled && ShouldUseLocalControls(path, query, hasLocalGenreFilter); + if (useLocalControls) + query["Fields"] = string.Join(',', (BrowseFields + "," + GetQueryString(query, "Fields")) + .Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries) + .Distinct(StringComparer.OrdinalIgnoreCase).OrderBy(field => field, StringComparer.OrdinalIgnoreCase)); var controls = useLocalControls ? rawControls - : new ItemsProxyControls(string.Empty, string.Empty, string.Empty, 0, 0, rawControls.ClientLocale, rawControls.GenreIds); - var localCacheKey = string.Join('|', - controls.SortBy, - controls.SortOrder, - controls.NameStartsWith, - controls.StartIndex.ToString(CultureInfo.InvariantCulture), - controls.Limit.ToString(CultureInfo.InvariantCulture), - controls.ClientLocale, - controls.GenreIds); + : new ItemsProxyControls(string.Empty, string.Empty, string.Empty, 0, 0, rawControls.ClientLocale, rawControls.GenreIds, ApplyLocalPaging: false); var pairs = query .Where(kv => !IsLocalProxyQuery(kv.Key, hasLocalGenreFilter, useLocalControls)) .SelectMany(kv => kv.Value.Select(v => KeyValuePair.Create(kv.Key, v ?? string.Empty))) .ToList(); pairs.Add(KeyValuePair.Create("userId", userId)); - if (hasLocalGenreFilter) - { - pairs.Add(KeyValuePair.Create("EnableImageTypes", "Primary,Backdrop,Banner,Thumb")); - pairs.Add(KeyValuePair.Create("Fields", "Genres,GenreItems")); - } - - var cacheQuery = QueryHelpers.AddQueryString(string.Empty, pairs).TrimStart('?'); - builder.Query = QueryHelpers.AddQueryString(string.Empty, pairs).TrimStart('?'); + var cacheQuery = QueryHelpers.AddQueryString(string.Empty, pairs.OrderBy(p => p.Key, StringComparer.OrdinalIgnoreCase)).TrimStart('?'); + builder.Query = cacheQuery; var normalizedForCache = builder.Path + (cacheQuery.Length > 0 ? "?" + cacheQuery : string.Empty); return new ItemsProxyRequest( builder.Uri, controls, - $"{userId}|{normalizedForCache}|{localCacheKey}", + $"{userId}|{normalizedForCache}", normalizedForCache, IsGenresPath(path), GenreMediaFromQuery(query)); @@ -91,6 +76,22 @@ public static class ItemsProxyRequestBuilder .Distinct() .ToArray(); + private static bool IsSupportedRoute(string path) + { + var parts = path.ToLowerInvariant().Split('/', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 3 && parts[0] == "users" && Guid.TryParse(parts[1], out _)) + parts = parts[2..]; + return parts switch + { + ["items"] or ["genres"] => true, + ["items", "latest" or "resume" or "nextup"] => true, + ["items" or "genres", var id] => Guid.TryParse(id, out _), + ["shows", "nextup"] => true, + ["shows", var id, "seasons" or "episodes"] => Guid.TryParse(id, out _), + _ => false + }; + } + private static bool ShouldUseLocalControls( string normalizedPath, IReadOnlyDictionary query, @@ -135,17 +136,14 @@ public static class ItemsProxyRequestBuilder key.Equals("Limit", StringComparison.OrdinalIgnoreCase) || key.Equals("NameStartsWith", StringComparison.OrdinalIgnoreCase))) || (hasGenreFilter && (key.Equals("GenreIds", StringComparison.OrdinalIgnoreCase) || - key.Equals("EnableTotalRecordCount", StringComparison.OrdinalIgnoreCase) || - key.Equals("EnableImageTypes", StringComparison.OrdinalIgnoreCase) || - key.Equals("Fields", StringComparison.OrdinalIgnoreCase))); + key.Equals("EnableTotalRecordCount", StringComparison.OrdinalIgnoreCase))); private static string NormalizeUpstreamPath(string path, string userId) { var segments = path.Split('/', StringSplitOptions.RemoveEmptyEntries).ToList(); if (segments.Count >= 2 && segments[0].Equals("Users", StringComparison.OrdinalIgnoreCase) && - segments[1].Length == 32 && - segments[1].All(Uri.IsHexDigit)) + Guid.TryParse(segments[1], out _)) { segments[1] = userId; return "/" + string.Join('/', segments); diff --git a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxySorting.cs b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxySorting.cs index c71ddae..25324dd 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxySorting.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxySorting.cs @@ -12,7 +12,8 @@ public sealed record ItemsProxyControls( int StartIndex, int Limit, string ClientLocale, - string GenreIds); + string GenreIds, + bool ApplyLocalPaging = true); public static class ItemsProxySorting { @@ -33,6 +34,8 @@ public static class ItemsProxySorting public static void Apply(JsonNode root, ItemsProxyControls controls, CultureInfo culture, Func titleArticlesForItem) { + if (!controls.ApplyLocalPaging) + return; if (!TryGetItemsArray(root, out var itemsArray)) return; diff --git a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyTransformer.cs b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyTransformer.cs index 73063ea..a15da40 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyTransformer.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/ItemsProxyTransformer.cs @@ -22,6 +22,9 @@ public readonly record struct ItemsProxyCategoryMatch( UserCategoryRule? Effective, bool DuplicateLabelResolved); +public sealed record ItemsProxyResolutionAttempt( + string Action, string Source, string LookupKey, bool HasValue, bool Chosen, string? Value, string Reason); + public sealed class ItemsProxyTransformer { private readonly TranslationStore _store; @@ -157,8 +160,6 @@ public sealed class ItemsProxyTransformer { foreach (var action in GetAllActionTokens(category.FieldActionLists)) AddLanguageAction(langs, action); - foreach (var action in (category.FieldActions ?? []).Values) - AddLanguageAction(langs, action); } foreach (var action in GetAllActionTokens(rules.FallbackFieldActions)) @@ -184,29 +185,28 @@ public sealed class ItemsProxyTransformer } public FactsData GetClassificationFacts(UserRulesDocument rules, FactsData facts) + => GetClassificationFacts(rules, new Dictionary { [facts.ItemId] = facts })[facts.ItemId]; + + private IReadOnlyDictionary GetClassificationFacts(UserRulesDocument rules, IReadOnlyDictionary facts) { - if (rules.TrustTmdbCollections || - !facts.Kind.Equals("collection", StringComparison.OrdinalIgnoreCase) || - !Guid.TryParseExact(facts.ItemId, "N", out var itemGuid)) - { + if (rules.TrustTmdbCollections) return facts; - } + var children = facts.Values.Where(fact => fact.Kind == "collection") + .ToDictionary(fact => fact.ItemId, fact => + (_libraryManager.GetItemById(Guid.Parse(fact.ItemId)) as Folder)?.GetLinkedChildren() + .OfType() + .Select(child => TranslationStore.ToItemId32(child.Id)).ToArray() ?? []); + var childFacts = _store.GetFactsForItems(children.Values.SelectMany(ids => ids)); + var result = new Dictionary(facts, StringComparer.OrdinalIgnoreCase); + foreach (var (id, ids) in children) + result[id] = AggregateCollectionFacts(facts[id], ids.Where(childFacts.ContainsKey).Select(id => childFacts[id]).ToArray()); + return result; + } - var item = _libraryManager.GetItemById(itemGuid); - var children = (item as Folder)?.GetLinkedChildren() - .Where(child => child.GetType().Name.Equals("Movie", StringComparison.OrdinalIgnoreCase)) - .ToArray() ?? []; - - var childIds = children.Select(child => TranslationStore.ToItemId32(child.Id)).ToArray(); - if (childIds.Length == 0) - return facts; - - var childFacts = _store.GetFactsForItems(childIds).Values - .Where(child => child.Kind.Equals("movie", StringComparison.OrdinalIgnoreCase)) - .ToArray(); + public static FactsData AggregateCollectionFacts(FactsData facts, FactsData[] childFacts) + { if (childFacts.Length == 0) return facts; - return facts with { OriginalLanguage = SharedSingle(childFacts.Select(child => child.OriginalLanguage)), @@ -228,74 +228,61 @@ public sealed class ItemsProxyTransformer if (actions is not null && actions.TryGetValue(field, out var list)) return list; - return category is not null && category.FieldActions is not null - ? LegacyActionToList(field, category.FieldActions) - : [JellyfinAction]; + return [JellyfinAction]; } public static ItemsProxyResolvedText ResolveField( - string field, - IReadOnlyList actions, - FactsData facts, - Dictionary>? byLang) + string field, IReadOnlyList actions, FactsData facts, + Dictionary>? byLang, + Action? trace = null) { foreach (var action in actions) { if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)) - return new ItemsProxyResolvedText(false, null); - if (field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) && - action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase) && - !string.IsNullOrWhiteSpace(facts.OriginalTitle)) - return new ItemsProxyResolvedText(true, facts.OriginalTitle, facts.OriginalLanguage); - if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase)) { - var originalValue = GetTranslatedField(byLang, OriginalAction, field); - if (!string.IsNullOrWhiteSpace(originalValue)) - return new ItemsProxyResolvedText(true, originalValue, facts.OriginalLanguage); - continue; + trace?.Invoke(new(action, JellyfinAction, "", true, true, null, "Use Jellyfin value")); + return new(false, null); } - - if (!action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)) + var original = action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase); + if (!original && !action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)) continue; - - var value = GetTranslatedField(byLang, action[LanguagePrefix.Length..], field); - if (!string.IsNullOrWhiteSpace(value)) - return new ItemsProxyResolvedText(true, value, action[LanguagePrefix.Length..]); + var lang = original ? OriginalAction : action[LanguagePrefix.Length..]; + var originalTitle = original && field == TitleField && !string.IsNullOrWhiteSpace(facts.OriginalTitle); + var value = originalTitle ? facts.OriginalTitle : GetTranslatedField(byLang, lang, field); + var found = !string.IsNullOrWhiteSpace(value); + trace?.Invoke(new(action, originalTitle ? "OriginalTitle" : "Translation", + originalTitle ? "facts.original_title" : lang + "/" + field, + found, found, value, found ? "Value found" : "Value missing or empty")); + if (found) + return new(true, value, original ? facts.OriginalLanguage : lang); } - - return field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) - ? new ItemsProxyResolvedText(false, null) - : new ItemsProxyResolvedText(true, string.Empty); + return field == TitleField ? new(false, null) : new(true, string.Empty); } public static ItemsProxyResolvedAsset ResolveAsset( - string kind, - IReadOnlyList actions, - Dictionary>? byKind) + string kind, IReadOnlyList actions, + Dictionary>? byKind, + Action? trace = null) { foreach (var action in actions) { if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)) - return new ItemsProxyResolvedAsset(false, null); - if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase)) { - var originalValue = GetAsset(byKind, kind, OriginalAction); - if (!string.IsNullOrWhiteSpace(originalValue)) - return new ItemsProxyResolvedAsset(true, originalValue); - continue; + trace?.Invoke(new(action, JellyfinAction, "", true, true, null, "Use Jellyfin image")); + return new(false, null); } - - if (!action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)) + var original = action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase); + if (!original && !action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)) continue; - - var value = GetAsset(byKind, kind, action[LanguagePrefix.Length..]); - if (!string.IsNullOrWhiteSpace(value)) - return new ItemsProxyResolvedAsset(true, value); + var lang = original ? OriginalAction : action[LanguagePrefix.Length..]; + var value = GetAsset(byKind, kind, lang); + var found = !string.IsNullOrWhiteSpace(value); + trace?.Invoke(new(action, "Asset", kind + "/" + lang, found, found, value, + found ? "Asset found" : "Asset missing")); + if (found) + return new(true, value); } - - return kind.Equals(PosterKind, StringComparison.OrdinalIgnoreCase) - ? new ItemsProxyResolvedAsset(false, null) - : new ItemsProxyResolvedAsset(true, string.Empty); + return kind == PosterKind ? new(false, null) : new(true, string.Empty); } public static string? GetTranslatedField( @@ -373,6 +360,7 @@ public sealed class ItemsProxyTransformer var langs = GetNeededLanguages(rules); var translations = _store.GetTranslations(ids, langs); var assets = _store.GetAssets(ids, langs); + var classification = GetClassificationFacts(rules, factsByItem); var titleLanguages = new Dictionary(StringComparer.OrdinalIgnoreCase); var movieGenreNames = _store.GetGenreNames("movie", clientLocale); var tvGenreNames = _store.GetGenreNames("tv", clientLocale); @@ -384,7 +372,7 @@ public sealed class ItemsProxyTransformer continue; translations.TryGetValue(itemId, out var byLang); - var classificationFacts = GetClassificationFacts(rules, facts); + var classificationFacts = classification[itemId]; var category = PickCategoryMatch(rules, classificationFacts).Effective; var title = ResolveField(TitleField, GetActionList(rules, category, TitleField), facts, byLang); @@ -583,15 +571,6 @@ public sealed class ItemsProxyTransformer langs.Add(action[LanguagePrefix.Length..]); } - private static string[] LegacyActionToList(string field, IReadOnlyDictionary? legacyActions) - { - if (legacyActions is null || !legacyActions.TryGetValue(field, out var action)) - return [JellyfinAction]; - if (string.IsNullOrWhiteSpace(action) || action.Equals(FallbackAction, StringComparison.OrdinalIgnoreCase)) - return [JellyfinAction]; - return [action]; - } - private static string SharedSingle(IEnumerable values) { var unique = values diff --git a/src/Jellyfin.Plugin.Multilang/Services/Providers/FanartClient.cs b/src/Jellyfin.Plugin.Multilang/Services/Providers/FanartClient.cs index 9aefb69..de69743 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Providers/FanartClient.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Providers/FanartClient.cs @@ -32,8 +32,9 @@ public sealed class FanartClient var url = $"https://webservice.fanart.tv/v3/movies/{Uri.EscapeDataString(tmdbId)}?api_key={Uri.EscapeDataString(apiKey)}"; using var request = new HttpRequestMessage(HttpMethod.Get, url); using var response = await ProviderHttp.SendWithRetryAsync(http, request, cancellationToken).ConfigureAwait(false); - if (!response.IsSuccessStatusCode) + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) return null; + response.EnsureSuccessStatusCode(); await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); using var doc = await System.Text.Json.JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); diff --git a/src/Jellyfin.Plugin.Multilang/Services/Providers/FetchRateLimiter.cs b/src/Jellyfin.Plugin.Multilang/Services/Providers/FetchRateLimiter.cs index 4070464..7d79a65 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Providers/FetchRateLimiter.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Providers/FetchRateLimiter.cs @@ -6,6 +6,7 @@ public sealed class FetchRateLimiter { private readonly TimeSpan _minimumSpacing; private long _lastStartTicks; + private readonly SemaphoreSlim _admission = new(1, 1); public FetchRateLimiter(TimeSpan minimumSpacing) { @@ -14,15 +15,18 @@ public sealed class FetchRateLimiter public async Task WaitAsync(CancellationToken cancellationToken) { - var last = Interlocked.Read(ref _lastStartTicks); - if (last != 0) + await _admission.WaitAsync(cancellationToken).ConfigureAwait(false); + try { - var elapsed = TimeSpan.FromSeconds((Stopwatch.GetTimestamp() - last) / (double)Stopwatch.Frequency); - var remaining = _minimumSpacing - elapsed; - if (remaining > TimeSpan.Zero) - await Task.Delay(remaining, cancellationToken).ConfigureAwait(false); - } + if (_lastStartTicks != 0) + { + var remaining = _minimumSpacing - Stopwatch.GetElapsedTime(_lastStartTicks); + if (remaining > TimeSpan.Zero) + await Task.Delay(remaining, cancellationToken).ConfigureAwait(false); + } - Interlocked.Exchange(ref _lastStartTicks, Stopwatch.GetTimestamp()); + _lastStartTicks = Stopwatch.GetTimestamp(); + } + finally { _admission.Release(); } } } diff --git a/src/Jellyfin.Plugin.Multilang/Services/Providers/ProviderHttp.cs b/src/Jellyfin.Plugin.Multilang/Services/Providers/ProviderHttp.cs index 6ce9fca..4bb9878 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Providers/ProviderHttp.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Providers/ProviderHttp.cs @@ -39,29 +39,33 @@ public static class ProviderHttp public static async Task SendWithRetryAsync( HttpClient http, HttpRequestMessage request, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + FetchRateLimiter? rateLimiter = null) { const int maxAttempts = 3; for (var attempt = 1; attempt <= maxAttempts; attempt++) { try { - var response = await http.SendAsync(Clone(request), cancellationToken).ConfigureAwait(false); + if (rateLimiter is not null) + await rateLimiter.WaitAsync(cancellationToken).ConfigureAwait(false); + using var attemptRequest = Clone(request); + var response = await http.SendAsync(attemptRequest, cancellationToken).ConfigureAwait(false); + Record(request, "HTTP " + (int)response.StatusCode, attempt); if (response.IsSuccessStatusCode || Classify(response.StatusCode) == ProviderFailureKind.Terminal || attempt == maxAttempts) { - Record(request, "HTTP " + (int)response.StatusCode, attempt); return response; } var delay = GetRetryDelay(response, attempt); response.Dispose(); - await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + await Task.Delay(delay < TimeSpan.Zero ? TimeSpan.Zero : delay, cancellationToken).ConfigureAwait(false); } - catch (Exception ex) + catch (Exception ex) when (!cancellationToken.IsCancellationRequested && ex is HttpRequestException or OperationCanceledException) { + Record(request, ex.GetType().Name, attempt); if (attempt >= maxAttempts) { - Record(request, ex.GetType().Name, attempt); throw; } diff --git a/src/Jellyfin.Plugin.Multilang/Services/Providers/TmdbClient.cs b/src/Jellyfin.Plugin.Multilang/Services/Providers/TmdbClient.cs index 3c59bdf..ab315de 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Providers/TmdbClient.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Providers/TmdbClient.cs @@ -23,11 +23,10 @@ public sealed class TmdbClient if (string.IsNullOrWhiteSpace(apiKey)) throw new InvalidOperationException("TMDb API key is required."); - await rateLimiter.WaitAsync(cancellationToken).ConfigureAwait(false); var http = _httpClientFactory.CreateClient(); var url = $"https://api.themoviedb.org/3/{BuildPath(item)}?api_key={Uri.EscapeDataString(apiKey)}&language={Uri.EscapeDataString(language)}"; using var request = new HttpRequestMessage(HttpMethod.Get, url); - using var response = await ProviderHttp.SendWithRetryAsync(http, request, cancellationToken).ConfigureAwait(false); + using var response = await ProviderHttp.SendWithRetryAsync(http, request, cancellationToken, rateLimiter).ConfigureAwait(false); if (response.StatusCode == HttpStatusCode.NotFound) return null; response.EnsureSuccessStatusCode(); @@ -58,11 +57,10 @@ public sealed class TmdbClient if (item.Kind is "tvepisode") return new TmdbImages([], [], []); - await rateLimiter.WaitAsync(cancellationToken).ConfigureAwait(false); var http = _httpClientFactory.CreateClient(); var url = $"https://api.themoviedb.org/3/{BuildPath(item)}/images?api_key={Uri.EscapeDataString(apiKey)}"; using var request = new HttpRequestMessage(HttpMethod.Get, url); - using var response = await ProviderHttp.SendWithRetryAsync(http, request, cancellationToken).ConfigureAwait(false); + using var response = await ProviderHttp.SendWithRetryAsync(http, request, cancellationToken, rateLimiter).ConfigureAwait(false); if (response.StatusCode == HttpStatusCode.NotFound) return null; response.EnsureSuccessStatusCode(); diff --git a/src/Jellyfin.Plugin.Multilang/Services/Refresh/ItemFetcher.cs b/src/Jellyfin.Plugin.Multilang/Services/Refresh/ItemFetcher.cs new file mode 100644 index 0000000..2b946e1 --- /dev/null +++ b/src/Jellyfin.Plugin.Multilang/Services/Refresh/ItemFetcher.cs @@ -0,0 +1,220 @@ +using Jellyfin.Plugin.Multilang.Configuration; +using Jellyfin.Plugin.Multilang.Data; +using Jellyfin.Plugin.Multilang.Services.Assets; +using Jellyfin.Plugin.Multilang.Services.Providers; +using Microsoft.Extensions.Logging; +using static Jellyfin.Plugin.Multilang.MultilangConstants; + +namespace Jellyfin.Plugin.Multilang.Services.Refresh; + +public sealed class ItemFetcher( + TranslationStore store, TmdbClient tmdb, FanartClient fanart, + AssetStorageService storage, ILogger logger) +{ + private readonly FetchRateLimiter _limiter = new(TimeSpan.FromMilliseconds(250)); + + private static string Scope(RefreshItemInfo item) + => $"{item.Kind}:{item.TmdbId}:{item.SeasonNumber}:{item.EpisodeNumber}"; + + public static bool NeedsFetch(FetchState? state, string scope, bool dataMissing, bool full, long cutoff) + => full || state is null || state.Scope != scope || + (dataMissing && (state.Complete || state.CheckedAt <= cutoff)) || + (!state.Complete && state.CheckedAt <= cutoff); + + public bool NeedsLocalWork(string itemId, PluginConfiguration cfg) + => cfg.AssetStorageMode == "local" && store.GetStoredAssets(itemId).Any(asset => asset.SourceUrl.Length > 0 && + (!store.TryNormalizeLocalAssetPath(asset.Path, out var path) || !File.Exists(path))); + + public bool NeedsRefresh(RefreshItemInfo item, PluginConfiguration cfg) + { + var states = store.GetFetchStates(item.ItemId); + var languages = Languages(cfg); + var texts = store.GetTranslations([item.ItemId], languages.Append(OriginalAction)).GetValueOrDefault(item.ItemId) ?? []; + var cutoff = MissingCutoff(cfg); + var original = store.GetFacts(item.ItemId)?.OriginalLanguage ?? ""; + foreach (var lang in languages) + if (NeedsMetadata(item, lang, lang, texts, states, false, cutoff)) + return true; + if (original.Length > 0 && !languages.Any(lang => SameLanguage(lang, original)) && + NeedsMetadata(item, OriginalAction, original, texts, states, false, cutoff)) + return true; + + if (original.Length > 0 && languages.Any(lang => SameLanguage(lang, original)) && !texts.ContainsKey(OriginalAction)) + return true; + var assets = store.GetStoredAssets(item.ItemId); + if (cfg.AssetStorageMode == "local" && assets.Any(asset => asset.SourceUrl.Length > 0 && + (!store.TryNormalizeLocalAssetPath(asset.Path, out var path) || !File.Exists(path)))) + return true; + var providers = Providers(item, cfg); + var scope = ArtworkScope(item, languages, original, providers); + return providers.Any(provider => NeedsFetch(states.GetValueOrDefault("artwork:" + provider), scope, + ArtworkMissing(assets, languages.Append(OriginalAction).Where(lang => lang != OriginalAction || original.Length > 0), providers, provider), false, cutoff)); + } + + public async Task RefreshAsync(RefreshItemInfo item, PluginConfiguration cfg, RefreshJobType job, CancellationToken ct) + { + var full = job == RefreshJobType.Full; + var localOnly = job == RefreshJobType.Aggregate; + var states = store.GetFetchStates(item.ItemId); + var languages = Languages(cfg); + var texts = store.GetTranslations([item.ItemId], languages.Append(OriginalAction)).GetValueOrDefault(item.ItemId) ?? []; + var cutoff = MissingCutoff(cfg); + var fetched = 0; + async Task FetchMetadata(string storedLanguage, string requestedLanguage) + { + if (localOnly || !NeedsMetadata(item, storedLanguage, requestedLanguage, texts, states, full, cutoff)) + return; + var metadata = await tmdb.FetchMetadataAsync(item, requestedLanguage, cfg.TmdbApiKey, _limiter, ct).ConfigureAwait(false); + var complete = metadata is not null && metadata.Title.Length > 0 && metadata.Overview.Length > 0 && + (item.Kind != "movie" || metadata.Tagline.Length > 0); + store.SaveMetadata(item, storedLanguage, metadata, + new(Scope(item) + ":" + requestedLanguage, TranslationStore.NowUnixUtc(), complete), full, + item.Kind is "movie" or "tv" || storedLanguage == languages[0]); + if (metadata is not null) + Interlocked.Increment(ref fetched); + if (cfg.EnableLogging) + logger.LogInformation("TMDb metadata {Result} item={ItemId} lang={Language} storedAs={StoredLanguage}", + metadata is null ? "404" : "fetched", item.ItemId, requestedLanguage, storedLanguage); + } + + await Parallel.ForEachAsync(languages, new ParallelOptions { MaxDegreeOfParallelism = 4, CancellationToken = ct }, + async (lang, _) => await FetchMetadata(lang, lang).ConfigureAwait(false)).ConfigureAwait(false); + var original = store.GetFacts(item.ItemId)?.OriginalLanguage ?? ""; + if (original.Length > 0) + { + var configured = languages.FirstOrDefault(lang => SameLanguage(lang, original)); + if (configured is not null) + store.CopyOriginalTranslations(item.ItemId, configured); + else + await FetchMetadata(OriginalAction, original).ConfigureAwait(false); + } + + var artworkFetched = await RefreshArtwork(item, cfg, languages, original, states, full, localOnly, cutoff, ct).ConfigureAwait(false); + if (!localOnly) + store.MarkFactsChecked(item, TranslationStore.NowUnixUtc(), full ? TranslationStore.NowUnixUtc() : 0); + return fetched > 0 || artworkFetched; + } + + private static bool NeedsMetadata(RefreshItemInfo item, string storedLanguage, string requestedLanguage, + Dictionary> texts, Dictionary states, bool full, long cutoff) + { + var fields = texts.GetValueOrDefault(storedLanguage); + var missing = fields is null || TextFields.Any(field => !fields.ContainsKey(field) || + ((field != TaglineField || item.Kind == "movie") && string.IsNullOrWhiteSpace(fields[field]))); + return NeedsFetch(states.GetValueOrDefault("metadata:" + storedLanguage), + Scope(item) + ":" + requestedLanguage, missing, full, cutoff); + } + + private async Task RefreshArtwork(RefreshItemInfo item, PluginConfiguration cfg, string[] languages, + string original, Dictionary states, bool full, bool localOnly, long cutoff, CancellationToken ct) + { + if (localOnly) + { + if (cfg.AssetStorageMode != "local") + return false; + var assets = store.GetStoredAssets(item.ItemId); + var repaired = new List(); + foreach (var asset in assets) + { + var path = asset.SourceUrl.Length == 0 ? asset.Path + : await storage.StoreAsync(item.ItemId, asset.SourceUrl, true, ct).ConfigureAwait(false); + repaired.Add(asset with { Path = path! }); + } + store.SaveArtwork(item.ItemId, repaired, new Dictionary()); + return true; + } + var providers = Providers(item, cfg); + var priority = providers.Select((provider, index) => (provider, index)).ToDictionary(p => p.provider, p => p.index); + var wanted = languages.ToDictionary(lang => lang, BaseLanguage, StringComparer.OrdinalIgnoreCase); + if (original.Length > 0) + wanted[OriginalAction] = BaseLanguage(original); + var previous = store.GetStoredAssets(item.ItemId); + var selected = previous.Where(asset => wanted.ContainsKey(asset.Language) && priority.ContainsKey(asset.Provider)) + .ToDictionary(asset => (asset.Language, asset.Kind)); + var updates = new Dictionary(); + var scope = ArtworkScope(item, languages, original, providers); + foreach (var provider in providers) + { + var kinds = ArtworkKinds(provider); + var keys = wanted.Keys.SelectMany(lang => kinds.Select(kind => (Language: lang, Kind: kind))).ToArray(); + var missing = keys.Any(key => !selected.TryGetValue(key, out var asset) || priority[asset.Provider] > priority[provider]); + if (localOnly || !NeedsFetch(states.GetValueOrDefault("artwork:" + provider), scope, missing, full, cutoff)) + continue; + + Dictionary> images; + if (provider == "tmdb") + { + var response = await tmdb.FetchImagesAsync(item, cfg.TmdbApiKey, _limiter, ct).ConfigureAwait(false); + images = new() { [PosterKind] = response?.Posters ?? [], [LogoKind] = response?.Logos ?? [], [BackdropKind] = response?.Backdrops ?? [] }; + } + else + { + var response = await fanart.FetchMovieImagesAsync(item.TmdbId, cfg.FanartApiKey, ct).ConfigureAwait(false); + images = new() { [PosterKind] = response?.Posters ?? [], [LogoKind] = response?.Logos ?? [], [BannerKind] = response?.Banners ?? [], [ThumbKind] = response?.Thumbs ?? [] }; + } + + var complete = true; + foreach (var key in keys) + { + selected.TryGetValue(key, out var existing); + var url = images[key.Kind].GetValueOrDefault(wanted[key.Language]); + if (string.IsNullOrWhiteSpace(url)) + { + complete = false; + if (existing?.Provider == provider) + selected.Remove(key); + } + else if (existing is null || priority[provider] <= priority[existing.Provider]) + { + // Keep an existing local path until the scheduled storage-mode cleanup. + var path = existing?.SourceUrl == url ? existing.Path : url; + selected[key] = new(key.Language, key.Kind, path, url, provider); + } + } + updates["artwork:" + provider] = new(scope, TranslationStore.NowUnixUtc(), complete); + if (cfg.EnableLogging && cfg.VerboseLogging) + logger.LogInformation("Artwork fetched item={ItemId} provider={Provider} languages={Languages}", + item.ItemId, provider, string.Join(",", wanted.Keys)); + } + + var changed = updates.Count > 0; + if (cfg.AssetStorageMode == "local") + { + foreach (var key in selected.Keys.ToArray()) + { + var asset = selected[key]; + var path = await storage.StoreAsync(item.ItemId, asset.SourceUrl, true, ct).ConfigureAwait(false); + selected[key] = asset with { Path = path! }; + changed |= path != asset.Path; + } + } + var result = selected.Values.ToArray(); + if (changed || !previous.OrderBy(a => a.Language).ThenBy(a => a.Kind).SequenceEqual(result.OrderBy(a => a.Language).ThenBy(a => a.Kind))) + store.SaveArtwork(item.ItemId, result, updates); + return changed; + } + + private static string[] ArtworkKinds(string provider) + => provider == "tmdb" ? [PosterKind, LogoKind, BackdropKind] : [PosterKind, LogoKind, BannerKind, ThumbKind]; + + private static bool ArtworkMissing(IEnumerable assets, IEnumerable languages, string[] providers, string provider) + { + var available = assets.Where(asset => Array.IndexOf(providers, asset.Provider) is var rank && rank >= 0 && rank <= Array.IndexOf(providers, provider)) + .Select(asset => (asset.Language, asset.Kind)).ToHashSet(); + return languages.Any(lang => ArtworkKinds(provider).Any(kind => !available.Contains((lang, kind)))); + } + + private static string[] Providers(RefreshItemInfo item, PluginConfiguration cfg) + => item.Kind == "tvepisode" ? [] : cfg.Providers.Where(p => p.ArtworkOrder >= 0) + .OrderBy(p => p.ArtworkOrder).Select(p => p.Id.Trim().ToLowerInvariant()) + .Where(p => p == "tmdb" || (p == "fanart" && item.Kind == "movie" && !string.IsNullOrWhiteSpace(cfg.FanartApiKey))).ToArray(); + + private static string ArtworkScope(RefreshItemInfo item, string[] languages, string original, string[] providers) + => Scope(item) + "|" + string.Join(",", languages.Order(StringComparer.OrdinalIgnoreCase)) + "|" + original + "|" + string.Join(",", providers); + private static long MissingCutoff(PluginConfiguration cfg) + => TranslationStore.NowUnixUtc() - (long)TimeSpan.FromDays(Math.Max(0, cfg.WaitDaysForMissingData)).TotalSeconds; + private static string[] Languages(PluginConfiguration cfg) + => cfg.Languages.Select(lang => lang.Trim()).Where(lang => lang.Length > 0).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); + private static string BaseLanguage(string lang) => lang.Split('-')[0].ToLowerInvariant(); + private static bool SameLanguage(string left, string right) => BaseLanguage(left) == BaseLanguage(right); +} diff --git a/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshCoordinator.cs b/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshCoordinator.cs index 24261fa..9569d4e 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshCoordinator.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshCoordinator.cs @@ -3,7 +3,7 @@ using System.Diagnostics; namespace Jellyfin.Plugin.Multilang.Services.Refresh; -public sealed class RefreshCoordinator +public sealed class RefreshCoordinator : IDisposable { private sealed class QueueEntry { @@ -20,22 +20,23 @@ public sealed class RefreshCoordinator public required TaskCompletionSource Completion { get; init; } public long Sequence { get; init; } + public bool Active { get; set; } } private readonly object _lock = new(); private readonly Dictionary _entries = new(StringComparer.OrdinalIgnoreCase); private readonly Queue _recent = new(); private readonly SemaphoreSlim _signal = new(0, int.MaxValue); - private readonly ItemsProxyCache _itemsProxyCache; private long _sequence; - private QueueEntry? _active; + private readonly CancellationTokenSource _stop = new(); public SemaphoreSlim ScanMutex { get; } = new(1, 1); - public RefreshCoordinator(ItemsProxyCache itemsProxyCache) + public RefreshCoordinator(int concurrency = MultilangConstants.RefreshConcurrency) { - _itemsProxyCache = itemsProxyCache; - _ = Task.Run(WorkerLoop); + ArgumentOutOfRangeException.ThrowIfLessThan(concurrency, 1); + for (var i = 0; i < concurrency; i++) + _ = Task.Run(WorkerLoop); } public Task EnqueueAsync( @@ -53,6 +54,7 @@ public sealed class RefreshCoordinator lock (_lock) { + ObjectDisposedException.ThrowIf(_stop.IsCancellationRequested, this); if (_entries.TryGetValue(itemId, out var existing)) { if (jobType > existing.JobType) @@ -64,7 +66,7 @@ public sealed class RefreshCoordinator if (sourceTier > existing.SourceTier) existing.SourceTier = sourceTier; - return existing.Completion.Task; + return cancellationToken.CanBeCanceled ? existing.Completion.Task.WaitAsync(cancellationToken) : existing.Completion.Task; } var entry = new QueueEntry @@ -80,7 +82,7 @@ public sealed class RefreshCoordinator _entries.Add(itemId, entry); _signal.Release(); - return entry.Completion.Task; + return cancellationToken.CanBeCanceled ? entry.Completion.Task.WaitAsync(cancellationToken) : entry.Completion.Task; } } @@ -88,7 +90,8 @@ public sealed class RefreshCoordinator { while (true) { - await _signal.WaitAsync().ConfigureAwait(false); + try { await _signal.WaitAsync(_stop.Token).ConfigureAwait(false); } + catch (OperationCanceledException) { return; } QueueEntry? entry; lock (_lock) @@ -102,12 +105,24 @@ public sealed class RefreshCoordinator var error = string.Empty; try { - lock (_lock) - _active = entry; - - ok = await entry.Work(entry.JobType, CancellationToken.None).ConfigureAwait(false); - if (ok) - _itemsProxyCache.InvalidateItems([entry.ItemId]); + while (true) + { + RefreshJobType executing; + Func> work; + lock (_lock) + { + executing = entry.JobType; + work = entry.Work; + } + ok |= await work(executing, _stop.Token).ConfigureAwait(false); + lock (_lock) + { + if (entry.JobType > executing) + continue; + _entries.Remove(entry.ItemId); + break; + } + } entry.Completion.TrySetResult(ok); } catch (Exception ex) @@ -120,9 +135,8 @@ public sealed class RefreshCoordinator sw.Stop(); lock (_lock) { - if (ReferenceEquals(_active, entry)) - _active = null; - _entries.Remove(entry.ItemId); + if (_entries.TryGetValue(entry.ItemId, out var current) && ReferenceEquals(current, entry)) + _entries.Remove(entry.ItemId); _recent.Enqueue(new RefreshRecentItem( entry.ItemId, entry.SourceTier.ToString(), @@ -144,11 +158,14 @@ public sealed class RefreshCoordinator if (_entries.Count == 0) return null; - return _entries.Values + var entry = _entries.Values + .Where(e => !e.Active) .OrderByDescending(e => e.SourceTier) .ThenBy(e => e.WorkClass) .ThenBy(e => e.Sequence) - .First(); + .FirstOrDefault(); + if (entry is not null) entry.Active = true; + return entry; } public RefreshQueueDiagnostics GetDiagnostics() @@ -156,7 +173,7 @@ public sealed class RefreshCoordinator lock (_lock) { var queued = _entries.Values - .Where(e => !ReferenceEquals(e, _active)) + .Where(e => !e.Active) .OrderByDescending(e => e.SourceTier) .ThenBy(e => e.WorkClass) .ThenBy(e => e.Sequence) @@ -166,7 +183,7 @@ public sealed class RefreshCoordinator return new RefreshQueueDiagnostics( queued.Length, queued, - _active is null ? null : ToDiagnostics(_active), + _entries.Values.Where(e => e.Active).Select(ToDiagnostics).ToArray(), _recent.Reverse().ToArray()); } } @@ -178,4 +195,15 @@ public sealed class RefreshCoordinator entry.WorkClass.ToString(), entry.JobType.ToString(), entry.Sequence); + + public void Dispose() + { + _stop.Cancel(); + lock (_lock) + { + foreach (var entry in _entries.Values) + entry.Completion.TrySetCanceled(); + _entries.Clear(); + } + } } diff --git a/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshModels.cs b/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshModels.cs index 7c76f97..115f2a6 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshModels.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshModels.cs @@ -53,7 +53,7 @@ public sealed record RefreshRecentItem( public sealed record RefreshQueueDiagnostics( int QueuedCount, RefreshQueueEntry[] Queued, - RefreshQueueEntry? Active, + RefreshQueueEntry[] ActiveItems, RefreshRecentItem[] Recent); public sealed record RefreshScanDiagnostics( diff --git a/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshService.cs b/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshService.cs index 3eed2e1..364b8cc 100644 --- a/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshService.cs +++ b/src/Jellyfin.Plugin.Multilang/Services/Refresh/RefreshService.cs @@ -2,7 +2,6 @@ using System.Diagnostics; using Jellyfin.Data.Enums; using Jellyfin.Plugin.Multilang.Configuration; using Jellyfin.Plugin.Multilang.Data; -using Jellyfin.Plugin.Multilang.Services.Assets; using Jellyfin.Plugin.Multilang.Services.Providers; using static Jellyfin.Plugin.Multilang.MultilangConstants; using MediaBrowser.Controller.Entities; @@ -16,9 +15,7 @@ public sealed class RefreshService { private readonly ILibraryManager _libraryManager; private readonly TranslationStore _store; - private readonly TmdbClient _tmdbClient; - private readonly FanartClient _fanartClient; - private readonly AssetStorageService _assetStorage; + private readonly ItemFetcher _fetcher; private readonly RefreshCoordinator _coordinator; private readonly ILogger _logger; private readonly object _diagnosticsLock = new(); @@ -27,17 +24,13 @@ public sealed class RefreshService public RefreshService( ILibraryManager libraryManager, TranslationStore store, - TmdbClient tmdbClient, - FanartClient fanartClient, - AssetStorageService assetStorage, + ItemFetcher fetcher, RefreshCoordinator coordinator, ILogger logger) { _libraryManager = libraryManager; _store = store; - _tmdbClient = tmdbClient; - _fanartClient = fanartClient; - _assetStorage = assetStorage; + _fetcher = fetcher; _coordinator = coordinator; _logger = logger; } @@ -151,7 +144,6 @@ public sealed class RefreshService UpdateRunningScan(); var items = GetAllItemsWithTmdbId(); var now = TranslationStore.NowUnixUtc(); - var missingCutoff = now - (long)TimeSpan.FromDays(Math.Max(0, cfg.WaitDaysForMissingData)).TotalSeconds; var fullCutoff = now - (long)TimeSpan.FromDays(Math.Max(0, cfg.WaitDaysForExistingData)).TotalSeconds; var ordered = items.OrderBy(i => i.WorkClass).ToList(); itemCount = ordered.Count; @@ -162,7 +154,14 @@ public sealed class RefreshService // Run local housekeeping only from full-library scans. ItemsProxy browsing may enqueue // missing items, but it must not delete local assets just because the admin briefly // changed artwork storage mode. - var localCleanup = _store.CleanupForConfiguration(liveIds, NormalizeLanguages(cfg.Languages), IsLocalAssetStorage(cfg)); + LocalCleanupResult localCleanup; + using (await _store.EnterMaintenanceAsync(cancellationToken).ConfigureAwait(false)) + { + cfg = Plugin.Instance?.Configuration ?? throw new InvalidOperationException("Plugin unavailable."); + if (Plugin.Instance.Uninstalling) + throw new OperationCanceledException("Plugin is being uninstalled."); + localCleanup = _store.CleanupForConfiguration(liveIds, NormalizeLanguages(cfg.Languages), IsLocalAssetStorage(cfg)); + } if (localCleanup.TranslationsDeleted > 0 || localCleanup.AssetsDeleted > 0 || localCleanup.GenresDeleted > 0 || localCleanup.AssetFilesDeleted > 0) { _logger.LogInformation( @@ -176,72 +175,90 @@ public sealed class RefreshService _logger.LogInformation("Multilang refresh started mode={Mode} source={Source} items={Items} new={New} deleted={Deleted}", jobType, runSource, ordered.Count, newCount, deletedCount); var index = 0; UpdateRunningScan(); - foreach (var item in ordered) - { - cancellationToken.ThrowIfCancellationRequested(); - index++; - SetCurrent(item, index, "checking"); - var status = _store.GetFactsStatus(item.ItemId); - var isNew = !status.Exists; - var missingConfiguredData = jobType == RefreshJobType.Missing && status.Exists && ItemNeedsMissingRefresh(item, cfg, langs: null); - var due = jobType == RefreshJobType.Full - ? !status.Exists || status.FullCheckedAt < fullCutoff - : !status.Exists || status.MissingCheckedAt < missingCutoff || missingConfiguredData; - - if (!due) + var completed = 0; + var scanLock = new object(); + foreach (var group in ordered.GroupBy(item => item.WorkClass)) + await Parallel.ForEachAsync(group, + new ParallelOptions { MaxDegreeOfParallelism = RefreshConcurrency, CancellationToken = cancellationToken }, + async (item, ct) => { - skippedNotDueCount++; - if (cfg.VerboseLogging && cfg.EnableLogging) + ct.ThrowIfCancellationRequested(); + var scanIndex = Interlocked.Increment(ref index); + var status = _store.GetFactsStatus(item.ItemId); + var isNew = !status.Exists; + var fullDue = jobType == RefreshJobType.Full && (!status.Exists || status.FullCheckedAt < fullCutoff); + var due = fullDue || (jobType == RefreshJobType.Missing + ? !status.Exists || _fetcher.NeedsRefresh(item, cfg) + : _fetcher.NeedsLocalWork(item.ItemId, cfg)); + + if (!due) { - _logger.LogInformation( - "Multilang refresh skip [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind} reason=not-due", - index, - ordered.Count, - item.ItemId, - item.DisplayName, - item.Kind); + Interlocked.Increment(ref skippedNotDueCount); + if (cfg.VerboseLogging && cfg.EnableLogging) + { + _logger.LogInformation( + "Multilang refresh skip [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind} reason=not-due", + scanIndex, + ordered.Count, + item.ItemId, + item.DisplayName, + item.Kind); + } + + lock (scanLock) + { + SetCurrent(item, completed + 1, "skipped-not-due"); + progress.Report(++completed * 100.0 / Math.Max(1, ordered.Count)); + UpdateRunningScan(); + } + return; } - progress.Report(index * 100.0 / Math.Max(1, ordered.Count)); - UpdateRunningScan("skipped-not-due"); - continue; - } + var actualJobType = fullDue || !status.Exists ? RefreshJobType.Full + : jobType == RefreshJobType.Full ? RefreshJobType.Aggregate : RefreshJobType.Missing; + Interlocked.Increment(ref dueCount); + Log( + cfg.EnableLogging, + LogLevel.Information, + "Multilang refresh fetch [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind} tmdb={TmdbId} mode={Mode} new={New}", + scanIndex, + ordered.Count, + item.ItemId, + item.DisplayName, + item.Kind, + item.TmdbId, + actualJobType, + isNew); - var scanIndex = index; - var actualJobType = jobType == RefreshJobType.Full || !status.Exists ? RefreshJobType.Full : RefreshJobType.Missing; - dueCount++; - SetCurrent(item, scanIndex, "fetching"); - Log( - cfg.EnableLogging, - LogLevel.Information, - "Multilang refresh fetch [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind} tmdb={TmdbId} mode={Mode} new={New}", - scanIndex, - ordered.Count, - item.ItemId, - item.DisplayName, - item.Kind, - item.TmdbId, - actualJobType, - isNew); + var ok = await _coordinator.EnqueueAsync( + item.ItemId, + RefreshSourceTier.Background, + item.WorkClass, + actualJobType, + async (actualJobType, ct) => await RefreshItemCoreAsync(item, actualJobType, ct).ConfigureAwait(false), + ct).ConfigureAwait(false); + lock (scanLock) + { + if (ok) + refreshedCount++; + else + skippedNoDataCount++; - var ok = await _coordinator.EnqueueAsync( - item.ItemId, - RefreshSourceTier.Background, - item.WorkClass, - actualJobType, - async (actualJobType, ct) => await RefreshItemCoreAsync(item, cfg, actualJobType, scanIndex, ordered.Count, ct).ConfigureAwait(false), - cancellationToken).ConfigureAwait(false); - if (ok) - refreshedCount++; - else - skippedNoDataCount++; + progress.Report(++completed * 100.0 / Math.Max(1, ordered.Count)); + SetCurrent(item, completed, ok ? "refreshed" : "skipped-no-data"); + UpdateRunningScan(); + } + }).ConfigureAwait(false); - progress.Report(index * 100.0 / Math.Max(1, ordered.Count)); - UpdateRunningScan(ok ? "refreshed" : "skipped-no-data"); + using (await _store.EnterMaintenanceAsync(cancellationToken).ConfigureAwait(false)) + { + if (Plugin.Instance!.Uninstalling) + throw new OperationCanceledException("Plugin is being uninstalled."); + _store.CleanupUnreferencedAssetFiles(); + _store.SetLastScanStarted(now); } sw.Stop(); - _store.SetLastScanStarted(now); CompleteScan(true, string.Empty); _logger.LogInformation( "Multilang refresh complete mode={Mode} source={Source} elapsed={ElapsedMs}ms items={Items} due={Due} refreshed={Refreshed} skippedNotDue={SkippedNotDue} skippedNoData={SkippedNoData} new={New} deleted={Deleted}", @@ -268,7 +285,8 @@ public sealed class RefreshService } } - public async Task RefreshItemAsync(Guid itemId, bool includeChildren, CancellationToken cancellationToken) + public async Task RefreshItemAsync(Guid itemId, bool includeChildren, CancellationToken cancellationToken, + Jellyfin.Database.Implementations.Entities.User user) { var cfg = Plugin.Instance?.Configuration ?? throw new InvalidOperationException("Plugin configuration unavailable."); ValidateRefreshConfiguration(cfg); @@ -279,43 +297,46 @@ public sealed class RefreshService return; } - var items = BuildItemInfos(GetItemAndChildren(root, includeChildren)).ToList(); + var items = BuildItemInfos(GetItemAndChildren(root, includeChildren) + .Where(item => item.IsVisibleStandalone(user))).ToList(); var total = items.Count; var index = 0; _logger.LogInformation("Manual Multilang refresh started item={ItemId} name={Name} items={Items} includeChildren={IncludeChildren}", itemId, root.Name ?? string.Empty, total, includeChildren); - foreach (var item in items) - { - index++; - var scanIndex = index; - Log( - cfg.EnableLogging, - LogLevel.Information, - "Manual Multilang refresh fetch [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind} tmdb={TmdbId}", - scanIndex, - total, - item.ItemId, - item.DisplayName, - item.Kind, - item.TmdbId); + foreach (var group in items.OrderBy(item => item.WorkClass).GroupBy(item => item.WorkClass)) + await Parallel.ForEachAsync(group, + new ParallelOptions { MaxDegreeOfParallelism = RefreshConcurrency, CancellationToken = cancellationToken }, + async (item, ct) => + { + var scanIndex = Interlocked.Increment(ref index); + Log( + cfg.EnableLogging, + LogLevel.Information, + "Manual Multilang refresh fetch [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind} tmdb={TmdbId}", + scanIndex, + total, + item.ItemId, + item.DisplayName, + item.Kind, + item.TmdbId); - var ok = await _coordinator.EnqueueAsync( - item.ItemId, - RefreshSourceTier.Manual, - item.WorkClass, - RefreshJobType.Full, - async (actualJobType, ct) => await RefreshItemCoreAsync(item, cfg, actualJobType, scanIndex, total, ct).ConfigureAwait(false), - cancellationToken).ConfigureAwait(false); - Log( - cfg.EnableLogging, - ok ? LogLevel.Information : LogLevel.Warning, - "Manual Multilang refresh item {Status} [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind}", - ok ? "complete" : "skipped", - scanIndex, - total, - item.ItemId, - item.DisplayName, - item.Kind); - } + var ok = await _coordinator.EnqueueAsync( + item.ItemId, + RefreshSourceTier.Manual, + item.WorkClass, + RefreshJobType.Full, + async (actualJobType, ct) => await RefreshItemCoreAsync(item, actualJobType, ct).ConfigureAwait(false), + ct).ConfigureAwait(false); + Log( + cfg.EnableLogging, + ok ? LogLevel.Information : LogLevel.Warning, + "Manual Multilang refresh item {Status} [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind}", + ok ? "complete" : "skipped", + scanIndex, + total, + item.ItemId, + item.DisplayName, + item.Kind); + }).ConfigureAwait(false); } public bool EnqueueOnTheFlyMissing(IEnumerable itemIds) @@ -365,7 +386,7 @@ public sealed class RefreshService sourceTier, info.WorkClass, RefreshJobType.Full, - async (actualJobType, ct) => await RefreshItemCoreAsync(info, cfg, actualJobType, 0, 0, ct).ConfigureAwait(false), + async (actualJobType, ct) => await RefreshItemCoreAsync(info, actualJobType, ct).ConfigureAwait(false), CancellationToken.None) .ContinueWith( task => @@ -388,245 +409,16 @@ public sealed class RefreshService private async Task RefreshItemCoreAsync( RefreshItemInfo item, - PluginConfiguration cfg, RefreshJobType jobType, - int scanIndex, - int totalItems, CancellationToken cancellationToken) { - var langs = NormalizeLanguages(cfg.Languages); - if (langs.Length == 0) - throw new InvalidOperationException("No Multilang languages are configured."); - - if (cfg.VerboseLogging && cfg.EnableLogging) - _logger.LogInformation("[{Index}/{Total}] Fetching item={ItemId} name=\"{Name}\" kind={Kind} tmdb={TmdbId}", scanIndex, totalItems, item.ItemId, item.DisplayName, item.Kind, item.TmdbId); - - var now = TranslationStore.NowUnixUtc(); - var limiter = new FetchRateLimiter(TimeSpan.FromMilliseconds(250)); - var wroteAny = false; - TmdbMetadata? firstMeta = null; - var fetchedMetadata = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (var lang in langs) - { - var meta = await _tmdbClient.FetchMetadataAsync(item, lang, cfg.TmdbApiKey, limiter, cancellationToken).ConfigureAwait(false); - if (meta is null) - { - Log(cfg.EnableLogging, LogLevel.Information, "[{Index}/{Total}] TMDb 404, skipping item={ItemId} name=\"{Name}\" tmdb={TmdbId} kind={Kind}", scanIndex, totalItems, item.ItemId, item.DisplayName, item.TmdbId, item.Kind); - _store.MarkFactsChecked(item, now, jobType == RefreshJobType.Full ? now : 0); - return false; - } - - Log(cfg.VerboseLogging && cfg.EnableLogging, LogLevel.Information, "[{Index}/{Total}] TMDb metadata ok item={ItemId} lang={Lang}", scanIndex, totalItems, item.ItemId, lang); - _store.UpsertTranslation(item.ItemId, lang, TitleField, meta.Title); - _store.UpsertTranslation(item.ItemId, lang, OverviewField, meta.Overview); - _store.UpsertTranslation(item.ItemId, lang, TaglineField, meta.Tagline); - StoreGenres(item, lang, meta); - fetchedMetadata[lang] = meta; - firstMeta ??= meta; - if (!wroteAny) - { - _store.UpsertFacts(item, meta, now, jobType == RefreshJobType.Full ? now : 0); - wroteAny = true; - } - } - - var artworkLanguages = langs; - if (firstMeta is not null && !string.IsNullOrWhiteSpace(firstMeta.OriginalLanguage)) - { - var originalLang = firstMeta.OriginalLanguage.Trim(); - var existingLang = langs.FirstOrDefault(lang => SameLanguageBase(lang, originalLang)); - Log( - cfg.EnableLogging, - LogLevel.Information, - "[{Index}/{Total}] Original language item={ItemId} lang={OriginalLang} metadataSource={Source}", - scanIndex, - totalItems, - item.ItemId, - originalLang, - existingLang is not null ? "configured-language:" + existingLang : "extra-fetch"); - var originalMeta = existingLang is not null - ? fetchedMetadata[existingLang] - : await _tmdbClient.FetchMetadataAsync(item, originalLang, cfg.TmdbApiKey, limiter, cancellationToken).ConfigureAwait(false); - - if (originalMeta is not null) - { - _store.UpsertTranslation(item.ItemId, OriginalAction, TitleField, originalMeta.Title); - _store.UpsertTranslation(item.ItemId, OriginalAction, OverviewField, originalMeta.Overview); - _store.UpsertTranslation(item.ItemId, OriginalAction, TaglineField, originalMeta.Tagline); - StoreGenres(item, OriginalAction, originalMeta); - Log( - cfg.VerboseLogging && cfg.EnableLogging, - LogLevel.Information, - "[{Index}/{Total}] Original metadata stored item={ItemId} lang={OriginalLang} titlePresent={TitlePresent} overviewPresent={OverviewPresent} taglinePresent={TaglinePresent}", - scanIndex, - totalItems, - item.ItemId, - originalLang, - !string.IsNullOrWhiteSpace(originalMeta.Title), - !string.IsNullOrWhiteSpace(originalMeta.Overview), - !string.IsNullOrWhiteSpace(originalMeta.Tagline)); - } - else - { - Log( - cfg.EnableLogging, - LogLevel.Warning, - "[{Index}/{Total}] Original metadata unavailable item={ItemId} lang={OriginalLang}", - scanIndex, - totalItems, - item.ItemId, - originalLang); - } - - artworkLanguages = AppendOriginalLanguage(langs, originalLang); - Log( - cfg.VerboseLogging && cfg.EnableLogging, - LogLevel.Information, - "[{Index}/{Total}] Artwork language set item={ItemId} langs={Languages}", - scanIndex, - totalItems, - item.ItemId, - string.Join(",", artworkLanguages)); - } - - await StoreArtworkAsync(item, cfg, artworkLanguages, limiter, now, cancellationToken).ConfigureAwait(false); - - Log(cfg.EnableLogging, LogLevel.Information, "Multilang refresh item complete item={ItemId} kind={Kind} tmdb={TmdbId} mode={Mode}", item.ItemId, item.Kind, item.TmdbId, jobType); - return true; - } - - private void StoreGenres(RefreshItemInfo item, string lang, TmdbMetadata metadata) - { - var media = GenreMediaFor(item.Kind); - if (media is null) - return; - - foreach (var genre in metadata.Genres) - _store.UpsertGenre(genre.Id, media, lang, genre.Name); - } - - private static string? GenreMediaFor(string kind) - => kind.Equals("movie", StringComparison.OrdinalIgnoreCase) || - kind.Equals("collection", StringComparison.OrdinalIgnoreCase) - ? "movie" - : kind.Equals("tv", StringComparison.OrdinalIgnoreCase) || - kind.Equals("tvseason", StringComparison.OrdinalIgnoreCase) || - kind.Equals("tvepisode", StringComparison.OrdinalIgnoreCase) - ? "tv" - : null; - - private async Task StoreArtworkAsync( - RefreshItemInfo item, - PluginConfiguration cfg, - string[] languages, - FetchRateLimiter limiter, - long updatedAt, - CancellationToken cancellationToken) - { - if (item.Kind is "tvepisode") - return; - - var providers = cfg.Providers - .Where(p => p.ArtworkOrder >= 0) - .OrderBy(p => p.ArtworkOrder) - .Select(p => p.Id.Trim().ToLowerInvariant()) - .ToArray(); - if (providers.Length == 0) - return; - - TmdbImages? tmdbImages = null; - FanartImages? fanartImages = null; - var written = new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (var provider in providers) - { - cancellationToken.ThrowIfCancellationRequested(); - - if (provider.Equals("tmdb", StringComparison.OrdinalIgnoreCase)) - { - tmdbImages ??= await _tmdbClient.FetchImagesAsync(item, cfg.TmdbApiKey, limiter, cancellationToken).ConfigureAwait(false); - if (tmdbImages is null) - continue; - - await StoreImageMapAsync(item.ItemId, languages, PosterKind, tmdbImages.Posters, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false); - await StoreImageMapAsync(item.ItemId, languages, LogoKind, tmdbImages.Logos, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false); - await StoreImageMapAsync(item.ItemId, languages, BackdropKind, tmdbImages.Backdrops, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false); - Log(cfg.VerboseLogging && cfg.EnableLogging, LogLevel.Information, "TMDb artwork stored item={ItemId} langs={Languages}", item.ItemId, string.Join(",", languages)); - continue; - } - - if (provider.Equals("fanart", StringComparison.OrdinalIgnoreCase) && item.Kind.Equals("movie", StringComparison.OrdinalIgnoreCase)) - { - fanartImages ??= await _fanartClient.FetchMovieImagesAsync(item.TmdbId, cfg.FanartApiKey, cancellationToken).ConfigureAwait(false); - if (fanartImages is null) - continue; - - await StoreImageMapAsync(item.ItemId, languages, PosterKind, fanartImages.Posters, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false); - await StoreImageMapAsync(item.ItemId, languages, LogoKind, fanartImages.Logos, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false); - await StoreImageMapAsync(item.ItemId, languages, BannerKind, fanartImages.Banners, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false); - await StoreImageMapAsync(item.ItemId, languages, ThumbKind, fanartImages.Thumbs, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false); - Log(cfg.VerboseLogging && cfg.EnableLogging, LogLevel.Information, "Fanart artwork stored item={ItemId} langs={Languages}", item.ItemId, string.Join(",", languages)); - } - } - } - - private async Task StoreImageMapAsync( - string itemId, - IEnumerable languages, - string kind, - IReadOnlyDictionary images, - PluginConfiguration cfg, - long updatedAt, - HashSet written, - CancellationToken cancellationToken) - { - foreach (var lang in languages) - { - var storeLang = lang.StartsWith(OriginalAction + ":", StringComparison.OrdinalIgnoreCase) - ? OriginalAction - : lang; - var key = kind + ":" + storeLang; - if (written.Contains(key)) - continue; - - var iso = ToIso639(lang); - if (!images.TryGetValue(iso, out var url) || string.IsNullOrWhiteSpace(url)) - continue; - - var storedPath = await _assetStorage.StoreAsync(itemId, storeLang, kind, url, IsLocalAssetStorage(cfg), cancellationToken).ConfigureAwait(false); - if (string.IsNullOrWhiteSpace(storedPath)) - continue; - - _store.UpsertAsset(itemId, storeLang, kind, storedPath, updatedAt); - written.Add(key); - } - } - - private bool ItemNeedsMissingRefresh(RefreshItemInfo item, PluginConfiguration cfg, string[]? langs) - { - langs ??= NormalizeLanguages(cfg.Languages); - if (_store.HasMissingConfiguredTranslations(item.ItemId, langs)) - return true; - - var assetPresence = _store.GetAssetPresence(item.ItemId, langs.Append(OriginalAction)); - return IsLocalAssetStorage(cfg) && assetPresence.AnyRemote; - } - - private static string[] AppendOriginalLanguage(string[] languages, string originalLanguage) - => languages - .Concat([OriginalAction + ":" + originalLanguage]) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); - - private static bool SameLanguageBase(string left, string right) - => ToIso639(left).Equals(ToIso639(right), StringComparison.OrdinalIgnoreCase); - - private static string ToIso639(string language) - { - if (language.StartsWith(OriginalAction + ":", StringComparison.OrdinalIgnoreCase)) - language = language[(OriginalAction.Length + 1)..]; - var index = language.IndexOf('-', StringComparison.Ordinal); - return (index > 0 ? language[..index] : language).Trim().ToLowerInvariant(); + using var lease = await _store.EnterRefreshAsync(cancellationToken).ConfigureAwait(false); + var plugin = Plugin.Instance ?? throw new InvalidOperationException("Plugin unavailable."); + if (plugin.Uninstalling) + throw new OperationCanceledException("Plugin is being uninstalled."); + var cfg = plugin.Configuration; + ValidateRefreshConfiguration(cfg); + return await _fetcher.RefreshAsync(item, cfg, jobType, cancellationToken).ConfigureAwait(false); } private List GetAllItemsWithTmdbId() diff --git a/src/Jellyfin.Plugin.Multilang/wwwroot/inject.js b/src/Jellyfin.Plugin.Multilang/wwwroot/inject.js index d8b6479..7a9cde1 100644 --- a/src/Jellyfin.Plugin.Multilang/wwwroot/inject.js +++ b/src/Jellyfin.Plugin.Multilang/wwwroot/inject.js @@ -1,16 +1,8 @@ (() => { - if (!Element.prototype.replaceChildren) { - Element.prototype.replaceChildren = function () { - this.textContent = ""; - for (let index = 0; index < arguments.length; index++) this.appendChild(arguments[index]); - }; - } - const BASE_PATH = window.location.pathname.split("/web/")[0] || ""; const MENU_ID = "mlUserRulesMenuItem"; const USER_RULES_ROUTE = "multilang/user-rules"; let lastMenuItemId = ""; - let userRulesRenderRun = 0; function getClientToken() { try { @@ -227,6 +219,7 @@ function closeUserRulesShellIfInactive() { if (isUserRulesRoute()) return; + document.querySelector("#ml-user-shell #ml-user")?._mlDispose?.(); document.getElementById("ml-user-shell")?.remove(); } @@ -234,7 +227,6 @@ event?.preventDefault?.(); event?.stopPropagation?.(); event?.stopImmediatePropagation?.(); - document.querySelector("#app-user-menu .MuiBackdrop-root")?.click(); history.pushState({}, "", userRulesHref()); scheduleUserRulesPageRender(); } @@ -250,32 +242,25 @@ const next = document.createElement("script"); [...script.attributes].forEach((attr) => next.setAttribute(attr.name, attr.value)); if (script.src) { - next.onload = () => resolve(); - next.onerror = () => reject(new Error(`Failed to load ${script.src}`)); + next.onload = () => { next.remove(); resolve(); }; + next.onerror = () => { next.remove(); reject(new Error(`Failed to load ${script.src}`)); }; next.src = script.src; } else { next.text = script.textContent || ""; } document.body.appendChild(next); - if (!script.src) resolve(); + if (!script.src) { next.remove(); resolve(); } }); } - async function renderUserRulesPage(run) { - if (!isUserRulesRoute() || run !== userRulesRenderRun) return; - - const host = userRulesHost(); - if (!host) return; - if (host.dataset.mlUserRulesRendered === "1" && document.getElementById("ml-user")) return; - - host.dataset.mlUserRulesRendered = "1"; + async function renderUserRulesPage(host) { host.innerHTML = "
Loading Multilang settings...
"; const response = await fetch(`${BASE_PATH}/web/configurationpage?name=MultilangUser`, { cache: "no-store" }); if (!response.ok) throw new Error(`HTTP ${response.status}`); const fragment = await response.text(); - if (run !== userRulesRenderRun || !isUserRulesRoute()) return; + if (!host.isConnected || !isUserRulesRoute()) return; const parsed = new DOMParser().parseFromString(fragment, "text/html"); const sourcePage = parsed.getElementById("ml-user"); @@ -289,26 +274,26 @@ page.id = "ml-user"; page.className = "ml-root"; while (sourceContent.firstChild) page.appendChild(sourceContent.firstChild); - host.replaceChildren(page); + host.textContent = ""; + host.appendChild(page); for (const script of scripts) { - if (run !== userRulesRenderRun || !isUserRulesRoute()) return; + if (!host.isConnected || !isUserRulesRoute()) return; await executeScript(script); } } function scheduleUserRulesPageRender() { - userRulesRenderRun++; if (!isUserRulesRoute()) { closeUserRulesShellIfInactive(); return; } - const run = userRulesRenderRun; - [0, 100, 350].forEach((delay) => { - setTimeout(() => { - renderUserRulesPage(run).catch((err) => console.error("[Multilang] failed to render user rules page", err)); - }, delay); + const host = userRulesHost(); + if (!host || host._mlLoad) return; + host._mlLoad = renderUserRulesPage(host).catch((err) => { + console.error("[Multilang] failed to render user rules page", err); + if (host.isConnected) host.textContent = `Unable to load Multilang settings: ${err.message}`; }); } @@ -392,6 +377,11 @@ if (!url.startsWith("/")) url = "/" + url; if (BASE_PATH && !url.toLowerCase().startsWith(BASE_PATH.toLowerCase() + "/")) url = BASE_PATH + url; } + const local = new URL(url, window.location.origin); + if (local.origin === window.location.origin && local.pathname.startsWith(BASE_PATH + "/Multilang/Assets/")) { + local.searchParams.set("api_key", getClientToken()); + return local.href; + } return url; } diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/AssetStorageServiceTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/AssetStorageServiceTests.cs index fa487a6..8691957 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/AssetStorageServiceTests.cs +++ b/tests/Jellyfin.Plugin.Multilang.Tests/AssetStorageServiceTests.cs @@ -19,7 +19,7 @@ public sealed class AssetStorageServiceTests }); var service = new AssetStorageService(factory, store); - var path = await service.StoreAsync("item", "fi", "poster", "https://images.example/poster.jpg", true, CancellationToken.None); + var path = await service.StoreAsync("item", "https://images.example/poster.jpg", true, CancellationToken.None); Assert.NotNull(path); Assert.StartsWith(TranslationStore.LocalAssetUrlPrefix, path, StringComparison.Ordinal); @@ -28,6 +28,19 @@ public sealed class AssetStorageServiceTests Assert.True(service.TryResolveLocalAsset(path[TranslationStore.LocalAssetUrlPrefix.Length..], out var resolved, out var contentType)); Assert.Equal(fullPath, resolved); Assert.Equal("image/jpeg", contentType); + + store.UpsertAsset("item", "fi", "poster", path, 1); + var original = await service.StoreAsync("item", "https://images.example/poster.jpg", true, CancellationToken.None); + store.UpsertAsset("item", "Original", "poster", original!, 1); + Assert.Equal(path, original); + Assert.Single(factory.Requests); + Assert.True(store.IsAssetReferenced("item", path)); + Assert.True(File.Exists(fullPath)); + File.Delete(fullPath); + Assert.False(File.Exists(fullPath)); + await service.StoreAsync("item", "https://images.example/poster.jpg", true, CancellationToken.None); + Assert.True(File.Exists(fullPath)); + Assert.Equal(2, factory.Requests.Count); } finally { @@ -46,7 +59,7 @@ public sealed class AssetStorageServiceTests var service = new AssetStorageService(factory, store); const string url = "https://images.example/poster.webp"; - var result = await service.StoreAsync("item", "fi", "poster", url, false, CancellationToken.None); + var result = await service.StoreAsync("item", url, false, CancellationToken.None); Assert.Equal(url, result); Assert.Empty(factory.Requests); @@ -73,7 +86,7 @@ public sealed class AssetStorageServiceTests }); var service = new AssetStorageService(factory, store); - await Assert.ThrowsAsync(() => service.StoreAsync("item", "fi", "poster", "https://images.example/poster.jpg", true, CancellationToken.None)); + await Assert.ThrowsAsync(() => service.StoreAsync("item", "https://images.example/poster.jpg", true, CancellationToken.None)); Assert.Empty(Directory.EnumerateFiles(store.AssetsDirectory, "*", SearchOption.AllDirectories)); } diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/CategoryRuleEvaluatorTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/CategoryRuleEvaluatorTests.cs index 7032db7..110230b 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/CategoryRuleEvaluatorTests.cs +++ b/tests/Jellyfin.Plugin.Multilang.Tests/CategoryRuleEvaluatorTests.cs @@ -6,6 +6,24 @@ namespace Jellyfin.Plugin.Multilang.Tests; public sealed class CategoryRuleEvaluatorTests { + [Fact] + public void FastMatchingAndDiagnosticTraceAgreeAcrossAllRuleModes() + { + foreach (var relation in new[] { "is", "is_not", "contains", "not_contains" }) + foreach (var fieldOr in new[] { false, true }) + foreach (var valueOr in new[] { false, true }) + foreach (var matchAll in new[] { false, true }) + foreach (var facts in new[] { Facts(originalLanguage: "sv"), Facts(spokenLanguages: ["fi", "en"]), Facts(kind: "tv") }) + { + var rule = Rule("original_language", relation, valueOr, "sv", "fi"); + rule.Requirements[0].Fields = ["original_language", "spoken_languages"]; + rule.Requirements[0].UseFieldOr = fieldOr; + rule.Requirements = [rule.Requirements[0], Requirement("production_countries", "contains", false, "FI")]; + rule.MatchAllConditions = matchAll; + Assert.Equal(CategoryRuleEvaluator.Trace(rule, facts).Result, CategoryRuleEvaluator.Matches(rule, facts)); + } + } + [Fact] public void IsWithAndListRequiresExactSet() { @@ -95,11 +113,9 @@ public sealed class CategoryRuleEvaluatorTests { Id = rule.Id, Label = rule.Label, - CriteriaText = rule.CriteriaText, Requirements = rule.Requirements, MatchAllConditions = matchAll, Scopes = rule.Scopes, - FieldActions = rule.FieldActions, FieldActionLists = rule.FieldActionLists }; diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/ItemFetcherTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/ItemFetcherTests.cs new file mode 100644 index 0000000..d8f0356 --- /dev/null +++ b/tests/Jellyfin.Plugin.Multilang.Tests/ItemFetcherTests.cs @@ -0,0 +1,232 @@ +using System.Net; +using System.Text.Json; +using Jellyfin.Plugin.Multilang.Configuration; +using Jellyfin.Plugin.Multilang.Data; +using Jellyfin.Plugin.Multilang.Services.Assets; +using Jellyfin.Plugin.Multilang.Services.Providers; +using Jellyfin.Plugin.Multilang.Services.Refresh; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Jellyfin.Plugin.Multilang.Tests; + +public sealed class ItemFetcherTests : IDisposable +{ + private readonly string _root = TestPaths.CreateRoot(); + private readonly TranslationStore _store; + private readonly TestHttpClientFactory _http; + private readonly ItemFetcher _fetcher; + private readonly RefreshItemInfo _item = new("11111111111111111111111111111111", "1", "movie", RefreshWorkClass.Movies, 0, 0, null, "Movie"); + private readonly PluginConfiguration _config = new() + { + Languages = ["en", "fi"], + TmdbApiKey = "test", + Providers = [new() { Id = "tmdb", MetadataOrder = 0 }] + }; + private string _original = "en"; + private string _overview = "Overview"; + private bool _missingFinnishTagline; + private bool _notFound; + private bool _missingTmdbPoster; + private bool _fanartUnauthorized; + + public ItemFetcherTests() + { + _store = new(new TestApplicationPaths(_root)); + _http = new(request => + { + if (_notFound) return new(HttpStatusCode.NotFound); + var url = request.RequestUri!; + if (url.Host == "webservice.fanart.tv") + return _fanartUnauthorized ? new(HttpStatusCode.Unauthorized) + : Json(new { movieposter = new[] { new { lang = "en", url = "https://assets.fanart.tv/poster.jpg", likes = "5" } } }); + if (url.Host == "image.tmdb.org") + return new(HttpStatusCode.OK) { Content = new ByteArrayContent([1, 2, 3]) }; + if (url.AbsolutePath.EndsWith("/images")) + return Json(new { posters = _missingTmdbPoster ? [] : new[] { new { iso_639_1 = "en", file_path = "/poster.jpg" } }, logos = Array.Empty(), backdrops = Array.Empty() }); + var language = Microsoft.AspNetCore.WebUtilities.QueryHelpers.ParseQuery(url.Query)["language"].ToString(); + return Json(new + { + title = language + " title", + overview = _overview, + tagline = language == "fi" && _missingFinnishTagline ? "" : "Tagline", + original_language = _original, + original_title = "Original title", + genres = Array.Empty() + }); + }); + _fetcher = new(_store, new(_http), new(_http), new(_http, _store), NullLogger.Instance); + } + + private static HttpResponseMessage Json(object value) + => new(HttpStatusCode.OK) { Content = new StringContent(JsonSerializer.Serialize(value)) }; + private Task Fetch(RefreshJobType job = RefreshJobType.Missing) => _fetcher.RefreshAsync(_item, _config, job, CancellationToken.None); + private string Text(string language, string field) => _store.GetTranslations([_item.ItemId], [language])[_item.ItemId][language][field]; + + [Fact] + public async Task CompleteMetadataIsNotRefetchedAndOriginalReusesConfiguredLanguage() + { + Assert.True(await Fetch()); + Assert.Equal(2, _http.Requests.Count); + Assert.Equal(Text("en", "title"), Text("Original", "title")); + Assert.False(_fetcher.NeedsRefresh(_item, _config)); + Assert.False(await Fetch()); + Assert.Equal(2, _http.Requests.Count); + await Fetch(RefreshJobType.Full); + Assert.Equal(4, _http.Requests.Count); + } + + [Fact] + public async Task MissingRetryFetchesOnlyIncompleteLanguageAndUpdatesWholeResponse() + { + _missingFinnishTagline = true; + await Fetch(); + Assert.False(_fetcher.NeedsRefresh(_item, _config)); + await Fetch(); + Assert.Equal(2, _http.Requests.Count); + _config.WaitDaysForMissingData = 0; + Assert.True(_fetcher.NeedsRefresh(_item, _config)); + _missingFinnishTagline = false; + _overview = "Updated overview"; + await Fetch(); + Assert.Equal(3, _http.Requests.Count); + Assert.Contains("language=fi", _http.Requests.Last()); + Assert.Equal("Updated overview", Text("fi", "overview")); + Assert.Equal("Overview", Text("en", "overview")); + Assert.False(_fetcher.NeedsRefresh(_item, _config)); + } + + [Fact] + public async Task AdditionalOriginalLanguageIsFetchedOnceAndNewConfiguredLanguageDoesNotRefetchOthers() + { + _original = "sv"; + await Fetch(); + Assert.Equal(3, _http.Requests.Count); + Assert.Equal("sv title", Text("Original", "title")); + await Fetch(); + Assert.Equal(3, _http.Requests.Count); + _config.Languages = ["en", "fi", "fr"]; + await Fetch(); + Assert.Equal(4, _http.Requests.Count); + Assert.Contains("language=fr", _http.Requests.Last()); + } + + [Fact] + public async Task NotFoundIsRememberedUntilRetryCutoff() + { + _notFound = true; + Assert.False(await Fetch()); + Assert.False(_fetcher.NeedsRefresh(_item, _config)); + await Fetch(); + Assert.Equal(2, _http.Requests.Count); + _config.WaitDaysForMissingData = 0; + Assert.True(_fetcher.NeedsRefresh(_item, _config)); + await Fetch(); + Assert.Equal(4, _http.Requests.Count); + } + + [Fact] + public async Task LocalRepairDoesNotFetchMetadataAndUrlCleanupOnlyHappensInScheduledHousekeeping() + { + _config.Providers[0].ArtworkOrder = 0; + await Fetch(); + Assert.Equal(3, _http.Requests.Count); + _config.AssetStorageMode = "local"; + Assert.True(_fetcher.NeedsLocalWork(_item.ItemId, _config)); + await Fetch(RefreshJobType.Aggregate); + Assert.Equal(4, _http.Requests.Count); + Assert.Contains("image.tmdb.org", _http.Requests.Last()); + var path = _store.GetAssetPath(_item.ItemId, "en", "poster")!; + Assert.Equal(path, _store.GetAssetPath(_item.ItemId, "Original", "poster")); + Assert.True(_store.TryNormalizeLocalAssetPath(path, out var file)); + File.Delete(file); + await Fetch(); + Assert.Equal(5, _http.Requests.Count); + Assert.True(File.Exists(file)); + _config.AssetStorageMode = "url"; + await Fetch(); + Assert.Equal(path, _store.GetAssetPath(_item.ItemId, "en", "poster")); + Assert.True(File.Exists(file)); + _store.CleanupForConfiguration(new HashSet { _item.ItemId }, _config.Languages, false); + Assert.StartsWith("https://", _store.GetAssetPath(_item.ItemId, "en", "poster")); + Assert.False(File.Exists(file)); + Assert.Equal(5, _http.Requests.Count); + } + + [Fact] + public async Task ArtworkUsesProviderPriorityAndFallsBackWhenHigherPriorityImageDisappears() + { + _config.Providers = [new() { Id = "tmdb", MetadataOrder = 0, ArtworkOrder = 0 }, new() { Id = "fanart", ArtworkOrder = 1 }]; + _config.FanartApiKey = "test"; + await Fetch(); + Assert.Contains("image.tmdb.org", _store.GetAssetPath(_item.ItemId, "en", "poster")); + _missingTmdbPoster = true; + await Fetch(RefreshJobType.Full); + Assert.Contains("assets.fanart.tv", _store.GetAssetPath(_item.ItemId, "en", "poster")); + _fanartUnauthorized = true; + await Assert.ThrowsAsync(() => Fetch(RefreshJobType.Full)); + Assert.Contains("assets.fanart.tv", _store.GetAssetPath(_item.ItemId, "en", "poster")); + } + + [Fact] + public async Task BackupPreservesSourceUrlsAndFetchStateAndFiltersUnwantedLanguages() + { + _config.Providers[0].ArtworkOrder = 0; + await Fetch(); + var backup = Path.Combine(_root, "export.sqlite"); + _store.ExportTranslationDatabase(backup); + _store.ResetAll(); + _store.ImportTranslationDatabase(backup, new HashSet { _item.ItemId }, new HashSet { "en" }); + Assert.Equal(2, _store.GetFetchStates(_item.ItemId).Count); + Assert.DoesNotContain("metadata:fi", _store.GetFetchStates(_item.ItemId).Keys); + Assert.Equal("https://image.tmdb.org/t/p/original/poster.jpg", _store.GetStoredAssets(_item.ItemId).First().SourceUrl); + Assert.Equal("tmdb", _store.GetStoredAssets(_item.ItemId).First().Provider); + } + + [Theory] + [InlineData(true, false, false)] + [InlineData(true, true, true)] + [InlineData(false, false, false)] + [InlineData(false, true, false)] + public void YoungFetchStateHonorsKnownMissingVersusDeletedData(bool complete, bool missing, bool expected) + => Assert.Equal(expected, ItemFetcher.NeedsFetch(new("scope", 100, complete), "scope", missing, false, 50)); + + [Fact] + public void FailedArtworkPacketRollsBackDeletionAndNewRowsTogether() + { + var original = new StoredAsset("en", "poster", "https://example.test/old.jpg", "https://example.test/old.jpg", "tmdb"); + _store.SaveArtwork(_item.ItemId, [original], new Dictionary()); + Assert.Throws(() => _store.SaveArtwork(_item.ItemId, + [original with { Path = "new" }, original], new Dictionary())); + Assert.Equal(original, Assert.Single(_store.GetStoredAssets(_item.ItemId))); + } + + [Fact] + public async Task ParallelMetadataPacketsDoNotLoseRows() + { + await Task.WhenAll(Enumerable.Range(0, 12).Select(index => Task.Run(() => + _store.SaveMetadata(_item with { ItemId = index.ToString() }, "en", + new("Title", "Overview", "Tagline", "Title", "en", [], [], [], []), + new("scope", 1, true), true)))); + foreach (var index in Enumerable.Range(0, 12)) + { + Assert.NotNull(_store.GetFacts(index.ToString())); + Assert.Equal(3, _store.GetTranslations([index.ToString()], ["en"])[index.ToString()]["en"].Count); + Assert.True(_store.GetFetchStates(index.ToString())["metadata:en"].Complete); + } + } + + [Fact] + public void CollectionAggregationKeepsOnlySharedSingleLanguageAndUnionsCountrySets() + { + var collection = new FactsData("collection", "1", "collection", "Collection", "en", "en", "[]", "[]", "[]", "", "[]", 1, 1); + var finnish = collection with { Kind = "movie", OriginalLanguage = "fi", ProductionCountriesJson = "[\"FI\"]" }; + var swedish = finnish with { OriginalLanguage = "sv", ProductionCountriesJson = "[\"SE\",\"FI\"]" }; + var mixed = Services.ItemsProxyTransformer.AggregateCollectionFacts(collection, [finnish, swedish]); + Assert.Equal("", mixed.OriginalLanguage); + Assert.Equal("fi,sv", mixed.OriginalLanguageAll); + Assert.Equal(new HashSet { "FI", "SE" }, JsonSerializer.Deserialize>(mixed.ProductionCountriesJson)); + Assert.Equal("fi", Services.ItemsProxyTransformer.AggregateCollectionFacts(collection, [finnish, finnish]).OriginalLanguage); + } + + public void Dispose() => TestPaths.DeleteRoot(_root); +} diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyCacheTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyCacheTests.cs index 665b279..0f62adc 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyCacheTests.cs +++ b/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyCacheTests.cs @@ -5,16 +5,29 @@ namespace Jellyfin.Plugin.Multilang.Tests; public sealed class ItemsProxyCacheTests { [Fact] - public void StoreRetrievesAndInvalidatesByItemId() + public void PolicyChangesInvalidateCachedDataAndOldInFlightWrites() + { + var cache = new ItemsProxyCache(); + var generation = cache.ObserveUserPolicy("user", "all libraries"); + Store(cache, "key"); + Assert.Equal(generation, cache.ObserveUserPolicy("user", "all libraries")); + Assert.True(cache.TryGet("key", TimeSpan.FromMinutes(1), out _)); + Assert.True(cache.ObserveUserPolicy("user", "no libraries") > generation); + Assert.False(cache.TryGet("key", TimeSpan.FromMinutes(1), out _)); + Assert.False(cache.Store("key", "user", "/Items", 1, "body", "application/json", 1, 100, TimeSpan.FromMinutes(1), generation)); + } + + [Fact] + public void StoreRetrievesAndClearsCachedResponses() { var cache = new ItemsProxyCache(); - Assert.True(Store(cache, "key", ["A"])); + Assert.True(Store(cache, "key")); Assert.True(cache.TryGet("key", TimeSpan.FromMinutes(1), out var response)); Assert.Equal("body-key", response.Body); Assert.Equal(1, response.ItemCount); - cache.InvalidateItems(["a"]); + cache.ClearAll(); Assert.False(cache.TryGet("key", TimeSpan.FromMinutes(1), out _)); } @@ -24,11 +37,11 @@ public sealed class ItemsProxyCacheTests { var cache = new ItemsProxyCache(); - Assert.True(cache.StoreMicro("key", "user", "/Items", ["A"], "micro", "application/json", 1, 1024)); + Assert.True(cache.StoreMicro("key", "user", "/Items", 1, "micro", "application/json", 1, 1024)); Assert.True(cache.TryGet("key", TimeSpan.FromMinutes(1), out var response)); Assert.Equal("micro", response.Body); - cache.InvalidateItems(["A"]); + cache.ClearAll(); Assert.False(cache.TryGet("key", TimeSpan.FromMinutes(1), out _)); } @@ -38,9 +51,9 @@ public sealed class ItemsProxyCacheTests { var cache = new ItemsProxyCache(); - Assert.True(Store(cache, "first", ["A"], body: "1111", maxBytes: 6)); + Assert.True(Store(cache, "first", body: "1111", maxBytes: 6)); Thread.Sleep(10); - Assert.True(Store(cache, "second", ["B"], body: "2222", maxBytes: 6)); + Assert.True(Store(cache, "second", body: "2222", maxBytes: 6)); Assert.False(cache.TryGet("first", TimeSpan.FromMinutes(1), out _)); Assert.True(cache.TryGet("second", TimeSpan.FromMinutes(1), out _)); @@ -51,7 +64,7 @@ public sealed class ItemsProxyCacheTests { var cache = new ItemsProxyCache(); - Assert.True(Store(cache, "key", ["A"], ttl: TimeSpan.FromMilliseconds(1))); + Assert.True(Store(cache, "key", ttl: TimeSpan.FromMilliseconds(1))); Thread.Sleep(20); Assert.False(cache.TryGet("key", TimeSpan.FromMilliseconds(1), out _)); @@ -62,14 +75,14 @@ public sealed class ItemsProxyCacheTests { var cache = new ItemsProxyCache(); - Assert.False(Store(cache, "key", ["A"], body: "12345", maxBytes: 4)); + Assert.False(Store(cache, "key", body: "12345", maxBytes: 4)); Assert.False(cache.TryGet("key", TimeSpan.FromMinutes(1), out _)); } private static bool Store( ItemsProxyCache cache, string key, - string[] itemIds, + int itemCount = 1, string? body = null, long maxBytes = 1024, TimeSpan? ttl = null) @@ -77,10 +90,43 @@ public sealed class ItemsProxyCacheTests key, "user", "/Items", - itemIds, + itemCount, body ?? "body-" + key, "application/json", 100, maxBytes, ttl ?? TimeSpan.FromMinutes(1)); + + [Fact] + public void MicroAndMainEntriesShareOneBudget() + { + var cache = new ItemsProxyCache(); + Assert.True(Store(cache, "main", body: "1111", maxBytes: 6)); + Assert.True(cache.StoreMicro("micro1", "user", "/Items", 1, "2222", "application/json", 1, 6)); + Assert.False(cache.TryGet("main", TimeSpan.FromMinutes(1), out _)); + Assert.True(cache.StoreMicro("micro2", "user", "/Items", 1, "3333", "application/json", 1, 6)); + Assert.False(cache.TryGet("micro1", TimeSpan.FromMinutes(1), out _)); + Assert.True(cache.TryGet("micro2", TimeSpan.FromMinutes(1), out _)); + } + + [Fact] + public void InvalidationRejectsStaleInFlightWrites() + { + var cache = new ItemsProxyCache(); + var generation = cache.Generation; + cache.ClearAll(); + Assert.False(cache.Store("key", "user", "/Items", 1, "body", "application/json", 1, 100, TimeSpan.FromMinutes(1), generation)); + Assert.False(cache.StoreMicro("key", "user", "/Items", 1, "body", "application/json", 1, 100, generation)); + } + + [Fact] + public void ClearingOneUserPreservesOtherUsersCachedResponses() + { + var cache = new ItemsProxyCache(); + Store(cache, "one"); + cache.Store("two", "other", "/Items", 1, "body", "application/json", 1, 100, TimeSpan.FromMinutes(1)); + cache.ClearUser("user"); + Assert.False(cache.TryGet("one", TimeSpan.FromMinutes(1), out _)); + Assert.True(cache.TryGet("two", TimeSpan.FromMinutes(1), out _)); + } } diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyRequestBuilderTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyRequestBuilderTests.cs index 4c9eca0..24a537e 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyRequestBuilderTests.cs +++ b/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyRequestBuilderTests.cs @@ -16,7 +16,6 @@ public sealed class ItemsProxyRequestBuilderTests var result = ItemsProxyRequestBuilder.Build( request, "/Items?Recursive=true&api_key=old", - "token", UserId, multilangEnabled: true); @@ -36,7 +35,6 @@ public sealed class ItemsProxyRequestBuilderTests var result = ItemsProxyRequestBuilder.Build( request, $"/Users/{OtherUserId}/Items?Limit=5", - "token", UserId, multilangEnabled: true); @@ -52,7 +50,6 @@ public sealed class ItemsProxyRequestBuilderTests var result = ItemsProxyRequestBuilder.Build( request, "/Items?SortBy=SortName&SortOrder=Descending&StartIndex=20&Limit=10&NameStartsWith=L", - "token", UserId, multilangEnabled: true); @@ -65,7 +62,7 @@ public sealed class ItemsProxyRequestBuilderTests Assert.Equal("fi-FI", result.Controls.ClientLocale); Assert.DoesNotContain("SortBy=", result.Upstream.Query); Assert.DoesNotContain("Limit=", result.Upstream.Query); - Assert.Contains("|SortName|Descending|L|20|10|fi-FI|", result.CacheKey); + Assert.DoesNotContain("SortBy=", result.CacheKey); } [Fact] @@ -76,20 +73,18 @@ public sealed class ItemsProxyRequestBuilderTests var first = ItemsProxyRequestBuilder.Build( request, "/Items?SortBy=SortName&StartIndex=0&Limit=100", - "token", UserId, multilangEnabled: true); var second = ItemsProxyRequestBuilder.Build( request, "/Items?SortBy=SortName&StartIndex=100&Limit=100", - "token", UserId, multilangEnabled: true); Assert.NotNull(first); Assert.NotNull(second); Assert.Equal(first.NormalizedUrlForCache, second.NormalizedUrlForCache); - Assert.NotEqual(first.CacheKey, second.CacheKey); + Assert.Equal(first.CacheKey, second.CacheKey); } [Fact] @@ -100,7 +95,6 @@ public sealed class ItemsProxyRequestBuilderTests var result = ItemsProxyRequestBuilder.Build( request, "/Items?SortBy=SortName&Limit=10", - "token", UserId, multilangEnabled: false); @@ -119,7 +113,6 @@ public sealed class ItemsProxyRequestBuilderTests var result = ItemsProxyRequestBuilder.Build( request, "http://other-host:8096/Items", - "token", UserId, multilangEnabled: true); @@ -132,6 +125,18 @@ public sealed class ItemsProxyRequestBuilderTests Assert.Equal([1, 2], ItemsProxyRequestBuilder.ParseLocalGenreIds("tmdb-1,abc,2,tmdb-2,tmdb-x,1")); } + [Fact] + public void GenreFilteringRetainsTheFieldsNeededForLocalFiltering() + { + var result = ItemsProxyRequestBuilder.Build(HttpRequest(), "/Items?GenreIds=tmdb-18&Fields=Overview", UserId, true); + Assert.NotNull(result); + var query = Microsoft.AspNetCore.WebUtilities.QueryHelpers.ParseQuery(result.Upstream.Query); + Assert.False(query.ContainsKey("GenreIds")); + Assert.Equal(1, query["Fields"].Count); + Assert.Contains("GenreItems", query["Fields"].ToString()); + Assert.Contains("Overview", query["Fields"].ToString()); + } + private static HttpRequest HttpRequest(string query = "") { var context = new DefaultHttpContext(); @@ -140,4 +145,40 @@ public sealed class ItemsProxyRequestBuilderTests context.Request.QueryString = new QueryString(query); return context.Request; } + + [Theory] + [InlineData("//other-host:8096/Items")] + [InlineData("/\\other-host/Items")] + [InlineData("https://server.local:8096/Items")] + [InlineData("/Multilang/ItemsProxy")] + [InlineData("/System/Configuration")] + [InlineData("/Items/Filters")] + [InlineData("/Items/../System/Info")] + public void RejectsUnsupportedDestinations(string url) + => Assert.Null(ItemsProxyRequestBuilder.Build(HttpRequest(), url, UserId, true)); + + [Fact] + public void EquivalentQueryOrderSharesCacheAndInFlightKeys() + { + var first = ItemsProxyRequestBuilder.Build(HttpRequest(), "/Items?Fields=Overview&Recursive=true", UserId, true)!; + var second = ItemsProxyRequestBuilder.Build(HttpRequest(), "/Items?Recursive=true&Fields=Overview", UserId, true)!; + Assert.Equal(first.CacheKey, second.CacheKey); + Assert.Equal(first.NormalizedUrlForCache, second.NormalizedUrlForCache); + } + + [Fact] + public void BasePathAndDashedUserIdsAreNormalized() + { + var request = HttpRequest(); + request.PathBase = "/jellyfin"; + var result = ItemsProxyRequestBuilder.Build(request, "/jellyfin/Users/22222222-2222-2222-2222-222222222222/Items", UserId, true)!; + Assert.Equal($"/jellyfin/Users/{UserId}/Items", result.Upstream.AbsolutePath); + Assert.Null(ItemsProxyRequestBuilder.Build(request, "/Items", UserId, true)); + } + + [Theory] + [InlineData("/Items/Resume?StartIndex=100&Limit=20")] + [InlineData("/Shows/NextUp?StartIndex=100&Limit=20")] + public void BoundedListsKeepUpstreamPaging(string url) + => Assert.False(ItemsProxyRequestBuilder.Build(HttpRequest(), url, UserId, true)!.Controls.ApplyLocalPaging); } diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxySortingTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxySortingTests.cs index b6cd88d..a3a4997 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxySortingTests.cs +++ b/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxySortingTests.cs @@ -7,6 +7,14 @@ namespace Jellyfin.Plugin.Multilang.Tests; public sealed class ItemsProxySortingTests { + [Fact] + public void TranslationOnlyDoesNotRewriteUpstreamPaging() + { + var root = JsonNode.Parse("""{"Items":[{"Name":"One"},{"Name":"Two"}],"TotalRecordCount":200,"StartIndex":100}""")!; + var original = root.ToJsonString(); + ItemsProxySorting.Apply(root, new ItemsProxyControls("", "", "", 0, 0, "", "", false), CultureInfo.InvariantCulture, []); + Assert.Equal(original, root.ToJsonString()); + } [Fact] public void GetSortArticlesUsesBuiltInLanguageFallback() { diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyTransformerResolutionTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyTransformerResolutionTests.cs index 691a6aa..4ae8417 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyTransformerResolutionTests.cs +++ b/tests/Jellyfin.Plugin.Multilang.Tests/ItemsProxyTransformerResolutionTests.cs @@ -13,6 +13,22 @@ public sealed class ItemsProxyTransformerResolutionTests private const string OriginalAction = "Original"; private const string LanguagePrefix = "Language:"; + [Fact] + public void TraceUsesStoredOriginalTranslationWhenOriginalTitleIsEmpty() + { + var facts = Facts(originalTitle: "", originalLanguage: "fi"); + var translations = TextTranslations((OriginalAction, TitleField, "Alkuperainen nimi")); + var attempts = new List(); + var normal = ItemsProxyTransformer.ResolveField(TitleField, [OriginalAction], facts, translations); + var traced = ItemsProxyTransformer.ResolveField(TitleField, [OriginalAction], facts, translations, attempts.Add); + Assert.Equal(normal, traced); + Assert.Equal("fi", traced.Language); + var chosen = Assert.Single(attempts); + Assert.True(chosen.Chosen); + Assert.Equal("Original/title", chosen.LookupKey); + Assert.Equal(traced.Value, chosen.Value); + } + [Fact] public void ResolveFieldUsesFirstLanguageWithData() { diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/Jellyfin.Plugin.Multilang.Tests.csproj b/tests/Jellyfin.Plugin.Multilang.Tests/Jellyfin.Plugin.Multilang.Tests.csproj index 8f72023..dfb1053 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/Jellyfin.Plugin.Multilang.Tests.csproj +++ b/tests/Jellyfin.Plugin.Multilang.Tests/Jellyfin.Plugin.Multilang.Tests.csproj @@ -13,8 +13,8 @@ - - + + diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/MaintenanceAndProviderTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/MaintenanceAndProviderTests.cs new file mode 100644 index 0000000..570473f --- /dev/null +++ b/tests/Jellyfin.Plugin.Multilang.Tests/MaintenanceAndProviderTests.cs @@ -0,0 +1,99 @@ +using System.Diagnostics; +using System.Net; +using Jellyfin.Plugin.Multilang.Data; +using Jellyfin.Plugin.Multilang.Services.Providers; + +namespace Jellyfin.Plugin.Multilang.Tests; + +public sealed class MaintenanceAndProviderTests +{ + [Fact] + public async Task ConcurrentRefreshesCanOverlapButMaintenanceWaitsForAllOfThem() + { + var root = TestPaths.CreateRoot(); + try + { + var store = new TranslationStore(new TestApplicationPaths(root)); + var first = await store.EnterRefreshAsync(CancellationToken.None); + var second = await store.EnterRefreshAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5)); + using var cancel = new CancellationTokenSource(); + var maintenance = store.EnterMaintenanceAsync(cancel.Token); + Assert.False(maintenance.IsCompleted); + first.Dispose(); + Assert.False(maintenance.IsCompleted); + cancel.Cancel(); + await Assert.ThrowsAnyAsync(() => maintenance); + second.Dispose(); + using var allSlots = await store.EnterMaintenanceAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5)); + } + finally { TestPaths.DeleteRoot(root); } + } + + [Fact] + public async Task MaintenanceWaitsForWriterAndCanceledWaiterDoesNotLeakLease() + { + var root = TestPaths.CreateRoot(); + try + { + var store = new TranslationStore(new TestApplicationPaths(root)); + var writer = await store.EnterMaintenanceAsync(CancellationToken.None); + using var canceled = new CancellationTokenSource(); + var waiting = store.EnterMaintenanceAsync(canceled.Token); + Assert.False(waiting.IsCompleted); + canceled.Cancel(); + await Assert.ThrowsAnyAsync(() => waiting); + var cleanup = store.EnterMaintenanceAsync(CancellationToken.None); + Assert.False(cleanup.IsCompleted); + writer.Dispose(); + using var lease = await cleanup.WaitAsync(TimeSpan.FromSeconds(5)); + } + finally { TestPaths.DeleteRoot(root); } + } + + [Fact] + public void ResetKeepsExistingConnectionsOnTheSameDatabase() + { + var root = TestPaths.CreateRoot(); + try + { + var store = new TranslationStore(new TestApplicationPaths(root)); + store.SaveUserRules("user", new UserRulesDocument()); + using var connection = store.Open(); + connection.Open(); + using var query = connection.CreateCommand(); + query.CommandText = "SELECT COUNT(*) FROM user_rules;"; + Assert.Equal(1L, query.ExecuteScalar()); + store.ResetAll(); + Assert.Equal(0L, query.ExecuteScalar()); + } + finally { TestPaths.DeleteRoot(root); } + } + + [Fact] + public async Task ProviderDoesNotRetryProgrammingErrors() + { + var attempts = 0; + using var http = new HttpClient(new TestHttpMessageHandler(_ => + { + attempts++; + throw new InvalidOperationException("Programming error"); + })); + using var request = new HttpRequestMessage(HttpMethod.Get, "https://example.test/"); + await Assert.ThrowsAsync(() => ProviderHttp.SendWithRetryAsync(http, request, CancellationToken.None)); + Assert.Equal(1, attempts); + } + + [Fact] + public async Task ConcurrentProviderAdmissionsKeepTheirSpacing() + { + var limiter = new FetchRateLimiter(TimeSpan.FromMilliseconds(50)); + var starts = await Task.WhenAll(Enumerable.Range(0, 4).Select(async _ => + { + await limiter.WaitAsync(CancellationToken.None); + return Stopwatch.GetTimestamp(); + })); + Array.Sort(starts); + Assert.All(starts.Zip(starts.Skip(1)), pair => + Assert.True(Stopwatch.GetElapsedTime(pair.First, pair.Second) >= TimeSpan.FromMilliseconds(40))); + } +} diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/RefreshCoordinatorTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/RefreshCoordinatorTests.cs index 4b7fb8c..6bfcbf7 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/RefreshCoordinatorTests.cs +++ b/tests/Jellyfin.Plugin.Multilang.Tests/RefreshCoordinatorTests.cs @@ -5,10 +5,76 @@ namespace Jellyfin.Plugin.Multilang.Tests; public sealed class RefreshCoordinatorTests { + [Fact] + public async Task IndependentItemsOverlapWithinWorkerLimitAndDuplicateWorkIsShared() + { + using var coordinator = new RefreshCoordinator(concurrency: 2); + var twoStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var count = 0; + Task Queue(string id) => coordinator.EnqueueAsync(id, RefreshSourceTier.Manual, RefreshWorkClass.Movies, + RefreshJobType.Full, async (_, _) => + { + if (Interlocked.Increment(ref count) == 2) twoStarted.SetResult(); + await release.Task; + return true; + }, CancellationToken.None); + var first = Queue("first"); + var second = Queue("second"); + await twoStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Same(first, Queue("first")); + var third = Queue("third"); + Assert.Equal(2, count); + Assert.Equal(2, coordinator.GetDiagnostics().ActiveItems.Length); + release.SetResult(); + await Task.WhenAll(first, second, third).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(3, count); + } + + [Fact] + public async Task ActiveMissingRefreshRunsFullUpgradeBeforeCompleting() + { + using var coordinator = new RefreshCoordinator(concurrency: 1); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var fullRan = false; + var first = coordinator.EnqueueAsync("item", RefreshSourceTier.Background, RefreshWorkClass.Movies, RefreshJobType.Missing, async (_, _) => + { + started.SetResult(); + await release.Task; + return false; + }, CancellationToken.None); + await started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var full = coordinator.EnqueueAsync("item", RefreshSourceTier.Manual, RefreshWorkClass.Movies, RefreshJobType.Full, (_, _) => + { + fullRan = true; + return Task.FromResult(true); + }, CancellationToken.None); + release.SetResult(); + Assert.True(await full.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.True(fullRan); + Assert.True(await first); + } + + [Fact] + public async Task CanceledWaiterDoesNotCancelAnotherWaiter() + { + using var coordinator = new RefreshCoordinator(concurrency: 1); + using var cancel = new CancellationTokenSource(); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var canceled = coordinator.EnqueueAsync("item", RefreshSourceTier.Background, RefreshWorkClass.Movies, RefreshJobType.Full, + async (_, _) => { await release.Task; return true; }, cancel.Token); + var survivor = coordinator.EnqueueAsync("item", RefreshSourceTier.Background, RefreshWorkClass.Movies, RefreshJobType.Full, + (_, _) => Task.FromResult(false), CancellationToken.None); + cancel.Cancel(); + await Assert.ThrowsAnyAsync(() => canceled); + release.SetResult(); + Assert.True(await survivor.WaitAsync(TimeSpan.FromSeconds(5))); + } [Fact] public async Task QueuedItemsUseSourceTierThenWorkClassPriority() { - var coordinator = new RefreshCoordinator(new ItemsProxyCache()); + using var coordinator = new RefreshCoordinator(concurrency: 1); var activeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseActive = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var order = new List(); @@ -44,11 +110,11 @@ public sealed class RefreshCoordinatorTests } [Fact] - public async Task DuplicateQueuedItemUsesTheHighestJobAndTierAndInvalidatesCache() + public async Task DuplicateQueuedItemUsesHighestJobAndTierWithoutInvalidatingUpstreamData() { var cache = new ItemsProxyCache(); - Assert.True(cache.Store("target", "user", "/Items", ["target"], "body", "application/json", 1, 1024, TimeSpan.FromMinutes(1))); - var coordinator = new RefreshCoordinator(cache); + Assert.True(cache.Store("target", "user", "/Items", 1, "body", "application/json", 1, 1024, TimeSpan.FromMinutes(1))); + using var coordinator = new RefreshCoordinator(concurrency: 1); var activeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseActive = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var runs = new List(); @@ -77,7 +143,7 @@ public sealed class RefreshCoordinatorTests await Task.WhenAll(active, full); Assert.Equal([RefreshJobType.Full], runs); - Assert.False(cache.TryGet("target", TimeSpan.FromMinutes(1), out _)); + Assert.True(cache.TryGet("target", TimeSpan.FromMinutes(1), out _)); var recent = coordinator.GetDiagnostics().Recent.Single(item => item.ItemId == "target"); Assert.Equal("Manual", recent.SourceTier); Assert.Equal("Full", recent.JobType); diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/TranslationStoreBatchTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/TranslationStoreBatchTests.cs new file mode 100644 index 0000000..594a601 --- /dev/null +++ b/tests/Jellyfin.Plugin.Multilang.Tests/TranslationStoreBatchTests.cs @@ -0,0 +1,33 @@ +using System.Diagnostics; +using Jellyfin.Plugin.Multilang.Data; +using Jellyfin.Plugin.Multilang.Services.Refresh; +using Xunit.Abstractions; + +namespace Jellyfin.Plugin.Multilang.Tests; + +public sealed class TranslationStoreBatchTests(ITestOutputHelper output) +{ + [Fact] + public void BatchFactsLookupMatchesIndividualReads() + { + var root = TestPaths.CreateRoot(); + try + { + var store = new TranslationStore(new TestApplicationPaths(root)); + var ids = Enumerable.Range(0, 100).Select(index => index.ToString()).ToArray(); + foreach (var id in ids) + store.SaveMetadata(new(id, id, "movie", RefreshWorkClass.Movies, 0, 0, null, "Movie"), "fi", + new("Title", "Overview", "Tagline", "Title", "fi", [], [], [], []), new("scope", 1, true), true); + var timer = Stopwatch.StartNew(); + var individual = ids.Select(id => store.GetFacts(id)!).ToArray(); + var separateMs = timer.Elapsed.TotalMilliseconds; + timer.Restart(); + var batch = store.GetFactsForItems(ids.Concat(ids)); + var batchMs = timer.Elapsed.TotalMilliseconds; + Assert.Equal(individual, ids.Select(id => batch[id])); + Assert.Equal(ids.Length, batch.Count); + output.WriteLine($"100 individual fact reads: {separateMs:F2} ms; one batched read: {batchMs:F2} ms. Timing is diagnostic, not a pass condition."); + } + finally { TestPaths.DeleteRoot(root); } + } +} diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/TranslationStoreCleanupTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/TranslationStoreCleanupTests.cs index b4b4c1d..d7ee5fb 100644 --- a/tests/Jellyfin.Plugin.Multilang.Tests/TranslationStoreCleanupTests.cs +++ b/tests/Jellyfin.Plugin.Multilang.Tests/TranslationStoreCleanupTests.cs @@ -34,7 +34,7 @@ public sealed class TranslationStoreCleanupTests ["en"], localAssetStorage: false); - Assert.Equal(new LocalCleanupResult(TranslationsDeleted: 1, AssetsDeleted: 2, GenresDeleted: 1, AssetFilesDeleted: 2), result); + Assert.Equal(new LocalCleanupResult(TranslationsDeleted: 1, AssetsDeleted: 2, GenresDeleted: 1, AssetFilesDeleted: 4), result); Assert.NotNull(store.GetFacts(live)); Assert.Null(store.GetFacts(stale)); Assert.Equal("English", Translation(store, live, "en", "title")); diff --git a/tests/Jellyfin.Plugin.Multilang.Tests/UserRulesNormalizerTests.cs b/tests/Jellyfin.Plugin.Multilang.Tests/UserRulesNormalizerTests.cs new file mode 100644 index 0000000..648fcd4 --- /dev/null +++ b/tests/Jellyfin.Plugin.Multilang.Tests/UserRulesNormalizerTests.cs @@ -0,0 +1,61 @@ +using System.Text.Json; +using Jellyfin.Plugin.Multilang.Data; +using Jellyfin.Plugin.Multilang.Rules; + +namespace Jellyfin.Plugin.Multilang.Tests; + +public sealed class UserRulesNormalizerTests +{ + [Fact] + public void LegacyFieldsAreNotConvertedOrWrittenBack() + { + var rules = JsonSerializer.Deserialize(""" + {"Categories":[{"CriteriaText":"original_language is fi", + "FieldActions":{"title":"Language:fi"}, + "Requirements":[{"Field":"original_language","Relation":"is","Values":["fi"]}]}]} + """)!; + UserRulesNormalizer.Normalize(rules, ["fi"]); + var category = Assert.Single(rules.Categories); + Assert.Empty(category.Requirements); + Assert.Equal(["Jellyfin"], category.FieldActionLists["title"]); + var saved = JsonSerializer.Serialize(rules); + Assert.DoesNotContain("CriteriaText", saved); + Assert.DoesNotContain("\"Field\":", saved); + Assert.DoesNotContain("\"FieldActions\":", saved); + } + + [Fact] + public void CurrentRulesPreserveBooleanModesAndNormalizeOnlyCompleteConditions() + { + var rules = new UserRulesDocument + { + Categories = [new() + { + Label = "Finnish", MatchAllConditions = false, Scopes = ["M", "C"], + Requirements = [new() { Fields = ["audio_language", "original_language", "origin_countries"], + UseFieldOr = true, Relation = "contains", UseOr = true, Values = ["fi", "sv"] }, new()], + FieldActionLists = new() { ["title"] = ["Language:FI", "Language:de", "Original"], ["overview"] = [] } + }] + }; + UserRulesNormalizer.Normalize(rules, ["fi"]); + var category = Assert.Single(rules.Categories); + Assert.False(category.MatchAllConditions); + Assert.Equal(["M", "C"], category.Scopes); + var condition = Assert.Single(category.Requirements); + Assert.Equal(["original_language", "audio_language"], condition.Fields); + Assert.True(condition.UseFieldOr); + Assert.True(condition.UseOr); + Assert.Equal(["Language:fi", "Original", "Jellyfin"], category.FieldActionLists["title"]); + Assert.Empty(category.FieldActionLists["overview"]); + Assert.Equal(JsonSerializer.Serialize(rules), JsonSerializer.Serialize(UserRulesNormalizer.Normalize(rules, ["fi"]))); + } + + [Fact] + public void NullExternalCollectionsNormalizeToDefaults() + { + var rules = JsonSerializer.Deserialize("""{"Categories":null,"FallbackFieldActions":null}""")!; + UserRulesNormalizer.Normalize(rules); + Assert.Empty(rules.Categories); + Assert.Equal(["Jellyfin"], rules.FallbackFieldActions["poster"]); + } +} diff --git a/tests/browser/multilang.spec.js b/tests/browser/multilang.spec.js index b89397f..6c0a38c 100644 --- a/tests/browser/multilang.spec.js +++ b/tests/browser/multilang.spec.js @@ -101,9 +101,15 @@ test('pre-caches the fixture movie library after user activity', async ({ reques const entries = await apiGet(request, '/Multilang/CacheEntries'); return entries.some(entry => entry.Url.includes(`ParentId=${fixture.libraryId}`)); }, { timeout: 60000 }).toBeTruthy(); + await getFixtureItems(request); + const recent = await apiGet(request, '/Multilang/ItemsProxyRequests'); + expect(recent[0].CacheHit).toBeTruthy(); }); test('shows configured translations and uses the injected ItemsProxy route', async ({ page, request }) => { + const rules = await apiGet(request, '/Multilang/UserRules/self'); + rules.FallbackFieldActions.poster = ['Language:fi', 'Jellyfin']; + await apiPost(request, '/Multilang/UserRules/self', rules); const proxiedRequests = []; page.on('request', request => { if (new URL(request.url()).pathname === '/Multilang/ItemsProxy') { @@ -112,7 +118,10 @@ test('shows configured translations and uses the injected ItemsProxy route', asy }); await signIn(page); + const artworkResponse = page.waitForResponse(response => + new URL(response.url()).pathname === `/Multilang/Assets/${fixture.orderedItemIds[0]}/fixture.png`); await openFixtureLibrary(page, fixture.translatedTitles[0]); + expect((await artworkResponse).status()).toBe(200); for (const title of fixture.translatedTitles) { await expect(page.getByText(title, { exact: true })).toBeVisible(); @@ -151,7 +160,6 @@ test('applies a structured category rule before fallback translation actions', a Id: 'browser-fixture-swedish', Label: 'Swedish originals', Requirements: [{ - Field: 'original_language', Fields: ['original_language'], Relation: 'is', Values: ['sv'], @@ -161,7 +169,6 @@ test('applies a structured category rule before fallback translation actions', a MatchAllConditions: true, Scopes: ['M'], FieldActionLists: categoryActions, - FieldActions: {} }]; await apiPost(request, '/Multilang/UserRules/self', rules); @@ -257,3 +264,81 @@ test('saves a structured classification rule through the user settings page', as Values: ['sv'] })]); }); + +test('rejects unsafe proxy targets and enforces restricted-user item access', async ({ request }) => { + for (const url of ['//outside.invalid/Items', '/System/Info', '/Multilang/ItemsProxy']) { + const response = await request.get('/Multilang/ItemsProxy', { headers: authHeaders(), params: { url } }); + expect(response.status()).toBe(400); + } + const name = 'multilang-access-test-' + Date.now(); + const user = await apiPost(request, '/Users/New', { Name: name }); + const policy = { ...user.Policy, IsAdministrator: false, EnableAllFolders: true }; + let response = await request.post(`/Users/${user.Id}/Policy`, { headers: authHeaders(), data: policy }); + expect(response.ok()).toBeTruthy(); + response = await request.post('/Users/AuthenticateByName', { + headers: { Authorization: 'MediaBrowser Client="Multilang tests", Device="test", DeviceId="multilang-access-test", Version="1"' }, + data: { Username: name, Pw: '' } + }); + expect(response.ok()).toBeTruthy(); + const login = await response.json(); + const headers = { Authorization: `MediaBrowser Token="${login.AccessToken}"` }; + const asset = `/Multilang/Assets/${fixture.orderedItemIds[0]}/fixture.png`; + expect((await request.get(asset)).status()).toBe(401); + expect((await request.get(asset, { headers })).status()).toBe(200); + response = await request.post('/Multilang/UserRules/self', { headers, data: { Enabled: true, FallbackFieldActions: jellyfinActions() } }); + expect(response.ok()).toBeTruthy(); + const params = { url: `/Items?ParentId=${fixture.libraryId}&IncludeItemTypes=Movie&Recursive=true&SortBy=SortName` }; + response = await request.get('/Multilang/ItemsProxy', { headers, params }); + expect(response.ok()).toBeTruthy(); + expect((await response.json()).Items).toHaveLength(3); + let entries = await apiGet(request, '/Multilang/CacheEntries'); + expect(entries.some(entry => entry.UserId === user.Id)).toBeTruthy(); + + policy.EnableAllFolders = false; + policy.EnabledFolders = []; + response = await request.post(`/Users/${user.Id}/Policy`, { headers: authHeaders(), data: policy }); + expect(response.ok()).toBeTruthy(); + response = await request.get('/Multilang/ItemsProxy', { headers, params }); + const native = await request.get(`/Users/${user.Id}/Items?ParentId=${fixture.libraryId}`, { headers }); + expect(native.status()).toBe(401); + expect(response.status()).toBe(native.status()); + const recent = await apiGet(request, '/Multilang/ItemsProxyRequests'); + expect(recent[0].CacheHit).toBeFalsy(); + expect((await request.get(asset, { headers })).status()).toBe(404); + entries = await apiGet(request, '/Multilang/CacheEntries'); + expect(entries.some(entry => entry.UserId === user.Id && entry.ItemCount > 0)).toBeFalsy(); + for (const path of [`Debug/${fixture.orderedItemIds[0]}`, `Debug/${fixture.orderedItemIds[0]}/self`]) { + expect((await request.get(`/Multilang/${path}`, { headers })).status()).toBe(404); + } + expect((await request.post(`/Multilang/RefreshItem/${fixture.orderedItemIds[0]}?includeChildren=true`, { headers })).status()).toBe(404); + response = await request.get('/Multilang/ItemsProxy', { headers, params }); + expect(response.status()).toBe(401); +}); + +test('reuses a library fetch across different pages and sorting controls', async ({ request }) => { + const base = `/Items?ParentId=${fixture.libraryId}&IncludeItemTypes=Movie&Recursive=true`; + let response = await request.get('/Multilang/ItemsProxy', { headers: authHeaders(), params: { url: base + '&SortBy=SortName&Limit=1', mlLocale: 'fi-FI' } }); + expect((await response.json()).Items.map(item => item.Name)).toEqual(['Ankkuri']); + response = await request.get('/Multilang/ItemsProxy', { headers: authHeaders(), params: { url: base + '&SortBy=SortName&StartIndex=1&Limit=1', mlLocale: 'fi-FI' } }); + expect((await response.json()).Items.map(item => item.Name)).toEqual(['The Clock']); + const recent = await apiGet(request, '/Multilang/ItemsProxyRequests'); + expect(recent[0].CacheHit).toBeTruthy(); +}); + +test('reopens user settings without duplicate loads or a native replaceChildren dependency', async ({ page }) => { + await page.addInitScript(() => { + Object.defineProperty(Element.prototype, 'replaceChildren', { configurable: true, value: undefined }); + }); + await signIn(page); + await page.goto(`/web/#/mypreferencesmenu?userId=${fixture.userId}`); + let loads = 0; + page.on('request', request => { if (new URL(request.url()).pathname === '/Multilang/UserRules/self') loads++; }); + for (let i = 0; i < 3; i++) { + await expect(page.locator('#mlUserRulesMenuItem')).toBeVisible(); + await page.locator('#mlUserRulesMenuItem').click(); + await expect(page.locator('#ml-sort-locale option[value="en"]')).toHaveCount(1); + expect(loads).toBe(i + 1); + await page.evaluate(userId => { location.hash = '#/mypreferencesmenu?userId=' + userId; }, fixture.userId); + await expect(page.locator('#ml-user-shell')).toHaveCount(0); + } +}); diff --git a/tools/run_jellyfin12_browser_tests.py b/tools/run_jellyfin12_browser_tests.py index e97e0af..d9ab845 100644 --- a/tools/run_jellyfin12_browser_tests.py +++ b/tools/run_jellyfin12_browser_tests.py @@ -144,6 +144,14 @@ for row in rows: INSERT INTO genres(tmdb_id, media, lang, name, name_norm) VALUES (?, 'movie', 'fi', ?, ?) ON CONFLICT(tmdb_id, media, lang) DO UPDATE SET name=excluded.name, name_norm=excluded.name_norm """, (row["genre_id"], row["genre"], row["genre"].lower())) + relative = row["item_id"] + "/fixture.png" + destination = os.path.join(os.path.dirname(database), "assets", relative) + os.makedirs(os.path.dirname(destination), exist_ok=True) + with open(destination, "wb") as image: + image.write(base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jZ1kAAAAASUVORK5CYII=")) + path = "/Multilang/Assets/" + relative + connection.execute("INSERT OR REPLACE INTO assets(item_id,lang,kind,path,path_low,updated_at) VALUES (?, 'fi', 'poster', ?, ?, ?)", + (row["item_id"], path, path.lower(), now)) connection.commit() ''' remote.stop()