Release 0.3.4: Jellyfin 12 stable support and audit fixes

Harden proxy permissions and cache behavior, streamline metadata fetching, remove obsolete rules, and validate the stable SDK with expanded regression coverage. Document backup format changes.
This commit is contained in:
ajp_anton
2026-09-09 01:18:43 +00:00
parent 8f7462fd3c
commit 096c6be8c7
50 changed files with 2126 additions and 1784 deletions
+19 -4
View File
@@ -4,7 +4,19 @@ Multilang is a Jellyfin plugin that lets different users see different metadata
languages without changing Jellyfin's stored metadata. languages without changing Jellyfin's stored metadata.
The `main` branch is built for the Jellyfin 10.11.x stable series. The The `main` branch is built for the Jellyfin 10.11.x stable series. The
`jellyfin12` branch targets Jellyfin 12 and is currently early beta software. `jellyfin12` branch targets Jellyfin 12.0 stable on Linux x64. The plugin remains
beta software.
## Upgrading to 0.2.5 or 0.3.4
Existing plugin data is upgraded automatically. Make fresh database exports after
upgrading: older translation-database exports cannot be imported by these versions.
User-settings-only exports keep their existing format. Obsolete text-based rules
are no longer supported; current classification rules are unaffected.
The first missing-data scan records the new per-language and per-provider fetch
state, so it may fetch more than subsequent scans. Later scans retry only missing
data when due and repair downloaded artwork without refetching its metadata.
## What It Does ## What It Does
@@ -87,9 +99,12 @@ the UI.
Some requests can be slow because Multilang may need to ask Jellyfin for a Some requests can be slow because Multilang may need to ask Jellyfin for a
larger list than the UI will finally display. This is needed so the plugin can larger list than the UI will finally display. This is needed so the plugin can
translate, filter, sort, and slice the list using the user's own rules. translate, filter, sort, and slice the list using the user's own rules.
Multilang caches slow transformed responses for a configurable time and also Multilang caches Jellyfin's responses for a configurable time, sharing the fetched
uses a very short burst cache for repeated requests during navigation. Cache list across pages and sort orders. Current translations and user rules are applied
entries are invalidated when affected items are refreshed. each time that list is used. Simultaneous requests share an in-progress fetch, and
a short burst cache also handles fast repeated requests. Library, permission and
watched-state changes invalidate affected caches. Movie and show libraries can
optionally be fetched in advance when a user becomes active.
## Original-Language Support ## Original-Language Support
@@ -1,4 +1,6 @@
using System.Reflection; using System.Reflection;
using Jellyfin.Data;
using Jellyfin.Database.Implementations.Enums;
using System.Diagnostics; using System.Diagnostics;
using System.Globalization; using System.Globalization;
using System.Text.Json; using System.Text.Json;
@@ -27,15 +29,6 @@ namespace Jellyfin.Plugin.Multilang.Api;
[Route("Multilang")] [Route("Multilang")]
public sealed class MultilangController : ControllerBase public sealed class MultilangController : ControllerBase
{ {
private static readonly string[] RequirementFieldOrder =
[
"original_language",
"spoken_languages",
"audio_language",
"origin_countries",
"production_countries"
];
private readonly TranslationStore _store; private readonly TranslationStore _store;
private readonly ISessionManager _sessionManager; private readonly ISessionManager _sessionManager;
private readonly IAuthorizationContext _authorizationContext; private readonly IAuthorizationContext _authorizationContext;
@@ -108,10 +101,18 @@ public sealed class MultilangController : ControllerBase
=> ServeEmbedded("Jellyfin.Plugin.Multilang.Configuration.shared.css", "text/css"); => ServeEmbedded("Jellyfin.Plugin.Multilang.Configuration.shared.css", "text/css");
[HttpGet("Assets/{**path}")] [HttpGet("Assets/{**path}")]
public IActionResult Asset(string path) public async Task<IActionResult> Asset(string path)
=> _assetStorage.TryResolveLocalAsset(path, out var fullPath, out var contentType) {
? PhysicalFile(fullPath, contentType) var userId = await GetUserIdAsync().ConfigureAwait(false);
: NotFound(); if (string.IsNullOrWhiteSpace(userId))
return Unauthorized(new { Error = "NotAuthenticated" });
if (!Guid.TryParse(path.Split('/')[0], out var itemId) || !CanAccessItem(itemId, userId) ||
!_store.IsAssetReferenced(itemId.ToString("N"), TranslationStore.LocalAssetUrlPrefix + path) ||
!_assetStorage.TryResolveLocalAsset(path, out var fullPath, out var contentType))
return NotFound();
Response.Headers.CacheControl = "private, no-cache";
return PhysicalFile(fullPath, contentType);
}
[HttpGet("Providers")] [HttpGet("Providers")]
public IActionResult Providers() public IActionResult Providers()
@@ -214,10 +215,9 @@ public sealed class MultilangController : ControllerBase
return Unauthorized(new { Error = "NotAuthenticated" }); return Unauthorized(new { Error = "NotAuthenticated" });
var allowed = (Plugin.Instance?.Configuration?.Languages ?? []).ToHashSet(StringComparer.OrdinalIgnoreCase); var allowed = (Plugin.Instance?.Configuration?.Languages ?? []).ToHashSet(StringComparer.OrdinalIgnoreCase);
request.Categories = NormalizeCategories(request.Categories, allowed); UserRulesNormalizer.Normalize(request, allowed);
request.FallbackFieldActions = NormalizeActionLists(request.FallbackFieldActions, allowed);
_store.SaveUserRules(userId, request); _store.SaveUserRules(userId, request);
_itemsProxyCache.ClearAll(); _itemsProxyCache.ClearUser(userId);
return Ok(request); return Ok(request);
} }
@@ -265,7 +265,11 @@ public sealed class MultilangController : ControllerBase
if (!TryParseItemId(itemId, out _, out var itemGuid)) if (!TryParseItemId(itemId, out _, out var itemGuid))
return BadRequest(new { Error = "InvalidItemId" }); return BadRequest(new { Error = "InvalidItemId" });
await _refreshService.RefreshItemAsync(itemGuid, includeChildren, cancellationToken).ConfigureAwait(false); var user = _userManager.GetUserById(Guid.Parse(userId));
if (user is null || !CanAccessItem(itemGuid, userId))
return NotFound();
await _refreshService.RefreshItemAsync(itemGuid, includeChildren, cancellationToken, user).ConfigureAwait(false);
return Ok(new { ItemId = itemGuid.ToString("N"), IncludeChildren = includeChildren }); return Ok(new { ItemId = itemGuid.ToString("N"), IncludeChildren = includeChildren });
} }
@@ -283,8 +287,13 @@ public sealed class MultilangController : ControllerBase
if (string.IsNullOrWhiteSpace(userId)) if (string.IsNullOrWhiteSpace(userId))
return Unauthorized(new { Error = "NotAuthenticated" }); return Unauthorized(new { Error = "NotAuthenticated" });
var user = _userManager.GetUserById(Guid.Parse(userId));
if (user is null || user.HasPermission(PermissionKind.IsDisabled))
return Unauthorized(new { Error = "NotAuthenticated" });
// Jellyfin's UpdatePolicyAsync does not publish a user-update event.
var generation = _itemsProxyCache.ObserveUserPolicy(userId, JsonSerializer.Serialize(_userManager.GetUserDto(user).Policy));
var rules = GetRulesOrDefault(userId); var rules = GetRulesOrDefault(userId);
var proxyRequest = ItemsProxyRequestBuilder.Build(Request, url, token, userId, rules.Enabled); var proxyRequest = ItemsProxyRequestBuilder.Build(Request, url, userId, rules.Enabled);
if (proxyRequest is null) if (proxyRequest is null)
return BadRequest(new { Error = "InvalidUrl" }); return BadRequest(new { Error = "InvalidUrl" });
@@ -304,6 +313,7 @@ public sealed class MultilangController : ControllerBase
if (cacheAllowed && _itemsProxyCache.TryGet(proxyRequest.CacheKey, cacheTtl, out var cached)) if (cacheAllowed && _itemsProxyCache.TryGet(proxyRequest.CacheKey, cacheTtl, out var cached))
{ {
var transformed = _itemsProxyTransformer.TransformItemsResponse(cached.Body, rules, proxyRequest.Controls);
totalSw.Stop(); totalSw.Stop();
_itemsProxyCache.RecordRequest( _itemsProxyCache.RecordRequest(
proxyRequest.NormalizedUrlForCache, proxyRequest.NormalizedUrlForCache,
@@ -313,9 +323,9 @@ public sealed class MultilangController : ControllerBase
itemCount: cached.ItemCount, itemCount: cached.ItemCount,
totalMs: totalSw.ElapsedMilliseconds, totalMs: totalSw.ElapsedMilliseconds,
upstreamMs: 0, upstreamMs: 0,
transformMs: 0, transformMs: totalSw.ElapsedMilliseconds,
sizeBytes: cached.SizeBytes); sizeBytes: cached.SizeBytes);
return Content(cached.Body, cached.ContentType); return Content(transformed.Body, cached.ContentType);
} }
if (rules.Enabled && proxyRequest.IsGenresRequest && _itemsProxyTransformer.TryBuildGenresResponse(proxyRequest.GenreMedia, proxyRequest.Controls.ClientLocale, out var genresBody)) if (rules.Enabled && proxyRequest.IsGenresRequest && _itemsProxyTransformer.TryBuildGenresResponse(proxyRequest.GenreMedia, proxyRequest.Controls.ClientLocale, out var genresBody))
@@ -337,11 +347,12 @@ public sealed class MultilangController : ControllerBase
var upstreamSw = Stopwatch.StartNew(); var upstreamSw = Stopwatch.StartNew();
var upstream = await _itemsProxyRequestCoalescer.GetOrFetchAsync( var upstream = await _itemsProxyRequestCoalescer.GetOrFetchAsync(
proxyRequest.NormalizedUrlForCache, generation + "|" + proxyRequest.NormalizedUrlForCache,
async sharedCancellationToken => async sharedCancellationToken =>
{ {
var http = _httpClientFactory.CreateClient(); var http = _httpClientFactory.CreateClient("Multilang.Jellyfin");
using var request = new HttpRequestMessage(HttpMethod.Get, proxyRequest.Upstream); var localUri = new Uri(_itemsProxyPrecacheService.LocalApiUri, proxyRequest.Upstream.PathAndQuery);
using var request = new HttpRequestMessage(HttpMethod.Get, localUri);
request.Headers.TryAddWithoutValidation("Authorization", $"MediaBrowser Token=\"{token}\""); request.Headers.TryAddWithoutValidation("Authorization", $"MediaBrowser Token=\"{token}\"");
request.Headers.TryAddWithoutValidation("X-Emby-Token", token); request.Headers.TryAddWithoutValidation("X-Emby-Token", token);
request.Headers.TryAddWithoutValidation("X-MediaBrowser-Token", token); request.Headers.TryAddWithoutValidation("X-MediaBrowser-Token", token);
@@ -380,12 +391,13 @@ public sealed class MultilangController : ControllerBase
proxyRequest.CacheKey, proxyRequest.CacheKey,
userId, userId,
proxyRequest.NormalizedUrlForCache, proxyRequest.NormalizedUrlForCache,
responseItemIds, responseItemIds.Length,
body, upstream.Response.Body,
contentType, contentType,
totalSw.ElapsedMilliseconds, totalSw.ElapsedMilliseconds,
cfg.ItemsProxyCacheMaxMiB * 1024L * 1024L, cfg.ItemsProxyCacheMaxMiB * 1024L * 1024L,
cacheTtl); cacheTtl,
generation);
} }
if (cacheAllowed && if (cacheAllowed &&
@@ -397,11 +409,13 @@ public sealed class MultilangController : ControllerBase
proxyRequest.CacheKey, proxyRequest.CacheKey,
userId, userId,
proxyRequest.NormalizedUrlForCache, proxyRequest.NormalizedUrlForCache,
responseItemIds, responseItemIds.Length,
body, upstream.Response.Body,
contentType, contentType,
totalSw.ElapsedMilliseconds, totalSw.ElapsedMilliseconds,
cfg.ItemsProxyCacheMaxMiB * 1024L * 1024L); cfg.ItemsProxyCacheMaxMiB * 1024L * 1024L,
generation,
cacheTtl);
} }
_itemsProxyCache.RecordRequest( _itemsProxyCache.RecordRequest(
@@ -424,19 +438,10 @@ public sealed class MultilangController : ControllerBase
}; };
} }
private sealed record DebugResolutionAttempt(
string Action,
string Source,
string LookupKey,
bool HasValue,
bool Chosen,
string? Value,
string Reason);
private sealed record DebugTextResolution( private sealed record DebugTextResolution(
string Field, string Field,
string[] Actions, string[] Actions,
DebugResolutionAttempt[] Attempts, ItemsProxyResolutionAttempt[] Attempts,
ItemsProxyResolvedText Result, ItemsProxyResolvedText Result,
string SelectedAction, string SelectedAction,
string Reason); string Reason);
@@ -444,7 +449,7 @@ public sealed class MultilangController : ControllerBase
private sealed record DebugAssetResolution( private sealed record DebugAssetResolution(
string Kind, string Kind,
string[] Actions, string[] Actions,
DebugResolutionAttempt[] Attempts, ItemsProxyResolutionAttempt[] Attempts,
ItemsProxyResolvedAsset Result, ItemsProxyResolvedAsset Result,
string SelectedAction, string SelectedAction,
string Reason); string Reason);
@@ -515,6 +520,7 @@ public sealed class MultilangController : ControllerBase
if (downloadedAssets && !translationsDatabase) if (downloadedAssets && !translationsDatabase)
return BadRequest(new { Error = "AssetsRequireDatabase" }); return BadRequest(new { Error = "AssetsRequireDatabase" });
using var maintenance = await _store.EnterMaintenanceAsync(HttpContext.RequestAborted).ConfigureAwait(false);
var bytes = _backupService.Export( var bytes = _backupService.Export(
new BackupExportOptions(pluginSettings, userSettings, translationsDatabase, downloadedAssets), new BackupExportOptions(pluginSettings, userSettings, translationsDatabase, downloadedAssets),
Plugin.Instance?.Configuration ?? new PluginConfiguration()); Plugin.Instance?.Configuration ?? new PluginConfiguration());
@@ -544,6 +550,7 @@ public sealed class MultilangController : ControllerBase
try try
{ {
await using var stream = file.OpenReadStream(); await using var stream = file.OpenReadStream();
using var maintenance = await _store.EnterMaintenanceAsync(HttpContext.RequestAborted).ConfigureAwait(false);
var result = _backupService.ImportAdmin( var result = _backupService.ImportAdmin(
stream, stream,
options, options,
@@ -567,6 +574,7 @@ public sealed class MultilangController : ControllerBase
if (request.Confirm != true) if (request.Confirm != true)
return BadRequest(new { Error = "ConfirmationRequired" }); return BadRequest(new { Error = "ConfirmationRequired" });
using var maintenance = await _store.EnterMaintenanceAsync(HttpContext.RequestAborted).ConfigureAwait(false);
Plugin.Instance?.UpdateConfiguration(new PluginConfiguration()); Plugin.Instance?.UpdateConfiguration(new PluginConfiguration());
_store.ResetAll(); _store.ResetAll();
_itemsProxyCache.ClearAll(); _itemsProxyCache.ClearAll();
@@ -625,7 +633,7 @@ public sealed class MultilangController : ControllerBase
await using var stream = file.OpenReadStream(); await using var stream = file.OpenReadStream();
var result = _backupService.ImportUser(stream, userId, form.TryGetValue("sourceUserId", out var sourceUserId) ? sourceUserId.ToString() : null); var result = _backupService.ImportUser(stream, userId, form.TryGetValue("sourceUserId", out var sourceUserId) ? sourceUserId.ToString() : null);
if (result.UserSettingsImported > 0) if (result.UserSettingsImported > 0)
_itemsProxyCache.ClearAll(); _itemsProxyCache.ClearUser(userId);
return Ok(result); return Ok(result);
} }
catch (InvalidOperationException ex) catch (InvalidOperationException ex)
@@ -641,9 +649,12 @@ public sealed class MultilangController : ControllerBase
if (string.IsNullOrWhiteSpace(userId)) if (string.IsNullOrWhiteSpace(userId))
return Unauthorized(new { Error = "NotAuthenticated" }); return Unauthorized(new { Error = "NotAuthenticated" });
if (!TryParseItemId(itemId, out var itemId32, out _)) if (!TryParseItemId(itemId, out var itemId32, out var itemGuid))
return BadRequest(new { Error = "InvalidItemId" }); return BadRequest(new { Error = "InvalidItemId" });
if (!CanAccessItem(itemGuid, userId))
return NotFound();
return Ok(new return Ok(new
{ {
ItemId = itemId32, ItemId = itemId32,
@@ -673,7 +684,7 @@ public sealed class MultilangController : ControllerBase
if (normalizedUserId is null) if (normalizedUserId is null)
return BadRequest(new { Error = "InvalidUserId" }); return BadRequest(new { Error = "InvalidUserId" });
return DebugResolveForUser(itemId, normalizedUserId); return DebugResolveForUser(itemId, normalizedUserId, administrator: true);
} }
[HttpGet("DebugResolve/{itemId}")] [HttpGet("DebugResolve/{itemId}")]
@@ -686,11 +697,14 @@ public sealed class MultilangController : ControllerBase
return DebugResolveForUser(itemId, userId); return DebugResolveForUser(itemId, userId);
} }
private IActionResult DebugResolveForUser(string itemId, string userId) private IActionResult DebugResolveForUser(string itemId, string userId, bool administrator = false)
{ {
if (!TryParseItemId(itemId, out var itemId32, out _)) if (!TryParseItemId(itemId, out var itemId32, out var itemGuid))
return BadRequest(new { Error = "InvalidItemId" }); return BadRequest(new { Error = "InvalidItemId" });
if (!administrator && !CanAccessItem(itemGuid, userId))
return NotFound();
var facts = _store.GetFacts(itemId32); var facts = _store.GetFacts(itemId32);
if (facts is null) if (facts is null)
return NotFound(new { Error = "FactsMissing" }); return NotFound(new { Error = "FactsMissing" });
@@ -809,128 +823,27 @@ public sealed class MultilangController : ControllerBase
} }
private static DebugTextResolution TraceResolveField( private static DebugTextResolution TraceResolveField(
string field, string field, IReadOnlyList<string> actions, FactsData facts,
IReadOnlyList<string> actions,
FactsData facts,
Dictionary<string, Dictionary<string, string>>? byLang) Dictionary<string, Dictionary<string, string>>? byLang)
{ {
var attempts = new List<DebugResolutionAttempt>(); var attempts = new List<ItemsProxyResolutionAttempt>();
foreach (var action in actions) var result = ItemsProxyTransformer.ResolveField(field, actions, facts, byLang, attempts.Add);
{ var chosen = attempts.FirstOrDefault(attempt => attempt.Chosen);
if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)) return new(field, actions.ToArray(), attempts.ToArray(), result,
{ chosen?.Action ?? (result.Change ? ClearAction : JellyfinAction),
var result = new ItemsProxyResolvedText(false, null); chosen?.Reason ?? (result.Change ? "No action had data; field is cleared" : "No action had data; use Jellyfin"));
attempts.Add(new DebugResolutionAttempt(action, JellyfinAction, string.Empty, true, true, null, "Use Jellyfin value"));
return new DebugTextResolution(field, actions.ToArray(), attempts.ToArray(), result, JellyfinAction, "Jellyfin action reached");
}
if (field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) &&
action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase))
{
var hasOriginalTitle = !string.IsNullOrWhiteSpace(facts.OriginalTitle);
attempts.Add(new DebugResolutionAttempt(action, "OriginalTitle", "facts.original_title", hasOriginalTitle, hasOriginalTitle, facts.OriginalTitle, hasOriginalTitle ? "Original title found" : "Original title missing"));
if (hasOriginalTitle)
{
var result = new ItemsProxyResolvedText(true, facts.OriginalTitle);
return new DebugTextResolution(field, actions.ToArray(), attempts.ToArray(), result, action, "Original title found");
}
continue;
}
if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase))
{
var originalValue = ItemsProxyTransformer.GetTranslatedField(byLang, OriginalAction, field);
var hasOriginalValue = !string.IsNullOrWhiteSpace(originalValue);
attempts.Add(new DebugResolutionAttempt(action, "Translation", "Original/" + field, hasOriginalValue, hasOriginalValue, originalValue, hasOriginalValue ? "Original-language translation found" : "Original-language translation missing"));
if (hasOriginalValue)
{
var result = new ItemsProxyResolvedText(true, originalValue);
return new DebugTextResolution(field, actions.ToArray(), attempts.ToArray(), result, action, "Original-language translation found");
}
continue;
}
if (!action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase))
{
attempts.Add(new DebugResolutionAttempt(action, "Unknown", string.Empty, false, false, null, "Unsupported action token"));
continue;
}
var lang = action[LanguagePrefix.Length..];
var value = ItemsProxyTransformer.GetTranslatedField(byLang, lang, field);
var hasValue = !string.IsNullOrWhiteSpace(value);
attempts.Add(new DebugResolutionAttempt(action, "Translation", lang + "/" + field, hasValue, hasValue, value, hasValue ? "Translation found" : "Translation missing or empty"));
if (hasValue)
{
var result = new ItemsProxyResolvedText(true, value);
return new DebugTextResolution(field, actions.ToArray(), attempts.ToArray(), result, action, "Translation found");
}
}
var fallback = field.Equals(TitleField, StringComparison.OrdinalIgnoreCase)
? new ItemsProxyResolvedText(false, null)
: new ItemsProxyResolvedText(true, string.Empty);
var reason = field.Equals(TitleField, StringComparison.OrdinalIgnoreCase)
? "No action had data; title falls back to Jellyfin"
: "No action had data; field is cleared";
return new DebugTextResolution(field, actions.ToArray(), attempts.ToArray(), fallback, field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) ? JellyfinAction : ClearAction, reason);
} }
private static DebugAssetResolution TraceResolveAsset( private static DebugAssetResolution TraceResolveAsset(
string kind, string kind, IReadOnlyList<string> actions,
IReadOnlyList<string> actions,
Dictionary<string, Dictionary<string, string>>? byKind) Dictionary<string, Dictionary<string, string>>? byKind)
{ {
var attempts = new List<DebugResolutionAttempt>(); var attempts = new List<ItemsProxyResolutionAttempt>();
foreach (var action in actions) var result = ItemsProxyTransformer.ResolveAsset(kind, actions, byKind, attempts.Add);
{ var chosen = attempts.FirstOrDefault(attempt => attempt.Chosen);
if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)) return new(kind, actions.ToArray(), attempts.ToArray(), result,
{ chosen?.Action ?? (result.Change ? ClearAction : JellyfinAction),
var result = new ItemsProxyResolvedAsset(false, null); chosen?.Reason ?? (result.Change ? "No action had data; image is cleared" : "No action had data; use Jellyfin"));
attempts.Add(new DebugResolutionAttempt(action, JellyfinAction, string.Empty, true, true, null, "Use Jellyfin image"));
return new DebugAssetResolution(kind, actions.ToArray(), attempts.ToArray(), result, JellyfinAction, "Jellyfin action reached");
}
if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase))
{
var originalValue = ItemsProxyTransformer.GetAsset(byKind, kind, OriginalAction);
var hasOriginalValue = !string.IsNullOrWhiteSpace(originalValue);
attempts.Add(new DebugResolutionAttempt(action, "Asset", kind + "/Original", hasOriginalValue, hasOriginalValue, originalValue, hasOriginalValue ? "Original-language asset found" : "Original-language asset missing"));
if (hasOriginalValue)
{
var result = new ItemsProxyResolvedAsset(true, originalValue);
return new DebugAssetResolution(kind, actions.ToArray(), attempts.ToArray(), result, action, "Original-language asset found");
}
continue;
}
if (!action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase))
{
attempts.Add(new DebugResolutionAttempt(action, "Unknown", string.Empty, false, false, null, "Unsupported action token"));
continue;
}
var lang = action[LanguagePrefix.Length..];
var value = ItemsProxyTransformer.GetAsset(byKind, kind, lang);
var hasValue = !string.IsNullOrWhiteSpace(value);
attempts.Add(new DebugResolutionAttempt(action, "Asset", kind + "/" + lang, hasValue, hasValue, value, hasValue ? "Asset found" : "Asset missing"));
if (hasValue)
{
var result = new ItemsProxyResolvedAsset(true, value);
return new DebugAssetResolution(kind, actions.ToArray(), attempts.ToArray(), result, action, "Asset found");
}
}
var fallback = kind.Equals(PosterKind, StringComparison.OrdinalIgnoreCase)
? new ItemsProxyResolvedAsset(false, null)
: new ItemsProxyResolvedAsset(true, string.Empty);
var reason = kind.Equals(PosterKind, StringComparison.OrdinalIgnoreCase)
? "No action had data; poster falls back to Jellyfin"
: "No action had data; image is cleared";
return new DebugAssetResolution(kind, actions.ToArray(), attempts.ToArray(), fallback, kind.Equals(PosterKind, StringComparison.OrdinalIgnoreCase) ? JellyfinAction : ClearAction, reason);
} }
private static object[] BuildMissingDataReport( private static object[] BuildMissingDataReport(
@@ -1025,238 +938,7 @@ public sealed class MultilangController : ControllerBase
} }
private UserRulesDocument GetRulesOrDefault(string userId) private UserRulesDocument GetRulesOrDefault(string userId)
{ => _store.GetUserRules(userId) ?? new UserRulesDocument { Enabled = false };
var existing = _store.GetUserRules(userId);
if (existing is not null)
{
existing.Categories ??= [];
existing.FallbackFieldActions = NormalizeActionLists(existing.FallbackFieldActions, (Plugin.Instance?.Configuration?.Languages ?? []).ToHashSet(StringComparer.OrdinalIgnoreCase));
existing.Categories = NormalizeCategories(existing.Categories, (Plugin.Instance?.Configuration?.Languages ?? []).ToHashSet(StringComparer.OrdinalIgnoreCase));
return existing;
}
return new UserRulesDocument
{
Enabled = false,
SortLocale = "Auto",
TrustTmdbCollections = true,
FallbackFieldActions = DefaultActionLists(),
Categories = []
};
}
private static UserCategoryRule[] NormalizeCategories(IEnumerable<UserCategoryRule>? categories, IReadOnlySet<string> allowedLanguages)
=> (categories ?? [])
.Select(category => new UserCategoryRule
{
Id = string.IsNullOrWhiteSpace(category.Id) ? Guid.NewGuid().ToString("N") : category.Id.Trim(),
Label = string.IsNullOrWhiteSpace(category.Label) ? "Category" : category.Label.Trim(),
CriteriaText = category.CriteriaText?.Trim() ?? string.Empty,
Requirements = NormalizeRequirements(category.Requirements),
MatchAllConditions = category.MatchAllConditions,
Scopes = (category.Scopes ?? [])
.Where(scope => scope is "M" or "S" or "C")
.Distinct(StringComparer.OrdinalIgnoreCase)
.DefaultIfEmpty("M")
.ToArray(),
FieldActions = NormalizeFieldActions(category.FieldActions, allowedLanguages),
FieldActionLists = NormalizeActionLists(category.FieldActionLists, allowedLanguages, category.FieldActions)
})
.ToArray();
private static UserCategoryRequirement[] NormalizeRequirements(IEnumerable<UserCategoryRequirement>? requirements)
=> (requirements ?? [])
.Select(requirement =>
{
var fields = NormalizeRequirementFields(requirement.Fields, requirement.Field);
return new UserCategoryRequirement
{
Fields = fields,
Field = fields.FirstOrDefault() ?? string.Empty,
UseFieldOr = requirement.UseFieldOr,
Relation = NormalizeRequirementRelation(requirement.Relation),
UseOr = requirement.UseOr,
Values = (requirement.Values ?? [])
.Where(v => !string.IsNullOrWhiteSpace(v))
.Select(v => v.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray()
};
})
.Where(requirement =>
requirement.Fields.Length > 0 &&
requirement.Relation.Length > 0 &&
requirement.Values.Length > 0)
.ToArray();
private static string[] NormalizeRequirementFields(IEnumerable<string>? fields, string? legacyField)
{
var requested = (fields ?? [])
.Append(legacyField ?? string.Empty)
.Select(NormalizeRequirementField)
.Where(field => field.Length > 0)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var result = new List<string>();
string? kind = null;
foreach (var field in RequirementFieldOrder)
{
if (!requested.Contains(field))
continue;
var currentKind = IsCountryRequirementField(field) ? "country" : "language";
kind ??= currentKind;
if (currentKind == kind)
result.Add(field);
}
return result.ToArray();
}
private static string NormalizeRequirementField(string? field)
{
var normalized = (field ?? string.Empty).Trim().ToLowerInvariant();
return normalized is "original_language" or "spoken_languages" or "audio_language" or "origin_countries" or "production_countries"
? normalized
: string.Empty;
}
private static bool IsCountryRequirementField(string field)
=> field.Equals("origin_countries", StringComparison.OrdinalIgnoreCase) ||
field.Equals("production_countries", StringComparison.OrdinalIgnoreCase);
private static string NormalizeRequirementRelation(string? relation)
{
var normalized = (relation ?? string.Empty).Trim().ToLowerInvariant();
return normalized is "is" or "is_not" or "contains" or "not_contains"
? normalized
: string.Empty;
}
private static Dictionary<string, string[]> NormalizeActionLists(
IReadOnlyDictionary<string, string[]>? actions,
IReadOnlySet<string> allowedLanguages,
IReadOnlyDictionary<string, string>? legacyActions = null)
{
if (legacyActions is not null && LooksLikeDefaultActionLists(actions) && HasMeaningfulLegacyActions(legacyActions))
actions = null;
var result = DefaultActionLists();
foreach (var field in result.Keys.ToArray())
{
var raw = actions is not null && actions.TryGetValue(field, out var configured)
? configured
: LegacyActionToList(field, legacyActions);
result[field] = NormalizeActionList(field, raw, allowedLanguages);
}
return result;
}
private static bool LooksLikeDefaultActionLists(IReadOnlyDictionary<string, string[]>? actions)
=> actions is null ||
DefaultActionLists().Keys.All(field =>
actions.TryGetValue(field, out var list) &&
list.Length == 1 &&
list[0].Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase));
private static bool HasMeaningfulLegacyActions(IReadOnlyDictionary<string, string> legacyActions)
=> legacyActions.Values.Any(action =>
!string.IsNullOrWhiteSpace(action) &&
!action.Equals(FallbackAction, StringComparison.OrdinalIgnoreCase) &&
!action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase));
private static string[] LegacyActionToList(string field, IReadOnlyDictionary<string, string>? legacyActions)
{
if (legacyActions is null || !legacyActions.TryGetValue(field, out var action))
return [JellyfinAction];
if (string.IsNullOrWhiteSpace(action) || action.Equals(FallbackAction, StringComparison.OrdinalIgnoreCase))
return [JellyfinAction];
return [action];
}
private static Dictionary<string, string> NormalizeFieldActions(
IReadOnlyDictionary<string, string>? actions,
IReadOnlySet<string> allowedLanguages)
{
var result = DefaultFieldActions();
if (actions is null)
return result;
foreach (var field in result.Keys.ToArray())
{
if (!actions.TryGetValue(field, out var raw))
continue;
result[field] = NormalizeAction(field, raw, allowedLanguages);
}
return result;
}
private static string NormalizeAction(string field, string? raw, IReadOnlySet<string> allowedLanguages)
{
var action = (raw ?? string.Empty).Trim();
if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase) ||
action.Equals(FallbackAction, StringComparison.OrdinalIgnoreCase))
return action;
if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase))
return OriginalAction;
if ((field is OverviewField or TaglineField or LogoKind) &&
action.Equals(ClearAction, StringComparison.OrdinalIgnoreCase))
return ClearAction;
if (action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase))
{
var lang = action[LanguagePrefix.Length..].Trim();
var canonical = allowedLanguages.FirstOrDefault(l => l.Equals(lang, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrWhiteSpace(canonical))
return LanguagePrefix + canonical;
}
return FallbackAction;
}
private static string[] NormalizeActionList(string field, IEnumerable<string>? raw, IReadOnlySet<string> allowedLanguages)
{
var result = new List<string>();
foreach (var action in raw ?? [])
{
var normalized = NormalizeActionToken(field, action, allowedLanguages);
if (normalized.Length == 0)
continue;
if (result.Contains(normalized, StringComparer.OrdinalIgnoreCase))
continue;
result.Add(normalized);
}
if (RequiresJellyfinFallback(field))
{
if (!result.Contains(JellyfinAction, StringComparer.OrdinalIgnoreCase))
result.Add(JellyfinAction);
}
return result.ToArray();
}
private static string NormalizeActionToken(string field, string? raw, IReadOnlySet<string> allowedLanguages)
{
var action = (raw ?? string.Empty).Trim();
if (action.Equals("J", StringComparison.OrdinalIgnoreCase))
return JellyfinAction;
if (action.Equals("O", StringComparison.OrdinalIgnoreCase))
return OriginalAction;
if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase))
return JellyfinAction;
if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase))
return OriginalAction;
var lang = action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)
? action[LanguagePrefix.Length..].Trim()
: action;
var canonical = allowedLanguages.FirstOrDefault(l => l.Equals(lang, StringComparison.OrdinalIgnoreCase));
return string.IsNullOrWhiteSpace(canonical) ? string.Empty : LanguagePrefix + canonical;
}
private IActionResult ServeEmbedded(string resource, string contentType) private IActionResult ServeEmbedded(string resource, string contentType)
{ {
@@ -1314,24 +996,16 @@ public sealed class MultilangController : ControllerBase
private async Task<bool> IsRequesterAdminAsync() private async Task<bool> IsRequesterAdminAsync()
{ {
var info = await _authorizationContext.GetAuthorizationInfo(HttpContext).ConfigureAwait(false);
if (info?.User is not null && TryGetIsAdmin(info.User, out var authInfoAdmin))
return authInfoAdmin;
var userId = await GetUserIdAsync().ConfigureAwait(false); var userId = await GetUserIdAsync().ConfigureAwait(false);
if (string.IsNullOrWhiteSpace(userId) || !Guid.TryParseExact(userId, "N", out var guid)) return Guid.TryParse(userId, out var guid) &&
return false; _userManager.GetUserById(guid)?.HasPermission(PermissionKind.IsAdministrator) == true;
}
var user = _userManager.GetUserById(guid); private bool CanAccessItem(Guid itemId, string userId)
if (user is null) {
return false; var user = _userManager.GetUserById(Guid.Parse(userId));
return user is not null && !user.HasPermission(PermissionKind.IsDisabled) &&
if (TryGetIsAdmin(user, out var isAdmin)) _libraryManager.GetItemById(itemId)?.IsVisibleStandalone(user) == true;
return isAdmin;
var remoteEndpoint = HttpContext.Connection.RemoteIpAddress?.ToString() ?? string.Empty;
var dto = _userManager.GetUserDto(user, remoteEndpoint);
return dto.Policy?.IsAdministrator == true;
} }
private static bool FormBool(IFormCollection form, string key, bool fallback) private static bool FormBool(IFormCollection form, string key, bool fallback)
@@ -1344,28 +1018,6 @@ public sealed class MultilangController : ControllerBase
value.Equals("on", StringComparison.OrdinalIgnoreCase); value.Equals("on", StringComparison.OrdinalIgnoreCase);
} }
private static bool TryGetIsAdmin(object user, out bool isAdmin)
{
isAdmin = false;
var policy = user.GetType().GetProperty("Policy")?.GetValue(user);
var policyAdmin = policy?.GetType().GetProperty("IsAdministrator")?.GetValue(policy);
if (policyAdmin is bool isPolicyAdmin)
{
isAdmin = isPolicyAdmin;
return true;
}
var directAdmin = user.GetType().GetProperty("IsAdministrator")?.GetValue(user);
if (directAdmin is bool isDirectAdmin)
{
isAdmin = isDirectAdmin;
return true;
}
return false;
}
private string? GetTokenFromRequest() private string? GetTokenFromRequest()
{ {
var req = HttpContext?.Request; var req = HttpContext?.Request;
@@ -276,7 +276,7 @@
function renderJellyfinTitleLanguages() { function renderJellyfinTitleLanguages() {
var select = document.getElementById("ml-jellyfin-title-language"); var select = document.getElementById("ml-jellyfin-title-language");
var selected = M.prop(state.config, "JellyfinTitleLanguageFallback", "jellyfinTitleLanguageFallback", ""); var selected = M.prop(state.config, "JellyfinTitleLanguageFallback", "jellyfinTitleLanguageFallback", "");
select.replaceChildren(); M.replaceChildren(select);
var defaultOption = make("option", "", "Jellyfin's default"); var defaultOption = make("option", "", "Jellyfin's default");
defaultOption.value = ""; defaultOption.value = "";
select.appendChild(defaultOption); select.appendChild(defaultOption);
@@ -317,7 +317,7 @@
} }
function renderProviderInputs() { function renderProviderInputs() {
providerInputs.replaceChildren(); M.replaceChildren(providerInputs);
state.providers.forEach(function (provider) { state.providers.forEach(function (provider) {
var id = providerId(provider); var id = providerId(provider);
var keyField = providerKeyField(provider); var keyField = providerKeyField(provider);
@@ -371,7 +371,7 @@
.map(providerId) .map(providerId)
.filter(function (id) { return providerSupports(providerById(id), bucket) && orderFor(id, bucket) >= 0; }) .filter(function (id) { return providerSupports(providerById(id), bucket) && orderFor(id, bucket) >= 0; })
.sort(function (a, b) { return orderFor(a, bucket) - orderFor(b, bucket); }); .sort(function (a, b) { return orderFor(a, bucket) - orderFor(b, bucket); });
list.replaceChildren(); M.replaceChildren(list);
ids.forEach(function (id) { list.appendChild(makeProviderChip(id)); }); ids.forEach(function (id) { list.appendChild(makeProviderChip(id)); });
} }
@@ -412,7 +412,7 @@
} }
function renderArticles() { function renderArticles() {
articleList.replaceChildren(); M.replaceChildren(articleList);
var configured = new Map(); var configured = new Map();
(state.config.ArticleEntries || []).forEach(function (entry) { (state.config.ArticleEntries || []).forEach(function (entry) {
var lang = normalizeLang(entry.Language || entry.language); var lang = normalizeLang(entry.Language || entry.language);
@@ -541,7 +541,7 @@
function renderCacheEntries(entries, error) { function renderCacheEntries(entries, error) {
var root = document.getElementById("ml-cache-entries"); var root = document.getElementById("ml-cache-entries");
root.replaceChildren(); M.replaceChildren(root);
if (error) { if (error) {
setDiagnosticsSummary("ml-cache-entries-summary", "Cached responses"); setDiagnosticsSummary("ml-cache-entries-summary", "Cached responses");
root.appendChild(make("div", "ml-error", error)); root.appendChild(make("div", "ml-error", error));
@@ -576,7 +576,7 @@
function renderProxyRequests(entries, error) { function renderProxyRequests(entries, error) {
var root = document.getElementById("ml-proxy-requests"); var root = document.getElementById("ml-proxy-requests");
root.replaceChildren(); M.replaceChildren(root);
if (error) { if (error) {
setDiagnosticsSummary("ml-proxy-requests-summary", "Recent proxy requests"); setDiagnosticsSummary("ml-proxy-requests-summary", "Recent proxy requests");
root.appendChild(make("div", "ml-error", error)); root.appendChild(make("div", "ml-error", error));
@@ -674,9 +674,9 @@
var scanRoot = document.getElementById("ml-refresh-scan"); var scanRoot = document.getElementById("ml-refresh-scan");
var queueRoot = document.getElementById("ml-refresh-queue"); var queueRoot = document.getElementById("ml-refresh-queue");
var httpRoot = document.getElementById("ml-provider-http"); var httpRoot = document.getElementById("ml-provider-http");
scanRoot.replaceChildren(); M.replaceChildren(scanRoot);
queueRoot.replaceChildren(); M.replaceChildren(queueRoot);
httpRoot.replaceChildren(); M.replaceChildren(httpRoot);
if (error) { if (error) {
setDiagnosticsSummary("ml-refresh-scan-summary", "Latest scan"); setDiagnosticsSummary("ml-refresh-scan-summary", "Latest scan");
@@ -728,18 +728,18 @@
var scanError = String(prop(scan, "Error", "error", "") || ""); var scanError = String(prop(scan, "Error", "error", "") || "");
if (scanError) scanRoot.appendChild(make("div", "ml-error", scanError)); if (scanError) scanRoot.appendChild(make("div", "ml-error", scanError));
var active = prop(queue, "Active", "active", null); var activeItems = queue.ActiveItems || [];
var queued = prop(queue, "Queued", "queued", []) || []; var queued = prop(queue, "Queued", "queued", []) || [];
var recent = prop(queue, "Recent", "recent", []) || []; var recent = prop(queue, "Recent", "recent", []) || [];
setDiagnosticsSummary("ml-refresh-queue-summary", "Queue (" + Number(prop(queue, "QueuedCount", "queuedCount", queued.length) || 0) + " pending)"); setDiagnosticsSummary("ml-refresh-queue-summary", "Queue (" + Number(prop(queue, "QueuedCount", "queuedCount", queued.length) || 0) + " pending)");
if (active) { activeItems.forEach(function (active) {
queueRoot.appendChild(makeDiagnosticRow("Active item: " + String(prop(active, "ItemId", "itemId", "")), [ queueRoot.appendChild(makeDiagnosticRow("Active item: " + String(prop(active, "ItemId", "itemId", "")), [
"running now", "running now",
"source: " + String(prop(active, "SourceTier", "sourceTier", "")), "source: " + String(prop(active, "SourceTier", "sourceTier", "")),
"type: " + String(prop(active, "WorkClass", "workClass", "")), "type: " + String(prop(active, "WorkClass", "workClass", "")),
"refresh: " + String(prop(active, "JobType", "jobType", "")) "refresh: " + String(prop(active, "JobType", "jobType", ""))
])); ]));
} });
if (queued.length === 0) { if (queued.length === 0) {
queueRoot.appendChild(make("div", "ml-muted", "No pending refresh items.")); queueRoot.appendChild(make("div", "ml-muted", "No pending refresh items."));
} else { } else {
@@ -185,7 +185,7 @@
} }
function renderSummary(data) { function renderSummary(data) {
summary.replaceChildren(); M.replaceChildren(summary);
if (mode.value !== "resolved") return; if (mode.value !== "resolved") return;
var top = make("div", "ml-diagnostic-meta"); var top = make("div", "ml-diagnostic-meta");
top.append( top.append(
@@ -207,7 +207,7 @@
M.setStatus(status, "Loaded."); M.setStatus(status, "Loaded.");
} catch (err) { } catch (err) {
lastJson = ""; lastJson = "";
summary.replaceChildren(); M.replaceChildren(summary);
output.textContent = authHint(err); output.textContent = authHint(err);
M.setStatus(status, err.message, true); M.setStatus(status, err.message, true);
} }
@@ -1,11 +1,4 @@
(function () { (function () {
if (!Element.prototype.replaceChildren) {
Element.prototype.replaceChildren = function () {
this.textContent = "";
for (let index = 0; index < arguments.length; index++) this.appendChild(arguments[index]);
};
}
const basePath = () => { const basePath = () => {
const path = window.location.pathname || ""; const path = window.location.pathname || "";
const marker = "/web/"; const marker = "/web/";
@@ -21,18 +14,6 @@
} }
} }
function headers() {
const qs = new URLSearchParams(location.search);
const t = token() || qs.get("token") || qs.get("api_key") || "";
const h = { "Content-Type": "application/json" };
if (t) {
h.Authorization = `MediaBrowser Token="${t}"`;
h["X-Emby-Token"] = t;
h["X-MediaBrowser-Token"] = t;
}
return h;
}
function authHeaders(contentType) { function authHeaders(contentType) {
const qs = new URLSearchParams(location.search); const qs = new URLSearchParams(location.search);
const t = token() || qs.get("token") || qs.get("api_key") || ""; const t = token() || qs.get("token") || qs.get("api_key") || "";
@@ -58,7 +39,7 @@
async function request(path, options) { async function request(path, options) {
const response = await fetch(`${basePath()}${path}`, { const response = await fetch(`${basePath()}${path}`, {
cache: "no-store", cache: "no-store",
headers: headers(), headers: authHeaders("application/json"),
...options ...options
}); });
if (!response.ok) throw new Error(`HTTP ${response.status}`); if (!response.ok) throw new Error(`HTTP ${response.status}`);
@@ -113,41 +94,9 @@
URL.revokeObjectURL(url); URL.revokeObjectURL(url);
} }
function chip(label) { function replaceChildren(parent, ...children) {
const el = document.createElement("span"); parent.textContent = "";
el.className = "ml-chip"; parent.append(...children);
el.textContent = label;
el.draggable = true;
el.dataset.value = label;
el.addEventListener("dragstart", (event) => {
event.dataTransfer.setData("text/plain", label);
event.dataTransfer.effectAllowed = "move";
el.classList.add("ml-dragging");
});
el.addEventListener("dragend", () => el.classList.remove("ml-dragging"));
return el;
}
function bucketValues(bucket) {
return [...bucket.querySelectorAll(".ml-chip")].map((el) => el.dataset.value || el.textContent.trim()).filter(Boolean);
}
function renderBucket(bucket, values) {
bucket.replaceChildren(...values.map(chip));
}
function wireBucket(bucket, onDrop) {
bucket.addEventListener("dragover", (event) => {
event.preventDefault();
bucket.classList.add("ml-over");
});
bucket.addEventListener("dragleave", () => bucket.classList.remove("ml-over"));
bucket.addEventListener("drop", (event) => {
event.preventDefault();
bucket.classList.remove("ml-over");
const value = event.dataTransfer.getData("text/plain");
if (value) onDrop(value, bucket);
});
} }
const staticData = { const staticData = {
@@ -169,10 +118,7 @@
showModal, showModal,
hideModals, hideModals,
downloadBlob, downloadBlob,
chip, replaceChildren,
bucketValues,
renderBucket,
wireBucket,
staticData staticData
}; };
})(); })();
@@ -109,8 +109,19 @@
var ISO3166 = M.staticData.iso3166; var ISO3166 = M.staticData.iso3166;
var LANG_REGION_COMBOS = M.staticData.langRegionCombos; var LANG_REGION_COMBOS = M.staticData.langRegionCombos;
var listeners = [];
function listen(target, type, handler, capture) {
target.addEventListener(type, handler, capture);
listeners.push(() => target.removeEventListener(type, handler, capture));
}
page._mlDispose = function () {
listeners.forEach(remove => remove());
listeners = [];
cleanupActionDrag();
};
function $(id) { function $(id) {
return document.getElementById(id); return page.querySelector("#" + id);
} }
function make(tag, className, text) { function make(tag, className, text) {
@@ -144,18 +155,18 @@
} }
function categoryId(category) { function categoryId(category) {
return category.Id || category.id || ""; return category.Id;
} }
function categoryLabel(category, index) { function categoryLabel(category, index) {
var label = String(category.Label || category.label || "").trim(); var label = String(category.Label || "").trim();
return label || categoryPlaceholder(index); return label || categoryPlaceholder(index);
} }
function categoryPlaceholder(index) { function categoryPlaceholder(index) {
var used = {}; var used = {};
categories().forEach(function (category) { categories().forEach(function (category) {
var label = String(category.Label || category.label || "").trim().toLowerCase(); var label = String(category.Label || "").trim().toLowerCase();
if (label) used[label] = true; if (label) used[label] = true;
}); });
var n = Math.max(1, index + 1); var n = Math.max(1, index + 1);
@@ -163,11 +174,10 @@
return "Category " + n; return "Category " + n;
} }
function newCategory(label, criteria) { function newCategory(label) {
return { return {
Id: randomId(), Id: randomId(),
Label: label || "", Label: label || "",
CriteriaText: criteria || "",
Requirements: [emptyRequirement()], Requirements: [emptyRequirement()],
MatchAllConditions: true, MatchAllConditions: true,
Scopes: ["M", "S", "C"], Scopes: ["M", "S", "C"],
@@ -176,7 +186,7 @@
} }
function emptyRequirement() { function emptyRequirement() {
return { Field: "", Fields: [], UseFieldOr: false, Relation: "", UseOr: false, Values: [] }; return { Fields: [], UseFieldOr: false, Relation: "", UseOr: false, Values: [] };
} }
function normalizeActionValue(fieldId, value) { function normalizeActionValue(fieldId, value) {
@@ -189,13 +199,6 @@
return canonical ? "Language:" + canonical : ""; return canonical ? "Language:" + canonical : "";
} }
function legacyActionToList(fieldId, legacy) {
if (!legacy || legacy.toLowerCase() === "fallback") {
return ["Jellyfin"];
}
return [legacy];
}
function normalizeActionList(fieldId, values) { function normalizeActionList(fieldId, values) {
var field = fields.find(function (f) { return f.id === fieldId; }); var field = fields.find(function (f) { return f.id === fieldId; });
var result = []; var result = [];
@@ -211,64 +214,31 @@
return result.length || !(field && field.lockedJ) ? result : ["Jellyfin"]; return result.length || !(field && field.lockedJ) ? result : ["Jellyfin"];
} }
function normalizeActionMap(source, legacy) { function normalizeActionMap(source) {
if (legacy && looksLikeDefaultActionMap(source) && hasMeaningfulLegacyActions(legacy)) {
source = null;
}
var result = {}; var result = {};
fields.forEach(function (field) { fields.forEach(function (field) {
var values = source && (source[field.id] || source[field.label]); result[field.id] = normalizeActionList(field.id, source[field.id]);
if (!values && legacy) values = legacyActionToList(field.id, legacy[field.id] || legacy[field.label]);
result[field.id] = normalizeActionList(field.id, Array.isArray(values) ? values : values ? [values] : ["Jellyfin"]);
}); });
return result; return result;
} }
function looksLikeDefaultActionMap(source) {
if (!source) return true;
return fields.every(function (field) {
var values = source[field.id] || source[field.label];
return Array.isArray(values) && values.length === 1 && String(values[0]).toLowerCase() === "jellyfin";
});
}
function hasMeaningfulLegacyActions(legacy) {
return Object.keys(legacy || {}).some(function (key) {
var action = String(legacy[key] || "").toLowerCase();
return action && action !== "fallback" && action !== "jellyfin";
});
}
function normalizeRules() { function normalizeRules() {
state.rules = state.rules || {}; state.rules.Categories.forEach(function (category) {
state.rules.Categories = (state.rules.Categories || []).filter(function (category) { category.Requirements = normalizeRequirements(category.Requirements);
var id = categoryId(category).toLowerCase(); category.FieldActionLists = normalizeActionMap(category.FieldActionLists);
var label = String(category.Label || category.label || "").trim().toLowerCase();
var criteria = String(category.CriteriaText || category.criteriaText || "").trim();
return !(id === "fallback" && label === "fallback" && criteria.length === 0);
}).map(function (category, index) {
category.Id = categoryId(category) || randomId();
category.Label = String(category.Label || category.label || "").trim();
category.CriteriaText = category.CriteriaText || category.criteriaText || "";
category.Requirements = normalizeRequirements(category.Requirements || category.requirements);
category.MatchAllConditions = category.MatchAllConditions !== false && category.matchAllConditions !== false;
category.Scopes = category.Scopes || category.scopes || ["M", "S", "C"];
category.FieldActionLists = normalizeActionMap(category.FieldActionLists || category.fieldActionLists, category.FieldActions || category.fieldActions);
return category;
}); });
state.rules.FallbackFieldActions = normalizeActionMap(state.rules.FallbackFieldActions || state.rules.fallbackFieldActions, null); state.rules.FallbackFieldActions = normalizeActionMap(state.rules.FallbackFieldActions);
} }
function normalizeRequirements(requirements) { function normalizeRequirements(requirements) {
var normalized = (requirements || []).map(function (requirement) { var normalized = (requirements || []).map(function (requirement) {
var fields = normalizeRequirementFields(requirement.Fields || requirement.fields, requirement.Field || requirement.field); var fields = normalizeRequirementFields(requirement.Fields);
return { return {
Field: fields[0] || "",
Fields: fields, Fields: fields,
UseFieldOr: requirement.UseFieldOr === true || requirement.useFieldOr === true, UseFieldOr: requirement.UseFieldOr === true,
Relation: String(requirement.Relation || requirement.relation || "").trim(), Relation: String(requirement.Relation || "").trim(),
UseOr: requirement.UseOr === true || requirement.useOr === true, UseOr: requirement.UseOr === true,
Values: orderedRequirementValues(fields[0] || "", requirement.Values || requirement.values || []) Values: orderedRequirementValues(fields[0] || "", requirement.Values || [])
}; };
}); });
return normalized.length ? normalized : [emptyRequirement()]; return normalized.length ? normalized : [emptyRequirement()];
@@ -294,14 +264,12 @@
return criteriaFields.find(function (item) { return item.id === id; }); return criteriaFields.find(function (item) { return item.id === id; });
} }
function normalizeRequirementFields(values, legacy) { function normalizeRequirementFields(values) {
var raw = []; var raw = [];
(values || []).forEach(function (value) { (values || []).forEach(function (value) {
value = String(value || "").trim(); value = String(value || "").trim();
if (value) raw.push(value); if (value) raw.push(value);
}); });
legacy = String(legacy || "").trim();
if (legacy) raw.push(legacy);
var wanted = {}; var wanted = {};
raw.forEach(function (value) { raw.forEach(function (value) {
var id = value.toLowerCase(); var id = value.toLowerCase();
@@ -319,7 +287,7 @@
} }
function requirementFields(requirement) { function requirementFields(requirement) {
return normalizeRequirementFields(requirement && (requirement.Fields || requirement.fields), requirement && (requirement.Field || requirement.field)); return normalizeRequirementFields(requirement.Fields);
} }
function criteriaLabel(id) { function criteriaLabel(id) {
@@ -329,7 +297,7 @@
function renderSortLocales() { function renderSortLocales() {
var select = $("ml-sort-locale"); var select = $("ml-sort-locale");
select.replaceChildren(option("Auto", "Auto", false)); M.replaceChildren(select, option("Auto", "Auto", false));
state.allowed.forEach(function (lang) { state.allowed.forEach(function (lang) {
select.appendChild(option(lang, lang, false)); select.appendChild(option(lang, lang, false));
}); });
@@ -363,7 +331,6 @@
function syncCategoryFromRow(row, category) { function syncCategoryFromRow(row, category) {
category.Label = row.querySelector("[data-category-label]").value; category.Label = row.querySelector("[data-category-label]").value;
category.MatchAllConditions = row.querySelector("[data-category-match-all]").checked; category.MatchAllConditions = row.querySelector("[data-category-match-all]").checked;
category.CriteriaText = categoryCriteriaText(category);
category.Scopes = Array.prototype.slice.call(row.querySelectorAll("[data-category-scope]:checked")) category.Scopes = Array.prototype.slice.call(row.querySelectorAll("[data-category-scope]:checked"))
.map(function (input) { return input.value; }); .map(function (input) { return input.value; });
} }
@@ -376,7 +343,7 @@
} }
function addSelectOptions(select, items, selectedValue) { function addSelectOptions(select, items, selectedValue) {
select.replaceChildren(); M.replaceChildren(select);
items.forEach(function (item) { items.forEach(function (item) {
select.appendChild(option(item.label, item.id, item.id === selectedValue)); select.appendChild(option(item.label, item.id, item.id === selectedValue));
}); });
@@ -462,23 +429,6 @@
return values.join(joiner); return values.join(joiner);
} }
function requirementText(requirement) {
if (!completeRequirement(requirement)) return "";
var values = orderedRequirementValues(requirementFields(requirement)[0] || "", requirement.Values || []);
var right = values.length === 1 ? values[0] : "(" + values.join(requirement.UseOr ? " or " : " and ") + ")";
var fields = requirementFields(requirement);
var left = fields.length === 1 ? fields[0] : "(" + fields.join(requirement.UseFieldOr ? " or " : " and ") + ")";
return left + " " + requirement.Relation + " " + right;
}
function categoryCriteriaText(category) {
var joiner = category.MatchAllConditions === false ? " or " : " and ";
return (category.Requirements || [])
.filter(completeRequirement)
.map(requirementText)
.join(joiner);
}
function makeRequirementRow(category, requirement, index) { function makeRequirementRow(category, requirement, index) {
var row = make("div", "ml-requirement-row"); var row = make("div", "ml-requirement-row");
var selectedFields = requirementFields(requirement); var selectedFields = requirementFields(requirement);
@@ -524,7 +474,7 @@
var current = requirement.Relation || ""; var current = requirement.Relation || "";
var fields = requirementFields(requirement); var fields = requirementFields(requirement);
var multiple = fields.length > 1; var multiple = fields.length > 1;
select.replaceChildren(option("Choose relation", "", !current)); M.replaceChildren(select, option("Choose relation", "", !current));
if (fields.length) { if (fields.length) {
relations.forEach(function (item) { relations.forEach(function (item) {
select.appendChild(option(multiple ? item.pluralLabel : item.label, item.id, item.id === current)); select.appendChild(option(multiple ? item.pluralLabel : item.label, item.id, item.id === current));
@@ -569,7 +519,6 @@
fields = normalizeRequirementFields(fields, ""); fields = normalizeRequirementFields(fields, "");
var nextKind = fields.length ? fieldKind(fields[0]) : ""; var nextKind = fields.length ? fieldKind(fields[0]) : "";
requirement.Fields = fields; requirement.Fields = fields;
requirement.Field = fields[0] || "";
button.textContent = fieldSummary(requirement); button.textContent = fieldSummary(requirement);
refreshFieldPanelOptions(panel, requirement); refreshFieldPanelOptions(panel, requirement);
if (onChange) onChange(previousKind, nextKind); if (onChange) onChange(previousKind, nextKind);
@@ -674,7 +623,6 @@
function cloneRequirements(requirements) { function cloneRequirements(requirements) {
return normalizeRequirements(requirements).map(function (requirement) { return normalizeRequirements(requirements).map(function (requirement) {
return { return {
Field: requirement.Field,
Fields: (requirement.Fields || []).slice(), Fields: (requirement.Fields || []).slice(),
UseFieldOr: requirement.UseFieldOr === true, UseFieldOr: requirement.UseFieldOr === true,
Relation: requirement.Relation, Relation: requirement.Relation,
@@ -688,7 +636,6 @@
return { return {
Id: randomId(), Id: randomId(),
Label: "", Label: "",
CriteriaText: categoryCriteriaText(category),
Requirements: cloneRequirements(category.Requirements), Requirements: cloneRequirements(category.Requirements),
MatchAllConditions: category.MatchAllConditions !== false, MatchAllConditions: category.MatchAllConditions !== false,
Scopes: (category.Scopes || ["M", "S", "C"]).slice(), Scopes: (category.Scopes || ["M", "S", "C"]).slice(),
@@ -698,7 +645,7 @@
function renderCategories() { function renderCategories() {
var root = $("ml-categories"); var root = $("ml-categories");
root.replaceChildren(); M.replaceChildren(root);
categories().forEach(function (category, index) { categories().forEach(function (category, index) {
var row = make("div", "ml-classification-rule"); var row = make("div", "ml-classification-rule");
row.dataset.categoryId = categoryId(category); row.dataset.categoryId = categoryId(category);
@@ -809,7 +756,7 @@
function renderActionSource() { function renderActionSource() {
var root = $("ml-action-source"); var root = $("ml-action-source");
root.replaceChildren(); M.replaceChildren(root);
actionSourceValues().forEach(function (value) { actionSourceValues().forEach(function (value) {
root.appendChild(makeActionToken(value, false, true)); root.appendChild(makeActionToken(value, false, true));
}); });
@@ -831,7 +778,7 @@
function renderActionMatrix() { function renderActionMatrix() {
renderActionSource(); renderActionSource();
var root = $("ml-action-matrix"); var root = $("ml-action-matrix");
root.replaceChildren(); M.replaceChildren(root);
actionRows().forEach(function (row) { actionRows().forEach(function (row) {
var line = make("div", "ml-action-category-row"); var line = make("div", "ml-action-category-row");
if (row.fallback) line.classList.add("ml-action-category-row-fallback"); if (row.fallback) line.classList.add("ml-action-category-row-fallback");
@@ -1068,7 +1015,7 @@
function clearCategoryActions(categoryId) { function clearCategoryActions(categoryId) {
boxesForCategory(categoryId).forEach(function (box) { boxesForCategory(categoryId).forEach(function (box) {
box.replaceChildren(); M.replaceChildren(box);
if (actionFieldLocksJ(box.dataset.fieldId)) box.appendChild(makeActionToken("Jellyfin", true, false)); if (actionFieldLocksJ(box.dataset.fieldId)) box.appendChild(makeActionToken("Jellyfin", true, false));
}); });
} }
@@ -1085,7 +1032,7 @@
return token.dataset.value !== "Jellyfin"; return token.dataset.value !== "Jellyfin";
}); });
var token = makeActionToken(value, actionFieldLocksJ(box.dataset.fieldId) && value === "Jellyfin", false); var token = makeActionToken(value, actionFieldLocksJ(box.dataset.fieldId) && value === "Jellyfin", false);
box.replaceChildren(); M.replaceChildren(box);
if (prepend) box.appendChild(token); if (prepend) box.appendChild(token);
existing.forEach(function (item) { box.appendChild(item); }); existing.forEach(function (item) { box.appendChild(item); });
if (!prepend) box.appendChild(token); if (!prepend) box.appendChild(token);
@@ -1128,7 +1075,6 @@
categories().forEach(function (category) { categories().forEach(function (category) {
var complete = (category.Requirements || []).filter(completeRequirement); var complete = (category.Requirements || []).filter(completeRequirement);
category.Requirements = complete.length ? complete : [emptyRequirement()]; category.Requirements = complete.length ? complete : [emptyRequirement()];
category.CriteriaText = complete.map(requirementText).join(" and ");
}); });
return { return {
Enabled: $("ml-enabled").checked, Enabled: $("ml-enabled").checked,
@@ -1140,12 +1086,10 @@
return { return {
Id: category.Id, Id: category.Id,
Label: categoryLabel(category, index), Label: categoryLabel(category, index),
CriteriaText: complete.map(requirementText).join(category.MatchAllConditions === false ? " or " : " and "),
Requirements: complete, Requirements: complete,
MatchAllConditions: category.MatchAllConditions !== false, MatchAllConditions: category.MatchAllConditions !== false,
Scopes: category.Scopes, Scopes: category.Scopes,
FieldActionLists: category.FieldActionLists, FieldActionLists: category.FieldActionLists
FieldActions: {}
}; };
}) })
}; };
@@ -1183,7 +1127,7 @@
$("ml-user-import-modal").hidden = !show; $("ml-user-import-modal").hidden = !show;
if (!show) { if (!show) {
$("ml-user-import-choice").hidden = true; $("ml-user-import-choice").hidden = true;
$("ml-user-import-source").replaceChildren(); M.replaceChildren($("ml-user-import-source"));
} }
} }
@@ -1251,7 +1195,7 @@
} }
var select = $("ml-user-import-source"); var select = $("ml-user-import-source");
select.replaceChildren(); M.replaceChildren(select);
ids.forEach(function (id) { select.appendChild(option(id, id, false)); }); ids.forEach(function (id) { select.appendChild(option(id, id, false)); });
choice.hidden = false; choice.hidden = false;
M.setStatus(status, "No matching user ID was found. Choose which exported user settings to import.", true); M.setStatus(status, "No matching user ID was found. Choose which exported user settings to import.", true);
@@ -1295,26 +1239,26 @@
$("ml-user-import-run").addEventListener("click", function () { $("ml-user-import-run").addEventListener("click", function () {
runUserImport().catch(function (err) { M.setStatus(status, String(err), true); }); runUserImport().catch(function (err) { M.setStatus(status, String(err), true); });
}); });
document.addEventListener("click", function () { closeValuePanels(null); }); listen(document, "click", function () { closeValuePanels(null); });
$("ml-add-category").addEventListener("click", function () { $("ml-add-category").addEventListener("click", function () {
syncCategoriesFromDom(); syncCategoriesFromDom();
updateActionStateFromBoxes(); updateActionStateFromBoxes();
categories().push(newCategory("", "")); categories().push(newCategory(""));
renderAll(); renderAll();
}); });
document.addEventListener("dragstart", function (event) { listen(document, "dragstart", function (event) {
if (event.target.closest && event.target.closest(".ml-action-token")) event.preventDefault(); if (event.target.closest && event.target.closest(".ml-action-token")) event.preventDefault();
}, true); }, true);
page.addEventListener("pointerdown", beginActionDrag, true); page.addEventListener("pointerdown", beginActionDrag, true);
document.addEventListener("pointermove", moveActionDrag, true); listen(document, "pointermove", moveActionDrag, true);
document.addEventListener("pointerup", endActionDrag, true); listen(document, "pointerup", endActionDrag, true);
document.addEventListener("pointercancel", function (event) { listen(document, "pointercancel", function (event) {
if (!actionDrag || event.pointerId !== actionDrag.pointerId) return; if (!actionDrag || event.pointerId !== actionDrag.pointerId) return;
restoreActionDrag(); restoreActionDrag();
cleanupActionDrag(); cleanupActionDrag();
updateActionStateFromBoxes(); updateActionStateFromBoxes();
}, true); }, true);
document.addEventListener("viewshow", function (event) { listen(document, "viewshow", function (event) {
if (event.target && event.target.id === "ml-user") load(); if (event.target && event.target.id === "ml-user") load();
}); });
} }
@@ -42,13 +42,13 @@ public sealed partial class TranslationStore
delete.CommandText = @" delete.CommandText = @"
DELETE FROM facts WHERE item_id = $item_id; DELETE FROM facts WHERE item_id = $item_id;
DELETE FROM translations WHERE item_id = $item_id; DELETE FROM translations WHERE item_id = $item_id;
DELETE FROM assets WHERE item_id = $item_id;"; DELETE FROM assets WHERE item_id = $item_id;
DELETE FROM fetch_state WHERE item_id = $item_id;";
delete.Parameters.AddWithValue("$item_id", itemId); delete.Parameters.AddWithValue("$item_id", itemId);
delete.ExecuteNonQuery(); delete.ExecuteNonQuery();
} }
tx.Commit(); tx.Commit();
CleanupUnreferencedAssetFiles();
} }
public LocalCleanupResult CleanupForConfiguration(IReadOnlySet<string> liveItemIds, IEnumerable<string> allowedLanguages, bool localAssetStorage) public LocalCleanupResult CleanupForConfiguration(IReadOnlySet<string> liveItemIds, IEnumerable<string> allowedLanguages, bool localAssetStorage)
@@ -62,10 +62,18 @@ DELETE FROM assets WHERE item_id = $item_id;";
.Distinct(StringComparer.OrdinalIgnoreCase) .Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray(); .ToArray();
using (var con = Open())
{
con.Open();
using var cmd = con.CreateCommand();
var parameters = AddParams(cmd, allowed, "$lang");
cmd.CommandText = $"DELETE FROM fetch_state WHERE source LIKE 'metadata:%' AND substr(source, 10) NOT IN ({parameters});";
cmd.ExecuteNonQuery();
}
var translationsDeleted = DeleteRowsOutsideLanguages("translations", allowed); var translationsDeleted = DeleteRowsOutsideLanguages("translations", allowed);
var genresDeleted = DeleteRowsOutsideLanguages("genres", allowed); var genresDeleted = DeleteRowsOutsideLanguages("genres", allowed);
var assetsDeleted = DeleteRowsOutsideLanguages("assets", allowed); var assetsDeleted = DeleteRowsOutsideLanguages("assets", allowed);
var modeRowsDeleted = localAssetStorage ? 0 : DeleteLocalAssetRows(resetMissingCheckedAt: true); var modeRowsDeleted = localAssetStorage ? 0 : SwitchLocalAssetsToUrls();
var assetFilesDeleted = CleanupUnreferencedAssetFiles(); var assetFilesDeleted = CleanupUnreferencedAssetFiles();
return new LocalCleanupResult(translationsDeleted, assetsDeleted + modeRowsDeleted, genresDeleted, assetFilesDeleted); return new LocalCleanupResult(translationsDeleted, assetsDeleted + modeRowsDeleted, genresDeleted, assetFilesDeleted);
} }
@@ -80,38 +88,22 @@ DELETE FROM assets WHERE item_id = $item_id;";
return cmd.ExecuteNonQuery(); return cmd.ExecuteNonQuery();
} }
private int DeleteLocalAssetRows(bool resetMissingCheckedAt) private int SwitchLocalAssetsToUrls()
{ {
using var con = Open(); using var con = Open();
con.Open(); con.Open();
using var tx = con.BeginTransaction(); using var tx = con.BeginTransaction();
var affected = new List<string>(); using var cmd = con.CreateCommand();
using (var select = con.CreateCommand()) cmd.Transaction = tx;
{ cmd.CommandText = @"UPDATE assets SET path = source_url, path_low = lower(source_url)
select.Transaction = tx; WHERE path_low LIKE '/multilang/assets/%' AND source_url <> '';
select.CommandText = "SELECT DISTINCT item_id FROM assets WHERE path_low LIKE '/multilang/assets/%';"; DELETE FROM fetch_state WHERE source LIKE 'artwork:%' AND item_id IN
using var reader = select.ExecuteReader(); (SELECT item_id FROM assets WHERE path_low LIKE '/multilang/assets/%');
while (reader.Read()) UPDATE facts SET missing_checked_at = 0 WHERE item_id IN
affected.Add(reader.GetString(0)); (SELECT item_id FROM assets WHERE path_low LIKE '/multilang/assets/%');";
} cmd.ExecuteNonQuery();
cmd.CommandText = "DELETE FROM assets WHERE path_low LIKE '/multilang/assets/%';";
int deleted; var deleted = cmd.ExecuteNonQuery();
using (var delete = con.CreateCommand())
{
delete.Transaction = tx;
delete.CommandText = "DELETE FROM assets WHERE path_low LIKE '/multilang/assets/%';";
deleted = delete.ExecuteNonQuery();
}
if (resetMissingCheckedAt && affected.Count > 0)
{
using var update = con.CreateCommand();
update.Transaction = tx;
var idParams = AddParams(update, affected, "$item");
update.CommandText = $"UPDATE facts SET missing_checked_at = 0 WHERE item_id IN ({idParams});";
update.ExecuteNonQuery();
}
tx.Commit(); tx.Commit();
return deleted; return deleted;
} }
@@ -15,21 +15,34 @@ public sealed partial class TranslationStore
destination.Open(); destination.Open();
using (var schema = destination.CreateCommand()) using (var schema = destination.CreateCommand())
{ {
schema.CommandText = SchemaSql; schema.CommandText = SchemaSql + "\nPRAGMA user_version = 1;";
schema.ExecuteNonQuery(); schema.ExecuteNonQuery();
} }
using var source = Open(); using var source = Open();
source.Open(); source.Open();
CopyTable(source, destination, "facts", "item_id, tmdb_id, kind, original_title, original_language, original_language_all, origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, genre_tmdb_ids_json, missing_checked_at, full_checked_at"); using var snapshot = source.BeginTransaction();
CopyTable(source, destination, "translations", "item_id, lang, field, text"); CopyTable(snapshot, destination, "facts", "item_id, tmdb_id, kind, original_title, original_language, original_language_all, origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, genre_tmdb_ids_json, missing_checked_at, full_checked_at");
CopyTable(source, destination, "assets", "item_id, lang, kind, path, path_low, updated_at"); CopyTable(snapshot, destination, "translations", "item_id, lang, field, text");
CopyTable(source, destination, "genres", "tmdb_id, media, lang, name, name_norm"); CopyTable(snapshot, destination, "assets", "item_id, lang, kind, path, path_low, updated_at, source_url, provider");
CopyTable(source, destination, "scan_state", "id, last_scan_started"); CopyTable(snapshot, destination, "fetch_state", "item_id, source, scope, checked_at, complete");
CopyTable(snapshot, destination, "genres", "tmdb_id, media, lang, name, name_norm");
CopyTable(snapshot, destination, "scan_state", "id, last_scan_started");
}
public static void ValidateTranslationBackup(string path)
{
using var connection = new SqliteConnection($"Data Source={path};Mode=ReadOnly");
connection.Open();
using var command = connection.CreateCommand();
command.CommandText = "PRAGMA user_version;";
if (Convert.ToInt32(command.ExecuteScalar()) != 1)
throw new InvalidOperationException("Unsupported translations backup version. Export a new backup using the current plugin.");
} }
public ImportDatabaseResult ImportTranslationDatabase(string sourcePath, IReadOnlySet<string> liveItemIds, IReadOnlySet<string> allowedLanguages) public ImportDatabaseResult ImportTranslationDatabase(string sourcePath, IReadOnlySet<string> liveItemIds, IReadOnlySet<string> allowedLanguages)
{ {
ValidateTranslationBackup(sourcePath);
var allowed = allowedLanguages.ToHashSet(StringComparer.OrdinalIgnoreCase); var allowed = allowedLanguages.ToHashSet(StringComparer.OrdinalIgnoreCase);
allowed.Add(MultilangConstants.OriginalAction); allowed.Add(MultilangConstants.OriginalAction);
@@ -39,187 +52,62 @@ public sealed partial class TranslationStore
source.Open(); source.Open();
using var tx = destination.BeginTransaction(); using var tx = destination.BeginTransaction();
var facts = ImportFacts(source, destination, tx, liveItemIds); bool Live(SqliteDataReader row) => liveItemIds.Contains(row.GetString(0));
var translations = ImportTranslations(source, destination, tx, liveItemIds, allowed); bool Translation(SqliteDataReader row) => Live(row) && allowed.Contains(row.GetString(1));
var assets = ImportAssets(source, destination, tx, liveItemIds, allowed); var facts = ImportTable(source, tx, "facts", "item_id, tmdb_id, kind, original_title, original_language, original_language_all, origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, genre_tmdb_ids_json, missing_checked_at, full_checked_at", Live);
var genres = ImportGenres(source, destination, tx, allowed); var translations = ImportTable(source, tx, "translations", "item_id, lang, field, text", Translation);
var assets = ImportTable(source, tx, "assets", "item_id, lang, kind, path, path_low, updated_at, source_url, provider", Translation);
var genres = ImportTable(source, tx, "genres", "tmdb_id, media, lang, name, name_norm", row => allowed.Contains(row.GetString(2)));
ImportTable(source, tx, "fetch_state", "item_id, source, scope, checked_at, complete", row =>
Live(row) && (row.GetString(1).StartsWith("artwork:") || (row.GetString(1).StartsWith("metadata:", StringComparison.Ordinal) && allowed.Contains(row.GetString(1)["metadata:".Length..]))));
tx.Commit(); tx.Commit();
return new ImportDatabaseResult(facts, translations, assets, genres); return new ImportDatabaseResult(facts, translations, assets, genres);
} }
private static void CopyTable(SqliteConnection source, SqliteConnection destination, string table, string columns) private static void CopyTable(SqliteTransaction snapshot, SqliteConnection destination, string table, string columns)
{ {
using var read = source.CreateCommand(); using var read = snapshot.Connection!.CreateCommand();
read.Transaction = snapshot;
read.CommandText = $"SELECT {columns} FROM {table};"; read.CommandText = $"SELECT {columns} FROM {table};";
using var reader = read.ExecuteReader(); using var reader = read.ExecuteReader();
using var tx = destination.BeginTransaction(); using var tx = destination.BeginTransaction();
while (reader.Read()) var names = columns.Split(',', StringSplitOptions.TrimEntries);
{
using var insert = destination.CreateCommand(); using var insert = destination.CreateCommand();
insert.Transaction = tx; insert.Transaction = tx;
var names = columns.Split(',', StringSplitOptions.TrimEntries);
insert.CommandText = $"INSERT OR REPLACE INTO {table}({columns}) VALUES({string.Join(",", names.Select((_, i) => "$v" + i.ToString(CultureInfo.InvariantCulture)))});"; insert.CommandText = $"INSERT OR REPLACE INTO {table}({columns}) VALUES({string.Join(",", names.Select((_, i) => "$v" + i.ToString(CultureInfo.InvariantCulture)))});";
for (var i = 0; i < names.Length; i++) for (var i = 0; i < names.Length; i++)
insert.Parameters.AddWithValue("$v" + i.ToString(CultureInfo.InvariantCulture), reader.GetValue(i)); insert.Parameters.Add(new SqliteParameter("$v" + i.ToString(CultureInfo.InvariantCulture), DBNull.Value));
while (reader.Read())
{
for (var i = 0; i < names.Length; i++)
insert.Parameters[i].Value = reader.GetValue(i);
insert.ExecuteNonQuery(); insert.ExecuteNonQuery();
} }
tx.Commit(); tx.Commit();
} }
private static int ImportFacts(SqliteConnection source, SqliteConnection destination, SqliteTransaction tx, IReadOnlySet<string> liveItemIds) private static int ImportTable(SqliteConnection source, SqliteTransaction tx, string table, string columns, Func<SqliteDataReader, bool> include)
{ {
using var read = source.CreateCommand(); using var read = source.CreateCommand();
read.CommandText = @" read.CommandText = $"SELECT {columns} FROM {table};";
SELECT item_id, tmdb_id, kind, original_title, original_language, original_language_all,
origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language,
genre_tmdb_ids_json, missing_checked_at, full_checked_at
FROM facts;";
using var reader = read.ExecuteReader(); using var reader = read.ExecuteReader();
var count = 0; using var insert = tx.Connection!.CreateCommand();
while (reader.Read())
{
var itemId = reader.GetString(0);
if (!liveItemIds.Contains(itemId))
continue;
using var insert = destination.CreateCommand();
insert.Transaction = tx; insert.Transaction = tx;
insert.CommandText = @" var names = columns.Split(',', StringSplitOptions.TrimEntries);
INSERT INTO facts(item_id, tmdb_id, kind, original_title, original_language, original_language_all, insert.CommandText = $"INSERT OR REPLACE INTO {table}({columns}) VALUES({string.Join(",", names.Select((_, i) => "$v" + i))});";
origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language,
genre_tmdb_ids_json, missing_checked_at, full_checked_at)
VALUES($item_id, $tmdb_id, $kind, $original_title, $original_language, $original_language_all,
$origin_countries_json, $production_countries_json, $spoken_languages_json, $audio_track_language,
$genre_tmdb_ids_json, $missing_checked_at, $full_checked_at)
ON CONFLICT(item_id) DO UPDATE SET
tmdb_id = excluded.tmdb_id,
kind = excluded.kind,
original_title = excluded.original_title,
original_language = excluded.original_language,
original_language_all = excluded.original_language_all,
origin_countries_json = excluded.origin_countries_json,
production_countries_json = excluded.production_countries_json,
spoken_languages_json = excluded.spoken_languages_json,
audio_track_language = excluded.audio_track_language,
genre_tmdb_ids_json = excluded.genre_tmdb_ids_json,
missing_checked_at = excluded.missing_checked_at,
full_checked_at = excluded.full_checked_at;";
var names = new[]
{
"$item_id",
"$tmdb_id",
"$kind",
"$original_title",
"$original_language",
"$original_language_all",
"$origin_countries_json",
"$production_countries_json",
"$spoken_languages_json",
"$audio_track_language",
"$genre_tmdb_ids_json",
"$missing_checked_at",
"$full_checked_at"
};
for (var i = 0; i < names.Length; i++) for (var i = 0; i < names.Length; i++)
insert.Parameters.AddWithValue(names[i], reader.GetValue(i)); insert.Parameters.Add(new SqliteParameter("$v" + i, DBNull.Value));
insert.ExecuteNonQuery();
count++;
}
return count;
}
private static int ImportTranslations(SqliteConnection source, SqliteConnection destination, SqliteTransaction tx, IReadOnlySet<string> liveItemIds, IReadOnlySet<string> allowedLanguages)
{
using var read = source.CreateCommand();
read.CommandText = "SELECT item_id, lang, field, text FROM translations;";
using var reader = read.ExecuteReader();
var count = 0; var count = 0;
while (reader.Read()) while (reader.Read())
{ {
var itemId = reader.GetString(0); if (!include(reader))
var lang = reader.GetString(1);
if (!liveItemIds.Contains(itemId) || !allowedLanguages.Contains(lang))
continue; continue;
using var insert = destination.CreateCommand(); for (var i = 0; i < names.Length; i++)
insert.Transaction = tx; insert.Parameters[i].Value = reader.GetValue(i);
insert.CommandText = @"
INSERT INTO translations(item_id, lang, field, text)
VALUES($item_id, $lang, $field, $text)
ON CONFLICT(item_id, lang, field) DO UPDATE SET text = excluded.text;";
insert.Parameters.AddWithValue("$item_id", itemId);
insert.Parameters.AddWithValue("$lang", lang);
insert.Parameters.AddWithValue("$field", reader.GetString(2));
insert.Parameters.AddWithValue("$text", reader.GetString(3));
insert.ExecuteNonQuery(); insert.ExecuteNonQuery();
count++; count++;
} }
return count;
}
private static int ImportAssets(SqliteConnection source, SqliteConnection destination, SqliteTransaction tx, IReadOnlySet<string> liveItemIds, IReadOnlySet<string> allowedLanguages)
{
using var read = source.CreateCommand();
read.CommandText = "SELECT item_id, lang, kind, path, path_low, updated_at FROM assets;";
using var reader = read.ExecuteReader();
var count = 0;
while (reader.Read())
{
var itemId = reader.GetString(0);
var lang = reader.GetString(1);
if (!liveItemIds.Contains(itemId) || !allowedLanguages.Contains(lang))
continue;
using var insert = destination.CreateCommand();
insert.Transaction = tx;
insert.CommandText = @"
INSERT INTO assets(item_id, lang, kind, path, path_low, updated_at)
VALUES($item_id, $lang, $kind, $path, $path_low, $updated_at)
ON CONFLICT(item_id, lang, kind) DO UPDATE SET
path = excluded.path,
path_low = excluded.path_low,
updated_at = excluded.updated_at;";
insert.Parameters.AddWithValue("$item_id", itemId);
insert.Parameters.AddWithValue("$lang", lang);
insert.Parameters.AddWithValue("$kind", reader.GetString(2));
insert.Parameters.AddWithValue("$path", reader.GetString(3));
insert.Parameters.AddWithValue("$path_low", reader.GetString(4));
insert.Parameters.AddWithValue("$updated_at", reader.GetInt64(5));
insert.ExecuteNonQuery();
count++;
}
return count;
}
private static int ImportGenres(SqliteConnection source, SqliteConnection destination, SqliteTransaction tx, IReadOnlySet<string> allowedLanguages)
{
using var read = source.CreateCommand();
read.CommandText = "SELECT tmdb_id, media, lang, name, name_norm FROM genres;";
using var reader = read.ExecuteReader();
var count = 0;
while (reader.Read())
{
var lang = reader.GetString(2);
if (!allowedLanguages.Contains(lang))
continue;
using var insert = destination.CreateCommand();
insert.Transaction = tx;
insert.CommandText = @"
INSERT INTO genres(tmdb_id, media, lang, name, name_norm)
VALUES($tmdb_id, $media, $lang, $name, $name_norm)
ON CONFLICT(tmdb_id, media, lang) DO UPDATE SET
name = excluded.name,
name_norm = excluded.name_norm;";
insert.Parameters.AddWithValue("$tmdb_id", reader.GetInt32(0));
insert.Parameters.AddWithValue("$media", reader.GetString(1));
insert.Parameters.AddWithValue("$lang", lang);
insert.Parameters.AddWithValue("$name", reader.GetString(3));
insert.Parameters.AddWithValue("$name_norm", reader.GetString(4));
insert.ExecuteNonQuery();
count++;
}
return count; return count;
} }
} }
@@ -4,8 +4,6 @@ namespace Jellyfin.Plugin.Multilang.Data;
public readonly record struct FactsStatus(bool Exists, long MissingCheckedAt, long FullCheckedAt); public readonly record struct FactsStatus(bool Exists, long MissingCheckedAt, long FullCheckedAt);
public readonly record struct AssetPresence(bool Any, bool AnyRemote);
public readonly record struct LocalCleanupResult(int TranslationsDeleted, int AssetsDeleted, int GenresDeleted, int AssetFilesDeleted); public readonly record struct LocalCleanupResult(int TranslationsDeleted, int AssetsDeleted, int GenresDeleted, int AssetFilesDeleted);
public readonly record struct ImportDatabaseResult(int Facts, int Translations, int Assets, int Genres); public readonly record struct ImportDatabaseResult(int Facts, int Translations, int Assets, int Genres);
@@ -46,23 +44,17 @@ public sealed class UserCategoryRule
public string Label { get; set; } = string.Empty; public string Label { get; set; } = string.Empty;
public string CriteriaText { get; set; } = string.Empty;
public UserCategoryRequirement[] Requirements { get; set; } = []; public UserCategoryRequirement[] Requirements { get; set; } = [];
public bool MatchAllConditions { get; set; } = true; public bool MatchAllConditions { get; set; } = true;
public string[] Scopes { get; set; } = ["M", "S", "C"]; public string[] Scopes { get; set; } = ["M", "S", "C"];
public Dictionary<string, string> FieldActions { get; set; } = MultilangConstants.DefaultFieldActions();
public Dictionary<string, string[]> FieldActionLists { get; set; } = MultilangConstants.DefaultActionLists(); public Dictionary<string, string[]> FieldActionLists { get; set; } = MultilangConstants.DefaultActionLists();
} }
public sealed class UserCategoryRequirement public sealed class UserCategoryRequirement
{ {
public string Field { get; set; } = string.Empty;
public string[] Fields { get; set; } = []; public string[] Fields { get; set; } = [];
public bool UseFieldOr { get; set; } public bool UseFieldOr { get; set; }
@@ -0,0 +1,125 @@
using Jellyfin.Plugin.Multilang.Services.Providers;
using Jellyfin.Plugin.Multilang.Services.Refresh;
using Microsoft.Data.Sqlite;
using static Jellyfin.Plugin.Multilang.MultilangConstants;
namespace Jellyfin.Plugin.Multilang.Data;
public sealed record FetchState(string Scope, long CheckedAt, bool Complete);
public sealed record StoredAsset(string Language, string Kind, string Path, string SourceUrl, string Provider);
public sealed partial class TranslationStore
{
private void Write(SqliteTransaction? transaction, Action<SqliteTransaction> action)
{
if (transaction is not null)
{
action(transaction);
return;
}
using var connection = Open();
connection.Open();
using var tx = connection.BeginTransaction();
action(tx);
tx.Commit();
}
public Dictionary<string, FetchState> GetFetchStates(string itemId)
{
using var connection = Open();
connection.Open();
using var cmd = connection.CreateCommand();
cmd.CommandText = "SELECT source, scope, checked_at, complete FROM fetch_state WHERE item_id = $id;";
cmd.Parameters.AddWithValue("$id", itemId);
using var reader = cmd.ExecuteReader();
var result = new Dictionary<string, FetchState>(StringComparer.OrdinalIgnoreCase);
while (reader.Read())
result.Add(reader.GetString(0), new(reader.GetString(1), reader.GetInt64(2), reader.GetBoolean(3)));
return result;
}
private static void SaveFetchState(SqliteTransaction tx, string itemId, string source, FetchState state)
{
using var cmd = tx.Connection!.CreateCommand();
cmd.Transaction = tx;
cmd.CommandText = "INSERT OR REPLACE INTO fetch_state(item_id, source, scope, checked_at, complete) VALUES($id, $source, $scope, $time, $complete);";
cmd.Parameters.AddWithValue("$id", itemId);
cmd.Parameters.AddWithValue("$source", source);
cmd.Parameters.AddWithValue("$scope", state.Scope);
cmd.Parameters.AddWithValue("$time", state.CheckedAt);
cmd.Parameters.AddWithValue("$complete", state.Complete);
cmd.ExecuteNonQuery();
}
public void SaveMetadata(RefreshItemInfo item, string language, TmdbMetadata? metadata, FetchState state, bool full, bool updateFacts = true)
=> Write(null, tx =>
{
if (metadata is not null)
{
if (updateFacts)
UpsertFacts(item, metadata, state.CheckedAt, full ? state.CheckedAt : 0, tx);
UpsertTranslation(item.ItemId, language, TitleField, metadata.Title, tx);
UpsertTranslation(item.ItemId, language, OverviewField, metadata.Overview, tx);
UpsertTranslation(item.ItemId, language, TaglineField, metadata.Tagline, tx);
foreach (var genre in metadata.Genres)
UpsertGenre(genre.Id, item.Kind is "movie" or "collection" ? "movie" : "tv", language, genre.Name, tx);
}
else
MarkFactsChecked(item, state.CheckedAt, full ? state.CheckedAt : 0, tx);
SaveFetchState(tx, item.ItemId, "metadata:" + language, state);
});
public void CopyOriginalTranslations(string itemId, string language)
=> Write(null, tx =>
{
using var cmd = tx.Connection!.CreateCommand();
cmd.Transaction = tx;
cmd.CommandText = @"DELETE FROM translations WHERE item_id = $id AND lang = 'Original';
INSERT INTO translations(item_id, lang, field, text)
SELECT item_id, 'Original', field, text FROM translations WHERE item_id = $id AND lang = $lang;";
cmd.Parameters.AddWithValue("$id", itemId);
cmd.Parameters.AddWithValue("$lang", language);
cmd.ExecuteNonQuery();
});
public List<StoredAsset> GetStoredAssets(string itemId)
{
using var connection = Open();
connection.Open();
using var cmd = connection.CreateCommand();
cmd.CommandText = "SELECT lang, kind, path, source_url, provider FROM assets WHERE item_id = $id;";
cmd.Parameters.AddWithValue("$id", itemId);
using var reader = cmd.ExecuteReader();
var result = new List<StoredAsset>();
while (reader.Read())
result.Add(new(reader.GetString(0), reader.GetString(1), reader.GetString(2), reader.GetString(3), reader.GetString(4)));
return result;
}
public void SaveArtwork(string itemId, IEnumerable<StoredAsset> assets, IReadOnlyDictionary<string, FetchState> states)
=> Write(null, tx =>
{
using var cmd = tx.Connection!.CreateCommand();
cmd.Transaction = tx;
cmd.CommandText = "DELETE FROM assets WHERE item_id = $id;";
cmd.Parameters.AddWithValue("$id", itemId);
cmd.ExecuteNonQuery();
cmd.CommandText = @"INSERT INTO assets(item_id, lang, kind, path, path_low, source_url, provider, updated_at)
VALUES($id, $lang, $kind, $path, $low, $url, $provider, $time);";
foreach (var name in new[] { "$lang", "$kind", "$path", "$low", "$url", "$provider", "$time" })
cmd.Parameters.Add(new SqliteParameter(name, DBNull.Value));
foreach (var asset in assets)
{
cmd.Parameters["$lang"].Value = asset.Language;
cmd.Parameters["$kind"].Value = asset.Kind;
cmd.Parameters["$path"].Value = asset.Path;
cmd.Parameters["$low"].Value = asset.Path.ToLowerInvariant();
cmd.Parameters["$url"].Value = asset.SourceUrl;
cmd.Parameters["$provider"].Value = asset.Provider;
cmd.Parameters["$time"].Value = NowUnixUtc();
cmd.ExecuteNonQuery();
}
foreach (var (provider, state) in states)
SaveFetchState(tx, itemId, provider, state);
});
}
@@ -38,11 +38,22 @@ CREATE TABLE IF NOT EXISTS assets (
kind TEXT NOT NULL, kind TEXT NOT NULL,
path TEXT NOT NULL, path TEXT NOT NULL,
path_low TEXT NOT NULL DEFAULT '', path_low TEXT NOT NULL DEFAULT '',
source_url TEXT NOT NULL DEFAULT '',
provider TEXT NOT NULL DEFAULT '',
updated_at INTEGER NOT NULL, updated_at INTEGER NOT NULL,
PRIMARY KEY (item_id, lang, kind) PRIMARY KEY (item_id, lang, kind)
); );
CREATE INDEX IF NOT EXISTS idx_assets_item ON assets(item_id); CREATE INDEX IF NOT EXISTS idx_assets_item ON assets(item_id);
CREATE TABLE IF NOT EXISTS fetch_state (
item_id TEXT NOT NULL,
source TEXT NOT NULL,
scope TEXT NOT NULL,
checked_at INTEGER NOT NULL,
complete INTEGER NOT NULL,
PRIMARY KEY (item_id, source)
);
CREATE TABLE IF NOT EXISTS genres ( CREATE TABLE IF NOT EXISTS genres (
tmdb_id INTEGER NOT NULL, tmdb_id INTEGER NOT NULL,
media TEXT NOT NULL, media TEXT NOT NULL,
@@ -15,6 +15,34 @@ public sealed partial class TranslationStore
private readonly string _dataDir; private readonly string _dataDir;
private readonly string _assetsDir; private readonly string _assetsDir;
private readonly string _dbPath; private readonly string _dbPath;
private readonly SemaphoreSlim _maintenance = new(1, 1);
private readonly SemaphoreSlim _refreshSlots = new(MultilangConstants.RefreshConcurrency, MultilangConstants.RefreshConcurrency);
public Task<IDisposable> EnterRefreshAsync(CancellationToken ct) => EnterAsync(1, ct);
public Task<IDisposable> EnterMaintenanceAsync(CancellationToken ct) => EnterAsync(MultilangConstants.RefreshConcurrency, ct);
private async Task<IDisposable> EnterAsync(int slots, CancellationToken ct)
{
await _maintenance.WaitAsync(ct).ConfigureAwait(false);
var acquired = 0;
try
{
for (; acquired < slots; acquired++)
await _refreshSlots.WaitAsync(ct).ConfigureAwait(false);
return new MaintenanceLease(_refreshSlots, slots);
}
catch
{
if (acquired > 0) _refreshSlots.Release(acquired);
throw;
}
finally { _maintenance.Release(); }
}
private sealed class MaintenanceLease(SemaphoreSlim semaphore, int slots) : IDisposable
{
public void Dispose() => semaphore.Release(slots);
}
public TranslationStore(IApplicationPaths appPaths) public TranslationStore(IApplicationPaths appPaths)
{ {
@@ -24,6 +52,8 @@ public sealed partial class TranslationStore
Directory.CreateDirectory(_assetsDir); Directory.CreateDirectory(_assetsDir);
_dbPath = Path.Combine(_dataDir, "translations.sqlite"); _dbPath = Path.Combine(_dataDir, "translations.sqlite");
Initialize(); Initialize();
if (Plugin.Instance is { } plugin)
plugin.Store = this;
} }
public string DataDirectory => _dataDir; public string DataDirectory => _dataDir;
@@ -42,27 +72,38 @@ public sealed partial class TranslationStore
=> JsonSerializer.Serialize(values.Where(v => !string.IsNullOrWhiteSpace(v)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray()); => JsonSerializer.Serialize(values.Where(v => !string.IsNullOrWhiteSpace(v)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray());
public SqliteConnection Open() public SqliteConnection Open()
=> new($"Data Source={_dbPath};Cache=Shared;Pooling=False"); => new($"Data Source={_dbPath};Pooling=False");
private void Initialize() private void Initialize()
{ {
using var con = Open(); using var con = Open();
con.Open(); con.Open();
using var cmd = con.CreateCommand(); using var cmd = con.CreateCommand();
cmd.CommandText = "PRAGMA journal_mode=WAL;";
cmd.ExecuteNonQuery();
cmd.CommandText = SchemaSql; cmd.CommandText = SchemaSql;
cmd.ExecuteNonQuery(); cmd.ExecuteNonQuery();
EnsureColumn(con, "assets", "path_low", "TEXT NOT NULL DEFAULT ''"); EnsureColumn(con, "assets", "path_low", "TEXT NOT NULL DEFAULT ''");
EnsureColumn(con, "assets", "source_url", "TEXT NOT NULL DEFAULT ''");
EnsureColumn(con, "assets", "provider", "TEXT NOT NULL DEFAULT ''");
cmd.CommandText = "UPDATE assets SET source_url = path WHERE source_url = '' AND (path_low LIKE 'https://%' OR path_low LIKE 'http://%');";
cmd.ExecuteNonQuery();
} }
public void ResetAll() public void ResetAll()
{ {
SqliteConnection.ClearAllPools(); using var con = Open();
if (Directory.Exists(_dataDir)) con.Open();
Directory.Delete(_dataDir, recursive: true); using var tx = con.BeginTransaction();
Directory.CreateDirectory(_dataDir); using var cmd = con.CreateCommand();
cmd.Transaction = tx;
cmd.CommandText = "DELETE FROM facts; DELETE FROM translations; DELETE FROM assets; DELETE FROM fetch_state; DELETE FROM genres; " +
"DELETE FROM user_rules; DELETE FROM user_display_langs; UPDATE scan_state SET last_scan_started = 0;";
cmd.ExecuteNonQuery();
tx.Commit();
if (Directory.Exists(_assetsDir))
Directory.Delete(_assetsDir, recursive: true);
Directory.CreateDirectory(_assetsDir); Directory.CreateDirectory(_assetsDir);
Initialize();
SqliteConnection.ClearAllPools();
} }
public (long DatabaseBytes, long AssetsBytes) GetStorageUsage() public (long DatabaseBytes, long AssetsBytes) GetStorageUsage()
@@ -147,11 +188,12 @@ ON CONFLICT(id) DO UPDATE SET last_scan_started = excluded.last_scan_started;";
return result; return result;
} }
public void UpsertFacts(RefreshItemInfo item, TmdbMetadata metadata, long missingCheckedAt, long fullCheckedAt) public void UpsertFacts(RefreshItemInfo item, TmdbMetadata metadata, long missingCheckedAt, long fullCheckedAt, SqliteTransaction? transaction = null)
{ {
using var con = Open(); Write(transaction, tx =>
con.Open(); {
using var cmd = con.CreateCommand(); using var cmd = tx.Connection!.CreateCommand();
cmd.Transaction = tx;
cmd.CommandText = @" cmd.CommandText = @"
INSERT INTO facts( INSERT INTO facts(
item_id, tmdb_id, kind, original_title, original_language, original_language_all, item_id, tmdb_id, kind, original_title, original_language, original_language_all,
@@ -188,13 +230,15 @@ ON CONFLICT(item_id) DO UPDATE SET
cmd.Parameters.AddWithValue("$missing_checked_at", missingCheckedAt); cmd.Parameters.AddWithValue("$missing_checked_at", missingCheckedAt);
cmd.Parameters.AddWithValue("$full_checked_at", fullCheckedAt); cmd.Parameters.AddWithValue("$full_checked_at", fullCheckedAt);
cmd.ExecuteNonQuery(); cmd.ExecuteNonQuery();
});
} }
public void MarkFactsChecked(RefreshItemInfo item, long missingCheckedAt, long fullCheckedAt) public void MarkFactsChecked(RefreshItemInfo item, long missingCheckedAt, long fullCheckedAt, SqliteTransaction? transaction = null)
{ {
using var con = Open(); Write(transaction, tx =>
con.Open(); {
using var cmd = con.CreateCommand(); using var cmd = tx.Connection!.CreateCommand();
cmd.Transaction = tx;
cmd.CommandText = @" cmd.CommandText = @"
INSERT INTO facts( INSERT INTO facts(
item_id, tmdb_id, kind, original_title, original_language, original_language_all, item_id, tmdb_id, kind, original_title, original_language, original_language_all,
@@ -215,6 +259,7 @@ ON CONFLICT(item_id) DO UPDATE SET
cmd.Parameters.AddWithValue("$missing_checked_at", missingCheckedAt); cmd.Parameters.AddWithValue("$missing_checked_at", missingCheckedAt);
cmd.Parameters.AddWithValue("$full_checked_at", fullCheckedAt); cmd.Parameters.AddWithValue("$full_checked_at", fullCheckedAt);
cmd.ExecuteNonQuery(); cmd.ExecuteNonQuery();
});
} }
public FactsData? GetFacts(string itemId) public FactsData? GetFacts(string itemId)
@@ -290,11 +335,12 @@ WHERE item_id IN ({idParams});";
return rows.ToArray(); return rows.ToArray();
} }
public void UpsertTranslation(string itemId, string lang, string field, string text) public void UpsertTranslation(string itemId, string lang, string field, string text, SqliteTransaction? transaction = null)
{ {
using var con = Open(); Write(transaction, tx =>
con.Open(); {
using var cmd = con.CreateCommand(); using var cmd = tx.Connection!.CreateCommand();
cmd.Transaction = tx;
cmd.CommandText = @" cmd.CommandText = @"
INSERT INTO translations(item_id, lang, field, text) INSERT INTO translations(item_id, lang, field, text)
VALUES($item_id, $lang, $field, $text) VALUES($item_id, $lang, $field, $text)
@@ -304,27 +350,32 @@ ON CONFLICT(item_id, lang, field) DO UPDATE SET text = excluded.text;";
cmd.Parameters.AddWithValue("$field", field); cmd.Parameters.AddWithValue("$field", field);
cmd.Parameters.AddWithValue("$text", text); cmd.Parameters.AddWithValue("$text", text);
cmd.ExecuteNonQuery(); cmd.ExecuteNonQuery();
});
} }
public void UpsertAsset(string itemId, string lang, string kind, string path, long updatedAt) public void UpsertAsset(string itemId, string lang, string kind, string path, long updatedAt, SqliteTransaction? transaction = null)
{ {
using var con = Open(); Write(transaction, tx =>
con.Open(); {
using var cmd = con.CreateCommand(); using var cmd = tx.Connection!.CreateCommand();
cmd.Transaction = tx;
cmd.CommandText = @" cmd.CommandText = @"
INSERT INTO assets(item_id, lang, kind, path, path_low, updated_at) INSERT INTO assets(item_id, lang, kind, path, path_low, updated_at, source_url)
VALUES($item_id, $lang, $kind, $path, $path_low, $updated_at) VALUES($item_id, $lang, $kind, $path, $path_low, $updated_at, $source_url)
ON CONFLICT(item_id, lang, kind) DO UPDATE SET ON CONFLICT(item_id, lang, kind) DO UPDATE SET
path = excluded.path, path = excluded.path,
path_low = excluded.path_low, path_low = excluded.path_low,
source_url = excluded.source_url,
updated_at = excluded.updated_at;"; updated_at = excluded.updated_at;";
cmd.Parameters.AddWithValue("$item_id", itemId); cmd.Parameters.AddWithValue("$item_id", itemId);
cmd.Parameters.AddWithValue("$lang", lang); cmd.Parameters.AddWithValue("$lang", lang);
cmd.Parameters.AddWithValue("$kind", kind); cmd.Parameters.AddWithValue("$kind", kind);
cmd.Parameters.AddWithValue("$path", path); cmd.Parameters.AddWithValue("$path", path);
cmd.Parameters.AddWithValue("$path_low", path.ToLowerInvariant()); cmd.Parameters.AddWithValue("$path_low", path.ToLowerInvariant());
cmd.Parameters.AddWithValue("$source_url", Uri.TryCreate(path, UriKind.Absolute, out var url) && url.Scheme is "https" or "http" ? path : "");
cmd.Parameters.AddWithValue("$updated_at", updatedAt); cmd.Parameters.AddWithValue("$updated_at", updatedAt);
cmd.ExecuteNonQuery(); cmd.ExecuteNonQuery();
});
} }
public string? GetAssetPath(string itemId, string lang, string kind) public string? GetAssetPath(string itemId, string lang, string kind)
@@ -339,64 +390,26 @@ ON CONFLICT(item_id, lang, kind) DO UPDATE SET
return cmd.ExecuteScalar() as string; return cmd.ExecuteScalar() as string;
} }
public AssetPresence GetAssetPresence(string itemId, IEnumerable<string> languages) public bool IsAssetReferenced(string itemId, string path)
{ {
var langs = languages.Where(l => !string.IsNullOrWhiteSpace(l)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
if (langs.Length == 0)
return new AssetPresence(false, false);
using var con = Open(); using var con = Open();
con.Open(); con.Open();
using var cmd = con.CreateCommand(); using var cmd = con.CreateCommand();
var langParams = AddParams(cmd, langs, "$lang"); cmd.CommandText = "SELECT 1 FROM assets WHERE item_id = $item AND path = $path LIMIT 1;";
cmd.CommandText = $"SELECT path_low FROM assets WHERE item_id = $item_id AND lang IN ({langParams});"; cmd.Parameters.AddWithValue("$item", itemId);
cmd.Parameters.AddWithValue("$item_id", itemId); cmd.Parameters.AddWithValue("$path", path);
using var reader = cmd.ExecuteReader(); return cmd.ExecuteScalar() is not null;
var any = false;
var anyRemote = false;
while (reader.Read())
{
any = true;
var path = reader.GetString(0);
if (path.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || path.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
anyRemote = true;
} }
return new AssetPresence(any, anyRemote); public void UpsertGenre(int tmdbId, string media, string lang, string name, SqliteTransaction? transaction = null)
}
public bool HasMissingConfiguredTranslations(string itemId, IEnumerable<string> languages)
{
var langs = languages.Where(l => !string.IsNullOrWhiteSpace(l)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
if (langs.Length == 0)
return false;
using var con = Open();
con.Open();
using var cmd = con.CreateCommand();
var langParams = AddParams(cmd, langs, "$lang");
cmd.CommandText = $@"
SELECT lang, COUNT(DISTINCT field)
FROM translations
WHERE item_id = $item_id AND lang IN ({langParams}) AND field IN ('title', 'overview', 'tagline')
GROUP BY lang;";
cmd.Parameters.AddWithValue("$item_id", itemId);
using var reader = cmd.ExecuteReader();
var counts = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
while (reader.Read())
counts[reader.GetString(0)] = reader.GetInt32(1);
return langs.Any(lang => !counts.TryGetValue(lang, out var count) || count < 3);
}
public void UpsertGenre(int tmdbId, string media, string lang, string name)
{ {
if (tmdbId <= 0 || string.IsNullOrWhiteSpace(media) || string.IsNullOrWhiteSpace(lang) || string.IsNullOrWhiteSpace(name)) if (tmdbId <= 0 || string.IsNullOrWhiteSpace(media) || string.IsNullOrWhiteSpace(lang) || string.IsNullOrWhiteSpace(name))
return; return;
using var con = Open(); Write(transaction, tx =>
con.Open(); {
using var cmd = con.CreateCommand(); using var cmd = tx.Connection!.CreateCommand();
cmd.Transaction = tx;
cmd.CommandText = @" cmd.CommandText = @"
INSERT INTO genres(tmdb_id, media, lang, name, name_norm) INSERT INTO genres(tmdb_id, media, lang, name, name_norm)
VALUES($tmdb_id, $media, $lang, $name, $name_norm) VALUES($tmdb_id, $media, $lang, $name, $name_norm)
@@ -409,6 +422,7 @@ ON CONFLICT(tmdb_id, media, lang) DO UPDATE SET
cmd.Parameters.AddWithValue("$name", name.Trim()); cmd.Parameters.AddWithValue("$name", name.Trim());
cmd.Parameters.AddWithValue("$name_norm", name.Trim().ToLowerInvariant()); cmd.Parameters.AddWithValue("$name_norm", name.Trim().ToLowerInvariant());
cmd.ExecuteNonQuery(); cmd.ExecuteNonQuery();
});
} }
public GenreRow[] GetGenres(string media) public GenreRow[] GetGenres(string media)
@@ -7,9 +7,9 @@
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<GenerateDocumentationFile>false</GenerateDocumentationFile> <GenerateDocumentationFile>false</GenerateDocumentationFile>
<Version>0.3.3</Version> <Version>0.3.4</Version>
<AssemblyVersion>0.3.3.0</AssemblyVersion> <AssemblyVersion>0.3.4.0</AssemblyVersion>
<FileVersion>0.3.3.0</FileVersion> <FileVersion>0.3.4.0</FileVersion>
<Authors>ajp_anton</Authors> <Authors>ajp_anton</Authors>
</PropertyGroup> </PropertyGroup>
@@ -18,10 +18,10 @@
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Jellyfin.Controller" Version="12.0.0-rc2"> <PackageReference Include="Jellyfin.Controller" Version="12.0.0">
<ExcludeAssets>runtime</ExcludeAssets> <ExcludeAssets>runtime</ExcludeAssets>
</PackageReference> </PackageReference>
<PackageReference Include="Jellyfin.Model" Version="12.0.0-rc2"> <PackageReference Include="Jellyfin.Model" Version="12.0.0">
<ExcludeAssets>runtime</ExcludeAssets> <ExcludeAssets>runtime</ExcludeAssets>
</PackageReference> </PackageReference>
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.0" /> <PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.0" />
@@ -2,6 +2,7 @@ namespace Jellyfin.Plugin.Multilang;
internal static class MultilangConstants internal static class MultilangConstants
{ {
public const int RefreshConcurrency = 4;
public const string JellyfinAction = "Jellyfin"; public const string JellyfinAction = "Jellyfin";
public const string OriginalAction = "Original"; public const string OriginalAction = "Original";
public const string FallbackAction = "Fallback"; public const string FallbackAction = "Fallback";
@@ -25,12 +26,6 @@ internal static class MultilangConstants
=> field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) || => field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) ||
field.Equals(PosterKind, StringComparison.OrdinalIgnoreCase); field.Equals(PosterKind, StringComparison.OrdinalIgnoreCase);
public static Dictionary<string, string> DefaultFieldActions()
=> ActionFields.ToDictionary(
field => field,
_ => FallbackAction,
StringComparer.OrdinalIgnoreCase);
public static Dictionary<string, string[]> DefaultActionLists() public static Dictionary<string, string[]> DefaultActionLists()
=> ActionFields.ToDictionary( => ActionFields.ToDictionary(
field => field, field => field,
+4
View File
@@ -10,6 +10,8 @@ namespace Jellyfin.Plugin.Multilang;
public sealed class Plugin : BasePlugin<PluginConfiguration>, IHasWebPages public sealed class Plugin : BasePlugin<PluginConfiguration>, IHasWebPages
{ {
internal Data.TranslationStore? Store { get; set; }
internal bool Uninstalling { get; private set; }
public static Plugin? Instance { get; private set; } public static Plugin? Instance { get; private set; }
public Plugin(IApplicationPaths applicationPaths, IXmlSerializer xmlSerializer) public Plugin(IApplicationPaths applicationPaths, IXmlSerializer xmlSerializer)
@@ -24,6 +26,8 @@ public sealed class Plugin : BasePlugin<PluginConfiguration>, IHasWebPages
public override void OnUninstalling() public override void OnUninstalling()
{ {
Uninstalling = true;
using var maintenance = Store?.EnterMaintenanceAsync(CancellationToken.None).GetAwaiter().GetResult();
WebScriptInjector.RemoveInjected(ApplicationPaths.WebPath); WebScriptInjector.RemoveInjected(ApplicationPaths.WebPath);
if (Configuration.CleanupDataOnUninstall) if (Configuration.CleanupDataOnUninstall)
@@ -17,6 +17,8 @@ public sealed class PluginServiceRegistrator : IPluginServiceRegistrator
public void RegisterServices(IServiceCollection services, IServerApplicationHost applicationHost) public void RegisterServices(IServiceCollection services, IServerApplicationHost applicationHost)
{ {
services.AddMvc().AddApplicationPart(typeof(Plugin).Assembly); services.AddMvc().AddApplicationPart(typeof(Plugin).Assembly);
services.AddHttpClient("Multilang.Jellyfin")
.ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler { AllowAutoRedirect = false });
services.AddSingleton<TranslationStore>(); services.AddSingleton<TranslationStore>();
services.AddSingleton<ProviderCatalog>(); services.AddSingleton<ProviderCatalog>();
@@ -25,7 +27,9 @@ public sealed class PluginServiceRegistrator : IPluginServiceRegistrator
services.AddSingleton<FanartClient>(); services.AddSingleton<FanartClient>();
services.AddSingleton<RefreshCoordinator>(); services.AddSingleton<RefreshCoordinator>();
services.AddSingleton<RefreshService>(); services.AddSingleton<RefreshService>();
services.AddSingleton<ItemFetcher>();
services.AddSingleton<ItemsProxyCache>(); services.AddSingleton<ItemsProxyCache>();
services.AddHostedService<ItemsProxyInvalidationService>();
services.AddSingleton<ItemsProxyRequestCoalescer>(); services.AddSingleton<ItemsProxyRequestCoalescer>();
services.AddSingleton<ItemsProxyPrecacheService>(); services.AddSingleton<ItemsProxyPrecacheService>();
services.AddSingleton<ItemsProxyTransformer>(); services.AddSingleton<ItemsProxyTransformer>();
@@ -10,10 +10,8 @@ public sealed record CategoryRuleTrace(
string[] Scopes, string[] Scopes,
bool ScopeMatches, bool ScopeMatches,
bool MatchAllConditions, bool MatchAllConditions,
bool UsesStructuredRequirements,
bool Result, bool Result,
RequirementTrace[] Requirements, RequirementTrace[] Requirements);
string CriteriaText);
public sealed record RequirementTrace( public sealed record RequirementTrace(
string[] Fields, string[] Fields,
@@ -33,59 +31,40 @@ public static class CategoryRuleEvaluator
{ {
public static bool Matches(UserCategoryRule category, FactsData facts) public static bool Matches(UserCategoryRule category, FactsData facts)
{ {
return Trace(category, facts).Result; if (!ScopeMatches(category, facts.Kind))
return false;
if (category.Requirements is not { Length: > 0 })
return false;
return category.MatchAllConditions
? category.Requirements.All(requirement => MatchesRequirement(requirement, facts))
: category.Requirements.Any(requirement => MatchesRequirement(requirement, facts));
} }
private static bool MatchesRequirement(UserCategoryRequirement requirement, FactsData facts)
{
var fields = RequirementFields(requirement);
var expected = NormalizeValues(requirement.Values ?? []);
if (fields.Length == 0 || expected.Length == 0 || string.IsNullOrWhiteSpace(requirement.Relation))
return false;
bool Match(string field) => CompareRequirement(NormalizeValues(FieldValues(field, facts)),
expected, requirement.Relation.Trim(), requirement.UseOr);
return requirement.UseFieldOr ? fields.Any(Match) : fields.All(Match);
}
private static string[] NormalizeValues(IEnumerable<string> values)
=> values.Where(v => !string.IsNullOrWhiteSpace(v)).Select(v => v.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
public static CategoryRuleTrace Trace(UserCategoryRule category, FactsData facts) public static CategoryRuleTrace Trace(UserCategoryRule category, FactsData facts)
{ {
var scopeMatches = ScopeMatches(category, facts.Kind); var scopeMatches = ScopeMatches(category, facts.Kind);
if (!scopeMatches)
{
return new CategoryRuleTrace(
category.Id,
category.Label,
facts.Kind,
category.Scopes ?? [],
false,
category.MatchAllConditions,
category.Requirements is { Length: > 0 },
false,
[],
category.CriteriaText ?? string.Empty);
}
if (category.Requirements is { Length: > 0 })
{
var requirements = category.Requirements.Select(requirement => TraceRequirement(requirement, facts)).ToArray(); var requirements = category.Requirements.Select(requirement => TraceRequirement(requirement, facts)).ToArray();
var result = category.MatchAllConditions var result = scopeMatches && requirements.Length > 0 && (category.MatchAllConditions
? requirements.All(requirement => requirement.Result) ? requirements.All(requirement => requirement.Result)
: requirements.Any(requirement => requirement.Result); : requirements.Any(requirement => requirement.Result));
return new CategoryRuleTrace( return new(category.Id, category.Label, facts.Kind, category.Scopes,
category.Id, scopeMatches, category.MatchAllConditions, result, requirements);
category.Label,
facts.Kind,
category.Scopes ?? [],
true,
category.MatchAllConditions,
true,
result,
requirements,
category.CriteriaText ?? string.Empty);
}
var criteria = (category.CriteriaText ?? string.Empty).Trim();
var criteriaResult = criteria.Length > 0 && EvaluateCriteria(criteria, facts);
return new CategoryRuleTrace(
category.Id,
category.Label,
facts.Kind,
category.Scopes ?? [],
true,
category.MatchAllConditions,
false,
criteriaResult,
[],
criteria);
} }
public static string[] ParseJsonStringArray(string json) public static string[] ParseJsonStringArray(string json)
@@ -120,42 +99,6 @@ public static class CategoryRuleEvaluator
return scopes.Contains(wanted, StringComparer.OrdinalIgnoreCase); return scopes.Contains(wanted, StringComparer.OrdinalIgnoreCase);
} }
private static bool EvaluateCriteria(string criteria, FactsData facts)
{
var orGroups = SplitByOperator(criteria, "or");
foreach (var group in orGroups)
{
var terms = SplitByOperator(group, "and");
if (terms.All(term => EvaluateTerm(term, facts)))
return true;
}
return false;
}
private static bool EvaluateTerm(string term, FactsData facts)
{
term = term.Trim();
if (term.StartsWith("not ", StringComparison.OrdinalIgnoreCase))
return !EvaluateTerm(term[4..], facts);
var relation = FindRelation(term);
if (relation is null)
return false;
var field = term[..relation.Value.Index].Trim();
var rawValue = term[(relation.Value.Index + relation.Value.Token.Length)..].Trim().Trim('"', '\'');
var result = relation.Value.Token switch
{
"contains" => FieldValues(field, facts).Any(v => ValueMatches(v, rawValue, IsLanguageField(field))),
"not_contains" => !FieldValues(field, facts).Any(v => ValueMatches(v, rawValue, IsLanguageField(field))),
"is" => FieldValues(field, facts) is { Length: 1 } values && ValueMatches(values[0], rawValue, IsLanguageField(field)),
"is_not" => !(FieldValues(field, facts) is { Length: 1 } values && ValueMatches(values[0], rawValue, IsLanguageField(field))),
_ => false
};
return result;
}
private static RequirementTrace TraceRequirement(UserCategoryRequirement requirement, FactsData facts) private static RequirementTrace TraceRequirement(UserCategoryRequirement requirement, FactsData facts)
{ {
var fields = RequirementFields(requirement); var fields = RequirementFields(requirement);
@@ -201,7 +144,11 @@ public static class CategoryRuleEvaluator
.Distinct(StringComparer.OrdinalIgnoreCase) .Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray(); .ToArray();
var result = relation switch return new RequirementFieldTrace(field, actual, CompareRequirement(actual, expected, relation, useOr));
}
private static bool CompareRequirement(string[] actual, string[] expected, string relation, bool useOr)
=> relation switch
{ {
"is" => RequirementIs(actual, expected, useOr), "is" => RequirementIs(actual, expected, useOr),
"is_not" => !RequirementIs(actual, expected, useOr), "is_not" => !RequirementIs(actual, expected, useOr),
@@ -209,23 +156,9 @@ public static class CategoryRuleEvaluator
"not_contains" => !RequirementContains(actual, expected, useOr), "not_contains" => !RequirementContains(actual, expected, useOr),
_ => false _ => false
}; };
return new RequirementFieldTrace(field, actual, result);
}
private static string[] RequirementFields(UserCategoryRequirement requirement) private static string[] RequirementFields(UserCategoryRequirement requirement)
{ => NormalizeValues(requirement.Fields);
var fields = (requirement.Fields ?? [])
.Where(v => !string.IsNullOrWhiteSpace(v))
.Select(v => v.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
if (fields.Length > 0)
return fields;
var field = (requirement.Field ?? string.Empty).Trim();
return field.Length == 0 ? [] : [field];
}
private static bool RequirementIs(string[] actual, string[] expected, bool useOr) private static bool RequirementIs(string[] actual, string[] expected, bool useOr)
{ {
@@ -252,19 +185,6 @@ public static class CategoryRuleEvaluator
private static bool EqualsValue(string actual, string expected) private static bool EqualsValue(string actual, string expected)
=> string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase); => string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase);
private static (string Token, int Index)? FindRelation(string term)
{
foreach (var token in new[] { "not_contains", "contains", "is_not", "is" })
{
var needle = " " + token + " ";
var index = term.IndexOf(needle, StringComparison.OrdinalIgnoreCase);
if (index >= 0)
return (token, index + 1);
}
return null;
}
private static string[] FieldValues(string field, FactsData facts) private static string[] FieldValues(string field, FactsData facts)
{ {
field = field.Trim().ToLowerInvariant(); field = field.Trim().ToLowerInvariant();
@@ -279,44 +199,4 @@ public static class CategoryRuleEvaluator
}; };
} }
private static bool IsLanguageField(string field)
=> field.Equals("spoken_languages", StringComparison.OrdinalIgnoreCase) ||
field.Equals("original_language", StringComparison.OrdinalIgnoreCase) ||
field.Equals("audio_language", StringComparison.OrdinalIgnoreCase);
private static bool ValueMatches(string actual, string expected, bool language)
{
if (expected.Equals("empty", StringComparison.OrdinalIgnoreCase))
return string.IsNullOrWhiteSpace(actual);
if (!language)
return string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase);
if (expected.Contains('-', StringComparison.Ordinal))
return string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase);
return string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase) ||
actual.StartsWith(expected + "-", StringComparison.OrdinalIgnoreCase);
}
private static string[] SplitByOperator(string text, string op)
{
var parts = new List<string>();
var tokens = text.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var current = new List<string>();
foreach (var token in tokens)
{
if (token.Equals(op, StringComparison.OrdinalIgnoreCase))
{
parts.Add(string.Join(' ', current));
current.Clear();
}
else
{
current.Add(token);
}
}
if (current.Count > 0)
parts.Add(string.Join(' ', current));
return parts.Where(p => p.Trim().Length > 0).ToArray();
}
} }
@@ -0,0 +1,166 @@
using Jellyfin.Plugin.Multilang.Data;
using static Jellyfin.Plugin.Multilang.MultilangConstants;
namespace Jellyfin.Plugin.Multilang.Rules;
public static class UserRulesNormalizer
{
private static readonly string[] RequirementFieldOrder =
[
"original_language",
"spoken_languages",
"audio_language",
"origin_countries",
"production_countries"
];
public static UserRulesDocument Normalize(UserRulesDocument rules, IEnumerable<string>? languages = null)
{
var allowed = (languages ?? (rules.Categories ?? []).SelectMany(c => (c.FieldActionLists ?? []).Values)
.Concat((rules.FallbackFieldActions ?? []).Values).SelectMany(actions => actions ?? [])
.Where(action => action is not null && action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)).Select(action => action[LanguagePrefix.Length..]))
.ToHashSet(StringComparer.OrdinalIgnoreCase);
rules.Categories = NormalizeCategories(rules.Categories, allowed);
rules.FallbackFieldActions = NormalizeActionLists(rules.FallbackFieldActions, allowed);
return rules;
}
private static UserCategoryRule[] NormalizeCategories(IEnumerable<UserCategoryRule>? categories, IReadOnlySet<string> allowedLanguages)
=> (categories ?? [])
.Select(category => new UserCategoryRule
{
Id = string.IsNullOrWhiteSpace(category.Id) ? Guid.NewGuid().ToString("N") : category.Id.Trim(),
Label = string.IsNullOrWhiteSpace(category.Label) ? "Category" : category.Label.Trim(),
Requirements = NormalizeRequirements(category.Requirements),
MatchAllConditions = category.MatchAllConditions,
Scopes = (category.Scopes ?? [])
.Where(scope => scope is "M" or "S" or "C")
.Distinct(StringComparer.OrdinalIgnoreCase)
.DefaultIfEmpty("M")
.ToArray(),
FieldActionLists = NormalizeActionLists(category.FieldActionLists, allowedLanguages)
})
.ToArray();
private static UserCategoryRequirement[] NormalizeRequirements(IEnumerable<UserCategoryRequirement>? requirements)
=> (requirements ?? [])
.Select(requirement =>
{
var fields = NormalizeRequirementFields(requirement.Fields);
return new UserCategoryRequirement
{
Fields = fields,
UseFieldOr = requirement.UseFieldOr,
Relation = NormalizeRequirementRelation(requirement.Relation),
UseOr = requirement.UseOr,
Values = (requirement.Values ?? [])
.Where(v => !string.IsNullOrWhiteSpace(v))
.Select(v => v.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray()
};
})
.Where(requirement =>
requirement.Fields.Length > 0 &&
requirement.Relation.Length > 0 &&
requirement.Values.Length > 0)
.ToArray();
private static string[] NormalizeRequirementFields(IEnumerable<string>? fields)
{
var requested = (fields ?? [])
.Select(NormalizeRequirementField)
.Where(field => field.Length > 0)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var result = new List<string>();
string? kind = null;
foreach (var field in RequirementFieldOrder)
{
if (!requested.Contains(field))
continue;
var currentKind = IsCountryRequirementField(field) ? "country" : "language";
kind ??= currentKind;
if (currentKind == kind)
result.Add(field);
}
return result.ToArray();
}
private static string NormalizeRequirementField(string? field)
{
var normalized = (field ?? string.Empty).Trim().ToLowerInvariant();
return normalized is "original_language" or "spoken_languages" or "audio_language" or "origin_countries" or "production_countries"
? normalized
: string.Empty;
}
private static bool IsCountryRequirementField(string field)
=> field.Equals("origin_countries", StringComparison.OrdinalIgnoreCase) ||
field.Equals("production_countries", StringComparison.OrdinalIgnoreCase);
private static string NormalizeRequirementRelation(string? relation)
{
var normalized = (relation ?? string.Empty).Trim().ToLowerInvariant();
return normalized is "is" or "is_not" or "contains" or "not_contains"
? normalized
: string.Empty;
}
private static Dictionary<string, string[]> NormalizeActionLists(
IReadOnlyDictionary<string, string[]>? actions,
IReadOnlySet<string> allowedLanguages)
{
var result = DefaultActionLists();
foreach (var field in result.Keys.ToArray())
{
var raw = actions is not null && actions.TryGetValue(field, out var configured)
? configured
: [JellyfinAction];
result[field] = NormalizeActionList(field, raw, allowedLanguages);
}
return result;
}
private static string[] NormalizeActionList(string field, IEnumerable<string>? raw, IReadOnlySet<string> allowedLanguages)
{
var result = new List<string>();
foreach (var action in raw ?? [])
{
var normalized = NormalizeActionToken(action, allowedLanguages);
if (normalized.Length == 0)
continue;
if (result.Contains(normalized, StringComparer.OrdinalIgnoreCase))
continue;
result.Add(normalized);
}
if (RequiresJellyfinFallback(field))
{
if (!result.Contains(JellyfinAction, StringComparer.OrdinalIgnoreCase))
result.Add(JellyfinAction);
}
return result.ToArray();
}
private static string NormalizeActionToken(string? raw, IReadOnlySet<string> allowedLanguages)
{
var action = (raw ?? string.Empty).Trim();
if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase))
return JellyfinAction;
if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase))
return OriginalAction;
var lang = action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)
? action[LanguagePrefix.Length..].Trim()
: string.Empty;
var canonical = allowedLanguages.FirstOrDefault(l => l.Equals(lang, StringComparison.OrdinalIgnoreCase));
return string.IsNullOrWhiteSpace(canonical) ? string.Empty : LanguagePrefix + canonical;
}
}
@@ -1,4 +1,3 @@
using System.Net.Http.Headers;
using System.Security.Cryptography; using System.Security.Cryptography;
using Jellyfin.Plugin.Multilang.Data; using Jellyfin.Plugin.Multilang.Data;
@@ -18,8 +17,6 @@ public sealed class AssetStorageService
public async Task<string?> StoreAsync( public async Task<string?> StoreAsync(
string itemId, string itemId,
string lang,
string kind,
string sourceUrl, string sourceUrl,
bool localStorage, bool localStorage,
CancellationToken cancellationToken) CancellationToken cancellationToken)
@@ -30,22 +27,17 @@ public sealed class AssetStorageService
if (!localStorage) if (!localStorage)
return sourceUrl; return sourceUrl;
var relative = BuildRelativePath(itemId, lang, kind, sourceUrl); var relative = BuildRelativePath(itemId, sourceUrl);
var expectedUrl = TranslationStore.LocalAssetUrlPrefix + relative.Replace('\\', '/'); var expectedUrl = TranslationStore.LocalAssetUrlPrefix + relative.Replace('\\', '/');
if (_store.GetAssetPath(itemId, lang, kind) is { } existing &&
existing.Equals(expectedUrl, StringComparison.OrdinalIgnoreCase) &&
_store.TryNormalizeLocalAssetPath(existing, out var existingPath) &&
File.Exists(existingPath))
{
return existing;
}
var destination = Path.GetFullPath(Path.Combine(_store.AssetsDirectory, relative)); var destination = Path.GetFullPath(Path.Combine(_store.AssetsDirectory, relative));
var root = Path.GetFullPath(_store.AssetsDirectory) + Path.DirectorySeparatorChar; var root = Path.GetFullPath(_store.AssetsDirectory) + Path.DirectorySeparatorChar;
if (!destination.StartsWith(root, StringComparison.OrdinalIgnoreCase)) if (!destination.StartsWith(root, StringComparison.OrdinalIgnoreCase))
throw new InvalidOperationException("Resolved asset path escaped the Multilang assets directory."); throw new InvalidOperationException("Resolved asset path escaped the Multilang assets directory.");
Directory.CreateDirectory(Path.GetDirectoryName(destination) ?? _store.AssetsDirectory); if (File.Exists(destination))
return expectedUrl;
Directory.CreateDirectory(Path.GetDirectoryName(destination)!);
await DownloadAsync(sourceUrl, destination, cancellationToken).ConfigureAwait(false); await DownloadAsync(sourceUrl, destination, cancellationToken).ConfigureAwait(false);
return expectedUrl; return expectedUrl;
} }
@@ -101,11 +93,11 @@ public sealed class AssetStorageService
File.Move(temp, destination, overwrite: true); File.Move(temp, destination, overwrite: true);
} }
private static string BuildRelativePath(string itemId, string lang, string kind, string sourceUrl) private static string BuildRelativePath(string itemId, string sourceUrl)
{ {
var extension = ExtensionFromUrl(sourceUrl); var extension = ExtensionFromUrl(sourceUrl);
var hash = Convert.ToHexString(SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(sourceUrl))).ToLowerInvariant()[..16]; var hash = Convert.ToHexString(SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(sourceUrl))).ToLowerInvariant()[..16];
return Path.Combine(SafeSegment(itemId), SafeSegment(lang), SafeSegment(kind) + "-" + hash + extension); return Path.Combine(SafeSegment(itemId), hash + extension);
} }
private static string SafeSegment(string value) private static string SafeSegment(string value)
@@ -133,6 +125,6 @@ public sealed class AssetStorageService
".png" => "image/png", ".png" => "image/png",
".webp" => "image/webp", ".webp" => "image/webp",
".gif" => "image/gif", ".gif" => "image/gif",
_ => MediaTypeHeaderValue.Parse("application/octet-stream").MediaType ?? "application/octet-stream" _ => "application/octet-stream"
}; };
} }
@@ -1,3 +1,4 @@
using Jellyfin.Plugin.Multilang.Rules;
using System.Globalization; using System.Globalization;
using System.IO.Compression; using System.IO.Compression;
using System.Text.Json; using System.Text.Json;
@@ -142,6 +143,14 @@ public sealed class MultilangBackupService
using var zip = ZipFile.OpenRead(zipPath); using var zip = ZipFile.OpenRead(zipPath);
ValidateManifest(zip); ValidateManifest(zip);
var dbPath = Path.Combine(tempDir, DatabaseName);
var dbEntry = options.TranslationsDatabase ? zip.GetEntry(DatabaseName) : null;
if (dbEntry is not null)
{
dbEntry.ExtractToFile(dbPath);
TranslationStore.ValidateTranslationBackup(dbPath);
}
var importedConfig = false; var importedConfig = false;
var importedUsers = 0; var importedUsers = 0;
var ignoredUsers = 0; var ignoredUsers = 0;
@@ -173,21 +182,13 @@ public sealed class MultilangBackupService
continue; continue;
} }
_store.SaveUserRules(userId, payload.Rules); _store.SaveUserRules(userId, UserRulesNormalizer.Normalize(payload.Rules, currentConfiguration.Languages));
importedUsers++; importedUsers++;
} }
} }
if (options.TranslationsDatabase)
{
var dbEntry = zip.GetEntry(DatabaseName);
if (dbEntry is not null) if (dbEntry is not null)
{
var dbPath = Path.Combine(tempDir, DatabaseName);
dbEntry.ExtractToFile(dbPath, overwrite: true);
database = _store.ImportTranslationDatabase(dbPath, GetLiveItemIds(), GetAllowedLanguages(currentConfiguration)); database = _store.ImportTranslationDatabase(dbPath, GetLiveItemIds(), GetAllowedLanguages(currentConfiguration));
}
}
if (options.DownloadedAssets) if (options.DownloadedAssets)
{ {
@@ -268,7 +269,7 @@ public sealed class MultilangBackupService
if (payload?.Rules is null) if (payload?.Rules is null)
return new BackupImportResult(false, 0, 0, 0, 0, 0, 0, 0, 0, [], false); return new BackupImportResult(false, 0, 0, 0, 0, 0, 0, 0, 0, [], false);
_store.SaveUserRules(current, payload.Rules); _store.SaveUserRules(current, UserRulesNormalizer.Normalize(payload.Rules, Plugin.Instance?.Configuration.Languages));
return new BackupImportResult(false, 1, 0, 0, 0, 0, 0, 0, 0, [], false); return new BackupImportResult(false, 1, 0, 0, 0, 0, 0, 0, 0, [], false);
} }
finally finally
@@ -10,7 +10,7 @@ public sealed class ItemsProxyCache
public required string Url { get; init; } public required string Url { get; init; }
public required HashSet<string> ItemIds { get; init; } public required int ItemCount { get; init; }
public required string Body { get; init; } public required string Body { get; init; }
@@ -21,6 +21,7 @@ public sealed class ItemsProxyCache
public required long DurationMs { get; init; } public required long DurationMs { get; init; }
public required long SizeBytes { get; init; } public required long SizeBytes { get; init; }
public bool Micro { get; init; }
} }
private sealed class RecentRequest private sealed class RecentRequest
@@ -50,50 +51,44 @@ public sealed class ItemsProxyCache
private readonly object _lock = new(); private readonly object _lock = new();
private readonly Dictionary<string, Entry> _entries = new(StringComparer.Ordinal); private readonly Dictionary<string, Entry> _entries = new(StringComparer.Ordinal);
private readonly Dictionary<string, Entry> _microEntries = new(StringComparer.Ordinal); private readonly Dictionary<string, string> _userPolicies = new(StringComparer.Ordinal);
private readonly Queue<RecentRequest> _recentRequests = new(); private readonly Queue<RecentRequest> _recentRequests = new();
private long _totalBytes; private long _totalBytes;
private long _generation;
public long Generation { get { lock (_lock) return _generation; } }
private const int MaxRecentRequests = 100; private const int MaxRecentRequests = 100;
private const int MaxMicroEntries = 50; private const int MaxMicroEntries = 50;
private static readonly TimeSpan MicroTtl = TimeSpan.FromSeconds(3); private static readonly TimeSpan MicroTtl = TimeSpan.FromSeconds(3);
public long ObserveUserPolicy(string userId, string policy)
{
lock (_lock)
{
if (_userPolicies.TryGetValue(userId, out var previous) && previous != policy)
ClearUser(userId);
_userPolicies[userId] = policy;
return _generation;
}
}
public void ClearAll() public void ClearAll()
{ {
lock (_lock) lock (_lock)
{ {
_entries.Clear(); _entries.Clear();
_microEntries.Clear();
_totalBytes = 0; _totalBytes = 0;
_generation++;
} }
} }
public void InvalidateItems(IEnumerable<string> itemIds) public void ClearUser(string userId)
{ {
var ids = itemIds
.Where(id => !string.IsNullOrWhiteSpace(id))
.Select(id => id.Trim().ToLowerInvariant())
.ToHashSet(StringComparer.OrdinalIgnoreCase);
if (ids.Count == 0)
return;
lock (_lock) lock (_lock)
{ {
foreach (var key in _entries.Values _generation++;
.Where(e => e.ItemIds.Overlaps(ids)) foreach (var key in _entries.Values.Where(e => e.UserId == userId).Select(e => e.Key).ToArray())
.Select(e => e.Key)
.ToArray())
{
Remove(key); Remove(key);
} }
foreach (var key in _microEntries.Values
.Where(e => e.ItemIds.Overlaps(ids))
.Select(e => e.Key)
.ToArray())
{
_microEntries.Remove(key);
}
}
} }
public bool TryGet(string key, TimeSpan ttl, out CachedItemsProxyResponse response) public bool TryGet(string key, TimeSpan ttl, out CachedItemsProxyResponse response)
@@ -102,24 +97,17 @@ public sealed class ItemsProxyCache
{ {
PurgeExpired(ttl); PurgeExpired(ttl);
if (!_entries.TryGetValue(key, out var entry)) if (!_entries.TryGetValue(key, out var entry))
{
PurgeExpired(_microEntries, MicroTtl);
if (!_microEntries.TryGetValue(key, out var microEntry))
{ {
response = default; response = default;
return false; return false;
} }
response = new CachedItemsProxyResponse(microEntry.Body, microEntry.ContentType, microEntry.ItemIds.Count, microEntry.SizeBytes); response = new CachedItemsProxyResponse(entry.Body, entry.ContentType, entry.ItemCount, entry.SizeBytes);
return true;
}
response = new CachedItemsProxyResponse(entry.Body, entry.ContentType, entry.ItemIds.Count, entry.SizeBytes);
return true; return true;
} }
} }
public bool Store(string key, string userId, string url, IEnumerable<string> itemIds, string body, string contentType, long durationMs, long maxBytes, TimeSpan ttl) public bool Store(string key, string userId, string url, int itemCount, string body, string contentType, long durationMs, long maxBytes, TimeSpan ttl, long? generation = null, bool micro = false)
{ {
if (maxBytes <= 0 || ttl <= TimeSpan.Zero) if (maxBytes <= 0 || ttl <= TimeSpan.Zero)
return false; return false;
@@ -130,76 +118,40 @@ public sealed class ItemsProxyCache
lock (_lock) lock (_lock)
{ {
if (generation.HasValue && generation != _generation)
return false;
PurgeExpired(ttl); PurgeExpired(ttl);
Remove(key); Remove(key);
if (micro && _entries.Values.Count(e => e.Micro) >= MaxMicroEntries)
Remove(_entries.Values.Where(e => e.Micro).MinBy(e => e.CreatedUtc)!.Key);
while (_totalBytes + sizeBytes > maxBytes && _entries.Count > 0) while (_totalBytes + sizeBytes > maxBytes && _entries.Count > 0)
{ {
var victim = _entries.Values.OrderBy(e => e.CreatedUtc).First(); var victim = _entries.Values.OrderBy(e => e.CreatedUtc).First();
Remove(victim.Key); Remove(victim.Key);
} }
if (_totalBytes + sizeBytes > maxBytes)
return false;
_entries[key] = new Entry _entries[key] = new Entry
{ {
Key = key, Key = key,
UserId = userId, UserId = userId,
Url = url, Url = url,
ItemIds = itemIds ItemCount = itemCount,
.Where(id => !string.IsNullOrWhiteSpace(id))
.Select(id => id.Trim().ToLowerInvariant())
.ToHashSet(StringComparer.OrdinalIgnoreCase),
Body = body, Body = body,
ContentType = contentType, ContentType = contentType,
CreatedUtc = DateTimeOffset.UtcNow, CreatedUtc = DateTimeOffset.UtcNow,
DurationMs = durationMs, DurationMs = durationMs,
SizeBytes = sizeBytes SizeBytes = sizeBytes,
Micro = micro
}; };
_totalBytes += sizeBytes; _totalBytes += sizeBytes;
return true; return true;
} }
} }
public bool StoreMicro(string key, string userId, string url, IEnumerable<string> itemIds, string body, string contentType, long durationMs, long maxBytes) public bool StoreMicro(string key, string userId, string url, int itemCount, string body, string contentType, long durationMs, long maxBytes, long? generation = null, TimeSpan? ttl = null)
{ => Store(key, userId, url, itemCount, body, contentType, durationMs, maxBytes, ttl ?? TimeSpan.FromMinutes(1), generation, micro: true);
if (maxBytes <= 0)
return false;
var sizeBytes = System.Text.Encoding.UTF8.GetByteCount(body);
if (sizeBytes > maxBytes)
return false;
lock (_lock)
{
PurgeExpired(_microEntries, MicroTtl);
_microEntries.Remove(key);
while (_microEntries.Count >= MaxMicroEntries)
{
var victim = _microEntries.Values.OrderBy(e => e.CreatedUtc).First();
_microEntries.Remove(victim.Key);
}
_microEntries[key] = new Entry
{
Key = key,
UserId = userId,
Url = url,
ItemIds = itemIds
.Where(id => !string.IsNullOrWhiteSpace(id))
.Select(id => id.Trim().ToLowerInvariant())
.ToHashSet(StringComparer.OrdinalIgnoreCase),
Body = body,
ContentType = contentType,
CreatedUtc = DateTimeOffset.UtcNow,
DurationMs = durationMs,
SizeBytes = sizeBytes
};
return true;
}
}
public object[] GetEntries() public object[] GetEntries()
{ {
@@ -207,6 +159,7 @@ public sealed class ItemsProxyCache
{ {
var now = DateTimeOffset.UtcNow; var now = DateTimeOffset.UtcNow;
return _entries.Values return _entries.Values
.Where(e => !e.Micro)
.OrderByDescending(e => e.CreatedUtc) .OrderByDescending(e => e.CreatedUtc)
.Select(e => new .Select(e => new
{ {
@@ -215,7 +168,7 @@ public sealed class ItemsProxyCache
AgeSeconds = (long)(now - e.CreatedUtc).TotalSeconds, AgeSeconds = (long)(now - e.CreatedUtc).TotalSeconds,
e.DurationMs, e.DurationMs,
e.SizeBytes, e.SizeBytes,
ItemCount = e.ItemIds.Count ItemCount = e.ItemCount
}) })
.Cast<object>() .Cast<object>()
.ToArray(); .ToArray();
@@ -288,26 +241,16 @@ public sealed class ItemsProxyCache
return; return;
var now = DateTimeOffset.UtcNow; var now = DateTimeOffset.UtcNow;
foreach (var key in _entries.Values.Where(e => now - e.CreatedUtc > ttl).Select(e => e.Key).ToArray()) foreach (var key in _entries.Values.Where(e => now - e.CreatedUtc > (e.Micro ? MicroTtl : ttl)).Select(e => e.Key).ToArray())
Remove(key); Remove(key);
} }
private static void PurgeExpired(Dictionary<string, Entry> entries, TimeSpan ttl)
{
if (ttl <= TimeSpan.Zero || entries.Count == 0)
return;
var now = DateTimeOffset.UtcNow;
foreach (var key in entries.Values.Where(e => now - e.CreatedUtc > ttl).Select(e => e.Key).ToArray())
entries.Remove(key);
}
private void Remove(string key) private void Remove(string key)
{ {
if (!_entries.Remove(key, out var existing)) if (!_entries.Remove(key, out var existing))
return; return;
_totalBytes = Math.Max(0, _totalBytes - existing.SizeBytes); _totalBytes -= existing.SizeBytes;
} }
} }
@@ -0,0 +1,35 @@
using Jellyfin.Data.Events;
using Jellyfin.Database.Implementations.Entities;
using MediaBrowser.Controller.Library;
using Microsoft.Extensions.Hosting;
namespace Jellyfin.Plugin.Multilang.Services;
public sealed class ItemsProxyInvalidationService(
ILibraryManager library, IUserManager users, IUserDataManager userData, ItemsProxyCache cache) : IHostedService
{
public Task StartAsync(CancellationToken cancellationToken)
{
library.ItemAdded += LibraryChanged;
library.ItemUpdated += LibraryChanged;
library.ItemRemoved += LibraryChanged;
users.OnUserUpdated += UserChanged;
userData.UserDataSaved += UserDataChanged;
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken cancellationToken)
{
library.ItemAdded -= LibraryChanged;
library.ItemUpdated -= LibraryChanged;
library.ItemRemoved -= LibraryChanged;
users.OnUserUpdated -= UserChanged;
userData.UserDataSaved -= UserDataChanged;
return Task.CompletedTask;
}
// Membership and native filters can change even when an item wasn't in a cached result.
private void LibraryChanged(object? sender, ItemChangeEventArgs args) => cache.ClearAll();
private void UserChanged(object? sender, GenericEventArgs<User> args) => cache.ClearUser(args.Argument.Id.ToString("N"));
private void UserDataChanged(object? sender, UserDataSaveEventArgs args) => cache.ClearUser(args.UserId.ToString("N"));
}
@@ -60,8 +60,7 @@ public sealed class ItemsProxyPrecacheService
{ {
try try
{ {
var port = _configurationManager.GetNetworkConfiguration().InternalHttpPort; var baseUri = LocalApiUri;
var baseUri = new Uri(_applicationHost.GetLocalApiUrl("127.0.0.1", "http", port).TrimEnd('/') + "/");
var views = await GetJsonAsync(new Uri(baseUri, $"Users/{userId}/Views"), token).ConfigureAwait(false); var views = await GetJsonAsync(new Uri(baseUri, $"Users/{userId}/Views"), token).ConfigureAwait(false);
var items = views["Items"]?.AsArray() ?? []; var items = views["Items"]?.AsArray() ?? [];
var targets = items var targets = items
@@ -86,7 +85,7 @@ public sealed class ItemsProxyPrecacheService
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(baseUri, endpoint)); using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(baseUri, endpoint));
AddTokenHeaders(request, token); AddTokenHeaders(request, token);
using var response = await _httpClientFactory.CreateClient().SendAsync(request, CancellationToken.None).ConfigureAwait(false); using var response = await _httpClientFactory.CreateClient("Multilang.Jellyfin").SendAsync(request, CancellationToken.None).ConfigureAwait(false);
if (!response.IsSuccessStatusCode) if (!response.IsSuccessStatusCode)
_logger.LogWarning("Multilang pre-cache failed user={UserId} library={LibraryId} status={StatusCode}", userId, target.Id, (int)response.StatusCode); _logger.LogWarning("Multilang pre-cache failed user={UserId} library={LibraryId} status={StatusCode}", userId, target.Id, (int)response.StatusCode);
} }
@@ -106,7 +105,7 @@ public sealed class ItemsProxyPrecacheService
{ {
using var request = new HttpRequestMessage(HttpMethod.Get, uri); using var request = new HttpRequestMessage(HttpMethod.Get, uri);
AddTokenHeaders(request, token); AddTokenHeaders(request, token);
using var response = await _httpClientFactory.CreateClient().SendAsync(request, CancellationToken.None).ConfigureAwait(false); using var response = await _httpClientFactory.CreateClient("Multilang.Jellyfin").SendAsync(request, CancellationToken.None).ConfigureAwait(false);
response.EnsureSuccessStatusCode(); response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsStringAsync(CancellationToken.None).ConfigureAwait(false); var body = await response.Content.ReadAsStringAsync(CancellationToken.None).ConfigureAwait(false);
return JsonNode.Parse(body)?.AsObject() ?? throw new InvalidOperationException("Jellyfin views response was not a JSON object."); return JsonNode.Parse(body)?.AsObject() ?? throw new InvalidOperationException("Jellyfin views response was not a JSON object.");
@@ -118,4 +117,7 @@ public sealed class ItemsProxyPrecacheService
request.Headers.TryAddWithoutValidation("X-Emby-Token", token); request.Headers.TryAddWithoutValidation("X-Emby-Token", token);
request.Headers.TryAddWithoutValidation("X-MediaBrowser-Token", token); request.Headers.TryAddWithoutValidation("X-MediaBrowser-Token", token);
} }
public Uri LocalApiUri => new(_applicationHost.GetLocalApiUrl("127.0.0.1", "http",
_configurationManager.GetNetworkConfiguration().InternalHttpPort).TrimEnd('/') + "/");
} }
@@ -15,28 +15,23 @@ public sealed record ItemsProxyRequest(
public static class ItemsProxyRequestBuilder public static class ItemsProxyRequestBuilder
{ {
public static ItemsProxyRequest? Build(HttpRequest request, string rawUrl, string token, string userId, bool multilangEnabled) private const string BrowseFields = "PrimaryImageAspectRatio,MediaSourceCount,SortName,Overview,Genres,GenreItems";
public static ItemsProxyRequest? Build(HttpRequest request, string rawUrl, string userId, bool multilangEnabled)
{ {
var baseUri = $"{request.Scheme}://{request.Host}{request.PathBase}"; var baseAddress = new Uri($"{request.Scheme}://{request.Host}{request.PathBase}/");
var baseAddress = new Uri(baseUri); if (rawUrl.Contains('\\') || !Uri.TryCreate(baseAddress, rawUrl, out var uri) ||
var uri = new Uri(baseAddress, rawUrl.StartsWith('/') ? rawUrl : "/" + rawUrl); uri.Scheme != baseAddress.Scheme || uri.Authority != baseAddress.Authority ||
if ((rawUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || uri.UserInfo.Length > 0 || uri.Fragment.Length > 0)
rawUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) &&
Uri.TryCreate(rawUrl, UriKind.Absolute, out var absolute))
{
if (!absolute.Scheme.Equals(Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) &&
!absolute.Scheme.Equals(Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase))
return null; return null;
if (!absolute.Host.Equals(baseAddress.Host, StringComparison.OrdinalIgnoreCase) || var pathBase = request.PathBase.Value ?? string.Empty;
absolute.Port != baseAddress.Port) if (!uri.AbsolutePath.StartsWith(pathBase + "/", StringComparison.OrdinalIgnoreCase))
return null; return null;
var route = uri.AbsolutePath[pathBase.Length..];
uri = absolute; if (!IsSupportedRoute(route))
} return null;
var path = NormalizeUpstreamPath(route, userId);
var path = NormalizeUpstreamPath(uri.AbsolutePath, userId); var builder = new UriBuilder(uri) { Path = pathBase + path };
var builder = new UriBuilder(uri) { Path = path };
var query = QueryHelpers.ParseQuery(builder.Query); var query = QueryHelpers.ParseQuery(builder.Query);
var rawControls = new ItemsProxyControls( var rawControls = new ItemsProxyControls(
GetQueryString(query, "SortBy"), GetQueryString(query, "SortBy"),
@@ -48,35 +43,25 @@ public static class ItemsProxyRequestBuilder
GetQueryString(query, "GenreIds")); GetQueryString(query, "GenreIds"));
var hasLocalGenreFilter = multilangEnabled && ParseLocalGenreIds(rawControls.GenreIds).Length > 0; var hasLocalGenreFilter = multilangEnabled && ParseLocalGenreIds(rawControls.GenreIds).Length > 0;
var useLocalControls = multilangEnabled && ShouldUseLocalControls(path, query, hasLocalGenreFilter); var useLocalControls = multilangEnabled && ShouldUseLocalControls(path, query, hasLocalGenreFilter);
if (useLocalControls)
query["Fields"] = string.Join(',', (BrowseFields + "," + GetQueryString(query, "Fields"))
.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries)
.Distinct(StringComparer.OrdinalIgnoreCase).OrderBy(field => field, StringComparer.OrdinalIgnoreCase));
var controls = useLocalControls var controls = useLocalControls
? rawControls ? rawControls
: new ItemsProxyControls(string.Empty, string.Empty, string.Empty, 0, 0, rawControls.ClientLocale, rawControls.GenreIds); : new ItemsProxyControls(string.Empty, string.Empty, string.Empty, 0, 0, rawControls.ClientLocale, rawControls.GenreIds, ApplyLocalPaging: false);
var localCacheKey = string.Join('|',
controls.SortBy,
controls.SortOrder,
controls.NameStartsWith,
controls.StartIndex.ToString(CultureInfo.InvariantCulture),
controls.Limit.ToString(CultureInfo.InvariantCulture),
controls.ClientLocale,
controls.GenreIds);
var pairs = query var pairs = query
.Where(kv => !IsLocalProxyQuery(kv.Key, hasLocalGenreFilter, useLocalControls)) .Where(kv => !IsLocalProxyQuery(kv.Key, hasLocalGenreFilter, useLocalControls))
.SelectMany(kv => kv.Value.Select(v => KeyValuePair.Create<string, string?>(kv.Key, v ?? string.Empty))) .SelectMany(kv => kv.Value.Select(v => KeyValuePair.Create<string, string?>(kv.Key, v ?? string.Empty)))
.ToList(); .ToList();
pairs.Add(KeyValuePair.Create<string, string?>("userId", userId)); pairs.Add(KeyValuePair.Create<string, string?>("userId", userId));
if (hasLocalGenreFilter) var cacheQuery = QueryHelpers.AddQueryString(string.Empty, pairs.OrderBy(p => p.Key, StringComparer.OrdinalIgnoreCase)).TrimStart('?');
{ builder.Query = cacheQuery;
pairs.Add(KeyValuePair.Create<string, string?>("EnableImageTypes", "Primary,Backdrop,Banner,Thumb"));
pairs.Add(KeyValuePair.Create<string, string?>("Fields", "Genres,GenreItems"));
}
var cacheQuery = QueryHelpers.AddQueryString(string.Empty, pairs).TrimStart('?');
builder.Query = QueryHelpers.AddQueryString(string.Empty, pairs).TrimStart('?');
var normalizedForCache = builder.Path + (cacheQuery.Length > 0 ? "?" + cacheQuery : string.Empty); var normalizedForCache = builder.Path + (cacheQuery.Length > 0 ? "?" + cacheQuery : string.Empty);
return new ItemsProxyRequest( return new ItemsProxyRequest(
builder.Uri, builder.Uri,
controls, controls,
$"{userId}|{normalizedForCache}|{localCacheKey}", $"{userId}|{normalizedForCache}",
normalizedForCache, normalizedForCache,
IsGenresPath(path), IsGenresPath(path),
GenreMediaFromQuery(query)); GenreMediaFromQuery(query));
@@ -91,6 +76,22 @@ public static class ItemsProxyRequestBuilder
.Distinct() .Distinct()
.ToArray(); .ToArray();
private static bool IsSupportedRoute(string path)
{
var parts = path.ToLowerInvariant().Split('/', StringSplitOptions.RemoveEmptyEntries);
if (parts.Length >= 3 && parts[0] == "users" && Guid.TryParse(parts[1], out _))
parts = parts[2..];
return parts switch
{
["items"] or ["genres"] => true,
["items", "latest" or "resume" or "nextup"] => true,
["items" or "genres", var id] => Guid.TryParse(id, out _),
["shows", "nextup"] => true,
["shows", var id, "seasons" or "episodes"] => Guid.TryParse(id, out _),
_ => false
};
}
private static bool ShouldUseLocalControls( private static bool ShouldUseLocalControls(
string normalizedPath, string normalizedPath,
IReadOnlyDictionary<string, StringValues> query, IReadOnlyDictionary<string, StringValues> query,
@@ -135,17 +136,14 @@ public static class ItemsProxyRequestBuilder
key.Equals("Limit", StringComparison.OrdinalIgnoreCase) || key.Equals("Limit", StringComparison.OrdinalIgnoreCase) ||
key.Equals("NameStartsWith", StringComparison.OrdinalIgnoreCase))) || key.Equals("NameStartsWith", StringComparison.OrdinalIgnoreCase))) ||
(hasGenreFilter && (key.Equals("GenreIds", StringComparison.OrdinalIgnoreCase) || (hasGenreFilter && (key.Equals("GenreIds", StringComparison.OrdinalIgnoreCase) ||
key.Equals("EnableTotalRecordCount", StringComparison.OrdinalIgnoreCase) || key.Equals("EnableTotalRecordCount", StringComparison.OrdinalIgnoreCase)));
key.Equals("EnableImageTypes", StringComparison.OrdinalIgnoreCase) ||
key.Equals("Fields", StringComparison.OrdinalIgnoreCase)));
private static string NormalizeUpstreamPath(string path, string userId) private static string NormalizeUpstreamPath(string path, string userId)
{ {
var segments = path.Split('/', StringSplitOptions.RemoveEmptyEntries).ToList(); var segments = path.Split('/', StringSplitOptions.RemoveEmptyEntries).ToList();
if (segments.Count >= 2 && if (segments.Count >= 2 &&
segments[0].Equals("Users", StringComparison.OrdinalIgnoreCase) && segments[0].Equals("Users", StringComparison.OrdinalIgnoreCase) &&
segments[1].Length == 32 && Guid.TryParse(segments[1], out _))
segments[1].All(Uri.IsHexDigit))
{ {
segments[1] = userId; segments[1] = userId;
return "/" + string.Join('/', segments); return "/" + string.Join('/', segments);
@@ -12,7 +12,8 @@ public sealed record ItemsProxyControls(
int StartIndex, int StartIndex,
int Limit, int Limit,
string ClientLocale, string ClientLocale,
string GenreIds); string GenreIds,
bool ApplyLocalPaging = true);
public static class ItemsProxySorting public static class ItemsProxySorting
{ {
@@ -33,6 +34,8 @@ public static class ItemsProxySorting
public static void Apply(JsonNode root, ItemsProxyControls controls, CultureInfo culture, Func<JsonObject, string[]> titleArticlesForItem) public static void Apply(JsonNode root, ItemsProxyControls controls, CultureInfo culture, Func<JsonObject, string[]> titleArticlesForItem)
{ {
if (!controls.ApplyLocalPaging)
return;
if (!TryGetItemsArray(root, out var itemsArray)) if (!TryGetItemsArray(root, out var itemsArray))
return; return;
@@ -22,6 +22,9 @@ public readonly record struct ItemsProxyCategoryMatch(
UserCategoryRule? Effective, UserCategoryRule? Effective,
bool DuplicateLabelResolved); bool DuplicateLabelResolved);
public sealed record ItemsProxyResolutionAttempt(
string Action, string Source, string LookupKey, bool HasValue, bool Chosen, string? Value, string Reason);
public sealed class ItemsProxyTransformer public sealed class ItemsProxyTransformer
{ {
private readonly TranslationStore _store; private readonly TranslationStore _store;
@@ -157,8 +160,6 @@ public sealed class ItemsProxyTransformer
{ {
foreach (var action in GetAllActionTokens(category.FieldActionLists)) foreach (var action in GetAllActionTokens(category.FieldActionLists))
AddLanguageAction(langs, action); AddLanguageAction(langs, action);
foreach (var action in (category.FieldActions ?? []).Values)
AddLanguageAction(langs, action);
} }
foreach (var action in GetAllActionTokens(rules.FallbackFieldActions)) foreach (var action in GetAllActionTokens(rules.FallbackFieldActions))
@@ -184,29 +185,28 @@ public sealed class ItemsProxyTransformer
} }
public FactsData GetClassificationFacts(UserRulesDocument rules, FactsData facts) public FactsData GetClassificationFacts(UserRulesDocument rules, FactsData facts)
=> GetClassificationFacts(rules, new Dictionary<string, FactsData> { [facts.ItemId] = facts })[facts.ItemId];
private IReadOnlyDictionary<string, FactsData> GetClassificationFacts(UserRulesDocument rules, IReadOnlyDictionary<string, FactsData> facts)
{ {
if (rules.TrustTmdbCollections || if (rules.TrustTmdbCollections)
!facts.Kind.Equals("collection", StringComparison.OrdinalIgnoreCase) ||
!Guid.TryParseExact(facts.ItemId, "N", out var itemGuid))
{
return facts; return facts;
var children = facts.Values.Where(fact => fact.Kind == "collection")
.ToDictionary(fact => fact.ItemId, fact =>
(_libraryManager.GetItemById(Guid.Parse(fact.ItemId)) as Folder)?.GetLinkedChildren()
.OfType<MediaBrowser.Controller.Entities.Movies.Movie>()
.Select(child => TranslationStore.ToItemId32(child.Id)).ToArray() ?? []);
var childFacts = _store.GetFactsForItems(children.Values.SelectMany(ids => ids));
var result = new Dictionary<string, FactsData>(facts, StringComparer.OrdinalIgnoreCase);
foreach (var (id, ids) in children)
result[id] = AggregateCollectionFacts(facts[id], ids.Where(childFacts.ContainsKey).Select(id => childFacts[id]).ToArray());
return result;
} }
var item = _libraryManager.GetItemById(itemGuid); public static FactsData AggregateCollectionFacts(FactsData facts, FactsData[] childFacts)
var children = (item as Folder)?.GetLinkedChildren() {
.Where(child => child.GetType().Name.Equals("Movie", StringComparison.OrdinalIgnoreCase))
.ToArray() ?? [];
var childIds = children.Select(child => TranslationStore.ToItemId32(child.Id)).ToArray();
if (childIds.Length == 0)
return facts;
var childFacts = _store.GetFactsForItems(childIds).Values
.Where(child => child.Kind.Equals("movie", StringComparison.OrdinalIgnoreCase))
.ToArray();
if (childFacts.Length == 0) if (childFacts.Length == 0)
return facts; return facts;
return facts with return facts with
{ {
OriginalLanguage = SharedSingle(childFacts.Select(child => child.OriginalLanguage)), OriginalLanguage = SharedSingle(childFacts.Select(child => child.OriginalLanguage)),
@@ -228,74 +228,61 @@ public sealed class ItemsProxyTransformer
if (actions is not null && actions.TryGetValue(field, out var list)) if (actions is not null && actions.TryGetValue(field, out var list))
return list; return list;
return category is not null && category.FieldActions is not null return [JellyfinAction];
? LegacyActionToList(field, category.FieldActions)
: [JellyfinAction];
} }
public static ItemsProxyResolvedText ResolveField( public static ItemsProxyResolvedText ResolveField(
string field, string field, IReadOnlyList<string> actions, FactsData facts,
IReadOnlyList<string> actions, Dictionary<string, Dictionary<string, string>>? byLang,
FactsData facts, Action<ItemsProxyResolutionAttempt>? trace = null)
Dictionary<string, Dictionary<string, string>>? byLang)
{ {
foreach (var action in actions) foreach (var action in actions)
{ {
if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)) if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase))
return new ItemsProxyResolvedText(false, null);
if (field.Equals(TitleField, StringComparison.OrdinalIgnoreCase) &&
action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrWhiteSpace(facts.OriginalTitle))
return new ItemsProxyResolvedText(true, facts.OriginalTitle, facts.OriginalLanguage);
if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase))
{ {
var originalValue = GetTranslatedField(byLang, OriginalAction, field); trace?.Invoke(new(action, JellyfinAction, "", true, true, null, "Use Jellyfin value"));
if (!string.IsNullOrWhiteSpace(originalValue)) return new(false, null);
return new ItemsProxyResolvedText(true, originalValue, facts.OriginalLanguage);
continue;
} }
var original = action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase);
if (!action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)) if (!original && !action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase))
continue; continue;
var lang = original ? OriginalAction : action[LanguagePrefix.Length..];
var value = GetTranslatedField(byLang, action[LanguagePrefix.Length..], field); var originalTitle = original && field == TitleField && !string.IsNullOrWhiteSpace(facts.OriginalTitle);
if (!string.IsNullOrWhiteSpace(value)) var value = originalTitle ? facts.OriginalTitle : GetTranslatedField(byLang, lang, field);
return new ItemsProxyResolvedText(true, value, action[LanguagePrefix.Length..]); var found = !string.IsNullOrWhiteSpace(value);
trace?.Invoke(new(action, originalTitle ? "OriginalTitle" : "Translation",
originalTitle ? "facts.original_title" : lang + "/" + field,
found, found, value, found ? "Value found" : "Value missing or empty"));
if (found)
return new(true, value, original ? facts.OriginalLanguage : lang);
} }
return field == TitleField ? new(false, null) : new(true, string.Empty);
return field.Equals(TitleField, StringComparison.OrdinalIgnoreCase)
? new ItemsProxyResolvedText(false, null)
: new ItemsProxyResolvedText(true, string.Empty);
} }
public static ItemsProxyResolvedAsset ResolveAsset( public static ItemsProxyResolvedAsset ResolveAsset(
string kind, string kind, IReadOnlyList<string> actions,
IReadOnlyList<string> actions, Dictionary<string, Dictionary<string, string>>? byKind,
Dictionary<string, Dictionary<string, string>>? byKind) Action<ItemsProxyResolutionAttempt>? trace = null)
{ {
foreach (var action in actions) foreach (var action in actions)
{ {
if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase)) if (action.Equals(JellyfinAction, StringComparison.OrdinalIgnoreCase))
return new ItemsProxyResolvedAsset(false, null);
if (action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase))
{ {
var originalValue = GetAsset(byKind, kind, OriginalAction); trace?.Invoke(new(action, JellyfinAction, "", true, true, null, "Use Jellyfin image"));
if (!string.IsNullOrWhiteSpace(originalValue)) return new(false, null);
return new ItemsProxyResolvedAsset(true, originalValue);
continue;
} }
var original = action.Equals(OriginalAction, StringComparison.OrdinalIgnoreCase);
if (!action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase)) if (!original && !action.StartsWith(LanguagePrefix, StringComparison.OrdinalIgnoreCase))
continue; continue;
var lang = original ? OriginalAction : action[LanguagePrefix.Length..];
var value = GetAsset(byKind, kind, action[LanguagePrefix.Length..]); var value = GetAsset(byKind, kind, lang);
if (!string.IsNullOrWhiteSpace(value)) var found = !string.IsNullOrWhiteSpace(value);
return new ItemsProxyResolvedAsset(true, value); trace?.Invoke(new(action, "Asset", kind + "/" + lang, found, found, value,
found ? "Asset found" : "Asset missing"));
if (found)
return new(true, value);
} }
return kind == PosterKind ? new(false, null) : new(true, string.Empty);
return kind.Equals(PosterKind, StringComparison.OrdinalIgnoreCase)
? new ItemsProxyResolvedAsset(false, null)
: new ItemsProxyResolvedAsset(true, string.Empty);
} }
public static string? GetTranslatedField( public static string? GetTranslatedField(
@@ -373,6 +360,7 @@ public sealed class ItemsProxyTransformer
var langs = GetNeededLanguages(rules); var langs = GetNeededLanguages(rules);
var translations = _store.GetTranslations(ids, langs); var translations = _store.GetTranslations(ids, langs);
var assets = _store.GetAssets(ids, langs); var assets = _store.GetAssets(ids, langs);
var classification = GetClassificationFacts(rules, factsByItem);
var titleLanguages = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase); var titleLanguages = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var movieGenreNames = _store.GetGenreNames("movie", clientLocale); var movieGenreNames = _store.GetGenreNames("movie", clientLocale);
var tvGenreNames = _store.GetGenreNames("tv", clientLocale); var tvGenreNames = _store.GetGenreNames("tv", clientLocale);
@@ -384,7 +372,7 @@ public sealed class ItemsProxyTransformer
continue; continue;
translations.TryGetValue(itemId, out var byLang); translations.TryGetValue(itemId, out var byLang);
var classificationFacts = GetClassificationFacts(rules, facts); var classificationFacts = classification[itemId];
var category = PickCategoryMatch(rules, classificationFacts).Effective; var category = PickCategoryMatch(rules, classificationFacts).Effective;
var title = ResolveField(TitleField, GetActionList(rules, category, TitleField), facts, byLang); var title = ResolveField(TitleField, GetActionList(rules, category, TitleField), facts, byLang);
@@ -583,15 +571,6 @@ public sealed class ItemsProxyTransformer
langs.Add(action[LanguagePrefix.Length..]); langs.Add(action[LanguagePrefix.Length..]);
} }
private static string[] LegacyActionToList(string field, IReadOnlyDictionary<string, string>? legacyActions)
{
if (legacyActions is null || !legacyActions.TryGetValue(field, out var action))
return [JellyfinAction];
if (string.IsNullOrWhiteSpace(action) || action.Equals(FallbackAction, StringComparison.OrdinalIgnoreCase))
return [JellyfinAction];
return [action];
}
private static string SharedSingle(IEnumerable<string> values) private static string SharedSingle(IEnumerable<string> values)
{ {
var unique = values var unique = values
@@ -32,8 +32,9 @@ public sealed class FanartClient
var url = $"https://webservice.fanart.tv/v3/movies/{Uri.EscapeDataString(tmdbId)}?api_key={Uri.EscapeDataString(apiKey)}"; var url = $"https://webservice.fanart.tv/v3/movies/{Uri.EscapeDataString(tmdbId)}?api_key={Uri.EscapeDataString(apiKey)}";
using var request = new HttpRequestMessage(HttpMethod.Get, url); using var request = new HttpRequestMessage(HttpMethod.Get, url);
using var response = await ProviderHttp.SendWithRetryAsync(http, request, cancellationToken).ConfigureAwait(false); using var response = await ProviderHttp.SendWithRetryAsync(http, request, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode) if (response.StatusCode == System.Net.HttpStatusCode.NotFound)
return null; return null;
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
using var doc = await System.Text.Json.JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); using var doc = await System.Text.Json.JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
@@ -6,6 +6,7 @@ public sealed class FetchRateLimiter
{ {
private readonly TimeSpan _minimumSpacing; private readonly TimeSpan _minimumSpacing;
private long _lastStartTicks; private long _lastStartTicks;
private readonly SemaphoreSlim _admission = new(1, 1);
public FetchRateLimiter(TimeSpan minimumSpacing) public FetchRateLimiter(TimeSpan minimumSpacing)
{ {
@@ -14,15 +15,18 @@ public sealed class FetchRateLimiter
public async Task WaitAsync(CancellationToken cancellationToken) public async Task WaitAsync(CancellationToken cancellationToken)
{ {
var last = Interlocked.Read(ref _lastStartTicks); await _admission.WaitAsync(cancellationToken).ConfigureAwait(false);
if (last != 0) try
{ {
var elapsed = TimeSpan.FromSeconds((Stopwatch.GetTimestamp() - last) / (double)Stopwatch.Frequency); if (_lastStartTicks != 0)
var remaining = _minimumSpacing - elapsed; {
var remaining = _minimumSpacing - Stopwatch.GetElapsedTime(_lastStartTicks);
if (remaining > TimeSpan.Zero) if (remaining > TimeSpan.Zero)
await Task.Delay(remaining, cancellationToken).ConfigureAwait(false); await Task.Delay(remaining, cancellationToken).ConfigureAwait(false);
} }
Interlocked.Exchange(ref _lastStartTicks, Stopwatch.GetTimestamp()); _lastStartTicks = Stopwatch.GetTimestamp();
}
finally { _admission.Release(); }
} }
} }
@@ -39,29 +39,33 @@ public static class ProviderHttp
public static async Task<HttpResponseMessage> SendWithRetryAsync( public static async Task<HttpResponseMessage> SendWithRetryAsync(
HttpClient http, HttpClient http,
HttpRequestMessage request, HttpRequestMessage request,
CancellationToken cancellationToken) CancellationToken cancellationToken,
FetchRateLimiter? rateLimiter = null)
{ {
const int maxAttempts = 3; const int maxAttempts = 3;
for (var attempt = 1; attempt <= maxAttempts; attempt++) for (var attempt = 1; attempt <= maxAttempts; attempt++)
{ {
try try
{ {
var response = await http.SendAsync(Clone(request), cancellationToken).ConfigureAwait(false); if (rateLimiter is not null)
await rateLimiter.WaitAsync(cancellationToken).ConfigureAwait(false);
using var attemptRequest = Clone(request);
var response = await http.SendAsync(attemptRequest, cancellationToken).ConfigureAwait(false);
Record(request, "HTTP " + (int)response.StatusCode, attempt);
if (response.IsSuccessStatusCode || Classify(response.StatusCode) == ProviderFailureKind.Terminal || attempt == maxAttempts) if (response.IsSuccessStatusCode || Classify(response.StatusCode) == ProviderFailureKind.Terminal || attempt == maxAttempts)
{ {
Record(request, "HTTP " + (int)response.StatusCode, attempt);
return response; return response;
} }
var delay = GetRetryDelay(response, attempt); var delay = GetRetryDelay(response, attempt);
response.Dispose(); response.Dispose();
await Task.Delay(delay, cancellationToken).ConfigureAwait(false); await Task.Delay(delay < TimeSpan.Zero ? TimeSpan.Zero : delay, cancellationToken).ConfigureAwait(false);
} }
catch (Exception ex) catch (Exception ex) when (!cancellationToken.IsCancellationRequested && ex is HttpRequestException or OperationCanceledException)
{
if (attempt >= maxAttempts)
{ {
Record(request, ex.GetType().Name, attempt); Record(request, ex.GetType().Name, attempt);
if (attempt >= maxAttempts)
{
throw; throw;
} }
@@ -23,11 +23,10 @@ public sealed class TmdbClient
if (string.IsNullOrWhiteSpace(apiKey)) if (string.IsNullOrWhiteSpace(apiKey))
throw new InvalidOperationException("TMDb API key is required."); throw new InvalidOperationException("TMDb API key is required.");
await rateLimiter.WaitAsync(cancellationToken).ConfigureAwait(false);
var http = _httpClientFactory.CreateClient(); var http = _httpClientFactory.CreateClient();
var url = $"https://api.themoviedb.org/3/{BuildPath(item)}?api_key={Uri.EscapeDataString(apiKey)}&language={Uri.EscapeDataString(language)}"; var url = $"https://api.themoviedb.org/3/{BuildPath(item)}?api_key={Uri.EscapeDataString(apiKey)}&language={Uri.EscapeDataString(language)}";
using var request = new HttpRequestMessage(HttpMethod.Get, url); using var request = new HttpRequestMessage(HttpMethod.Get, url);
using var response = await ProviderHttp.SendWithRetryAsync(http, request, cancellationToken).ConfigureAwait(false); using var response = await ProviderHttp.SendWithRetryAsync(http, request, cancellationToken, rateLimiter).ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.NotFound) if (response.StatusCode == HttpStatusCode.NotFound)
return null; return null;
response.EnsureSuccessStatusCode(); response.EnsureSuccessStatusCode();
@@ -58,11 +57,10 @@ public sealed class TmdbClient
if (item.Kind is "tvepisode") if (item.Kind is "tvepisode")
return new TmdbImages([], [], []); return new TmdbImages([], [], []);
await rateLimiter.WaitAsync(cancellationToken).ConfigureAwait(false);
var http = _httpClientFactory.CreateClient(); var http = _httpClientFactory.CreateClient();
var url = $"https://api.themoviedb.org/3/{BuildPath(item)}/images?api_key={Uri.EscapeDataString(apiKey)}"; var url = $"https://api.themoviedb.org/3/{BuildPath(item)}/images?api_key={Uri.EscapeDataString(apiKey)}";
using var request = new HttpRequestMessage(HttpMethod.Get, url); using var request = new HttpRequestMessage(HttpMethod.Get, url);
using var response = await ProviderHttp.SendWithRetryAsync(http, request, cancellationToken).ConfigureAwait(false); using var response = await ProviderHttp.SendWithRetryAsync(http, request, cancellationToken, rateLimiter).ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.NotFound) if (response.StatusCode == HttpStatusCode.NotFound)
return null; return null;
response.EnsureSuccessStatusCode(); response.EnsureSuccessStatusCode();
@@ -0,0 +1,220 @@
using Jellyfin.Plugin.Multilang.Configuration;
using Jellyfin.Plugin.Multilang.Data;
using Jellyfin.Plugin.Multilang.Services.Assets;
using Jellyfin.Plugin.Multilang.Services.Providers;
using Microsoft.Extensions.Logging;
using static Jellyfin.Plugin.Multilang.MultilangConstants;
namespace Jellyfin.Plugin.Multilang.Services.Refresh;
public sealed class ItemFetcher(
TranslationStore store, TmdbClient tmdb, FanartClient fanart,
AssetStorageService storage, ILogger<ItemFetcher> logger)
{
private readonly FetchRateLimiter _limiter = new(TimeSpan.FromMilliseconds(250));
private static string Scope(RefreshItemInfo item)
=> $"{item.Kind}:{item.TmdbId}:{item.SeasonNumber}:{item.EpisodeNumber}";
public static bool NeedsFetch(FetchState? state, string scope, bool dataMissing, bool full, long cutoff)
=> full || state is null || state.Scope != scope ||
(dataMissing && (state.Complete || state.CheckedAt <= cutoff)) ||
(!state.Complete && state.CheckedAt <= cutoff);
public bool NeedsLocalWork(string itemId, PluginConfiguration cfg)
=> cfg.AssetStorageMode == "local" && store.GetStoredAssets(itemId).Any(asset => asset.SourceUrl.Length > 0 &&
(!store.TryNormalizeLocalAssetPath(asset.Path, out var path) || !File.Exists(path)));
public bool NeedsRefresh(RefreshItemInfo item, PluginConfiguration cfg)
{
var states = store.GetFetchStates(item.ItemId);
var languages = Languages(cfg);
var texts = store.GetTranslations([item.ItemId], languages.Append(OriginalAction)).GetValueOrDefault(item.ItemId) ?? [];
var cutoff = MissingCutoff(cfg);
var original = store.GetFacts(item.ItemId)?.OriginalLanguage ?? "";
foreach (var lang in languages)
if (NeedsMetadata(item, lang, lang, texts, states, false, cutoff))
return true;
if (original.Length > 0 && !languages.Any(lang => SameLanguage(lang, original)) &&
NeedsMetadata(item, OriginalAction, original, texts, states, false, cutoff))
return true;
if (original.Length > 0 && languages.Any(lang => SameLanguage(lang, original)) && !texts.ContainsKey(OriginalAction))
return true;
var assets = store.GetStoredAssets(item.ItemId);
if (cfg.AssetStorageMode == "local" && assets.Any(asset => asset.SourceUrl.Length > 0 &&
(!store.TryNormalizeLocalAssetPath(asset.Path, out var path) || !File.Exists(path))))
return true;
var providers = Providers(item, cfg);
var scope = ArtworkScope(item, languages, original, providers);
return providers.Any(provider => NeedsFetch(states.GetValueOrDefault("artwork:" + provider), scope,
ArtworkMissing(assets, languages.Append(OriginalAction).Where(lang => lang != OriginalAction || original.Length > 0), providers, provider), false, cutoff));
}
public async Task<bool> RefreshAsync(RefreshItemInfo item, PluginConfiguration cfg, RefreshJobType job, CancellationToken ct)
{
var full = job == RefreshJobType.Full;
var localOnly = job == RefreshJobType.Aggregate;
var states = store.GetFetchStates(item.ItemId);
var languages = Languages(cfg);
var texts = store.GetTranslations([item.ItemId], languages.Append(OriginalAction)).GetValueOrDefault(item.ItemId) ?? [];
var cutoff = MissingCutoff(cfg);
var fetched = 0;
async Task FetchMetadata(string storedLanguage, string requestedLanguage)
{
if (localOnly || !NeedsMetadata(item, storedLanguage, requestedLanguage, texts, states, full, cutoff))
return;
var metadata = await tmdb.FetchMetadataAsync(item, requestedLanguage, cfg.TmdbApiKey, _limiter, ct).ConfigureAwait(false);
var complete = metadata is not null && metadata.Title.Length > 0 && metadata.Overview.Length > 0 &&
(item.Kind != "movie" || metadata.Tagline.Length > 0);
store.SaveMetadata(item, storedLanguage, metadata,
new(Scope(item) + ":" + requestedLanguage, TranslationStore.NowUnixUtc(), complete), full,
item.Kind is "movie" or "tv" || storedLanguage == languages[0]);
if (metadata is not null)
Interlocked.Increment(ref fetched);
if (cfg.EnableLogging)
logger.LogInformation("TMDb metadata {Result} item={ItemId} lang={Language} storedAs={StoredLanguage}",
metadata is null ? "404" : "fetched", item.ItemId, requestedLanguage, storedLanguage);
}
await Parallel.ForEachAsync(languages, new ParallelOptions { MaxDegreeOfParallelism = 4, CancellationToken = ct },
async (lang, _) => await FetchMetadata(lang, lang).ConfigureAwait(false)).ConfigureAwait(false);
var original = store.GetFacts(item.ItemId)?.OriginalLanguage ?? "";
if (original.Length > 0)
{
var configured = languages.FirstOrDefault(lang => SameLanguage(lang, original));
if (configured is not null)
store.CopyOriginalTranslations(item.ItemId, configured);
else
await FetchMetadata(OriginalAction, original).ConfigureAwait(false);
}
var artworkFetched = await RefreshArtwork(item, cfg, languages, original, states, full, localOnly, cutoff, ct).ConfigureAwait(false);
if (!localOnly)
store.MarkFactsChecked(item, TranslationStore.NowUnixUtc(), full ? TranslationStore.NowUnixUtc() : 0);
return fetched > 0 || artworkFetched;
}
private static bool NeedsMetadata(RefreshItemInfo item, string storedLanguage, string requestedLanguage,
Dictionary<string, Dictionary<string, string>> texts, Dictionary<string, FetchState> states, bool full, long cutoff)
{
var fields = texts.GetValueOrDefault(storedLanguage);
var missing = fields is null || TextFields.Any(field => !fields.ContainsKey(field) ||
((field != TaglineField || item.Kind == "movie") && string.IsNullOrWhiteSpace(fields[field])));
return NeedsFetch(states.GetValueOrDefault("metadata:" + storedLanguage),
Scope(item) + ":" + requestedLanguage, missing, full, cutoff);
}
private async Task<bool> RefreshArtwork(RefreshItemInfo item, PluginConfiguration cfg, string[] languages,
string original, Dictionary<string, FetchState> states, bool full, bool localOnly, long cutoff, CancellationToken ct)
{
if (localOnly)
{
if (cfg.AssetStorageMode != "local")
return false;
var assets = store.GetStoredAssets(item.ItemId);
var repaired = new List<StoredAsset>();
foreach (var asset in assets)
{
var path = asset.SourceUrl.Length == 0 ? asset.Path
: await storage.StoreAsync(item.ItemId, asset.SourceUrl, true, ct).ConfigureAwait(false);
repaired.Add(asset with { Path = path! });
}
store.SaveArtwork(item.ItemId, repaired, new Dictionary<string, FetchState>());
return true;
}
var providers = Providers(item, cfg);
var priority = providers.Select((provider, index) => (provider, index)).ToDictionary(p => p.provider, p => p.index);
var wanted = languages.ToDictionary(lang => lang, BaseLanguage, StringComparer.OrdinalIgnoreCase);
if (original.Length > 0)
wanted[OriginalAction] = BaseLanguage(original);
var previous = store.GetStoredAssets(item.ItemId);
var selected = previous.Where(asset => wanted.ContainsKey(asset.Language) && priority.ContainsKey(asset.Provider))
.ToDictionary(asset => (asset.Language, asset.Kind));
var updates = new Dictionary<string, FetchState>();
var scope = ArtworkScope(item, languages, original, providers);
foreach (var provider in providers)
{
var kinds = ArtworkKinds(provider);
var keys = wanted.Keys.SelectMany(lang => kinds.Select(kind => (Language: lang, Kind: kind))).ToArray();
var missing = keys.Any(key => !selected.TryGetValue(key, out var asset) || priority[asset.Provider] > priority[provider]);
if (localOnly || !NeedsFetch(states.GetValueOrDefault("artwork:" + provider), scope, missing, full, cutoff))
continue;
Dictionary<string, Dictionary<string, string>> images;
if (provider == "tmdb")
{
var response = await tmdb.FetchImagesAsync(item, cfg.TmdbApiKey, _limiter, ct).ConfigureAwait(false);
images = new() { [PosterKind] = response?.Posters ?? [], [LogoKind] = response?.Logos ?? [], [BackdropKind] = response?.Backdrops ?? [] };
}
else
{
var response = await fanart.FetchMovieImagesAsync(item.TmdbId, cfg.FanartApiKey, ct).ConfigureAwait(false);
images = new() { [PosterKind] = response?.Posters ?? [], [LogoKind] = response?.Logos ?? [], [BannerKind] = response?.Banners ?? [], [ThumbKind] = response?.Thumbs ?? [] };
}
var complete = true;
foreach (var key in keys)
{
selected.TryGetValue(key, out var existing);
var url = images[key.Kind].GetValueOrDefault(wanted[key.Language]);
if (string.IsNullOrWhiteSpace(url))
{
complete = false;
if (existing?.Provider == provider)
selected.Remove(key);
}
else if (existing is null || priority[provider] <= priority[existing.Provider])
{
// Keep an existing local path until the scheduled storage-mode cleanup.
var path = existing?.SourceUrl == url ? existing.Path : url;
selected[key] = new(key.Language, key.Kind, path, url, provider);
}
}
updates["artwork:" + provider] = new(scope, TranslationStore.NowUnixUtc(), complete);
if (cfg.EnableLogging && cfg.VerboseLogging)
logger.LogInformation("Artwork fetched item={ItemId} provider={Provider} languages={Languages}",
item.ItemId, provider, string.Join(",", wanted.Keys));
}
var changed = updates.Count > 0;
if (cfg.AssetStorageMode == "local")
{
foreach (var key in selected.Keys.ToArray())
{
var asset = selected[key];
var path = await storage.StoreAsync(item.ItemId, asset.SourceUrl, true, ct).ConfigureAwait(false);
selected[key] = asset with { Path = path! };
changed |= path != asset.Path;
}
}
var result = selected.Values.ToArray();
if (changed || !previous.OrderBy(a => a.Language).ThenBy(a => a.Kind).SequenceEqual(result.OrderBy(a => a.Language).ThenBy(a => a.Kind)))
store.SaveArtwork(item.ItemId, result, updates);
return changed;
}
private static string[] ArtworkKinds(string provider)
=> provider == "tmdb" ? [PosterKind, LogoKind, BackdropKind] : [PosterKind, LogoKind, BannerKind, ThumbKind];
private static bool ArtworkMissing(IEnumerable<StoredAsset> assets, IEnumerable<string> languages, string[] providers, string provider)
{
var available = assets.Where(asset => Array.IndexOf(providers, asset.Provider) is var rank && rank >= 0 && rank <= Array.IndexOf(providers, provider))
.Select(asset => (asset.Language, asset.Kind)).ToHashSet();
return languages.Any(lang => ArtworkKinds(provider).Any(kind => !available.Contains((lang, kind))));
}
private static string[] Providers(RefreshItemInfo item, PluginConfiguration cfg)
=> item.Kind == "tvepisode" ? [] : cfg.Providers.Where(p => p.ArtworkOrder >= 0)
.OrderBy(p => p.ArtworkOrder).Select(p => p.Id.Trim().ToLowerInvariant())
.Where(p => p == "tmdb" || (p == "fanart" && item.Kind == "movie" && !string.IsNullOrWhiteSpace(cfg.FanartApiKey))).ToArray();
private static string ArtworkScope(RefreshItemInfo item, string[] languages, string original, string[] providers)
=> Scope(item) + "|" + string.Join(",", languages.Order(StringComparer.OrdinalIgnoreCase)) + "|" + original + "|" + string.Join(",", providers);
private static long MissingCutoff(PluginConfiguration cfg)
=> TranslationStore.NowUnixUtc() - (long)TimeSpan.FromDays(Math.Max(0, cfg.WaitDaysForMissingData)).TotalSeconds;
private static string[] Languages(PluginConfiguration cfg)
=> cfg.Languages.Select(lang => lang.Trim()).Where(lang => lang.Length > 0).Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
private static string BaseLanguage(string lang) => lang.Split('-')[0].ToLowerInvariant();
private static bool SameLanguage(string left, string right) => BaseLanguage(left) == BaseLanguage(right);
}
@@ -3,7 +3,7 @@ using System.Diagnostics;
namespace Jellyfin.Plugin.Multilang.Services.Refresh; namespace Jellyfin.Plugin.Multilang.Services.Refresh;
public sealed class RefreshCoordinator public sealed class RefreshCoordinator : IDisposable
{ {
private sealed class QueueEntry private sealed class QueueEntry
{ {
@@ -20,21 +20,22 @@ public sealed class RefreshCoordinator
public required TaskCompletionSource<bool> Completion { get; init; } public required TaskCompletionSource<bool> Completion { get; init; }
public long Sequence { get; init; } public long Sequence { get; init; }
public bool Active { get; set; }
} }
private readonly object _lock = new(); private readonly object _lock = new();
private readonly Dictionary<string, QueueEntry> _entries = new(StringComparer.OrdinalIgnoreCase); private readonly Dictionary<string, QueueEntry> _entries = new(StringComparer.OrdinalIgnoreCase);
private readonly Queue<RefreshRecentItem> _recent = new(); private readonly Queue<RefreshRecentItem> _recent = new();
private readonly SemaphoreSlim _signal = new(0, int.MaxValue); private readonly SemaphoreSlim _signal = new(0, int.MaxValue);
private readonly ItemsProxyCache _itemsProxyCache;
private long _sequence; private long _sequence;
private QueueEntry? _active; private readonly CancellationTokenSource _stop = new();
public SemaphoreSlim ScanMutex { get; } = new(1, 1); public SemaphoreSlim ScanMutex { get; } = new(1, 1);
public RefreshCoordinator(ItemsProxyCache itemsProxyCache) public RefreshCoordinator(int concurrency = MultilangConstants.RefreshConcurrency)
{ {
_itemsProxyCache = itemsProxyCache; ArgumentOutOfRangeException.ThrowIfLessThan(concurrency, 1);
for (var i = 0; i < concurrency; i++)
_ = Task.Run(WorkerLoop); _ = Task.Run(WorkerLoop);
} }
@@ -53,6 +54,7 @@ public sealed class RefreshCoordinator
lock (_lock) lock (_lock)
{ {
ObjectDisposedException.ThrowIf(_stop.IsCancellationRequested, this);
if (_entries.TryGetValue(itemId, out var existing)) if (_entries.TryGetValue(itemId, out var existing))
{ {
if (jobType > existing.JobType) if (jobType > existing.JobType)
@@ -64,7 +66,7 @@ public sealed class RefreshCoordinator
if (sourceTier > existing.SourceTier) if (sourceTier > existing.SourceTier)
existing.SourceTier = sourceTier; existing.SourceTier = sourceTier;
return existing.Completion.Task; return cancellationToken.CanBeCanceled ? existing.Completion.Task.WaitAsync(cancellationToken) : existing.Completion.Task;
} }
var entry = new QueueEntry var entry = new QueueEntry
@@ -80,7 +82,7 @@ public sealed class RefreshCoordinator
_entries.Add(itemId, entry); _entries.Add(itemId, entry);
_signal.Release(); _signal.Release();
return entry.Completion.Task; return cancellationToken.CanBeCanceled ? entry.Completion.Task.WaitAsync(cancellationToken) : entry.Completion.Task;
} }
} }
@@ -88,7 +90,8 @@ public sealed class RefreshCoordinator
{ {
while (true) while (true)
{ {
await _signal.WaitAsync().ConfigureAwait(false); try { await _signal.WaitAsync(_stop.Token).ConfigureAwait(false); }
catch (OperationCanceledException) { return; }
QueueEntry? entry; QueueEntry? entry;
lock (_lock) lock (_lock)
@@ -102,12 +105,24 @@ public sealed class RefreshCoordinator
var error = string.Empty; var error = string.Empty;
try try
{ {
while (true)
{
RefreshJobType executing;
Func<RefreshJobType, CancellationToken, Task<bool>> work;
lock (_lock) lock (_lock)
_active = entry; {
executing = entry.JobType;
ok = await entry.Work(entry.JobType, CancellationToken.None).ConfigureAwait(false); work = entry.Work;
if (ok) }
_itemsProxyCache.InvalidateItems([entry.ItemId]); ok |= await work(executing, _stop.Token).ConfigureAwait(false);
lock (_lock)
{
if (entry.JobType > executing)
continue;
_entries.Remove(entry.ItemId);
break;
}
}
entry.Completion.TrySetResult(ok); entry.Completion.TrySetResult(ok);
} }
catch (Exception ex) catch (Exception ex)
@@ -120,8 +135,7 @@ public sealed class RefreshCoordinator
sw.Stop(); sw.Stop();
lock (_lock) lock (_lock)
{ {
if (ReferenceEquals(_active, entry)) if (_entries.TryGetValue(entry.ItemId, out var current) && ReferenceEquals(current, entry))
_active = null;
_entries.Remove(entry.ItemId); _entries.Remove(entry.ItemId);
_recent.Enqueue(new RefreshRecentItem( _recent.Enqueue(new RefreshRecentItem(
entry.ItemId, entry.ItemId,
@@ -144,11 +158,14 @@ public sealed class RefreshCoordinator
if (_entries.Count == 0) if (_entries.Count == 0)
return null; return null;
return _entries.Values var entry = _entries.Values
.Where(e => !e.Active)
.OrderByDescending(e => e.SourceTier) .OrderByDescending(e => e.SourceTier)
.ThenBy(e => e.WorkClass) .ThenBy(e => e.WorkClass)
.ThenBy(e => e.Sequence) .ThenBy(e => e.Sequence)
.First(); .FirstOrDefault();
if (entry is not null) entry.Active = true;
return entry;
} }
public RefreshQueueDiagnostics GetDiagnostics() public RefreshQueueDiagnostics GetDiagnostics()
@@ -156,7 +173,7 @@ public sealed class RefreshCoordinator
lock (_lock) lock (_lock)
{ {
var queued = _entries.Values var queued = _entries.Values
.Where(e => !ReferenceEquals(e, _active)) .Where(e => !e.Active)
.OrderByDescending(e => e.SourceTier) .OrderByDescending(e => e.SourceTier)
.ThenBy(e => e.WorkClass) .ThenBy(e => e.WorkClass)
.ThenBy(e => e.Sequence) .ThenBy(e => e.Sequence)
@@ -166,7 +183,7 @@ public sealed class RefreshCoordinator
return new RefreshQueueDiagnostics( return new RefreshQueueDiagnostics(
queued.Length, queued.Length,
queued, queued,
_active is null ? null : ToDiagnostics(_active), _entries.Values.Where(e => e.Active).Select(ToDiagnostics).ToArray(),
_recent.Reverse().ToArray()); _recent.Reverse().ToArray());
} }
} }
@@ -178,4 +195,15 @@ public sealed class RefreshCoordinator
entry.WorkClass.ToString(), entry.WorkClass.ToString(),
entry.JobType.ToString(), entry.JobType.ToString(),
entry.Sequence); entry.Sequence);
public void Dispose()
{
_stop.Cancel();
lock (_lock)
{
foreach (var entry in _entries.Values)
entry.Completion.TrySetCanceled();
_entries.Clear();
}
}
} }
@@ -53,7 +53,7 @@ public sealed record RefreshRecentItem(
public sealed record RefreshQueueDiagnostics( public sealed record RefreshQueueDiagnostics(
int QueuedCount, int QueuedCount,
RefreshQueueEntry[] Queued, RefreshQueueEntry[] Queued,
RefreshQueueEntry? Active, RefreshQueueEntry[] ActiveItems,
RefreshRecentItem[] Recent); RefreshRecentItem[] Recent);
public sealed record RefreshScanDiagnostics( public sealed record RefreshScanDiagnostics(
@@ -2,7 +2,6 @@ using System.Diagnostics;
using Jellyfin.Data.Enums; using Jellyfin.Data.Enums;
using Jellyfin.Plugin.Multilang.Configuration; using Jellyfin.Plugin.Multilang.Configuration;
using Jellyfin.Plugin.Multilang.Data; using Jellyfin.Plugin.Multilang.Data;
using Jellyfin.Plugin.Multilang.Services.Assets;
using Jellyfin.Plugin.Multilang.Services.Providers; using Jellyfin.Plugin.Multilang.Services.Providers;
using static Jellyfin.Plugin.Multilang.MultilangConstants; using static Jellyfin.Plugin.Multilang.MultilangConstants;
using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities;
@@ -16,9 +15,7 @@ public sealed class RefreshService
{ {
private readonly ILibraryManager _libraryManager; private readonly ILibraryManager _libraryManager;
private readonly TranslationStore _store; private readonly TranslationStore _store;
private readonly TmdbClient _tmdbClient; private readonly ItemFetcher _fetcher;
private readonly FanartClient _fanartClient;
private readonly AssetStorageService _assetStorage;
private readonly RefreshCoordinator _coordinator; private readonly RefreshCoordinator _coordinator;
private readonly ILogger<RefreshService> _logger; private readonly ILogger<RefreshService> _logger;
private readonly object _diagnosticsLock = new(); private readonly object _diagnosticsLock = new();
@@ -27,17 +24,13 @@ public sealed class RefreshService
public RefreshService( public RefreshService(
ILibraryManager libraryManager, ILibraryManager libraryManager,
TranslationStore store, TranslationStore store,
TmdbClient tmdbClient, ItemFetcher fetcher,
FanartClient fanartClient,
AssetStorageService assetStorage,
RefreshCoordinator coordinator, RefreshCoordinator coordinator,
ILogger<RefreshService> logger) ILogger<RefreshService> logger)
{ {
_libraryManager = libraryManager; _libraryManager = libraryManager;
_store = store; _store = store;
_tmdbClient = tmdbClient; _fetcher = fetcher;
_fanartClient = fanartClient;
_assetStorage = assetStorage;
_coordinator = coordinator; _coordinator = coordinator;
_logger = logger; _logger = logger;
} }
@@ -151,7 +144,6 @@ public sealed class RefreshService
UpdateRunningScan(); UpdateRunningScan();
var items = GetAllItemsWithTmdbId(); var items = GetAllItemsWithTmdbId();
var now = TranslationStore.NowUnixUtc(); var now = TranslationStore.NowUnixUtc();
var missingCutoff = now - (long)TimeSpan.FromDays(Math.Max(0, cfg.WaitDaysForMissingData)).TotalSeconds;
var fullCutoff = now - (long)TimeSpan.FromDays(Math.Max(0, cfg.WaitDaysForExistingData)).TotalSeconds; var fullCutoff = now - (long)TimeSpan.FromDays(Math.Max(0, cfg.WaitDaysForExistingData)).TotalSeconds;
var ordered = items.OrderBy(i => i.WorkClass).ToList(); var ordered = items.OrderBy(i => i.WorkClass).ToList();
itemCount = ordered.Count; itemCount = ordered.Count;
@@ -162,7 +154,14 @@ public sealed class RefreshService
// Run local housekeeping only from full-library scans. ItemsProxy browsing may enqueue // Run local housekeeping only from full-library scans. ItemsProxy browsing may enqueue
// missing items, but it must not delete local assets just because the admin briefly // missing items, but it must not delete local assets just because the admin briefly
// changed artwork storage mode. // changed artwork storage mode.
var localCleanup = _store.CleanupForConfiguration(liveIds, NormalizeLanguages(cfg.Languages), IsLocalAssetStorage(cfg)); LocalCleanupResult localCleanup;
using (await _store.EnterMaintenanceAsync(cancellationToken).ConfigureAwait(false))
{
cfg = Plugin.Instance?.Configuration ?? throw new InvalidOperationException("Plugin unavailable.");
if (Plugin.Instance.Uninstalling)
throw new OperationCanceledException("Plugin is being uninstalled.");
localCleanup = _store.CleanupForConfiguration(liveIds, NormalizeLanguages(cfg.Languages), IsLocalAssetStorage(cfg));
}
if (localCleanup.TranslationsDeleted > 0 || localCleanup.AssetsDeleted > 0 || localCleanup.GenresDeleted > 0 || localCleanup.AssetFilesDeleted > 0) if (localCleanup.TranslationsDeleted > 0 || localCleanup.AssetsDeleted > 0 || localCleanup.GenresDeleted > 0 || localCleanup.AssetFilesDeleted > 0)
{ {
_logger.LogInformation( _logger.LogInformation(
@@ -176,41 +175,48 @@ public sealed class RefreshService
_logger.LogInformation("Multilang refresh started mode={Mode} source={Source} items={Items} new={New} deleted={Deleted}", jobType, runSource, ordered.Count, newCount, deletedCount); _logger.LogInformation("Multilang refresh started mode={Mode} source={Source} items={Items} new={New} deleted={Deleted}", jobType, runSource, ordered.Count, newCount, deletedCount);
var index = 0; var index = 0;
UpdateRunningScan(); UpdateRunningScan();
foreach (var item in ordered) var completed = 0;
var scanLock = new object();
foreach (var group in ordered.GroupBy(item => item.WorkClass))
await Parallel.ForEachAsync(group,
new ParallelOptions { MaxDegreeOfParallelism = RefreshConcurrency, CancellationToken = cancellationToken },
async (item, ct) =>
{ {
cancellationToken.ThrowIfCancellationRequested(); ct.ThrowIfCancellationRequested();
index++; var scanIndex = Interlocked.Increment(ref index);
SetCurrent(item, index, "checking");
var status = _store.GetFactsStatus(item.ItemId); var status = _store.GetFactsStatus(item.ItemId);
var isNew = !status.Exists; var isNew = !status.Exists;
var missingConfiguredData = jobType == RefreshJobType.Missing && status.Exists && ItemNeedsMissingRefresh(item, cfg, langs: null); var fullDue = jobType == RefreshJobType.Full && (!status.Exists || status.FullCheckedAt < fullCutoff);
var due = jobType == RefreshJobType.Full var due = fullDue || (jobType == RefreshJobType.Missing
? !status.Exists || status.FullCheckedAt < fullCutoff ? !status.Exists || _fetcher.NeedsRefresh(item, cfg)
: !status.Exists || status.MissingCheckedAt < missingCutoff || missingConfiguredData; : _fetcher.NeedsLocalWork(item.ItemId, cfg));
if (!due) if (!due)
{ {
skippedNotDueCount++; Interlocked.Increment(ref skippedNotDueCount);
if (cfg.VerboseLogging && cfg.EnableLogging) if (cfg.VerboseLogging && cfg.EnableLogging)
{ {
_logger.LogInformation( _logger.LogInformation(
"Multilang refresh skip [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind} reason=not-due", "Multilang refresh skip [{Index}/{Total}] item={ItemId} name=\"{Name}\" kind={Kind} reason=not-due",
index, scanIndex,
ordered.Count, ordered.Count,
item.ItemId, item.ItemId,
item.DisplayName, item.DisplayName,
item.Kind); item.Kind);
} }
progress.Report(index * 100.0 / Math.Max(1, ordered.Count)); lock (scanLock)
UpdateRunningScan("skipped-not-due"); {
continue; SetCurrent(item, completed + 1, "skipped-not-due");
progress.Report(++completed * 100.0 / Math.Max(1, ordered.Count));
UpdateRunningScan();
}
return;
} }
var scanIndex = index; var actualJobType = fullDue || !status.Exists ? RefreshJobType.Full
var actualJobType = jobType == RefreshJobType.Full || !status.Exists ? RefreshJobType.Full : RefreshJobType.Missing; : jobType == RefreshJobType.Full ? RefreshJobType.Aggregate : RefreshJobType.Missing;
dueCount++; Interlocked.Increment(ref dueCount);
SetCurrent(item, scanIndex, "fetching");
Log( Log(
cfg.EnableLogging, cfg.EnableLogging,
LogLevel.Information, LogLevel.Information,
@@ -229,19 +235,30 @@ public sealed class RefreshService
RefreshSourceTier.Background, RefreshSourceTier.Background,
item.WorkClass, item.WorkClass,
actualJobType, actualJobType,
async (actualJobType, ct) => await RefreshItemCoreAsync(item, cfg, actualJobType, scanIndex, ordered.Count, ct).ConfigureAwait(false), async (actualJobType, ct) => await RefreshItemCoreAsync(item, actualJobType, ct).ConfigureAwait(false),
cancellationToken).ConfigureAwait(false); ct).ConfigureAwait(false);
lock (scanLock)
{
if (ok) if (ok)
refreshedCount++; refreshedCount++;
else else
skippedNoDataCount++; skippedNoDataCount++;
progress.Report(index * 100.0 / Math.Max(1, ordered.Count)); progress.Report(++completed * 100.0 / Math.Max(1, ordered.Count));
UpdateRunningScan(ok ? "refreshed" : "skipped-no-data"); SetCurrent(item, completed, ok ? "refreshed" : "skipped-no-data");
UpdateRunningScan();
}
}).ConfigureAwait(false);
using (await _store.EnterMaintenanceAsync(cancellationToken).ConfigureAwait(false))
{
if (Plugin.Instance!.Uninstalling)
throw new OperationCanceledException("Plugin is being uninstalled.");
_store.CleanupUnreferencedAssetFiles();
_store.SetLastScanStarted(now);
} }
sw.Stop(); sw.Stop();
_store.SetLastScanStarted(now);
CompleteScan(true, string.Empty); CompleteScan(true, string.Empty);
_logger.LogInformation( _logger.LogInformation(
"Multilang refresh complete mode={Mode} source={Source} elapsed={ElapsedMs}ms items={Items} due={Due} refreshed={Refreshed} skippedNotDue={SkippedNotDue} skippedNoData={SkippedNoData} new={New} deleted={Deleted}", "Multilang refresh complete mode={Mode} source={Source} elapsed={ElapsedMs}ms items={Items} due={Due} refreshed={Refreshed} skippedNotDue={SkippedNotDue} skippedNoData={SkippedNoData} new={New} deleted={Deleted}",
@@ -268,7 +285,8 @@ public sealed class RefreshService
} }
} }
public async Task RefreshItemAsync(Guid itemId, bool includeChildren, CancellationToken cancellationToken) public async Task RefreshItemAsync(Guid itemId, bool includeChildren, CancellationToken cancellationToken,
Jellyfin.Database.Implementations.Entities.User user)
{ {
var cfg = Plugin.Instance?.Configuration ?? throw new InvalidOperationException("Plugin configuration unavailable."); var cfg = Plugin.Instance?.Configuration ?? throw new InvalidOperationException("Plugin configuration unavailable.");
ValidateRefreshConfiguration(cfg); ValidateRefreshConfiguration(cfg);
@@ -279,14 +297,17 @@ public sealed class RefreshService
return; return;
} }
var items = BuildItemInfos(GetItemAndChildren(root, includeChildren)).ToList(); var items = BuildItemInfos(GetItemAndChildren(root, includeChildren)
.Where(item => item.IsVisibleStandalone(user))).ToList();
var total = items.Count; var total = items.Count;
var index = 0; var index = 0;
_logger.LogInformation("Manual Multilang refresh started item={ItemId} name={Name} items={Items} includeChildren={IncludeChildren}", itemId, root.Name ?? string.Empty, total, includeChildren); _logger.LogInformation("Manual Multilang refresh started item={ItemId} name={Name} items={Items} includeChildren={IncludeChildren}", itemId, root.Name ?? string.Empty, total, includeChildren);
foreach (var item in items) foreach (var group in items.OrderBy(item => item.WorkClass).GroupBy(item => item.WorkClass))
await Parallel.ForEachAsync(group,
new ParallelOptions { MaxDegreeOfParallelism = RefreshConcurrency, CancellationToken = cancellationToken },
async (item, ct) =>
{ {
index++; var scanIndex = Interlocked.Increment(ref index);
var scanIndex = index;
Log( Log(
cfg.EnableLogging, cfg.EnableLogging,
LogLevel.Information, LogLevel.Information,
@@ -303,8 +324,8 @@ public sealed class RefreshService
RefreshSourceTier.Manual, RefreshSourceTier.Manual,
item.WorkClass, item.WorkClass,
RefreshJobType.Full, RefreshJobType.Full,
async (actualJobType, ct) => await RefreshItemCoreAsync(item, cfg, actualJobType, scanIndex, total, ct).ConfigureAwait(false), async (actualJobType, ct) => await RefreshItemCoreAsync(item, actualJobType, ct).ConfigureAwait(false),
cancellationToken).ConfigureAwait(false); ct).ConfigureAwait(false);
Log( Log(
cfg.EnableLogging, cfg.EnableLogging,
ok ? LogLevel.Information : LogLevel.Warning, ok ? LogLevel.Information : LogLevel.Warning,
@@ -315,7 +336,7 @@ public sealed class RefreshService
item.ItemId, item.ItemId,
item.DisplayName, item.DisplayName,
item.Kind); item.Kind);
} }).ConfigureAwait(false);
} }
public bool EnqueueOnTheFlyMissing(IEnumerable<string> itemIds) public bool EnqueueOnTheFlyMissing(IEnumerable<string> itemIds)
@@ -365,7 +386,7 @@ public sealed class RefreshService
sourceTier, sourceTier,
info.WorkClass, info.WorkClass,
RefreshJobType.Full, RefreshJobType.Full,
async (actualJobType, ct) => await RefreshItemCoreAsync(info, cfg, actualJobType, 0, 0, ct).ConfigureAwait(false), async (actualJobType, ct) => await RefreshItemCoreAsync(info, actualJobType, ct).ConfigureAwait(false),
CancellationToken.None) CancellationToken.None)
.ContinueWith( .ContinueWith(
task => task =>
@@ -388,245 +409,16 @@ public sealed class RefreshService
private async Task<bool> RefreshItemCoreAsync( private async Task<bool> RefreshItemCoreAsync(
RefreshItemInfo item, RefreshItemInfo item,
PluginConfiguration cfg,
RefreshJobType jobType, RefreshJobType jobType,
int scanIndex,
int totalItems,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
var langs = NormalizeLanguages(cfg.Languages); using var lease = await _store.EnterRefreshAsync(cancellationToken).ConfigureAwait(false);
if (langs.Length == 0) var plugin = Plugin.Instance ?? throw new InvalidOperationException("Plugin unavailable.");
throw new InvalidOperationException("No Multilang languages are configured."); if (plugin.Uninstalling)
throw new OperationCanceledException("Plugin is being uninstalled.");
if (cfg.VerboseLogging && cfg.EnableLogging) var cfg = plugin.Configuration;
_logger.LogInformation("[{Index}/{Total}] Fetching item={ItemId} name=\"{Name}\" kind={Kind} tmdb={TmdbId}", scanIndex, totalItems, item.ItemId, item.DisplayName, item.Kind, item.TmdbId); ValidateRefreshConfiguration(cfg);
return await _fetcher.RefreshAsync(item, cfg, jobType, cancellationToken).ConfigureAwait(false);
var now = TranslationStore.NowUnixUtc();
var limiter = new FetchRateLimiter(TimeSpan.FromMilliseconds(250));
var wroteAny = false;
TmdbMetadata? firstMeta = null;
var fetchedMetadata = new Dictionary<string, TmdbMetadata>(StringComparer.OrdinalIgnoreCase);
foreach (var lang in langs)
{
var meta = await _tmdbClient.FetchMetadataAsync(item, lang, cfg.TmdbApiKey, limiter, cancellationToken).ConfigureAwait(false);
if (meta is null)
{
Log(cfg.EnableLogging, LogLevel.Information, "[{Index}/{Total}] TMDb 404, skipping item={ItemId} name=\"{Name}\" tmdb={TmdbId} kind={Kind}", scanIndex, totalItems, item.ItemId, item.DisplayName, item.TmdbId, item.Kind);
_store.MarkFactsChecked(item, now, jobType == RefreshJobType.Full ? now : 0);
return false;
}
Log(cfg.VerboseLogging && cfg.EnableLogging, LogLevel.Information, "[{Index}/{Total}] TMDb metadata ok item={ItemId} lang={Lang}", scanIndex, totalItems, item.ItemId, lang);
_store.UpsertTranslation(item.ItemId, lang, TitleField, meta.Title);
_store.UpsertTranslation(item.ItemId, lang, OverviewField, meta.Overview);
_store.UpsertTranslation(item.ItemId, lang, TaglineField, meta.Tagline);
StoreGenres(item, lang, meta);
fetchedMetadata[lang] = meta;
firstMeta ??= meta;
if (!wroteAny)
{
_store.UpsertFacts(item, meta, now, jobType == RefreshJobType.Full ? now : 0);
wroteAny = true;
}
}
var artworkLanguages = langs;
if (firstMeta is not null && !string.IsNullOrWhiteSpace(firstMeta.OriginalLanguage))
{
var originalLang = firstMeta.OriginalLanguage.Trim();
var existingLang = langs.FirstOrDefault(lang => SameLanguageBase(lang, originalLang));
Log(
cfg.EnableLogging,
LogLevel.Information,
"[{Index}/{Total}] Original language item={ItemId} lang={OriginalLang} metadataSource={Source}",
scanIndex,
totalItems,
item.ItemId,
originalLang,
existingLang is not null ? "configured-language:" + existingLang : "extra-fetch");
var originalMeta = existingLang is not null
? fetchedMetadata[existingLang]
: await _tmdbClient.FetchMetadataAsync(item, originalLang, cfg.TmdbApiKey, limiter, cancellationToken).ConfigureAwait(false);
if (originalMeta is not null)
{
_store.UpsertTranslation(item.ItemId, OriginalAction, TitleField, originalMeta.Title);
_store.UpsertTranslation(item.ItemId, OriginalAction, OverviewField, originalMeta.Overview);
_store.UpsertTranslation(item.ItemId, OriginalAction, TaglineField, originalMeta.Tagline);
StoreGenres(item, OriginalAction, originalMeta);
Log(
cfg.VerboseLogging && cfg.EnableLogging,
LogLevel.Information,
"[{Index}/{Total}] Original metadata stored item={ItemId} lang={OriginalLang} titlePresent={TitlePresent} overviewPresent={OverviewPresent} taglinePresent={TaglinePresent}",
scanIndex,
totalItems,
item.ItemId,
originalLang,
!string.IsNullOrWhiteSpace(originalMeta.Title),
!string.IsNullOrWhiteSpace(originalMeta.Overview),
!string.IsNullOrWhiteSpace(originalMeta.Tagline));
}
else
{
Log(
cfg.EnableLogging,
LogLevel.Warning,
"[{Index}/{Total}] Original metadata unavailable item={ItemId} lang={OriginalLang}",
scanIndex,
totalItems,
item.ItemId,
originalLang);
}
artworkLanguages = AppendOriginalLanguage(langs, originalLang);
Log(
cfg.VerboseLogging && cfg.EnableLogging,
LogLevel.Information,
"[{Index}/{Total}] Artwork language set item={ItemId} langs={Languages}",
scanIndex,
totalItems,
item.ItemId,
string.Join(",", artworkLanguages));
}
await StoreArtworkAsync(item, cfg, artworkLanguages, limiter, now, cancellationToken).ConfigureAwait(false);
Log(cfg.EnableLogging, LogLevel.Information, "Multilang refresh item complete item={ItemId} kind={Kind} tmdb={TmdbId} mode={Mode}", item.ItemId, item.Kind, item.TmdbId, jobType);
return true;
}
private void StoreGenres(RefreshItemInfo item, string lang, TmdbMetadata metadata)
{
var media = GenreMediaFor(item.Kind);
if (media is null)
return;
foreach (var genre in metadata.Genres)
_store.UpsertGenre(genre.Id, media, lang, genre.Name);
}
private static string? GenreMediaFor(string kind)
=> kind.Equals("movie", StringComparison.OrdinalIgnoreCase) ||
kind.Equals("collection", StringComparison.OrdinalIgnoreCase)
? "movie"
: kind.Equals("tv", StringComparison.OrdinalIgnoreCase) ||
kind.Equals("tvseason", StringComparison.OrdinalIgnoreCase) ||
kind.Equals("tvepisode", StringComparison.OrdinalIgnoreCase)
? "tv"
: null;
private async Task StoreArtworkAsync(
RefreshItemInfo item,
PluginConfiguration cfg,
string[] languages,
FetchRateLimiter limiter,
long updatedAt,
CancellationToken cancellationToken)
{
if (item.Kind is "tvepisode")
return;
var providers = cfg.Providers
.Where(p => p.ArtworkOrder >= 0)
.OrderBy(p => p.ArtworkOrder)
.Select(p => p.Id.Trim().ToLowerInvariant())
.ToArray();
if (providers.Length == 0)
return;
TmdbImages? tmdbImages = null;
FanartImages? fanartImages = null;
var written = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var provider in providers)
{
cancellationToken.ThrowIfCancellationRequested();
if (provider.Equals("tmdb", StringComparison.OrdinalIgnoreCase))
{
tmdbImages ??= await _tmdbClient.FetchImagesAsync(item, cfg.TmdbApiKey, limiter, cancellationToken).ConfigureAwait(false);
if (tmdbImages is null)
continue;
await StoreImageMapAsync(item.ItemId, languages, PosterKind, tmdbImages.Posters, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false);
await StoreImageMapAsync(item.ItemId, languages, LogoKind, tmdbImages.Logos, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false);
await StoreImageMapAsync(item.ItemId, languages, BackdropKind, tmdbImages.Backdrops, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false);
Log(cfg.VerboseLogging && cfg.EnableLogging, LogLevel.Information, "TMDb artwork stored item={ItemId} langs={Languages}", item.ItemId, string.Join(",", languages));
continue;
}
if (provider.Equals("fanart", StringComparison.OrdinalIgnoreCase) && item.Kind.Equals("movie", StringComparison.OrdinalIgnoreCase))
{
fanartImages ??= await _fanartClient.FetchMovieImagesAsync(item.TmdbId, cfg.FanartApiKey, cancellationToken).ConfigureAwait(false);
if (fanartImages is null)
continue;
await StoreImageMapAsync(item.ItemId, languages, PosterKind, fanartImages.Posters, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false);
await StoreImageMapAsync(item.ItemId, languages, LogoKind, fanartImages.Logos, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false);
await StoreImageMapAsync(item.ItemId, languages, BannerKind, fanartImages.Banners, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false);
await StoreImageMapAsync(item.ItemId, languages, ThumbKind, fanartImages.Thumbs, cfg, updatedAt, written, cancellationToken).ConfigureAwait(false);
Log(cfg.VerboseLogging && cfg.EnableLogging, LogLevel.Information, "Fanart artwork stored item={ItemId} langs={Languages}", item.ItemId, string.Join(",", languages));
}
}
}
private async Task StoreImageMapAsync(
string itemId,
IEnumerable<string> languages,
string kind,
IReadOnlyDictionary<string, string> images,
PluginConfiguration cfg,
long updatedAt,
HashSet<string> written,
CancellationToken cancellationToken)
{
foreach (var lang in languages)
{
var storeLang = lang.StartsWith(OriginalAction + ":", StringComparison.OrdinalIgnoreCase)
? OriginalAction
: lang;
var key = kind + ":" + storeLang;
if (written.Contains(key))
continue;
var iso = ToIso639(lang);
if (!images.TryGetValue(iso, out var url) || string.IsNullOrWhiteSpace(url))
continue;
var storedPath = await _assetStorage.StoreAsync(itemId, storeLang, kind, url, IsLocalAssetStorage(cfg), cancellationToken).ConfigureAwait(false);
if (string.IsNullOrWhiteSpace(storedPath))
continue;
_store.UpsertAsset(itemId, storeLang, kind, storedPath, updatedAt);
written.Add(key);
}
}
private bool ItemNeedsMissingRefresh(RefreshItemInfo item, PluginConfiguration cfg, string[]? langs)
{
langs ??= NormalizeLanguages(cfg.Languages);
if (_store.HasMissingConfiguredTranslations(item.ItemId, langs))
return true;
var assetPresence = _store.GetAssetPresence(item.ItemId, langs.Append(OriginalAction));
return IsLocalAssetStorage(cfg) && assetPresence.AnyRemote;
}
private static string[] AppendOriginalLanguage(string[] languages, string originalLanguage)
=> languages
.Concat([OriginalAction + ":" + originalLanguage])
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
private static bool SameLanguageBase(string left, string right)
=> ToIso639(left).Equals(ToIso639(right), StringComparison.OrdinalIgnoreCase);
private static string ToIso639(string language)
{
if (language.StartsWith(OriginalAction + ":", StringComparison.OrdinalIgnoreCase))
language = language[(OriginalAction.Length + 1)..];
var index = language.IndexOf('-', StringComparison.Ordinal);
return (index > 0 ? language[..index] : language).Trim().ToLowerInvariant();
} }
private List<RefreshItemInfo> GetAllItemsWithTmdbId() private List<RefreshItemInfo> GetAllItemsWithTmdbId()
+19 -29
View File
@@ -1,16 +1,8 @@
(() => { (() => {
if (!Element.prototype.replaceChildren) {
Element.prototype.replaceChildren = function () {
this.textContent = "";
for (let index = 0; index < arguments.length; index++) this.appendChild(arguments[index]);
};
}
const BASE_PATH = window.location.pathname.split("/web/")[0] || ""; const BASE_PATH = window.location.pathname.split("/web/")[0] || "";
const MENU_ID = "mlUserRulesMenuItem"; const MENU_ID = "mlUserRulesMenuItem";
const USER_RULES_ROUTE = "multilang/user-rules"; const USER_RULES_ROUTE = "multilang/user-rules";
let lastMenuItemId = ""; let lastMenuItemId = "";
let userRulesRenderRun = 0;
function getClientToken() { function getClientToken() {
try { try {
@@ -227,6 +219,7 @@
function closeUserRulesShellIfInactive() { function closeUserRulesShellIfInactive() {
if (isUserRulesRoute()) return; if (isUserRulesRoute()) return;
document.querySelector("#ml-user-shell #ml-user")?._mlDispose?.();
document.getElementById("ml-user-shell")?.remove(); document.getElementById("ml-user-shell")?.remove();
} }
@@ -234,7 +227,6 @@
event?.preventDefault?.(); event?.preventDefault?.();
event?.stopPropagation?.(); event?.stopPropagation?.();
event?.stopImmediatePropagation?.(); event?.stopImmediatePropagation?.();
document.querySelector("#app-user-menu .MuiBackdrop-root")?.click();
history.pushState({}, "", userRulesHref()); history.pushState({}, "", userRulesHref());
scheduleUserRulesPageRender(); scheduleUserRulesPageRender();
} }
@@ -250,32 +242,25 @@
const next = document.createElement("script"); const next = document.createElement("script");
[...script.attributes].forEach((attr) => next.setAttribute(attr.name, attr.value)); [...script.attributes].forEach((attr) => next.setAttribute(attr.name, attr.value));
if (script.src) { if (script.src) {
next.onload = () => resolve(); next.onload = () => { next.remove(); resolve(); };
next.onerror = () => reject(new Error(`Failed to load ${script.src}`)); next.onerror = () => { next.remove(); reject(new Error(`Failed to load ${script.src}`)); };
next.src = script.src; next.src = script.src;
} else { } else {
next.text = script.textContent || ""; next.text = script.textContent || "";
} }
document.body.appendChild(next); document.body.appendChild(next);
if (!script.src) resolve(); if (!script.src) { next.remove(); resolve(); }
}); });
} }
async function renderUserRulesPage(run) { async function renderUserRulesPage(host) {
if (!isUserRulesRoute() || run !== userRulesRenderRun) return;
const host = userRulesHost();
if (!host) return;
if (host.dataset.mlUserRulesRendered === "1" && document.getElementById("ml-user")) return;
host.dataset.mlUserRulesRendered = "1";
host.innerHTML = "<div class=\"fieldDescription\">Loading Multilang settings...</div>"; host.innerHTML = "<div class=\"fieldDescription\">Loading Multilang settings...</div>";
const response = await fetch(`${BASE_PATH}/web/configurationpage?name=MultilangUser`, { cache: "no-store" }); const response = await fetch(`${BASE_PATH}/web/configurationpage?name=MultilangUser`, { cache: "no-store" });
if (!response.ok) throw new Error(`HTTP ${response.status}`); if (!response.ok) throw new Error(`HTTP ${response.status}`);
const fragment = await response.text(); const fragment = await response.text();
if (run !== userRulesRenderRun || !isUserRulesRoute()) return; if (!host.isConnected || !isUserRulesRoute()) return;
const parsed = new DOMParser().parseFromString(fragment, "text/html"); const parsed = new DOMParser().parseFromString(fragment, "text/html");
const sourcePage = parsed.getElementById("ml-user"); const sourcePage = parsed.getElementById("ml-user");
@@ -289,26 +274,26 @@
page.id = "ml-user"; page.id = "ml-user";
page.className = "ml-root"; page.className = "ml-root";
while (sourceContent.firstChild) page.appendChild(sourceContent.firstChild); while (sourceContent.firstChild) page.appendChild(sourceContent.firstChild);
host.replaceChildren(page); host.textContent = "";
host.appendChild(page);
for (const script of scripts) { for (const script of scripts) {
if (run !== userRulesRenderRun || !isUserRulesRoute()) return; if (!host.isConnected || !isUserRulesRoute()) return;
await executeScript(script); await executeScript(script);
} }
} }
function scheduleUserRulesPageRender() { function scheduleUserRulesPageRender() {
userRulesRenderRun++;
if (!isUserRulesRoute()) { if (!isUserRulesRoute()) {
closeUserRulesShellIfInactive(); closeUserRulesShellIfInactive();
return; return;
} }
const run = userRulesRenderRun; const host = userRulesHost();
[0, 100, 350].forEach((delay) => { if (!host || host._mlLoad) return;
setTimeout(() => { host._mlLoad = renderUserRulesPage(host).catch((err) => {
renderUserRulesPage(run).catch((err) => console.error("[Multilang] failed to render user rules page", err)); console.error("[Multilang] failed to render user rules page", err);
}, delay); if (host.isConnected) host.textContent = `Unable to load Multilang settings: ${err.message}`;
}); });
} }
@@ -392,6 +377,11 @@
if (!url.startsWith("/")) url = "/" + url; if (!url.startsWith("/")) url = "/" + url;
if (BASE_PATH && !url.toLowerCase().startsWith(BASE_PATH.toLowerCase() + "/")) url = BASE_PATH + url; if (BASE_PATH && !url.toLowerCase().startsWith(BASE_PATH.toLowerCase() + "/")) url = BASE_PATH + url;
} }
const local = new URL(url, window.location.origin);
if (local.origin === window.location.origin && local.pathname.startsWith(BASE_PATH + "/Multilang/Assets/")) {
local.searchParams.set("api_key", getClientToken());
return local.href;
}
return url; return url;
} }
@@ -19,7 +19,7 @@ public sealed class AssetStorageServiceTests
}); });
var service = new AssetStorageService(factory, store); var service = new AssetStorageService(factory, store);
var path = await service.StoreAsync("item", "fi", "poster", "https://images.example/poster.jpg", true, CancellationToken.None); var path = await service.StoreAsync("item", "https://images.example/poster.jpg", true, CancellationToken.None);
Assert.NotNull(path); Assert.NotNull(path);
Assert.StartsWith(TranslationStore.LocalAssetUrlPrefix, path, StringComparison.Ordinal); Assert.StartsWith(TranslationStore.LocalAssetUrlPrefix, path, StringComparison.Ordinal);
@@ -28,6 +28,19 @@ public sealed class AssetStorageServiceTests
Assert.True(service.TryResolveLocalAsset(path[TranslationStore.LocalAssetUrlPrefix.Length..], out var resolved, out var contentType)); Assert.True(service.TryResolveLocalAsset(path[TranslationStore.LocalAssetUrlPrefix.Length..], out var resolved, out var contentType));
Assert.Equal(fullPath, resolved); Assert.Equal(fullPath, resolved);
Assert.Equal("image/jpeg", contentType); Assert.Equal("image/jpeg", contentType);
store.UpsertAsset("item", "fi", "poster", path, 1);
var original = await service.StoreAsync("item", "https://images.example/poster.jpg", true, CancellationToken.None);
store.UpsertAsset("item", "Original", "poster", original!, 1);
Assert.Equal(path, original);
Assert.Single(factory.Requests);
Assert.True(store.IsAssetReferenced("item", path));
Assert.True(File.Exists(fullPath));
File.Delete(fullPath);
Assert.False(File.Exists(fullPath));
await service.StoreAsync("item", "https://images.example/poster.jpg", true, CancellationToken.None);
Assert.True(File.Exists(fullPath));
Assert.Equal(2, factory.Requests.Count);
} }
finally finally
{ {
@@ -46,7 +59,7 @@ public sealed class AssetStorageServiceTests
var service = new AssetStorageService(factory, store); var service = new AssetStorageService(factory, store);
const string url = "https://images.example/poster.webp"; const string url = "https://images.example/poster.webp";
var result = await service.StoreAsync("item", "fi", "poster", url, false, CancellationToken.None); var result = await service.StoreAsync("item", url, false, CancellationToken.None);
Assert.Equal(url, result); Assert.Equal(url, result);
Assert.Empty(factory.Requests); Assert.Empty(factory.Requests);
@@ -73,7 +86,7 @@ public sealed class AssetStorageServiceTests
}); });
var service = new AssetStorageService(factory, store); var service = new AssetStorageService(factory, store);
await Assert.ThrowsAsync<InvalidOperationException>(() => service.StoreAsync("item", "fi", "poster", "https://images.example/poster.jpg", true, CancellationToken.None)); await Assert.ThrowsAsync<InvalidOperationException>(() => service.StoreAsync("item", "https://images.example/poster.jpg", true, CancellationToken.None));
Assert.Empty(Directory.EnumerateFiles(store.AssetsDirectory, "*", SearchOption.AllDirectories)); Assert.Empty(Directory.EnumerateFiles(store.AssetsDirectory, "*", SearchOption.AllDirectories));
} }
@@ -6,6 +6,24 @@ namespace Jellyfin.Plugin.Multilang.Tests;
public sealed class CategoryRuleEvaluatorTests public sealed class CategoryRuleEvaluatorTests
{ {
[Fact]
public void FastMatchingAndDiagnosticTraceAgreeAcrossAllRuleModes()
{
foreach (var relation in new[] { "is", "is_not", "contains", "not_contains" })
foreach (var fieldOr in new[] { false, true })
foreach (var valueOr in new[] { false, true })
foreach (var matchAll in new[] { false, true })
foreach (var facts in new[] { Facts(originalLanguage: "sv"), Facts(spokenLanguages: ["fi", "en"]), Facts(kind: "tv") })
{
var rule = Rule("original_language", relation, valueOr, "sv", "fi");
rule.Requirements[0].Fields = ["original_language", "spoken_languages"];
rule.Requirements[0].UseFieldOr = fieldOr;
rule.Requirements = [rule.Requirements[0], Requirement("production_countries", "contains", false, "FI")];
rule.MatchAllConditions = matchAll;
Assert.Equal(CategoryRuleEvaluator.Trace(rule, facts).Result, CategoryRuleEvaluator.Matches(rule, facts));
}
}
[Fact] [Fact]
public void IsWithAndListRequiresExactSet() public void IsWithAndListRequiresExactSet()
{ {
@@ -95,11 +113,9 @@ public sealed class CategoryRuleEvaluatorTests
{ {
Id = rule.Id, Id = rule.Id,
Label = rule.Label, Label = rule.Label,
CriteriaText = rule.CriteriaText,
Requirements = rule.Requirements, Requirements = rule.Requirements,
MatchAllConditions = matchAll, MatchAllConditions = matchAll,
Scopes = rule.Scopes, Scopes = rule.Scopes,
FieldActions = rule.FieldActions,
FieldActionLists = rule.FieldActionLists FieldActionLists = rule.FieldActionLists
}; };
@@ -0,0 +1,232 @@
using System.Net;
using System.Text.Json;
using Jellyfin.Plugin.Multilang.Configuration;
using Jellyfin.Plugin.Multilang.Data;
using Jellyfin.Plugin.Multilang.Services.Assets;
using Jellyfin.Plugin.Multilang.Services.Providers;
using Jellyfin.Plugin.Multilang.Services.Refresh;
using Microsoft.Extensions.Logging.Abstractions;
namespace Jellyfin.Plugin.Multilang.Tests;
public sealed class ItemFetcherTests : IDisposable
{
private readonly string _root = TestPaths.CreateRoot();
private readonly TranslationStore _store;
private readonly TestHttpClientFactory _http;
private readonly ItemFetcher _fetcher;
private readonly RefreshItemInfo _item = new("11111111111111111111111111111111", "1", "movie", RefreshWorkClass.Movies, 0, 0, null, "Movie");
private readonly PluginConfiguration _config = new()
{
Languages = ["en", "fi"],
TmdbApiKey = "test",
Providers = [new() { Id = "tmdb", MetadataOrder = 0 }]
};
private string _original = "en";
private string _overview = "Overview";
private bool _missingFinnishTagline;
private bool _notFound;
private bool _missingTmdbPoster;
private bool _fanartUnauthorized;
public ItemFetcherTests()
{
_store = new(new TestApplicationPaths(_root));
_http = new(request =>
{
if (_notFound) return new(HttpStatusCode.NotFound);
var url = request.RequestUri!;
if (url.Host == "webservice.fanart.tv")
return _fanartUnauthorized ? new(HttpStatusCode.Unauthorized)
: Json(new { movieposter = new[] { new { lang = "en", url = "https://assets.fanart.tv/poster.jpg", likes = "5" } } });
if (url.Host == "image.tmdb.org")
return new(HttpStatusCode.OK) { Content = new ByteArrayContent([1, 2, 3]) };
if (url.AbsolutePath.EndsWith("/images"))
return Json(new { posters = _missingTmdbPoster ? [] : new[] { new { iso_639_1 = "en", file_path = "/poster.jpg" } }, logos = Array.Empty<object>(), backdrops = Array.Empty<object>() });
var language = Microsoft.AspNetCore.WebUtilities.QueryHelpers.ParseQuery(url.Query)["language"].ToString();
return Json(new
{
title = language + " title",
overview = _overview,
tagline = language == "fi" && _missingFinnishTagline ? "" : "Tagline",
original_language = _original,
original_title = "Original title",
genres = Array.Empty<object>()
});
});
_fetcher = new(_store, new(_http), new(_http), new(_http, _store), NullLogger<ItemFetcher>.Instance);
}
private static HttpResponseMessage Json(object value)
=> new(HttpStatusCode.OK) { Content = new StringContent(JsonSerializer.Serialize(value)) };
private Task<bool> Fetch(RefreshJobType job = RefreshJobType.Missing) => _fetcher.RefreshAsync(_item, _config, job, CancellationToken.None);
private string Text(string language, string field) => _store.GetTranslations([_item.ItemId], [language])[_item.ItemId][language][field];
[Fact]
public async Task CompleteMetadataIsNotRefetchedAndOriginalReusesConfiguredLanguage()
{
Assert.True(await Fetch());
Assert.Equal(2, _http.Requests.Count);
Assert.Equal(Text("en", "title"), Text("Original", "title"));
Assert.False(_fetcher.NeedsRefresh(_item, _config));
Assert.False(await Fetch());
Assert.Equal(2, _http.Requests.Count);
await Fetch(RefreshJobType.Full);
Assert.Equal(4, _http.Requests.Count);
}
[Fact]
public async Task MissingRetryFetchesOnlyIncompleteLanguageAndUpdatesWholeResponse()
{
_missingFinnishTagline = true;
await Fetch();
Assert.False(_fetcher.NeedsRefresh(_item, _config));
await Fetch();
Assert.Equal(2, _http.Requests.Count);
_config.WaitDaysForMissingData = 0;
Assert.True(_fetcher.NeedsRefresh(_item, _config));
_missingFinnishTagline = false;
_overview = "Updated overview";
await Fetch();
Assert.Equal(3, _http.Requests.Count);
Assert.Contains("language=fi", _http.Requests.Last());
Assert.Equal("Updated overview", Text("fi", "overview"));
Assert.Equal("Overview", Text("en", "overview"));
Assert.False(_fetcher.NeedsRefresh(_item, _config));
}
[Fact]
public async Task AdditionalOriginalLanguageIsFetchedOnceAndNewConfiguredLanguageDoesNotRefetchOthers()
{
_original = "sv";
await Fetch();
Assert.Equal(3, _http.Requests.Count);
Assert.Equal("sv title", Text("Original", "title"));
await Fetch();
Assert.Equal(3, _http.Requests.Count);
_config.Languages = ["en", "fi", "fr"];
await Fetch();
Assert.Equal(4, _http.Requests.Count);
Assert.Contains("language=fr", _http.Requests.Last());
}
[Fact]
public async Task NotFoundIsRememberedUntilRetryCutoff()
{
_notFound = true;
Assert.False(await Fetch());
Assert.False(_fetcher.NeedsRefresh(_item, _config));
await Fetch();
Assert.Equal(2, _http.Requests.Count);
_config.WaitDaysForMissingData = 0;
Assert.True(_fetcher.NeedsRefresh(_item, _config));
await Fetch();
Assert.Equal(4, _http.Requests.Count);
}
[Fact]
public async Task LocalRepairDoesNotFetchMetadataAndUrlCleanupOnlyHappensInScheduledHousekeeping()
{
_config.Providers[0].ArtworkOrder = 0;
await Fetch();
Assert.Equal(3, _http.Requests.Count);
_config.AssetStorageMode = "local";
Assert.True(_fetcher.NeedsLocalWork(_item.ItemId, _config));
await Fetch(RefreshJobType.Aggregate);
Assert.Equal(4, _http.Requests.Count);
Assert.Contains("image.tmdb.org", _http.Requests.Last());
var path = _store.GetAssetPath(_item.ItemId, "en", "poster")!;
Assert.Equal(path, _store.GetAssetPath(_item.ItemId, "Original", "poster"));
Assert.True(_store.TryNormalizeLocalAssetPath(path, out var file));
File.Delete(file);
await Fetch();
Assert.Equal(5, _http.Requests.Count);
Assert.True(File.Exists(file));
_config.AssetStorageMode = "url";
await Fetch();
Assert.Equal(path, _store.GetAssetPath(_item.ItemId, "en", "poster"));
Assert.True(File.Exists(file));
_store.CleanupForConfiguration(new HashSet<string> { _item.ItemId }, _config.Languages, false);
Assert.StartsWith("https://", _store.GetAssetPath(_item.ItemId, "en", "poster"));
Assert.False(File.Exists(file));
Assert.Equal(5, _http.Requests.Count);
}
[Fact]
public async Task ArtworkUsesProviderPriorityAndFallsBackWhenHigherPriorityImageDisappears()
{
_config.Providers = [new() { Id = "tmdb", MetadataOrder = 0, ArtworkOrder = 0 }, new() { Id = "fanart", ArtworkOrder = 1 }];
_config.FanartApiKey = "test";
await Fetch();
Assert.Contains("image.tmdb.org", _store.GetAssetPath(_item.ItemId, "en", "poster"));
_missingTmdbPoster = true;
await Fetch(RefreshJobType.Full);
Assert.Contains("assets.fanart.tv", _store.GetAssetPath(_item.ItemId, "en", "poster"));
_fanartUnauthorized = true;
await Assert.ThrowsAsync<HttpRequestException>(() => Fetch(RefreshJobType.Full));
Assert.Contains("assets.fanart.tv", _store.GetAssetPath(_item.ItemId, "en", "poster"));
}
[Fact]
public async Task BackupPreservesSourceUrlsAndFetchStateAndFiltersUnwantedLanguages()
{
_config.Providers[0].ArtworkOrder = 0;
await Fetch();
var backup = Path.Combine(_root, "export.sqlite");
_store.ExportTranslationDatabase(backup);
_store.ResetAll();
_store.ImportTranslationDatabase(backup, new HashSet<string> { _item.ItemId }, new HashSet<string> { "en" });
Assert.Equal(2, _store.GetFetchStates(_item.ItemId).Count);
Assert.DoesNotContain("metadata:fi", _store.GetFetchStates(_item.ItemId).Keys);
Assert.Equal("https://image.tmdb.org/t/p/original/poster.jpg", _store.GetStoredAssets(_item.ItemId).First().SourceUrl);
Assert.Equal("tmdb", _store.GetStoredAssets(_item.ItemId).First().Provider);
}
[Theory]
[InlineData(true, false, false)]
[InlineData(true, true, true)]
[InlineData(false, false, false)]
[InlineData(false, true, false)]
public void YoungFetchStateHonorsKnownMissingVersusDeletedData(bool complete, bool missing, bool expected)
=> Assert.Equal(expected, ItemFetcher.NeedsFetch(new("scope", 100, complete), "scope", missing, false, 50));
[Fact]
public void FailedArtworkPacketRollsBackDeletionAndNewRowsTogether()
{
var original = new StoredAsset("en", "poster", "https://example.test/old.jpg", "https://example.test/old.jpg", "tmdb");
_store.SaveArtwork(_item.ItemId, [original], new Dictionary<string, FetchState>());
Assert.Throws<Microsoft.Data.Sqlite.SqliteException>(() => _store.SaveArtwork(_item.ItemId,
[original with { Path = "new" }, original], new Dictionary<string, FetchState>()));
Assert.Equal(original, Assert.Single(_store.GetStoredAssets(_item.ItemId)));
}
[Fact]
public async Task ParallelMetadataPacketsDoNotLoseRows()
{
await Task.WhenAll(Enumerable.Range(0, 12).Select(index => Task.Run(() =>
_store.SaveMetadata(_item with { ItemId = index.ToString() }, "en",
new("Title", "Overview", "Tagline", "Title", "en", [], [], [], []),
new("scope", 1, true), true))));
foreach (var index in Enumerable.Range(0, 12))
{
Assert.NotNull(_store.GetFacts(index.ToString()));
Assert.Equal(3, _store.GetTranslations([index.ToString()], ["en"])[index.ToString()]["en"].Count);
Assert.True(_store.GetFetchStates(index.ToString())["metadata:en"].Complete);
}
}
[Fact]
public void CollectionAggregationKeepsOnlySharedSingleLanguageAndUnionsCountrySets()
{
var collection = new FactsData("collection", "1", "collection", "Collection", "en", "en", "[]", "[]", "[]", "", "[]", 1, 1);
var finnish = collection with { Kind = "movie", OriginalLanguage = "fi", ProductionCountriesJson = "[\"FI\"]" };
var swedish = finnish with { OriginalLanguage = "sv", ProductionCountriesJson = "[\"SE\",\"FI\"]" };
var mixed = Services.ItemsProxyTransformer.AggregateCollectionFacts(collection, [finnish, swedish]);
Assert.Equal("", mixed.OriginalLanguage);
Assert.Equal("fi,sv", mixed.OriginalLanguageAll);
Assert.Equal(new HashSet<string> { "FI", "SE" }, JsonSerializer.Deserialize<HashSet<string>>(mixed.ProductionCountriesJson));
Assert.Equal("fi", Services.ItemsProxyTransformer.AggregateCollectionFacts(collection, [finnish, finnish]).OriginalLanguage);
}
public void Dispose() => TestPaths.DeleteRoot(_root);
}
@@ -5,16 +5,29 @@ namespace Jellyfin.Plugin.Multilang.Tests;
public sealed class ItemsProxyCacheTests public sealed class ItemsProxyCacheTests
{ {
[Fact] [Fact]
public void StoreRetrievesAndInvalidatesByItemId() public void PolicyChangesInvalidateCachedDataAndOldInFlightWrites()
{
var cache = new ItemsProxyCache();
var generation = cache.ObserveUserPolicy("user", "all libraries");
Store(cache, "key");
Assert.Equal(generation, cache.ObserveUserPolicy("user", "all libraries"));
Assert.True(cache.TryGet("key", TimeSpan.FromMinutes(1), out _));
Assert.True(cache.ObserveUserPolicy("user", "no libraries") > generation);
Assert.False(cache.TryGet("key", TimeSpan.FromMinutes(1), out _));
Assert.False(cache.Store("key", "user", "/Items", 1, "body", "application/json", 1, 100, TimeSpan.FromMinutes(1), generation));
}
[Fact]
public void StoreRetrievesAndClearsCachedResponses()
{ {
var cache = new ItemsProxyCache(); var cache = new ItemsProxyCache();
Assert.True(Store(cache, "key", ["A"])); Assert.True(Store(cache, "key"));
Assert.True(cache.TryGet("key", TimeSpan.FromMinutes(1), out var response)); Assert.True(cache.TryGet("key", TimeSpan.FromMinutes(1), out var response));
Assert.Equal("body-key", response.Body); Assert.Equal("body-key", response.Body);
Assert.Equal(1, response.ItemCount); Assert.Equal(1, response.ItemCount);
cache.InvalidateItems(["a"]); cache.ClearAll();
Assert.False(cache.TryGet("key", TimeSpan.FromMinutes(1), out _)); Assert.False(cache.TryGet("key", TimeSpan.FromMinutes(1), out _));
} }
@@ -24,11 +37,11 @@ public sealed class ItemsProxyCacheTests
{ {
var cache = new ItemsProxyCache(); var cache = new ItemsProxyCache();
Assert.True(cache.StoreMicro("key", "user", "/Items", ["A"], "micro", "application/json", 1, 1024)); Assert.True(cache.StoreMicro("key", "user", "/Items", 1, "micro", "application/json", 1, 1024));
Assert.True(cache.TryGet("key", TimeSpan.FromMinutes(1), out var response)); Assert.True(cache.TryGet("key", TimeSpan.FromMinutes(1), out var response));
Assert.Equal("micro", response.Body); Assert.Equal("micro", response.Body);
cache.InvalidateItems(["A"]); cache.ClearAll();
Assert.False(cache.TryGet("key", TimeSpan.FromMinutes(1), out _)); Assert.False(cache.TryGet("key", TimeSpan.FromMinutes(1), out _));
} }
@@ -38,9 +51,9 @@ public sealed class ItemsProxyCacheTests
{ {
var cache = new ItemsProxyCache(); var cache = new ItemsProxyCache();
Assert.True(Store(cache, "first", ["A"], body: "1111", maxBytes: 6)); Assert.True(Store(cache, "first", body: "1111", maxBytes: 6));
Thread.Sleep(10); Thread.Sleep(10);
Assert.True(Store(cache, "second", ["B"], body: "2222", maxBytes: 6)); Assert.True(Store(cache, "second", body: "2222", maxBytes: 6));
Assert.False(cache.TryGet("first", TimeSpan.FromMinutes(1), out _)); Assert.False(cache.TryGet("first", TimeSpan.FromMinutes(1), out _));
Assert.True(cache.TryGet("second", TimeSpan.FromMinutes(1), out _)); Assert.True(cache.TryGet("second", TimeSpan.FromMinutes(1), out _));
@@ -51,7 +64,7 @@ public sealed class ItemsProxyCacheTests
{ {
var cache = new ItemsProxyCache(); var cache = new ItemsProxyCache();
Assert.True(Store(cache, "key", ["A"], ttl: TimeSpan.FromMilliseconds(1))); Assert.True(Store(cache, "key", ttl: TimeSpan.FromMilliseconds(1)));
Thread.Sleep(20); Thread.Sleep(20);
Assert.False(cache.TryGet("key", TimeSpan.FromMilliseconds(1), out _)); Assert.False(cache.TryGet("key", TimeSpan.FromMilliseconds(1), out _));
@@ -62,14 +75,14 @@ public sealed class ItemsProxyCacheTests
{ {
var cache = new ItemsProxyCache(); var cache = new ItemsProxyCache();
Assert.False(Store(cache, "key", ["A"], body: "12345", maxBytes: 4)); Assert.False(Store(cache, "key", body: "12345", maxBytes: 4));
Assert.False(cache.TryGet("key", TimeSpan.FromMinutes(1), out _)); Assert.False(cache.TryGet("key", TimeSpan.FromMinutes(1), out _));
} }
private static bool Store( private static bool Store(
ItemsProxyCache cache, ItemsProxyCache cache,
string key, string key,
string[] itemIds, int itemCount = 1,
string? body = null, string? body = null,
long maxBytes = 1024, long maxBytes = 1024,
TimeSpan? ttl = null) TimeSpan? ttl = null)
@@ -77,10 +90,43 @@ public sealed class ItemsProxyCacheTests
key, key,
"user", "user",
"/Items", "/Items",
itemIds, itemCount,
body ?? "body-" + key, body ?? "body-" + key,
"application/json", "application/json",
100, 100,
maxBytes, maxBytes,
ttl ?? TimeSpan.FromMinutes(1)); ttl ?? TimeSpan.FromMinutes(1));
[Fact]
public void MicroAndMainEntriesShareOneBudget()
{
var cache = new ItemsProxyCache();
Assert.True(Store(cache, "main", body: "1111", maxBytes: 6));
Assert.True(cache.StoreMicro("micro1", "user", "/Items", 1, "2222", "application/json", 1, 6));
Assert.False(cache.TryGet("main", TimeSpan.FromMinutes(1), out _));
Assert.True(cache.StoreMicro("micro2", "user", "/Items", 1, "3333", "application/json", 1, 6));
Assert.False(cache.TryGet("micro1", TimeSpan.FromMinutes(1), out _));
Assert.True(cache.TryGet("micro2", TimeSpan.FromMinutes(1), out _));
}
[Fact]
public void InvalidationRejectsStaleInFlightWrites()
{
var cache = new ItemsProxyCache();
var generation = cache.Generation;
cache.ClearAll();
Assert.False(cache.Store("key", "user", "/Items", 1, "body", "application/json", 1, 100, TimeSpan.FromMinutes(1), generation));
Assert.False(cache.StoreMicro("key", "user", "/Items", 1, "body", "application/json", 1, 100, generation));
}
[Fact]
public void ClearingOneUserPreservesOtherUsersCachedResponses()
{
var cache = new ItemsProxyCache();
Store(cache, "one");
cache.Store("two", "other", "/Items", 1, "body", "application/json", 1, 100, TimeSpan.FromMinutes(1));
cache.ClearUser("user");
Assert.False(cache.TryGet("one", TimeSpan.FromMinutes(1), out _));
Assert.True(cache.TryGet("two", TimeSpan.FromMinutes(1), out _));
}
} }
@@ -16,7 +16,6 @@ public sealed class ItemsProxyRequestBuilderTests
var result = ItemsProxyRequestBuilder.Build( var result = ItemsProxyRequestBuilder.Build(
request, request,
"/Items?Recursive=true&api_key=old", "/Items?Recursive=true&api_key=old",
"token",
UserId, UserId,
multilangEnabled: true); multilangEnabled: true);
@@ -36,7 +35,6 @@ public sealed class ItemsProxyRequestBuilderTests
var result = ItemsProxyRequestBuilder.Build( var result = ItemsProxyRequestBuilder.Build(
request, request,
$"/Users/{OtherUserId}/Items?Limit=5", $"/Users/{OtherUserId}/Items?Limit=5",
"token",
UserId, UserId,
multilangEnabled: true); multilangEnabled: true);
@@ -52,7 +50,6 @@ public sealed class ItemsProxyRequestBuilderTests
var result = ItemsProxyRequestBuilder.Build( var result = ItemsProxyRequestBuilder.Build(
request, request,
"/Items?SortBy=SortName&SortOrder=Descending&StartIndex=20&Limit=10&NameStartsWith=L", "/Items?SortBy=SortName&SortOrder=Descending&StartIndex=20&Limit=10&NameStartsWith=L",
"token",
UserId, UserId,
multilangEnabled: true); multilangEnabled: true);
@@ -65,7 +62,7 @@ public sealed class ItemsProxyRequestBuilderTests
Assert.Equal("fi-FI", result.Controls.ClientLocale); Assert.Equal("fi-FI", result.Controls.ClientLocale);
Assert.DoesNotContain("SortBy=", result.Upstream.Query); Assert.DoesNotContain("SortBy=", result.Upstream.Query);
Assert.DoesNotContain("Limit=", result.Upstream.Query); Assert.DoesNotContain("Limit=", result.Upstream.Query);
Assert.Contains("|SortName|Descending|L|20|10|fi-FI|", result.CacheKey); Assert.DoesNotContain("SortBy=", result.CacheKey);
} }
[Fact] [Fact]
@@ -76,20 +73,18 @@ public sealed class ItemsProxyRequestBuilderTests
var first = ItemsProxyRequestBuilder.Build( var first = ItemsProxyRequestBuilder.Build(
request, request,
"/Items?SortBy=SortName&StartIndex=0&Limit=100", "/Items?SortBy=SortName&StartIndex=0&Limit=100",
"token",
UserId, UserId,
multilangEnabled: true); multilangEnabled: true);
var second = ItemsProxyRequestBuilder.Build( var second = ItemsProxyRequestBuilder.Build(
request, request,
"/Items?SortBy=SortName&StartIndex=100&Limit=100", "/Items?SortBy=SortName&StartIndex=100&Limit=100",
"token",
UserId, UserId,
multilangEnabled: true); multilangEnabled: true);
Assert.NotNull(first); Assert.NotNull(first);
Assert.NotNull(second); Assert.NotNull(second);
Assert.Equal(first.NormalizedUrlForCache, second.NormalizedUrlForCache); Assert.Equal(first.NormalizedUrlForCache, second.NormalizedUrlForCache);
Assert.NotEqual(first.CacheKey, second.CacheKey); Assert.Equal(first.CacheKey, second.CacheKey);
} }
[Fact] [Fact]
@@ -100,7 +95,6 @@ public sealed class ItemsProxyRequestBuilderTests
var result = ItemsProxyRequestBuilder.Build( var result = ItemsProxyRequestBuilder.Build(
request, request,
"/Items?SortBy=SortName&Limit=10", "/Items?SortBy=SortName&Limit=10",
"token",
UserId, UserId,
multilangEnabled: false); multilangEnabled: false);
@@ -119,7 +113,6 @@ public sealed class ItemsProxyRequestBuilderTests
var result = ItemsProxyRequestBuilder.Build( var result = ItemsProxyRequestBuilder.Build(
request, request,
"http://other-host:8096/Items", "http://other-host:8096/Items",
"token",
UserId, UserId,
multilangEnabled: true); multilangEnabled: true);
@@ -132,6 +125,18 @@ public sealed class ItemsProxyRequestBuilderTests
Assert.Equal([1, 2], ItemsProxyRequestBuilder.ParseLocalGenreIds("tmdb-1,abc,2,tmdb-2,tmdb-x,1")); Assert.Equal([1, 2], ItemsProxyRequestBuilder.ParseLocalGenreIds("tmdb-1,abc,2,tmdb-2,tmdb-x,1"));
} }
[Fact]
public void GenreFilteringRetainsTheFieldsNeededForLocalFiltering()
{
var result = ItemsProxyRequestBuilder.Build(HttpRequest(), "/Items?GenreIds=tmdb-18&Fields=Overview", UserId, true);
Assert.NotNull(result);
var query = Microsoft.AspNetCore.WebUtilities.QueryHelpers.ParseQuery(result.Upstream.Query);
Assert.False(query.ContainsKey("GenreIds"));
Assert.Equal(1, query["Fields"].Count);
Assert.Contains("GenreItems", query["Fields"].ToString());
Assert.Contains("Overview", query["Fields"].ToString());
}
private static HttpRequest HttpRequest(string query = "") private static HttpRequest HttpRequest(string query = "")
{ {
var context = new DefaultHttpContext(); var context = new DefaultHttpContext();
@@ -140,4 +145,40 @@ public sealed class ItemsProxyRequestBuilderTests
context.Request.QueryString = new QueryString(query); context.Request.QueryString = new QueryString(query);
return context.Request; return context.Request;
} }
[Theory]
[InlineData("//other-host:8096/Items")]
[InlineData("/\\other-host/Items")]
[InlineData("https://server.local:8096/Items")]
[InlineData("/Multilang/ItemsProxy")]
[InlineData("/System/Configuration")]
[InlineData("/Items/Filters")]
[InlineData("/Items/../System/Info")]
public void RejectsUnsupportedDestinations(string url)
=> Assert.Null(ItemsProxyRequestBuilder.Build(HttpRequest(), url, UserId, true));
[Fact]
public void EquivalentQueryOrderSharesCacheAndInFlightKeys()
{
var first = ItemsProxyRequestBuilder.Build(HttpRequest(), "/Items?Fields=Overview&Recursive=true", UserId, true)!;
var second = ItemsProxyRequestBuilder.Build(HttpRequest(), "/Items?Recursive=true&Fields=Overview", UserId, true)!;
Assert.Equal(first.CacheKey, second.CacheKey);
Assert.Equal(first.NormalizedUrlForCache, second.NormalizedUrlForCache);
}
[Fact]
public void BasePathAndDashedUserIdsAreNormalized()
{
var request = HttpRequest();
request.PathBase = "/jellyfin";
var result = ItemsProxyRequestBuilder.Build(request, "/jellyfin/Users/22222222-2222-2222-2222-222222222222/Items", UserId, true)!;
Assert.Equal($"/jellyfin/Users/{UserId}/Items", result.Upstream.AbsolutePath);
Assert.Null(ItemsProxyRequestBuilder.Build(request, "/Items", UserId, true));
}
[Theory]
[InlineData("/Items/Resume?StartIndex=100&Limit=20")]
[InlineData("/Shows/NextUp?StartIndex=100&Limit=20")]
public void BoundedListsKeepUpstreamPaging(string url)
=> Assert.False(ItemsProxyRequestBuilder.Build(HttpRequest(), url, UserId, true)!.Controls.ApplyLocalPaging);
} }
@@ -7,6 +7,14 @@ namespace Jellyfin.Plugin.Multilang.Tests;
public sealed class ItemsProxySortingTests public sealed class ItemsProxySortingTests
{ {
[Fact]
public void TranslationOnlyDoesNotRewriteUpstreamPaging()
{
var root = JsonNode.Parse("""{"Items":[{"Name":"One"},{"Name":"Two"}],"TotalRecordCount":200,"StartIndex":100}""")!;
var original = root.ToJsonString();
ItemsProxySorting.Apply(root, new ItemsProxyControls("", "", "", 0, 0, "", "", false), CultureInfo.InvariantCulture, []);
Assert.Equal(original, root.ToJsonString());
}
[Fact] [Fact]
public void GetSortArticlesUsesBuiltInLanguageFallback() public void GetSortArticlesUsesBuiltInLanguageFallback()
{ {
@@ -13,6 +13,22 @@ public sealed class ItemsProxyTransformerResolutionTests
private const string OriginalAction = "Original"; private const string OriginalAction = "Original";
private const string LanguagePrefix = "Language:"; private const string LanguagePrefix = "Language:";
[Fact]
public void TraceUsesStoredOriginalTranslationWhenOriginalTitleIsEmpty()
{
var facts = Facts(originalTitle: "", originalLanguage: "fi");
var translations = TextTranslations((OriginalAction, TitleField, "Alkuperainen nimi"));
var attempts = new List<ItemsProxyResolutionAttempt>();
var normal = ItemsProxyTransformer.ResolveField(TitleField, [OriginalAction], facts, translations);
var traced = ItemsProxyTransformer.ResolveField(TitleField, [OriginalAction], facts, translations, attempts.Add);
Assert.Equal(normal, traced);
Assert.Equal("fi", traced.Language);
var chosen = Assert.Single(attempts);
Assert.True(chosen.Chosen);
Assert.Equal("Original/title", chosen.LookupKey);
Assert.Equal(traced.Value, chosen.Value);
}
[Fact] [Fact]
public void ResolveFieldUsesFirstLanguageWithData() public void ResolveFieldUsesFirstLanguageWithData()
{ {
@@ -13,8 +13,8 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="coverlet.collector" Version="6.0.2" /> <PackageReference Include="coverlet.collector" Version="6.0.2" />
<PackageReference Include="Jellyfin.Common" Version="12.0.0-rc2" /> <PackageReference Include="Jellyfin.Common" Version="12.0.0" />
<PackageReference Include="Jellyfin.Controller" Version="12.0.0-rc2" /> <PackageReference Include="Jellyfin.Controller" Version="12.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" /> <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
<PackageReference Include="xunit" Version="2.9.2" /> <PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" /> <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
@@ -0,0 +1,99 @@
using System.Diagnostics;
using System.Net;
using Jellyfin.Plugin.Multilang.Data;
using Jellyfin.Plugin.Multilang.Services.Providers;
namespace Jellyfin.Plugin.Multilang.Tests;
public sealed class MaintenanceAndProviderTests
{
[Fact]
public async Task ConcurrentRefreshesCanOverlapButMaintenanceWaitsForAllOfThem()
{
var root = TestPaths.CreateRoot();
try
{
var store = new TranslationStore(new TestApplicationPaths(root));
var first = await store.EnterRefreshAsync(CancellationToken.None);
var second = await store.EnterRefreshAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5));
using var cancel = new CancellationTokenSource();
var maintenance = store.EnterMaintenanceAsync(cancel.Token);
Assert.False(maintenance.IsCompleted);
first.Dispose();
Assert.False(maintenance.IsCompleted);
cancel.Cancel();
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => maintenance);
second.Dispose();
using var allSlots = await store.EnterMaintenanceAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5));
}
finally { TestPaths.DeleteRoot(root); }
}
[Fact]
public async Task MaintenanceWaitsForWriterAndCanceledWaiterDoesNotLeakLease()
{
var root = TestPaths.CreateRoot();
try
{
var store = new TranslationStore(new TestApplicationPaths(root));
var writer = await store.EnterMaintenanceAsync(CancellationToken.None);
using var canceled = new CancellationTokenSource();
var waiting = store.EnterMaintenanceAsync(canceled.Token);
Assert.False(waiting.IsCompleted);
canceled.Cancel();
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => waiting);
var cleanup = store.EnterMaintenanceAsync(CancellationToken.None);
Assert.False(cleanup.IsCompleted);
writer.Dispose();
using var lease = await cleanup.WaitAsync(TimeSpan.FromSeconds(5));
}
finally { TestPaths.DeleteRoot(root); }
}
[Fact]
public void ResetKeepsExistingConnectionsOnTheSameDatabase()
{
var root = TestPaths.CreateRoot();
try
{
var store = new TranslationStore(new TestApplicationPaths(root));
store.SaveUserRules("user", new UserRulesDocument());
using var connection = store.Open();
connection.Open();
using var query = connection.CreateCommand();
query.CommandText = "SELECT COUNT(*) FROM user_rules;";
Assert.Equal(1L, query.ExecuteScalar());
store.ResetAll();
Assert.Equal(0L, query.ExecuteScalar());
}
finally { TestPaths.DeleteRoot(root); }
}
[Fact]
public async Task ProviderDoesNotRetryProgrammingErrors()
{
var attempts = 0;
using var http = new HttpClient(new TestHttpMessageHandler(_ =>
{
attempts++;
throw new InvalidOperationException("Programming error");
}));
using var request = new HttpRequestMessage(HttpMethod.Get, "https://example.test/");
await Assert.ThrowsAsync<InvalidOperationException>(() => ProviderHttp.SendWithRetryAsync(http, request, CancellationToken.None));
Assert.Equal(1, attempts);
}
[Fact]
public async Task ConcurrentProviderAdmissionsKeepTheirSpacing()
{
var limiter = new FetchRateLimiter(TimeSpan.FromMilliseconds(50));
var starts = await Task.WhenAll(Enumerable.Range(0, 4).Select(async _ =>
{
await limiter.WaitAsync(CancellationToken.None);
return Stopwatch.GetTimestamp();
}));
Array.Sort(starts);
Assert.All(starts.Zip(starts.Skip(1)), pair =>
Assert.True(Stopwatch.GetElapsedTime(pair.First, pair.Second) >= TimeSpan.FromMilliseconds(40)));
}
}
@@ -5,10 +5,76 @@ namespace Jellyfin.Plugin.Multilang.Tests;
public sealed class RefreshCoordinatorTests public sealed class RefreshCoordinatorTests
{ {
[Fact]
public async Task IndependentItemsOverlapWithinWorkerLimitAndDuplicateWorkIsShared()
{
using var coordinator = new RefreshCoordinator(concurrency: 2);
var twoStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var count = 0;
Task<bool> Queue(string id) => coordinator.EnqueueAsync(id, RefreshSourceTier.Manual, RefreshWorkClass.Movies,
RefreshJobType.Full, async (_, _) =>
{
if (Interlocked.Increment(ref count) == 2) twoStarted.SetResult();
await release.Task;
return true;
}, CancellationToken.None);
var first = Queue("first");
var second = Queue("second");
await twoStarted.Task.WaitAsync(TimeSpan.FromSeconds(5));
Assert.Same(first, Queue("first"));
var third = Queue("third");
Assert.Equal(2, count);
Assert.Equal(2, coordinator.GetDiagnostics().ActiveItems.Length);
release.SetResult();
await Task.WhenAll(first, second, third).WaitAsync(TimeSpan.FromSeconds(5));
Assert.Equal(3, count);
}
[Fact]
public async Task ActiveMissingRefreshRunsFullUpgradeBeforeCompleting()
{
using var coordinator = new RefreshCoordinator(concurrency: 1);
var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var fullRan = false;
var first = coordinator.EnqueueAsync("item", RefreshSourceTier.Background, RefreshWorkClass.Movies, RefreshJobType.Missing, async (_, _) =>
{
started.SetResult();
await release.Task;
return false;
}, CancellationToken.None);
await started.Task.WaitAsync(TimeSpan.FromSeconds(5));
var full = coordinator.EnqueueAsync("item", RefreshSourceTier.Manual, RefreshWorkClass.Movies, RefreshJobType.Full, (_, _) =>
{
fullRan = true;
return Task.FromResult(true);
}, CancellationToken.None);
release.SetResult();
Assert.True(await full.WaitAsync(TimeSpan.FromSeconds(5)));
Assert.True(fullRan);
Assert.True(await first);
}
[Fact]
public async Task CanceledWaiterDoesNotCancelAnotherWaiter()
{
using var coordinator = new RefreshCoordinator(concurrency: 1);
using var cancel = new CancellationTokenSource();
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var canceled = coordinator.EnqueueAsync("item", RefreshSourceTier.Background, RefreshWorkClass.Movies, RefreshJobType.Full,
async (_, _) => { await release.Task; return true; }, cancel.Token);
var survivor = coordinator.EnqueueAsync("item", RefreshSourceTier.Background, RefreshWorkClass.Movies, RefreshJobType.Full,
(_, _) => Task.FromResult(false), CancellationToken.None);
cancel.Cancel();
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => canceled);
release.SetResult();
Assert.True(await survivor.WaitAsync(TimeSpan.FromSeconds(5)));
}
[Fact] [Fact]
public async Task QueuedItemsUseSourceTierThenWorkClassPriority() public async Task QueuedItemsUseSourceTierThenWorkClassPriority()
{ {
var coordinator = new RefreshCoordinator(new ItemsProxyCache()); using var coordinator = new RefreshCoordinator(concurrency: 1);
var activeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var activeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var releaseActive = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseActive = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var order = new List<string>(); var order = new List<string>();
@@ -44,11 +110,11 @@ public sealed class RefreshCoordinatorTests
} }
[Fact] [Fact]
public async Task DuplicateQueuedItemUsesTheHighestJobAndTierAndInvalidatesCache() public async Task DuplicateQueuedItemUsesHighestJobAndTierWithoutInvalidatingUpstreamData()
{ {
var cache = new ItemsProxyCache(); var cache = new ItemsProxyCache();
Assert.True(cache.Store("target", "user", "/Items", ["target"], "body", "application/json", 1, 1024, TimeSpan.FromMinutes(1))); Assert.True(cache.Store("target", "user", "/Items", 1, "body", "application/json", 1, 1024, TimeSpan.FromMinutes(1)));
var coordinator = new RefreshCoordinator(cache); using var coordinator = new RefreshCoordinator(concurrency: 1);
var activeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var activeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var releaseActive = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseActive = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var runs = new List<RefreshJobType>(); var runs = new List<RefreshJobType>();
@@ -77,7 +143,7 @@ public sealed class RefreshCoordinatorTests
await Task.WhenAll(active, full); await Task.WhenAll(active, full);
Assert.Equal([RefreshJobType.Full], runs); Assert.Equal([RefreshJobType.Full], runs);
Assert.False(cache.TryGet("target", TimeSpan.FromMinutes(1), out _)); Assert.True(cache.TryGet("target", TimeSpan.FromMinutes(1), out _));
var recent = coordinator.GetDiagnostics().Recent.Single(item => item.ItemId == "target"); var recent = coordinator.GetDiagnostics().Recent.Single(item => item.ItemId == "target");
Assert.Equal("Manual", recent.SourceTier); Assert.Equal("Manual", recent.SourceTier);
Assert.Equal("Full", recent.JobType); Assert.Equal("Full", recent.JobType);
@@ -0,0 +1,33 @@
using System.Diagnostics;
using Jellyfin.Plugin.Multilang.Data;
using Jellyfin.Plugin.Multilang.Services.Refresh;
using Xunit.Abstractions;
namespace Jellyfin.Plugin.Multilang.Tests;
public sealed class TranslationStoreBatchTests(ITestOutputHelper output)
{
[Fact]
public void BatchFactsLookupMatchesIndividualReads()
{
var root = TestPaths.CreateRoot();
try
{
var store = new TranslationStore(new TestApplicationPaths(root));
var ids = Enumerable.Range(0, 100).Select(index => index.ToString()).ToArray();
foreach (var id in ids)
store.SaveMetadata(new(id, id, "movie", RefreshWorkClass.Movies, 0, 0, null, "Movie"), "fi",
new("Title", "Overview", "Tagline", "Title", "fi", [], [], [], []), new("scope", 1, true), true);
var timer = Stopwatch.StartNew();
var individual = ids.Select(id => store.GetFacts(id)!).ToArray();
var separateMs = timer.Elapsed.TotalMilliseconds;
timer.Restart();
var batch = store.GetFactsForItems(ids.Concat(ids));
var batchMs = timer.Elapsed.TotalMilliseconds;
Assert.Equal(individual, ids.Select(id => batch[id]));
Assert.Equal(ids.Length, batch.Count);
output.WriteLine($"100 individual fact reads: {separateMs:F2} ms; one batched read: {batchMs:F2} ms. Timing is diagnostic, not a pass condition.");
}
finally { TestPaths.DeleteRoot(root); }
}
}
@@ -34,7 +34,7 @@ public sealed class TranslationStoreCleanupTests
["en"], ["en"],
localAssetStorage: false); localAssetStorage: false);
Assert.Equal(new LocalCleanupResult(TranslationsDeleted: 1, AssetsDeleted: 2, GenresDeleted: 1, AssetFilesDeleted: 2), result); Assert.Equal(new LocalCleanupResult(TranslationsDeleted: 1, AssetsDeleted: 2, GenresDeleted: 1, AssetFilesDeleted: 4), result);
Assert.NotNull(store.GetFacts(live)); Assert.NotNull(store.GetFacts(live));
Assert.Null(store.GetFacts(stale)); Assert.Null(store.GetFacts(stale));
Assert.Equal("English", Translation(store, live, "en", "title")); Assert.Equal("English", Translation(store, live, "en", "title"));
@@ -0,0 +1,61 @@
using System.Text.Json;
using Jellyfin.Plugin.Multilang.Data;
using Jellyfin.Plugin.Multilang.Rules;
namespace Jellyfin.Plugin.Multilang.Tests;
public sealed class UserRulesNormalizerTests
{
[Fact]
public void LegacyFieldsAreNotConvertedOrWrittenBack()
{
var rules = JsonSerializer.Deserialize<UserRulesDocument>("""
{"Categories":[{"CriteriaText":"original_language is fi",
"FieldActions":{"title":"Language:fi"},
"Requirements":[{"Field":"original_language","Relation":"is","Values":["fi"]}]}]}
""")!;
UserRulesNormalizer.Normalize(rules, ["fi"]);
var category = Assert.Single(rules.Categories);
Assert.Empty(category.Requirements);
Assert.Equal(["Jellyfin"], category.FieldActionLists["title"]);
var saved = JsonSerializer.Serialize(rules);
Assert.DoesNotContain("CriteriaText", saved);
Assert.DoesNotContain("\"Field\":", saved);
Assert.DoesNotContain("\"FieldActions\":", saved);
}
[Fact]
public void CurrentRulesPreserveBooleanModesAndNormalizeOnlyCompleteConditions()
{
var rules = new UserRulesDocument
{
Categories = [new()
{
Label = "Finnish", MatchAllConditions = false, Scopes = ["M", "C"],
Requirements = [new() { Fields = ["audio_language", "original_language", "origin_countries"],
UseFieldOr = true, Relation = "contains", UseOr = true, Values = ["fi", "sv"] }, new()],
FieldActionLists = new() { ["title"] = ["Language:FI", "Language:de", "Original"], ["overview"] = [] }
}]
};
UserRulesNormalizer.Normalize(rules, ["fi"]);
var category = Assert.Single(rules.Categories);
Assert.False(category.MatchAllConditions);
Assert.Equal(["M", "C"], category.Scopes);
var condition = Assert.Single(category.Requirements);
Assert.Equal(["original_language", "audio_language"], condition.Fields);
Assert.True(condition.UseFieldOr);
Assert.True(condition.UseOr);
Assert.Equal(["Language:fi", "Original", "Jellyfin"], category.FieldActionLists["title"]);
Assert.Empty(category.FieldActionLists["overview"]);
Assert.Equal(JsonSerializer.Serialize(rules), JsonSerializer.Serialize(UserRulesNormalizer.Normalize(rules, ["fi"])));
}
[Fact]
public void NullExternalCollectionsNormalizeToDefaults()
{
var rules = JsonSerializer.Deserialize<UserRulesDocument>("""{"Categories":null,"FallbackFieldActions":null}""")!;
UserRulesNormalizer.Normalize(rules);
Assert.Empty(rules.Categories);
Assert.Equal(["Jellyfin"], rules.FallbackFieldActions["poster"]);
}
}
+87 -2
View File
@@ -101,9 +101,15 @@ test('pre-caches the fixture movie library after user activity', async ({ reques
const entries = await apiGet(request, '/Multilang/CacheEntries'); const entries = await apiGet(request, '/Multilang/CacheEntries');
return entries.some(entry => entry.Url.includes(`ParentId=${fixture.libraryId}`)); return entries.some(entry => entry.Url.includes(`ParentId=${fixture.libraryId}`));
}, { timeout: 60000 }).toBeTruthy(); }, { timeout: 60000 }).toBeTruthy();
await getFixtureItems(request);
const recent = await apiGet(request, '/Multilang/ItemsProxyRequests');
expect(recent[0].CacheHit).toBeTruthy();
}); });
test('shows configured translations and uses the injected ItemsProxy route', async ({ page, request }) => { test('shows configured translations and uses the injected ItemsProxy route', async ({ page, request }) => {
const rules = await apiGet(request, '/Multilang/UserRules/self');
rules.FallbackFieldActions.poster = ['Language:fi', 'Jellyfin'];
await apiPost(request, '/Multilang/UserRules/self', rules);
const proxiedRequests = []; const proxiedRequests = [];
page.on('request', request => { page.on('request', request => {
if (new URL(request.url()).pathname === '/Multilang/ItemsProxy') { if (new URL(request.url()).pathname === '/Multilang/ItemsProxy') {
@@ -112,7 +118,10 @@ test('shows configured translations and uses the injected ItemsProxy route', asy
}); });
await signIn(page); await signIn(page);
const artworkResponse = page.waitForResponse(response =>
new URL(response.url()).pathname === `/Multilang/Assets/${fixture.orderedItemIds[0]}/fixture.png`);
await openFixtureLibrary(page, fixture.translatedTitles[0]); await openFixtureLibrary(page, fixture.translatedTitles[0]);
expect((await artworkResponse).status()).toBe(200);
for (const title of fixture.translatedTitles) { for (const title of fixture.translatedTitles) {
await expect(page.getByText(title, { exact: true })).toBeVisible(); await expect(page.getByText(title, { exact: true })).toBeVisible();
@@ -151,7 +160,6 @@ test('applies a structured category rule before fallback translation actions', a
Id: 'browser-fixture-swedish', Id: 'browser-fixture-swedish',
Label: 'Swedish originals', Label: 'Swedish originals',
Requirements: [{ Requirements: [{
Field: 'original_language',
Fields: ['original_language'], Fields: ['original_language'],
Relation: 'is', Relation: 'is',
Values: ['sv'], Values: ['sv'],
@@ -161,7 +169,6 @@ test('applies a structured category rule before fallback translation actions', a
MatchAllConditions: true, MatchAllConditions: true,
Scopes: ['M'], Scopes: ['M'],
FieldActionLists: categoryActions, FieldActionLists: categoryActions,
FieldActions: {}
}]; }];
await apiPost(request, '/Multilang/UserRules/self', rules); await apiPost(request, '/Multilang/UserRules/self', rules);
@@ -257,3 +264,81 @@ test('saves a structured classification rule through the user settings page', as
Values: ['sv'] Values: ['sv']
})]); })]);
}); });
test('rejects unsafe proxy targets and enforces restricted-user item access', async ({ request }) => {
for (const url of ['//outside.invalid/Items', '/System/Info', '/Multilang/ItemsProxy']) {
const response = await request.get('/Multilang/ItemsProxy', { headers: authHeaders(), params: { url } });
expect(response.status()).toBe(400);
}
const name = 'multilang-access-test-' + Date.now();
const user = await apiPost(request, '/Users/New', { Name: name });
const policy = { ...user.Policy, IsAdministrator: false, EnableAllFolders: true };
let response = await request.post(`/Users/${user.Id}/Policy`, { headers: authHeaders(), data: policy });
expect(response.ok()).toBeTruthy();
response = await request.post('/Users/AuthenticateByName', {
headers: { Authorization: 'MediaBrowser Client="Multilang tests", Device="test", DeviceId="multilang-access-test", Version="1"' },
data: { Username: name, Pw: '' }
});
expect(response.ok()).toBeTruthy();
const login = await response.json();
const headers = { Authorization: `MediaBrowser Token="${login.AccessToken}"` };
const asset = `/Multilang/Assets/${fixture.orderedItemIds[0]}/fixture.png`;
expect((await request.get(asset)).status()).toBe(401);
expect((await request.get(asset, { headers })).status()).toBe(200);
response = await request.post('/Multilang/UserRules/self', { headers, data: { Enabled: true, FallbackFieldActions: jellyfinActions() } });
expect(response.ok()).toBeTruthy();
const params = { url: `/Items?ParentId=${fixture.libraryId}&IncludeItemTypes=Movie&Recursive=true&SortBy=SortName` };
response = await request.get('/Multilang/ItemsProxy', { headers, params });
expect(response.ok()).toBeTruthy();
expect((await response.json()).Items).toHaveLength(3);
let entries = await apiGet(request, '/Multilang/CacheEntries');
expect(entries.some(entry => entry.UserId === user.Id)).toBeTruthy();
policy.EnableAllFolders = false;
policy.EnabledFolders = [];
response = await request.post(`/Users/${user.Id}/Policy`, { headers: authHeaders(), data: policy });
expect(response.ok()).toBeTruthy();
response = await request.get('/Multilang/ItemsProxy', { headers, params });
const native = await request.get(`/Users/${user.Id}/Items?ParentId=${fixture.libraryId}`, { headers });
expect(native.status()).toBe(401);
expect(response.status()).toBe(native.status());
const recent = await apiGet(request, '/Multilang/ItemsProxyRequests');
expect(recent[0].CacheHit).toBeFalsy();
expect((await request.get(asset, { headers })).status()).toBe(404);
entries = await apiGet(request, '/Multilang/CacheEntries');
expect(entries.some(entry => entry.UserId === user.Id && entry.ItemCount > 0)).toBeFalsy();
for (const path of [`Debug/${fixture.orderedItemIds[0]}`, `Debug/${fixture.orderedItemIds[0]}/self`]) {
expect((await request.get(`/Multilang/${path}`, { headers })).status()).toBe(404);
}
expect((await request.post(`/Multilang/RefreshItem/${fixture.orderedItemIds[0]}?includeChildren=true`, { headers })).status()).toBe(404);
response = await request.get('/Multilang/ItemsProxy', { headers, params });
expect(response.status()).toBe(401);
});
test('reuses a library fetch across different pages and sorting controls', async ({ request }) => {
const base = `/Items?ParentId=${fixture.libraryId}&IncludeItemTypes=Movie&Recursive=true`;
let response = await request.get('/Multilang/ItemsProxy', { headers: authHeaders(), params: { url: base + '&SortBy=SortName&Limit=1', mlLocale: 'fi-FI' } });
expect((await response.json()).Items.map(item => item.Name)).toEqual(['Ankkuri']);
response = await request.get('/Multilang/ItemsProxy', { headers: authHeaders(), params: { url: base + '&SortBy=SortName&StartIndex=1&Limit=1', mlLocale: 'fi-FI' } });
expect((await response.json()).Items.map(item => item.Name)).toEqual(['The Clock']);
const recent = await apiGet(request, '/Multilang/ItemsProxyRequests');
expect(recent[0].CacheHit).toBeTruthy();
});
test('reopens user settings without duplicate loads or a native replaceChildren dependency', async ({ page }) => {
await page.addInitScript(() => {
Object.defineProperty(Element.prototype, 'replaceChildren', { configurable: true, value: undefined });
});
await signIn(page);
await page.goto(`/web/#/mypreferencesmenu?userId=${fixture.userId}`);
let loads = 0;
page.on('request', request => { if (new URL(request.url()).pathname === '/Multilang/UserRules/self') loads++; });
for (let i = 0; i < 3; i++) {
await expect(page.locator('#mlUserRulesMenuItem')).toBeVisible();
await page.locator('#mlUserRulesMenuItem').click();
await expect(page.locator('#ml-sort-locale option[value="en"]')).toHaveCount(1);
expect(loads).toBe(i + 1);
await page.evaluate(userId => { location.hash = '#/mypreferencesmenu?userId=' + userId; }, fixture.userId);
await expect(page.locator('#ml-user-shell')).toHaveCount(0);
}
});
+8
View File
@@ -144,6 +144,14 @@ for row in rows:
INSERT INTO genres(tmdb_id, media, lang, name, name_norm) VALUES (?, 'movie', 'fi', ?, ?) INSERT INTO genres(tmdb_id, media, lang, name, name_norm) VALUES (?, 'movie', 'fi', ?, ?)
ON CONFLICT(tmdb_id, media, lang) DO UPDATE SET name=excluded.name, name_norm=excluded.name_norm ON CONFLICT(tmdb_id, media, lang) DO UPDATE SET name=excluded.name, name_norm=excluded.name_norm
""", (row["genre_id"], row["genre"], row["genre"].lower())) """, (row["genre_id"], row["genre"], row["genre"].lower()))
relative = row["item_id"] + "/fixture.png"
destination = os.path.join(os.path.dirname(database), "assets", relative)
os.makedirs(os.path.dirname(destination), exist_ok=True)
with open(destination, "wb") as image:
image.write(base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jZ1kAAAAASUVORK5CYII="))
path = "/Multilang/Assets/" + relative
connection.execute("INSERT OR REPLACE INTO assets(item_id,lang,kind,path,path_low,updated_at) VALUES (?, 'fi', 'poster', ?, ?, ?)",
(row["item_id"], path, path.lower(), now))
connection.commit() connection.commit()
''' '''
remote.stop() remote.stop()