Add backup export and local artwork storage

This commit is contained in:
ajp_anton committed 2026-06-08 04:19:08 +00:00
1 parent 63dbc3a6d4
commit a4bb9a98c2
14 files changed
+1745 -46

No files matched your search

@@ -9,35 +9,9 @@ namespace Jellyfin.Plugin.Multilang.Data;
public sealed class TranslationStore
{
private readonly string _dbPath;
public const string LocalAssetUrlPrefix = "/Multilang/Assets/";
public TranslationStore(IApplicationPaths appPaths)
{
var dir = Path.Combine(appPaths.DataPath, "multilang");
Directory.CreateDirectory(dir);
Directory.CreateDirectory(Path.Combine(dir, "assets"));
_dbPath = Path.Combine(dir, "translations.sqlite");
Initialize();
}
public static string ToItemId32(Guid id)
=> id.ToString("N", CultureInfo.InvariantCulture).ToLowerInvariant();
public static long NowUnixUtc()
=> DateTimeOffset.UtcNow.ToUnixTimeSeconds();
public static string JsonArray(IEnumerable<string> values)
=> JsonSerializer.Serialize(values.Where(v => !string.IsNullOrWhiteSpace(v)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray());
public SqliteConnection Open()
=> new($"Data Source={_dbPath};Cache=Shared");
private void Initialize()
{
using var con = Open();
con.Open();
using var cmd = con.CreateCommand();
cmd.CommandText = @"
private const string SchemaSql = @"
CREATE TABLE IF NOT EXISTS facts (
item_id TEXT PRIMARY KEY,
tmdb_id TEXT NOT NULL,
@@ -107,10 +81,70 @@ CREATE TABLE IF NOT EXISTS scan_state (
);
INSERT OR IGNORE INTO scan_state(id, last_scan_started) VALUES (1, 0);
";
private readonly string _dataDir;
private readonly string _assetsDir;
private readonly string _dbPath;
public TranslationStore(IApplicationPaths appPaths)
{
_dataDir = Path.Combine(appPaths.DataPath, "multilang");
_assetsDir = Path.Combine(_dataDir, "assets");
Directory.CreateDirectory(_dataDir);
Directory.CreateDirectory(_assetsDir);
_dbPath = Path.Combine(_dataDir, "translations.sqlite");
Initialize();
}
public string DataDirectory => _dataDir;
public string AssetsDirectory => _assetsDir;
public string DatabasePath => _dbPath;
public static string ToItemId32(Guid id)
=> id.ToString("N", CultureInfo.InvariantCulture).ToLowerInvariant();
public static long NowUnixUtc()
=> DateTimeOffset.UtcNow.ToUnixTimeSeconds();
public static string JsonArray(IEnumerable<string> values)
=> JsonSerializer.Serialize(values.Where(v => !string.IsNullOrWhiteSpace(v)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray());
public SqliteConnection Open()
=> new($"Data Source={_dbPath};Cache=Shared;Pooling=False");
private void Initialize()
{
using var con = Open();
con.Open();
using var cmd = con.CreateCommand();
cmd.CommandText = SchemaSql;
cmd.ExecuteNonQuery();
EnsureColumn(con, "assets", "path_low", "TEXT NOT NULL DEFAULT ''");
}
public void ResetAll()
{
SqliteConnection.ClearAllPools();
if (Directory.Exists(_dataDir))
Directory.Delete(_dataDir, recursive: true);
Directory.CreateDirectory(_dataDir);
Directory.CreateDirectory(_assetsDir);
Initialize();
SqliteConnection.ClearAllPools();
}
public (long DatabaseBytes, long AssetsBytes) GetStorageUsage()
=> (File.Exists(_dbPath) ? new FileInfo(_dbPath).Length : 0, DirectorySize(_assetsDir));
private static long DirectorySize(string path)
{
if (!Directory.Exists(path))
return 0;
return Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories).Sum(file => new FileInfo(file).Length);
}
private static void EnsureColumn(SqliteConnection con, string table, string column, string definition)
{
using (var info = con.CreateCommand())
@@ -173,6 +207,137 @@ DELETE FROM assets WHERE item_id = $item_id;";
}
tx.Commit();
CleanupUnreferencedAssetFiles();
}
public LocalCleanupResult CleanupForConfiguration(IReadOnlySet<string> liveItemIds, IEnumerable<string> allowedLanguages, bool localAssetStorage)
{
CleanupNotInLibrary(liveItemIds);
var allowed = allowedLanguages
.Where(l => !string.IsNullOrWhiteSpace(l))
.Select(l => l.Trim())
.Append("Original")
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
var translationsDeleted = DeleteRowsOutsideLanguages("translations", allowed);
var genresDeleted = DeleteRowsOutsideLanguages("genres", allowed);
var assetsDeleted = DeleteRowsOutsideLanguages("assets", allowed);
var modeRowsDeleted = localAssetStorage ? 0 : DeleteLocalAssetRows(resetMissingCheckedAt: true);
var assetFilesDeleted = CleanupUnreferencedAssetFiles();
return new LocalCleanupResult(translationsDeleted, assetsDeleted + modeRowsDeleted, genresDeleted, assetFilesDeleted);
}
private int DeleteRowsOutsideLanguages(string table, IReadOnlyList<string> allowedLanguages)
{
using var con = Open();
con.Open();
using var cmd = con.CreateCommand();
var langParams = AddParams(cmd, allowedLanguages, "$lang");
cmd.CommandText = $"DELETE FROM {table} WHERE lang NOT IN ({langParams});";
return cmd.ExecuteNonQuery();
}
private int DeleteLocalAssetRows(bool resetMissingCheckedAt)
{
using var con = Open();
con.Open();
using var tx = con.BeginTransaction();
var affected = new List<string>();
using (var select = con.CreateCommand())
{
select.Transaction = tx;
select.CommandText = "SELECT DISTINCT item_id FROM assets WHERE path_low LIKE '/multilang/assets/%';";
using var reader = select.ExecuteReader();
while (reader.Read())
affected.Add(reader.GetString(0));
}
int deleted;
using (var delete = con.CreateCommand())
{
delete.Transaction = tx;
delete.CommandText = "DELETE FROM assets WHERE path_low LIKE '/multilang/assets/%';";
deleted = delete.ExecuteNonQuery();
}
if (resetMissingCheckedAt && affected.Count > 0)
{
using var update = con.CreateCommand();
update.Transaction = tx;
var idParams = AddParams(update, affected, "$item");
update.CommandText = $"UPDATE facts SET missing_checked_at = 0 WHERE item_id IN ({idParams});";
update.ExecuteNonQuery();
}
tx.Commit();
return deleted;
}
public int CleanupUnreferencedAssetFiles()
{
if (!Directory.Exists(_assetsDir))
return 0;
var referenced = GetReferencedLocalAssetPaths();
var deleted = 0;
foreach (var file in Directory.EnumerateFiles(_assetsDir, "*", SearchOption.AllDirectories))
{
var full = Path.GetFullPath(file);
if (referenced.Contains(full))
continue;
File.Delete(file);
deleted++;
}
foreach (var dir in Directory.EnumerateDirectories(_assetsDir, "*", SearchOption.AllDirectories).OrderByDescending(d => d.Length))
{
if (!Directory.EnumerateFileSystemEntries(dir).Any())
Directory.Delete(dir);
}
return deleted;
}
private HashSet<string> GetReferencedLocalAssetPaths()
{
using var con = Open();
con.Open();
using var cmd = con.CreateCommand();
cmd.CommandText = "SELECT path FROM assets;";
using var reader = cmd.ExecuteReader();
var result = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
while (reader.Read())
{
var path = reader.GetString(0);
if (TryNormalizeLocalAssetPath(path, out var fullPath))
result.Add(fullPath);
}
return result;
}
public bool TryNormalizeLocalAssetPath(string path, out string fullPath)
{
fullPath = string.Empty;
if (string.IsNullOrWhiteSpace(path) || Uri.TryCreate(path, UriKind.Absolute, out _))
return false;
var relative = path;
if (relative.StartsWith(LocalAssetUrlPrefix, StringComparison.OrdinalIgnoreCase))
relative = relative[LocalAssetUrlPrefix.Length..];
else if (Path.IsPathRooted(relative))
return false;
var candidate = Path.GetFullPath(Path.Combine(_assetsDir, relative));
var root = Path.GetFullPath(_assetsDir) + Path.DirectorySeparatorChar;
if (!candidate.StartsWith(root, StringComparison.OrdinalIgnoreCase))
return false;
fullPath = candidate;
return true;
}
public long GetLastScanStarted()
@@ -421,6 +586,68 @@ ON CONFLICT(item_id, lang, kind) DO UPDATE SET
cmd.ExecuteNonQuery();
}
public string? GetAssetPath(string itemId, string lang, string kind)
{
using var con = Open();
con.Open();
using var cmd = con.CreateCommand();
cmd.CommandText = "SELECT path FROM assets WHERE item_id = $item_id AND lang = $lang AND kind = $kind;";
cmd.Parameters.AddWithValue("$item_id", itemId);
cmd.Parameters.AddWithValue("$lang", lang);
cmd.Parameters.AddWithValue("$kind", kind);
return cmd.ExecuteScalar() as string;
}
public AssetPresence GetAssetPresence(string itemId, IEnumerable<string> languages)
{
var langs = languages.Where(l => !string.IsNullOrWhiteSpace(l)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
if (langs.Length == 0)
return new AssetPresence(false, false);
using var con = Open();
con.Open();
using var cmd = con.CreateCommand();
var langParams = AddParams(cmd, langs, "$lang");
cmd.CommandText = $"SELECT path_low FROM assets WHERE item_id = $item_id AND lang IN ({langParams});";
cmd.Parameters.AddWithValue("$item_id", itemId);
using var reader = cmd.ExecuteReader();
var any = false;
var anyRemote = false;
while (reader.Read())
{
any = true;
var path = reader.GetString(0);
if (path.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || path.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
anyRemote = true;
}
return new AssetPresence(any, anyRemote);
}
public bool HasMissingConfiguredTranslations(string itemId, IEnumerable<string> languages)
{
var langs = languages.Where(l => !string.IsNullOrWhiteSpace(l)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
if (langs.Length == 0)
return false;
using var con = Open();
con.Open();
using var cmd = con.CreateCommand();
var langParams = AddParams(cmd, langs, "$lang");
cmd.CommandText = $@"
SELECT lang, COUNT(DISTINCT field)
FROM translations
WHERE item_id = $item_id AND lang IN ({langParams}) AND field IN ('title', 'overview', 'tagline')
GROUP BY lang;";
cmd.Parameters.AddWithValue("$item_id", itemId);
using var reader = cmd.ExecuteReader();
var counts = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
while (reader.Read())
counts[reader.GetString(0)] = reader.GetInt32(1);
return langs.Any(lang => !counts.TryGetValue(lang, out var count) || count < 3);
}
public void UpsertGenre(int tmdbId, string media, string lang, string name)
{
if (tmdbId <= 0 || string.IsNullOrWhiteSpace(media) || string.IsNullOrWhiteSpace(lang) || string.IsNullOrWhiteSpace(name))
@@ -623,6 +850,243 @@ ON CONFLICT(user_id) DO UPDATE SET langs_json = excluded.langs_json, updated_at
return string.IsNullOrWhiteSpace(json) ? null : JsonSerializer.Deserialize<UserRulesDocument>(json);
}
public Dictionary<string, UserRulesDocument> GetAllUserRules()
{
using var con = Open();
con.Open();
using var cmd = con.CreateCommand();
cmd.CommandText = "SELECT user_id, rules_json FROM user_rules ORDER BY user_id;";
using var reader = cmd.ExecuteReader();
var result = new Dictionary<string, UserRulesDocument>(StringComparer.OrdinalIgnoreCase);
while (reader.Read())
{
var userId = reader.GetString(0);
var json = reader.GetString(1);
var rules = JsonSerializer.Deserialize<UserRulesDocument>(json);
if (rules is not null)
result[userId] = rules;
}
return result;
}
public void ExportTranslationDatabase(string destinationPath)
{
if (File.Exists(destinationPath))
File.Delete(destinationPath);
using var destination = new SqliteConnection($"Data Source={destinationPath}");
destination.Open();
using (var schema = destination.CreateCommand())
{
schema.CommandText = SchemaSql;
schema.ExecuteNonQuery();
}
using var source = Open();
source.Open();
CopyTable(source, destination, "facts", "item_id, tmdb_id, kind, original_title, original_language, original_language_all, origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language, genre_tmdb_ids_json, missing_checked_at, full_checked_at");
CopyTable(source, destination, "translations", "item_id, lang, field, text");
CopyTable(source, destination, "assets", "item_id, lang, kind, path, path_low, updated_at");
CopyTable(source, destination, "genres", "tmdb_id, media, lang, name, name_norm");
CopyTable(source, destination, "scan_state", "id, last_scan_started");
}
public ImportDatabaseResult ImportTranslationDatabase(string sourcePath, IReadOnlySet<string> liveItemIds, IReadOnlySet<string> allowedLanguages)
{
var allowed = allowedLanguages.ToHashSet(StringComparer.OrdinalIgnoreCase);
allowed.Add("Original");
using var destination = Open();
destination.Open();
using var source = new SqliteConnection($"Data Source={sourcePath};Mode=ReadOnly");
source.Open();
using var tx = destination.BeginTransaction();
var facts = ImportFacts(source, destination, tx, liveItemIds);
var translations = ImportTranslations(source, destination, tx, liveItemIds, allowed);
var assets = ImportAssets(source, destination, tx, liveItemIds, allowed);
var genres = ImportGenres(source, destination, tx, allowed);
tx.Commit();
return new ImportDatabaseResult(facts, translations, assets, genres);
}
private static void CopyTable(SqliteConnection source, SqliteConnection destination, string table, string columns)
{
using var read = source.CreateCommand();
read.CommandText = $"SELECT {columns} FROM {table};";
using var reader = read.ExecuteReader();
using var tx = destination.BeginTransaction();
while (reader.Read())
{
using var insert = destination.CreateCommand();
insert.Transaction = tx;
var names = columns.Split(',', StringSplitOptions.TrimEntries);
insert.CommandText = $"INSERT OR REPLACE INTO {table}({columns}) VALUES({string.Join(",", names.Select((_, i) => "$v" + i.ToString(CultureInfo.InvariantCulture)))});";
for (var i = 0; i < names.Length; i++)
insert.Parameters.AddWithValue("$v" + i.ToString(CultureInfo.InvariantCulture), reader.GetValue(i));
insert.ExecuteNonQuery();
}
tx.Commit();
}
private static int ImportFacts(SqliteConnection source, SqliteConnection destination, SqliteTransaction tx, IReadOnlySet<string> liveItemIds)
{
using var read = source.CreateCommand();
read.CommandText = @"
SELECT item_id, tmdb_id, kind, original_title, original_language, original_language_all,
origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language,
genre_tmdb_ids_json, missing_checked_at, full_checked_at
FROM facts;";
using var reader = read.ExecuteReader();
var count = 0;
while (reader.Read())
{
var itemId = reader.GetString(0);
if (!liveItemIds.Contains(itemId))
continue;
using var insert = destination.CreateCommand();
insert.Transaction = tx;
insert.CommandText = @"
INSERT INTO facts(item_id, tmdb_id, kind, original_title, original_language, original_language_all,
origin_countries_json, production_countries_json, spoken_languages_json, audio_track_language,
genre_tmdb_ids_json, missing_checked_at, full_checked_at)
VALUES($item_id, $tmdb_id, $kind, $original_title, $original_language, $original_language_all,
$origin_countries_json, $production_countries_json, $spoken_languages_json, $audio_track_language,
$genre_tmdb_ids_json, $missing_checked_at, $full_checked_at)
ON CONFLICT(item_id) DO UPDATE SET
tmdb_id = excluded.tmdb_id,
kind = excluded.kind,
original_title = excluded.original_title,
original_language = excluded.original_language,
original_language_all = excluded.original_language_all,
origin_countries_json = excluded.origin_countries_json,
production_countries_json = excluded.production_countries_json,
spoken_languages_json = excluded.spoken_languages_json,
audio_track_language = excluded.audio_track_language,
genre_tmdb_ids_json = excluded.genre_tmdb_ids_json,
missing_checked_at = excluded.missing_checked_at,
full_checked_at = excluded.full_checked_at;";
var names = new[]
{
"$item_id",
"$tmdb_id",
"$kind",
"$original_title",
"$original_language",
"$original_language_all",
"$origin_countries_json",
"$production_countries_json",
"$spoken_languages_json",
"$audio_track_language",
"$genre_tmdb_ids_json",
"$missing_checked_at",
"$full_checked_at"
};
for (var i = 0; i < names.Length; i++)
insert.Parameters.AddWithValue(names[i], reader.GetValue(i));
insert.ExecuteNonQuery();
count++;
}
return count;
}
private static int ImportTranslations(SqliteConnection source, SqliteConnection destination, SqliteTransaction tx, IReadOnlySet<string> liveItemIds, IReadOnlySet<string> allowedLanguages)
{
using var read = source.CreateCommand();
read.CommandText = "SELECT item_id, lang, field, text FROM translations;";
using var reader = read.ExecuteReader();
var count = 0;
while (reader.Read())
{
var itemId = reader.GetString(0);
var lang = reader.GetString(1);
if (!liveItemIds.Contains(itemId) || !allowedLanguages.Contains(lang))
continue;
using var insert = destination.CreateCommand();
insert.Transaction = tx;
insert.CommandText = @"
INSERT INTO translations(item_id, lang, field, text)
VALUES($item_id, $lang, $field, $text)
ON CONFLICT(item_id, lang, field) DO UPDATE SET text = excluded.text;";
insert.Parameters.AddWithValue("$item_id", itemId);
insert.Parameters.AddWithValue("$lang", lang);
insert.Parameters.AddWithValue("$field", reader.GetString(2));
insert.Parameters.AddWithValue("$text", reader.GetString(3));
insert.ExecuteNonQuery();
count++;
}
return count;
}
private static int ImportAssets(SqliteConnection source, SqliteConnection destination, SqliteTransaction tx, IReadOnlySet<string> liveItemIds, IReadOnlySet<string> allowedLanguages)
{
using var read = source.CreateCommand();
read.CommandText = "SELECT item_id, lang, kind, path, path_low, updated_at FROM assets;";
using var reader = read.ExecuteReader();
var count = 0;
while (reader.Read())
{
var itemId = reader.GetString(0);
var lang = reader.GetString(1);
if (!liveItemIds.Contains(itemId) || !allowedLanguages.Contains(lang))
continue;
using var insert = destination.CreateCommand();
insert.Transaction = tx;
insert.CommandText = @"
INSERT INTO assets(item_id, lang, kind, path, path_low, updated_at)
VALUES($item_id, $lang, $kind, $path, $path_low, $updated_at)
ON CONFLICT(item_id, lang, kind) DO UPDATE SET
path = excluded.path,
path_low = excluded.path_low,
updated_at = excluded.updated_at;";
insert.Parameters.AddWithValue("$item_id", itemId);
insert.Parameters.AddWithValue("$lang", lang);
insert.Parameters.AddWithValue("$kind", reader.GetString(2));
insert.Parameters.AddWithValue("$path", reader.GetString(3));
insert.Parameters.AddWithValue("$path_low", reader.GetString(4));
insert.Parameters.AddWithValue("$updated_at", reader.GetInt64(5));
insert.ExecuteNonQuery();
count++;
}
return count;
}
private static int ImportGenres(SqliteConnection source, SqliteConnection destination, SqliteTransaction tx, IReadOnlySet<string> allowedLanguages)
{
using var read = source.CreateCommand();
read.CommandText = "SELECT tmdb_id, media, lang, name, name_norm FROM genres;";
using var reader = read.ExecuteReader();
var count = 0;
while (reader.Read())
{
var lang = reader.GetString(2);
if (!allowedLanguages.Contains(lang))
continue;
using var insert = destination.CreateCommand();
insert.Transaction = tx;
insert.CommandText = @"
INSERT INTO genres(tmdb_id, media, lang, name, name_norm)
VALUES($tmdb_id, $media, $lang, $name, $name_norm)
ON CONFLICT(tmdb_id, media, lang) DO UPDATE SET
name = excluded.name,
name_norm = excluded.name_norm;";
insert.Parameters.AddWithValue("$tmdb_id", reader.GetInt32(0));
insert.Parameters.AddWithValue("$media", reader.GetString(1));
insert.Parameters.AddWithValue("$lang", lang);
insert.Parameters.AddWithValue("$name", reader.GetString(3));
insert.Parameters.AddWithValue("$name_norm", reader.GetString(4));
insert.ExecuteNonQuery();
count++;
}
return count;
}
private static string AddParams(SqliteCommand cmd, IReadOnlyList<string> values, string prefix)
{
var names = new string[values.Count];
@@ -639,6 +1103,12 @@ ON CONFLICT(user_id) DO UPDATE SET langs_json = excluded.langs_json, updated_at
public readonly record struct FactsStatus(bool Exists, long MissingCheckedAt, long FullCheckedAt);
public readonly record struct AssetPresence(bool Any, bool AnyRemote);
public readonly record struct LocalCleanupResult(int TranslationsDeleted, int AssetsDeleted, int GenresDeleted, int AssetFilesDeleted);
public readonly record struct ImportDatabaseResult(int Facts, int Translations, int Assets, int Genres);
public readonly record struct GenreRow(int TmdbId, string Media, string Lang, string Name);
public sealed record FactsData(