diff --git a/StarterNG.Tests/Fakes/InMemoryFileSystem.cs b/StarterNG.Tests/Fakes/InMemoryFileSystem.cs index 1a12a86..fca2cbd 100644 --- a/StarterNG.Tests/Fakes/InMemoryFileSystem.cs +++ b/StarterNG.Tests/Fakes/InMemoryFileSystem.cs @@ -134,6 +134,8 @@ public sealed class InMemoryFileSystem : IFileSystem public DateTime GetLastWriteTimeUtc(string path) => LastWriteTimeUtc; /// Files added through carry the execute bit. + public long GetFileSize(string path) => Bytes(path).LongLength; + public bool IsExecutable(string path) => _executable.Contains(Normalize(path)); public InMemoryFileSystem WithExecutable(string path, string contents = "\u007fELF") diff --git a/StarterNG.Tests/Fakes/TestInstallation.cs b/StarterNG.Tests/Fakes/TestInstallation.cs index 724a8d7..fbb39a3 100644 --- a/StarterNG.Tests/Fakes/TestInstallation.cs +++ b/StarterNG.Tests/Fakes/TestInstallation.cs @@ -26,12 +26,15 @@ public sealed class TestInstallation MiniTextures = new MiniTextureIndex(Files, Paths); Physics = new FizPhysicsRepository(Files, Paths, Log); Vehicles = new TexturesTxtVehicleRepository(Files, Paths, Log, new TexturesTxtParser()); + VehicleJson = new VehicleJsonSerializer(); Sceneries = new SceneryRepository(Files, Paths, Log, new SceneryParser(Clock, Random)); MissingAssets = new MissingAssetScanner(Files, Paths); - Library = new GameLibrary(Vehicles, Sceneries, MiniTextures, Physics, Log); - Environment = new FakeEnvironment(); SettingsPaths = new SettingsPaths(Environment, Paths); + VehicleCache = new VehicleCacheRepository(Files, Paths, Log, Vehicles, VehicleJson, + SettingsPaths.VehicleCacheDirectory()); + Library = new GameLibrary(VehicleCache, Sceneries, MiniTextures, Physics, Log); + Executables = new ExecutableLocator(Files, Paths, Environment, Log); SettingsStore = new SettingsStore(Files, Clock, Log, SettingsPaths, new SettingsSerializer(), Executables); } @@ -45,6 +48,8 @@ public sealed class TestInstallation public MiniTextureIndex MiniTextures { get; } public FizPhysicsRepository Physics { get; } public TexturesTxtVehicleRepository Vehicles { get; } + public VehicleJsonSerializer VehicleJson { get; } + public VehicleCacheRepository VehicleCache { get; } public SceneryRepository Sceneries { get; } public MissingAssetScanner MissingAssets { get; } public GameLibrary Library { get; } diff --git a/StarterNG.Tests/VehicleCacheTests.cs b/StarterNG.Tests/VehicleCacheTests.cs new file mode 100644 index 0000000..5931ca8 --- /dev/null +++ b/StarterNG.Tests/VehicleCacheTests.cs @@ -0,0 +1,111 @@ +using System.Text; +using StarterNG.Domain.Vehicles; +using StarterNG.Tests.Fakes; + +namespace StarterNG.Tests; + +public class VehicleCacheTests +{ + private const string Ep07 = "ep07-135.mat=ep07.e3d,ep07"; + + private const string En57 = "en57-1000ra.mat=en57.e3d,en57"; + + private static string Dynamic(string maker, string vehicle) => + TestInstallation.At("dynamic", maker, vehicle, "textures.txt"); + + private static string Cached(TestInstallation installation, string file) => + Path.Combine(installation.SettingsPaths.VehicleCacheDirectory(), file); + + [Fact] + public void The_first_load_migrates_textures_txt_into_the_starters_own_copy() + { + var files = new InMemoryFileSystem().WithLegacyFile(Dynamic("pkp", "ep07"), Ep07); + var installation = new TestInstallation(files); + + int liveries = installation.VehicleCache.Load(installation.Library.Vehicles); + + Assert.Equal(1, liveries); + Assert.True(files.FileExists(Cached(installation, "vehicles.json"))); + Assert.Contains("ep07-135", files.ReadAllText(Cached(installation, "vehicles.json"))); + } + + [Fact] + public void An_untouched_installation_is_served_from_the_copy_without_reading_textures_txt() + { + var files = new InMemoryFileSystem().WithLegacyFile(Dynamic("pkp", "ep07"), Ep07); + var installation = new TestInstallation(files); + installation.VehicleCache.Load(installation.Library.Vehicles); + + // Different liveries, same length and same timestamp: nothing the stamp + // can see has changed, so an answer of ep07-135 can only come from the copy. + files.WithLegacyFile(Dynamic("pkp", "ep07"), "ep07-999.mat=ep07.e3d,ep07"); + var catalog = new VehicleCatalog(installation.MiniTextures); + installation.VehicleCache.Load(catalog); + + Assert.Equal("legacy:pkp/ep07/ep07-135", Assert.Single(catalog.Textures).Uuid); + } + + [Fact] + public void An_edited_textures_txt_makes_the_copy_stale_and_it_is_rebuilt() + { + var files = new InMemoryFileSystem().WithLegacyFile(Dynamic("pkp", "ep07"), Ep07); + var installation = new TestInstallation(files); + installation.VehicleCache.Load(installation.Library.Vehicles); + + files.WithLegacyFile(Dynamic("pkp", "ep07"), Ep07 + "\nep07-174.mat=ep07.e3d,ep07"); + var catalog = new VehicleCatalog(installation.MiniTextures); + int liveries = installation.VehicleCache.Load(catalog); + + Assert.Equal(2, liveries); + Assert.Contains("ep07-174", files.ReadAllText(Cached(installation, "vehicles.json"))); + } + + [Fact] + public void A_vehicle_folder_that_disappeared_upstream_leaves_the_copy_too() + { + var files = new InMemoryFileSystem() + .WithLegacyFile(Dynamic("pkp", "ep07"), Ep07) + .WithLegacyFile(Dynamic("pkp", "en57"), En57); + var installation = new TestInstallation(files); + installation.VehicleCache.Load(installation.Library.Vehicles); + + files.DeleteFile(Dynamic("pkp", "en57")); + var catalog = new VehicleCatalog(installation.MiniTextures); + installation.VehicleCache.Load(catalog); + + Assert.Equal("legacy:pkp/ep07/ep07-135", Assert.Single(catalog.Textures).Uuid); + Assert.DoesNotContain("en57", files.ReadAllText(Cached(installation, "vehicles.json"))); + } + + [Fact] + public void A_damaged_copy_costs_the_parse_it_saved_and_nothing_else() + { + var files = new InMemoryFileSystem().WithLegacyFile(Dynamic("pkp", "ep07"), Ep07); + var installation = new TestInstallation(files); + installation.VehicleCache.Load(installation.Library.Vehicles); + files.WithFile(Cached(installation, "vehicles.json"), "{ obcieta kop"); + + var catalog = new VehicleCatalog(installation.MiniTextures); + int liveries = installation.VehicleCache.Load(catalog); + + Assert.Equal(1, liveries); + Assert.Contains("kopia nie do odczytu", installation.Log.Text); + } + + [Fact] + public void The_copy_reads_the_same_catalogue_the_legacy_reader_does() + { + var files = new InMemoryFileSystem() + .WithLegacyFile(Dynamic("pkp", "ep07"), Ep07) + .WithLegacyFile(Dynamic("pkp", "en57"), En57); + var installation = new TestInstallation(files); + + var direct = new VehicleCatalog(installation.MiniTextures); + installation.Vehicles.Load(direct); + var cached = new VehicleCatalog(installation.MiniTextures); + installation.VehicleCache.Load(cached); + + Assert.Equal(direct.Textures.Select(t => t.Uuid), cached.Textures.Select(t => t.Uuid)); + Assert.Equal(direct.GroupsById.Keys.Order(), cached.GroupsById.Keys.Order()); + } +} diff --git a/StarterNG/Application/Abstractions/IFileSystem.cs b/StarterNG/Application/Abstractions/IFileSystem.cs index c49c726..b30f657 100644 --- a/StarterNG/Application/Abstractions/IFileSystem.cs +++ b/StarterNG/Application/Abstractions/IFileSystem.cs @@ -43,5 +43,7 @@ public interface IFileSystem DateTime GetLastWriteTimeUtc(string path); + long GetFileSize(string path); + bool IsExecutable(string path); } diff --git a/StarterNG/Application/AppServices.cs b/StarterNG/Application/AppServices.cs index e223746..2df8944 100644 --- a/StarterNG/Application/AppServices.cs +++ b/StarterNG/Application/AppServices.cs @@ -31,7 +31,16 @@ public sealed class AppServices Sceneries = new SceneryRepository(files, paths, log, new SceneryParser(clock, Random)); SceneryImages = new SceneryImageLocator(files); MiniTextures = new MiniTextureIndex(files, paths); - Vehicles = new TexturesTxtVehicleRepository(files, paths, log, new TexturesTxtParser()); + var settingsPaths = new SettingsPaths(environment, paths); + SettingsPaths = settingsPaths; + + // textures.txt stays the truth; the starter reads its own JSON copy of it + // and rebuilds that copy whenever the installation has moved on. + TexturesTxt = new TexturesTxtVehicleRepository(files, paths, log, new TexturesTxtParser()); + VehicleJson = new VehicleJsonSerializer(); + VehicleCache = new VehicleCacheRepository(files, paths, log, TexturesTxt, VehicleJson, + settingsPaths.VehicleCacheDirectory()); + Vehicles = VehicleCache; Physics = new FizPhysicsRepository(files, paths, log); MissingAssets = new MissingAssetScanner(files, paths); SceneryTexts = new SceneryTranslations(files, log); @@ -40,10 +49,8 @@ public sealed class AppServices Library = new GameLibrary(Vehicles, Sceneries, MiniTextures, Physics, log); State = new AppState(); - var settingsPaths = new SettingsPaths(environment, paths); Executables = new ExecutableLocator(files, paths, environment, log); SettingsStore = new SettingsStore(files, clock, log, settingsPaths, new SettingsSerializer(), Executables); - SettingsPaths = settingsPaths; MissingVehicleLog = new MissingVehicleLog(SettingsStore.Settings, MissingAssets, Library, log); StartSimulation = new StartSimulation(State, SettingsStore, files, Processes, Random, log); } @@ -85,6 +92,14 @@ public sealed class AppServices public IVehicleRepository Vehicles { get; } + /// The same object as , typed for the developer tab. + public VehicleCacheRepository VehicleCache { get; } + + /// The legacy reader behind the cache, for tools that go to the source. + public TexturesTxtVehicleRepository TexturesTxt { get; } + + public VehicleJsonSerializer VehicleJson { get; } + public IPhysicsRepository Physics { get; } public MissingAssetScanner MissingAssets { get; } diff --git a/StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs b/StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs index fb64b2b..69e0c0f 100644 --- a/StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs +++ b/StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs @@ -51,6 +51,8 @@ public sealed class PhysicalFileSystem : IFileSystem public DateTime GetLastWriteTimeUtc(string path) => File.GetLastWriteTimeUtc(path); + public long GetFileSize(string path) => new FileInfo(path).Length; + public bool IsExecutable(string path) { if (OperatingSystem.IsWindows()) diff --git a/StarterNG/Infrastructure/Settings/SettingsPaths.cs b/StarterNG/Infrastructure/Settings/SettingsPaths.cs index d9eba18..75639c2 100644 --- a/StarterNG/Infrastructure/Settings/SettingsPaths.cs +++ b/StarterNG/Infrastructure/Settings/SettingsPaths.cs @@ -43,6 +43,13 @@ public sealed class SettingsPaths public string InstallationConfigPath() => _paths.FromRoot(ConfigFileName); + /// + /// Where the starter keeps its own copy of the vehicle database. Per user + /// rather than under the installation: that one can be read-only, is a working + /// copy someone updates from under us, and the copy is ours, not the game's. + /// + public string VehicleCacheDirectory() => Path.Combine(UserConfigDirectory(), "vehicles"); + public string UserConfigDirectory() => Path.GetDirectoryName(UserConfigPath()) ?? _environment.BaseDirectory; } diff --git a/StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs b/StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs index 71e5f2f..986a55c 100644 --- a/StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs +++ b/StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using StarterNG.Application.Abstractions; using StarterNG.Domain.Vehicles; @@ -31,9 +32,9 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository catalog.BeginLoad(); int liveries = 0; - foreach (string file in TexturesFiles()) + foreach (string file in Sources()) { - var entry = ReadEntry(file); + var entry = Read(file); if (entry is null) continue; @@ -45,7 +46,27 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository return liveries; } - private VehicleEntry? ReadEntry(string path) + /// + /// Every textures.txt of the installation, in name order. Sorted rather than + /// taken as the file system hands them over, so that which of two folders + /// claiming the same skin wins is the same on every machine. + /// + public IEnumerable Sources() + { + foreach (string maker in Sorted(_files.GetDirectories(_paths.Dynamic))) + foreach (string vehicle in Sorted(_files.GetDirectories(maker))) + { + string path = Path.Combine(vehicle, TexturesFileName); + if (_files.FileExists(path)) + yield return path; + } + } + + private static IEnumerable Sorted(IEnumerable directories) => + directories.OrderBy(Path.GetFileName, StringComparer.OrdinalIgnoreCase); + + /// Reads one textures.txt; a file that will not parse is logged and skipped. + public VehicleEntry? Read(string path) { try { @@ -59,17 +80,6 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository } } - private IEnumerable TexturesFiles() - { - foreach (string maker in _files.GetDirectories(_paths.Dynamic)) - foreach (string vehicle in _files.GetDirectories(maker)) - { - string path = Path.Combine(vehicle, TexturesFileName); - if (_files.FileExists(path)) - yield return path; - } - } - private string RelativeDirectory(string texturesPath) { string directory = Path.GetDirectoryName(texturesPath) ?? ""; diff --git a/StarterNG/Infrastructure/Vehicles/VehicleCacheRepository.cs b/StarterNG/Infrastructure/Vehicles/VehicleCacheRepository.cs new file mode 100644 index 0000000..5befa5a --- /dev/null +++ b/StarterNG/Infrastructure/Vehicles/VehicleCacheRepository.cs @@ -0,0 +1,189 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using StarterNG.Application.Abstractions; +using StarterNG.Domain.Vehicles; + +namespace StarterNG.Infrastructure.Vehicles; + +/// +/// Loads the catalogue from the starter's own JSON copy of the installation, +/// rebuilding that copy from textures.txt whenever the installation has moved on. +/// +/// +/// The copy is a plain merged database, the same format a package is written in, +/// so it can be read by anything that reads the format - and it is a copy, never +/// a dependency: a missing, stale, unreadable or unwritable cache only costs the +/// parse it was meant to save. textures.txt stays the truth. +/// +public sealed class VehicleCacheRepository : IVehicleRepository +{ + private const string DatabaseFileName = "vehicles.json"; + + /// What the copy was built from, and what makes it stale. + private const string SourcesFileName = "sources.txt"; + + private readonly IFileSystem _files; + private readonly IGamePaths _paths; + private readonly IDiagnosticsLog _log; + private readonly TexturesTxtVehicleRepository _legacy; + private readonly VehicleJsonSerializer _serializer; + private readonly string _directory; + + public VehicleCacheRepository(IFileSystem files, IGamePaths paths, IDiagnosticsLog log, + TexturesTxtVehicleRepository legacy, VehicleJsonSerializer serializer, + string directory) + { + _files = files; + _paths = paths; + _log = log; + _legacy = legacy; + _serializer = serializer; + _directory = directory; + } + + /// What the last load did, for the developer tab to report. + public VehicleCacheStatus Status { get; private set; } = VehicleCacheStatus.None; + + public int Load(VehicleCatalog catalog) => Load(catalog, useCopy: true); + + /// Throws the copy away and reads the installation again. + public int Rebuild(VehicleCatalog catalog) => Load(catalog, useCopy: false); + + private int Load(VehicleCatalog catalog, bool useCopy) + { + var (stamp, sources) = Stamp(); + var cached = useCopy ? Cached(stamp) : null; + var entries = cached ?? Migrate(stamp); + + Status = new VehicleCacheStatus(cached is not null, sources, WrittenAt(), _directory); + + catalog.BeginLoad(); + + int liveries = 0; + foreach (var entry in entries) + { + liveries += entry.Textures.Count; + catalog.Ingest(entry); + } + + catalog.EndLoad(); + return liveries; + } + + /// + /// The installation as one comparable line per textures.txt. Every way the + /// installation can move on - a file edited, added, removed, or the whole + /// installation swapped for another one - shows up as a different text. + /// + private (string Text, int Sources) Stamp() + { + var stamp = new StringBuilder().Append("root|").Append(Path.GetFullPath(_paths.Root)).Append('\n'); + int sources = 0; + + foreach (string path in _legacy.Sources()) + { + sources++; + long ticks = 0; + long size = 0; + try + { + ticks = _files.GetLastWriteTimeUtc(path).Ticks; + size = _files.GetFileSize(path); + } + catch (Exception ex) + { + _log.Log($"textures.txt {path}", ex); + } + + stamp.Append(ticks).Append('|').Append(size).Append('|') + .Append(path.Replace('\\', '/')).Append('\n'); + } + + return (stamp.ToString(), sources); + } + + private DateTime? WrittenAt() + { + try + { + return _files.FileExists(DatabasePath) ? _files.GetLastWriteTimeUtc(DatabasePath) : null; + } + catch (Exception ex) + { + _log.Log($"{DatabaseFileName}: nie znam daty kopii", ex); + return null; + } + } + + private List? Cached(string stamp) + { + try + { + if (!_files.FileExists(SourcesPath) || !_files.FileExists(DatabasePath)) + return null; + + if (!string.Equals(_files.ReadAllText(SourcesPath), stamp, StringComparison.Ordinal)) + return null; + + var entries = _serializer.ReadCollection(_files.ReadAllText(DatabasePath)); + foreach (var entry in entries) + if (!VehicleJsonSerializer.IsReadable(entry)) + { + _log.Log($"{DatabaseFileName}: kopia w wersji {entry.SchemaVersion}, " + + $"ten starter czyta do {VehicleJsonSerializer.SchemaVersion} - odtwarzam"); + return null; + } + + return entries; + } + catch (Exception ex) + { + _log.Log($"{DatabaseFileName}: kopia nie do odczytu, odtwarzam", ex); + return null; + } + } + + private List Migrate(string stamp) + { + var entries = new List(); + foreach (string path in _legacy.Sources()) + if (_legacy.Read(path) is { } entry) + entries.Add(entry); + + Store(entries, stamp); + return entries; + } + + /// + /// Writes the copy out. Best effort by design - an installation on a read-only + /// disk, or a home folder we cannot write, still gets its vehicles. + /// + private void Store(List entries, string stamp) + { + try + { + _files.CreateDirectory(_directory); + _files.WriteAllText(DatabasePath, _serializer.WriteCollection(entries)); + _files.WriteAllText(SourcesPath, stamp); + } + catch (Exception ex) + { + _log.Log($"{DatabaseFileName}: kopia nie do zapisania", ex); + } + } + + private string DatabasePath => Path.Combine(_directory, DatabaseFileName); + + private string SourcesPath => Path.Combine(_directory, SourcesFileName); +} + +/// +/// How the catalogue was last filled: from the starter's copy or by reading the +/// installation, over how many textures.txt, and when the copy was last written. +/// +public sealed record VehicleCacheStatus(bool ServedFromCopy, int Sources, DateTime? Written, string Directory) +{ + public static readonly VehicleCacheStatus None = new(false, 0, null, ""); +}