Load vehicles from a JSON copy of textures.txt, rebuilt when the installation changes

StarterNG.Tests/Fakes/InMemoryFileSystem.cs

@@ -134,6 +134,8 @@ public sealed class InMemoryFileSystem : IFileSystem
     public DateTime GetLastWriteTimeUtc(string path) => LastWriteTimeUtc;
 
     /// <summary>Files added through <see cref="WithExecutable"/> carry the execute bit.</summary>
+    public long GetFileSize(string path) => Bytes(path).LongLength;
+
     public bool IsExecutable(string path) => _executable.Contains(Normalize(path));
 
     public InMemoryFileSystem WithExecutable(string path, string contents = "\u007fELF")

StarterNG.Tests/Fakes/TestInstallation.cs

@@ -26,12 +26,15 @@ public sealed class TestInstallation
         MiniTextures = new MiniTextureIndex(Files, Paths);
         Physics = new FizPhysicsRepository(Files, Paths, Log);
         Vehicles = new TexturesTxtVehicleRepository(Files, Paths, Log, new TexturesTxtParser());
+        VehicleJson = new VehicleJsonSerializer();
         Sceneries = new SceneryRepository(Files, Paths, Log, new SceneryParser(Clock, Random));
         MissingAssets = new MissingAssetScanner(Files, Paths);
-        Library = new GameLibrary(Vehicles, Sceneries, MiniTextures, Physics, Log);
-
         Environment = new FakeEnvironment();
         SettingsPaths = new SettingsPaths(Environment, Paths);
+        VehicleCache = new VehicleCacheRepository(Files, Paths, Log, Vehicles, VehicleJson,
+                                                  SettingsPaths.VehicleCacheDirectory());
+        Library = new GameLibrary(VehicleCache, Sceneries, MiniTextures, Physics, Log);
+
         Executables = new ExecutableLocator(Files, Paths, Environment, Log);
         SettingsStore = new SettingsStore(Files, Clock, Log, SettingsPaths, new SettingsSerializer(), Executables);
     }
@@ -45,6 +48,8 @@ public sealed class TestInstallation
     public MiniTextureIndex MiniTextures { get; }
     public FizPhysicsRepository Physics { get; }
     public TexturesTxtVehicleRepository Vehicles { get; }
+    public VehicleJsonSerializer VehicleJson { get; }
+    public VehicleCacheRepository VehicleCache { get; }
     public SceneryRepository Sceneries { get; }
     public MissingAssetScanner MissingAssets { get; }
     public GameLibrary Library { get; }

StarterNG.Tests/VehicleCacheTests.cs

@@ -0,0 +1,111 @@
+using System.Text;
+using StarterNG.Domain.Vehicles;
+using StarterNG.Tests.Fakes;
+
+namespace StarterNG.Tests;
+
+public class VehicleCacheTests
+{
+    private const string Ep07 = "ep07-135.mat=ep07.e3d,ep07";
+
+    private const string En57 = "en57-1000ra.mat=en57.e3d,en57";
+
+    private static string Dynamic(string maker, string vehicle) =>
+        TestInstallation.At("dynamic", maker, vehicle, "textures.txt");
+
+    private static string Cached(TestInstallation installation, string file) =>
+        Path.Combine(installation.SettingsPaths.VehicleCacheDirectory(), file);
+
+    [Fact]
+    public void The_first_load_migrates_textures_txt_into_the_starters_own_copy()
+    {
+        var files = new InMemoryFileSystem().WithLegacyFile(Dynamic("pkp", "ep07"), Ep07);
+        var installation = new TestInstallation(files);
+
+        int liveries = installation.VehicleCache.Load(installation.Library.Vehicles);
+
+        Assert.Equal(1, liveries);
+        Assert.True(files.FileExists(Cached(installation, "vehicles.json")));
+        Assert.Contains("ep07-135", files.ReadAllText(Cached(installation, "vehicles.json")));
+    }
+
+    [Fact]
+    public void An_untouched_installation_is_served_from_the_copy_without_reading_textures_txt()
+    {
+        var files = new InMemoryFileSystem().WithLegacyFile(Dynamic("pkp", "ep07"), Ep07);
+        var installation = new TestInstallation(files);
+        installation.VehicleCache.Load(installation.Library.Vehicles);
+
+        // Different liveries, same length and same timestamp: nothing the stamp
+        // can see has changed, so an answer of ep07-135 can only come from the copy.
+        files.WithLegacyFile(Dynamic("pkp", "ep07"), "ep07-999.mat=ep07.e3d,ep07");
+        var catalog = new VehicleCatalog(installation.MiniTextures);
+        installation.VehicleCache.Load(catalog);
+
+        Assert.Equal("legacy:pkp/ep07/ep07-135", Assert.Single(catalog.Textures).Uuid);
+    }
+
+    [Fact]
+    public void An_edited_textures_txt_makes_the_copy_stale_and_it_is_rebuilt()
+    {
+        var files = new InMemoryFileSystem().WithLegacyFile(Dynamic("pkp", "ep07"), Ep07);
+        var installation = new TestInstallation(files);
+        installation.VehicleCache.Load(installation.Library.Vehicles);
+
+        files.WithLegacyFile(Dynamic("pkp", "ep07"), Ep07 + "\nep07-174.mat=ep07.e3d,ep07");
+        var catalog = new VehicleCatalog(installation.MiniTextures);
+        int liveries = installation.VehicleCache.Load(catalog);
+
+        Assert.Equal(2, liveries);
+        Assert.Contains("ep07-174", files.ReadAllText(Cached(installation, "vehicles.json")));
+    }
+
+    [Fact]
+    public void A_vehicle_folder_that_disappeared_upstream_leaves_the_copy_too()
+    {
+        var files = new InMemoryFileSystem()
+            .WithLegacyFile(Dynamic("pkp", "ep07"), Ep07)
+            .WithLegacyFile(Dynamic("pkp", "en57"), En57);
+        var installation = new TestInstallation(files);
+        installation.VehicleCache.Load(installation.Library.Vehicles);
+
+        files.DeleteFile(Dynamic("pkp", "en57"));
+        var catalog = new VehicleCatalog(installation.MiniTextures);
+        installation.VehicleCache.Load(catalog);
+
+        Assert.Equal("legacy:pkp/ep07/ep07-135", Assert.Single(catalog.Textures).Uuid);
+        Assert.DoesNotContain("en57", files.ReadAllText(Cached(installation, "vehicles.json")));
+    }
+
+    [Fact]
+    public void A_damaged_copy_costs_the_parse_it_saved_and_nothing_else()
+    {
+        var files = new InMemoryFileSystem().WithLegacyFile(Dynamic("pkp", "ep07"), Ep07);
+        var installation = new TestInstallation(files);
+        installation.VehicleCache.Load(installation.Library.Vehicles);
+        files.WithFile(Cached(installation, "vehicles.json"), "{ obcieta kop");
+
+        var catalog = new VehicleCatalog(installation.MiniTextures);
+        int liveries = installation.VehicleCache.Load(catalog);
+
+        Assert.Equal(1, liveries);
+        Assert.Contains("kopia nie do odczytu", installation.Log.Text);
+    }
+
+    [Fact]
+    public void The_copy_reads_the_same_catalogue_the_legacy_reader_does()
+    {
+        var files = new InMemoryFileSystem()
+            .WithLegacyFile(Dynamic("pkp", "ep07"), Ep07)
+            .WithLegacyFile(Dynamic("pkp", "en57"), En57);
+        var installation = new TestInstallation(files);
+
+        var direct = new VehicleCatalog(installation.MiniTextures);
+        installation.Vehicles.Load(direct);
+        var cached = new VehicleCatalog(installation.MiniTextures);
+        installation.VehicleCache.Load(cached);
+
+        Assert.Equal(direct.Textures.Select(t => t.Uuid), cached.Textures.Select(t => t.Uuid));
+        Assert.Equal(direct.GroupsById.Keys.Order(), cached.GroupsById.Keys.Order());
+    }
+}

StarterNG/Application/Abstractions/IFileSystem.cs

@@ -43,5 +43,7 @@ public interface IFileSystem
 
     DateTime GetLastWriteTimeUtc(string path);
 
+    long GetFileSize(string path);
+
     bool IsExecutable(string path);
 }

StarterNG/Application/AppServices.cs

@@ -31,7 +31,16 @@ public sealed class AppServices
         Sceneries = new SceneryRepository(files, paths, log, new SceneryParser(clock, Random));
         SceneryImages = new SceneryImageLocator(files);
         MiniTextures = new MiniTextureIndex(files, paths);
-        Vehicles = new TexturesTxtVehicleRepository(files, paths, log, new TexturesTxtParser());
+        var settingsPaths = new SettingsPaths(environment, paths);
+        SettingsPaths = settingsPaths;
+
+        // textures.txt stays the truth; the starter reads its own JSON copy of it
+        // and rebuilds that copy whenever the installation has moved on.
+        TexturesTxt = new TexturesTxtVehicleRepository(files, paths, log, new TexturesTxtParser());
+        VehicleJson = new VehicleJsonSerializer();
+        VehicleCache = new VehicleCacheRepository(files, paths, log, TexturesTxt, VehicleJson,
+                                                  settingsPaths.VehicleCacheDirectory());
+        Vehicles = VehicleCache;
         Physics = new FizPhysicsRepository(files, paths, log);
         MissingAssets = new MissingAssetScanner(files, paths);
         SceneryTexts = new SceneryTranslations(files, log);
@@ -40,10 +49,8 @@ public sealed class AppServices
         Library = new GameLibrary(Vehicles, Sceneries, MiniTextures, Physics, log);
         State = new AppState();
 
-        var settingsPaths = new SettingsPaths(environment, paths);
         Executables = new ExecutableLocator(files, paths, environment, log);
         SettingsStore = new SettingsStore(files, clock, log, settingsPaths, new SettingsSerializer(), Executables);
-        SettingsPaths = settingsPaths;
         MissingVehicleLog = new MissingVehicleLog(SettingsStore.Settings, MissingAssets, Library, log);
         StartSimulation = new StartSimulation(State, SettingsStore, files, Processes, Random, log);
     }
@@ -85,6 +92,14 @@ public sealed class AppServices
 
     public IVehicleRepository Vehicles { get; }
 
+    /// <summary>The same object as <see cref="Vehicles"/>, typed for the developer tab.</summary>
+    public VehicleCacheRepository VehicleCache { get; }
+
+    /// <summary>The legacy reader behind the cache, for tools that go to the source.</summary>
+    public TexturesTxtVehicleRepository TexturesTxt { get; }
+
+    public VehicleJsonSerializer VehicleJson { get; }
+
     public IPhysicsRepository Physics { get; }
 
     public MissingAssetScanner MissingAssets { get; }

StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs

@@ -51,6 +51,8 @@ public sealed class PhysicalFileSystem : IFileSystem
 
     public DateTime GetLastWriteTimeUtc(string path) => File.GetLastWriteTimeUtc(path);
 
+    public long GetFileSize(string path) => new FileInfo(path).Length;
+
     public bool IsExecutable(string path)
     {
         if (OperatingSystem.IsWindows())

StarterNG/Infrastructure/Settings/SettingsPaths.cs

@@ -43,6 +43,13 @@ public sealed class SettingsPaths
 
     public string InstallationConfigPath() => _paths.FromRoot(ConfigFileName);
 
+    /// <summary>
+    /// Where the starter keeps its own copy of the vehicle database. Per user
+    /// rather than under the installation: that one can be read-only, is a working
+    /// copy someone updates from under us, and the copy is ours, not the game's.
+    /// </summary>
+    public string VehicleCacheDirectory() => Path.Combine(UserConfigDirectory(), "vehicles");
+
     public string UserConfigDirectory() =>
         Path.GetDirectoryName(UserConfigPath()) ?? _environment.BaseDirectory;
 }

StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text;
 using StarterNG.Application.Abstractions;
 using StarterNG.Domain.Vehicles;
@@ -31,9 +32,9 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository
         catalog.BeginLoad();
 
         int liveries = 0;
-        foreach (string file in TexturesFiles())
+        foreach (string file in Sources())
         {
-            var entry = ReadEntry(file);
+            var entry = Read(file);
             if (entry is null)
                 continue;
 
@@ -45,7 +46,27 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository
         return liveries;
     }
 
-    private VehicleEntry? ReadEntry(string path)
+    /// <summary>
+    /// Every textures.txt of the installation, in name order. Sorted rather than
+    /// taken as the file system hands them over, so that which of two folders
+    /// claiming the same skin wins is the same on every machine.
+    /// </summary>
+    public IEnumerable<string> Sources()
+    {
+        foreach (string maker in Sorted(_files.GetDirectories(_paths.Dynamic)))
+        foreach (string vehicle in Sorted(_files.GetDirectories(maker)))
+        {
+            string path = Path.Combine(vehicle, TexturesFileName);
+            if (_files.FileExists(path))
+                yield return path;
+        }
+    }
+
+    private static IEnumerable<string> Sorted(IEnumerable<string> directories) =>
+        directories.OrderBy(Path.GetFileName, StringComparer.OrdinalIgnoreCase);
+
+    /// <summary>Reads one textures.txt; a file that will not parse is logged and skipped.</summary>
+    public VehicleEntry? Read(string path)
     {
         try
         {
@@ -59,17 +80,6 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository
         }
     }
 
-    private IEnumerable<string> TexturesFiles()
-    {
-        foreach (string maker in _files.GetDirectories(_paths.Dynamic))
-        foreach (string vehicle in _files.GetDirectories(maker))
-        {
-            string path = Path.Combine(vehicle, TexturesFileName);
-            if (_files.FileExists(path))
-                yield return path;
-        }
-    }
-
     private string RelativeDirectory(string texturesPath)
     {
         string directory = Path.GetDirectoryName(texturesPath) ?? "";

StarterNG/Infrastructure/Vehicles/VehicleCacheRepository.cs

@@ -0,0 +1,189 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using StarterNG.Application.Abstractions;
+using StarterNG.Domain.Vehicles;
+
+namespace StarterNG.Infrastructure.Vehicles;
+
+/// <summary>
+/// Loads the catalogue from the starter's own JSON copy of the installation,
+/// rebuilding that copy from textures.txt whenever the installation has moved on.
+/// </summary>
+/// <remarks>
+/// The copy is a plain merged database, the same format a package is written in,
+/// so it can be read by anything that reads the format - and it is a copy, never
+/// a dependency: a missing, stale, unreadable or unwritable cache only costs the
+/// parse it was meant to save. textures.txt stays the truth.
+/// </remarks>
+public sealed class VehicleCacheRepository : IVehicleRepository
+{
+    private const string DatabaseFileName = "vehicles.json";
+
+    /// <summary>What the copy was built from, and what makes it stale.</summary>
+    private const string SourcesFileName = "sources.txt";
+
+    private readonly IFileSystem _files;
+    private readonly IGamePaths _paths;
+    private readonly IDiagnosticsLog _log;
+    private readonly TexturesTxtVehicleRepository _legacy;
+    private readonly VehicleJsonSerializer _serializer;
+    private readonly string _directory;
+
+    public VehicleCacheRepository(IFileSystem files, IGamePaths paths, IDiagnosticsLog log,
+                                  TexturesTxtVehicleRepository legacy, VehicleJsonSerializer serializer,
+                                  string directory)
+    {
+        _files = files;
+        _paths = paths;
+        _log = log;
+        _legacy = legacy;
+        _serializer = serializer;
+        _directory = directory;
+    }
+
+    /// <summary>What the last load did, for the developer tab to report.</summary>
+    public VehicleCacheStatus Status { get; private set; } = VehicleCacheStatus.None;
+
+    public int Load(VehicleCatalog catalog) => Load(catalog, useCopy: true);
+
+    /// <summary>Throws the copy away and reads the installation again.</summary>
+    public int Rebuild(VehicleCatalog catalog) => Load(catalog, useCopy: false);
+
+    private int Load(VehicleCatalog catalog, bool useCopy)
+    {
+        var (stamp, sources) = Stamp();
+        var cached = useCopy ? Cached(stamp) : null;
+        var entries = cached ?? Migrate(stamp);
+
+        Status = new VehicleCacheStatus(cached is not null, sources, WrittenAt(), _directory);
+
+        catalog.BeginLoad();
+
+        int liveries = 0;
+        foreach (var entry in entries)
+        {
+            liveries += entry.Textures.Count;
+            catalog.Ingest(entry);
+        }
+
+        catalog.EndLoad();
+        return liveries;
+    }
+
+    /// <summary>
+    /// The installation as one comparable line per textures.txt. Every way the
+    /// installation can move on - a file edited, added, removed, or the whole
+    /// installation swapped for another one - shows up as a different text.
+    /// </summary>
+    private (string Text, int Sources) Stamp()
+    {
+        var stamp = new StringBuilder().Append("root|").Append(Path.GetFullPath(_paths.Root)).Append('\n');
+        int sources = 0;
+
+        foreach (string path in _legacy.Sources())
+        {
+            sources++;
+            long ticks = 0;
+            long size = 0;
+            try
+            {
+                ticks = _files.GetLastWriteTimeUtc(path).Ticks;
+                size = _files.GetFileSize(path);
+            }
+            catch (Exception ex)
+            {
+                _log.Log($"textures.txt {path}", ex);
+            }
+
+            stamp.Append(ticks).Append('|').Append(size).Append('|')
+                 .Append(path.Replace('\\', '/')).Append('\n');
+        }
+
+        return (stamp.ToString(), sources);
+    }
+
+    private DateTime? WrittenAt()
+    {
+        try
+        {
+            return _files.FileExists(DatabasePath) ? _files.GetLastWriteTimeUtc(DatabasePath) : null;
+        }
+        catch (Exception ex)
+        {
+            _log.Log($"{DatabaseFileName}: nie znam daty kopii", ex);
+            return null;
+        }
+    }
+
+    private List<VehicleEntry>? Cached(string stamp)
+    {
+        try
+        {
+            if (!_files.FileExists(SourcesPath) || !_files.FileExists(DatabasePath))
+                return null;
+
+            if (!string.Equals(_files.ReadAllText(SourcesPath), stamp, StringComparison.Ordinal))
+                return null;
+
+            var entries = _serializer.ReadCollection(_files.ReadAllText(DatabasePath));
+            foreach (var entry in entries)
+                if (!VehicleJsonSerializer.IsReadable(entry))
+                {
+                    _log.Log($"{DatabaseFileName}: kopia w wersji {entry.SchemaVersion}, " +
+                             $"ten starter czyta do {VehicleJsonSerializer.SchemaVersion} - odtwarzam");
+                    return null;
+                }
+
+            return entries;
+        }
+        catch (Exception ex)
+        {
+            _log.Log($"{DatabaseFileName}: kopia nie do odczytu, odtwarzam", ex);
+            return null;
+        }
+    }
+
+    private List<VehicleEntry> Migrate(string stamp)
+    {
+        var entries = new List<VehicleEntry>();
+        foreach (string path in _legacy.Sources())
+            if (_legacy.Read(path) is { } entry)
+                entries.Add(entry);
+
+        Store(entries, stamp);
+        return entries;
+    }
+
+    /// <summary>
+    /// Writes the copy out. Best effort by design - an installation on a read-only
+    /// disk, or a home folder we cannot write, still gets its vehicles.
+    /// </summary>
+    private void Store(List<VehicleEntry> entries, string stamp)
+    {
+        try
+        {
+            _files.CreateDirectory(_directory);
+            _files.WriteAllText(DatabasePath, _serializer.WriteCollection(entries));
+            _files.WriteAllText(SourcesPath, stamp);
+        }
+        catch (Exception ex)
+        {
+            _log.Log($"{DatabaseFileName}: kopia nie do zapisania", ex);
+        }
+    }
+
+    private string DatabasePath => Path.Combine(_directory, DatabaseFileName);
+
+    private string SourcesPath => Path.Combine(_directory, SourcesFileName);
+}
+
+/// <summary>
+/// How the catalogue was last filled: from the starter's copy or by reading the
+/// installation, over how many textures.txt, and when the copy was last written.
+/// </summary>
+public sealed record VehicleCacheStatus(bool ServedFromCopy, int Sources, DateTime? Written, string Directory)
+{
+    public static readonly VehicleCacheStatus None = new(false, 0, null, "");
+}