From c492c260470bc9d51b2dc772f7bce6d0160f9c42 Mon Sep 17 00:00:00 2001 From: maj00r Date: Fri, 4 Sep 2026 23:50:41 +0200 Subject: [PATCH 01/55] fallback decode utf8 strict to win 1250 --- StarterNG.Tests/Fakes/InMemoryFileSystem.cs | 2 + StarterNG.Tests/LegacyDecodingTests.cs | 55 +++++++++++++++++++ .../Application/Abstractions/IFileSystem.cs | 2 + .../Infrastructure/Adapters/LegacyText.cs | 21 +++++++ .../Adapters/PhysicalFileSystem.cs | 2 + .../Sceneries/SceneryRepository.cs | 4 +- .../Vehicles/FizPhysicsRepository.cs | 4 +- .../Vehicles/LoadWeightsRepository.cs | 2 +- .../Vehicles/TexturesTxtVehicleRepository.cs | 4 +- 9 files changed, 86 insertions(+), 10 deletions(-) create mode 100644 StarterNG.Tests/LegacyDecodingTests.cs diff --git a/StarterNG.Tests/Fakes/InMemoryFileSystem.cs b/StarterNG.Tests/Fakes/InMemoryFileSystem.cs index 2dddd26..1a12a86 100644 --- a/StarterNG.Tests/Fakes/InMemoryFileSystem.cs +++ b/StarterNG.Tests/Fakes/InMemoryFileSystem.cs @@ -63,6 +63,8 @@ public sealed class InMemoryFileSystem : IFileSystem public string[] ReadAllLines(string path) => ReadAllText(path).Split('\n').Select(line => line.TrimEnd('\r')).ToArray(); + public byte[] ReadAllBytes(string path) => Bytes(path); + public Stream OpenRead(string path) => new MemoryStream(Bytes(path), writable: false); public void WriteAllText(string path, string contents) => WithFile(path, contents); diff --git a/StarterNG.Tests/LegacyDecodingTests.cs b/StarterNG.Tests/LegacyDecodingTests.cs new file mode 100644 index 0000000..4325bf5 --- /dev/null +++ b/StarterNG.Tests/LegacyDecodingTests.cs @@ -0,0 +1,55 @@ +using System.Text; +using StarterNG.Infrastructure.Adapters; +using StarterNG.Tests.Fakes; + +namespace StarterNG.Tests; + +public class LegacyDecodingTests +{ + private const string Polish = "Zakład_Świętokrzyski_w_Kielcach"; + + [Fact] + public void A_code_page_1250_file_decodes_to_its_letters() + { + Assert.Equal(Polish, LegacyText.Decode(LegacyText.CodePage1250.GetBytes(Polish))); + } + + [Fact] + public void A_file_re_saved_as_utf8_decodes_too() + { + // A handful of textures.txt in the wild are UTF-8; read as 1250 they turn + // "Zakład" into "ZakĹ‚ad". + Assert.Equal(Polish, LegacyText.Decode(Encoding.UTF8.GetBytes(Polish))); + } + + [Fact] + public void Plain_ascii_reads_the_same_either_way() + { + Assert.Equal("PESA_Bydgoszcz", LegacyText.Decode(Encoding.ASCII.GetBytes("PESA_Bydgoszcz"))); + } + + [Fact] + public void A_utf8_textures_file_reaches_the_catalogue_intact() + { + var files = new InMemoryFileSystem().WithFile( + TestInstallation.At("dynamic", "pkp", "elf", "textures.txt"), + "en96-001.mat=34we,EN96,EN96-001 // v2,EN96-001,PREG,Zakład_Świętokrzyski,29.09.2011"); + var installation = new TestInstallation(files); + + installation.Vehicles.Load(installation.Library.Vehicles); + + var meta = Assert.Single(installation.Library.Vehicles.Textures).Meta; + Assert.Equal("Zakład_Świętokrzyski".Replace('_', ' '), meta!.Depot); + } + + [Fact] + public void A_code_page_1250_scenery_reaches_the_list_intact() + { + var files = new InMemoryFileSystem().WithLegacyFile( + TestInstallation.At("scenery", "td.scn"), "//$n Strzebiń - Woźniki Śl."); + + var scenery = Assert.Single(new TestInstallation(files).Sceneries.LoadAll()); + + Assert.Equal("Strzebiń - Woźniki Śl.", scenery.Name); + } +} diff --git a/StarterNG/Application/Abstractions/IFileSystem.cs b/StarterNG/Application/Abstractions/IFileSystem.cs index 93c1083..c49c726 100644 --- a/StarterNG/Application/Abstractions/IFileSystem.cs +++ b/StarterNG/Application/Abstractions/IFileSystem.cs @@ -17,6 +17,8 @@ public interface IFileSystem string[] ReadAllLines(string path); + byte[] ReadAllBytes(string path); + Stream OpenRead(string path); void WriteAllText(string path, string contents); diff --git a/StarterNG/Infrastructure/Adapters/LegacyText.cs b/StarterNG/Infrastructure/Adapters/LegacyText.cs index 991b023..8997cfe 100644 --- a/StarterNG/Infrastructure/Adapters/LegacyText.cs +++ b/StarterNG/Infrastructure/Adapters/LegacyText.cs @@ -21,6 +21,27 @@ public static class LegacyText public static bool IsFallback { get; private set; } + /// + /// Decodes a game file. Most are code page 1250, but a few have been re-saved + /// as UTF-8 over the years, and reading those as 1250 turns "Zakład" into + /// "ZakĹ‚ad". Valid UTF-8 is taken at its word; anything that fails a strict + /// decode is 1250, which cannot fail. + /// + public static string Decode(byte[] bytes) + { + try + { + return Utf8Strict.GetString(bytes); + } + catch (DecoderFallbackException) + { + return CodePage1250.GetString(bytes); + } + } + + private static readonly Encoding Utf8Strict = + new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + private static Encoding Resolve() { try diff --git a/StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs b/StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs index 3c3effe..fb64b2b 100644 --- a/StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs +++ b/StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs @@ -18,6 +18,8 @@ public sealed class PhysicalFileSystem : IFileSystem public string[] ReadAllLines(string path) => File.ReadAllLines(path); + public byte[] ReadAllBytes(string path) => File.ReadAllBytes(path); + public Stream OpenRead(string path) => File.OpenRead(path); public void WriteAllText(string path, string contents) => File.WriteAllText(path, contents); diff --git a/StarterNG/Infrastructure/Sceneries/SceneryRepository.cs b/StarterNG/Infrastructure/Sceneries/SceneryRepository.cs index 5da5dee..cbbde38 100644 --- a/StarterNG/Infrastructure/Sceneries/SceneryRepository.cs +++ b/StarterNG/Infrastructure/Sceneries/SceneryRepository.cs @@ -20,7 +20,6 @@ public sealed class SceneryRepository : ISceneryRepository private readonly IGamePaths _paths; private readonly IDiagnosticsLog _log; private readonly SceneryParser _parser; - private readonly Encoding _encoding; public SceneryRepository(IFileSystem files, IGamePaths paths, IDiagnosticsLog log, SceneryParser parser) { @@ -28,7 +27,6 @@ public sealed class SceneryRepository : ISceneryRepository _paths = paths; _log = log; _parser = parser; - _encoding = LegacyText.CodePage1250; } public IReadOnlyList LoadAll(IProgress? progress = null) @@ -60,7 +58,7 @@ public sealed class SceneryRepository : ISceneryRepository if (!_files.FileExists(path)) return null; - var scenery = _parser.Parse(path, _files.ReadAllText(path, _encoding)); + var scenery = _parser.Parse(path, LegacyText.Decode(_files.ReadAllBytes(path))); scenery.HasCompanionTimetable = _files.FileExists(Path.ChangeExtension(path, ".sbt")); return scenery; } diff --git a/StarterNG/Infrastructure/Vehicles/FizPhysicsRepository.cs b/StarterNG/Infrastructure/Vehicles/FizPhysicsRepository.cs index 52f52eb..36a5a2c 100644 --- a/StarterNG/Infrastructure/Vehicles/FizPhysicsRepository.cs +++ b/StarterNG/Infrastructure/Vehicles/FizPhysicsRepository.cs @@ -25,7 +25,6 @@ public sealed class FizPhysicsRepository : IPhysicsRepository private readonly IFileSystem _files; private readonly IGamePaths _paths; private readonly IDiagnosticsLog _log; - private readonly Encoding _encoding; private readonly ConcurrentDictionary _cache = new(StringComparer.OrdinalIgnoreCase); @@ -38,7 +37,6 @@ public sealed class FizPhysicsRepository : IPhysicsRepository _files = files; _paths = paths; _log = log; - _encoding = LegacyText.CodePage1250; } public void Preload() => Index(); @@ -118,7 +116,7 @@ public sealed class FizPhysicsRepository : IPhysicsRepository if (depth > MaxIncludeDepth || !_files.FileExists(path)) return; - var tokens = Tokenize(_files.ReadAllText(path, _encoding)); + var tokens = Tokenize(LegacyText.Decode(_files.ReadAllBytes(path))); string section = ""; string? pendingKey = null; diff --git a/StarterNG/Infrastructure/Vehicles/LoadWeightsRepository.cs b/StarterNG/Infrastructure/Vehicles/LoadWeightsRepository.cs index 53a4f1f..dea430e 100644 --- a/StarterNG/Infrastructure/Vehicles/LoadWeightsRepository.cs +++ b/StarterNG/Infrastructure/Vehicles/LoadWeightsRepository.cs @@ -44,7 +44,7 @@ public sealed class LoadWeightsRepository try { - return new LoadWeightsTable(ParsePairs(_files.ReadAllText(path, LegacyText.CodePage1250))); + return new LoadWeightsTable(ParsePairs(LegacyText.Decode(_files.ReadAllBytes(path)))); } catch (Exception ex) { diff --git a/StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs b/StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs index 4a9a30d..71e5f2f 100644 --- a/StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs +++ b/StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs @@ -16,7 +16,6 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository private readonly IGamePaths _paths; private readonly IDiagnosticsLog _log; private readonly TexturesTxtParser _parser; - private readonly Encoding _encoding; public TexturesTxtVehicleRepository(IFileSystem files, IGamePaths paths, IDiagnosticsLog log, TexturesTxtParser parser) @@ -25,7 +24,6 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository _paths = paths; _log = log; _parser = parser; - _encoding = LegacyText.CodePage1250; } public int Load(VehicleCatalog catalog) @@ -51,7 +49,7 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository { try { - string[] lines = _files.ReadAllText(path, _encoding).Split('\n'); + string[] lines = LegacyText.Decode(_files.ReadAllBytes(path)).Split('\n'); return _parser.Parse(RelativeDirectory(path), lines); } catch (Exception ex) From 2545daeb82e07aff3f5d6c96985f8b03d9597c03 Mon Sep 17 00:00:00 2001 From: maj00r Date: Sun, 4 Oct 2026 12:18:36 +0200 Subject: [PATCH 02/55] Close the vehicle catalogue on release, not on the double-click press Closing from DoubleTapped left the pointer captured by a dead window, and on Windows the main window stopped taking clicks. --- StarterNG/Views/TextureBaseWindow.axaml.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/StarterNG/Views/TextureBaseWindow.axaml.cs b/StarterNG/Views/TextureBaseWindow.axaml.cs index b081ded..c697356 100644 --- a/StarterNG/Views/TextureBaseWindow.axaml.cs +++ b/StarterNG/Views/TextureBaseWindow.axaml.cs @@ -50,6 +50,7 @@ public partial class TextureBaseWindow : Window modelList.SelectionChanged += (_, _) => Refresh(); textureList.ContextRequested += TextureList_OnContextRequested; + textureList.AddHandler(PointerReleasedEvent, TextureList_OnPointerReleased, handledEventsToo: true); Refresh(); } @@ -133,7 +134,16 @@ public partial class TextureBaseWindow : Window { if (textureList.SelectedItem is not TextureRow row) return; Picked = row.Texture; - Close(); + } + + // DoubleTapped is raised from the second press, while the list still holds the + // pointer. Closing there leaves the capture on a dead window and on Windows the + // owner stops taking clicks. The release ends the gesture and drops the capture, + // so that is where the window goes. + private void TextureList_OnPointerReleased(object? sender, PointerReleasedEventArgs e) + { + if (Picked != null) + Close(); } private void TextureList_OnContextRequested(object? sender, ContextRequestedEventArgs e) From 3ddd00895f90d9eced45b44f25f50a2c81c54f80 Mon Sep 17 00:00:00 2001 From: maj00r Date: Sun, 4 Oct 2026 12:18:36 +0200 Subject: [PATCH 03/55] Tell UTF-8 from 1250 without throwing, and decode timetables the same way The strict decode threw on every 1250 file, hundreds of them per load. --- StarterNG.Tests/LegacyDecodingTests.cs | 35 +++++++++++++++++++ .../Infrastructure/Adapters/LegacyText.cs | 23 ++++++------ StarterNG/Views/Scenarios.axaml.cs | 2 +- 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/StarterNG.Tests/LegacyDecodingTests.cs b/StarterNG.Tests/LegacyDecodingTests.cs index 4325bf5..c37e438 100644 --- a/StarterNG.Tests/LegacyDecodingTests.cs +++ b/StarterNG.Tests/LegacyDecodingTests.cs @@ -1,3 +1,4 @@ +using System.Runtime.ExceptionServices; using System.Text; using StarterNG.Infrastructure.Adapters; using StarterNG.Tests.Fakes; @@ -22,6 +23,40 @@ public class LegacyDecodingTests Assert.Equal(Polish, LegacyText.Decode(Encoding.UTF8.GetBytes(Polish))); } + [Fact] + public void A_byte_order_mark_does_not_stick_to_the_text() + { + byte[] bytes = Encoding.UTF8.GetPreamble().Concat(Encoding.UTF8.GetBytes("$a\n" + Polish)).ToArray(); + + Assert.Equal("$a\n" + Polish, LegacyText.Decode(bytes)); + } + + [Fact] + public void Telling_the_two_apart_does_not_throw() + { + // Decoding used to try strict UTF-8 and catch the failure: a first-chance + // exception for every 1250 file, hundreds of them on a load. + int thread = Environment.CurrentManagedThreadId; + int thrown = 0; + EventHandler count = (_, e) => + { + if (e.Exception is DecoderFallbackException && Environment.CurrentManagedThreadId == thread) + thrown++; + }; + + AppDomain.CurrentDomain.FirstChanceException += count; + try + { + LegacyText.Decode(LegacyText.CodePage1250.GetBytes(Polish)); + } + finally + { + AppDomain.CurrentDomain.FirstChanceException -= count; + } + + Assert.Equal(0, thrown); + } + [Fact] public void Plain_ascii_reads_the_same_either_way() { diff --git a/StarterNG/Infrastructure/Adapters/LegacyText.cs b/StarterNG/Infrastructure/Adapters/LegacyText.cs index 8997cfe..5eebd01 100644 --- a/StarterNG/Infrastructure/Adapters/LegacyText.cs +++ b/StarterNG/Infrastructure/Adapters/LegacyText.cs @@ -1,5 +1,6 @@ using System; using System.Text; +using System.Text.Unicode; namespace StarterNG.Infrastructure.Adapters; @@ -24,23 +25,21 @@ public static class LegacyText /// /// Decodes a game file. Most are code page 1250, but a few have been re-saved /// as UTF-8 over the years, and reading those as 1250 turns "Zakład" into - /// "ZakĹ‚ad". Valid UTF-8 is taken at its word; anything that fails a strict - /// decode is 1250, which cannot fail. + /// "ZakĹ‚ad". Valid UTF-8 is taken at its word; anything else is 1250, which + /// cannot fail. Checked up front rather than by catching a failed strict decode: + /// most files are 1250, and that was an exception per file on every load. + /// A byte order mark is dropped, or it would stick to the first token. /// public static string Decode(byte[] bytes) { - try - { - return Utf8Strict.GetString(bytes); - } - catch (DecoderFallbackException) - { - return CodePage1250.GetString(bytes); - } + ReadOnlySpan text = bytes; + if (text.StartsWith(Utf8Bom)) + text = text[Utf8Bom.Length..]; + + return Utf8.IsValid(text) ? Encoding.UTF8.GetString(text) : CodePage1250.GetString(text); } - private static readonly Encoding Utf8Strict = - new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + private static ReadOnlySpan Utf8Bom => [0xEF, 0xBB, 0xBF]; private static Encoding Resolve() { diff --git a/StarterNG/Views/Scenarios.axaml.cs b/StarterNG/Views/Scenarios.axaml.cs index a55b984..99a992a 100644 --- a/StarterNG/Views/Scenarios.axaml.cs +++ b/StarterNG/Views/Scenarios.axaml.cs @@ -709,7 +709,7 @@ public partial class Scenarios : UserControl try { - timetableContent.Text = File.ReadAllText(path, LegacyText.CodePage1250); + timetableContent.Text = LegacyText.Decode(File.ReadAllBytes(path)); UpdateTimetableTab(true); } catch From d1d458f8eefae04d79bb6c3df298c487c2985b2f Mon Sep 17 00:00:00 2001 From: maj00r Date: Sun, 4 Oct 2026 12:18:36 +0200 Subject: [PATCH 04/55] Add a JSON format for the vehicle database --- StarterNG.Tests/VehicleJsonTests.cs | 199 ++++++++++++++++ .../Application/Abstractions/IGamePaths.cs | 3 + StarterNG/Domain/Vehicles/VehicleTexture.cs | 23 ++ .../Infrastructure/Adapters/GamePaths.cs | 2 + .../Vehicles/TexturesTxtParser.cs | 12 +- .../Vehicles/VehicleJsonDocument.cs | 124 ++++++++++ .../Vehicles/VehicleJsonRepository.cs | 101 ++++++++ .../Vehicles/VehicleJsonSerializer.cs | 215 ++++++++++++++++++ 8 files changed, 678 insertions(+), 1 deletion(-) create mode 100644 StarterNG.Tests/VehicleJsonTests.cs create mode 100644 StarterNG/Infrastructure/Vehicles/VehicleJsonDocument.cs create mode 100644 StarterNG/Infrastructure/Vehicles/VehicleJsonRepository.cs create mode 100644 StarterNG/Infrastructure/Vehicles/VehicleJsonSerializer.cs diff --git a/StarterNG.Tests/VehicleJsonTests.cs b/StarterNG.Tests/VehicleJsonTests.cs new file mode 100644 index 0000000..2e48eed --- /dev/null +++ b/StarterNG.Tests/VehicleJsonTests.cs @@ -0,0 +1,199 @@ +using System.Text; +using StarterNG.Domain.Vehicles; +using StarterNG.Infrastructure.Vehicles; +using StarterNG.Tests.Fakes; + +namespace StarterNG.Tests; + +public class VehicleJsonTests +{ + private static readonly TexturesTxtParser Parser = new(); + + private static readonly VehicleJsonSerializer Serializer = new(); + + /// + /// A vehicle folder exercising every piece of the legacy syntax at once: the + /// headers and comments that carry no data, a category marker, a credit line + /// with Polish text, an alternative model, a wreck and an archival section. + /// + private const string Ep07 = """ + # EP07 - lokomotywy elektryczne + @ep07 + !=e + ep07-135.mat=ep07.e3d,ep07 // 1.0,EP07-135,PKP Intercity,Warszawa Grochów,2019-04-01,Wrocław,Zażółć,Gęślą Jaźń + ep07-174.mat=ep07.e3d,ep07=ep07p.e3d,ep07p + ep07wrak.mat=ep07.e3d,ep07 + linia bez znaku rownosci + $a + ep07-stary.mat=ep07.e3d,ep07 + """; + + /// A unit whose cars are coupled into a fixed set. + private const string En57 = """ + ^2 + en57-1000ra.mat=en57.e3d,en57 + en57-1000rb.mat=en57.e3d,en57 + """; + + [Fact] + public void An_entry_written_out_and_read_back_is_the_entry_it_started_as() + { + var entry = Parser.Parse("pkp/ep07/", Lines(Ep07))!; + + var restored = Serializer.Read(Serializer.Write(entry)); + + Assert.Equal(Describe(entry), Describe(restored!)); + } + + [Fact] + public void The_catalogue_built_from_json_matches_the_one_built_from_textures_txt() + { + var files = new InMemoryFileSystem() + .WithLegacyFile(Dynamic("pkp", "ep07"), Ep07) + .WithLegacyFile(Dynamic("pkp", "en57"), En57); + var installation = new TestInstallation(files); + + installation.Vehicles.Load(installation.Library.Vehicles); + var fromJson = ReloadThroughJson(installation); + + Assert.Equal(5, installation.Library.Vehicles.Textures.Count); + Assert.Equal(Describe(installation.Library.Vehicles), Describe(fromJson)); + } + + [Fact] + public void Headers_and_unparsable_lines_survive_the_trip_so_nothing_is_dropped() + { + var entry = Parser.Parse("pkp/ep07/", Lines(Ep07))!; + + var restored = Serializer.Read(Serializer.Write(entry))!; + + Assert.Equal(new[] { "# EP07 - lokomotywy elektryczne", "@ep07", "linia bez znaku rownosci" }, + restored.Unknown); + } + + [Fact] + public void Polish_text_is_written_as_itself_rather_than_as_escapes() + { + var entry = Parser.Parse("pkp/ep07/", Lines(Ep07))!; + + string json = Serializer.Write(entry); + + Assert.Contains("Warszawa Grochów", json); + Assert.DoesNotContain("\\u", json); + } + + [Fact] + public void The_merged_database_is_read_before_the_packages_layered_on_it() + { + var entry = Parser.Parse("pkp/ep07/", Lines(Ep07))!; + var package = Parser.Parse("pkp/en57/", Lines(En57))!; + var files = new InMemoryFileSystem() + .WithFile(Database("vehicles.json"), Serializer.WriteCollection(new[] { entry })) + .WithFile(Database("aaa-pakiet.json"), Serializer.Write(package)); + var installation = new TestInstallation(files); + var catalog = new VehicleCatalog(installation.MiniTextures); + + int liveries = Repository(installation).Load(catalog); + + Assert.Equal(6, liveries); + Assert.Equal("legacy:pkp/ep07/ep07-135", catalog.Textures[0].Uuid); + } + + [Fact] + public void An_entry_from_a_newer_format_is_refused_instead_of_half_read() + { + var files = new InMemoryFileSystem() + .WithFile(Database("z-przyszlosci.json"), + """{"uuid":"x","schema_version":99,"textures":[{"skinfile":"ep07-135"}]}"""); + var installation = new TestInstallation(files); + var catalog = new VehicleCatalog(installation.MiniTextures); + + int liveries = Repository(installation).Load(catalog); + + Assert.Equal(0, liveries); + Assert.Empty(catalog.Textures); + Assert.Contains("z-przyszlosci.json", installation.Log.Text); + } + + [Fact] + public void An_unreadable_package_is_logged_and_leaves_the_rest_of_the_database_alone() + { + var entry = Parser.Parse("pkp/ep07/", Lines(Ep07))!; + var files = new InMemoryFileSystem() + .WithFile(Database("aaa-polamany.json"), "{ to nie jest json") + .WithFile(Database("bbb-dobry.json"), Serializer.Write(entry)); + var installation = new TestInstallation(files); + var catalog = new VehicleCatalog(installation.MiniTextures); + + int liveries = Repository(installation).Load(catalog); + + Assert.Equal(4, liveries); + Assert.Contains("aaa-polamany.json", installation.Log.Text); + } + + private static VehicleJsonRepository Repository(TestInstallation installation) => + new(installation.Files, installation.Paths, installation.Log, Serializer); + + /// + /// Writes every vehicle folder of an installation out as its own package file + /// and loads the catalogue back from those, which is the migration the starter + /// will do silently once this format is the one it reads. + /// + private static VehicleCatalog ReloadThroughJson(TestInstallation installation) + { + foreach (string maker in installation.Files.GetDirectories(installation.Paths.Dynamic)) + foreach (string vehicle in installation.Files.GetDirectories(maker)) + { + string path = Path.Combine(vehicle, "textures.txt"); + if (!installation.Files.FileExists(path)) + continue; + + string directory = $"{Path.GetFileName(maker)}/{Path.GetFileName(vehicle)}/"; + var entry = Parser.Parse(directory, + installation.Files.ReadAllText(path, Encoding.GetEncoding(1250)).Split('\n')); + if (entry is not null) + installation.Files.WithFile(Database($"{directory.Replace('/', '.')}json"), + Serializer.Write(entry)); + } + + var catalog = new VehicleCatalog(installation.MiniTextures); + Repository(installation).Load(catalog); + return catalog; + } + + private static string[] Lines(string file) => file.Split('\n'); + + private static string Dynamic(string maker, string vehicle) => + TestInstallation.At("dynamic", maker, vehicle, "textures.txt"); + + private static string Database(string file) => TestInstallation.At("databases", "vehicles", file); + + /// Everything an entry carries, flattened so a difference reads as one. + private static string Describe(VehicleEntry entry) => + string.Join('\n', + new[] { $"entry {entry.Uuid}" } + .Concat(entry.Groups.Select(Describe)) + .Concat(entry.Textures.Select(Describe)) + .Concat(entry.Sets.Select(Describe)) + .Concat(entry.Unknown.Select(line => $"unknown {line}"))); + + /// The same for a whole catalogue, wrecks and resolved fields included. + private static string Describe(VehicleCatalog catalog) => + string.Join('\n', + catalog.GroupsById.OrderBy(pair => pair.Key, StringComparer.Ordinal).Select(pair => Describe(pair.Value)) + .Concat(catalog.TextureByUuid.OrderBy(pair => pair.Key, StringComparer.Ordinal) + .Select(pair => Describe(pair.Value))) + .Concat(catalog.Sets.Select(Describe))); + + private static string Describe(VehicleGroup group) => + $"group {group.Id}|{group.Category}|{group.Mini}|{group.Archived}|{group.Implicit}"; + + private static string Describe(VehicleSet set) => + $"set {set.Uuid}|{set.Mode}|{set.Count}|{string.Join(',', set.TextureRefs)}"; + + private static string Describe(VehicleTexture texture) => + $"texture {texture.Uuid}|{texture.Directory}|{texture.Skinfile}|{texture.Model}|{texture.Group}|" + + $"{texture.MiniRef}|{texture.TextureMini}|{texture.Wreck}|{texture.ResolvedClass}|" + + $"{texture.ResolvedCategory}|{texture.ResolvedArchived}|{texture.Meta?.Raw}|" + + string.Join(';', texture.Aliases.Select(alias => $"{alias.Model},{alias.MiniRef},{alias.TextureMini}")); +} diff --git a/StarterNG/Application/Abstractions/IGamePaths.cs b/StarterNG/Application/Abstractions/IGamePaths.cs index 314647c..5adcf5d 100644 --- a/StarterNG/Application/Abstractions/IGamePaths.cs +++ b/StarterNG/Application/Abstractions/IGamePaths.cs @@ -12,6 +12,9 @@ public interface IGamePaths string Data { get; } + /// Where the JSON vehicle database and the packages layered on it live. + string VehicleDatabase { get; } + string StarterConfig { get; } string DiagnosticsLog { get; } diff --git a/StarterNG/Domain/Vehicles/VehicleTexture.cs b/StarterNG/Domain/Vehicles/VehicleTexture.cs index 820b08a..e6469b2 100644 --- a/StarterNG/Domain/Vehicles/VehicleTexture.cs +++ b/StarterNG/Domain/Vehicles/VehicleTexture.cs @@ -5,9 +5,25 @@ namespace StarterNG.Domain.Vehicles; public sealed class VehicleEntry { public string? Uuid { get; set; } + + /// + /// Version of the file format the entry was read from, zero when it came from + /// a legacy textures.txt. Kept so a file written by a newer starter is refused + /// outright instead of being read with half its meaning missing. + /// + public int SchemaVersion { get; set; } + public List Groups { get; } = new(); public List Textures { get; } = new(); public List Sets { get; } = new(); + + /// + /// Source lines the reader did not turn into anything else - legacy headers, + /// comments and malformed liveries - kept verbatim so an entry written back + /// out does not quietly drop what its author put in the file. Content only: + /// blank lines and the original spacing are not preserved. + /// + public List Unknown { get; } = new(); } public sealed class VehicleGroup @@ -17,6 +33,13 @@ public sealed class VehicleGroup public string? Mini { get; set; } public bool Archived { get; set; } + + /// + /// True when nothing declared the group and it was inferred from the liveries + /// that share a thumbnail - which is every group textures.txt produces, as the + /// legacy format has no syntax for declaring one. + /// + public bool Implicit { get; set; } } public sealed class VehicleTexture diff --git a/StarterNG/Infrastructure/Adapters/GamePaths.cs b/StarterNG/Infrastructure/Adapters/GamePaths.cs index 3dd5c67..bbf86db 100644 --- a/StarterNG/Infrastructure/Adapters/GamePaths.cs +++ b/StarterNG/Infrastructure/Adapters/GamePaths.cs @@ -22,6 +22,8 @@ public sealed class GamePaths : IGamePaths public string Data => Path.Combine(Root, "data"); + public string VehicleDatabase => Path.Combine(Root, "databases", "vehicles"); + public string StarterConfig => Path.Combine(Root, "starter"); public string DiagnosticsLog => Path.Combine(StarterConfig, "bledy.txt"); diff --git a/StarterNG/Infrastructure/Vehicles/TexturesTxtParser.cs b/StarterNG/Infrastructure/Vehicles/TexturesTxtParser.cs index c99a88e..ba731ec 100644 --- a/StarterNG/Infrastructure/Vehicles/TexturesTxtParser.cs +++ b/StarterNG/Infrastructure/Vehicles/TexturesTxtParser.cs @@ -25,7 +25,10 @@ public sealed class TexturesTxtParser char first = line[0]; if (first is '#' or '@' or '*') + { + entry.Unknown.Add(line); continue; + } if (line.StartsWith("$a", StringComparison.OrdinalIgnoreCase)) { @@ -47,11 +50,17 @@ public sealed class TexturesTxtParser } if (line.IndexOf('=') < 0 || line.StartsWith("//", StringComparison.Ordinal)) + { + entry.Unknown.Add(line); continue; + } var texture = ParseLivery(line, directory, categorySign, archived, entry); if (texture is null) + { + entry.Unknown.Add(line); continue; + } entry.Textures.Add(texture); @@ -120,7 +129,8 @@ public sealed class TexturesTxtParser Id = groupId, Category = categorySign, Mini = lead.Mini, - Archived = archived + Archived = archived, + Implicit = true }); var texture = new VehicleTexture diff --git a/StarterNG/Infrastructure/Vehicles/VehicleJsonDocument.cs b/StarterNG/Infrastructure/Vehicles/VehicleJsonDocument.cs new file mode 100644 index 0000000..b75962f --- /dev/null +++ b/StarterNG/Infrastructure/Vehicles/VehicleJsonDocument.cs @@ -0,0 +1,124 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StarterNG.Infrastructure.Vehicles; + +// The vehicle database as it is written to disk (schema_version 1). Deliberately +// separate from the domain model: the model is free to change shape with the +// starter, the file format is not - packages written by someone else have to keep +// parsing. The property names are the ones the format shipped with, so a file +// written for the older reader still loads. +// +// Collections are nullable and left out when empty, which keeps a generated file +// close to what a person would have typed by hand. + +public sealed class VehicleEntryDocument +{ + [JsonPropertyName("uuid")] public string? Uuid { get; set; } + + [JsonPropertyName("schema_version")] public int SchemaVersion { get; set; } + + [JsonPropertyName("groups")] public List? Groups { get; set; } + + [JsonPropertyName("textures")] public List? Textures { get; set; } + + [JsonPropertyName("sets")] public List? Sets { get; set; } + + [JsonPropertyName("unknown")] public List? Unknown { get; set; } +} + +/// Root of a merged database holding many entries in one file. +public sealed class VehicleCollectionDocument +{ + [JsonPropertyName("vehicles")] public List? Vehicles { get; set; } +} + +public sealed class VehicleGroupDocument +{ + [JsonPropertyName("id")] public string? Id { get; set; } + + [JsonPropertyName("category")] public string? Category { get; set; } + + [JsonPropertyName("mini")] public string? Mini { get; set; } + + [JsonPropertyName("archived")] public bool Archived { get; set; } + + [JsonPropertyName("implicit")] public bool Implicit { get; set; } +} + +public sealed class VehicleTextureDocument +{ + [JsonPropertyName("uuid")] public string? Uuid { get; set; } + + [JsonPropertyName("directory")] public string? Directory { get; set; } + + [JsonPropertyName("skinfile")] public string? Skinfile { get; set; } + + [JsonPropertyName("model")] public string? Model { get; set; } + + [JsonPropertyName("group")] public string? Group { get; set; } + + [JsonPropertyName("mini_ref")] public string? MiniRef { get; set; } + + [JsonPropertyName("texture_mini")] public string? TextureMini { get; set; } + + [JsonPropertyName("wreck")] public bool Wreck { get; set; } + + [JsonPropertyName("aliases")] public List? Aliases { get; set; } + + [JsonPropertyName("meta")] public VehicleMetaDocument? Meta { get; set; } +} + +public sealed class VehicleAliasDocument +{ + [JsonPropertyName("model")] public string? Model { get; set; } + + [JsonPropertyName("group")] public string? Group { get; set; } + + [JsonPropertyName("mini_ref")] public string? MiniRef { get; set; } + + [JsonPropertyName("texture_mini")] public string? TextureMini { get; set; } +} + +public sealed class VehicleMetaDocument +{ + [JsonPropertyName("raw")] public string? Raw { get; set; } + + [JsonPropertyName("version")] public string? Version { get; set; } + + [JsonPropertyName("vehicle")] public string? Vehicle { get; set; } + + [JsonPropertyName("operator")] public string? Operator { get; set; } + + [JsonPropertyName("depot")] public string? Depot { get; set; } + + [JsonPropertyName("revision_date")] public string? RevisionDate { get; set; } + + [JsonPropertyName("revision_place")] public string? RevisionPlace { get; set; } + + [JsonPropertyName("texture_author")] public string? TextureAuthor { get; set; } + + [JsonPropertyName("photo_author")] public string? PhotoAuthor { get; set; } + + [JsonPropertyName("extra")] public List? Extra { get; set; } +} + +public sealed class VehicleSetDocument +{ + [JsonPropertyName("uuid")] public string? Uuid { get; set; } + + [JsonPropertyName("mode")] public string? Mode { get; set; } + + [JsonPropertyName("count")] public int Count { get; set; } + + [JsonPropertyName("texture_refs")] public List? TextureRefs { get; set; } +} + +// Source-generated metadata: PublishAot and PublishTrimmed rule out the reflection +// based serializer. The generator pulls in every nested document type on its own, +// and the reader's leniency is set on the options in VehicleJsonSerializer. +[JsonSerializable(typeof(VehicleEntryDocument))] +[JsonSerializable(typeof(VehicleCollectionDocument))] +internal partial class VehicleJsonContext : JsonSerializerContext +{ +} diff --git a/StarterNG/Infrastructure/Vehicles/VehicleJsonRepository.cs b/StarterNG/Infrastructure/Vehicles/VehicleJsonRepository.cs new file mode 100644 index 0000000..a0e0d72 --- /dev/null +++ b/StarterNG/Infrastructure/Vehicles/VehicleJsonRepository.cs @@ -0,0 +1,101 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using StarterNG.Application.Abstractions; +using StarterNG.Domain.Vehicles; + +namespace StarterNG.Infrastructure.Vehicles; + +/// +/// Reads the vehicle catalogue from databases/vehicles: a merged vehicles.json +/// holding the whole database, and one file per package layered on top of it. +/// +public sealed class VehicleJsonRepository : IVehicleRepository +{ + private const string MergedFileName = "vehicles.json"; + + private readonly IFileSystem _files; + private readonly IGamePaths _paths; + private readonly IDiagnosticsLog _log; + private readonly VehicleJsonSerializer _serializer; + + public VehicleJsonRepository(IFileSystem files, IGamePaths paths, IDiagnosticsLog log, + VehicleJsonSerializer serializer) + { + _files = files; + _paths = paths; + _log = log; + _serializer = serializer; + } + + public int Load(VehicleCatalog catalog) + { + catalog.BeginLoad(); + + int liveries = 0; + foreach (string file in DatabaseFiles()) + liveries += ReadInto(catalog, file); + + catalog.EndLoad(); + return liveries; + } + + private int ReadInto(VehicleCatalog catalog, string path) + { + try + { + string json = _files.ReadAllText(path); + var entries = IsMerged(path) + ? _serializer.ReadCollection(json) + : Listed(_serializer.Read(json)); + + int liveries = 0; + foreach (var entry in entries) + { + if (!VehicleJsonSerializer.IsReadable(entry)) + { + _log.Log($"{Name(path)}: format w wersji {entry.SchemaVersion}, " + + $"ten starter czyta do {VehicleJsonSerializer.SchemaVersion}"); + continue; + } + + liveries += entry.Textures.Count; + catalog.Ingest(entry); + } + + return liveries; + } + catch (Exception ex) + { + _log.Log(Name(path), ex); + return 0; + } + } + + /// + /// The merged database first and the packages after it, both in a fixed order, + /// so which entry wins a clash is the same on every machine - the file system + /// does not promise one. + /// + private IEnumerable DatabaseFiles() + { + var files = _files.GetFiles(_paths.VehicleDatabase, "*.json") + .OrderBy(Path.GetFileName, StringComparer.OrdinalIgnoreCase) + .ToList(); + + foreach (string file in files.Where(IsMerged)) + yield return file; + + foreach (string file in files.Where(file => !IsMerged(file))) + yield return file; + } + + private static bool IsMerged(string path) => + string.Equals(Path.GetFileName(path), MergedFileName, StringComparison.OrdinalIgnoreCase); + + private static string Name(string path) => $"databases/vehicles/{Path.GetFileName(path)}"; + + private static List Listed(VehicleEntry? entry) => + entry is null ? new List() : new List { entry }; +} diff --git a/StarterNG/Infrastructure/Vehicles/VehicleJsonSerializer.cs b/StarterNG/Infrastructure/Vehicles/VehicleJsonSerializer.cs new file mode 100644 index 0000000..d4fc885 --- /dev/null +++ b/StarterNG/Infrastructure/Vehicles/VehicleJsonSerializer.cs @@ -0,0 +1,215 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; +using StarterNG.Domain.Vehicles; + +namespace StarterNG.Infrastructure.Vehicles; + +/// +/// Reads and writes in the JSON database format, +/// mapping it on and off the document types that define the format. +/// +public sealed class VehicleJsonSerializer +{ + /// The format this build writes, and the newest one it can read. + public const int SchemaVersion = 1; + + // Leniency a hand-written package file deserves: names matched case + // insensitively, // and /* */ comments skipped, trailing commas allowed. The + // relaxed encoder keeps Polish text in the credits and in the preserved legacy + // comments readable instead of turning it into \uXXXX escapes, and the + // source-generated resolver keeps all of it AOT-safe. + private static readonly JsonSerializerOptions Options = new() + { + WriteIndented = true, + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + TypeInfoResolver = VehicleJsonContext.Default + }; + + private static readonly JsonTypeInfo EntryInfo = + (JsonTypeInfo)Options.GetTypeInfo(typeof(VehicleEntryDocument)); + + private static readonly JsonTypeInfo CollectionInfo = + (JsonTypeInfo)Options.GetTypeInfo(typeof(VehicleCollectionDocument)); + + public string Write(VehicleEntry entry) => JsonSerializer.Serialize(ToDocument(entry), EntryInfo); + + public string WriteCollection(IEnumerable entries) => + JsonSerializer.Serialize( + new VehicleCollectionDocument { Vehicles = entries.Select(ToDocument).ToList() }, + CollectionInfo); + + /// One entry per file, the form a package is authored in. + public VehicleEntry? Read(string json) + { + var document = JsonSerializer.Deserialize(json, EntryInfo); + return document is null ? null : FromDocument(document); + } + + /// Many entries in one file, the form a whole database is shipped in. + public List ReadCollection(string json) + { + var document = JsonSerializer.Deserialize(json, CollectionInfo); + if (document?.Vehicles is null) + return new List(); + + return document.Vehicles.Select(FromDocument).ToList(); + } + + /// + /// Whether this build understands the file. An entry written by a newer + /// starter is refused rather than half-read, an entry with no version at all + /// is taken as the current one - the format carried no version when the first + /// files were written by hand. + /// + public static bool IsReadable(VehicleEntry entry) => entry.SchemaVersion <= SchemaVersion; + + private static VehicleEntryDocument ToDocument(VehicleEntry entry) => + new() + { + Uuid = entry.Uuid, + SchemaVersion = SchemaVersion, + Groups = OrNull(entry.Groups.Select(group => new VehicleGroupDocument + { + Id = group.Id, + Category = group.Category, + Mini = group.Mini, + Archived = group.Archived, + Implicit = group.Implicit + }).ToList()), + Textures = OrNull(entry.Textures.Select(ToDocument).ToList()), + Sets = OrNull(entry.Sets.Select(set => new VehicleSetDocument + { + Uuid = set.Uuid, + Mode = set.Mode, + Count = set.Count, + TextureRefs = OrNull(set.TextureRefs) + }).ToList()), + Unknown = OrNull(entry.Unknown) + }; + + private static VehicleTextureDocument ToDocument(VehicleTexture texture) => + new() + { + Uuid = texture.Uuid, + Directory = texture.Directory, + Skinfile = texture.Skinfile, + Model = texture.Model, + Group = texture.Group, + MiniRef = texture.MiniRef, + TextureMini = texture.TextureMini, + Wreck = texture.Wreck, + Aliases = OrNull(texture.Aliases.Select(alias => new VehicleAliasDocument + { + Model = alias.Model, + Group = alias.Group, + MiniRef = alias.MiniRef, + TextureMini = alias.TextureMini + }).ToList()), + Meta = texture.Meta is null + ? null + : new VehicleMetaDocument + { + Raw = texture.Meta.Raw, + Version = texture.Meta.Version, + Vehicle = texture.Meta.Vehicle, + Operator = texture.Meta.Operator, + Depot = texture.Meta.Depot, + RevisionDate = texture.Meta.RevisionDate, + RevisionPlace = texture.Meta.RevisionPlace, + TextureAuthor = texture.Meta.TextureAuthor, + PhotoAuthor = texture.Meta.PhotoAuthor, + Extra = OrNull(texture.Meta.Extra) + } + }; + + private static VehicleEntry FromDocument(VehicleEntryDocument document) + { + var entry = new VehicleEntry + { + Uuid = document.Uuid, + SchemaVersion = document.SchemaVersion == 0 ? SchemaVersion : document.SchemaVersion + }; + + foreach (var group in document.Groups ?? Enumerable.Empty()) + entry.Groups.Add(new VehicleGroup + { + Id = group.Id ?? "", + Category = group.Category, + Mini = group.Mini, + Archived = group.Archived, + Implicit = group.Implicit + }); + + foreach (var texture in document.Textures ?? Enumerable.Empty()) + entry.Textures.Add(FromDocument(texture)); + + foreach (var set in document.Sets ?? Enumerable.Empty()) + entry.Sets.Add(new VehicleSet + { + Uuid = set.Uuid, + Mode = set.Mode, + Count = set.Count, + TextureRefs = set.TextureRefs ?? new List() + }); + + if (document.Unknown is not null) + entry.Unknown.AddRange(document.Unknown); + + return entry; + } + + private static VehicleTexture FromDocument(VehicleTextureDocument document) + { + var texture = new VehicleTexture + { + Uuid = document.Uuid, + Directory = document.Directory ?? "", + Skinfile = document.Skinfile ?? "", + Model = document.Model, + Group = document.Group, + MiniRef = document.MiniRef, + TextureMini = document.TextureMini, + Wreck = document.Wreck + }; + + foreach (var alias in document.Aliases ?? Enumerable.Empty()) + texture.Aliases.Add(new VehicleAlias + { + Model = alias.Model, + Group = alias.Group, + MiniRef = alias.MiniRef, + TextureMini = alias.TextureMini + }); + + if (document.Meta is not null) + { + texture.Meta = new VehicleMeta + { + Raw = document.Meta.Raw, + Version = document.Meta.Version, + Vehicle = document.Meta.Vehicle, + Operator = document.Meta.Operator, + Depot = document.Meta.Depot, + RevisionDate = document.Meta.RevisionDate, + RevisionPlace = document.Meta.RevisionPlace, + TextureAuthor = document.Meta.TextureAuthor, + PhotoAuthor = document.Meta.PhotoAuthor + }; + + if (document.Meta.Extra is not null) + texture.Meta.Extra.AddRange(document.Meta.Extra); + } + + return texture; + } + + private static List? OrNull(List items) => items.Count > 0 ? items : null; +} From 93463ae7f2ca99dfef8e1362386ba5a1d113f6ac Mon Sep 17 00:00:00 2001 From: maj00r Date: Sun, 4 Oct 2026 12:18:36 +0200 Subject: [PATCH 05/55] Load vehicles from a JSON copy of textures.txt, rebuilt when the installation changes --- StarterNG.Tests/Fakes/InMemoryFileSystem.cs | 2 + StarterNG.Tests/Fakes/TestInstallation.cs | 9 +- StarterNG.Tests/VehicleCacheTests.cs | 111 ++++++++++ .../Application/Abstractions/IFileSystem.cs | 2 + StarterNG/Application/AppServices.cs | 21 +- .../Adapters/PhysicalFileSystem.cs | 2 + .../Infrastructure/Settings/SettingsPaths.cs | 7 + .../Vehicles/TexturesTxtVehicleRepository.cs | 38 ++-- .../Vehicles/VehicleCacheRepository.cs | 189 ++++++++++++++++++ 9 files changed, 362 insertions(+), 19 deletions(-) create mode 100644 StarterNG.Tests/VehicleCacheTests.cs create mode 100644 StarterNG/Infrastructure/Vehicles/VehicleCacheRepository.cs diff --git a/StarterNG.Tests/Fakes/InMemoryFileSystem.cs b/StarterNG.Tests/Fakes/InMemoryFileSystem.cs index 1a12a86..fca2cbd 100644 --- a/StarterNG.Tests/Fakes/InMemoryFileSystem.cs +++ b/StarterNG.Tests/Fakes/InMemoryFileSystem.cs @@ -134,6 +134,8 @@ public sealed class InMemoryFileSystem : IFileSystem public DateTime GetLastWriteTimeUtc(string path) => LastWriteTimeUtc; /// Files added through carry the execute bit. + public long GetFileSize(string path) => Bytes(path).LongLength; + public bool IsExecutable(string path) => _executable.Contains(Normalize(path)); public InMemoryFileSystem WithExecutable(string path, string contents = "\u007fELF") diff --git a/StarterNG.Tests/Fakes/TestInstallation.cs b/StarterNG.Tests/Fakes/TestInstallation.cs index 724a8d7..fbb39a3 100644 --- a/StarterNG.Tests/Fakes/TestInstallation.cs +++ b/StarterNG.Tests/Fakes/TestInstallation.cs @@ -26,12 +26,15 @@ public sealed class TestInstallation MiniTextures = new MiniTextureIndex(Files, Paths); Physics = new FizPhysicsRepository(Files, Paths, Log); Vehicles = new TexturesTxtVehicleRepository(Files, Paths, Log, new TexturesTxtParser()); + VehicleJson = new VehicleJsonSerializer(); Sceneries = new SceneryRepository(Files, Paths, Log, new SceneryParser(Clock, Random)); MissingAssets = new MissingAssetScanner(Files, Paths); - Library = new GameLibrary(Vehicles, Sceneries, MiniTextures, Physics, Log); - Environment = new FakeEnvironment(); SettingsPaths = new SettingsPaths(Environment, Paths); + VehicleCache = new VehicleCacheRepository(Files, Paths, Log, Vehicles, VehicleJson, + SettingsPaths.VehicleCacheDirectory()); + Library = new GameLibrary(VehicleCache, Sceneries, MiniTextures, Physics, Log); + Executables = new ExecutableLocator(Files, Paths, Environment, Log); SettingsStore = new SettingsStore(Files, Clock, Log, SettingsPaths, new SettingsSerializer(), Executables); } @@ -45,6 +48,8 @@ public sealed class TestInstallation public MiniTextureIndex MiniTextures { get; } public FizPhysicsRepository Physics { get; } public TexturesTxtVehicleRepository Vehicles { get; } + public VehicleJsonSerializer VehicleJson { get; } + public VehicleCacheRepository VehicleCache { get; } public SceneryRepository Sceneries { get; } public MissingAssetScanner MissingAssets { get; } public GameLibrary Library { get; } diff --git a/StarterNG.Tests/VehicleCacheTests.cs b/StarterNG.Tests/VehicleCacheTests.cs new file mode 100644 index 0000000..5931ca8 --- /dev/null +++ b/StarterNG.Tests/VehicleCacheTests.cs @@ -0,0 +1,111 @@ +using System.Text; +using StarterNG.Domain.Vehicles; +using StarterNG.Tests.Fakes; + +namespace StarterNG.Tests; + +public class VehicleCacheTests +{ + private const string Ep07 = "ep07-135.mat=ep07.e3d,ep07"; + + private const string En57 = "en57-1000ra.mat=en57.e3d,en57"; + + private static string Dynamic(string maker, string vehicle) => + TestInstallation.At("dynamic", maker, vehicle, "textures.txt"); + + private static string Cached(TestInstallation installation, string file) => + Path.Combine(installation.SettingsPaths.VehicleCacheDirectory(), file); + + [Fact] + public void The_first_load_migrates_textures_txt_into_the_starters_own_copy() + { + var files = new InMemoryFileSystem().WithLegacyFile(Dynamic("pkp", "ep07"), Ep07); + var installation = new TestInstallation(files); + + int liveries = installation.VehicleCache.Load(installation.Library.Vehicles); + + Assert.Equal(1, liveries); + Assert.True(files.FileExists(Cached(installation, "vehicles.json"))); + Assert.Contains("ep07-135", files.ReadAllText(Cached(installation, "vehicles.json"))); + } + + [Fact] + public void An_untouched_installation_is_served_from_the_copy_without_reading_textures_txt() + { + var files = new InMemoryFileSystem().WithLegacyFile(Dynamic("pkp", "ep07"), Ep07); + var installation = new TestInstallation(files); + installation.VehicleCache.Load(installation.Library.Vehicles); + + // Different liveries, same length and same timestamp: nothing the stamp + // can see has changed, so an answer of ep07-135 can only come from the copy. + files.WithLegacyFile(Dynamic("pkp", "ep07"), "ep07-999.mat=ep07.e3d,ep07"); + var catalog = new VehicleCatalog(installation.MiniTextures); + installation.VehicleCache.Load(catalog); + + Assert.Equal("legacy:pkp/ep07/ep07-135", Assert.Single(catalog.Textures).Uuid); + } + + [Fact] + public void An_edited_textures_txt_makes_the_copy_stale_and_it_is_rebuilt() + { + var files = new InMemoryFileSystem().WithLegacyFile(Dynamic("pkp", "ep07"), Ep07); + var installation = new TestInstallation(files); + installation.VehicleCache.Load(installation.Library.Vehicles); + + files.WithLegacyFile(Dynamic("pkp", "ep07"), Ep07 + "\nep07-174.mat=ep07.e3d,ep07"); + var catalog = new VehicleCatalog(installation.MiniTextures); + int liveries = installation.VehicleCache.Load(catalog); + + Assert.Equal(2, liveries); + Assert.Contains("ep07-174", files.ReadAllText(Cached(installation, "vehicles.json"))); + } + + [Fact] + public void A_vehicle_folder_that_disappeared_upstream_leaves_the_copy_too() + { + var files = new InMemoryFileSystem() + .WithLegacyFile(Dynamic("pkp", "ep07"), Ep07) + .WithLegacyFile(Dynamic("pkp", "en57"), En57); + var installation = new TestInstallation(files); + installation.VehicleCache.Load(installation.Library.Vehicles); + + files.DeleteFile(Dynamic("pkp", "en57")); + var catalog = new VehicleCatalog(installation.MiniTextures); + installation.VehicleCache.Load(catalog); + + Assert.Equal("legacy:pkp/ep07/ep07-135", Assert.Single(catalog.Textures).Uuid); + Assert.DoesNotContain("en57", files.ReadAllText(Cached(installation, "vehicles.json"))); + } + + [Fact] + public void A_damaged_copy_costs_the_parse_it_saved_and_nothing_else() + { + var files = new InMemoryFileSystem().WithLegacyFile(Dynamic("pkp", "ep07"), Ep07); + var installation = new TestInstallation(files); + installation.VehicleCache.Load(installation.Library.Vehicles); + files.WithFile(Cached(installation, "vehicles.json"), "{ obcieta kop"); + + var catalog = new VehicleCatalog(installation.MiniTextures); + int liveries = installation.VehicleCache.Load(catalog); + + Assert.Equal(1, liveries); + Assert.Contains("kopia nie do odczytu", installation.Log.Text); + } + + [Fact] + public void The_copy_reads_the_same_catalogue_the_legacy_reader_does() + { + var files = new InMemoryFileSystem() + .WithLegacyFile(Dynamic("pkp", "ep07"), Ep07) + .WithLegacyFile(Dynamic("pkp", "en57"), En57); + var installation = new TestInstallation(files); + + var direct = new VehicleCatalog(installation.MiniTextures); + installation.Vehicles.Load(direct); + var cached = new VehicleCatalog(installation.MiniTextures); + installation.VehicleCache.Load(cached); + + Assert.Equal(direct.Textures.Select(t => t.Uuid), cached.Textures.Select(t => t.Uuid)); + Assert.Equal(direct.GroupsById.Keys.Order(), cached.GroupsById.Keys.Order()); + } +} diff --git a/StarterNG/Application/Abstractions/IFileSystem.cs b/StarterNG/Application/Abstractions/IFileSystem.cs index c49c726..b30f657 100644 --- a/StarterNG/Application/Abstractions/IFileSystem.cs +++ b/StarterNG/Application/Abstractions/IFileSystem.cs @@ -43,5 +43,7 @@ public interface IFileSystem DateTime GetLastWriteTimeUtc(string path); + long GetFileSize(string path); + bool IsExecutable(string path); } diff --git a/StarterNG/Application/AppServices.cs b/StarterNG/Application/AppServices.cs index e223746..2df8944 100644 --- a/StarterNG/Application/AppServices.cs +++ b/StarterNG/Application/AppServices.cs @@ -31,7 +31,16 @@ public sealed class AppServices Sceneries = new SceneryRepository(files, paths, log, new SceneryParser(clock, Random)); SceneryImages = new SceneryImageLocator(files); MiniTextures = new MiniTextureIndex(files, paths); - Vehicles = new TexturesTxtVehicleRepository(files, paths, log, new TexturesTxtParser()); + var settingsPaths = new SettingsPaths(environment, paths); + SettingsPaths = settingsPaths; + + // textures.txt stays the truth; the starter reads its own JSON copy of it + // and rebuilds that copy whenever the installation has moved on. + TexturesTxt = new TexturesTxtVehicleRepository(files, paths, log, new TexturesTxtParser()); + VehicleJson = new VehicleJsonSerializer(); + VehicleCache = new VehicleCacheRepository(files, paths, log, TexturesTxt, VehicleJson, + settingsPaths.VehicleCacheDirectory()); + Vehicles = VehicleCache; Physics = new FizPhysicsRepository(files, paths, log); MissingAssets = new MissingAssetScanner(files, paths); SceneryTexts = new SceneryTranslations(files, log); @@ -40,10 +49,8 @@ public sealed class AppServices Library = new GameLibrary(Vehicles, Sceneries, MiniTextures, Physics, log); State = new AppState(); - var settingsPaths = new SettingsPaths(environment, paths); Executables = new ExecutableLocator(files, paths, environment, log); SettingsStore = new SettingsStore(files, clock, log, settingsPaths, new SettingsSerializer(), Executables); - SettingsPaths = settingsPaths; MissingVehicleLog = new MissingVehicleLog(SettingsStore.Settings, MissingAssets, Library, log); StartSimulation = new StartSimulation(State, SettingsStore, files, Processes, Random, log); } @@ -85,6 +92,14 @@ public sealed class AppServices public IVehicleRepository Vehicles { get; } + /// The same object as , typed for the developer tab. + public VehicleCacheRepository VehicleCache { get; } + + /// The legacy reader behind the cache, for tools that go to the source. + public TexturesTxtVehicleRepository TexturesTxt { get; } + + public VehicleJsonSerializer VehicleJson { get; } + public IPhysicsRepository Physics { get; } public MissingAssetScanner MissingAssets { get; } diff --git a/StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs b/StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs index fb64b2b..69e0c0f 100644 --- a/StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs +++ b/StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs @@ -51,6 +51,8 @@ public sealed class PhysicalFileSystem : IFileSystem public DateTime GetLastWriteTimeUtc(string path) => File.GetLastWriteTimeUtc(path); + public long GetFileSize(string path) => new FileInfo(path).Length; + public bool IsExecutable(string path) { if (OperatingSystem.IsWindows()) diff --git a/StarterNG/Infrastructure/Settings/SettingsPaths.cs b/StarterNG/Infrastructure/Settings/SettingsPaths.cs index d9eba18..75639c2 100644 --- a/StarterNG/Infrastructure/Settings/SettingsPaths.cs +++ b/StarterNG/Infrastructure/Settings/SettingsPaths.cs @@ -43,6 +43,13 @@ public sealed class SettingsPaths public string InstallationConfigPath() => _paths.FromRoot(ConfigFileName); + /// + /// Where the starter keeps its own copy of the vehicle database. Per user + /// rather than under the installation: that one can be read-only, is a working + /// copy someone updates from under us, and the copy is ours, not the game's. + /// + public string VehicleCacheDirectory() => Path.Combine(UserConfigDirectory(), "vehicles"); + public string UserConfigDirectory() => Path.GetDirectoryName(UserConfigPath()) ?? _environment.BaseDirectory; } diff --git a/StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs b/StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs index 71e5f2f..986a55c 100644 --- a/StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs +++ b/StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using StarterNG.Application.Abstractions; using StarterNG.Domain.Vehicles; @@ -31,9 +32,9 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository catalog.BeginLoad(); int liveries = 0; - foreach (string file in TexturesFiles()) + foreach (string file in Sources()) { - var entry = ReadEntry(file); + var entry = Read(file); if (entry is null) continue; @@ -45,7 +46,27 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository return liveries; } - private VehicleEntry? ReadEntry(string path) + /// + /// Every textures.txt of the installation, in name order. Sorted rather than + /// taken as the file system hands them over, so that which of two folders + /// claiming the same skin wins is the same on every machine. + /// + public IEnumerable Sources() + { + foreach (string maker in Sorted(_files.GetDirectories(_paths.Dynamic))) + foreach (string vehicle in Sorted(_files.GetDirectories(maker))) + { + string path = Path.Combine(vehicle, TexturesFileName); + if (_files.FileExists(path)) + yield return path; + } + } + + private static IEnumerable Sorted(IEnumerable directories) => + directories.OrderBy(Path.GetFileName, StringComparer.OrdinalIgnoreCase); + + /// Reads one textures.txt; a file that will not parse is logged and skipped. + public VehicleEntry? Read(string path) { try { @@ -59,17 +80,6 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository } } - private IEnumerable TexturesFiles() - { - foreach (string maker in _files.GetDirectories(_paths.Dynamic)) - foreach (string vehicle in _files.GetDirectories(maker)) - { - string path = Path.Combine(vehicle, TexturesFileName); - if (_files.FileExists(path)) - yield return path; - } - } - private string RelativeDirectory(string texturesPath) { string directory = Path.GetDirectoryName(texturesPath) ?? ""; diff --git a/StarterNG/Infrastructure/Vehicles/VehicleCacheRepository.cs b/StarterNG/Infrastructure/Vehicles/VehicleCacheRepository.cs new file mode 100644 index 0000000..5befa5a --- /dev/null +++ b/StarterNG/Infrastructure/Vehicles/VehicleCacheRepository.cs @@ -0,0 +1,189 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using StarterNG.Application.Abstractions; +using StarterNG.Domain.Vehicles; + +namespace StarterNG.Infrastructure.Vehicles; + +/// +/// Loads the catalogue from the starter's own JSON copy of the installation, +/// rebuilding that copy from textures.txt whenever the installation has moved on. +/// +/// +/// The copy is a plain merged database, the same format a package is written in, +/// so it can be read by anything that reads the format - and it is a copy, never +/// a dependency: a missing, stale, unreadable or unwritable cache only costs the +/// parse it was meant to save. textures.txt stays the truth. +/// +public sealed class VehicleCacheRepository : IVehicleRepository +{ + private const string DatabaseFileName = "vehicles.json"; + + /// What the copy was built from, and what makes it stale. + private const string SourcesFileName = "sources.txt"; + + private readonly IFileSystem _files; + private readonly IGamePaths _paths; + private readonly IDiagnosticsLog _log; + private readonly TexturesTxtVehicleRepository _legacy; + private readonly VehicleJsonSerializer _serializer; + private readonly string _directory; + + public VehicleCacheRepository(IFileSystem files, IGamePaths paths, IDiagnosticsLog log, + TexturesTxtVehicleRepository legacy, VehicleJsonSerializer serializer, + string directory) + { + _files = files; + _paths = paths; + _log = log; + _legacy = legacy; + _serializer = serializer; + _directory = directory; + } + + /// What the last load did, for the developer tab to report. + public VehicleCacheStatus Status { get; private set; } = VehicleCacheStatus.None; + + public int Load(VehicleCatalog catalog) => Load(catalog, useCopy: true); + + /// Throws the copy away and reads the installation again. + public int Rebuild(VehicleCatalog catalog) => Load(catalog, useCopy: false); + + private int Load(VehicleCatalog catalog, bool useCopy) + { + var (stamp, sources) = Stamp(); + var cached = useCopy ? Cached(stamp) : null; + var entries = cached ?? Migrate(stamp); + + Status = new VehicleCacheStatus(cached is not null, sources, WrittenAt(), _directory); + + catalog.BeginLoad(); + + int liveries = 0; + foreach (var entry in entries) + { + liveries += entry.Textures.Count; + catalog.Ingest(entry); + } + + catalog.EndLoad(); + return liveries; + } + + /// + /// The installation as one comparable line per textures.txt. Every way the + /// installation can move on - a file edited, added, removed, or the whole + /// installation swapped for another one - shows up as a different text. + /// + private (string Text, int Sources) Stamp() + { + var stamp = new StringBuilder().Append("root|").Append(Path.GetFullPath(_paths.Root)).Append('\n'); + int sources = 0; + + foreach (string path in _legacy.Sources()) + { + sources++; + long ticks = 0; + long size = 0; + try + { + ticks = _files.GetLastWriteTimeUtc(path).Ticks; + size = _files.GetFileSize(path); + } + catch (Exception ex) + { + _log.Log($"textures.txt {path}", ex); + } + + stamp.Append(ticks).Append('|').Append(size).Append('|') + .Append(path.Replace('\\', '/')).Append('\n'); + } + + return (stamp.ToString(), sources); + } + + private DateTime? WrittenAt() + { + try + { + return _files.FileExists(DatabasePath) ? _files.GetLastWriteTimeUtc(DatabasePath) : null; + } + catch (Exception ex) + { + _log.Log($"{DatabaseFileName}: nie znam daty kopii", ex); + return null; + } + } + + private List? Cached(string stamp) + { + try + { + if (!_files.FileExists(SourcesPath) || !_files.FileExists(DatabasePath)) + return null; + + if (!string.Equals(_files.ReadAllText(SourcesPath), stamp, StringComparison.Ordinal)) + return null; + + var entries = _serializer.ReadCollection(_files.ReadAllText(DatabasePath)); + foreach (var entry in entries) + if (!VehicleJsonSerializer.IsReadable(entry)) + { + _log.Log($"{DatabaseFileName}: kopia w wersji {entry.SchemaVersion}, " + + $"ten starter czyta do {VehicleJsonSerializer.SchemaVersion} - odtwarzam"); + return null; + } + + return entries; + } + catch (Exception ex) + { + _log.Log($"{DatabaseFileName}: kopia nie do odczytu, odtwarzam", ex); + return null; + } + } + + private List Migrate(string stamp) + { + var entries = new List(); + foreach (string path in _legacy.Sources()) + if (_legacy.Read(path) is { } entry) + entries.Add(entry); + + Store(entries, stamp); + return entries; + } + + /// + /// Writes the copy out. Best effort by design - an installation on a read-only + /// disk, or a home folder we cannot write, still gets its vehicles. + /// + private void Store(List entries, string stamp) + { + try + { + _files.CreateDirectory(_directory); + _files.WriteAllText(DatabasePath, _serializer.WriteCollection(entries)); + _files.WriteAllText(SourcesPath, stamp); + } + catch (Exception ex) + { + _log.Log($"{DatabaseFileName}: kopia nie do zapisania", ex); + } + } + + private string DatabasePath => Path.Combine(_directory, DatabaseFileName); + + private string SourcesPath => Path.Combine(_directory, SourcesFileName); +} + +/// +/// How the catalogue was last filled: from the starter's copy or by reading the +/// installation, over how many textures.txt, and when the copy was last written. +/// +public sealed record VehicleCacheStatus(bool ServedFromCopy, int Sources, DateTime? Written, string Directory) +{ + public static readonly VehicleCacheStatus None = new(false, 0, null, ""); +} From 9d981cb0932c372074eaba25d5c917acbacc25d8 Mon Sep 17 00:00:00 2001 From: maj00r Date: Sun, 4 Oct 2026 12:18:36 +0200 Subject: [PATCH 06/55] Add a developer tab with the state of the vehicle database --- .../Domain/Settings/SimulatorSettings.cs | 1 + .../Settings/SettingsSerializer.cs | 2 + StarterNG/MainWindow.axaml | 5 ++ StarterNG/MainWindow.axaml.cs | 26 ++++++- StarterNG/Views/Developer.axaml | 65 ++++++++++++++++ StarterNG/Views/Developer.axaml.cs | 78 +++++++++++++++++++ StarterNG/Views/Settings.axaml | 3 + StarterNG/Views/Settings.axaml.cs | 19 +++++ StarterNG/startercfg/lang/cz.xml | 18 +++++ StarterNG/startercfg/lang/en.xml | 19 +++++ StarterNG/startercfg/lang/hu.xml | 18 +++++ StarterNG/startercfg/lang/pl.xml | 19 +++++ StarterNG/startercfg/lang/ru.xml | 18 +++++ 13 files changed, 289 insertions(+), 2 deletions(-) create mode 100644 StarterNG/Views/Developer.axaml create mode 100644 StarterNG/Views/Developer.axaml.cs diff --git a/StarterNG/Domain/Settings/SimulatorSettings.cs b/StarterNG/Domain/Settings/SimulatorSettings.cs index a93169e..e1dea3c 100644 --- a/StarterNG/Domain/Settings/SimulatorSettings.cs +++ b/StarterNG/Domain/Settings/SimulatorSettings.cs @@ -35,6 +35,7 @@ public sealed class SimulatorSettings public bool DebugMode; public bool VirtualShunting = true; public bool LogMissingVehicleFiles; + public bool DeveloperTools; public int RenderEngine; public int Width = 1280; diff --git a/StarterNG/Infrastructure/Settings/SettingsSerializer.cs b/StarterNG/Infrastructure/Settings/SettingsSerializer.cs index 498a50f..79260d8 100644 --- a/StarterNG/Infrastructure/Settings/SettingsSerializer.cs +++ b/StarterNG/Infrastructure/Settings/SettingsSerializer.cs @@ -56,6 +56,7 @@ public sealed class SettingsSerializer s.DebugMode = c.GetBool("debugmode", false); s.VirtualShunting = c.GetBool("ai.trainman", true); s.LogMissingVehicleFiles = c.GetBool("starter.logmissingvehicles", false); + s.DeveloperTools = c.GetBool("starter.developertools", false); s.RenderEngine = IndexOf(SimulatorSettings.RenderEngines, c.GetString("gfxrenderer", "full"), 0); s.Width = c.GetInt("width", 1280); @@ -195,6 +196,7 @@ public sealed class SettingsSerializer c.SetBool("debugmode", s.DebugMode); c.SetBool("ai.trainman", s.VirtualShunting); c.SetBool("starter.logmissingvehicles", s.LogMissingVehicleFiles); + c.SetBool("starter.developertools", s.DeveloperTools); c.Set("gfxrenderer", SimulatorSettings.RenderEngines[Clamp(s.RenderEngine, 0, SimulatorSettings.RenderEngines.Length - 1)]); c.SetInt("width", s.Width); diff --git a/StarterNG/MainWindow.axaml b/StarterNG/MainWindow.axaml index 9d2b8bc..c631bd1 100644 --- a/StarterNG/MainWindow.axaml +++ b/StarterNG/MainWindow.axaml @@ -37,6 +37,10 @@ Tag="settings" IsCheckedChanged="Nav_OnCheckedChanged" Content="{Binding [NavSettings], Source={x:Static starterNg:App.Loc}}" /> + +