Add a JSON format for the vehicle database

StarterNG/Application/Abstractions/IGamePaths.cs

@@ -12,6 +12,9 @@ public interface IGamePaths
 
     string Data { get; }
 
+    /// <summary>Where the JSON vehicle database and the packages layered on it live.</summary>
+    string VehicleDatabase { get; }
+
     string StarterConfig { get; }
 
     string DiagnosticsLog { get; }

StarterNG/Domain/Vehicles/VehicleTexture.cs

@@ -5,9 +5,25 @@ namespace StarterNG.Domain.Vehicles;
 public sealed class VehicleEntry
 {
     public string? Uuid { get; set; }
+
+    /// <summary>
+    /// Version of the file format the entry was read from, zero when it came from
+    /// a legacy textures.txt. Kept so a file written by a newer starter is refused
+    /// outright instead of being read with half its meaning missing.
+    /// </summary>
+    public int SchemaVersion { get; set; }
+
     public List<VehicleGroup> Groups { get; } = new();
     public List<VehicleTexture> Textures { get; } = new();
     public List<VehicleSet> Sets { get; } = new();
+
+    /// <summary>
+    /// Source lines the reader did not turn into anything else - legacy headers,
+    /// comments and malformed liveries - kept verbatim so an entry written back
+    /// out does not quietly drop what its author put in the file. Content only:
+    /// blank lines and the original spacing are not preserved.
+    /// </summary>
+    public List<string> Unknown { get; } = new();
 }
 
 public sealed class VehicleGroup
@@ -17,6 +33,13 @@ public sealed class VehicleGroup
     public string? Mini { get; set; }
 
     public bool Archived { get; set; }
+
+    /// <summary>
+    /// True when nothing declared the group and it was inferred from the liveries
+    /// that share a thumbnail - which is every group textures.txt produces, as the
+    /// legacy format has no syntax for declaring one.
+    /// </summary>
+    public bool Implicit { get; set; }
 }
 
 public sealed class VehicleTexture

StarterNG/Infrastructure/Adapters/GamePaths.cs

@@ -22,6 +22,8 @@ public sealed class GamePaths : IGamePaths
 
     public string Data => Path.Combine(Root, "data");
 
+    public string VehicleDatabase => Path.Combine(Root, "databases", "vehicles");
+
     public string StarterConfig => Path.Combine(Root, "starter");
 
     public string DiagnosticsLog => Path.Combine(StarterConfig, "bledy.txt");

StarterNG/Infrastructure/Vehicles/TexturesTxtParser.cs

@@ -25,7 +25,10 @@ public sealed class TexturesTxtParser
 
             char first = line[0];
             if (first is '#' or '@' or '*')
+            {
+                entry.Unknown.Add(line);
                 continue;
+            }
 
             if (line.StartsWith("$a", StringComparison.OrdinalIgnoreCase))
             {
@@ -47,11 +50,17 @@ public sealed class TexturesTxtParser
             }
 
             if (line.IndexOf('=') < 0 || line.StartsWith("//", StringComparison.Ordinal))
+            {
+                entry.Unknown.Add(line);
                 continue;
+            }
 
             var texture = ParseLivery(line, directory, categorySign, archived, entry);
             if (texture is null)
+            {
+                entry.Unknown.Add(line);
                 continue;
+            }
 
             entry.Textures.Add(texture);
 
@@ -120,7 +129,8 @@ public sealed class TexturesTxtParser
                 Id = groupId,
                 Category = categorySign,
                 Mini = lead.Mini,
-                Archived = archived
+                Archived = archived,
+                Implicit = true
             });
 
         var texture = new VehicleTexture

StarterNG/Infrastructure/Vehicles/VehicleJsonDocument.cs

@@ -0,0 +1,124 @@
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+
+namespace StarterNG.Infrastructure.Vehicles;
+
+// The vehicle database as it is written to disk (schema_version 1). Deliberately
+// separate from the domain model: the model is free to change shape with the
+// starter, the file format is not - packages written by someone else have to keep
+// parsing. The property names are the ones the format shipped with, so a file
+// written for the older reader still loads.
+//
+// Collections are nullable and left out when empty, which keeps a generated file
+// close to what a person would have typed by hand.
+
+public sealed class VehicleEntryDocument
+{
+    [JsonPropertyName("uuid")] public string? Uuid { get; set; }
+
+    [JsonPropertyName("schema_version")] public int SchemaVersion { get; set; }
+
+    [JsonPropertyName("groups")] public List<VehicleGroupDocument>? Groups { get; set; }
+
+    [JsonPropertyName("textures")] public List<VehicleTextureDocument>? Textures { get; set; }
+
+    [JsonPropertyName("sets")] public List<VehicleSetDocument>? Sets { get; set; }
+
+    [JsonPropertyName("unknown")] public List<string>? Unknown { get; set; }
+}
+
+/// <summary>Root of a merged database holding many entries in one file.</summary>
+public sealed class VehicleCollectionDocument
+{
+    [JsonPropertyName("vehicles")] public List<VehicleEntryDocument>? Vehicles { get; set; }
+}
+
+public sealed class VehicleGroupDocument
+{
+    [JsonPropertyName("id")] public string? Id { get; set; }
+
+    [JsonPropertyName("category")] public string? Category { get; set; }
+
+    [JsonPropertyName("mini")] public string? Mini { get; set; }
+
+    [JsonPropertyName("archived")] public bool Archived { get; set; }
+
+    [JsonPropertyName("implicit")] public bool Implicit { get; set; }
+}
+
+public sealed class VehicleTextureDocument
+{
+    [JsonPropertyName("uuid")] public string? Uuid { get; set; }
+
+    [JsonPropertyName("directory")] public string? Directory { get; set; }
+
+    [JsonPropertyName("skinfile")] public string? Skinfile { get; set; }
+
+    [JsonPropertyName("model")] public string? Model { get; set; }
+
+    [JsonPropertyName("group")] public string? Group { get; set; }
+
+    [JsonPropertyName("mini_ref")] public string? MiniRef { get; set; }
+
+    [JsonPropertyName("texture_mini")] public string? TextureMini { get; set; }
+
+    [JsonPropertyName("wreck")] public bool Wreck { get; set; }
+
+    [JsonPropertyName("aliases")] public List<VehicleAliasDocument>? Aliases { get; set; }
+
+    [JsonPropertyName("meta")] public VehicleMetaDocument? Meta { get; set; }
+}
+
+public sealed class VehicleAliasDocument
+{
+    [JsonPropertyName("model")] public string? Model { get; set; }
+
+    [JsonPropertyName("group")] public string? Group { get; set; }
+
+    [JsonPropertyName("mini_ref")] public string? MiniRef { get; set; }
+
+    [JsonPropertyName("texture_mini")] public string? TextureMini { get; set; }
+}
+
+public sealed class VehicleMetaDocument
+{
+    [JsonPropertyName("raw")] public string? Raw { get; set; }
+
+    [JsonPropertyName("version")] public string? Version { get; set; }
+
+    [JsonPropertyName("vehicle")] public string? Vehicle { get; set; }
+
+    [JsonPropertyName("operator")] public string? Operator { get; set; }
+
+    [JsonPropertyName("depot")] public string? Depot { get; set; }
+
+    [JsonPropertyName("revision_date")] public string? RevisionDate { get; set; }
+
+    [JsonPropertyName("revision_place")] public string? RevisionPlace { get; set; }
+
+    [JsonPropertyName("texture_author")] public string? TextureAuthor { get; set; }
+
+    [JsonPropertyName("photo_author")] public string? PhotoAuthor { get; set; }
+
+    [JsonPropertyName("extra")] public List<string>? Extra { get; set; }
+}
+
+public sealed class VehicleSetDocument
+{
+    [JsonPropertyName("uuid")] public string? Uuid { get; set; }
+
+    [JsonPropertyName("mode")] public string? Mode { get; set; }
+
+    [JsonPropertyName("count")] public int Count { get; set; }
+
+    [JsonPropertyName("texture_refs")] public List<string>? TextureRefs { get; set; }
+}
+
+// Source-generated metadata: PublishAot and PublishTrimmed rule out the reflection
+// based serializer. The generator pulls in every nested document type on its own,
+// and the reader's leniency is set on the options in VehicleJsonSerializer.
+[JsonSerializable(typeof(VehicleEntryDocument))]
+[JsonSerializable(typeof(VehicleCollectionDocument))]
+internal partial class VehicleJsonContext : JsonSerializerContext
+{
+}

StarterNG/Infrastructure/Vehicles/VehicleJsonRepository.cs

@@ -0,0 +1,101 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using StarterNG.Application.Abstractions;
+using StarterNG.Domain.Vehicles;
+
+namespace StarterNG.Infrastructure.Vehicles;
+
+/// <summary>
+/// Reads the vehicle catalogue from databases/vehicles: a merged vehicles.json
+/// holding the whole database, and one file per package layered on top of it.
+/// </summary>
+public sealed class VehicleJsonRepository : IVehicleRepository
+{
+    private const string MergedFileName = "vehicles.json";
+
+    private readonly IFileSystem _files;
+    private readonly IGamePaths _paths;
+    private readonly IDiagnosticsLog _log;
+    private readonly VehicleJsonSerializer _serializer;
+
+    public VehicleJsonRepository(IFileSystem files, IGamePaths paths, IDiagnosticsLog log,
+                                 VehicleJsonSerializer serializer)
+    {
+        _files = files;
+        _paths = paths;
+        _log = log;
+        _serializer = serializer;
+    }
+
+    public int Load(VehicleCatalog catalog)
+    {
+        catalog.BeginLoad();
+
+        int liveries = 0;
+        foreach (string file in DatabaseFiles())
+            liveries += ReadInto(catalog, file);
+
+        catalog.EndLoad();
+        return liveries;
+    }
+
+    private int ReadInto(VehicleCatalog catalog, string path)
+    {
+        try
+        {
+            string json = _files.ReadAllText(path);
+            var entries = IsMerged(path)
+                ? _serializer.ReadCollection(json)
+                : Listed(_serializer.Read(json));
+
+            int liveries = 0;
+            foreach (var entry in entries)
+            {
+                if (!VehicleJsonSerializer.IsReadable(entry))
+                {
+                    _log.Log($"{Name(path)}: format w wersji {entry.SchemaVersion}, " +
+                             $"ten starter czyta do {VehicleJsonSerializer.SchemaVersion}");
+                    continue;
+                }
+
+                liveries += entry.Textures.Count;
+                catalog.Ingest(entry);
+            }
+
+            return liveries;
+        }
+        catch (Exception ex)
+        {
+            _log.Log(Name(path), ex);
+            return 0;
+        }
+    }
+
+    /// <summary>
+    /// The merged database first and the packages after it, both in a fixed order,
+    /// so which entry wins a clash is the same on every machine - the file system
+    /// does not promise one.
+    /// </summary>
+    private IEnumerable<string> DatabaseFiles()
+    {
+        var files = _files.GetFiles(_paths.VehicleDatabase, "*.json")
+                          .OrderBy(Path.GetFileName, StringComparer.OrdinalIgnoreCase)
+                          .ToList();
+
+        foreach (string file in files.Where(IsMerged))
+            yield return file;
+
+        foreach (string file in files.Where(file => !IsMerged(file)))
+            yield return file;
+    }
+
+    private static bool IsMerged(string path) =>
+        string.Equals(Path.GetFileName(path), MergedFileName, StringComparison.OrdinalIgnoreCase);
+
+    private static string Name(string path) => $"databases/vehicles/{Path.GetFileName(path)}";
+
+    private static List<VehicleEntry> Listed(VehicleEntry? entry) =>
+        entry is null ? new List<VehicleEntry>() : new List<VehicleEntry> { entry };
+}

StarterNG/Infrastructure/Vehicles/VehicleJsonSerializer.cs

@@ -0,0 +1,215 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Encodings.Web;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using System.Text.Json.Serialization.Metadata;
+using StarterNG.Domain.Vehicles;
+
+namespace StarterNG.Infrastructure.Vehicles;
+
+/// <summary>
+/// Reads and writes <see cref="VehicleEntry"/> in the JSON database format,
+/// mapping it on and off the document types that define the format.
+/// </summary>
+public sealed class VehicleJsonSerializer
+{
+    /// <summary>The format this build writes, and the newest one it can read.</summary>
+    public const int SchemaVersion = 1;
+
+    // Leniency a hand-written package file deserves: names matched case
+    // insensitively, // and /* */ comments skipped, trailing commas allowed. The
+    // relaxed encoder keeps Polish text in the credits and in the preserved legacy
+    // comments readable instead of turning it into \uXXXX escapes, and the
+    // source-generated resolver keeps all of it AOT-safe.
+    private static readonly JsonSerializerOptions Options = new()
+    {
+        WriteIndented = true,
+        PropertyNameCaseInsensitive = true,
+        ReadCommentHandling = JsonCommentHandling.Skip,
+        AllowTrailingCommas = true,
+        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        TypeInfoResolver = VehicleJsonContext.Default
+    };
+
+    private static readonly JsonTypeInfo<VehicleEntryDocument> EntryInfo =
+        (JsonTypeInfo<VehicleEntryDocument>)Options.GetTypeInfo(typeof(VehicleEntryDocument));
+
+    private static readonly JsonTypeInfo<VehicleCollectionDocument> CollectionInfo =
+        (JsonTypeInfo<VehicleCollectionDocument>)Options.GetTypeInfo(typeof(VehicleCollectionDocument));
+
+    public string Write(VehicleEntry entry) => JsonSerializer.Serialize(ToDocument(entry), EntryInfo);
+
+    public string WriteCollection(IEnumerable<VehicleEntry> entries) =>
+        JsonSerializer.Serialize(
+            new VehicleCollectionDocument { Vehicles = entries.Select(ToDocument).ToList() },
+            CollectionInfo);
+
+    /// <summary>One entry per file, the form a package is authored in.</summary>
+    public VehicleEntry? Read(string json)
+    {
+        var document = JsonSerializer.Deserialize(json, EntryInfo);
+        return document is null ? null : FromDocument(document);
+    }
+
+    /// <summary>Many entries in one file, the form a whole database is shipped in.</summary>
+    public List<VehicleEntry> ReadCollection(string json)
+    {
+        var document = JsonSerializer.Deserialize(json, CollectionInfo);
+        if (document?.Vehicles is null)
+            return new List<VehicleEntry>();
+
+        return document.Vehicles.Select(FromDocument).ToList();
+    }
+
+    /// <summary>
+    /// Whether this build understands the file. An entry written by a newer
+    /// starter is refused rather than half-read, an entry with no version at all
+    /// is taken as the current one - the format carried no version when the first
+    /// files were written by hand.
+    /// </summary>
+    public static bool IsReadable(VehicleEntry entry) => entry.SchemaVersion <= SchemaVersion;
+
+    private static VehicleEntryDocument ToDocument(VehicleEntry entry) =>
+        new()
+        {
+            Uuid = entry.Uuid,
+            SchemaVersion = SchemaVersion,
+            Groups = OrNull(entry.Groups.Select(group => new VehicleGroupDocument
+            {
+                Id = group.Id,
+                Category = group.Category,
+                Mini = group.Mini,
+                Archived = group.Archived,
+                Implicit = group.Implicit
+            }).ToList()),
+            Textures = OrNull(entry.Textures.Select(ToDocument).ToList()),
+            Sets = OrNull(entry.Sets.Select(set => new VehicleSetDocument
+            {
+                Uuid = set.Uuid,
+                Mode = set.Mode,
+                Count = set.Count,
+                TextureRefs = OrNull(set.TextureRefs)
+            }).ToList()),
+            Unknown = OrNull(entry.Unknown)
+        };
+
+    private static VehicleTextureDocument ToDocument(VehicleTexture texture) =>
+        new()
+        {
+            Uuid = texture.Uuid,
+            Directory = texture.Directory,
+            Skinfile = texture.Skinfile,
+            Model = texture.Model,
+            Group = texture.Group,
+            MiniRef = texture.MiniRef,
+            TextureMini = texture.TextureMini,
+            Wreck = texture.Wreck,
+            Aliases = OrNull(texture.Aliases.Select(alias => new VehicleAliasDocument
+            {
+                Model = alias.Model,
+                Group = alias.Group,
+                MiniRef = alias.MiniRef,
+                TextureMini = alias.TextureMini
+            }).ToList()),
+            Meta = texture.Meta is null
+                ? null
+                : new VehicleMetaDocument
+                {
+                    Raw = texture.Meta.Raw,
+                    Version = texture.Meta.Version,
+                    Vehicle = texture.Meta.Vehicle,
+                    Operator = texture.Meta.Operator,
+                    Depot = texture.Meta.Depot,
+                    RevisionDate = texture.Meta.RevisionDate,
+                    RevisionPlace = texture.Meta.RevisionPlace,
+                    TextureAuthor = texture.Meta.TextureAuthor,
+                    PhotoAuthor = texture.Meta.PhotoAuthor,
+                    Extra = OrNull(texture.Meta.Extra)
+                }
+        };
+
+    private static VehicleEntry FromDocument(VehicleEntryDocument document)
+    {
+        var entry = new VehicleEntry
+        {
+            Uuid = document.Uuid,
+            SchemaVersion = document.SchemaVersion == 0 ? SchemaVersion : document.SchemaVersion
+        };
+
+        foreach (var group in document.Groups ?? Enumerable.Empty<VehicleGroupDocument>())
+            entry.Groups.Add(new VehicleGroup
+            {
+                Id = group.Id ?? "",
+                Category = group.Category,
+                Mini = group.Mini,
+                Archived = group.Archived,
+                Implicit = group.Implicit
+            });
+
+        foreach (var texture in document.Textures ?? Enumerable.Empty<VehicleTextureDocument>())
+            entry.Textures.Add(FromDocument(texture));
+
+        foreach (var set in document.Sets ?? Enumerable.Empty<VehicleSetDocument>())
+            entry.Sets.Add(new VehicleSet
+            {
+                Uuid = set.Uuid,
+                Mode = set.Mode,
+                Count = set.Count,
+                TextureRefs = set.TextureRefs ?? new List<string>()
+            });
+
+        if (document.Unknown is not null)
+            entry.Unknown.AddRange(document.Unknown);
+
+        return entry;
+    }
+
+    private static VehicleTexture FromDocument(VehicleTextureDocument document)
+    {
+        var texture = new VehicleTexture
+        {
+            Uuid = document.Uuid,
+            Directory = document.Directory ?? "",
+            Skinfile = document.Skinfile ?? "",
+            Model = document.Model,
+            Group = document.Group,
+            MiniRef = document.MiniRef,
+            TextureMini = document.TextureMini,
+            Wreck = document.Wreck
+        };
+
+        foreach (var alias in document.Aliases ?? Enumerable.Empty<VehicleAliasDocument>())
+            texture.Aliases.Add(new VehicleAlias
+            {
+                Model = alias.Model,
+                Group = alias.Group,
+                MiniRef = alias.MiniRef,
+                TextureMini = alias.TextureMini
+            });
+
+        if (document.Meta is not null)
+        {
+            texture.Meta = new VehicleMeta
+            {
+                Raw = document.Meta.Raw,
+                Version = document.Meta.Version,
+                Vehicle = document.Meta.Vehicle,
+                Operator = document.Meta.Operator,
+                Depot = document.Meta.Depot,
+                RevisionDate = document.Meta.RevisionDate,
+                RevisionPlace = document.Meta.RevisionPlace,
+                TextureAuthor = document.Meta.TextureAuthor,
+                PhotoAuthor = document.Meta.PhotoAuthor
+            };
+
+            if (document.Meta.Extra is not null)
+                texture.Meta.Extra.AddRange(document.Meta.Extra);
+        }
+
+        return texture;
+    }
+
+    private static List<T>? OrNull<T>(List<T> items) => items.Count > 0 ? items : null;
+}