fallback decode utf8 strict to win 1250

StarterNG.Tests/Fakes/InMemoryFileSystem.cs

@@ -63,6 +63,8 @@ public sealed class InMemoryFileSystem : IFileSystem
     public string[] ReadAllLines(string path) =>
         ReadAllText(path).Split('\n').Select(line => line.TrimEnd('\r')).ToArray();
 
+    public byte[] ReadAllBytes(string path) => Bytes(path);
+
     public Stream OpenRead(string path) => new MemoryStream(Bytes(path), writable: false);
 
     public void WriteAllText(string path, string contents) => WithFile(path, contents);

StarterNG.Tests/LegacyDecodingTests.cs

@@ -0,0 +1,55 @@
+using System.Text;
+using StarterNG.Infrastructure.Adapters;
+using StarterNG.Tests.Fakes;
+
+namespace StarterNG.Tests;
+
+public class LegacyDecodingTests
+{
+    private const string Polish = "Zakład_Świętokrzyski_w_Kielcach";
+
+    [Fact]
+    public void A_code_page_1250_file_decodes_to_its_letters()
+    {
+        Assert.Equal(Polish, LegacyText.Decode(LegacyText.CodePage1250.GetBytes(Polish)));
+    }
+
+    [Fact]
+    public void A_file_re_saved_as_utf8_decodes_too()
+    {
+        // A handful of textures.txt in the wild are UTF-8; read as 1250 they turn
+        // "Zakład" into "ZakĹ‚ad".
+        Assert.Equal(Polish, LegacyText.Decode(Encoding.UTF8.GetBytes(Polish)));
+    }
+
+    [Fact]
+    public void Plain_ascii_reads_the_same_either_way()
+    {
+        Assert.Equal("PESA_Bydgoszcz", LegacyText.Decode(Encoding.ASCII.GetBytes("PESA_Bydgoszcz")));
+    }
+
+    [Fact]
+    public void A_utf8_textures_file_reaches_the_catalogue_intact()
+    {
+        var files = new InMemoryFileSystem().WithFile(
+            TestInstallation.At("dynamic", "pkp", "elf", "textures.txt"),
+            "en96-001.mat=34we,EN96,EN96-001 // v2,EN96-001,PREG,Zakład_Świętokrzyski,29.09.2011");
+        var installation = new TestInstallation(files);
+
+        installation.Vehicles.Load(installation.Library.Vehicles);
+
+        var meta = Assert.Single(installation.Library.Vehicles.Textures).Meta;
+        Assert.Equal("Zakład_Świętokrzyski".Replace('_', ' '), meta!.Depot);
+    }
+
+    [Fact]
+    public void A_code_page_1250_scenery_reaches_the_list_intact()
+    {
+        var files = new InMemoryFileSystem().WithLegacyFile(
+            TestInstallation.At("scenery", "td.scn"), "//$n Strzebiń - Woźniki Śl.");
+
+        var scenery = Assert.Single(new TestInstallation(files).Sceneries.LoadAll());
+
+        Assert.Equal("Strzebiń - Woźniki Śl.", scenery.Name);
+    }
+}

StarterNG/Application/Abstractions/IFileSystem.cs

@@ -17,6 +17,8 @@ public interface IFileSystem
 
     string[] ReadAllLines(string path);
 
+    byte[] ReadAllBytes(string path);
+
     Stream OpenRead(string path);
 
     void WriteAllText(string path, string contents);

StarterNG/Infrastructure/Adapters/LegacyText.cs

@@ -21,6 +21,27 @@ public static class LegacyText
 
     public static bool IsFallback { get; private set; }
 
+    /// <summary>
+    /// Decodes a game file. Most are code page 1250, but a few have been re-saved
+    /// as UTF-8 over the years, and reading those as 1250 turns "Zakład" into
+    /// "Zakład". Valid UTF-8 is taken at its word; anything that fails a strict
+    /// decode is 1250, which cannot fail.
+    /// </summary>
+    public static string Decode(byte[] bytes)
+    {
+        try
+        {
+            return Utf8Strict.GetString(bytes);
+        }
+        catch (DecoderFallbackException)
+        {
+            return CodePage1250.GetString(bytes);
+        }
+    }
+
+    private static readonly Encoding Utf8Strict =
+        new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
+
     private static Encoding Resolve()
     {
         try

StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs

@@ -18,6 +18,8 @@ public sealed class PhysicalFileSystem : IFileSystem
 
     public string[] ReadAllLines(string path) => File.ReadAllLines(path);
 
+    public byte[] ReadAllBytes(string path) => File.ReadAllBytes(path);
+
     public Stream OpenRead(string path) => File.OpenRead(path);
 
     public void WriteAllText(string path, string contents) => File.WriteAllText(path, contents);

StarterNG/Infrastructure/Sceneries/SceneryRepository.cs

@@ -20,7 +20,6 @@ public sealed class SceneryRepository : ISceneryRepository
     private readonly IGamePaths _paths;
     private readonly IDiagnosticsLog _log;
     private readonly SceneryParser _parser;
-    private readonly Encoding _encoding;
 
     public SceneryRepository(IFileSystem files, IGamePaths paths, IDiagnosticsLog log, SceneryParser parser)
     {
@@ -28,7 +27,6 @@ public sealed class SceneryRepository : ISceneryRepository
         _paths = paths;
         _log = log;
         _parser = parser;
-        _encoding = LegacyText.CodePage1250;
     }
 
     public IReadOnlyList<Scenery> LoadAll(IProgress<SceneryLoadProgress>? progress = null)
@@ -60,7 +58,7 @@ public sealed class SceneryRepository : ISceneryRepository
             if (!_files.FileExists(path))
                 return null;
 
-            var scenery = _parser.Parse(path, _files.ReadAllText(path, _encoding));
+            var scenery = _parser.Parse(path, LegacyText.Decode(_files.ReadAllBytes(path)));
             scenery.HasCompanionTimetable = _files.FileExists(Path.ChangeExtension(path, ".sbt"));
             return scenery;
         }

StarterNG/Infrastructure/Vehicles/FizPhysicsRepository.cs

@@ -25,7 +25,6 @@ public sealed class FizPhysicsRepository : IPhysicsRepository
     private readonly IFileSystem _files;
     private readonly IGamePaths _paths;
     private readonly IDiagnosticsLog _log;
-    private readonly Encoding _encoding;
 
     private readonly ConcurrentDictionary<string, VehiclePhysics?> _cache =
         new(StringComparer.OrdinalIgnoreCase);
@@ -38,7 +37,6 @@ public sealed class FizPhysicsRepository : IPhysicsRepository
         _files = files;
         _paths = paths;
         _log = log;
-        _encoding = LegacyText.CodePage1250;
     }
 
     public void Preload() => Index();
@@ -118,7 +116,7 @@ public sealed class FizPhysicsRepository : IPhysicsRepository
         if (depth > MaxIncludeDepth || !_files.FileExists(path))
             return;
 
-        var tokens = Tokenize(_files.ReadAllText(path, _encoding));
+        var tokens = Tokenize(LegacyText.Decode(_files.ReadAllBytes(path)));
 
         string section = "";
         string? pendingKey = null;

StarterNG/Infrastructure/Vehicles/LoadWeightsRepository.cs

@@ -44,7 +44,7 @@ public sealed class LoadWeightsRepository
 
         try
         {
-            return new LoadWeightsTable(ParsePairs(_files.ReadAllText(path, LegacyText.CodePage1250)));
+            return new LoadWeightsTable(ParsePairs(LegacyText.Decode(_files.ReadAllBytes(path))));
         }
         catch (Exception ex)
         {

StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs

@@ -16,7 +16,6 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository
     private readonly IGamePaths _paths;
     private readonly IDiagnosticsLog _log;
     private readonly TexturesTxtParser _parser;
-    private readonly Encoding _encoding;
 
     public TexturesTxtVehicleRepository(IFileSystem files, IGamePaths paths, IDiagnosticsLog log,
                                         TexturesTxtParser parser)
@@ -25,7 +24,6 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository
         _paths = paths;
         _log = log;
         _parser = parser;
-        _encoding = LegacyText.CodePage1250;
     }
 
     public int Load(VehicleCatalog catalog)
@@ -51,7 +49,7 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository
     {
         try
         {
-            string[] lines = _files.ReadAllText(path, _encoding).Split('\n');
+            string[] lines = LegacyText.Decode(_files.ReadAllBytes(path)).Split('\n');
             return _parser.Parse(RelativeDirectory(path), lines);
         }
         catch (Exception ex)