diff --git a/StarterNG.Tests/Fakes/InMemoryFileSystem.cs b/StarterNG.Tests/Fakes/InMemoryFileSystem.cs index 2dddd26..1a12a86 100644 --- a/StarterNG.Tests/Fakes/InMemoryFileSystem.cs +++ b/StarterNG.Tests/Fakes/InMemoryFileSystem.cs @@ -63,6 +63,8 @@ public sealed class InMemoryFileSystem : IFileSystem public string[] ReadAllLines(string path) => ReadAllText(path).Split('\n').Select(line => line.TrimEnd('\r')).ToArray(); + public byte[] ReadAllBytes(string path) => Bytes(path); + public Stream OpenRead(string path) => new MemoryStream(Bytes(path), writable: false); public void WriteAllText(string path, string contents) => WithFile(path, contents); diff --git a/StarterNG.Tests/LegacyDecodingTests.cs b/StarterNG.Tests/LegacyDecodingTests.cs new file mode 100644 index 0000000..4325bf5 --- /dev/null +++ b/StarterNG.Tests/LegacyDecodingTests.cs @@ -0,0 +1,55 @@ +using System.Text; +using StarterNG.Infrastructure.Adapters; +using StarterNG.Tests.Fakes; + +namespace StarterNG.Tests; + +public class LegacyDecodingTests +{ + private const string Polish = "Zakład_Świętokrzyski_w_Kielcach"; + + [Fact] + public void A_code_page_1250_file_decodes_to_its_letters() + { + Assert.Equal(Polish, LegacyText.Decode(LegacyText.CodePage1250.GetBytes(Polish))); + } + + [Fact] + public void A_file_re_saved_as_utf8_decodes_too() + { + // A handful of textures.txt in the wild are UTF-8; read as 1250 they turn + // "Zakład" into "ZakĹ‚ad". + Assert.Equal(Polish, LegacyText.Decode(Encoding.UTF8.GetBytes(Polish))); + } + + [Fact] + public void Plain_ascii_reads_the_same_either_way() + { + Assert.Equal("PESA_Bydgoszcz", LegacyText.Decode(Encoding.ASCII.GetBytes("PESA_Bydgoszcz"))); + } + + [Fact] + public void A_utf8_textures_file_reaches_the_catalogue_intact() + { + var files = new InMemoryFileSystem().WithFile( + TestInstallation.At("dynamic", "pkp", "elf", "textures.txt"), + "en96-001.mat=34we,EN96,EN96-001 // v2,EN96-001,PREG,Zakład_Świętokrzyski,29.09.2011"); + var installation = new TestInstallation(files); + + installation.Vehicles.Load(installation.Library.Vehicles); + + var meta = Assert.Single(installation.Library.Vehicles.Textures).Meta; + Assert.Equal("Zakład_Świętokrzyski".Replace('_', ' '), meta!.Depot); + } + + [Fact] + public void A_code_page_1250_scenery_reaches_the_list_intact() + { + var files = new InMemoryFileSystem().WithLegacyFile( + TestInstallation.At("scenery", "td.scn"), "//$n Strzebiń - Woźniki Śl."); + + var scenery = Assert.Single(new TestInstallation(files).Sceneries.LoadAll()); + + Assert.Equal("Strzebiń - Woźniki Śl.", scenery.Name); + } +} diff --git a/StarterNG/Application/Abstractions/IFileSystem.cs b/StarterNG/Application/Abstractions/IFileSystem.cs index 93c1083..c49c726 100644 --- a/StarterNG/Application/Abstractions/IFileSystem.cs +++ b/StarterNG/Application/Abstractions/IFileSystem.cs @@ -17,6 +17,8 @@ public interface IFileSystem string[] ReadAllLines(string path); + byte[] ReadAllBytes(string path); + Stream OpenRead(string path); void WriteAllText(string path, string contents); diff --git a/StarterNG/Infrastructure/Adapters/LegacyText.cs b/StarterNG/Infrastructure/Adapters/LegacyText.cs index 991b023..8997cfe 100644 --- a/StarterNG/Infrastructure/Adapters/LegacyText.cs +++ b/StarterNG/Infrastructure/Adapters/LegacyText.cs @@ -21,6 +21,27 @@ public static class LegacyText public static bool IsFallback { get; private set; } + /// + /// Decodes a game file. Most are code page 1250, but a few have been re-saved + /// as UTF-8 over the years, and reading those as 1250 turns "Zakład" into + /// "ZakĹ‚ad". Valid UTF-8 is taken at its word; anything that fails a strict + /// decode is 1250, which cannot fail. + /// + public static string Decode(byte[] bytes) + { + try + { + return Utf8Strict.GetString(bytes); + } + catch (DecoderFallbackException) + { + return CodePage1250.GetString(bytes); + } + } + + private static readonly Encoding Utf8Strict = + new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + private static Encoding Resolve() { try diff --git a/StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs b/StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs index 3c3effe..fb64b2b 100644 --- a/StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs +++ b/StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs @@ -18,6 +18,8 @@ public sealed class PhysicalFileSystem : IFileSystem public string[] ReadAllLines(string path) => File.ReadAllLines(path); + public byte[] ReadAllBytes(string path) => File.ReadAllBytes(path); + public Stream OpenRead(string path) => File.OpenRead(path); public void WriteAllText(string path, string contents) => File.WriteAllText(path, contents); diff --git a/StarterNG/Infrastructure/Sceneries/SceneryRepository.cs b/StarterNG/Infrastructure/Sceneries/SceneryRepository.cs index 5da5dee..cbbde38 100644 --- a/StarterNG/Infrastructure/Sceneries/SceneryRepository.cs +++ b/StarterNG/Infrastructure/Sceneries/SceneryRepository.cs @@ -20,7 +20,6 @@ public sealed class SceneryRepository : ISceneryRepository private readonly IGamePaths _paths; private readonly IDiagnosticsLog _log; private readonly SceneryParser _parser; - private readonly Encoding _encoding; public SceneryRepository(IFileSystem files, IGamePaths paths, IDiagnosticsLog log, SceneryParser parser) { @@ -28,7 +27,6 @@ public sealed class SceneryRepository : ISceneryRepository _paths = paths; _log = log; _parser = parser; - _encoding = LegacyText.CodePage1250; } public IReadOnlyList LoadAll(IProgress? progress = null) @@ -60,7 +58,7 @@ public sealed class SceneryRepository : ISceneryRepository if (!_files.FileExists(path)) return null; - var scenery = _parser.Parse(path, _files.ReadAllText(path, _encoding)); + var scenery = _parser.Parse(path, LegacyText.Decode(_files.ReadAllBytes(path))); scenery.HasCompanionTimetable = _files.FileExists(Path.ChangeExtension(path, ".sbt")); return scenery; } diff --git a/StarterNG/Infrastructure/Vehicles/FizPhysicsRepository.cs b/StarterNG/Infrastructure/Vehicles/FizPhysicsRepository.cs index 52f52eb..36a5a2c 100644 --- a/StarterNG/Infrastructure/Vehicles/FizPhysicsRepository.cs +++ b/StarterNG/Infrastructure/Vehicles/FizPhysicsRepository.cs @@ -25,7 +25,6 @@ public sealed class FizPhysicsRepository : IPhysicsRepository private readonly IFileSystem _files; private readonly IGamePaths _paths; private readonly IDiagnosticsLog _log; - private readonly Encoding _encoding; private readonly ConcurrentDictionary _cache = new(StringComparer.OrdinalIgnoreCase); @@ -38,7 +37,6 @@ public sealed class FizPhysicsRepository : IPhysicsRepository _files = files; _paths = paths; _log = log; - _encoding = LegacyText.CodePage1250; } public void Preload() => Index(); @@ -118,7 +116,7 @@ public sealed class FizPhysicsRepository : IPhysicsRepository if (depth > MaxIncludeDepth || !_files.FileExists(path)) return; - var tokens = Tokenize(_files.ReadAllText(path, _encoding)); + var tokens = Tokenize(LegacyText.Decode(_files.ReadAllBytes(path))); string section = ""; string? pendingKey = null; diff --git a/StarterNG/Infrastructure/Vehicles/LoadWeightsRepository.cs b/StarterNG/Infrastructure/Vehicles/LoadWeightsRepository.cs index 53a4f1f..dea430e 100644 --- a/StarterNG/Infrastructure/Vehicles/LoadWeightsRepository.cs +++ b/StarterNG/Infrastructure/Vehicles/LoadWeightsRepository.cs @@ -44,7 +44,7 @@ public sealed class LoadWeightsRepository try { - return new LoadWeightsTable(ParsePairs(_files.ReadAllText(path, LegacyText.CodePage1250))); + return new LoadWeightsTable(ParsePairs(LegacyText.Decode(_files.ReadAllBytes(path)))); } catch (Exception ex) { diff --git a/StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs b/StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs index 4a9a30d..71e5f2f 100644 --- a/StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs +++ b/StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs @@ -16,7 +16,6 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository private readonly IGamePaths _paths; private readonly IDiagnosticsLog _log; private readonly TexturesTxtParser _parser; - private readonly Encoding _encoding; public TexturesTxtVehicleRepository(IFileSystem files, IGamePaths paths, IDiagnosticsLog log, TexturesTxtParser parser) @@ -25,7 +24,6 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository _paths = paths; _log = log; _parser = parser; - _encoding = LegacyText.CodePage1250; } public int Load(VehicleCatalog catalog) @@ -51,7 +49,7 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository { try { - string[] lines = _files.ReadAllText(path, _encoding).Split('\n'); + string[] lines = LegacyText.Decode(_files.ReadAllBytes(path)).Split('\n'); return _parser.Parse(RelativeDirectory(path), lines); } catch (Exception ex)