eu07/StarterNG
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/StarterNG.git synced 2026-10-16 23:49:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

fallback decode utf8 strict to win 1250

Browse Source
This commit is contained in:
maj00r
2026-09-04 23:50:41 +02:00
parent f7219cf705
commit c492c26047
9 changed files with 86 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
StarterNG.Tests/Fakes/InMemoryFileSystem.cs
View File

@@ -63,6 +63,8 @@ public sealed class InMemoryFileSystem : IFileSystem
public string[] ReadAllLines(string path) =>
ReadAllText(path).Split('\n').Select(line => line.TrimEnd('\r')).ToArray();
public byte[] ReadAllBytes(string path) => Bytes(path);
public Stream OpenRead(string path) => new MemoryStream(Bytes(path), writable: false);
public void WriteAllText(string path, string contents) => WithFile(path, contents);

55
StarterNG.Tests/LegacyDecodingTests.cs Normal file
View File

@@ -0,0 +1,55 @@
using System.Text;
using StarterNG.Infrastructure.Adapters;
using StarterNG.Tests.Fakes;
namespace StarterNG.Tests;
public class LegacyDecodingTests
{
private const string Polish = "Zakład_Świętokrzyski_w_Kielcach";
[Fact]
public void A_code_page_1250_file_decodes_to_its_letters()
{
Assert.Equal(Polish, LegacyText.Decode(LegacyText.CodePage1250.GetBytes(Polish)));
}
[Fact]
public void A_file_re_saved_as_utf8_decodes_too()
{
// A handful of textures.txt in the wild are UTF-8; read as 1250 they turn
// "Zakład" into "ZakĹ‚ad".
Assert.Equal(Polish, LegacyText.Decode(Encoding.UTF8.GetBytes(Polish)));
}
[Fact]
public void Plain_ascii_reads_the_same_either_way()
{
Assert.Equal("PESA_Bydgoszcz", LegacyText.Decode(Encoding.ASCII.GetBytes("PESA_Bydgoszcz")));
}
[Fact]
public void A_utf8_textures_file_reaches_the_catalogue_intact()
{
var files = new InMemoryFileSystem().WithFile(
TestInstallation.At("dynamic", "pkp", "elf", "textures.txt"),
"en96-001.mat=34we,EN96,EN96-001 // v2,EN96-001,PREG,Zakład_Świętokrzyski,29.09.2011");
var installation = new TestInstallation(files);
installation.Vehicles.Load(installation.Library.Vehicles);
var meta = Assert.Single(installation.Library.Vehicles.Textures).Meta;
Assert.Equal("Zakład_Świętokrzyski".Replace('_', ' '), meta!.Depot);
}
[Fact]
public void A_code_page_1250_scenery_reaches_the_list_intact()
{
var files = new InMemoryFileSystem().WithLegacyFile(
TestInstallation.At("scenery", "td.scn"), "//$n Strzebiń - Woźniki Śl.");
var scenery = Assert.Single(new TestInstallation(files).Sceneries.LoadAll());
Assert.Equal("Strzebiń - Woźniki Śl.", scenery.Name);
}
}

2
StarterNG/Application/Abstractions/IFileSystem.cs
View File

@@ -17,6 +17,8 @@ public interface IFileSystem
string[] ReadAllLines(string path);
byte[] ReadAllBytes(string path);
Stream OpenRead(string path);
void WriteAllText(string path, string contents);

21
StarterNG/Infrastructure/Adapters/LegacyText.cs
View File

@@ -21,6 +21,27 @@ public static class LegacyText
public static bool IsFallback { get; private set; }
/// <summary>
/// Decodes a game file. Most are code page 1250, but a few have been re-saved
/// as UTF-8 over the years, and reading those as 1250 turns "Zakład" into
/// "Zakład". Valid UTF-8 is taken at its word; anything that fails a strict
/// decode is 1250, which cannot fail.
/// </summary>
public static string Decode(byte[] bytes)
{
try
{
return Utf8Strict.GetString(bytes);
}
catch (DecoderFallbackException)
{
return CodePage1250.GetString(bytes);
}
}
private static readonly Encoding Utf8Strict =
new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
private static Encoding Resolve()
{
try

2
StarterNG/Infrastructure/Adapters/PhysicalFileSystem.cs
View File

@@ -18,6 +18,8 @@ public sealed class PhysicalFileSystem : IFileSystem
public string[] ReadAllLines(string path) => File.ReadAllLines(path);
public byte[] ReadAllBytes(string path) => File.ReadAllBytes(path);
public Stream OpenRead(string path) => File.OpenRead(path);
public void WriteAllText(string path, string contents) => File.WriteAllText(path, contents);

4
StarterNG/Infrastructure/Sceneries/SceneryRepository.cs
View File

@@ -20,7 +20,6 @@ public sealed class SceneryRepository : ISceneryRepository
private readonly IGamePaths _paths;
private readonly IDiagnosticsLog _log;
private readonly SceneryParser _parser;
private readonly Encoding _encoding;
public SceneryRepository(IFileSystem files, IGamePaths paths, IDiagnosticsLog log, SceneryParser parser)
{
@@ -28,7 +27,6 @@ public sealed class SceneryRepository : ISceneryRepository
_paths = paths;
_log = log;
_parser = parser;
_encoding = LegacyText.CodePage1250;
}
public IReadOnlyList<Scenery> LoadAll(IProgress<SceneryLoadProgress>? progress = null)
@@ -60,7 +58,7 @@ public sealed class SceneryRepository : ISceneryRepository
if (!_files.FileExists(path))
return null;
var scenery = _parser.Parse(path, _files.ReadAllText(path, _encoding));
var scenery = _parser.Parse(path, LegacyText.Decode(_files.ReadAllBytes(path)));
scenery.HasCompanionTimetable = _files.FileExists(Path.ChangeExtension(path, ".sbt"));
return scenery;
}

4
StarterNG/Infrastructure/Vehicles/FizPhysicsRepository.cs
View File

@@ -25,7 +25,6 @@ public sealed class FizPhysicsRepository : IPhysicsRepository
private readonly IFileSystem _files;
private readonly IGamePaths _paths;
private readonly IDiagnosticsLog _log;
private readonly Encoding _encoding;
private readonly ConcurrentDictionary<string, VehiclePhysics?> _cache =
new(StringComparer.OrdinalIgnoreCase);
@@ -38,7 +37,6 @@ public sealed class FizPhysicsRepository : IPhysicsRepository
_files = files;
_paths = paths;
_log = log;
_encoding = LegacyText.CodePage1250;
}
public void Preload() => Index();
@@ -118,7 +116,7 @@ public sealed class FizPhysicsRepository : IPhysicsRepository
if (depth > MaxIncludeDepth || !_files.FileExists(path))
return;
var tokens = Tokenize(_files.ReadAllText(path, _encoding));
var tokens = Tokenize(LegacyText.Decode(_files.ReadAllBytes(path)));
string section = "";
string? pendingKey = null;

2
StarterNG/Infrastructure/Vehicles/LoadWeightsRepository.cs
View File

@@ -44,7 +44,7 @@ public sealed class LoadWeightsRepository
try
{
return new LoadWeightsTable(ParsePairs(_files.ReadAllText(path, LegacyText.CodePage1250)));
return new LoadWeightsTable(ParsePairs(LegacyText.Decode(_files.ReadAllBytes(path))));
}
catch (Exception ex)
{

4
StarterNG/Infrastructure/Vehicles/TexturesTxtVehicleRepository.cs
View File

@@ -16,7 +16,6 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository
private readonly IGamePaths _paths;
private readonly IDiagnosticsLog _log;
private readonly TexturesTxtParser _parser;
private readonly Encoding _encoding;
public TexturesTxtVehicleRepository(IFileSystem files, IGamePaths paths, IDiagnosticsLog log,
TexturesTxtParser parser)
@@ -25,7 +24,6 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository
_paths = paths;
_log = log;
_parser = parser;
_encoding = LegacyText.CodePage1250;
}
public int Load(VehicleCatalog catalog)
@@ -51,7 +49,7 @@ public sealed class TexturesTxtVehicleRepository : IVehicleRepository
{
try
{
string[] lines = _files.ReadAllText(path, _encoding).Split('\n');
string[] lines = LegacyText.Decode(_files.ReadAllBytes(path)).Split('\n');
return _parser.Parse(RelativeDirectory(path), lines);
}
catch (Exception ex)