From a8f8cb0b7441287b37ab7ab525b25217a6ee7a51 Mon Sep 17 00:00:00 2001 From: maj00r Date: Tue, 3 Nov 2026 21:51:52 +0100 Subject: [PATCH] Keep each file's encoding in the copy written for a start The copy carries the scenario includes put in place of their include lines, read as UTF-8 where they are, and it was written all in code page 1250. A name with a Polish letter in a UTF-8 include then no longer matched the same name in the files the simulator reads apart, and what 1250 lacks became '?'. Each part now goes out in the encoding of the file it came from, the scenery's own file too, as a save of a trainset already did. --- StarterNG.Tests/SceneryRepositoryTests.cs | 21 ++++++++++++++++ StarterNG/Application/StartSimulation.cs | 6 ++--- StarterNG/Domain/Sceneries/Scenery.cs | 14 +++++++++++ StarterNG/Domain/Sceneries/SceneryIncludes.cs | 25 +++++++++++++++++++ .../Infrastructure/Adapters/LegacyText.cs | 7 ++++-- .../Sceneries/SceneryRepository.cs | 14 +++++++++-- 6 files changed, 79 insertions(+), 8 deletions(-) diff --git a/StarterNG.Tests/SceneryRepositoryTests.cs b/StarterNG.Tests/SceneryRepositoryTests.cs index facaa7e..08dbfcb 100644 --- a/StarterNG.Tests/SceneryRepositoryTests.cs +++ b/StarterNG.Tests/SceneryRepositoryTests.cs @@ -98,4 +98,25 @@ public class SceneryRepositoryTests Assert.Null(new SceneryImageLocator(files).Resolve(scenery)); } + + [Fact] + public void The_copy_written_for_a_start_keeps_each_file_in_its_own_encoding() + { + const string cars = + "trainset none droga 0 0\r\n" + + "node -1 0 żuk dynamic road\\zuk zuk zuk 0 headdriver 0 0 enddynamic\r\n" + + "endtrainset\r\n"; + var files = new InMemoryFileSystem() + .WithLegacyFile(Path.Combine("game", "scenery", "kato.scn"), + "//$n Łódź\r\nFirstInit\r\ninclude auta.scm end\r\n") + .WithFile(Path.Combine("game", "scenery", "auta.scm"), cars); + var scenery = Assert.Single(Build(files, out _).LoadAll()); + + byte[] exported = scenery.BuildExportBytes(skipDecorTrainsets: false, new StubRandom()); + + static bool Has(byte[] text, byte[] part) => text.AsSpan().IndexOf(part) >= 0; + Assert.True(Has(exported, LegacyText.CodePage1250.GetBytes("//$n Łódź\r\n"))); + Assert.True(Has(exported, Encoding.UTF8.GetBytes("żuk dynamic"))); + Assert.False(Has(exported, Encoding.UTF8.GetBytes("Łódź"))); + } } diff --git a/StarterNG/Application/StartSimulation.cs b/StarterNG/Application/StartSimulation.cs index 071e5e4..6cacc79 100644 --- a/StarterNG/Application/StartSimulation.cs +++ b/StarterNG/Application/StartSimulation.cs @@ -8,7 +8,6 @@ using StarterNG.Domain; using StarterNG.Domain.Settings; using StarterNG.Domain.Vehicles; using StarterNG.Infrastructure; -using StarterNG.Infrastructure.Adapters; using StarterNG.Infrastructure.Settings; namespace StarterNG.Application; @@ -132,9 +131,8 @@ public sealed class StartSimulation string exportPath = Path.Combine(Path.GetDirectoryName(scenery.Path) ?? "scenery", exportName); try { - _files.WriteAllText(exportPath, - scenery.BuildExportContent(_settings.Settings.IgnoreIrrelevantTrains, _random), - LegacyText.CodePage1250); + _files.WriteAllBytes(exportPath, + scenery.BuildExportBytes(_settings.Settings.IgnoreIrrelevantTrains, _random)); } catch (Exception ex) { diff --git a/StarterNG/Domain/Sceneries/Scenery.cs b/StarterNG/Domain/Sceneries/Scenery.cs index 07b0d32..8865bae 100644 --- a/StarterNG/Domain/Sceneries/Scenery.cs +++ b/StarterNG/Domain/Sceneries/Scenery.cs @@ -4,6 +4,7 @@ using System.Text; using System.Text.RegularExpressions; using StarterNG.Application.Abstractions; using StarterNG.Classes; +using StarterNG.Infrastructure.Adapters; namespace StarterNG.Domain.Sceneries; @@ -67,6 +68,12 @@ public sealed class Scenery /// public IReadOnlyDictionary Sources { get; set; } = new Dictionary(); + /// What its own file is written in, for the copy written for a start to be in the same. + public Encoding TextEncoding { get; set; } = LegacyText.CodePage1250; + + /// What each file put in place of its include is written in, by the name its marker carries. + public IReadOnlyDictionary IncludeEncodings { get; set; } = new Dictionary(); + public string DisplayName => System.IO.Path.GetFileNameWithoutExtension(Path); public string LocalizedDescription(ISceneryTranslations translations) => @@ -99,6 +106,13 @@ public sealed class Scenery return result; } + /// + /// as the file to write: each part in the encoding of + /// the file it came from, so the simulator gets every word as that file has it. + /// + public byte[] BuildExportBytes(bool skipDecorTrainsets, IRandomSource random) => + SceneryIncludes.Encode(BuildExportContent(skipDecorTrainsets, random), TextEncoding, IncludeEncodings); + private string AppendLooseVehicles(string text) { var sb = new StringBuilder(); diff --git a/StarterNG/Domain/Sceneries/SceneryIncludes.cs b/StarterNG/Domain/Sceneries/SceneryIncludes.cs index 08181b5..0f11dda 100644 --- a/StarterNG/Domain/Sceneries/SceneryIncludes.cs +++ b/StarterNG/Domain/Sceneries/SceneryIncludes.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Text; using System.Text.RegularExpressions; namespace StarterNG.Domain.Sceneries; @@ -75,4 +77,27 @@ public static class SceneryIncludes public static string? FileAt(IReadOnlyList<(int Start, int End, string File)> ranges, int index) => ranges.FirstOrDefault(r => index >= r.Start && index < r.End).File; + + /// + /// The text as bytes: each inlined file in the encoding it was read in, the rest in + /// . A name both a file put in place and one left to its include + /// spell alike then has the same bytes in both, as when the simulator reads them apart. + /// + /// By the name the marker carries; one not listed goes in . + public static byte[] Encode(string text, Encoding own, IReadOnlyDictionary inlined) + { + using var bytes = new MemoryStream(); + void Put(Encoding encoding, int start, int end) => + bytes.Write(encoding.GetBytes(text, start, end - start)); + + int at = 0; + foreach (var (start, end, file) in Ranges(text)) + { + Put(own, at, start); + Put(inlined.GetValueOrDefault(file, own), start, end); + at = end; + } + Put(own, at, text.Length); + return bytes.ToArray(); + } } diff --git a/StarterNG/Infrastructure/Adapters/LegacyText.cs b/StarterNG/Infrastructure/Adapters/LegacyText.cs index b7f5b85..2ef32d9 100644 --- a/StarterNG/Infrastructure/Adapters/LegacyText.cs +++ b/StarterNG/Infrastructure/Adapters/LegacyText.cs @@ -30,9 +30,12 @@ public static class LegacyText /// most files are 1250, and that was an exception per file on every load. /// A byte order mark is dropped, or it would stick to the first token. /// - public static string Decode(byte[] bytes) + public static string Decode(byte[] bytes) => Decode(bytes, out _); + + /// What the file is written in, for what is made of it to go back in the same. + public static string Decode(byte[] bytes, out Encoding encoding) { - var (encoding, bom) = Detect(bytes); + (encoding, var bom) = Detect(bytes); return encoding.GetString(bytes, bom.Length, bytes.Length - bom.Length); } diff --git a/StarterNG/Infrastructure/Sceneries/SceneryRepository.cs b/StarterNG/Infrastructure/Sceneries/SceneryRepository.cs index e83d8f7..0aa4008 100644 --- a/StarterNG/Infrastructure/Sceneries/SceneryRepository.cs +++ b/StarterNG/Infrastructure/Sceneries/SceneryRepository.cs @@ -3,6 +3,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; using StarterNG.Application.Abstractions; @@ -73,6 +74,7 @@ public sealed class SceneryRepository : ISceneryRepository // Looked at ahead of the reading: changed meanwhile, it counts as changed. var sources = new Dictionary { [path] = FileStamp.Of(_files, path) }; + var encodings = new Dictionary(StringComparer.Ordinal); // With its own .sbt the scenery's terrain comes from it, and the simulator // passes its terrain includes over without opening them. @@ -87,13 +89,21 @@ public sealed class SceneryRepository : ISceneryRepository string? file = ResolveInclude(name); string full = Path.Combine(_paths.Scenery, file ?? name.Replace('\\', '/')); sources[full] = FileStamp.Of(_files, full); - return file is null ? null : (file, LegacyText.Decode(_files.ReadAllBytes(full))); + if (file is null) + return null; + + string included = LegacyText.Decode(_files.ReadAllBytes(full), out var encoding); + encodings[file] = encoding; + return (file, included); } - string text = SceneryIncludes.Inline(LegacyText.Decode(_files.ReadAllBytes(path)), ReadInclude); + string own = LegacyText.Decode(_files.ReadAllBytes(path), out var ownEncoding); + string text = SceneryIncludes.Inline(own, ReadInclude); var scenery = _parser.Parse(path, text, FindInclude); scenery.HasCompanionTimetable = binaryTerrain; scenery.Sources = sources; + scenery.TextEncoding = ownEncoding; + scenery.IncludeEncodings = encodings; return scenery; } catch (Exception ex)