Keep each file's encoding in the copy written for a start

The copy carries the scenario includes put in place of their include lines,
read as UTF-8 where they are, and it was written all in code page 1250. A name
with a Polish letter in a UTF-8 include then no longer matched the same name in
the files the simulator reads apart, and what 1250 lacks became '?'. Each part
now goes out in the encoding of the file it came from, the scenery's own file
too, as a save of a trainset already did.

StarterNG.Tests/SceneryRepositoryTests.cs

@@ -98,4 +98,25 @@ public class SceneryRepositoryTests
 
         Assert.Null(new SceneryImageLocator(files).Resolve(scenery));
     }
+
+    [Fact]
+    public void The_copy_written_for_a_start_keeps_each_file_in_its_own_encoding()
+    {
+        const string cars =
+            "trainset none droga 0 0\r\n" +
+            "node -1 0 żuk dynamic road\\zuk zuk zuk 0 headdriver 0 0 enddynamic\r\n" +
+            "endtrainset\r\n";
+        var files = new InMemoryFileSystem()
+            .WithLegacyFile(Path.Combine("game", "scenery", "kato.scn"),
+                            "//$n Łódź\r\nFirstInit\r\ninclude auta.scm end\r\n")
+            .WithFile(Path.Combine("game", "scenery", "auta.scm"), cars);
+        var scenery = Assert.Single(Build(files, out _).LoadAll());
+
+        byte[] exported = scenery.BuildExportBytes(skipDecorTrainsets: false, new StubRandom());
+
+        static bool Has(byte[] text, byte[] part) => text.AsSpan().IndexOf(part) >= 0;
+        Assert.True(Has(exported, LegacyText.CodePage1250.GetBytes("//$n Łódź\r\n")));
+        Assert.True(Has(exported, Encoding.UTF8.GetBytes("żuk dynamic")));
+        Assert.False(Has(exported, Encoding.UTF8.GetBytes("Łódź")));
+    }
 }

StarterNG/Application/StartSimulation.cs

@@ -8,7 +8,6 @@ using StarterNG.Domain;
 using StarterNG.Domain.Settings;
 using StarterNG.Domain.Vehicles;
 using StarterNG.Infrastructure;
-using StarterNG.Infrastructure.Adapters;
 using StarterNG.Infrastructure.Settings;
 
 namespace StarterNG.Application;
@@ -132,9 +131,8 @@ public sealed class StartSimulation
         string exportPath = Path.Combine(Path.GetDirectoryName(scenery.Path) ?? "scenery", exportName);
         try
         {
-            _files.WriteAllText(exportPath,
-                                scenery.BuildExportContent(_settings.Settings.IgnoreIrrelevantTrains, _random),
-                                LegacyText.CodePage1250);
+            _files.WriteAllBytes(exportPath,
+                                 scenery.BuildExportBytes(_settings.Settings.IgnoreIrrelevantTrains, _random));
         }
         catch (Exception ex)
         {

StarterNG/Domain/Sceneries/Scenery.cs

@@ -4,6 +4,7 @@ using System.Text;
 using System.Text.RegularExpressions;
 using StarterNG.Application.Abstractions;
 using StarterNG.Classes;
+using StarterNG.Infrastructure.Adapters;
 
 namespace StarterNG.Domain.Sceneries;
 
@@ -67,6 +68,12 @@ public sealed class Scenery
     /// </summary>
     public IReadOnlyDictionary<string, FileStamp?> Sources { get; set; } = new Dictionary<string, FileStamp?>();
 
+    /// <summary>What its own file is written in, for the copy written for a start to be in the same.</summary>
+    public Encoding TextEncoding { get; set; } = LegacyText.CodePage1250;
+
+    /// <summary>What each file put in place of its include is written in, by the name its marker carries.</summary>
+    public IReadOnlyDictionary<string, Encoding> IncludeEncodings { get; set; } = new Dictionary<string, Encoding>();
+
     public string DisplayName => System.IO.Path.GetFileNameWithoutExtension(Path);
 
     public string LocalizedDescription(ISceneryTranslations translations) =>
@@ -99,6 +106,13 @@ public sealed class Scenery
         return result;
     }
 
+    /// <summary>
+    /// <see cref="BuildExportContent"/> as the file to write: each part in the encoding of
+    /// the file it came from, so the simulator gets every word as that file has it.
+    /// </summary>
+    public byte[] BuildExportBytes(bool skipDecorTrainsets, IRandomSource random) =>
+        SceneryIncludes.Encode(BuildExportContent(skipDecorTrainsets, random), TextEncoding, IncludeEncodings);
+
     private string AppendLooseVehicles(string text)
     {
         var sb = new StringBuilder();

StarterNG/Domain/Sceneries/SceneryIncludes.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
+using System.Text;
 using System.Text.RegularExpressions;
 
 namespace StarterNG.Domain.Sceneries;
@@ -75,4 +77,27 @@ public static class SceneryIncludes
 
     public static string? FileAt(IReadOnlyList<(int Start, int End, string File)> ranges, int index) =>
         ranges.FirstOrDefault(r => index >= r.Start && index < r.End).File;
+
+    /// <summary>
+    /// The text as bytes: each inlined file in the encoding it was read in, the rest in
+    /// <paramref name="own"/>. A name both a file put in place and one left to its include
+    /// spell alike then has the same bytes in both, as when the simulator reads them apart.
+    /// </summary>
+    /// <param name="inlined">By the name the marker carries; one not listed goes in <paramref name="own"/>.</param>
+    public static byte[] Encode(string text, Encoding own, IReadOnlyDictionary<string, Encoding> inlined)
+    {
+        using var bytes = new MemoryStream();
+        void Put(Encoding encoding, int start, int end) =>
+            bytes.Write(encoding.GetBytes(text, start, end - start));
+
+        int at = 0;
+        foreach (var (start, end, file) in Ranges(text))
+        {
+            Put(own, at, start);
+            Put(inlined.GetValueOrDefault(file, own), start, end);
+            at = end;
+        }
+        Put(own, at, text.Length);
+        return bytes.ToArray();
+    }
 }

StarterNG/Infrastructure/Adapters/LegacyText.cs

@@ -30,9 +30,12 @@ public static class LegacyText
     /// most files are 1250, and that was an exception per file on every load.
     /// A byte order mark is dropped, or it would stick to the first token.
     /// </summary>
-    public static string Decode(byte[] bytes)
+    public static string Decode(byte[] bytes) => Decode(bytes, out _);
+
+    /// <param name="encoding">What the file is written in, for what is made of it to go back in the same.</param>
+    public static string Decode(byte[] bytes, out Encoding encoding)
     {
-        var (encoding, bom) = Detect(bytes);
+        (encoding, var bom) = Detect(bytes);
         return encoding.GetString(bytes, bom.Length, bytes.Length - bom.Length);
     }
 

StarterNG/Infrastructure/Sceneries/SceneryRepository.cs

@@ -3,6 +3,7 @@ using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using StarterNG.Application.Abstractions;
@@ -73,6 +74,7 @@ public sealed class SceneryRepository : ISceneryRepository
 
             // Looked at ahead of the reading: changed meanwhile, it counts as changed.
             var sources = new Dictionary<string, FileStamp?> { [path] = FileStamp.Of(_files, path) };
+            var encodings = new Dictionary<string, Encoding>(StringComparer.Ordinal);
 
             // With its own .sbt the scenery's terrain comes from it, and the simulator
             // passes its terrain includes over without opening them.
@@ -87,13 +89,21 @@ public sealed class SceneryRepository : ISceneryRepository
                 string? file = ResolveInclude(name);
                 string full = Path.Combine(_paths.Scenery, file ?? name.Replace('\\', '/'));
                 sources[full] = FileStamp.Of(_files, full);
-                return file is null ? null : (file, LegacyText.Decode(_files.ReadAllBytes(full)));
+                if (file is null)
+                    return null;
+
+                string included = LegacyText.Decode(_files.ReadAllBytes(full), out var encoding);
+                encodings[file] = encoding;
+                return (file, included);
             }
 
-            string text = SceneryIncludes.Inline(LegacyText.Decode(_files.ReadAllBytes(path)), ReadInclude);
+            string own = LegacyText.Decode(_files.ReadAllBytes(path), out var ownEncoding);
+            string text = SceneryIncludes.Inline(own, ReadInclude);
             var scenery = _parser.Parse(path, text, FindInclude);
             scenery.HasCompanionTimetable = binaryTerrain;
             scenery.Sources = sources;
+            scenery.TextEncoding = ownEncoding;
+            scenery.IncludeEncodings = encodings;
             return scenery;
         }
         catch (Exception ex)