diff --git a/StarterNG.Tests/LegacyDecodingTests.cs b/StarterNG.Tests/LegacyDecodingTests.cs index 4325bf5..c37e438 100644 --- a/StarterNG.Tests/LegacyDecodingTests.cs +++ b/StarterNG.Tests/LegacyDecodingTests.cs @@ -1,3 +1,4 @@ +using System.Runtime.ExceptionServices; using System.Text; using StarterNG.Infrastructure.Adapters; using StarterNG.Tests.Fakes; @@ -22,6 +23,40 @@ public class LegacyDecodingTests Assert.Equal(Polish, LegacyText.Decode(Encoding.UTF8.GetBytes(Polish))); } + [Fact] + public void A_byte_order_mark_does_not_stick_to_the_text() + { + byte[] bytes = Encoding.UTF8.GetPreamble().Concat(Encoding.UTF8.GetBytes("$a\n" + Polish)).ToArray(); + + Assert.Equal("$a\n" + Polish, LegacyText.Decode(bytes)); + } + + [Fact] + public void Telling_the_two_apart_does_not_throw() + { + // Decoding used to try strict UTF-8 and catch the failure: a first-chance + // exception for every 1250 file, hundreds of them on a load. + int thread = Environment.CurrentManagedThreadId; + int thrown = 0; + EventHandler count = (_, e) => + { + if (e.Exception is DecoderFallbackException && Environment.CurrentManagedThreadId == thread) + thrown++; + }; + + AppDomain.CurrentDomain.FirstChanceException += count; + try + { + LegacyText.Decode(LegacyText.CodePage1250.GetBytes(Polish)); + } + finally + { + AppDomain.CurrentDomain.FirstChanceException -= count; + } + + Assert.Equal(0, thrown); + } + [Fact] public void Plain_ascii_reads_the_same_either_way() { diff --git a/StarterNG/Infrastructure/Adapters/LegacyText.cs b/StarterNG/Infrastructure/Adapters/LegacyText.cs index 8997cfe..5eebd01 100644 --- a/StarterNG/Infrastructure/Adapters/LegacyText.cs +++ b/StarterNG/Infrastructure/Adapters/LegacyText.cs @@ -1,5 +1,6 @@ using System; using System.Text; +using System.Text.Unicode; namespace StarterNG.Infrastructure.Adapters; @@ -24,23 +25,21 @@ public static class LegacyText /// /// Decodes a game file. Most are code page 1250, but a few have been re-saved /// as UTF-8 over the years, and reading those as 1250 turns "Zakład" into - /// "ZakĹ‚ad". Valid UTF-8 is taken at its word; anything that fails a strict - /// decode is 1250, which cannot fail. + /// "ZakĹ‚ad". Valid UTF-8 is taken at its word; anything else is 1250, which + /// cannot fail. Checked up front rather than by catching a failed strict decode: + /// most files are 1250, and that was an exception per file on every load. + /// A byte order mark is dropped, or it would stick to the first token. /// public static string Decode(byte[] bytes) { - try - { - return Utf8Strict.GetString(bytes); - } - catch (DecoderFallbackException) - { - return CodePage1250.GetString(bytes); - } + ReadOnlySpan text = bytes; + if (text.StartsWith(Utf8Bom)) + text = text[Utf8Bom.Length..]; + + return Utf8.IsValid(text) ? Encoding.UTF8.GetString(text) : CodePage1250.GetString(text); } - private static readonly Encoding Utf8Strict = - new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + private static ReadOnlySpan Utf8Bom => [0xEF, 0xBB, 0xBF]; private static Encoding Resolve() { diff --git a/StarterNG/Views/Scenarios.axaml.cs b/StarterNG/Views/Scenarios.axaml.cs index a55b984..99a992a 100644 --- a/StarterNG/Views/Scenarios.axaml.cs +++ b/StarterNG/Views/Scenarios.axaml.cs @@ -709,7 +709,7 @@ public partial class Scenarios : UserControl try { - timetableContent.Text = File.ReadAllText(path, LegacyText.CodePage1250); + timetableContent.Text = LegacyText.Decode(File.ReadAllBytes(path)); UpdateTimetableTab(true); } catch