Tell UTF-8 from 1250 without throwing, and decode timetables the same way

The strict decode threw on every 1250 file, hundreds of them per load.
This commit is contained in:
maj00r
2026-10-04 12:18:36 +02:00
parent 2545daeb82
commit 3ddd00895f
3 changed files with 47 additions and 13 deletions

View File

@@ -1,3 +1,4 @@
using System.Runtime.ExceptionServices;
using System.Text;
using StarterNG.Infrastructure.Adapters;
using StarterNG.Tests.Fakes;
@@ -22,6 +23,40 @@ public class LegacyDecodingTests
Assert.Equal(Polish, LegacyText.Decode(Encoding.UTF8.GetBytes(Polish)));
}
[Fact]
public void A_byte_order_mark_does_not_stick_to_the_text()
{
byte[] bytes = Encoding.UTF8.GetPreamble().Concat(Encoding.UTF8.GetBytes("$a\n" + Polish)).ToArray();
Assert.Equal("$a\n" + Polish, LegacyText.Decode(bytes));
}
[Fact]
public void Telling_the_two_apart_does_not_throw()
{
// Decoding used to try strict UTF-8 and catch the failure: a first-chance
// exception for every 1250 file, hundreds of them on a load.
int thread = Environment.CurrentManagedThreadId;
int thrown = 0;
EventHandler<FirstChanceExceptionEventArgs> count = (_, e) =>
{
if (e.Exception is DecoderFallbackException && Environment.CurrentManagedThreadId == thread)
thrown++;
};
AppDomain.CurrentDomain.FirstChanceException += count;
try
{
LegacyText.Decode(LegacyText.CodePage1250.GetBytes(Polish));
}
finally
{
AppDomain.CurrentDomain.FirstChanceException -= count;
}
Assert.Equal(0, thrown);
}
[Fact]
public void Plain_ascii_reads_the_same_either_way()
{

View File

@@ -1,5 +1,6 @@
using System;
using System.Text;
using System.Text.Unicode;
namespace StarterNG.Infrastructure.Adapters;
@@ -24,23 +25,21 @@ public static class LegacyText
/// <summary>
/// Decodes a game file. Most are code page 1250, but a few have been re-saved
/// as UTF-8 over the years, and reading those as 1250 turns "Zakład" into
/// "Zakład". Valid UTF-8 is taken at its word; anything that fails a strict
/// decode is 1250, which cannot fail.
/// "Zakład". Valid UTF-8 is taken at its word; anything else is 1250, which
/// cannot fail. Checked up front rather than by catching a failed strict decode:
/// most files are 1250, and that was an exception per file on every load.
/// A byte order mark is dropped, or it would stick to the first token.
/// </summary>
public static string Decode(byte[] bytes)
{
try
{
return Utf8Strict.GetString(bytes);
}
catch (DecoderFallbackException)
{
return CodePage1250.GetString(bytes);
}
ReadOnlySpan<byte> text = bytes;
if (text.StartsWith(Utf8Bom))
text = text[Utf8Bom.Length..];
return Utf8.IsValid(text) ? Encoding.UTF8.GetString(text) : CodePage1250.GetString(text);
}
private static readonly Encoding Utf8Strict =
new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
private static ReadOnlySpan<byte> Utf8Bom => [0xEF, 0xBB, 0xBF];
private static Encoding Resolve()
{

View File

@@ -709,7 +709,7 @@ public partial class Scenarios : UserControl
try
{
timetableContent.Text = File.ReadAllText(path, LegacyText.CodePage1250);
timetableContent.Text = LegacyText.Decode(File.ReadAllBytes(path));
UpdateTimetableTab(true);
}
catch