mirror of
https://github.com/MaSzyna-EU07/StarterNG.git
synced 2026-10-16 23:49:19 +02:00
Tell UTF-8 from 1250 without throwing, and decode timetables the same way
The strict decode threw on every 1250 file, hundreds of them per load.
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
using System.Runtime.ExceptionServices;
|
||||
using System.Text;
|
||||
using StarterNG.Infrastructure.Adapters;
|
||||
using StarterNG.Tests.Fakes;
|
||||
@@ -22,6 +23,40 @@ public class LegacyDecodingTests
|
||||
Assert.Equal(Polish, LegacyText.Decode(Encoding.UTF8.GetBytes(Polish)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_byte_order_mark_does_not_stick_to_the_text()
|
||||
{
|
||||
byte[] bytes = Encoding.UTF8.GetPreamble().Concat(Encoding.UTF8.GetBytes("$a\n" + Polish)).ToArray();
|
||||
|
||||
Assert.Equal("$a\n" + Polish, LegacyText.Decode(bytes));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Telling_the_two_apart_does_not_throw()
|
||||
{
|
||||
// Decoding used to try strict UTF-8 and catch the failure: a first-chance
|
||||
// exception for every 1250 file, hundreds of them on a load.
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
int thrown = 0;
|
||||
EventHandler<FirstChanceExceptionEventArgs> count = (_, e) =>
|
||||
{
|
||||
if (e.Exception is DecoderFallbackException && Environment.CurrentManagedThreadId == thread)
|
||||
thrown++;
|
||||
};
|
||||
|
||||
AppDomain.CurrentDomain.FirstChanceException += count;
|
||||
try
|
||||
{
|
||||
LegacyText.Decode(LegacyText.CodePage1250.GetBytes(Polish));
|
||||
}
|
||||
finally
|
||||
{
|
||||
AppDomain.CurrentDomain.FirstChanceException -= count;
|
||||
}
|
||||
|
||||
Assert.Equal(0, thrown);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Plain_ascii_reads_the_same_either_way()
|
||||
{
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Text;
|
||||
using System.Text.Unicode;
|
||||
|
||||
namespace StarterNG.Infrastructure.Adapters;
|
||||
|
||||
@@ -24,23 +25,21 @@ public static class LegacyText
|
||||
/// <summary>
|
||||
/// Decodes a game file. Most are code page 1250, but a few have been re-saved
|
||||
/// as UTF-8 over the years, and reading those as 1250 turns "Zakład" into
|
||||
/// "Zakład". Valid UTF-8 is taken at its word; anything that fails a strict
|
||||
/// decode is 1250, which cannot fail.
|
||||
/// "Zakład". Valid UTF-8 is taken at its word; anything else is 1250, which
|
||||
/// cannot fail. Checked up front rather than by catching a failed strict decode:
|
||||
/// most files are 1250, and that was an exception per file on every load.
|
||||
/// A byte order mark is dropped, or it would stick to the first token.
|
||||
/// </summary>
|
||||
public static string Decode(byte[] bytes)
|
||||
{
|
||||
try
|
||||
{
|
||||
return Utf8Strict.GetString(bytes);
|
||||
}
|
||||
catch (DecoderFallbackException)
|
||||
{
|
||||
return CodePage1250.GetString(bytes);
|
||||
}
|
||||
ReadOnlySpan<byte> text = bytes;
|
||||
if (text.StartsWith(Utf8Bom))
|
||||
text = text[Utf8Bom.Length..];
|
||||
|
||||
return Utf8.IsValid(text) ? Encoding.UTF8.GetString(text) : CodePage1250.GetString(text);
|
||||
}
|
||||
|
||||
private static readonly Encoding Utf8Strict =
|
||||
new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
|
||||
private static ReadOnlySpan<byte> Utf8Bom => [0xEF, 0xBB, 0xBF];
|
||||
|
||||
private static Encoding Resolve()
|
||||
{
|
||||
|
||||
@@ -709,7 +709,7 @@ public partial class Scenarios : UserControl
|
||||
|
||||
try
|
||||
{
|
||||
timetableContent.Text = File.ReadAllText(path, LegacyText.CodePage1250);
|
||||
timetableContent.Text = LegacyText.Decode(File.ReadAllBytes(path));
|
||||
UpdateTimetableTab(true);
|
||||
}
|
||||
catch
|
||||
|
||||
Reference in New Issue
Block a user