From 09b1c2de10463f69331841a81cc18999fd7846a2 Mon Sep 17 00:00:00 2001 From: maj00r <73280030+maj00r@users.noreply.github.com> Date: Thu, 5 Nov 2026 03:12:05 +0100 Subject: [PATCH] fix(utilities): win1250_to_utf8 turns the polish letters into utf-8, not into themselves (#252) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit utilities.cpp starts with a byte order mark, so the compiler reads it as utf-8. The game's targets don't get /utf-8, though: set_target_properties(... COMPILE_OPTIONS ...) in the MSVC block of CMakeLists.txt replaces the options add_compile_options gave them. So the string literals went into the program in the code page of the system, windows-1250 on a polish one, and the map of win1250_to_utf8 turned each letter into itself. What read windows-1250 through it got no utf-8: the names, descriptions and categories of the sceneries and the names of the textures in the launcher, and the descriptions of the templates in the editor. The editor writes such a description back through utf8_to_win1250, which makes a question mark of each byte that doesn't start a utf-8 sequence, so the letters it had read unchanged didn't come back. The utf-8 sequences are written as bytes now, the way utf8_to_win1250 has them, so they don't depend on the options. The options stay as they are: with /utf-8 other literals of the files with a byte order mark would change, the "±" of the table of Bezogonkow would take two bytes and move the letters after it. The unit tests check the letters and that the text given is valid utf-8. Co-authored-by: maj00r --- tests/unit/test_win1250.cpp | 26 ++++++++++++++++++++++++++ utilities/utilities.cpp | 7 +++++-- 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_win1250.cpp diff --git a/tests/unit/test_win1250.cpp b/tests/unit/test_win1250.cpp new file mode 100644 index 00000000..156c46ae --- /dev/null +++ b/tests/unit/test_win1250.cpp @@ -0,0 +1,26 @@ +#include "stdafx.h" +#include "doctest.h" +#include "utilities/utilities.h" + +// the polish letters, in windows-1250 and in utf-8; written as bytes, the build reads the sources in any code page +namespace +{ +std::string const kWin1250{"\xA5\xC6\xCA\xA3\xD1\xD3\x8C\x8F\xAF\xB9\xE6\xEA\xB3\xF1\xF3\x9C\x9F\xBF"}; +std::string const kUtf8{"\xC4\x84\xC4\x86\xC4\x98\xC5\x81\xC5\x83\xC3\x93\xC5\x9A\xC5\xB9\xC5\xBB" + "\xC4\x85\xC4\x87\xC4\x99\xC5\x82\xC5\x84\xC3\xB3\xC5\x9B\xC5\xBA\xC5\xBC"}; +} // namespace + +TEST_CASE("win1250: the polish letters of a scenery file read as utf-8") { + CHECK(win1250_to_utf8("Bia\xB3" "a Podlaska") == "Bia\xC5\x82" "a Podlaska"); + CHECK(win1250_to_utf8(kWin1250) == kUtf8); + // the rest of the text stays as it is + CHECK(win1250_to_utf8("tor 1, km 0.178") == "tor 1, km 0.178"); +} + +TEST_CASE("win1250: what win1250_to_utf8 gives is valid utf-8") { + std::u32string decoded; + CHECK_FALSE(utf8_to_utf32(kWin1250, decoded)); + REQUIRE(utf8_to_utf32(win1250_to_utf8(kWin1250), decoded)); + CHECK(decoded.size() == kWin1250.size()); + CHECK(decoded[12] == U'\x0142'); +} diff --git a/utilities/utilities.cpp b/utilities/utilities.cpp index 02608262..e84307e3 100644 --- a/utilities/utilities.cpp +++ b/utilities/utilities.cpp @@ -317,8 +317,11 @@ void win1250_to_ascii(std::string &Input) std::string win1250_to_utf8(const std::string &Input) { - std::unordered_map const charmap{{165, "Ą"}, {198, "Ć"}, {202, "Ę"}, {163, "Ł"}, {209, "Ń"}, {211, "Ó"}, {140, "Ś"}, {143, "Ź"}, {175, "Ż"}, - {185, "ą"}, {230, "ć"}, {234, "ę"}, {179, "ł"}, {241, "ń"}, {243, "ó"}, {156, "ś"}, {159, "ź"}, {191, "ż"}}; + // the utf-8 sequences written as bytes: the file has a byte order mark and the build doesn't give the compiler /utf-8, so letters + // written as letters would end up in the program in windows-1250, the same as the input + static std::unordered_map const charmap{{'\xA5', "\xC4\x84"}, {'\xC6', "\xC4\x86"}, {'\xCA', "\xC4\x98"}, {'\xA3', "\xC5\x81"}, {'\xD1', "\xC5\x83"}, {'\xD3', "\xC3\x93"}, + {'\x8C', "\xC5\x9A"}, {'\x8F', "\xC5\xB9"}, {'\xAF', "\xC5\xBB"}, {'\xB9', "\xC4\x85"}, {'\xE6', "\xC4\x87"}, {'\xEA', "\xC4\x99"}, + {'\xB3', "\xC5\x82"}, {'\xF1', "\xC5\x84"}, {'\xF3', "\xC3\xB3"}, {'\x9C', "\xC5\x9B"}, {'\x9F', "\xC5\xBA"}, {'\xBF', "\xC5\xBC"}}; std::string output; std::unordered_map::const_iterator lookup; for (auto &input : Input)