diff --git a/tests/unit/test_win1250.cpp b/tests/unit/test_win1250.cpp new file mode 100644 index 00000000..156c46ae --- /dev/null +++ b/tests/unit/test_win1250.cpp @@ -0,0 +1,26 @@ +#include "stdafx.h" +#include "doctest.h" +#include "utilities/utilities.h" + +// the polish letters, in windows-1250 and in utf-8; written as bytes, the build reads the sources in any code page +namespace +{ +std::string const kWin1250{"\xA5\xC6\xCA\xA3\xD1\xD3\x8C\x8F\xAF\xB9\xE6\xEA\xB3\xF1\xF3\x9C\x9F\xBF"}; +std::string const kUtf8{"\xC4\x84\xC4\x86\xC4\x98\xC5\x81\xC5\x83\xC3\x93\xC5\x9A\xC5\xB9\xC5\xBB" + "\xC4\x85\xC4\x87\xC4\x99\xC5\x82\xC5\x84\xC3\xB3\xC5\x9B\xC5\xBA\xC5\xBC"}; +} // namespace + +TEST_CASE("win1250: the polish letters of a scenery file read as utf-8") { + CHECK(win1250_to_utf8("Bia\xB3" "a Podlaska") == "Bia\xC5\x82" "a Podlaska"); + CHECK(win1250_to_utf8(kWin1250) == kUtf8); + // the rest of the text stays as it is + CHECK(win1250_to_utf8("tor 1, km 0.178") == "tor 1, km 0.178"); +} + +TEST_CASE("win1250: what win1250_to_utf8 gives is valid utf-8") { + std::u32string decoded; + CHECK_FALSE(utf8_to_utf32(kWin1250, decoded)); + REQUIRE(utf8_to_utf32(win1250_to_utf8(kWin1250), decoded)); + CHECK(decoded.size() == kWin1250.size()); + CHECK(decoded[12] == U'\x0142'); +} diff --git a/utilities/utilities.cpp b/utilities/utilities.cpp index 02608262..e84307e3 100644 --- a/utilities/utilities.cpp +++ b/utilities/utilities.cpp @@ -317,8 +317,11 @@ void win1250_to_ascii(std::string &Input) std::string win1250_to_utf8(const std::string &Input) { - std::unordered_map const charmap{{165, "Ą"}, {198, "Ć"}, {202, "Ę"}, {163, "Ł"}, {209, "Ń"}, {211, "Ó"}, {140, "Ś"}, {143, "Ź"}, {175, "Ż"}, - {185, "ą"}, {230, "ć"}, {234, "ę"}, {179, "ł"}, {241, "ń"}, {243, "ó"}, {156, "ś"}, {159, "ź"}, {191, "ż"}}; + // the utf-8 sequences written as bytes: the file has a byte order mark and the build doesn't give the compiler /utf-8, so letters + // written as letters would end up in the program in windows-1250, the same as the input + static std::unordered_map const charmap{{'\xA5', "\xC4\x84"}, {'\xC6', "\xC4\x86"}, {'\xCA', "\xC4\x98"}, {'\xA3', "\xC5\x81"}, {'\xD1', "\xC5\x83"}, {'\xD3', "\xC3\x93"}, + {'\x8C', "\xC5\x9A"}, {'\x8F', "\xC5\xB9"}, {'\xAF', "\xC5\xBB"}, {'\xB9', "\xC4\x85"}, {'\xE6', "\xC4\x87"}, {'\xEA', "\xC4\x99"}, + {'\xB3', "\xC5\x82"}, {'\xF1', "\xC5\x84"}, {'\xF3', "\xC3\xB3"}, {'\x9C', "\xC5\x9B"}, {'\x9F', "\xC5\xBA"}, {'\xBF', "\xC5\xBC"}}; std::string output; std::unordered_map::const_iterator lookup; for (auto &input : Input)