eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-17 22:39:17 +02:00
Code Issues 121 Wiki Activity

parser: keep UTF-8 multibyte chars intact when lowercasing tokens

Browse Source
This commit is contained in:
Hirek
2026-06-30 23:35:33 +02:00
parent 92626db147
commit 4867f60a65
1 changed files with 7 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
utilities/parser.cpp
View File

@@ -36,7 +36,13 @@ inline std::array<bool, 256> makeBreakTable(const char *brk)
inline char toLowerChar(char c)
{
return static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
// Only fold ASCII letters. Bytes >= 0x80 belong to multibyte UTF-8
// sequences and must be passed through untouched, otherwise a non-"C"
// global locale could remap them and corrupt UTF-8 encoded text.
const unsigned char uc = static_cast<unsigned char>(c);
if (uc < 0x80)
return static_cast<char>(std::tolower(uc));
return c;
}
inline bool startsWithBOM(const std::string &s)