From 4867f60a6596944232e6becd842fb52638f531e4 Mon Sep 17 00:00:00 2001 From: Hirek193 Date: Tue, 30 Jun 2026 23:35:33 +0200 Subject: [PATCH] parser: keep UTF-8 multibyte chars intact when lowercasing tokens --- utilities/parser.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/utilities/parser.cpp b/utilities/parser.cpp index 68656c5d..d6cc341a 100644 --- a/utilities/parser.cpp +++ b/utilities/parser.cpp @@ -36,7 +36,13 @@ inline std::array makeBreakTable(const char *brk) inline char toLowerChar(char c) { - return static_cast(std::tolower(static_cast(c))); + // Only fold ASCII letters. Bytes >= 0x80 belong to multibyte UTF-8 + // sequences and must be passed through untouched, otherwise a non-"C" + // global locale could remap them and corrupt UTF-8 encoded text. + const unsigned char uc = static_cast(c); + if (uc < 0x80) + return static_cast(std::tolower(uc)); + return c; } inline bool startsWithBOM(const std::string &s)