From 51521a135a8b83a63d0fa3c4c7fbad12ff203e3f Mon Sep 17 00:00:00 2001 From: pawelw Date: Sat, 3 Oct 2026 12:05:32 +0200 Subject: [PATCH] Text displays: accept characters above 127, in windows-1250 or utf-8 (#198) * Text displays: accept characters above 127, in windows-1250 or utf-8 A text display (sub-model of type "text") looks up the sub-model of each character in a 256 entry table, and indexed it with a plain char. With char being signed, any character above 127 gave a negative index: reading the text went outside the table, and so did filling the table if a character sub-model had such a name. In effect the displays could only show ASCII, and a text with national characters could crash. The lookup is now keyed by unicode code point. Both the displayed text and the names of the character sub-models are decoded as utf-8 if they are valid utf-8, and as windows-1250 otherwise, so legacy (ansi) sceneries and models work next to utf-8 ones, and the two can be mixed. A short ansi text can be valid utf-8 by accident (two upper case accented letters in a row will do); when both readings are possible the one which finds more of its characters in the model is used. The rest is unchanged: the first character of the text still isn't displayed, the characters are still drawn in the translucent pass only. A missing text pointer no longer gets dereferenced. * Text displays: prepare the characters once for each text The sequence of character sub-models was cached for the last text only, so displays of several instances drawn in turn decoded their texts again on every draw. It is now kept for each text the display shows; the texts of the instances don't change, so a draw costs a single lookup. Also the comments added in the renderers are now in English. * Text displays: move the text decoding to utilities utf8_to_utf32() and win1250_to_utf32(), with the windows-1250 table, now sit next to win1250_to_utf8(). The existing function maps only the Polish letters and passes other bytes above 127 through, so it can't serve the displays. * Text displays: address the SonarCloud findings std::array instead of the C-style arrays, the utf-8 sequence length worked out in a function of its own instead of nested conditional operators, std::byte for the bit operations, std::ranges::count_if and contains(). The character loop of the renderers moved to Render_Alpha_text(), which takes the nesting back to the limit, and the static pasText is accessed through the class. --- model/Model3d.cpp | 64 ++++++++++++++++++++++++++- model/Model3d.h | 7 ++- rendering/opengl33renderer.cpp | 39 +++++++--------- rendering/opengl33renderer.h | 1 + rendering/openglrenderer.cpp | 33 ++++++-------- rendering/openglrenderer.h | 3 ++ utilities/utilities.cpp | 81 ++++++++++++++++++++++++++++++++++ utilities/utilities.h | 4 ++ 8 files changed, 187 insertions(+), 45 deletions(-) diff --git a/model/Model3d.cpp b/model/Model3d.cpp index fba07446..e1a3a79e 100644 --- a/model/Model3d.cpp +++ b/model/Model3d.cpp @@ -17,6 +17,7 @@ Copyright (C) 2001-2004 Marcin Wozniak, Maciej Czapkiewicz and others #include "utilities/Globals.h" #include "utilities/Logs.h" +#include "utilities/utilities.h" #include "rendering/renderer.h" #include "utilities/Timer.h" #include "simulation/simulation.h" @@ -55,8 +56,6 @@ TSubModel::~TSubModel() SafeDelete(Child); delete fMatrix; // własny transform trzeba usunąć (zawsze jeden) } - delete[] smLetter; // używany tylko roboczo dla TP_TEXT, do przyspieszenia - // wyświetlania }; void TSubModel::Name_Material(std::string const &Name) @@ -892,6 +891,67 @@ void TSubModel::InitialRotate(bool doit) Next->InitialRotate(doit); }; +// text display (TP_TEXT): returns sub-models of the characters to draw for provided text, in drawing order. +// a character is matched with the first character of a child sub-model name. both the text and the names can be +// encoded either in utf-8 or in windows-1250, so the legacy (ansi) scenery and model files work along the utf-8 ones +std::vector const &TSubModel::text_letters(std::string const &Text) +{ + // the sequence is prepared once for each text the display shows. the texts of the model instances don't change, + // so drawing a display afterwards costs a single lookup, however many instances and passes take turns + auto const prepared{m_textlayouts.find(Text)}; + if (prepared != m_textlayouts.end()) + { + return prepared->second; + } + + std::u32string characters; + if (false == m_lettersready) + { + for (auto *letter = Child; letter != nullptr; letter = letter->Next) + { + if (letter->pName.empty()) + { + continue; + } + if (false == utf8_to_utf32(letter->pName, characters)) + { + win1250_to_utf32(letter->pName, characters); + } + if (characters.front() != 0) + { + m_letters[characters.front()] = letter; + } + } + m_lettersready = true; + } + + // the text is taken for utf-8 if it's valid as such, and for windows-1250 otherwise. a short ansi text can pass + // for utf-8 by accident (two upper case accented letters in a row are enough for this to happen), + // so if both readings are possible the one which finds more of its characters in the model wins + auto const known = [this](std::u32string const &Characters) { + return std::ranges::count_if(Characters, [this](char32_t const Character) { return m_letters.contains(Character); }); + }; + std::u32string ansi; + win1250_to_utf32(Text, ansi); + if (false == utf8_to_utf32(Text, characters) || known(ansi) > known(characters)) + { + characters = ansi; + } + + auto &textletters{m_textlayouts[Text]}; + // NOTE: the first character of the text isn't displayed. the displays always worked this way, and the existing + // content relies on it + for (std::size_t idx = 1; idx < characters.size(); ++idx) + { + auto const lookup{m_letters.find(characters[idx])}; + if (lookup != m_letters.end()) + { + textletters.emplace_back(lookup->second); + } + } + return textletters; +} + void TSubModel::ChildAdd(TSubModel *SubModel) { // dodanie submodelu potemnego (uzależnionego) // Ra: zmiana kolejności, żeby kolejne móc renderować po aktualnym (było diff --git a/model/Model3d.h b/model/Model3d.h index 0e053609..53e9084f 100644 --- a/model/Model3d.h +++ b/model/Model3d.h @@ -156,7 +156,10 @@ public: // chwilowo std::uintptr_t iAnimOwner{ 0 }; // roboczy numer egzemplarza, który ustawił animację TAnimType b_aAnim{ TAnimType::at_None }; // kody animacji oddzielnie, bo zerowane std::shared_ptr mAnimMatrix; // macierz do animacji kwaternionowych - TSubModel **smLetter{ nullptr }; // wskaźnik na tablicę submdeli do generoania tekstu (docelowo zapisać do E3D) + // text display (TP_TEXT) data, filled on first use + std::unordered_map m_letters; // sub-models of the available characters, by unicode code point + bool m_lettersready { false }; // whether the character lookup was built yet + std::unordered_map> m_textlayouts; // sub-models of consecutive characters, for each text shown TSubModel *Parent{ nullptr }; // nadrzędny, np. do wymnażania macierzy int iVisible { 1 }; // roboczy stan widoczności float fVisible { 1.f }; // visibility level @@ -202,6 +205,8 @@ public: void SetTranslate(float3 vNewTransVector); void SetRotateIK1(float3 vNewAngles); TSubModel * GetFromName( std::string const &search, bool i = true ); + // text display (TP_TEXT): returns sub-models of the characters to draw for provided text, in drawing order + std::vector const & text_letters( std::string const &Text ); inline float4x4 * GetMatrix() { return fMatrix; }; inline float4x4 const * GetMatrix() const { return fMatrix; }; // returns offset vector from root diff --git a/rendering/opengl33renderer.cpp b/rendering/opengl33renderer.cpp index c07efca4..2f587033 100644 --- a/rendering/opengl33renderer.cpp +++ b/rendering/opengl33renderer.cpp @@ -4634,6 +4634,21 @@ bool opengl33_renderer::Render_Alpha(TModel3d *Model, material_data const *Mater return result; } +// characters of a text display, laid out in a row. the text can be encoded in utf-8 or in windows-1250 +void opengl33_renderer::Render_Alpha_text(TSubModel *Submodel) +{ + if (TSubModel::pasText == nullptr) + { + return; + } + for (auto *p : Submodel->text_letters(*TSubModel::pasText)) + { // translucent only, for the time being + Render_Alpha(p); + if (p->fMatrix) + ::glMultMatrixf(p->fMatrix->readArray()); // move on to the place of the next character + } +} + void opengl33_renderer::Render_Alpha(TSubModel *Submodel) { // renderowanie przezroczystych przez DL @@ -4862,29 +4877,7 @@ void opengl33_renderer::Render_Alpha(TSubModel *Submodel) { if (Submodel->eType == TP_TEXT) { // tekst renderujemy w specjalny sposób, zamiast submodeli z łańcucha Child - int i, j = (int)Submodel->pasText->size(); - TSubModel *p; - if (!Submodel->smLetter) - { // jeśli nie ma tablicy, to ją stworzyć; miejsce nieodpowiednie, ale tymczasowo może być - Submodel->smLetter = new TSubModel *[256]; // tablica wskaźników submodeli dla wyświetlania tekstu - memset(Submodel->smLetter, 0, 256 * sizeof(TSubModel *)); // wypełnianie zerami - p = Submodel->Child; - while (p) - { - Submodel->smLetter[p->pName[0]] = p; - p = p->Next; // kolejny znak - } - } - for (i = 1; i <= j; ++i) - { - p = Submodel->smLetter[(*(Submodel->pasText))[i]]; // znak do wyświetlenia - if (p) - { // na razie tylko jako przezroczyste - Render_Alpha(p); - if (p->fMatrix) - ::glMultMatrixf(p->fMatrix->readArray()); // przesuwanie widoku - } - } + Render_Alpha_text(Submodel); } else if (Submodel->iAlpha & Submodel->iFlags & 0x002F0000) Render_Alpha(Submodel->Child); diff --git a/rendering/opengl33renderer.h b/rendering/opengl33renderer.h index 43cd24b0..562670d9 100644 --- a/rendering/opengl33renderer.h +++ b/rendering/opengl33renderer.h @@ -310,6 +310,7 @@ class opengl33_renderer : public gfx_renderer { bool Render_Alpha(TModel3d *Model, material_data const *Material, float const Squaredistance, glm::dvec3 const &Position, glm::vec3 const &Angle); bool Render_Alpha(TModel3d *Model, material_data const *Material, float const Squaredistance); void Render_Alpha(TSubModel *Submodel); + void Render_Alpha_text(TSubModel *Submodel); // characters of a text display void Update_Lights(light_array &Lights); glm::vec3 pick_color(std::size_t const Index); std::size_t pick_index(glm::ivec3 const &Color); diff --git a/rendering/openglrenderer.cpp b/rendering/openglrenderer.cpp index bc6c9d14..b675c38f 100644 --- a/rendering/openglrenderer.cpp +++ b/rendering/openglrenderer.cpp @@ -3813,6 +3813,19 @@ opengl_renderer::Render_Alpha( TModel3d *Model, material_data const *Material, f return result; } +// characters of a text display, laid out in a row. the text can be encoded in utf-8 or in windows-1250 +void +opengl_renderer::Render_Alpha_text( TSubModel *Submodel ) { + + if( TSubModel::pasText == nullptr ) { return; } + + for( auto *p : Submodel->text_letters( *TSubModel::pasText ) ) { // translucent only, for the time being + Render_Alpha( p ); + if( p->fMatrix ) + ::glMultMatrixf( p->fMatrix->readArray() ); // move on to the place of the next character + } +} + void opengl_renderer::Render_Alpha( TSubModel *Submodel ) { // renderowanie przezroczystych przez DL @@ -4028,25 +4041,7 @@ opengl_renderer::Render_Alpha( TSubModel *Submodel ) { if( Submodel->Child != nullptr ) { if( Submodel->eType == TP_TEXT ) { // tekst renderujemy w specjalny sposób, zamiast submodeli z łańcucha Child - int i, j = (int)Submodel->pasText->size(); - TSubModel *p; - if( !Submodel->smLetter ) { // jeśli nie ma tablicy, to ją stworzyć; miejsce nieodpowiednie, ale tymczasowo może być - Submodel->smLetter = new TSubModel *[ 256 ]; // tablica wskaźników submodeli dla wyświetlania tekstu - std::memset(Submodel->smLetter, 0, 256 * sizeof( TSubModel * )); // wypełnianie zerami - p = Submodel->Child; - while( p ) { - Submodel->smLetter[ p->pName[ 0 ] ] = p; - p = p->Next; // kolejny znak - } - } - for( i = 1; i <= j; ++i ) { - p = Submodel->smLetter[ ( *( Submodel->pasText) )[ i ] ]; // znak do wyświetlenia - if( p ) { // na razie tylko jako przezroczyste - Render_Alpha( p ); - if( p->fMatrix ) - ::glMultMatrixf( p->fMatrix->readArray() ); // przesuwanie widoku - } - } + Render_Alpha_text( Submodel ); } else if( Submodel->iAlpha & Submodel->iFlags & 0x002F0000 ) Render_Alpha( Submodel->Child ); diff --git a/rendering/openglrenderer.h b/rendering/openglrenderer.h index 820deb92..83dca483 100644 --- a/rendering/openglrenderer.h +++ b/rendering/openglrenderer.h @@ -270,6 +270,9 @@ private: Render_Alpha( TModel3d *Model, material_data const *Material, float const Squaredistance ); void Render_Alpha( TSubModel *Submodel ); + // characters of a text display + void + Render_Alpha_text( TSubModel *Submodel ); void Update_Lights( light_array &Lights ); bool diff --git a/utilities/utilities.cpp b/utilities/utilities.cpp index efaf3e6c..804ca66c 100644 --- a/utilities/utilities.cpp +++ b/utilities/utilities.cpp @@ -339,6 +339,87 @@ std::string win1250_to_utf8(const std::string &Input) return output; } +namespace +{ +// unicode code points of the upper half of the windows-1250 code page; 0 marks the unassigned bytes +std::array const win1250_upperhalf{ + 0x20AC, 0x0000, 0x201A, 0x0000, 0x201E, 0x2026, 0x2020, 0x2021, 0x0000, 0x2030, 0x0160, 0x2039, 0x015A, 0x0164, 0x017D, 0x0179, + 0x0000, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x0000, 0x2122, 0x0161, 0x203A, 0x015B, 0x0165, 0x017E, 0x017A, + 0x00A0, 0x02C7, 0x02D8, 0x0141, 0x00A4, 0x0104, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x015E, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x017B, + 0x00B0, 0x00B1, 0x02DB, 0x0142, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00B8, 0x0105, 0x015F, 0x00BB, 0x013D, 0x02DD, 0x013E, 0x017C, + 0x0154, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0139, 0x0106, 0x00C7, 0x010C, 0x00C9, 0x0118, 0x00CB, 0x011A, 0x00CD, 0x00CE, 0x010E, + 0x0110, 0x0143, 0x0147, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x00D7, 0x0158, 0x016E, 0x00DA, 0x0170, 0x00DC, 0x00DD, 0x0162, 0x00DF, + 0x0155, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x013A, 0x0107, 0x00E7, 0x010D, 0x00E9, 0x0119, 0x00EB, 0x011B, 0x00ED, 0x00EE, 0x010F, + 0x0111, 0x0144, 0x0148, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x00F7, 0x0159, 0x016F, 0x00FA, 0x0171, 0x00FC, 0x00FD, 0x0163, 0x02D9}; + +// returns number of bytes of the utf-8 sequence started by provided byte, or 0 if it can't start one +std::size_t utf8_sequence_length(std::byte const Lead) +{ + if ((Lead & std::byte{0x80}) == std::byte{0x00}) + { + return 1; + } + if ((Lead & std::byte{0xE0}) == std::byte{0xC0}) + { + return 2; + } + if ((Lead & std::byte{0xF0}) == std::byte{0xE0}) + { + return 3; + } + if ((Lead & std::byte{0xF8}) == std::byte{0xF0}) + { + return 4; + } + return 0; // a continuation byte, or a lead of a sequence utf-8 doesn't allow +} +} // namespace + +bool utf8_to_utf32(std::string const &Text, std::u32string &Output) +{ + // lowest code point which needs given number of bytes; shorter values are overlong forms + static std::array const minimum{0, 0, 0x80, 0x800, 0x10000}; + + Output.clear(); + auto const size{Text.size()}; + for (std::size_t idx = 0; idx < size;) + { + auto const lead{static_cast(Text[idx])}; + auto const length{utf8_sequence_length(lead)}; + if (length == 0 || idx + length > size) + { + return false; + } + auto codepoint{std::to_integer(lead & (std::byte{0xFF} >> (length > 1 ? length + 1 : 1)))}; + for (std::size_t offset = 1; offset < length; ++offset) + { + auto const next{static_cast(Text[idx + offset])}; + if ((next & std::byte{0xC0}) != std::byte{0x80}) + { + return false; + } + codepoint = (codepoint << 6) | std::to_integer(next & std::byte{0x3F}); + } + if (codepoint < minimum[length] || codepoint > 0x10FFFF || (codepoint >= 0xD800 && codepoint <= 0xDFFF)) + { + return false; + } + Output += codepoint; + idx += length; + } + return true; +} + +void win1250_to_utf32(std::string const &Text, std::u32string &Output) +{ + Output.clear(); + for (auto const character : Text) + { + auto const value{std::to_integer(static_cast(character))}; + Output += (value < 0x80 ? static_cast(value) : win1250_upperhalf[value - 0x80]); + } +} + // Ra: tymczasowe rozwiązanie kwestii zagranicznych (czeskich) napisów char charsetconversiontable[] = "E?,?\"_++?%Sstzz" " ^^L$A|S^CS<--RZo±,l'uP.,as>L\"lz" diff --git a/utilities/utilities.h b/utilities/utilities.h index 6e761380..6b8111e3 100644 --- a/utilities/utilities.h +++ b/utilities/utilities.h @@ -166,6 +166,10 @@ void win1250_to_ascii(std::string &Input); std::string Bezogonkow(std::string Input, bool const Underscorestospaces = false); std::string win1250_to_utf8(const std::string &input); +// decodes utf-8 encoded text into unicode code points. returns false if the text isn't valid utf-8 +bool utf8_to_utf32(std::string const &Text, std::u32string &Output); +// decodes windows-1250 (ansi) encoded text into unicode code points +void win1250_to_utf32(std::string const &Text, std::u32string &Output); inline std::string extract_value(std::string const &Key, std::string const &Input) {