From d0a36946579ad55e27b7748c14293bcd4cf42660 Mon Sep 17 00:00:00 2001 From: maj00r Date: Fri, 13 Nov 2026 15:41:40 +0100 Subject: [PATCH 1/7] fix(files): data named in another case is found on Linux, folders included The parser and the loaders lowercase every name they read, while the data made on Windows has files and folders named in any case (scenery/Braniewo, textures/siec_trakcyjna_ph/.../STS_*.tga, sounds/sip/Gorki.wav). FileCased matched the file name only, so a folder in another case wasn't found. - FileCased walks every part of the path on Linux, the folder listings kept by their lowercase and read again once the folder changed; Windows as it was - FileExists finds the file in any case on Linux, and the variant with names and extensions gives back the name as it's on the disk, so textures, materials, sounds, station announcements and scripts open that name - models, sound captions, timetables and nodebank previews open the file as it's named on the disk; the texture format is told by the extension in any case --- audio/audio.cpp | 4 +- editor/editorPreviews.cpp | 4 +- model/Model3d.cpp | 17 ++++--- model/Texture.cpp | 16 +++--- tests/unit/test_utilities.cpp | 32 ++++++++++++ utilities/utilities.cpp | 96 ++++++++++++++++++++++++++++++++--- utilities/utilities.h | 5 +- world/mtable.cpp | 9 ++++ 8 files changed, 157 insertions(+), 26 deletions(-) diff --git a/audio/audio.cpp b/audio/audio.cpp index 3be2810d1..c9aa56675 100644 --- a/audio/audio.cpp +++ b/audio/audio.cpp @@ -216,9 +216,9 @@ openal_buffer::fetch_caption() { std::string captionfilename { name }; captionfilename.erase( captionfilename.rfind( '.' ) ); // obcięcie rozszerzenia captionfilename += "-" + Global.asLang + ".txt"; // już może być w różnych językach - if( true == FileExists( captionfilename ) ) { + if( auto const casedfilename { FileCased( captionfilename ) }; false == casedfilename.empty() ) { // wczytanie - std::ifstream inputfile( captionfilename ); + std::ifstream inputfile( casedfilename ); caption.assign( std::istreambuf_iterator( inputfile ), std::istreambuf_iterator() ); } } diff --git a/editor/editorPreviews.cpp b/editor/editorPreviews.cpp index 27f84a9fe..68164460c 100644 --- a/editor/editorPreviews.cpp +++ b/editor/editorPreviews.cpp @@ -298,7 +298,9 @@ void image_cache::work() m_queue.pop_front(); } int width{0}, height{0}, components{0}; - auto *pixels{stbi_load(image.path.c_str(), &width, &height, &components, 4)}; + // the file as it's named on the disk, the path stays the key of the cache + auto const file{FileCased(image.path)}; + auto *pixels{file.empty() ? nullptr : stbi_load(file.c_str(), &width, &height, &components, 4)}; if (pixels != nullptr && width > 0 && height > 0) { // scaled down by the average of the pixels which make each one, the longer side to the size of the cache diff --git a/model/Model3d.cpp b/model/Model3d.cpp index 44f5f6255..e3507e216 100644 --- a/model/Model3d.cpp +++ b/model/Model3d.cpp @@ -1702,24 +1702,27 @@ bool TModel3d::LoadFromFile(std::string const &FileName, bool dynamic) m_filename = name; asBinary = name + ".e3d"; + // the files as they're named on the disk, the name given is in lowercase + auto const textfile{FileCased(name + ".t3d")}; + auto const binaryfile{FileCased(asBinary)}; // Hirek: Jesli mamy ustawione priorityLoadText3D na yes to wpierw ladujemy t3d - if (Global.priorityLoadText3D && FileExists(name + ".t3d")) + if (Global.priorityLoadText3D && (false == textfile.empty())) { - WriteLog("Forced loading text model \"" + name + ".t3d\""); - LoadFromTextFile(name + ".t3d", dynamic); + WriteLog("Forced loading text model \"" + textfile + "\""); + LoadFromTextFile(textfile, dynamic); if (!dynamic) Init(); } - else if (FileExists(asBinary)) + else if (false == binaryfile.empty()) { - LoadFromBinFile(asBinary, dynamic); + LoadFromBinFile(binaryfile, dynamic); asBinary = ""; Init(); } - else if (FileExists(name + ".t3d")) + else if (false == textfile.empty()) { - LoadFromTextFile(name + ".t3d", dynamic); + LoadFromTextFile(textfile, dynamic); if (!dynamic) Init(); } diff --git a/model/Texture.cpp b/model/Texture.cpp index cd275c9e0..e09e8c8ab 100644 --- a/model/Texture.cpp +++ b/model/Texture.cpp @@ -549,13 +549,15 @@ opengl_texture::load() { data_state = resource_state::loading; - if( type == ".dds" ) { load_DDS(); } - else if( type == ".tga" ) { load_TGA(); } - else if( type == ".png" ) { load_STBI(); } - else if( type == ".ktx" ) { load_KTX(); } - else if( type == ".bmp" ) { load_STBI(); } - else if( type == ".jpg" ) { load_STBI(); } - else if( type == ".tex" ) { load_TEX(); } + // the extension as the file has it on the disk, in any case + auto const format { ToLower( type ) }; + if( format == ".dds" ) { load_DDS(); } + else if( format == ".tga" ) { load_TGA(); } + else if( format == ".png" ) { load_STBI(); } + else if( format == ".ktx" ) { load_KTX(); } + else if( format == ".bmp" ) { load_STBI(); } + else if( format == ".jpg" ) { load_STBI(); } + else if( format == ".tex" ) { load_TEX(); } else { goto fail; } } diff --git a/tests/unit/test_utilities.cpp b/tests/unit/test_utilities.cpp index f04d9e34a..b8ad0b95c 100644 --- a/tests/unit/test_utilities.cpp +++ b/tests/unit/test_utilities.cpp @@ -578,6 +578,38 @@ TEST_CASE("utilities: a file is found by its name in another case") { CHECK(FileCased("eu07_unit_test_no_such_file.xyz").empty()); } +TEST_CASE("utilities: a file is found in folders named in another case, the way the parser gives the names in lowercase") { + test::scratch_folder const folder{"utilities_cased_folders"}; + folder.write("Scenery/Braniewo/rozklady/Os91121.txt", "x"); + folder.write("Textures/Tabl/L101-2m.BMP", "x"); + + // every part of the path in lowercase: the file as it's on the disk where the case matters, the name given where it doesn't + auto const timetable{FileCased(folder.path("scenery/braniewo/rozklady/os91121.txt"))}; + REQUIRE_FALSE(timetable.empty()); + CHECK(FileExists(timetable)); + CHECK(ToLower(timetable) == ToLower(folder.path("Scenery/Braniewo/rozklady/Os91121.txt"))); + CHECK(FileExists(folder.path("scenery/braniewo/rozklady/os91121.txt"))); + // the folders written the Windows way as well + CHECK_FALSE(FileCased(folder.path() + "scenery\\braniewo\\rozklady\\os91121.txt").empty()); + // a folder missing in any case + CHECK(FileCased(folder.path("scenery/nowhere/rozklady/os91121.txt")).empty()); + CHECK_FALSE(FileExists(folder.path("scenery/nowhere/rozklady/os91121.txt"))); + + // among names and extensions: given back as named on the disk, so it's opened by that name, the extension too + auto const texture{FileExists(std::vector{folder.path("textures/tabl/l101-2m")}, std::vector{".dds", ".bmp"})}; + REQUIRE_FALSE(texture.first.empty()); + CHECK(ToLower(texture.second) == ".bmp"); + CHECK(FileExists(texture.first + texture.second)); + CHECK(ToLower(texture.first) == ToLower(folder.path("Textures/Tabl/L101-2m"))); + + // a file written after the folder was gone through is found as well + CHECK(FileCased(folder.path("textures/tabl/l7-2m.bmp")).empty()); + folder.write("Textures/Tabl/L7-2m.bmp", "x"); + auto const added{FileCased(folder.path("textures/tabl/l7-2m.bmp"))}; + REQUIRE_FALSE(added.empty()); + CHECK(FileExists(added)); +} + TEST_CASE("utilities: the time a file was changed is given, zero for a file which isn't there") { test::scratch_folder const folder{"utilities_modified"}; auto const name{folder.write("fresh.txt", "x")}; diff --git a/utilities/utilities.cpp b/utilities/utilities.cpp index 550ad016a..85b40d330 100644 --- a/utilities/utilities.cpp +++ b/utilities/utilities.cpp @@ -513,9 +513,55 @@ template <> bool extract_value(bool &Variable, std::string const &Key, std::stri bool FileExists(std::string const &Filename) { +#ifdef _WIN32 return std::filesystem::exists(Filename); +#else + // the names come in lowercase from the parser, the files of the data are named in any case + return false == FileCased(Filename).empty(); +#endif } +#ifndef _WIN32 +namespace +{ + +// the names in a folder by their lowercase, and the time the folder was changed when they were read +struct cased_folder +{ + std::filesystem::file_time_type changed{}; + std::unordered_map names; +}; + +std::mutex cased_mutex; +std::unordered_map cased_folders; + +// the name in the folder matching the one given in any case, empty when there's none. the folder is read again once it was +// changed, as the editor writes files while it runs. to be called with cased_mutex held +std::string cased_name(std::string const &Folder, std::string const &Name) +{ + std::filesystem::path const folder{Folder.empty() ? std::string{"."} : Folder}; + std::error_code error; + auto const changed{std::filesystem::last_write_time(folder, error)}; + if (error) + return {}; + auto &cached{cased_folders[Folder]}; + if (cached.names.empty() || cached.changed != changed) + { + cached.changed = changed; + cached.names.clear(); + for (std::filesystem::directory_iterator entry{folder, error}, end; (false == bool(error)) && (entry != end); entry.increment(error)) + { + auto const name{entry->path().filename().string()}; + cached.names.try_emplace(ToLower(name), name); + } + } + auto const found{cached.names.find(ToLower(Name))}; + return found != cached.names.end() ? found->second : std::string{}; +} + +} // namespace +#endif + std::string FileCased(std::filesystem::path const &Filename) { std::error_code error; @@ -526,13 +572,40 @@ std::string FileCased(std::filesystem::path const &Filename) // code page to throw on return {}; #else - // only the name of the file itself, a folder named in another case isn't found - auto const directory{Filename.has_parent_path() ? Filename.parent_path() : std::filesystem::path{"."}}; - auto const wanted{ToLower(Filename.filename().string())}; - for (std::filesystem::directory_iterator entry{directory, error}, end; entry != end; entry.increment(error)) - if (ToLower(entry->path().filename().string()) == wanted) - return Filename.has_parent_path() ? (directory / entry->path().filename()).string() : entry->path().filename().string(); - return {}; + // every part of the path in turn, the folders can be named in another case as well as the file + auto const text{Filename.string()}; + std::string cased; + std::size_t start{0}; + if ((false == text.empty()) && (text.front() == '/')) + { + cased = "/"; + start = 1; + } + while (start < text.size()) + { + auto end{text.find_first_of("/\\", start)}; + if (end == std::string::npos) + end = text.size(); + auto const part{text.substr(start, end - start)}; + start = end + 1; + if (part.empty()) + continue; + if ((part == ".") || (part == "..") || std::filesystem::exists(cased + part, error)) + { + cased += part; + } + else + { + std::scoped_lock const lock{cased_mutex}; + auto const name{cased_name(cased, part)}; + if (name.empty()) + return {}; + cased += name; + } + if (end < text.size()) + cased += '/'; + } + return std::filesystem::exists(cased, error) ? cased : std::string{}; #endif } @@ -543,10 +616,19 @@ std::pair FileExists(std::vector const &N { for (auto const &extension : Extensions) { +#ifdef _WIN32 if (FileExists(name + extension)) { return {name, extension}; } +#else + // given back as it's named on the disk, so it's opened by that name + if (auto const cased{FileCased(name + extension)}; cased.size() >= extension.size()) + { + auto const split{cased.size() - extension.size()}; + return {cased.substr(0, split), cased.substr(split)}; + } +#endif } } // nothing found diff --git a/utilities/utilities.h b/utilities/utilities.h index 748c4c8d9..2e347b933 100644 --- a/utilities/utilities.h +++ b/utilities/utilities.h @@ -222,10 +222,11 @@ template bool extract_value(Type_ &Variable, std::string const template <> bool extract_value(bool &Variable, std::string const &Key, std::string const &Input, std::string const &Default); bool FileExists(std::string const &Filename); -// the file as it's named on the disk when only the case of its name differs, as in the sceneries made on Windows; empty when -// there's none. the folders have to be named in their case; on Windows, finding the names in any case itself, the file given or none +// the file as it's named on the disk when only the case of its name or of its folders differs, as in the sceneries made on Windows; +// empty when there's none. on Windows, finding the names in any case itself, the file given or none std::string FileCased(std::filesystem::path const &Filename); +// the first name with one of the extensions which has a file, in the order given, as the file is named on the disk std::pair FileExists(std::vector const &Names, std::vector const &Extensions); // returns time of last modification for specified file diff --git a/world/mtable.cpp b/world/mtable.cpp index 123ae00f6..0ac08bc20 100644 --- a/world/mtable.cpp +++ b/world/mtable.cpp @@ -277,6 +277,15 @@ bool TTrainParameters::LoadTTfile(std::string const &scnpath, int iPlus, double // Ra 2014-09: ustalić zasady wyznaczenia pierwotnego pliku przy przesuniętych rozkładach // (kolejny pociąg dostaje numer +2) fin.open(s.c_str()); // otwieranie pliku + if (false == fin.is_open()) + { + // the name of the train comes in lowercase, the file can be named in another case + if (auto const cased{FileCased(s)}; false == cased.empty()) + { + fin.clear(); + fin.open(cased); + } + } replace_slashes(s); if (!fin.is_open()) From afbc84c6bc9d5d06584d0ac6e43cef44f220dd23 Mon Sep 17 00:00:00 2001 From: maj00r Date: Fri, 13 Nov 2026 15:47:52 +0100 Subject: [PATCH 2/7] fix(files): the banks keep the names asked for, only the opening takes the case of the disk The textures, materials and sounds are kept by the names they're asked for. Given back as named on the disk, a file in another case missed the bank and was loaded again on every request. - FileExists with names and extensions finds the file in any case but gives back the name asked for; FileOnDisk gives the file to open, used where textures, sounds and scripts are opened - FileExists of a single name is a plain check again, the places checking it and opening the same name stay as they were - the texture format is told by the extension asked for again - a folder changed shortly before it was read is read again for a name missing from it, the time of a change being as coarse as the clock of the file system; a listing cut short isn't kept --- audio/audio.cpp | 3 ++- model/Texture.cpp | 28 ++++++++++---------- scripting/PyInt.cpp | 2 +- tests/unit/test_utilities.cpp | 16 ++++++------ utilities/utilities.cpp | 48 ++++++++++++++++++++--------------- utilities/utilities.h | 5 +++- 6 files changed, 55 insertions(+), 47 deletions(-) diff --git a/audio/audio.cpp b/audio/audio.cpp index c9aa56675..ad53d95de 100644 --- a/audio/audio.cpp +++ b/audio/audio.cpp @@ -35,7 +35,8 @@ decoded_sound decode_sound( std::string const &File ) { SF_INFO si; si.format = 0; - SNDFILE *sf = sf_open(File.c_str(), SFM_READ, &si); + // the file as it's named on the disk, the name of the buffer is in lowercase + SNDFILE *sf = sf_open(FileOnDisk(File).c_str(), SFM_READ, &si); if (sf == nullptr) throw std::runtime_error("sound: sf_open failed"); diff --git a/model/Texture.cpp b/model/Texture.cpp index e09e8c8ab..96fb3778d 100644 --- a/model/Texture.cpp +++ b/model/Texture.cpp @@ -549,15 +549,13 @@ opengl_texture::load() { data_state = resource_state::loading; - // the extension as the file has it on the disk, in any case - auto const format { ToLower( type ) }; - if( format == ".dds" ) { load_DDS(); } - else if( format == ".tga" ) { load_TGA(); } - else if( format == ".png" ) { load_STBI(); } - else if( format == ".ktx" ) { load_KTX(); } - else if( format == ".bmp" ) { load_STBI(); } - else if( format == ".jpg" ) { load_STBI(); } - else if( format == ".tex" ) { load_TEX(); } + if( type == ".dds" ) { load_DDS(); } + else if( type == ".tga" ) { load_TGA(); } + else if( type == ".png" ) { load_STBI(); } + else if( type == ".ktx" ) { load_KTX(); } + else if( type == ".bmp" ) { load_STBI(); } + else if( type == ".jpg" ) { load_STBI(); } + else if( type == ".tex" ) { load_TEX(); } else { goto fail; } } @@ -642,7 +640,7 @@ void opengl_texture::load_STBI() { int x, y, n; stbi_set_flip_vertically_on_load(1); - uint8_t *image = stbi_load((name + type).c_str(), &x, &y, &n, 4); + uint8_t *image = stbi_load(FileOnDisk(name + type).c_str(), &x, &y, &n, 4); if (!image) { data_state = resource_state::failed; @@ -820,7 +818,7 @@ DDSURFACEDESC2 opengl_texture::deserialize_ddsd(std::istream &s) void opengl_texture::load_DDS() { - std::ifstream file( name + type, std::ios::binary | std::ios::ate ); file.unsetf( std::ios::skipws ); + std::ifstream file( FileOnDisk( name + type ), std::ios::binary | std::ios::ate ); file.unsetf( std::ios::skipws ); std::size_t filesize = static_cast(file.tellg()); // ios::ate already positioned us at the end of the file file.seekg( 0, std::ios::beg ); // rewind the caret afterwards @@ -940,7 +938,7 @@ opengl_texture::load_DDS() { if( true == Global.gfx_texture_streaming ) { // the data is read by a worker thread, everything else about the texture is known already request = std::make_shared(); - request->file = name + type; + request->file = FileOnDisk( name + type ); request->offset = static_cast( file.tellg() ) + offset; request->size = datasize; request->flip = flip; @@ -1002,7 +1000,7 @@ opengl_texture::load_DDS() { void opengl_texture::load_KTX() { - std::ifstream file( name + type, std::ios::binary | std::ios::ate ); file.unsetf( std::ios::skipws ); + std::ifstream file( FileOnDisk( name + type ), std::ios::binary | std::ios::ate ); file.unsetf( std::ios::skipws ); std::size_t filesize = static_cast(file.tellg()); // ios::ate already positioned us at the end of the file file.seekg( 0, std::ios::beg ); // rewind the caret afterwards @@ -1063,7 +1061,7 @@ opengl_texture::load_KTX() { void opengl_texture::load_TEX() { - std::ifstream file( name + type, std::ios::binary ); file.unsetf( std::ios::skipws ); + std::ifstream file( FileOnDisk( name + type ), std::ios::binary ); file.unsetf( std::ios::skipws ); char head[ 5 ]; file.read( head, 4 ); @@ -1110,7 +1108,7 @@ opengl_texture::load_TEX() { void opengl_texture::load_TGA() { - std::ifstream file( name + type, std::ios::binary ); file.unsetf( std::ios::skipws ); + std::ifstream file( FileOnDisk( name + type ), std::ios::binary ); file.unsetf( std::ios::skipws ); // Read the header of the TGA, compare it with the known headers for compressed and uncompressed TGAs unsigned char tgaheader[ 18 ]; diff --git a/scripting/PyInt.cpp b/scripting/PyInt.cpp index 666609b0e..fac540019 100644 --- a/scripting/PyInt.cpp +++ b/scripting/PyInt.cpp @@ -443,7 +443,7 @@ auto python_taskqueue::run_file(std::string const &File, std::string const &Path return false; } - std::ifstream inputfile{filepath + fileextension}; + std::ifstream inputfile{FileOnDisk(filepath + fileextension)}; std::string input; input.assign(std::istreambuf_iterator(inputfile), std::istreambuf_iterator()); diff --git a/tests/unit/test_utilities.cpp b/tests/unit/test_utilities.cpp index b8ad0b95c..45a919568 100644 --- a/tests/unit/test_utilities.cpp +++ b/tests/unit/test_utilities.cpp @@ -586,28 +586,28 @@ TEST_CASE("utilities: a file is found in folders named in another case, the way // every part of the path in lowercase: the file as it's on the disk where the case matters, the name given where it doesn't auto const timetable{FileCased(folder.path("scenery/braniewo/rozklady/os91121.txt"))}; REQUIRE_FALSE(timetable.empty()); - CHECK(FileExists(timetable)); + CHECK(std::filesystem::exists(timetable)); CHECK(ToLower(timetable) == ToLower(folder.path("Scenery/Braniewo/rozklady/Os91121.txt"))); - CHECK(FileExists(folder.path("scenery/braniewo/rozklady/os91121.txt"))); + CHECK(FileOnDisk(folder.path("scenery/braniewo/rozklady/os91121.txt")) == timetable); // the folders written the Windows way as well CHECK_FALSE(FileCased(folder.path() + "scenery\\braniewo\\rozklady\\os91121.txt").empty()); // a folder missing in any case CHECK(FileCased(folder.path("scenery/nowhere/rozklady/os91121.txt")).empty()); - CHECK_FALSE(FileExists(folder.path("scenery/nowhere/rozklady/os91121.txt"))); + CHECK(FileOnDisk(folder.path("scenery/nowhere/rozklady/os91121.txt")) == folder.path("scenery/nowhere/rozklady/os91121.txt")); - // among names and extensions: given back as named on the disk, so it's opened by that name, the extension too + // among names and extensions: found in any case, given back as named, the name being the key of the banks auto const texture{FileExists(std::vector{folder.path("textures/tabl/l101-2m")}, std::vector{".dds", ".bmp"})}; REQUIRE_FALSE(texture.first.empty()); - CHECK(ToLower(texture.second) == ".bmp"); - CHECK(FileExists(texture.first + texture.second)); - CHECK(ToLower(texture.first) == ToLower(folder.path("Textures/Tabl/L101-2m"))); + CHECK(texture.first == folder.path("textures/tabl/l101-2m")); + CHECK(texture.second == ".bmp"); + CHECK(std::filesystem::exists(FileOnDisk(texture.first + texture.second))); // a file written after the folder was gone through is found as well CHECK(FileCased(folder.path("textures/tabl/l7-2m.bmp")).empty()); folder.write("Textures/Tabl/L7-2m.bmp", "x"); auto const added{FileCased(folder.path("textures/tabl/l7-2m.bmp"))}; REQUIRE_FALSE(added.empty()); - CHECK(FileExists(added)); + CHECK(std::filesystem::exists(added)); } TEST_CASE("utilities: the time a file was changed is given, zero for a file which isn't there") { diff --git a/utilities/utilities.cpp b/utilities/utilities.cpp index 85b40d330..deb63f2bd 100644 --- a/utilities/utilities.cpp +++ b/utilities/utilities.cpp @@ -513,22 +513,19 @@ template <> bool extract_value(bool &Variable, std::string const &Key, std::stri bool FileExists(std::string const &Filename) { -#ifdef _WIN32 return std::filesystem::exists(Filename); -#else - // the names come in lowercase from the parser, the files of the data are named in any case - return false == FileCased(Filename).empty(); -#endif } #ifndef _WIN32 namespace { -// the names in a folder by their lowercase, and the time the folder was changed when they were read +// the names in a folder by their lowercase, the time the folder was changed and the time they were read struct cased_folder { + bool read{false}; std::filesystem::file_time_type changed{}; + std::filesystem::file_time_type listed{}; std::unordered_map names; }; @@ -536,7 +533,8 @@ std::mutex cased_mutex; std::unordered_map cased_folders; // the name in the folder matching the one given in any case, empty when there's none. the folder is read again once it was -// changed, as the editor writes files while it runs. to be called with cased_mutex held +// changed, as the editor writes files while it runs; a name missing from a folder changed shortly before it was read is looked +// for again, the time of the change being as coarse as the clock of the file system. to be called with cased_mutex held std::string cased_name(std::string const &Folder, std::string const &Name) { std::filesystem::path const folder{Folder.empty() ? std::string{"."} : Folder}; @@ -545,17 +543,27 @@ std::string cased_name(std::string const &Folder, std::string const &Name) if (error) return {}; auto &cached{cased_folders[Folder]}; - if (cached.names.empty() || cached.changed != changed) - { - cached.changed = changed; + auto const wanted{ToLower(Name)}; + auto const read{[&]() { cached.names.clear(); + cached.changed = changed; + cached.listed = std::filesystem::file_time_type::clock::now(); for (std::filesystem::directory_iterator entry{folder, error}, end; (false == bool(error)) && (entry != end); entry.increment(error)) { auto const name{entry->path().filename().string()}; cached.names.try_emplace(ToLower(name), name); } + // a listing cut short isn't kept + cached.read = (false == bool(error)); + }}; + if ((false == cached.read) || (cached.changed != changed)) + read(); + auto found{cached.names.find(wanted)}; + if ((found == cached.names.end()) && (cached.listed - cached.changed < std::chrono::seconds{2})) + { + read(); + found = cached.names.find(wanted); } - auto const found{cached.names.find(ToLower(Name))}; return found != cached.names.end() ? found->second : std::string{}; } @@ -609,6 +617,12 @@ std::string FileCased(std::filesystem::path const &Filename) #endif } +std::string FileOnDisk(std::string const &Filename) +{ + auto cased{FileCased(Filename)}; + return cased.empty() ? Filename : cased; +} + std::pair FileExists(std::vector const &Names, std::vector const &Extensions) { @@ -616,19 +630,11 @@ std::pair FileExists(std::vector const &N { for (auto const &extension : Extensions) { -#ifdef _WIN32 - if (FileExists(name + extension)) + // found in any case, given back as named here: the name stays the key of the banks, the file is opened by FileOnDisk() + if (false == FileCased(name + extension).empty()) { return {name, extension}; } -#else - // given back as it's named on the disk, so it's opened by that name - if (auto const cased{FileCased(name + extension)}; cased.size() >= extension.size()) - { - auto const split{cased.size() - extension.size()}; - return {cased.substr(0, split), cased.substr(split)}; - } -#endif } } // nothing found diff --git a/utilities/utilities.h b/utilities/utilities.h index 2e347b933..d15e03234 100644 --- a/utilities/utilities.h +++ b/utilities/utilities.h @@ -225,8 +225,11 @@ bool FileExists(std::string const &Filename); // the file as it's named on the disk when only the case of its name or of its folders differs, as in the sceneries made on Windows; // empty when there's none. on Windows, finding the names in any case itself, the file given or none std::string FileCased(std::filesystem::path const &Filename); +// the file to open: as it's named on the disk, the name given when there's none +std::string FileOnDisk(std::string const &Filename); -// the first name with one of the extensions which has a file, in the order given, as the file is named on the disk +// the first name with one of the extensions which has a file in any case, in the order given; given back as named here, the file +// is opened by FileOnDisk() std::pair FileExists(std::vector const &Names, std::vector const &Extensions); // returns time of last modification for specified file From a30b5f30e156cd0804f0148be5c550708f78c2a6 Mon Sep 17 00:00:00 2001 From: maj00r Date: Fri, 13 Nov 2026 15:49:41 +0100 Subject: [PATCH 3/7] fix(files): a folder changed in the future isn't read again on every name missing from it --- utilities/utilities.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utilities/utilities.cpp b/utilities/utilities.cpp index deb63f2bd..6b869236e 100644 --- a/utilities/utilities.cpp +++ b/utilities/utilities.cpp @@ -559,7 +559,9 @@ std::string cased_name(std::string const &Folder, std::string const &Name) if ((false == cached.read) || (cached.changed != changed)) read(); auto found{cached.names.find(wanted)}; - if ((found == cached.names.end()) && (cached.listed - cached.changed < std::chrono::seconds{2})) + // a folder changed in the future, as from another clock, isn't read again on every miss + auto const age{cached.listed - cached.changed}; + if ((found == cached.names.end()) && (age >= std::chrono::seconds{0}) && (age < std::chrono::seconds{2})) { read(); found = cached.names.find(wanted); From 1e5b09515039aa81471553b3616196e82cb5aab9 Mon Sep 17 00:00:00 2001 From: maj00r Date: Fri, 13 Nov 2026 16:18:27 +0100 Subject: [PATCH 4/7] perf(files): a file missing costs a single check, as before Measured in a case sensitive folder (3000 textures, every 20th in mixed case, looked up the way Texture.cpp does), the walk over every part of the path made a texture lookup 4-5 times slower than master: each name tried and missing checked every folder of its path. - the folders found are kept by the name they're given with, the missing ones for a second, so a missing file checks its folder alone - a folder named as it is on the disk is taken without its listing - the time a folder was changed is looked at once a second at most The lookup is now as fast as master's (about 180 us per texture on NTFS in both) while finding the files named in another case too. --- utilities/utilities.cpp | 133 ++++++++++++++++++++++++++++++---------- 1 file changed, 100 insertions(+), 33 deletions(-) diff --git a/utilities/utilities.cpp b/utilities/utilities.cpp index 6b869236e..fbccbab6a 100644 --- a/utilities/utilities.cpp +++ b/utilities/utilities.cpp @@ -526,23 +526,46 @@ struct cased_folder bool read{false}; std::filesystem::file_time_type changed{}; std::filesystem::file_time_type listed{}; + // when the time of the change was last looked at + std::chrono::steady_clock::time_point checked{}; std::unordered_map names; }; std::mutex cased_mutex; std::unordered_map cased_folders; +// a folder by the name it's given with (parts split by '/', a '/' at the end): as it's named on the disk, or missing in any case +// since the time given +struct cased_path +{ + std::string name; + bool missing{false}; + std::chrono::steady_clock::time_point checked{}; +}; +std::unordered_map cased_paths; // the name in the folder matching the one given in any case, empty when there's none. the folder is read again once it was -// changed, as the editor writes files while it runs; a name missing from a folder changed shortly before it was read is looked -// for again, the time of the change being as coarse as the clock of the file system. to be called with cased_mutex held +// changed, as the editor writes files while it runs, the time of the change looked at once a second at most, which keeps a +// missing file as cheap as a single check; a name missing from a folder changed shortly before it was read is looked for again, +// the time of the change being as coarse as the clock of the file system. to be called with cased_mutex held std::string cased_name(std::string const &Folder, std::string const &Name) { std::filesystem::path const folder{Folder.empty() ? std::string{"."} : Folder}; std::error_code error; - auto const changed{std::filesystem::last_write_time(folder, error)}; - if (error) - return {}; auto &cached{cased_folders[Folder]}; + auto const now{std::chrono::steady_clock::now()}; + auto changed{cached.changed}; + if ((false == cached.read) || (now - cached.checked >= std::chrono::seconds{1})) + { + changed = std::filesystem::last_write_time(folder, error); + if (error) + { + // a folder found before may have been renamed or removed since + cased_folders.erase(Folder); + cased_paths.clear(); + return {}; + } + cached.checked = now; + } auto const wanted{ToLower(Name)}; auto const read{[&]() { cached.names.clear(); @@ -569,6 +592,53 @@ std::string cased_name(std::string const &Folder, std::string const &Name) return found != cached.names.end() ? found->second : std::string{}; } +// the folder as it's named on the disk into Cased, from the one given with '/' between its parts and at its end ("" the current +// folder); false when it's missing in any case. the folders found are kept, and the missing ones for a second, so a file missing +// costs a check of its folder alone. to be called with cased_mutex held +bool cased_folder_path(std::string const &Folder, std::string &Cased) +{ + if (Folder.empty() || (Folder == "/")) + { + Cased = Folder; + return true; + } + auto const now{std::chrono::steady_clock::now()}; + if (auto const found{cased_paths.find(Folder)}; found != cased_paths.end()) + { + if (false == found->second.missing) + { + Cased = found->second.name; + return true; + } + if (now - found->second.checked < std::chrono::seconds{1}) + return false; + } + auto const keep{[&](std::string const &Name) { + auto &path{cased_paths[Folder]}; + path.name = Name; + path.missing = Name.empty(); + path.checked = now; + Cased = Name; + return false == Name.empty(); + }}; + std::error_code error; + if (std::filesystem::is_directory(Folder, error)) + { + // named as it is on the disk + return keep(Folder); + } + // the parent, then the last part among the names in it + auto const last{Folder.size() - 1}; + auto const split{Folder.rfind('/', last - 1)}; + auto const parent{split == std::string::npos ? std::string{} : Folder.substr(0, split + 1)}; + auto const part{Folder.substr(parent.size(), last - parent.size())}; + std::string casedparent; + if (false == cased_folder_path(parent, casedparent)) + return keep({}); + auto const name{((part == ".") || (part == "..")) ? part : cased_name(casedparent, part)}; + return keep(name.empty() ? std::string{} : casedparent + name + '/'); +} + } // namespace #endif @@ -582,38 +652,35 @@ std::string FileCased(std::filesystem::path const &Filename) // code page to throw on return {}; #else - // every part of the path in turn, the folders can be named in another case as well as the file - auto const text{Filename.string()}; - std::string cased; - std::size_t start{0}; - if ((false == text.empty()) && (text.front() == '/')) + // the folder, then the file in it: the folders can be named in another case as well as the file + auto text{Filename.string()}; + std::replace(text.begin(), text.end(), '\\', '/'); + std::string folder{(false == text.empty()) && (text.front() == '/') ? "/" : ""}; + std::string file; + for (std::size_t start{0}; start < text.size();) { - cased = "/"; - start = 1; - } - while (start < text.size()) - { - auto end{text.find_first_of("/\\", start)}; + auto end{text.find('/', start)}; if (end == std::string::npos) end = text.size(); - auto const part{text.substr(start, end - start)}; + if (end > start) + { + if (false == file.empty()) + folder += file + '/'; + file = text.substr(start, end - start); + } start = end + 1; - if (part.empty()) - continue; - if ((part == ".") || (part == "..") || std::filesystem::exists(cased + part, error)) - { - cased += part; - } - else - { - std::scoped_lock const lock{cased_mutex}; - auto const name{cased_name(cased, part)}; - if (name.empty()) - return {}; - cased += name; - } - if (end < text.size()) - cased += '/'; + } + if (file.empty()) + return {}; + std::string cased; + { + std::scoped_lock const lock{cased_mutex}; + if (false == cased_folder_path(folder, cased)) + return {}; + auto const name{((file == ".") || (file == "..")) ? file : cased_name(cased, file)}; + if (name.empty()) + return {}; + cased += name; } return std::filesystem::exists(cased, error) ? cased : std::string{}; #endif From f9b7e9174c7dcf3818cb01d6da06379e512add58 Mon Sep 17 00:00:00 2001 From: maj00r Date: Fri, 13 Nov 2026 16:34:06 +0100 Subject: [PATCH 5/7] fix(files): terrain files named in another case are found on Linux The terrain directive names .txtf/.btf files in lowercase, as the parser gives every token. The name is now taken as it is on disk, from the text file first, so the binary file is made next to the text and the editor's references and layers keep one name. --- simulation/simulationstateserializer.cpp | 2 ++ tests/unit/test_utilities.cpp | 18 ++++++++++++++++++ utilities/utilities.cpp | 13 +++++++++++++ utilities/utilities.h | 3 +++ 4 files changed, 36 insertions(+) diff --git a/simulation/simulationstateserializer.cpp b/simulation/simulationstateserializer.cpp index 9d1e94869..0a8670c4e 100644 --- a/simulation/simulationstateserializer.cpp +++ b/simulation/simulationstateserializer.cpp @@ -1200,6 +1200,8 @@ include_terrain_file( std::string File, std::string &Includes, scene::scratch_da replace_slashes( File ); erase_extension( File ); + // the parser gives the name in lowercase, the files may be named in another case; the binary file is made next to the text + File = FileStemOnDisk( Global.asCurrentSceneryPath, File, { ".txtf", ".btf" } ); if( std::find( binary.terrain_textfiles.begin(), binary.terrain_textfiles.end(), File ) != binary.terrain_textfiles.end() ) { WriteLog( "Terrain file: " + File + " is already in use, ignored" ); return; diff --git a/tests/unit/test_utilities.cpp b/tests/unit/test_utilities.cpp index 45a919568..f8cd9a290 100644 --- a/tests/unit/test_utilities.cpp +++ b/tests/unit/test_utilities.cpp @@ -610,6 +610,24 @@ TEST_CASE("utilities: a file is found in folders named in another case, the way CHECK(std::filesystem::exists(added)); } +TEST_CASE("utilities: a terrain file is found in another case, by its text or by its binary file") { + test::scratch_folder const folder{"utilities_cased_terrain"}; + folder.write("Scenery/L204/Teren/Woda_ter.txtf", "x"); + folder.write("Scenery/Nmt/Gory_ter.btf", "x"); + auto const scenery{folder.path("Scenery/")}; + std::vector const extensions{".txtf", ".btf"}; + + // the name as the parser gives it: the files are there under the name given back, which differs only in case + auto const text{FileStemOnDisk(scenery, "l204/teren/woda_ter", extensions)}; + CHECK(ToLower(text) == "l204/teren/woda_ter"); + CHECK(std::filesystem::exists(scenery + text + ".txtf")); + auto const binary{FileStemOnDisk(scenery, "nmt/gory_ter", extensions)}; + CHECK(ToLower(binary) == "nmt/gory_ter"); + CHECK(std::filesystem::exists(scenery + binary + ".btf")); + // neither file in any case: the name stays as given + CHECK(FileStemOnDisk(scenery, "nmt/nowhere_ter", extensions) == "nmt/nowhere_ter"); +} + TEST_CASE("utilities: the time a file was changed is given, zero for a file which isn't there") { test::scratch_folder const folder{"utilities_modified"}; auto const name{folder.write("fresh.txt", "x")}; diff --git a/utilities/utilities.cpp b/utilities/utilities.cpp index fbccbab6a..d7406f3bc 100644 --- a/utilities/utilities.cpp +++ b/utilities/utilities.cpp @@ -692,6 +692,19 @@ std::string FileOnDisk(std::string const &Filename) return cased.empty() ? Filename : cased; } +std::string FileStemOnDisk(std::string const &Folder, std::string const &Name, std::vector const &Extensions) +{ + for (auto const &extension : Extensions) + { + auto const cased{FileCased(Folder + Name + extension)}; + if (cased.size() > Folder.size() + extension.size() && cased.starts_with(Folder) && cased.ends_with(extension)) + { + return cased.substr(Folder.size(), cased.size() - Folder.size() - extension.size()); + } + } + return Name; +} + std::pair FileExists(std::vector const &Names, std::vector const &Extensions) { diff --git a/utilities/utilities.h b/utilities/utilities.h index d15e03234..09c02d7fb 100644 --- a/utilities/utilities.h +++ b/utilities/utilities.h @@ -227,6 +227,9 @@ bool FileExists(std::string const &Filename); std::string FileCased(std::filesystem::path const &Filename); // the file to open: as it's named on the disk, the name given when there's none std::string FileOnDisk(std::string const &Filename); +// the name (relative to the folder, without extension) as it's named on the disk with the first of the extensions it has a file with, +// found in any case; the name given when there's none +std::string FileStemOnDisk(std::string const &Folder, std::string const &Name, std::vector const &Extensions); // the first name with one of the extensions which has a file in any case, in the order given; given back as named here, the file // is opened by FileOnDisk() From d76e6368b06ba17a2f107ab38a9815f0796ba32a Mon Sep 17 00:00:00 2001 From: maj00r Date: Fri, 13 Nov 2026 17:49:18 +0100 Subject: [PATCH 6/7] fix(files): models named in another case are found on Linux The manager of the models looked for the file with FileExists of a single name, a plain check again since the banks keep the names asked for, so a model named in another case (models/przejazdy/plyty3_L, dynamic/pkp/st44_v2/main/2M62-0685-A) was reported missing before TModel3d::LoadFromFile() could open it as it's named on the disk. find_on_disk() finds the file in any case and gives back the name asked for, the key of the bank. --- model/MdlMngr.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/model/MdlMngr.cpp b/model/MdlMngr.cpp index 680f67c42..52bc4047e 100644 --- a/model/MdlMngr.cpp +++ b/model/MdlMngr.cpp @@ -137,15 +137,19 @@ TModelsManager::find_in_databank( std::string const &Name ) { } // checks whether specified file exists. returns name of the located file, or empty string. +// the file is found in any case, as the scenery made on Windows names it, but the name is given back as asked for: it's the key of +// the bank, and TModel3d::LoadFromFile() opens the file as it's named on the disk std::string TModelsManager::find_on_disk( std::string const &Name ) { std::vector extensions { { ".e3d" }, { ".t3d" } }; for( auto const &extension : extensions ) { - auto lookup = FileExists(Name + extension) ? Name : FileExists(paths::models + Name + extension) ? paths::models + Name : ""; - if( false == lookup.empty() ) { - return lookup; + if( false == FileCased( Name + extension ).empty() ) { + return Name; + } + if( false == FileCased( paths::models + Name + extension ).empty() ) { + return paths::models + Name; } } From 38540d08f53b0b1d8f2be7b9aeea91153d997838 Mon Sep 17 00:00:00 2001 From: maj00r Date: Fri, 13 Nov 2026 17:49:18 +0100 Subject: [PATCH 7/7] refactor(files): named functions instead of the lambdas of the folder cache (Sonar S2259, S3608, S6004) Sonar took the folder captured by the lambda reading it for a reference to null, failing the quality gate on reliability. The folder is read and the path kept by functions in the anonymous namespace, the age of the listing declared in its if. --- utilities/utilities.cpp | 62 +++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/utilities/utilities.cpp b/utilities/utilities.cpp index d7406f3bc..d704fe9b7 100644 --- a/utilities/utilities.cpp +++ b/utilities/utilities.cpp @@ -543,6 +543,23 @@ struct cased_path }; std::unordered_map cased_paths; +// the names in the folder into Cached by their lowercase, with the time the folder was changed and the time they were read. a named +// function rather than a lambda capturing Cached, which Sonar takes for a reference to null (S2259) +void read_cased_folder(cased_folder &Cached, std::filesystem::path const &Folder, std::filesystem::file_time_type const Changed) +{ + Cached.names.clear(); + Cached.changed = Changed; + Cached.listed = std::filesystem::file_time_type::clock::now(); + std::error_code error; + for (std::filesystem::directory_iterator entry{Folder, error}, end; (false == bool(error)) && (entry != end); entry.increment(error)) + { + auto const name{entry->path().filename().string()}; + Cached.names.try_emplace(ToLower(name), name); + } + // a listing cut short isn't kept + Cached.read = (false == bool(error)); +} + // the name in the folder matching the one given in any case, empty when there's none. the folder is read again once it was // changed, as the editor writes files while it runs, the time of the change looked at once a second at most, which keeps a // missing file as cheap as a single check; a name missing from a folder changed shortly before it was read is looked for again, @@ -567,31 +584,30 @@ std::string cased_name(std::string const &Folder, std::string const &Name) cached.checked = now; } auto const wanted{ToLower(Name)}; - auto const read{[&]() { - cached.names.clear(); - cached.changed = changed; - cached.listed = std::filesystem::file_time_type::clock::now(); - for (std::filesystem::directory_iterator entry{folder, error}, end; (false == bool(error)) && (entry != end); entry.increment(error)) - { - auto const name{entry->path().filename().string()}; - cached.names.try_emplace(ToLower(name), name); - } - // a listing cut short isn't kept - cached.read = (false == bool(error)); - }}; if ((false == cached.read) || (cached.changed != changed)) - read(); + read_cased_folder(cached, folder, changed); auto found{cached.names.find(wanted)}; // a folder changed in the future, as from another clock, isn't read again on every miss - auto const age{cached.listed - cached.changed}; - if ((found == cached.names.end()) && (age >= std::chrono::seconds{0}) && (age < std::chrono::seconds{2})) + if (auto const age{cached.listed - cached.changed}; (found == cached.names.end()) && (age >= std::chrono::seconds{0}) && (age < std::chrono::seconds{2})) { - read(); + read_cased_folder(cached, folder, changed); found = cached.names.find(wanted); } return found != cached.names.end() ? found->second : std::string{}; } +// keeps the folder by the name it's given with as Name, or as missing since the time given when Name is empty, and puts Name into +// Cased; false when it's missing +bool keep_cased_path(std::string const &Folder, std::string const &Name, std::chrono::steady_clock::time_point const Now, std::string &Cased) +{ + auto &path{cased_paths[Folder]}; + path.name = Name; + path.missing = Name.empty(); + path.checked = Now; + Cased = Name; + return false == Name.empty(); +} + // the folder as it's named on the disk into Cased, from the one given with '/' between its parts and at its end ("" the current // folder); false when it's missing in any case. the folders found are kept, and the missing ones for a second, so a file missing // costs a check of its folder alone. to be called with cased_mutex held @@ -613,19 +629,11 @@ bool cased_folder_path(std::string const &Folder, std::string &Cased) if (now - found->second.checked < std::chrono::seconds{1}) return false; } - auto const keep{[&](std::string const &Name) { - auto &path{cased_paths[Folder]}; - path.name = Name; - path.missing = Name.empty(); - path.checked = now; - Cased = Name; - return false == Name.empty(); - }}; std::error_code error; if (std::filesystem::is_directory(Folder, error)) { // named as it is on the disk - return keep(Folder); + return keep_cased_path(Folder, Folder, now, Cased); } // the parent, then the last part among the names in it auto const last{Folder.size() - 1}; @@ -634,9 +642,9 @@ bool cased_folder_path(std::string const &Folder, std::string &Cased) auto const part{Folder.substr(parent.size(), last - parent.size())}; std::string casedparent; if (false == cased_folder_path(parent, casedparent)) - return keep({}); + return keep_cased_path(Folder, {}, now, Cased); auto const name{((part == ".") || (part == "..")) ? part : cased_name(casedparent, part)}; - return keep(name.empty() ? std::string{} : casedparent + name + '/'); + return keep_cased_path(Folder, name.empty() ? std::string{} : casedparent + name + '/', now, Cased); } } // namespace