From 2ee5f9335335ccc5004c79dac04a65ebd8c643a5 Mon Sep 17 00:00:00 2001 From: maj00r Date: Sun, 14 Jun 2026 21:22:50 +0200 Subject: [PATCH] Add EU7 PACK v11 UTEX, section unload, and LRU streaming caches Bake deduplicated section texture lists (UTEX) and read them at runtime so texture warm/prefetch no longer scans every instance. Evict PACK sections outside a 16 km keep ring and cap mesh/nodedata/texture warm caches to stop RAM and apply cost from growing without bound during long fast camera flight. - Bake v11 (format=4): UMES + UTEX + CHNK; mesh/texture ordering by frequency - Runtime: incremental UTEX warm, adaptive apply slices, preempt/catchup tuning - Unload: track instances per section, Region erase, pack pool release - LRU: mesh/nodedata (2000, ref-counted), warmed textures (2048) - Bench/diagnostics: unload and cache eviction counters - Fix pack instance erase in Region; resolve duplicate helper symbols --- docs/eu7-format.md | 24 + .../eu07/scene/binary/runtime_module.hpp | 238 +++++- model/AnimModel.cpp | 9 +- rendering/particles.cpp | 17 + rendering/particles.h | 2 + scene/eu7/eu7_chunks.h | 1 + scene/eu7/eu7_model_prefetch.cpp | 182 ++++- scene/eu7/eu7_model_prefetch.h | 16 + scene/eu7/eu7_pack_bench.cpp | 4 + scene/eu7/eu7_pack_bench.h | 8 + scene/eu7/eu7_reader.cpp | 50 +- scene/eu7/eu7_section_stream.cpp | 686 ++++++++++++++++-- scene/eu7/eu7_types.h | 11 + scene/scene.cpp | 27 + simulation/simulationstateserializer.cpp | 47 +- simulation/simulationstateserializer.h | 10 + 16 files changed, 1228 insertions(+), 104 deletions(-) diff --git a/docs/eu7-format.md b/docs/eu7-format.md index cee532bb..cdac2c34 100644 --- a/docs/eu7-format.md +++ b/docs/eu7-format.md @@ -853,6 +853,30 @@ Powtórzone chunk_count razy: Runtime: worker czyta `read_pack_section_chunk_load(row, col, chunk_index)` z O(1) seek. Sekcje v9 (format=2) i v7 flat pozostają kompatybilne (`chunk_count` implicit = 1). +### Format sekcji — v11 UTEX (`kPackSectionFormatV11 = 4`) + +Rozszerzenie v10: po UMES deduplikowana lista ścieżek tekstur (`texture_file`), posortowana po częstotliwości. + +``` +uint8 section_format — musi być = 4 +uint32 solo_count +uint32 inst_count +uint32 unique_mesh_count +uint32[unique_mesh_count] string_id + +uint32 unique_texture_count +uint32[unique_texture_count] string_id — texture_file (pomija none/make:/@/#/.e3d) + +uint32 chunk_count +Powtórzone chunk_count razy: + uint32 chunk_model_count + uint32 chunk_byte_offset + +[payload chunków] +``` + +Runtime używa UTEX do prefetch/warm tekstur bez skanowania wszystkich instancji. + --- ## 30. PROT — prototypy modeli (v8+) diff --git a/eu07-parser/include/eu07/scene/binary/runtime_module.hpp b/eu07-parser/include/eu07/scene/binary/runtime_module.hpp index 17ef6a47..6496ed47 100644 --- a/eu07-parser/include/eu07/scene/binary/runtime_module.hpp +++ b/eu07-parser/include/eu07/scene/binary/runtime_module.hpp @@ -25,6 +25,7 @@ #include #include #include +#include #include namespace eu07::scene::binary { @@ -49,10 +50,13 @@ inline constexpr std::array kChunkPlac{'P', 'L', 'A', 'C'}; inline constexpr std::array kChunkPidx{'P', 'I', 'D', 'X'}; inline constexpr std::array kChunkPack{'P', 'A', 'C', 'K'}; -// PACK section payload: v9 = UMES, v10 = UMES + CHNK (sub-chunk offset table). +// PACK section payload: v9 = UMES, v10 = UMES + CHNK, v11 = UMES + UTEX + CHNK. inline constexpr std::uint8_t kPackSectionFormatV9 = 2; inline constexpr std::uint8_t kPackSectionFormatV10 = 3; +inline constexpr std::uint8_t kPackSectionFormatV11 = 4; inline constexpr std::size_t kPackSectionChunkModels = 512; +inline constexpr std::size_t kPackSectionChunkModelsHeavy = 256; +inline constexpr std::size_t kPackSectionHeavyModelThreshold = 1024; struct PackWriteStats { std::size_t strings_total = 0; @@ -408,28 +412,161 @@ struct PackPayloadBuild { return !path.empty() && path != "notload"; } +[[nodiscard]] inline bool isPackTexturePath(const std::string_view path) { + if (path.empty() || path == "none" || path.front() == '*') { + return false; + } + if (path.starts_with("make:") || path.starts_with("@") || path.starts_with("none|")) { + return false; + } + if (path.ends_with(".e3d") || path.ends_with(".t3d")) { + return false; + } + if (path == "none" || path.ends_with("/none") || path.ends_with('/')) { + return false; + } + if (path.find("tr/none") != std::string_view::npos || path.find('#') != std::string_view::npos) { + return false; + } + return true; +} + +[[nodiscard]] inline runtime::Vec3 packSectionCenter( + const codec::TerrSectionKey section) noexcept { + runtime::Vec3 center{}; + center.x = + (static_cast(section.x) - + static_cast(codec::kEu07RegionSideSectionCount) / 2.0 + 0.5) * + static_cast(codec::kEu07SectionSize); + center.z = + (static_cast(section.z) - + static_cast(codec::kEu07RegionSideSectionCount) / 2.0 + 0.5) * + static_cast(codec::kEu07SectionSize); + return center; +} + +inline void sortPackSectionModelsForStreaming( + std::vector& models, + const codec::TerrSectionKey section) { + if (models.size() < 2) { + return; + } + + std::unordered_map mesh_frequency; + mesh_frequency.reserve(models.size()); + for (const runtime::RuntimeModelInstance& model : models) { + if (isLoadableMeshPath(model.modelFile)) { + ++mesh_frequency[model.modelFile]; + } + } + + const runtime::Vec3 section_center = packSectionCenter(section); + const auto mesh_rank = [&](const std::string& path) -> std::uint32_t { + const auto it = mesh_frequency.find(path); + return it == mesh_frequency.end() ? 0u : it->second; + }; + const auto distance_squared = [&](const runtime::RuntimeModelInstance& model) -> double { + const double dx = model.location.x - section_center.x; + const double dz = model.location.z - section_center.z; + return dx * dx + dz * dz; + }; + + std::stable_sort( + models.begin(), + models.end(), + [&](const runtime::RuntimeModelInstance& lhs, const runtime::RuntimeModelInstance& rhs) { + const std::uint32_t lhs_rank = mesh_rank(lhs.modelFile); + const std::uint32_t rhs_rank = mesh_rank(rhs.modelFile); + if (lhs_rank != rhs_rank) { + return lhs_rank > rhs_rank; + } + return distance_squared(lhs) < distance_squared(rhs); + }); +} + [[nodiscard]] inline std::vector collectUniqueMeshIds( StringTable& table, const std::vector& models, const std::vector* prototype_mesh_paths = nullptr) { - std::vector paths; - paths.reserve(models.size()); + std::unordered_map mesh_frequency; + mesh_frequency.reserve(models.size()); for (const runtime::RuntimeModelInstance& model : models) { if (isLoadableMeshPath(model.modelFile)) { - paths.emplace_back(model.modelFile); + ++mesh_frequency[model.modelFile]; + } + } + + std::vector paths; + paths.reserve(mesh_frequency.size()); + for (const auto& [path, frequency] : mesh_frequency) { + if (frequency > 0) { + paths.emplace_back(path); } } if (prototype_mesh_paths != nullptr) { for (const std::string_view path : *prototype_mesh_paths) { if (isLoadableMeshPath(path)) { - paths.emplace_back(path); + const std::string text(path); + if (false == mesh_frequency.contains(text)) { + paths.emplace_back(text); + mesh_frequency.emplace(text, 0u); + } } } } - std::stable_sort(paths.begin(), paths.end()); - paths.erase(std::unique(paths.begin(), paths.end()), paths.end()); + std::stable_sort( + paths.begin(), + paths.end(), + [&](const std::string& lhs, const std::string& rhs) { + const std::uint32_t lhs_rank = mesh_frequency[lhs]; + const std::uint32_t rhs_rank = mesh_frequency[rhs]; + if (lhs_rank != rhs_rank) { + return lhs_rank > rhs_rank; + } + return lhs < rhs; + }); + + std::vector ids; + ids.reserve(paths.size()); + for (const std::string& path : paths) { + ids.push_back(table.intern(path)); + } + return ids; +} + +[[nodiscard]] inline std::vector collectUniqueTextureIds( + StringTable& table, + const std::vector& models) { + std::unordered_map texture_frequency; + texture_frequency.reserve(models.size()); + + for (const runtime::RuntimeModelInstance& model : models) { + if (isPackTexturePath(model.textureFile)) { + ++texture_frequency[model.textureFile]; + } + } + + std::vector paths; + paths.reserve(texture_frequency.size()); + for (const auto& [path, frequency] : texture_frequency) { + if (frequency > 0) { + paths.emplace_back(path); + } + } + + std::stable_sort( + paths.begin(), + paths.end(), + [&](const std::string& lhs, const std::string& rhs) { + const std::uint32_t lhs_rank = texture_frequency[lhs]; + const std::uint32_t rhs_rank = texture_frequency[rhs]; + if (lhs_rank != rhs_rank) { + return lhs_rank > rhs_rank; + } + return lhs < rhs; + }); std::vector ids; ids.reserve(paths.size()); @@ -461,12 +598,14 @@ struct PackPayloadBuild { const std::vector* prototype_mesh_paths = nullptr) { const auto mesh_ids = collectUniqueMeshIds(table, models, prototype_mesh_paths); const std::uint32_t solo_count = static_cast(models.size()); + const std::size_t chunk_models = + models.size() > kPackSectionHeavyModelThreshold ? kPackSectionChunkModelsHeavy + : kPackSectionChunkModels; std::vector> chunk_payloads; - chunk_payloads.reserve((models.size() + kPackSectionChunkModels - 1) / kPackSectionChunkModels); - for (std::size_t offset = 0; offset < models.size(); offset += kPackSectionChunkModels) { - const std::size_t end = - std::min(offset + kPackSectionChunkModels, models.size()); + chunk_payloads.reserve((models.size() + chunk_models - 1) / chunk_models); + for (std::size_t offset = 0; offset < models.size(); offset += chunk_models) { + const std::size_t end = std::min(offset + chunk_models, models.size()); chunk_payloads.push_back(buildPackModelBlob(table, models, offset, end)); } if (chunk_payloads.empty()) { @@ -483,10 +622,8 @@ struct PackPayloadBuild { std::uint32_t payload_offset = header_size; for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) { chunk_offsets[chunk_index] = payload_offset; - const std::size_t model_begin = - static_cast(chunk_index) * kPackSectionChunkModels; - const std::size_t model_end = - std::min(model_begin + kPackSectionChunkModels, models.size()); + const std::size_t model_begin = static_cast(chunk_index) * chunk_models; + const std::size_t model_end = std::min(model_begin + chunk_models, models.size()); chunk_model_counts[chunk_index] = static_cast(model_end > model_begin ? model_end - model_begin : 0); payload_offset += static_cast(chunk_payloads[chunk_index].size()); @@ -512,6 +649,69 @@ struct PackPayloadBuild { return {blob.begin(), blob.end()}; } +[[nodiscard]] inline std::vector buildPackSectionPayloadV11( + StringTable& table, + const std::vector& models, + const std::uint32_t inst_count = 0, + const std::vector* prototype_mesh_paths = nullptr) { + const auto mesh_ids = collectUniqueMeshIds(table, models, prototype_mesh_paths); + const auto texture_ids = collectUniqueTextureIds(table, models); + const std::uint32_t solo_count = static_cast(models.size()); + const std::size_t chunk_models = + models.size() > kPackSectionHeavyModelThreshold ? kPackSectionChunkModelsHeavy + : kPackSectionChunkModels; + + std::vector> chunk_payloads; + chunk_payloads.reserve((models.size() + chunk_models - 1) / chunk_models); + for (std::size_t offset = 0; offset < models.size(); offset += chunk_models) { + const std::size_t end = std::min(offset + chunk_models, models.size()); + chunk_payloads.push_back(buildPackModelBlob(table, models, offset, end)); + } + if (chunk_payloads.empty()) { + chunk_payloads.emplace_back(); + } + + const std::uint32_t chunk_count = static_cast(chunk_payloads.size()); + const std::uint32_t header_size = + 1u + 4u + 4u + 4u + static_cast(mesh_ids.size()) * 4u + 4u + + static_cast(texture_ids.size()) * 4u + 4u + chunk_count * 8u; + + std::vector chunk_offsets(chunk_count); + std::vector chunk_model_counts(chunk_count); + std::uint32_t payload_offset = header_size; + for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) { + chunk_offsets[chunk_index] = payload_offset; + const std::size_t model_begin = static_cast(chunk_index) * chunk_models; + const std::size_t model_end = std::min(model_begin + chunk_models, models.size()); + chunk_model_counts[chunk_index] = + static_cast(model_end > model_begin ? model_end - model_begin : 0); + payload_offset += static_cast(chunk_payloads[chunk_index].size()); + } + + std::ostringstream packOut(std::ios::binary); + io::writeU8(packOut, kPackSectionFormatV11); + io::writeU32(packOut, solo_count); + io::writeU32(packOut, inst_count); + io::writeU32(packOut, static_cast(mesh_ids.size())); + for (const std::uint32_t id : mesh_ids) { + io::writeU32(packOut, id); + } + io::writeU32(packOut, static_cast(texture_ids.size())); + for (const std::uint32_t id : texture_ids) { + io::writeU32(packOut, id); + } + io::writeU32(packOut, chunk_count); + for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) { + io::writeU32(packOut, chunk_model_counts[chunk_index]); + io::writeU32(packOut, chunk_offsets[chunk_index]); + } + for (const std::vector& payload : chunk_payloads) { + packOut.write(payload.data(), static_cast(payload.size())); + } + const std::string blob = packOut.str(); + return {blob.begin(), blob.end()}; +} + [[nodiscard]] inline PackPayloadBuild buildPackPayloadV7( StringTable& table, const std::vector& batches) { @@ -540,7 +740,9 @@ struct PackPayloadBuild { entry.row = static_cast(batch.section.z); entry.column = static_cast(batch.section.x); entry.modelCount = static_cast(batch.models.size()); - section_payloads[batch_index] = buildPackSectionPayloadV10(table, batch.models); + std::vector section_models = batch.models; + sortPackSectionModelsForStreaming(section_models, batch.section); + section_payloads[batch_index] = buildPackSectionPayloadV11(table, section_models); } std::size_t total_size = 0; for (const std::vector& payload : section_payloads) { @@ -574,7 +776,9 @@ struct PackPayloadBuild { entry.column = static_cast(batch.section.x); entry.modelCount = static_cast(batch.models.size()); - section_payloads[batch_index] = buildPackSectionPayloadV10(table, batch.models); + std::vector section_models = batch.models; + sortPackSectionModelsForStreaming(section_models, batch.section); + section_payloads[batch_index] = buildPackSectionPayloadV11(table, section_models); } }); } diff --git a/model/AnimModel.cpp b/model/AnimModel.cpp index 6c6359ef..41291c76 100644 --- a/model/AnimModel.cpp +++ b/model/AnimModel.cpp @@ -668,13 +668,18 @@ TAnimModel::LoadEu7PackWarm( if( false == texture_file.empty() && texture_file != "none" && - texture_file.substr( 0, 1 ) != "*" ) { + texture_file.substr( 0, 1 ) != "*" && + false == texture_file.starts_with( "make:" ) && + false == texture_file.starts_with( "@" ) && + false == texture_file.starts_with( "none|" ) ) { auto normalized { texture_file }; replace_slashes( normalized ); if( normalized != "none" && false == normalized.ends_with( "/none" ) && - normalized.find( "tr/none" ) == std::string::npos ) { + false == normalized.ends_with( '/' ) && + normalized.find( "tr/none" ) == std::string::npos && + normalized.find( '#' ) == std::string::npos ) { m_materialdata.assign( normalized ); if( m_materialdata.replacable_skins[ 1 ] == null_handle ) { scene::eu7::pack_bench_inc( &scene::eu7::Eu7PackBench::main_texture_assign_fail ); diff --git a/rendering/particles.cpp b/rendering/particles.cpp index 12eaa737..d01abd3c 100644 --- a/rendering/particles.cpp +++ b/rendering/particles.cpp @@ -442,6 +442,23 @@ particle_manager::insert( std::string const &Sourcetemplate, TAnimModel const *N return true; } +void +particle_manager::erase_bound_node( TAnimModel const *const node ) { + if( node == nullptr ) { + return; + } + + m_sources.erase( + std::remove_if( + std::begin( m_sources ), + std::end( m_sources ), + [node]( smoke_source const &source ) { + return source.m_ownertype == smoke_source::owner_type::node + && source.m_owner.node == node; + } ), + std::end( m_sources ) ); +} + // updates state of all owned emitters void particle_manager::update() { diff --git a/rendering/particles.h b/rendering/particles.h index 5ee44230..d14a23f0 100644 --- a/rendering/particles.h +++ b/rendering/particles.h @@ -181,6 +181,8 @@ public: insert( std::string const &Sourcetemplate, TDynamicObject const *Vehicle, glm::dvec3 const Location ); bool insert( std::string const &Sourcetemplate, TAnimModel const *Node, glm::dvec3 const Location ); + void + erase_bound_node( TAnimModel const *Node ); // updates state of all owned emitters void update(); diff --git a/scene/eu7/eu7_chunks.h b/scene/eu7/eu7_chunks.h index b1ee94dd..454bf5f2 100644 --- a/scene/eu7/eu7_chunks.h +++ b/scene/eu7/eu7_chunks.h @@ -60,6 +60,7 @@ constexpr std::uint32_t kChunkProt { detail::make_id4( 'P', 'R', 'O', 'T' ) }; constexpr std::uint8_t kPackSectionFormatV8 { 1 }; constexpr std::uint8_t kPackSectionFormatV9 { 2 }; constexpr std::uint8_t kPackSectionFormatV10 { 3 }; +constexpr std::uint8_t kPackSectionFormatV11 { 4 }; constexpr std::size_t kPackSectionChunkModels { 512 }; } // namespace scene::eu7 diff --git a/scene/eu7/eu7_model_prefetch.cpp b/scene/eu7/eu7_model_prefetch.cpp index 31d1cff8..d1467144 100644 --- a/scene/eu7/eu7_model_prefetch.cpp +++ b/scene/eu7/eu7_model_prefetch.cpp @@ -17,27 +17,89 @@ http://mozilla.org/MPL/2.0/. #include "utilities/utilities.h" #include "vehicle/DynObj.h" +#include #include #include +#include #include #include namespace scene::eu7 { namespace { +constexpr std::size_t kMaxWarmedTextures { 2048 }; + std::mutex g_pack_mesh_load_mutex; std::unordered_set g_session_warmed_textures; +std::list g_session_warmed_textures_lru; +std::unordered_map::iterator> g_session_warmed_textures_iters; + +void +touch_warmed_texture_lru( std::string const &texture_file ) { + auto const found { g_session_warmed_textures_iters.find( texture_file ) }; + if( found == g_session_warmed_textures_iters.end() ) { + return; + } + g_session_warmed_textures_lru.splice( + g_session_warmed_textures_lru.end(), + g_session_warmed_textures_lru, + found->second ); +} + +void +evict_warmed_texture_lru() { + while( + g_session_warmed_textures.size() >= kMaxWarmedTextures && + false == g_session_warmed_textures_lru.empty() ) { + auto const oldest { g_session_warmed_textures_lru.front() }; + g_session_warmed_textures_lru.pop_front(); + g_session_warmed_textures_iters.erase( oldest ); + g_session_warmed_textures.erase( oldest ); + pack_bench_inc( &Eu7PackBench::stream_texture_cache_evictions ); + } +} + +void +remember_warmed_texture( std::string texture_file ) { + if( g_session_warmed_textures.contains( texture_file ) ) { + touch_warmed_texture_lru( texture_file ); + return; + } + + evict_warmed_texture_lru(); + g_session_warmed_textures.insert( texture_file ); + g_session_warmed_textures_lru.push_back( texture_file ); + g_session_warmed_textures_iters.emplace( + std::move( texture_file ), + std::prev( g_session_warmed_textures_lru.end() ) ); +} [[nodiscard]] bool pack_texture_usable( std::string texture_file ) { if( texture_file.empty() || texture_file == "none" || texture_file.front() == '*' ) { return false; } - replace_slashes( texture_file ); - if( texture_file == "none" || texture_file.ends_with( "/none" ) ) { + if( + texture_file.starts_with( "make:" ) || + texture_file.starts_with( "@" ) || + texture_file.starts_with( "none|" ) ) { return false; } - if( texture_file.find( "tr/none" ) != std::string::npos ) { + if( + texture_file.ends_with( ".e3d" ) || + texture_file.ends_with( ".t3d" ) ) { + return false; + } + replace_slashes( texture_file ); + if( + texture_file == "none" || + texture_file.ends_with( "/none" ) || + texture_file.ends_with( '/' ) ) { + return false; + } + if( + texture_file.find( "tr/none" ) != std::string::npos || + texture_file.find( '#' ) != std::string::npos ) { return false; } return true; @@ -69,30 +131,80 @@ warm_one_pack_texture( std::string texture_file, std::unordered_set auto const resolved { TextureTest( ToLower( texture_file ) ) }; if( resolved.empty() ) { - pack_bench_inc( &Eu7PackBench::main_texture_warm_miss ); - log_pack_texture_fail( texture_file ); - return false; + bool warmed { false }; + for( int skinindex { 1 }; skinindex <= 4; ++skinindex ) { + auto const multi { + TextureTest( ToLower( texture_file + "," + std::to_string( skinindex ) ) ) }; + if( multi.empty() ) { + break; + } + GfxRenderer->Fetch_Material( multi ); + warmed = true; + } + if( false == warmed ) { + pack_bench_inc( &Eu7PackBench::main_texture_warm_miss ); + log_pack_texture_fail( texture_file ); + return false; + } + } + else { + GfxRenderer->Fetch_Material( resolved ); } - g_session_warmed_textures.insert( texture_file ); - GfxRenderer->Fetch_Material( resolved ); + remember_warmed_texture( texture_file ); - for( int skinindex { 1 }; skinindex <= 4; ++skinindex ) { - auto const multi { - TextureTest( ToLower( texture_file + "," + std::to_string( skinindex ) ) ) }; - if( multi.empty() ) { - break; + if( false == resolved.empty() ) { + for( int skinindex { 1 }; skinindex <= 4; ++skinindex ) { + auto const multi { + TextureTest( ToLower( texture_file + "," + std::to_string( skinindex ) ) ) }; + if( multi.empty() ) { + break; + } + GfxRenderer->Fetch_Material( multi ); } - GfxRenderer->Fetch_Material( multi ); } return true; } } // namespace +[[nodiscard]] bool +ensure_pack_mesh_in_session_cache( + std::string model_file, + std::unordered_map &session_cache ) { + if( model_file.empty() || model_file == "notload" ) { + return false; + } + replace_slashes( model_file ); + if( session_cache.contains( model_file ) ) { + pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit ); + return true; + } + + auto const was_global_cached { TModelsManager::IsModelCached( model_file ) }; + TModel3d *mesh { nullptr }; + { + std::lock_guard lock { g_pack_mesh_load_mutex }; + mesh = TModelsManager::GetModel( model_file, false, false ); + } + if( was_global_cached ) { + pack_bench_inc( &Eu7PackBench::stream_mesh_global_hit ); + } + else { + pack_bench_inc( &Eu7PackBench::stream_mesh_disk_load ); + } + if( mesh != nullptr ) { + TAnimModel::warm_instanceable_cache( mesh ); + } + session_cache.emplace( model_file, mesh ); + return true; +} + void reset_pack_texture_warm_cache() { g_session_warmed_textures.clear(); + g_session_warmed_textures_lru.clear(); + g_session_warmed_textures_iters.clear(); } void @@ -147,6 +259,48 @@ preload_pack_models( preload_pack_models( models ); } +std::size_t +warm_pack_texture_paths_main( + std::string const *const paths, + std::size_t const count, + double const budget_ms, + std::size_t *const processed_out ) { + if( paths == nullptr || count == 0 || GfxRenderer == nullptr ) { + if( processed_out != nullptr ) { + *processed_out = 0; + } + return 0; + } + + std::unordered_set seen; + seen.reserve( std::min( count, std::size_t { 64 } ) ); + std::size_t warmed { 0 }; + std::size_t processed { 0 }; + auto const started { std::chrono::steady_clock::now() }; + + for( std::size_t i { 0 }; i < count; ++i ) { + if( + budget_ms > 0.0 && + processed > 0 ) { + auto const elapsed_ms { + std::chrono::duration( + std::chrono::steady_clock::now() - started ).count() }; + if( elapsed_ms >= budget_ms ) { + break; + } + } + if( warm_one_pack_texture( paths[ i ], seen ) ) { + ++warmed; + } + ++processed; + } + + if( processed_out != nullptr ) { + *processed_out = processed; + } + return warmed; +} + std::size_t warm_pack_textures_main( Eu7Model const *const models, std::size_t const count ) { if( models == nullptr || count == 0 || GfxRenderer == nullptr ) { diff --git a/scene/eu7/eu7_model_prefetch.h b/scene/eu7/eu7_model_prefetch.h index 31ae5495..5ae916fb 100644 --- a/scene/eu7/eu7_model_prefetch.h +++ b/scene/eu7/eu7_model_prefetch.h @@ -13,9 +13,12 @@ http://mozilla.org/MPL/2.0/. #include #include +#include #include #include +class TModel3d; + namespace scene::eu7 { // Tylko warm_instanceable_cache dla juz zcache'owanych meshy (bez cold GetModel na workerze). @@ -30,10 +33,23 @@ preload_pack_models( // Main thread: Fetch_Material dla unikalnych texture_file z chunka przed apply. // Zwraca liczbe unikalnych Fetch_Material w tym slice. +std::size_t +warm_pack_texture_paths_main( + std::string const *paths, + std::size_t count, + double budget_ms = 0.0, + std::size_t *processed_out = nullptr ); + std::size_t warm_pack_textures_main( Eu7Model const *models, std::size_t count ); void reset_pack_texture_warm_cache(); +// Thread-safe cold mesh load: session cache, then GetModel under g_pack_mesh_load_mutex. +[[nodiscard]] bool +ensure_pack_mesh_in_session_cache( + std::string model_file, + std::unordered_map &session_cache ); + } // namespace scene::eu7 diff --git a/scene/eu7/eu7_pack_bench.cpp b/scene/eu7/eu7_pack_bench.cpp index 34653ce2..d6c9d21f 100644 --- a/scene/eu7/eu7_pack_bench.cpp +++ b/scene/eu7/eu7_pack_bench.cpp @@ -189,6 +189,8 @@ log_pack_bench_impl( Eu7PackBench const &bench, char const *const title ) { " stuck_skip=" + std::to_string( bench.stream_apply_stuck_skip ) ); WriteLog( " diag ready_tex_ms=" + std::to_string( static_cast( bench.stream_prefetch_ready_tex_ms ) ) + + " ready_umes_ms=" + std::to_string( static_cast( bench.stream_prefetch_ready_umes_ms ) ) + + " umes_loads=" + std::to_string( bench.stream_prefetch_ready_umes_loads ) + " peak_pending=" + std::to_string( bench.peak_pending_total ) ); WriteLog( " peaks: ready_q=" + std::to_string( bench.peak_ready_queue ) + @@ -324,6 +326,7 @@ pack_bench_note_chunk( pack_bench_inc( &Eu7PackBench::main_chunk_cold_loads, cold_meshes ); g_pack_bench.last_chunk_ms = wall_ms; + g_pack_bench.last_apply_ms = pointer_apply_ms; g_pack_bench.last_chunk_instances = instances; g_pack_bench.last_chunk_cold = cold_meshes; if( wall_ms > g_pack_bench.peak_chunk_ms ) { @@ -344,6 +347,7 @@ pack_bench_note_chunk( auto &stream { g_pack_bench_stream }; stream.last_chunk_ms = wall_ms; + stream.last_apply_ms = pointer_apply_ms; stream.last_chunk_instances = instances; stream.last_chunk_cold = cold_meshes; if( wall_ms > stream.peak_chunk_ms ) { diff --git a/scene/eu7/eu7_pack_bench.h b/scene/eu7/eu7_pack_bench.h index 243d352e..bfc7543a 100644 --- a/scene/eu7/eu7_pack_bench.h +++ b/scene/eu7/eu7_pack_bench.h @@ -72,7 +72,14 @@ struct Eu7PackBench { std::uint64_t stream_dequeue_wait_near { 0 }; std::uint64_t stream_jobs_blocked_far { 0 }; std::uint64_t stream_apply_stuck_skip { 0 }; + std::uint64_t stream_sections_unloaded { 0 }; + std::uint64_t stream_unload_instances { 0 }; + std::uint64_t stream_mesh_cache_evictions { 0 }; + std::uint64_t stream_nodedata_cache_evictions { 0 }; + std::uint64_t stream_texture_cache_evictions { 0 }; double stream_prefetch_ready_tex_ms { 0.0 }; + double stream_prefetch_ready_umes_ms { 0.0 }; + std::uint64_t stream_prefetch_ready_umes_loads { 0 }; std::uint64_t peak_pending_total { 0 }; std::uint64_t stream_inst_planned { 0 }; std::uint64_t stream_inst_after_cold { 0 }; @@ -92,6 +99,7 @@ struct Eu7PackBench { std::uint64_t peak_chunk_instances { 0 }; std::uint64_t peak_chunk_cold { 0 }; double last_chunk_ms { 0.0 }; + double last_apply_ms { 0.0 }; std::uint64_t last_chunk_instances { 0 }; std::uint64_t last_chunk_cold { 0 }; diff --git a/scene/eu7/eu7_reader.cpp b/scene/eu7/eu7_reader.cpp index fa624183..765612a5 100644 --- a/scene/eu7/eu7_reader.cpp +++ b/scene/eu7/eu7_reader.cpp @@ -505,6 +505,48 @@ parse_pack_section_header( const std::uint8_t first_byte { peek == EOF ? std::uint8_t { 0 } : static_cast( peek ) }; + bool use_v11_header { false }; + if( first_byte == kPackSectionFormatV11 ) { + cursor.section_format = sn_utils::d_uint8( input ); + const std::uint32_t solo_total { sn_utils::ld_uint32( input ) }; + const std::uint32_t inst_total { sn_utils::ld_uint32( input ) }; + const std::uint32_t unique_mesh_count { sn_utils::ld_uint32( input ) }; + if( solo_total + inst_total == entry.model_count ) { + use_v11_header = true; + cursor.model_total = solo_total + inst_total; + cursor.solo_remaining = 0; + cursor.inst_remaining = 0; + + StringTable const strings { module.strings }; + cursor.unique_meshes.reserve( unique_mesh_count ); + for( std::uint32_t mesh_idx { 0 }; mesh_idx < unique_mesh_count; ++mesh_idx ) { + cursor.unique_meshes.push_back( + strings.resolve( sn_utils::ld_uint32( input ) ) ); + } + + const std::uint32_t unique_texture_count { sn_utils::ld_uint32( input ) }; + cursor.unique_textures.reserve( unique_texture_count ); + for( std::uint32_t tex_idx { 0 }; tex_idx < unique_texture_count; ++tex_idx ) { + cursor.unique_textures.push_back( + strings.resolve( sn_utils::ld_uint32( input ) ) ); + } + + cursor.chunk_count = sn_utils::ld_uint32( input ); + cursor.chunk_model_counts.resize( cursor.chunk_count ); + cursor.chunk_byte_offsets.resize( cursor.chunk_count ); + for( std::uint32_t chunk_idx { 0 }; chunk_idx < cursor.chunk_count; ++chunk_idx ) { + cursor.chunk_model_counts[ chunk_idx ] = sn_utils::ld_uint32( input ); + cursor.chunk_byte_offsets[ chunk_idx ] = sn_utils::ld_uint32( input ); + } + } + } + + if( use_v11_header ) { + cursor.models_read = 0; + cursor.header_parsed = true; + return; + } + bool use_v10_header { false }; if( first_byte == kPackSectionFormatV10 ) { cursor.section_format = sn_utils::d_uint8( input ); @@ -1168,11 +1210,13 @@ read_pack_section_chunk_load_impl( result.chunk_index = chunk_index; if( chunk_index == 0 ) { result.unique_meshes = header.unique_meshes; + result.unique_textures = header.unique_textures; } Eu7PackSectionCursor chunk_cursor { header }; if( - header.section_format == kPackSectionFormatV10 && + ( header.section_format == kPackSectionFormatV10 || + header.section_format == kPackSectionFormatV11 ) && false == header.chunk_byte_offsets.empty() ) { auto const section_start { static_cast( module.pack_payload_offset + entry->pack_offset ) }; @@ -1218,9 +1262,11 @@ read_pack_section_load_impl( Eu7Module const &module, int const row, int const c parse_pack_section_header( module, input, *entry, header ); result.unique_meshes = header.unique_meshes; + result.unique_textures = header.unique_textures; if( - header.section_format == kPackSectionFormatV10 && + ( header.section_format == kPackSectionFormatV10 || + header.section_format == kPackSectionFormatV11 ) && header.chunk_count > 0 ) { result.models.reserve( entry->model_count ); for( std::uint32_t chunk_idx { 0 }; chunk_idx < header.chunk_count; ++chunk_idx ) { diff --git a/scene/eu7/eu7_section_stream.cpp b/scene/eu7/eu7_section_stream.cpp index 34cbc6b4..e1ed784d 100644 --- a/scene/eu7/eu7_section_stream.cpp +++ b/scene/eu7/eu7_section_stream.cpp @@ -32,6 +32,7 @@ http://mozilla.org/MPL/2.0/. #include #include #include +#include #include #include #include @@ -41,6 +42,15 @@ http://mozilla.org/MPL/2.0/. #include namespace scene::eu7 { + +[[nodiscard]] bool +ring_section_in_radius( + int row, + int column, + int center_row, + int center_column, + int radius_km ); + namespace { constexpr double kDrainBudgetMs { 12.0 }; @@ -50,7 +60,7 @@ constexpr std::size_t kLoaderSectionsPerDrain { 8 }; constexpr double kGameplayApplyBudgetMs { 4.0 }; constexpr double kCatchupApplyBudgetMs { 6.0 }; constexpr std::size_t kGameplaySliceInstances { 96 }; -constexpr std::size_t kCatchupSliceInstances { 160 }; +constexpr std::size_t kCatchupSliceInstances { 64 }; constexpr std::size_t kLoaderSliceInstances { 768 }; constexpr std::size_t kGameplaySliceColdMeshes { 1 }; constexpr std::size_t kCatchupSliceColdMeshes { 2 }; @@ -60,6 +70,9 @@ constexpr double kCatchupColdBudgetMs { 5.0 }; constexpr int kInitialBootstrapRadius { 5 }; constexpr int kStreamRadius { 14 }; +constexpr int kSectionUnloadMarginKm { 2 }; +constexpr std::size_t kMeshCacheLruCap { 2000 }; +constexpr std::size_t kNodedataCacheLruCap { 2000 }; constexpr int kMovementLookahead { 10 }; constexpr std::size_t kMaxPackStreamWorkers { 8 }; constexpr std::size_t kMaxInFlightSections { 12 }; @@ -67,7 +80,9 @@ constexpr std::size_t kMaxReadySections { 6 }; constexpr double kStreamStatusLogIntervalSec { 5.0 }; constexpr double kReenqueueDistanceM { 500.0 }; constexpr double kCatchupReenqueueDistanceM { 40.0 }; -constexpr std::size_t kHeavySectionModelThreshold { 800 }; +constexpr std::size_t kHeavySectionModelThreshold { 32 }; +constexpr std::size_t kPackTextureWarmSlice { 8 }; +constexpr double kPackTextureWarmBudgetMs { 4.0 }; constexpr double kUrgentApplyBudgetMs { 32.0 }; constexpr std::size_t kUrgentSliceInstances { 96 }; constexpr std::size_t kUrgentSliceColdMeshes { 2 }; @@ -75,6 +90,10 @@ constexpr double kUrgentColdBudgetMs { 24.0 }; constexpr std::size_t kUrgentMaxChunksPerDrain { 8 }; constexpr double kReadyTexturePrefetchBudgetMs { 14.0 }; constexpr std::size_t kReadyTexturePrefetchSlice { 256 }; +constexpr int kUmesPrefetchMaxDistanceKm { 2 }; +constexpr double kReadyUmesPrefetchBudgetMs { 8.0 }; +constexpr std::size_t kReadyUmesPrefetchMaxMeshes { 6 }; +constexpr std::size_t kUmesPrefetchJobsBacklogThreshold { 4 }; constexpr std::size_t kCatchupMaxInFlightSections { 28 }; constexpr std::size_t kCatchupMaxReadySections { 14 }; constexpr int kUrgentSectionDistanceKm { 1 }; @@ -110,11 +129,13 @@ struct PackSectionReady { std::size_t section_idx { 0 }; std::unique_ptr> models; std::vector unique_meshes; + std::vector unique_textures; bool failed { false }; bool section_final { true }; std::size_t apply_offset { 0 }; std::size_t texture_warm_offset { 0 }; std::size_t umes_cold_offset { 0 }; + std::size_t umes_prefetch_offset { 0 }; }; struct SectionStreamState { @@ -139,6 +160,14 @@ struct SectionStreamState { std::unordered_map mesh_cache; std::unordered_map nodedata_cache; + std::list mesh_lru; + std::unordered_map::iterator> mesh_lru_iters; + std::unordered_map mesh_ref_counts; + std::unordered_map> section_mesh_keys; + std::list nodedata_lru; + std::unordered_map::iterator> nodedata_lru_iters; + std::unordered_map nodedata_ref_counts; + std::unordered_map> section_nodedata_keys; std::vector workers; std::atomic worker_exit { false }; @@ -171,6 +200,12 @@ enqueue_section_if_needed( void sync_stream_limits( glm::dvec3 const &world_position ); +void +maybe_unload_distant_sections( int const center_row, int const center_column ); + +void +evict_unreferenced_stream_caches(); + [[nodiscard]] bool try_dequeue_ready_batch(); @@ -186,6 +221,46 @@ enqueue_failed_section( PackSectionJob const &job ); void release_pending_buffer(); +void +prefetch_ready_queue_umes_worker( double budget_ms, std::size_t max_meshes ); + +[[nodiscard]] std::size_t +warm_pack_section_textures( + PackSectionReady &batch, + std::size_t const model_offset, + std::size_t const model_count, + std::size_t const max_textures = kPackTextureWarmSlice ) { + if( false == batch.unique_textures.empty() ) { + if( batch.texture_warm_offset >= batch.unique_textures.size() ) { + return 0; + } + auto const remaining { batch.unique_textures.size() - batch.texture_warm_offset }; + auto const slice { std::min( remaining, max_textures ) }; + std::size_t processed { 0 }; + auto const warmed { + warm_pack_texture_paths_main( + batch.unique_textures.data() + batch.texture_warm_offset, + slice, + kPackTextureWarmBudgetMs, + &processed ) }; + batch.texture_warm_offset += processed; + return warmed; + } + + if( batch.models == nullptr || model_count == 0 ) { + return 0; + } + auto const begin { model_offset }; + if( begin >= batch.models->size() ) { + return 0; + } + auto const remaining { std::min( model_count, batch.models->size() - begin ) }; + auto const warmed { + warm_pack_textures_main( batch.models->data() + begin, remaining ) }; + batch.texture_warm_offset = begin + remaining; + return warmed; +} + [[nodiscard]] bool section_has_pack_models( int const row, int const column ); @@ -216,6 +291,303 @@ stream_worker_count() { return std::min( requested, kMaxPackStreamWorkers ); } +void +touch_string_lru( + std::string const &key, + std::list &order, + std::unordered_map::iterator> &iters ) { + auto const found { iters.find( key ) }; + if( found == iters.end() ) { + order.push_back( key ); + iters.emplace( key, std::prev( order.end() ) ); + return; + } + order.splice( order.end(), order, found->second ); +} + +void +retain_cache_key( + std::size_t const section_idx, + std::string key, + std::unordered_map> §ion_keys, + std::unordered_map &ref_counts, + std::list &lru_order, + std::unordered_map::iterator> &lru_iters ) { + if( key.empty() ) { + return; + } + auto &keys { section_keys[ section_idx ] }; + if( false == keys.insert( key ).second ) { + touch_string_lru( key, lru_order, lru_iters ); + return; + } + ++ref_counts[ key ]; + touch_string_lru( key, lru_order, lru_iters ); +} + +void +release_section_key_refs( + std::size_t const section_idx, + std::unordered_map> §ion_keys, + std::unordered_map &ref_counts ) { + auto const found { section_keys.find( section_idx ) }; + if( found == section_keys.end() ) { + return; + } + + for( auto const &key : found->second ) { + auto ref_it { ref_counts.find( key ) }; + if( ref_it == ref_counts.end() ) { + continue; + } + if( ref_it->second <= 1 ) { + ref_counts.erase( ref_it ); + } + else { + --ref_it->second; + } + } + section_keys.erase( found ); +} + +void +evict_cache_lru_if_needed( + std::unordered_map &ref_counts, + std::unordered_map *const mesh_cache, + std::unordered_map *const nodedata_cache, + std::list &lru_order, + std::unordered_map::iterator> &lru_iters, + std::size_t const cap, + std::uint64_t Eu7PackBench::*const eviction_counter ) { + while( lru_order.size() > cap ) { + bool evicted { false }; + for( auto it { lru_order.begin() }; it != lru_order.end(); ++it ) { + auto const &key { *it }; + auto const ref_it { ref_counts.find( key ) }; + if( ref_it != ref_counts.end() && ref_it->second > 0 ) { + continue; + } + lru_iters.erase( key ); + lru_order.erase( it ); + if( mesh_cache != nullptr ) { + mesh_cache->erase( key ); + } + if( nodedata_cache != nullptr ) { + nodedata_cache->erase( key ); + } + pack_bench_inc( eviction_counter ); + evicted = true; + break; + } + if( false == evicted ) { + break; + } + } +} + +void +retain_section_mesh_keys( + std::size_t const section_idx, + Eu7Model const *const models, + std::size_t const offset, + std::size_t const count ) { + if( models == nullptr || count == 0 ) { + return; + } + for( std::size_t i { 0 }; i < count; ++i ) { + auto model_file { models[ offset + i ].model_file }; + if( model_file.empty() || model_file == "notload" ) { + continue; + } + replace_slashes( model_file ); + retain_cache_key( + section_idx, + model_file, + g_stream.section_mesh_keys, + g_stream.mesh_ref_counts, + g_stream.mesh_lru, + g_stream.mesh_lru_iters ); + } +} + +void +retain_section_nodedata_keys( + std::size_t const section_idx, + Eu7Model const *const models, + std::size_t const offset, + std::size_t const count ) { + if( models == nullptr || count == 0 ) { + return; + } + for( std::size_t i { 0 }; i < count; ++i ) { + retain_cache_key( + section_idx, + pack_nodedata_cache_key( models[ offset + i ] ), + g_stream.section_nodedata_keys, + g_stream.nodedata_ref_counts, + g_stream.nodedata_lru, + g_stream.nodedata_lru_iters ); + } +} + +void +release_section_cache_refs( std::size_t const section_idx ) { + release_section_key_refs( + section_idx, + g_stream.section_mesh_keys, + g_stream.mesh_ref_counts ); + release_section_key_refs( + section_idx, + g_stream.section_nodedata_keys, + g_stream.nodedata_ref_counts ); + evict_unreferenced_stream_caches(); +} + +void +evict_unreferenced_stream_caches() { + evict_cache_lru_if_needed( + g_stream.mesh_ref_counts, + &g_stream.mesh_cache, + nullptr, + g_stream.mesh_lru, + g_stream.mesh_lru_iters, + kMeshCacheLruCap, + &Eu7PackBench::stream_mesh_cache_evictions ); + evict_cache_lru_if_needed( + g_stream.nodedata_ref_counts, + nullptr, + &g_stream.nodedata_cache, + g_stream.nodedata_lru, + g_stream.nodedata_lru_iters, + kNodedataCacheLruCap, + &Eu7PackBench::stream_nodedata_cache_evictions ); +} + +[[nodiscard]] bool +ensure_stream_pack_mesh( std::string model_file ) { + if( model_file.empty() || model_file == "notload" ) { + return false; + } + replace_slashes( model_file ); + (void)ensure_pack_mesh_in_session_cache( model_file, g_stream.mesh_cache ); + touch_string_lru( model_file, g_stream.mesh_lru, g_stream.mesh_lru_iters ); + return true; +} + +void +cancel_section_stream_work( std::size_t const section_idx ) { + { + std::lock_guard lock { g_stream.jobs.mutex }; + auto &jobs { g_stream.jobs.data }; + jobs.erase( + std::remove_if( + jobs.begin(), + jobs.end(), + [&]( PackSectionJob const &job ) { + return job.section_idx == section_idx; + } ), + jobs.end() ); + } + { + std::lock_guard lock { g_stream.ready.mutex }; + for( auto it { g_stream.ready.data.begin() }; it != g_stream.ready.data.end(); ) { + if( it->section_idx != section_idx ) { + ++it; + continue; + } + if( it->models != nullptr ) { + it->models->clear(); + it->models->shrink_to_fit(); + it->models.reset(); + } + it = g_stream.ready.data.erase( it ); + } + } + if( + g_stream.pending_apply.has_value() && + g_stream.pending_apply->section_idx == section_idx ) { + release_pending_buffer(); + } + g_stream.in_flight_sections.erase( section_idx ); + g_stream.loaded_sections.erase( section_idx ); +} + +[[nodiscard]] std::size_t +unload_pack_section( int const row, int const column ) { + auto const section_idx { section_index( row, column ) }; + if( false == g_stream.loaded_sections.contains( section_idx ) ) { + return 0; + } + + std::size_t removed { 0 }; + if( g_stream.serializer != nullptr ) { + removed = g_stream.serializer->unload_eu7_pack_section( row, column ); + } + release_section_cache_refs( section_idx ); + cancel_section_stream_work( section_idx ); + + if( removed > 0 ) { + if( load_stats().pack_models >= removed ) { + load_stats().pack_models -= removed; + } + else { + load_stats().pack_models = 0; + } + pack_bench_inc( &Eu7PackBench::stream_unload_instances, removed ); + } + pack_bench_inc( &Eu7PackBench::stream_sections_unloaded ); + return removed; +} + +void +maybe_unload_distant_sections( int const center_row, int const center_column ) { + if( false == g_loading_screen_dismissed || g_stream.bootstrap_active ) { + return; + } + + auto const keep_radius { g_stream.radius + kSectionUnloadMarginKm }; + std::vector to_unload; + to_unload.reserve( g_stream.loaded_sections.size() ); + + for( auto const section_idx : g_stream.loaded_sections ) { + auto const row { + static_cast( section_idx / static_cast( kRegionSideSectionCount ) ) }; + auto const column { + static_cast( section_idx % static_cast( kRegionSideSectionCount ) ) }; + if( ring_section_in_radius( row, column, center_row, center_column, keep_radius ) ) { + continue; + } + to_unload.push_back( section_idx ); + } + + if( to_unload.empty() ) { + return; + } + + static double s_last_unload_log { 0.0 }; + auto const now { Timer::GetTime() }; + auto const should_log { + now - s_last_unload_log >= kStreamStatusLogIntervalSec }; + + std::size_t total_removed { 0 }; + for( auto const section_idx : to_unload ) { + auto const row { + static_cast( section_idx / static_cast( kRegionSideSectionCount ) ) }; + auto const column { + static_cast( section_idx % static_cast( kRegionSideSectionCount ) ) }; + total_removed += unload_pack_section( row, column ); + } + + if( should_log && total_removed > 0 ) { + s_last_unload_log = now; + WriteLog( + "EU7 PACK: unload " + std::to_string( to_unload.size() ) + " sekcji, inst=" + + std::to_string( total_removed ) + ", loaded=" + + std::to_string( g_stream.loaded_sections.size() ) + ", mesh_cache=" + + std::to_string( g_stream.mesh_cache.size() ) ); + } +} + void reset_stream_fields(); @@ -242,6 +614,14 @@ reset_stream_fields() { g_stream.has_anchor_position = false; g_stream.mesh_cache.clear(); g_stream.nodedata_cache.clear(); + g_stream.mesh_lru.clear(); + g_stream.mesh_lru_iters.clear(); + g_stream.mesh_ref_counts.clear(); + g_stream.section_mesh_keys.clear(); + g_stream.nodedata_lru.clear(); + g_stream.nodedata_lru_iters.clear(); + g_stream.nodedata_ref_counts.clear(); + g_stream.section_nodedata_keys.clear(); reset_pack_texture_warm_cache(); note_apply_progress(); } @@ -331,6 +711,33 @@ camera_stream_speed_mps() { return glm::length( rotated ) * 5.0; } +[[nodiscard]] bool pending_apply_is_urgent(); +[[nodiscard]] double gameplay_apply_budget_ms(); + +[[nodiscard]] double +ready_texture_prefetch_budget_ms() { + auto const speed { camera_stream_speed_mps() }; + if( + speed > 500.0 && + pending_apply_is_urgent() ) { + return 0.0; + } + + double budget { kReadyTexturePrefetchBudgetMs * 0.5 }; + if( g_stream_catchup || speed > 80.0 ) { + budget = kReadyTexturePrefetchBudgetMs; + } + + if( g_stream_catchup ) { + auto const apply_budget { + pending_apply_is_urgent() ? + kUrgentApplyBudgetMs : + gameplay_apply_budget_ms() }; + return std::min( budget, apply_budget * 0.25 ); + } + return std::min( budget, kDrainBudgetMs * 0.35 ); +} + [[nodiscard]] glm::dvec3 camera_travel_forward() { if( glm::length2( Global.pCamera.Velocity ) > 0.01 ) { @@ -404,6 +811,24 @@ pending_apply_is_urgent() { return pending_section_near_camera( max_dist ); } +[[nodiscard]] std::size_t +scene_apply_pressure_slice_cap() { + auto const applied { load_stats().pack_models }; + if( applied >= 200000 ) { + return 16; + } + if( applied >= 120000 ) { + return 24; + } + if( applied >= 80000 ) { + return 32; + } + if( applied >= 40000 ) { + return 48; + } + return std::numeric_limits::max(); +} + [[nodiscard]] std::size_t adaptive_slice_instances( std::size_t const section_total, bool const urgent = false ) { if( urgent ) { @@ -428,6 +853,14 @@ adaptive_slice_instances( std::size_t const section_total, bool const urgent = f else if( last_ms >= 12.0 ) { limit = std::min( limit, std::size_t { 48 } ); } + limit = std::min( limit, scene_apply_pressure_slice_cap() ); + auto const last_apply { pack_bench_stream().last_apply_ms }; + if( last_apply >= 20.0 ) { + limit = std::min( limit, std::size_t { 16 } ); + } + else if( last_apply >= 12.0 ) { + limit = std::min( limit, std::size_t { 24 } ); + } return std::max( limit, std::size_t { 16 } ); } @@ -438,7 +871,7 @@ adaptive_slice_instances( std::size_t const section_total, bool const urgent = f limit = std::min( limit, kCatchupSliceInstances ); } else if( speed > 600.0 ) { - limit = std::min( limit, std::size_t { 128 } ); + limit = std::min( limit, std::size_t { 48 } ); } else { limit = std::min( limit, std::size_t { 96 } ); @@ -463,6 +896,19 @@ adaptive_slice_instances( std::size_t const section_total, bool const urgent = f limit = std::min( limit, std::size_t { 64 } ); } + auto const last_apply { pack_bench_stream().last_apply_ms }; + if( last_apply >= 24.0 ) { + limit = std::min( limit, std::size_t { 16 } ); + } + else if( last_apply >= 16.0 ) { + limit = std::min( limit, std::size_t { 24 } ); + } + else if( last_apply >= 10.0 ) { + limit = std::min( limit, std::size_t { 32 } ); + } + + limit = std::min( limit, scene_apply_pressure_slice_cap() ); + return std::max( limit, std::size_t { 16 } ); } @@ -528,23 +974,11 @@ preload_slice_cold_meshes( break; } } - auto const was_global_cached { TModelsManager::IsModelCached( model_file ) }; - TModel3d *mesh { nullptr }; { PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms }; - mesh = TModelsManager::GetModel( model_file, false, false ); + (void)ensure_stream_pack_mesh( model_file ); pack_bench_inc( &Eu7PackBench::main_cold_getmodel_calls ); } - if( was_global_cached ) { - pack_bench_inc( &Eu7PackBench::stream_mesh_global_hit ); - } - else { - pack_bench_inc( &Eu7PackBench::stream_mesh_disk_load ); - } - if( mesh != nullptr ) { - TAnimModel::warm_instanceable_cache( mesh ); - } - g_stream.mesh_cache.emplace( model_file, mesh ); ++cold_loaded; } @@ -556,23 +990,11 @@ preload_slice_cold_meshes( } replace_slashes( model_file ); if( false == g_stream.mesh_cache.contains( model_file ) ) { - auto const was_global_cached { TModelsManager::IsModelCached( model_file ) }; - TModel3d *mesh { nullptr }; { PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms }; - mesh = TModelsManager::GetModel( model_file, false, false ); + (void)ensure_stream_pack_mesh( model_file ); pack_bench_inc( &Eu7PackBench::main_cold_getmodel_calls ); } - if( was_global_cached ) { - pack_bench_inc( &Eu7PackBench::stream_mesh_global_hit ); - } - else { - pack_bench_inc( &Eu7PackBench::stream_mesh_disk_load ); - } - if( mesh != nullptr ) { - TAnimModel::warm_instanceable_cache( mesh ); - } - g_stream.mesh_cache.emplace( model_file, mesh ); cold_loaded = 1; } slice_count = 1; @@ -619,23 +1041,11 @@ preload_umes_cold_meshes( break; } } - auto const was_global_cached { TModelsManager::IsModelCached( model_file ) }; - TModel3d *mesh { nullptr }; { PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms }; - mesh = TModelsManager::GetModel( model_file, false, false ); + (void)ensure_stream_pack_mesh( model_file ); pack_bench_inc( &Eu7PackBench::main_cold_getmodel_calls ); } - if( was_global_cached ) { - pack_bench_inc( &Eu7PackBench::stream_mesh_global_hit ); - } - else { - pack_bench_inc( &Eu7PackBench::stream_mesh_disk_load ); - } - if( mesh != nullptr ) { - TAnimModel::warm_instanceable_cache( mesh ); - } - g_stream.mesh_cache.emplace( model_file, mesh ); ++cold_loaded; offset_out = i + 1; } @@ -689,6 +1099,7 @@ maybe_skip_stuck_apply_offset() { ( model_file.empty() ? "" : " file=\"" + model_file + "\"" ) ); if( false == model_file.empty() && model_file != "notload" ) { replace_slashes( model_file ); + touch_string_lru( model_file, g_stream.mesh_lru, g_stream.mesh_lru_iters ); g_stream.mesh_cache.emplace( std::move( model_file ), nullptr ); } g_stream.pending_apply_offset = offset + 1; @@ -724,6 +1135,16 @@ maybe_preempt_distant_pending() { if( pending_dist <= kUrgentSectionDistanceKm ) { return; } + if( pending_dist < 5 ) { + return; + } + + auto const total { batch.models->size() }; + if( + total > 0 && + g_stream.pending_apply_offset > total / 2 ) { + return; + } auto const speed { camera_stream_speed_mps() }; auto const near_apply_km { @@ -749,7 +1170,7 @@ maybe_preempt_distant_pending() { break; } if( - ready_dist < pending_dist && + ready_dist + 3 < pending_dist && ready_dist <= near_apply_km ) { nearer_ready = true; break; @@ -847,7 +1268,10 @@ apply_pending_chunk( auto const cam_speed { camera_stream_speed_mps() }; auto const is_urgent { pending_apply_is_urgent() }; auto const is_heavy { - total > kHeavySectionModelThreshold || remaining > kHeavySectionModelThreshold }; + false == batch.unique_meshes.empty() || + false == batch.unique_textures.empty() || + total >= kHeavySectionModelThreshold || + remaining >= kHeavySectionModelThreshold }; auto const far_slow_drain { cam_speed > 300.0 && pending_dist > kFarApplySlowDistanceKm && @@ -932,7 +1356,7 @@ apply_pending_chunk( { auto const phase_started { std::chrono::steady_clock::now() }; - tex_fetches = warm_pack_textures_main( models_ptr + offset, chunk_count ); + tex_fetches = warm_pack_section_textures( batch, offset, chunk_count ); warm_ms = std::chrono::duration( std::chrono::steady_clock::now() - phase_started ).count(); } @@ -941,7 +1365,8 @@ apply_pending_chunk( scene::scratch_data scratch; simulation::eu7_pack_apply_session const session { &g_stream.mesh_cache, - &g_stream.nodedata_cache }; + &g_stream.nodedata_cache, + batch.section_idx }; { auto const phase_started { std::chrono::steady_clock::now() }; g_stream.serializer->apply_eu7_pack_models( @@ -953,6 +1378,9 @@ apply_pending_chunk( apply_ms = std::chrono::duration( std::chrono::steady_clock::now() - phase_started ).count(); } + retain_section_mesh_keys( batch.section_idx, models_ptr, offset, chunk_count ); + retain_section_nodedata_keys( batch.section_idx, models_ptr, offset, chunk_count ); + evict_unreferenced_stream_caches(); auto const chunk_wall_ms { std::chrono::duration( @@ -1121,6 +1549,7 @@ worker_loop( std::stop_token const stop_token ) { std::unique_ptr> models; std::vector unique_meshes; + std::vector unique_textures; std::uint32_t chunk_count { 1 }; { PackBenchTimer const read_timer { &Eu7PackBench::worker_read_pack_ms }; @@ -1129,6 +1558,7 @@ worker_loop( std::stop_token const stop_token ) { module, job.row, job.column, job.next_chunk ) }; chunk_count = chunk.chunk_count; unique_meshes = std::move( chunk.unique_meshes ); + unique_textures = std::move( chunk.unique_textures ); if( false == chunk.models.empty() ) { models = std::make_unique>( std::move( chunk.models ) ); @@ -1141,6 +1571,7 @@ worker_loop( std::stop_token const stop_token ) { result.section_idx = job.section_idx; result.models = std::move( models ); result.unique_meshes = std::move( unique_meshes ); + result.unique_textures = std::move( unique_textures ); result.failed = result.models == nullptr; result.section_final = job.next_chunk + 1 >= chunk_count; @@ -1169,6 +1600,18 @@ worker_loop( std::stop_token const stop_token ) { pack_bench_inc( &Eu7PackBench::worker_sections_done ); } + if( job.next_chunk == 0 ) { + std::size_t pending_jobs { 0 }; + { + std::lock_guard lock { g_stream.jobs.mutex }; + pending_jobs = g_stream.jobs.data.size(); + } + if( pending_jobs < kUmesPrefetchJobsBacklogThreshold ) { + prefetch_ready_queue_umes_worker( + kReadyUmesPrefetchBudgetMs, kReadyUmesPrefetchMaxMeshes ); + } + } + if( job.next_chunk + 1 < chunk_count ) { PackSectionJob continue_job { job }; continue_job.next_chunk = job.next_chunk + 1; @@ -1722,15 +2165,6 @@ try_dequeue_ready_batch() { pack_bench_inc( &Eu7PackBench::stream_dequeue_far ); } - if( - g_stream.pending_apply->models != nullptr && - g_stream.pending_apply->texture_warm_offset < g_stream.pending_apply->models->size() ) { - (void)warm_pack_textures_main( - g_stream.pending_apply->models->data() + g_stream.pending_apply->texture_warm_offset, - g_stream.pending_apply->models->size() - g_stream.pending_apply->texture_warm_offset ); - g_stream.pending_apply->texture_warm_offset = g_stream.pending_apply->models->size(); - } - return true; } @@ -1748,7 +2182,17 @@ prefetch_ready_queue_textures( double const budget_ms ) { { std::lock_guard lock { g_stream.ready.mutex }; for( auto it { g_stream.ready.data.begin() }; it != g_stream.ready.data.end(); ++it ) { - if( it->failed || it->models == nullptr || it->models->empty() ) { + if( it->failed ) { + continue; + } + if( false == it->unique_textures.empty() ) { + if( it->texture_warm_offset >= it->unique_textures.size() ) { + continue; + } + candidates.push_back( it ); + continue; + } + if( it->models == nullptr || it->models->empty() ) { continue; } if( it->texture_warm_offset >= it->models->size() ) { @@ -1784,6 +2228,30 @@ prefetch_ready_queue_textures( double const budget_ms ) { for( auto const it : candidates ) { auto &batch { *it }; + if( false == batch.unique_textures.empty() ) { + while( batch.texture_warm_offset < batch.unique_textures.size() ) { + auto const remaining { + batch.unique_textures.size() - batch.texture_warm_offset }; + auto const slice { std::min( remaining, kReadyTexturePrefetchSlice ) }; + auto const tex_slice { std::min( slice, kPackTextureWarmSlice ) }; + std::size_t processed { 0 }; + (void)warm_pack_texture_paths_main( + batch.unique_textures.data() + batch.texture_warm_offset, + tex_slice, + kPackTextureWarmBudgetMs, + &processed ); + batch.texture_warm_offset += processed; + + auto const elapsed_ms { + std::chrono::duration( + std::chrono::steady_clock::now() - started ).count() }; + if( elapsed_ms >= budget_ms ) { + return; + } + } + continue; + } + while( batch.texture_warm_offset < batch.models->size() ) { auto const remaining { batch.models->size() - batch.texture_warm_offset }; auto const slice { std::min( remaining, kReadyTexturePrefetchSlice ) }; @@ -1802,6 +2270,100 @@ prefetch_ready_queue_textures( double const budget_ms ) { } } +void +prefetch_ready_queue_umes_worker( double const budget_ms, std::size_t const max_meshes ) { + if( budget_ms <= 0.0 || max_meshes == 0 ) { + return; + } + + PackBenchTimer const prefetch_timer { &Eu7PackBench::stream_prefetch_ready_umes_ms }; + auto const started { std::chrono::steady_clock::now() }; + std::size_t loads { 0 }; + + std::lock_guard lock { g_stream.ready.mutex }; + + std::vector::iterator> candidates; + candidates.reserve( g_stream.ready.data.size() ); + for( auto it { g_stream.ready.data.begin() }; it != g_stream.ready.data.end(); ++it ) { + if( it->failed || it->unique_meshes.empty() ) { + continue; + } + if( it->umes_prefetch_offset >= it->unique_meshes.size() ) { + continue; + } + if( g_stream.center_row >= 0 && g_stream.center_column >= 0 ) { + auto const dist { + section_manhattan_sections( + it->row, + it->column, + g_stream.center_row, + g_stream.center_column ) }; + if( dist > kUmesPrefetchMaxDistanceKm ) { + continue; + } + } + candidates.push_back( it ); + } + + if( candidates.empty() ) { + return; + } + + std::sort( + candidates.begin(), + candidates.end(), + []( std::deque::iterator const &lhs, + std::deque::iterator const &rhs ) { + auto const lhs_dist { + section_manhattan_sections( + lhs->row, + lhs->column, + g_stream.center_row, + g_stream.center_column ) }; + auto const rhs_dist { + section_manhattan_sections( + rhs->row, + rhs->column, + g_stream.center_row, + g_stream.center_column ) }; + return lhs_dist < rhs_dist; + } ); + + for( auto const it : candidates ) { + auto &batch { *it }; + while( batch.umes_prefetch_offset < batch.unique_meshes.size() ) { + auto model_file { batch.unique_meshes[ batch.umes_prefetch_offset ] }; + + if( model_file.empty() || model_file == "notload" ) { + ++batch.umes_prefetch_offset; + continue; + } + replace_slashes( model_file ); + if( g_stream.mesh_cache.contains( model_file ) ) { + pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit ); + ++batch.umes_prefetch_offset; + continue; + } + if( loads >= max_meshes ) { + return; + } + if( loads > 0 ) { + auto const elapsed_ms { + std::chrono::duration( + std::chrono::steady_clock::now() - started ).count() }; + if( elapsed_ms >= budget_ms ) { + return; + } + } + + (void)ensure_stream_pack_mesh( model_file ); + ++loads; + pack_bench_inc( &Eu7PackBench::stream_prefetch_ready_umes_loads ); + ++batch.umes_prefetch_offset; + } + } +} + [[nodiscard]] bool gameplay_stream_mode() { return g_loading_screen_dismissed @@ -2107,6 +2669,7 @@ update_section_stream( glm::dvec3 const &world_position ) { if( center_moved ) { reprioritize_job_queue(); + maybe_unload_distant_sections( center_row, center_column ); } sync_stream_limits( stream_position ); @@ -2206,10 +2769,10 @@ drain_section_stream( kSectionStreamGameplayRadiusKm ) }; if( gameplay_stream_mode() && inner_ring_ready ) { - prefetch_ready_queue_textures( - g_stream_catchup || camera_stream_speed_mps() > 80.0 ? - kReadyTexturePrefetchBudgetMs : - kReadyTexturePrefetchBudgetMs * 0.5 ); + maybe_unload_distant_sections( + g_stream.center_row, + g_stream.center_column ); + prefetch_ready_queue_textures( ready_texture_prefetch_budget_ms() ); if( g_stream_catchup ) { auto const urgent { @@ -2689,6 +3252,9 @@ void reset_section_stream() { flush_pack_stream_bench(); stop_workers(); + if( g_stream.serializer != nullptr ) { + g_stream.serializer->reset_eu7_pack_section_instances(); + } reset_stream_fields(); g_ring_ready_since.reset(); g_loading_block_started.reset(); diff --git a/scene/eu7/eu7_types.h b/scene/eu7/eu7_types.h index 4acc8023..d8b08f3d 100644 --- a/scene/eu7/eu7_types.h +++ b/scene/eu7/eu7_types.h @@ -203,6 +203,14 @@ struct Eu7Model { bool is_terrain = false; }; +[[nodiscard]] inline std::string +pack_nodedata_cache_key( Eu7Model const &model ) { + return model.model_file + '\x1f' + model.texture_file + '\x1f' + + std::to_string( model.node.range_squared_min ) + '\x1f' + + std::to_string( model.node.range_squared_max ) + '\x1f' + + ( model.is_terrain ? '1' : '0' ); +} + // Wspolna definicja modelu (EU7B v8 PROT) — bez transformacji instancji. struct Eu7ModelPrototype { Eu7BasicNode node; @@ -369,6 +377,7 @@ struct Eu7PackSectionCursor { std::uint8_t section_format { 0 }; bool header_parsed { false }; std::vector unique_meshes; + std::vector unique_textures; std::uint32_t chunk_count { 0 }; std::vector chunk_model_counts; std::vector chunk_byte_offsets; @@ -377,11 +386,13 @@ struct Eu7PackSectionCursor { struct Eu7PackSectionLoad { std::vector models; std::vector unique_meshes; + std::vector unique_textures; }; struct Eu7PackSectionChunkLoad { std::vector models; std::vector unique_meshes; + std::vector unique_textures; std::uint32_t chunk_count { 1 }; std::uint32_t chunk_index { 0 }; }; diff --git a/scene/scene.cpp b/scene/scene.cpp index 738578d5..688a59a8 100644 --- a/scene/scene.cpp +++ b/scene/scene.cpp @@ -444,6 +444,33 @@ basic_cell::insert( TMemCell *Memorycell ) { void basic_cell::erase( TAnimModel *Instance ) { + if( Instance->m_eu7_pack && Instance->m_instanceable ) { + m_instancesopaque.erase( + std::remove_if( + std::begin( m_instancesopaque ), std::end( m_instancesopaque ), + [=]( TAnimModel *instance ) { + return instance == Instance; } ), + std::end( m_instancesopaque ) ); + if( Instance->Model() != nullptr ) { + instance_bucket_key key; + key.pModel = Instance->Model(); + auto const *mat { Instance->Material() }; + if( mat != nullptr ) { + for( int i { 0 }; i < 5; ++i ) { key.skins[ i ] = mat->replacable_skins[ i ]; } + } + auto bucket { m_instancebuckets_opaque.find( key ) }; + if( bucket != m_instancebuckets_opaque.end() ) { + bucket->second.erase( + std::remove( std::begin( bucket->second ), std::end( bucket->second ), Instance ), + std::end( bucket->second ) ); + if( bucket->second.empty() ) { + m_instancebuckets_opaque.erase( bucket ); + } + } + } + return; + } + auto const flags = Instance->Flags(); auto alpha = ( Instance->Material() != nullptr ? diff --git a/simulation/simulationstateserializer.cpp b/simulation/simulationstateserializer.cpp index 6fa799c3..555ba464 100644 --- a/simulation/simulationstateserializer.cpp +++ b/simulation/simulationstateserializer.cpp @@ -34,6 +34,7 @@ http://mozilla.org/MPL/2.0/. #include "scene/eu7/eu7_loader.h" #include "scene/eu7/eu7_load_stats.h" #include "scene/eu7/eu7_pack_bench.h" +#include "scene/eu7/eu7_section.h" #include "scene/eu7/eu7_transform.h" #include "world/Track.h" #include "world/Traction.h" @@ -285,19 +286,11 @@ pack_model_needs_full_load( scene::eu7::Eu7Model const &model ) { || false == model.transition; } -[[nodiscard]] std::string -pack_nodedata_cache_key( scene::eu7::Eu7Model const &model ) { - return model.model_file + '\x1f' + model.texture_file + '\x1f' - + std::to_string( model.node.range_squared_min ) + '\x1f' - + std::to_string( model.node.range_squared_max ) + '\x1f' - + ( model.is_terrain ? '1' : '0' ); -} - [[nodiscard]] scene::node_data const & pack_nodedata_cached( scene::eu7::Eu7Model const &model, std::unordered_map &cache ) { - auto const key { pack_nodedata_cache_key( model ) }; + auto const key { scene::eu7::pack_nodedata_cache_key( model ) }; auto const found { cache.find( key ) }; if( found != cache.end() ) { return found->second; @@ -581,6 +574,11 @@ state_serializer::insert_eu7_pack_models( scene::eu7::pack_bench_inc( &scene::eu7::Eu7PackBench::main_region_inserts ); scene::eu7::pack_bench_inc( &scene::eu7::Eu7PackBench::main_instances_applied ); } + if( + session != nullptr && + session->section_idx != std::numeric_limits::max() ) { + m_pack_section_instances[ session->section_idx ].push_back( instance ); + } } } @@ -894,6 +892,37 @@ state_serializer::apply_eu7_pack_models( insert_eu7_pack_models( models + offset, count, scratchpad, session ); } +std::size_t +state_serializer::unload_eu7_pack_section( int const row, int const column ) { + auto const section_idx { scene::eu7::section_index( row, column ) }; + auto found { m_pack_section_instances.find( section_idx ) }; + if( found == m_pack_section_instances.end() || found->second.empty() ) { + m_pack_section_instances.erase( section_idx ); + return 0; + } + + std::size_t removed { 0 }; + for( auto *instance : found->second ) { + if( instance == nullptr ) { + continue; + } + Particles.erase_bound_node( instance ); + if( simulation::Region != nullptr ) { + simulation::Region->erase( instance ); + } + TAnimModel::release_pack_instance( instance ); + ++removed; + } + + m_pack_section_instances.erase( found ); + return removed; +} + +void +state_serializer::reset_eu7_pack_section_instances() { + m_pack_section_instances.clear(); +} + void state_serializer::apply_eu7_scene( scene::eu7::Eu7Scene const &scene, diff --git a/simulation/simulationstateserializer.h b/simulation/simulationstateserializer.h index 7d7c1c06..ed0464f6 100644 --- a/simulation/simulationstateserializer.h +++ b/simulation/simulationstateserializer.h @@ -9,12 +9,15 @@ http://mozilla.org/MPL/2.0/. #pragma once +#include #include +#include #include "utilities/parser.h" #include "scene/eu7/eu7_types.h" #include "scene/scene.h" +class TAnimModel; class TModel3d; namespace simulation { @@ -22,6 +25,7 @@ namespace simulation { struct eu7_pack_apply_session { std::unordered_map *mesh_cache { nullptr }; std::unordered_map *nodedata_cache { nullptr }; + std::size_t section_idx { std::numeric_limits::max() }; }; struct deserializer_state { @@ -108,8 +112,14 @@ public: // Odlozone sklady AI — ladowane w tle po wejsciu w gre. void drain_deferred_eu7_trainsets( double MaxMs = 12.0 ); + // PACK streaming: usun instancje sekcji z Region i zwolnij pool. + [[nodiscard]] std::size_t + unload_eu7_pack_section( int Row, int Column ); + void + reset_eu7_pack_section_instances(); private: + std::unordered_map> m_pack_section_instances; struct eu7_transform_state; void insert_eu7_models(