diff --git a/docs/eu7-format.md b/docs/eu7-format.md index 74496aca..cd57522f 100644 --- a/docs/eu7-format.md +++ b/docs/eu7-format.md @@ -805,6 +805,32 @@ Powtórzone inst_count razy: Instancje v8 są rozwijane przez `expand_prototype_instance()` łącząc dane z PROT z lokalizacją instancji. +### Format sekcji — v9 UMES (`kPackSectionFormatV9 = 2`) + +Rozszerzenie v8/v7: przed payloadem modeli zapisana jest deduplikowana lista ścieżek mesh (`model_file`). + +``` +uint8 section_format — musi być = 2 +uint32 solo_count — modele pełne (bez prototypu) +uint32 inst_count — instancje odwołujące się do PROT (0 w bake v7) +uint32 unique_mesh_count +uint32[unique_mesh_count] string_id — model_file (posortowane, unikalne; pomija puste/"notload") + +Powtórzone solo_count razy: + (pełny rekord RuntimeModelInstance, transformacja zerowana) + +Powtórzone inst_count razy (gdy inst_count > 0): + uint32 proto_id + Vec3 location + Vec3 angles + Vec3 scale + string_id name +``` + +Sekcje v7 bez nagłówka (flat) pozostają czytelne — brak UMES, runtime skanuje instancje. + +Runtime używa UMES do cold preload meshów zamiast liniowego skanowania wszystkich instancji w sekcji. + --- ## 30. PROT — prototypy modeli (v8+) diff --git a/eu07-parser/include/eu07/scene/binary/runtime_module.hpp b/eu07-parser/include/eu07/scene/binary/runtime_module.hpp index e9bf47ca..7b7f05c8 100644 --- a/eu07-parser/include/eu07/scene/binary/runtime_module.hpp +++ b/eu07-parser/include/eu07/scene/binary/runtime_module.hpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -22,6 +23,7 @@ #include #include #include +#include #include #include @@ -47,6 +49,9 @@ inline constexpr std::array kChunkPlac{'P', 'L', 'A', 'C'}; inline constexpr std::array kChunkPidx{'P', 'I', 'D', 'X'}; inline constexpr std::array kChunkPack{'P', 'A', 'C', 'K'}; +// PACK section payload: v9 = UMES (unique mesh list per 1 km section). +inline constexpr std::uint8_t kPackSectionFormatV9 = 2; + struct PackWriteStats { std::size_t strings_total = 0; std::size_t pack_entries = 0; @@ -397,10 +402,57 @@ struct PackPayloadBuild { std::vector entries; }; -[[nodiscard]] inline std::vector buildPackSectionPayload( +[[nodiscard]] inline bool isLoadableMeshPath(const std::string_view path) { + return !path.empty() && path != "notload"; +} + +[[nodiscard]] inline std::vector collectUniqueMeshIds( StringTable& table, - const std::vector& models) { + const std::vector& models, + const std::vector* prototype_mesh_paths = nullptr) { + std::vector paths; + paths.reserve(models.size()); + + for (const runtime::RuntimeModelInstance& model : models) { + if (isLoadableMeshPath(model.modelFile)) { + paths.emplace_back(model.modelFile); + } + } + if (prototype_mesh_paths != nullptr) { + for (const std::string_view path : *prototype_mesh_paths) { + if (isLoadableMeshPath(path)) { + paths.emplace_back(path); + } + } + } + + std::stable_sort(paths.begin(), paths.end()); + paths.erase(std::unique(paths.begin(), paths.end()), paths.end()); + + std::vector ids; + ids.reserve(paths.size()); + for (const std::string& path : paths) { + ids.push_back(table.intern(path)); + } + return ids; +} + +[[nodiscard]] inline std::vector buildPackSectionPayloadV9( + StringTable& table, + const std::vector& models, + const std::uint32_t inst_count = 0, + const std::vector* prototype_mesh_paths = nullptr) { + const auto mesh_ids = collectUniqueMeshIds(table, models, prototype_mesh_paths); + const std::uint32_t solo_count = static_cast(models.size()); + std::ostringstream packOut(std::ios::binary); + io::writeU8(packOut, kPackSectionFormatV9); + io::writeU32(packOut, solo_count); + io::writeU32(packOut, inst_count); + io::writeU32(packOut, static_cast(mesh_ids.size())); + for (const std::uint32_t id : mesh_ids) { + io::writeU32(packOut, id); + } for (const runtime::RuntimeModelInstance& model : models) { runtime::RuntimeModelInstance worldModel = model; worldModel.node.transform = {}; @@ -438,7 +490,7 @@ struct PackPayloadBuild { entry.row = static_cast(batch.section.z); entry.column = static_cast(batch.section.x); entry.modelCount = static_cast(batch.models.size()); - section_payloads[batch_index] = buildPackSectionPayload(table, batch.models); + section_payloads[batch_index] = buildPackSectionPayloadV9(table, batch.models); } std::size_t total_size = 0; for (const std::vector& payload : section_payloads) { @@ -472,7 +524,7 @@ struct PackPayloadBuild { entry.column = static_cast(batch.section.x); entry.modelCount = static_cast(batch.models.size()); - section_payloads[batch_index] = buildPackSectionPayload(table, batch.models); + section_payloads[batch_index] = buildPackSectionPayloadV9(table, batch.models); } }); } diff --git a/scene/eu7/eu7_chunks.h b/scene/eu7/eu7_chunks.h index 67c09163..b984af37 100644 --- a/scene/eu7/eu7_chunks.h +++ b/scene/eu7/eu7_chunks.h @@ -58,5 +58,6 @@ constexpr std::uint32_t kChunkPack { detail::make_id4( 'P', 'A', 'C', 'K' ) }; constexpr std::uint32_t kChunkProt { detail::make_id4( 'P', 'R', 'O', 'T' ) }; constexpr std::uint8_t kPackSectionFormatV8 { 1 }; +constexpr std::uint8_t kPackSectionFormatV9 { 2 }; } // namespace scene::eu7 diff --git a/scene/eu7/eu7_model_prefetch.cpp b/scene/eu7/eu7_model_prefetch.cpp index cdc588ab..31d1cff8 100644 --- a/scene/eu7/eu7_model_prefetch.cpp +++ b/scene/eu7/eu7_model_prefetch.cpp @@ -95,6 +95,26 @@ reset_pack_texture_warm_cache() { g_session_warmed_textures.clear(); } +void +preload_pack_model_paths( std::vector const &model_files ) { + std::unordered_set seen; + seen.reserve( model_files.size() ); + + for( auto model_file : model_files ) { + if( model_file.empty() || model_file == "notload" ) { + continue; + } + + replace_slashes( model_file ); + if( false == seen.insert( model_file ).second ) { + continue; + } + + std::lock_guard lock { g_pack_mesh_load_mutex }; + preload_pack_model_file( model_file ); + } +} + void preload_pack_models( std::vector const &models ) { std::unordered_set seen; @@ -116,6 +136,17 @@ preload_pack_models( std::vector const &models ) { } } +void +preload_pack_models( + std::vector const &models, + std::vector const &unique_meshes ) { + if( false == unique_meshes.empty() ) { + preload_pack_model_paths( unique_meshes ); + return; + } + preload_pack_models( models ); +} + std::size_t warm_pack_textures_main( Eu7Model const *const models, std::size_t const count ) { if( models == nullptr || count == 0 || GfxRenderer == nullptr ) { diff --git a/scene/eu7/eu7_model_prefetch.h b/scene/eu7/eu7_model_prefetch.h index e7113982..31ae5495 100644 --- a/scene/eu7/eu7_model_prefetch.h +++ b/scene/eu7/eu7_model_prefetch.h @@ -22,6 +22,12 @@ namespace scene::eu7 { void preload_pack_models( std::vector const &Models ); +// Jak wyzej, ale iteruje precomputed UMES zamiast skanowac instancje. +void +preload_pack_models( + std::vector const &Models, + std::vector const &UniqueMeshes ); + // Main thread: Fetch_Material dla unikalnych texture_file z chunka przed apply. // Zwraca liczbe unikalnych Fetch_Material w tym slice. std::size_t diff --git a/scene/eu7/eu7_reader.cpp b/scene/eu7/eu7_reader.cpp index 46a8a77b..c26550d5 100644 --- a/scene/eu7/eu7_reader.cpp +++ b/scene/eu7/eu7_reader.cpp @@ -505,11 +505,35 @@ parse_pack_section_header( const std::uint8_t first_byte { peek == EOF ? std::uint8_t { 0 } : static_cast( peek ) }; - if( first_byte == 2 ) { - throw std::runtime_error( - "EU7 PACK: format v9 nieobslugiwany — przebake plik .eu7" ); + bool use_v9_header { false }; + if( first_byte == kPackSectionFormatV9 ) { + cursor.section_format = sn_utils::d_uint8( input ); + const std::uint32_t solo_total { sn_utils::ld_uint32( input ) }; + const std::uint32_t inst_total { sn_utils::ld_uint32( input ) }; + const std::uint32_t unique_mesh_count { sn_utils::ld_uint32( input ) }; + if( solo_total + inst_total == entry.model_count ) { + use_v9_header = true; + cursor.model_total = solo_total + inst_total; + cursor.solo_remaining = solo_total; + cursor.inst_remaining = inst_total; + + StringTable const strings { module.strings }; + cursor.unique_meshes.reserve( unique_mesh_count ); + for( std::uint32_t mesh_idx { 0 }; mesh_idx < unique_mesh_count; ++mesh_idx ) { + cursor.unique_meshes.push_back( + strings.resolve( sn_utils::ld_uint32( input ) ) ); + } + } } + if( use_v9_header ) { + cursor.models_read = 0; + cursor.header_parsed = true; + return; + } + + input.seekg( section_start ); + // v8 header starts with byte 1 — same as kNodeFlagHasName on flat v7 models. // Accept v8 only when PROT exists and solo+inst counts match PIDX model_count. bool use_v8_header { false }; @@ -1013,15 +1037,16 @@ find_pack_entry_impl( Eu7Module const &module, int const row, int const column ) return std::nullopt; } -[[nodiscard]] std::vector -read_pack_section_impl( Eu7Module const &module, int const row, int const column ) { +[[nodiscard]] Eu7PackSectionLoad +read_pack_section_load_impl( Eu7Module const &module, int const row, int const column ) { + Eu7PackSectionLoad result; if( false == module.has_pack_chunk || module.source_path.empty() ) { - return {}; + return result; } auto const entry { find_pack_entry_impl( module, row, column ) }; if( false == entry.has_value() || entry->model_count == 0 ) { - return {}; + return result; } std::ifstream input { module.source_path, std::ios::binary }; @@ -1034,8 +1059,15 @@ read_pack_section_impl( Eu7Module const &module, int const row, int const column Eu7PackSectionCursor header; parse_pack_section_header( module, input, *entry, header ); - return read_pack_models_chunk_impl( + result.unique_meshes = std::move( header.unique_meshes ); + result.models = read_pack_models_chunk_impl( module, input, header, std::numeric_limits::max(), strings ); + return result; +} + +[[nodiscard]] std::vector +read_pack_section_impl( Eu7Module const &module, int const row, int const column ) { + return read_pack_section_load_impl( module, row, column ).models; } } // namespace @@ -1119,4 +1151,9 @@ read_pack_section( Eu7Module const &module, int const row, int const column ) { return read_pack_section_impl( module, row, column ); } +Eu7PackSectionLoad +read_pack_section_load( Eu7Module const &module, int const row, int const column ) { + return read_pack_section_load_impl( module, row, column ); +} + } // namespace scene::eu7 diff --git a/scene/eu7/eu7_reader.h b/scene/eu7/eu7_reader.h index 40a984aa..bedd0ab9 100644 --- a/scene/eu7/eu7_reader.h +++ b/scene/eu7/eu7_reader.h @@ -31,4 +31,8 @@ find_pack_entry( Eu7Module const &Module, int Row, int Column ); [[nodiscard]] std::vector read_pack_section( Eu7Module const &Module, int Row, int Column ); +// Odczyt sekcji PACK: modele + precomputed UMES (v9). +[[nodiscard]] Eu7PackSectionLoad +read_pack_section_load( Eu7Module const &Module, int Row, int Column ); + } // namespace scene::eu7 diff --git a/scene/eu7/eu7_section_stream.cpp b/scene/eu7/eu7_section_stream.cpp index f8a37028..a40ae19d 100644 --- a/scene/eu7/eu7_section_stream.cpp +++ b/scene/eu7/eu7_section_stream.cpp @@ -69,8 +69,8 @@ constexpr double kReenqueueDistanceM { 500.0 }; constexpr double kCatchupReenqueueDistanceM { 40.0 }; constexpr std::size_t kHeavySectionModelThreshold { 800 }; constexpr double kUrgentApplyBudgetMs { 32.0 }; -constexpr std::size_t kUrgentSliceInstances { 512 }; -constexpr std::size_t kUrgentSliceColdMeshes { 20 }; +constexpr std::size_t kUrgentSliceInstances { 96 }; +constexpr std::size_t kUrgentSliceColdMeshes { 2 }; constexpr double kUrgentColdBudgetMs { 24.0 }; constexpr std::size_t kUrgentMaxChunksPerDrain { 8 }; constexpr double kReadyTexturePrefetchBudgetMs { 14.0 }; @@ -108,9 +108,11 @@ struct PackSectionReady { int column { 0 }; std::size_t section_idx { 0 }; std::unique_ptr> models; + std::vector unique_meshes; bool failed { false }; std::size_t apply_offset { 0 }; std::size_t texture_warm_offset { 0 }; + std::size_t umes_cold_offset { 0 }; }; struct SectionStreamState { @@ -404,13 +406,27 @@ pending_apply_is_urgent() { adaptive_slice_instances( std::size_t const section_total, bool const urgent = false ) { if( urgent ) { auto limit { kUrgentSliceInstances }; + auto const speed { camera_stream_speed_mps() }; + if( speed > 1200.0 ) { + limit = std::min( limit, std::size_t { 64 } ); + } + else if( speed > 600.0 ) { + limit = std::min( limit, std::size_t { 96 } ); + } if( section_total > 4000 ) { - limit = std::min( limit, std::size_t { 384 } ); + limit = std::min( limit, std::size_t { 64 } ); } else if( section_total > 2000 ) { - limit = std::min( limit, std::size_t { 448 } ); + limit = std::min( limit, std::size_t { 96 } ); } - return limit; + auto const last_ms { pack_bench_stream().last_chunk_ms }; + if( last_ms >= 24.0 ) { + limit = std::min( limit, std::size_t { 32 } ); + } + else if( last_ms >= 12.0 ) { + limit = std::min( limit, std::size_t { 48 } ); + } + return std::max( limit, std::size_t { 16 } ); } auto limit { gameplay_slice_instances() }; @@ -562,6 +578,69 @@ preload_slice_cold_meshes( return cold_loaded; } +[[nodiscard]] std::size_t +preload_umes_cold_meshes( + std::vector const &unique_meshes, + std::size_t const offset, + std::size_t const max_cold_meshes, + double const cold_budget_ms, + std::size_t &offset_out ) { + offset_out = offset; + if( unique_meshes.empty() || offset >= unique_meshes.size() ) { + return 0; + } + + std::size_t cold_loaded { 0 }; + auto const cold_started { std::chrono::steady_clock::now() }; + for( std::size_t i { offset }; i < unique_meshes.size(); ++i ) { + auto model_file { unique_meshes[ i ] }; + if( model_file.empty() || model_file == "notload" ) { + offset_out = i + 1; + continue; + } + replace_slashes( model_file ); + if( g_stream.mesh_cache.contains( model_file ) ) { + pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit ); + offset_out = i + 1; + continue; + } + if( cold_loaded >= max_cold_meshes ) { + pack_bench_inc( &Eu7PackBench::stream_cold_slice_truncated ); + break; + } + if( cold_budget_ms > 0.0 && cold_loaded > 0 ) { + auto const elapsed_ms { + std::chrono::duration( + std::chrono::steady_clock::now() - cold_started ).count() }; + if( elapsed_ms >= cold_budget_ms ) { + pack_bench_inc( &Eu7PackBench::stream_cold_slice_truncated ); + break; + } + } + auto const was_global_cached { TModelsManager::IsModelCached( model_file ) }; + TModel3d *mesh { nullptr }; + { + PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms }; + mesh = TModelsManager::GetModel( model_file, false, false ); + pack_bench_inc( &Eu7PackBench::main_cold_getmodel_calls ); + } + if( was_global_cached ) { + pack_bench_inc( &Eu7PackBench::stream_mesh_global_hit ); + } + else { + pack_bench_inc( &Eu7PackBench::stream_mesh_disk_load ); + } + if( mesh != nullptr ) { + TAnimModel::warm_instanceable_cache( mesh ); + } + g_stream.mesh_cache.emplace( model_file, mesh ); + ++cold_loaded; + offset_out = i + 1; + } + + return cold_loaded; +} + constexpr int kApplyStuckSkipFrames { 20 }; std::size_t g_apply_stuck_offset { std::numeric_limits::max() }; @@ -817,17 +896,27 @@ apply_pending_chunk( { auto const phase_started { std::chrono::steady_clock::now() }; - cold_loads = preload_slice_cold_meshes( - models_ptr + offset, - chunk_count, - cold_mesh_limit, - effective_cold_budget_ms, - chunk_count ); + if( false == batch.unique_meshes.empty() ) { + cold_loads = preload_umes_cold_meshes( + batch.unique_meshes, + batch.umes_cold_offset, + cold_mesh_limit, + effective_cold_budget_ms, + batch.umes_cold_offset ); + } + else { + cold_loads = preload_slice_cold_meshes( + models_ptr + offset, + chunk_count, + cold_mesh_limit, + effective_cold_budget_ms, + chunk_count ); + } cold_ms = std::chrono::duration( std::chrono::steady_clock::now() - phase_started ).count(); } pack_bench_inc( &Eu7PackBench::stream_inst_after_cold, chunk_count ); - if( chunk_count < planned_inst ) { + if( batch.unique_meshes.empty() && chunk_count < planned_inst ) { pack_bench_inc( &Eu7PackBench::stream_cold_slice_truncated ); } if( chunk_count == 0 ) { @@ -1025,11 +1114,13 @@ worker_loop( std::stop_token const stop_token ) { } std::unique_ptr> models; + std::vector unique_meshes; { PackBenchTimer const read_timer { &Eu7PackBench::worker_read_pack_ms }; - auto section { read_pack_section( module, job.row, job.column ) }; - if( false == section.empty() ) { - models = std::make_unique>( std::move( section ) ); + auto section { read_pack_section_load( module, job.row, job.column ) }; + unique_meshes = std::move( section.unique_meshes ); + if( false == section.models.empty() ) { + models = std::make_unique>( std::move( section.models ) ); } } @@ -1038,6 +1129,7 @@ worker_loop( std::stop_token const stop_token ) { result.column = job.column; result.section_idx = job.section_idx; result.models = std::move( models ); + result.unique_meshes = std::move( unique_meshes ); result.failed = result.models == nullptr; if( result.failed ) { @@ -1052,7 +1144,7 @@ worker_loop( std::stop_token const stop_token ) { pack_bench_inc( &Eu7PackBench::worker_models_decoded, result.models->size() ); - preload_pack_models( *result.models ); + preload_pack_models( *result.models, result.unique_meshes ); { std::lock_guard lock { g_stream.ready.mutex }; diff --git a/scene/eu7/eu7_types.h b/scene/eu7/eu7_types.h index eb4d67f9..901c4abf 100644 --- a/scene/eu7/eu7_types.h +++ b/scene/eu7/eu7_types.h @@ -368,6 +368,12 @@ struct Eu7PackSectionCursor { std::uint32_t model_total { 0 }; std::uint8_t section_format { 0 }; bool header_parsed { false }; + std::vector unique_meshes; +}; + +struct Eu7PackSectionLoad { + std::vector models; + std::vector unique_meshes; }; struct Eu7PackCatalog {