diff --git a/docs/eu7-format.md b/docs/eu7-format.md index cd57522f..cee532bb 100644 --- a/docs/eu7-format.md +++ b/docs/eu7-format.md @@ -831,6 +831,28 @@ Sekcje v7 bez nagłówka (flat) pozostają czytelne — brak UMES, runtime skanu Runtime używa UMES do cold preload meshów zamiast liniowego skanowania wszystkich instancji w sekcji. +### Format sekcji — v10 CHNK (`kPackSectionFormatV10 = 3`) + +Rozszerzenie v9: po UMES tabela sub-chunków (domyślnie 512 modeli/chunk przy bake). + +``` +uint8 section_format — musi być = 3 +uint32 solo_count +uint32 inst_count +uint32 unique_mesh_count +uint32[unique_mesh_count] string_id + +uint32 chunk_count +Powtórzone chunk_count razy: + uint32 chunk_model_count + uint32 chunk_byte_offset — offset od początku sekcji (względem PIDX pack_offset) + +[payload chunków — solo modele, chunk_model_count rekordów każdy] +``` + +Runtime: worker czyta `read_pack_section_chunk_load(row, col, chunk_index)` z O(1) seek. +Sekcje v9 (format=2) i v7 flat pozostają kompatybilne (`chunk_count` implicit = 1). + --- ## 30. PROT — prototypy modeli (v8+) diff --git a/eu07-parser/include/eu07/scene/binary/runtime_module.hpp b/eu07-parser/include/eu07/scene/binary/runtime_module.hpp index 7b7f05c8..17ef6a47 100644 --- a/eu07-parser/include/eu07/scene/binary/runtime_module.hpp +++ b/eu07-parser/include/eu07/scene/binary/runtime_module.hpp @@ -49,8 +49,10 @@ inline constexpr std::array kChunkPlac{'P', 'L', 'A', 'C'}; inline constexpr std::array kChunkPidx{'P', 'I', 'D', 'X'}; inline constexpr std::array kChunkPack{'P', 'A', 'C', 'K'}; -// PACK section payload: v9 = UMES (unique mesh list per 1 km section). +// PACK section payload: v9 = UMES, v10 = UMES + CHNK (sub-chunk offset table). inline constexpr std::uint8_t kPackSectionFormatV9 = 2; +inline constexpr std::uint8_t kPackSectionFormatV10 = 3; +inline constexpr std::size_t kPackSectionChunkModels = 512; struct PackWriteStats { std::size_t strings_total = 0; @@ -437,7 +439,22 @@ struct PackPayloadBuild { return ids; } -[[nodiscard]] inline std::vector buildPackSectionPayloadV9( +[[nodiscard]] inline std::vector buildPackModelBlob( + StringTable& table, + const std::vector& models, + const std::size_t begin, + const std::size_t end) { + std::ostringstream out(std::ios::binary); + for (std::size_t index = begin; index < end; ++index) { + runtime::RuntimeModelInstance worldModel = models[index]; + worldModel.node.transform = {}; + writeRuntimeModel(out, table, worldModel); + } + const std::string blob = out.str(); + return {blob.begin(), blob.end()}; +} + +[[nodiscard]] inline std::vector buildPackSectionPayloadV10( StringTable& table, const std::vector& models, const std::uint32_t inst_count = 0, @@ -445,18 +462,51 @@ struct PackPayloadBuild { const auto mesh_ids = collectUniqueMeshIds(table, models, prototype_mesh_paths); const std::uint32_t solo_count = static_cast(models.size()); + std::vector> chunk_payloads; + chunk_payloads.reserve((models.size() + kPackSectionChunkModels - 1) / kPackSectionChunkModels); + for (std::size_t offset = 0; offset < models.size(); offset += kPackSectionChunkModels) { + const std::size_t end = + std::min(offset + kPackSectionChunkModels, models.size()); + chunk_payloads.push_back(buildPackModelBlob(table, models, offset, end)); + } + if (chunk_payloads.empty()) { + chunk_payloads.emplace_back(); + } + + const std::uint32_t chunk_count = static_cast(chunk_payloads.size()); + const std::uint32_t header_size = + 1u + 4u + 4u + 4u + static_cast(mesh_ids.size()) * 4u + 4u + + chunk_count * 8u; + + std::vector chunk_offsets(chunk_count); + std::vector chunk_model_counts(chunk_count); + std::uint32_t payload_offset = header_size; + for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) { + chunk_offsets[chunk_index] = payload_offset; + const std::size_t model_begin = + static_cast(chunk_index) * kPackSectionChunkModels; + const std::size_t model_end = + std::min(model_begin + kPackSectionChunkModels, models.size()); + chunk_model_counts[chunk_index] = + static_cast(model_end > model_begin ? model_end - model_begin : 0); + payload_offset += static_cast(chunk_payloads[chunk_index].size()); + } + std::ostringstream packOut(std::ios::binary); - io::writeU8(packOut, kPackSectionFormatV9); + io::writeU8(packOut, kPackSectionFormatV10); io::writeU32(packOut, solo_count); io::writeU32(packOut, inst_count); io::writeU32(packOut, static_cast(mesh_ids.size())); for (const std::uint32_t id : mesh_ids) { io::writeU32(packOut, id); } - for (const runtime::RuntimeModelInstance& model : models) { - runtime::RuntimeModelInstance worldModel = model; - worldModel.node.transform = {}; - writeRuntimeModel(packOut, table, worldModel); + io::writeU32(packOut, chunk_count); + for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) { + io::writeU32(packOut, chunk_model_counts[chunk_index]); + io::writeU32(packOut, chunk_offsets[chunk_index]); + } + for (const std::vector& payload : chunk_payloads) { + packOut.write(payload.data(), static_cast(payload.size())); } const std::string blob = packOut.str(); return {blob.begin(), blob.end()}; @@ -490,7 +540,7 @@ struct PackPayloadBuild { entry.row = static_cast(batch.section.z); entry.column = static_cast(batch.section.x); entry.modelCount = static_cast(batch.models.size()); - section_payloads[batch_index] = buildPackSectionPayloadV9(table, batch.models); + section_payloads[batch_index] = buildPackSectionPayloadV10(table, batch.models); } std::size_t total_size = 0; for (const std::vector& payload : section_payloads) { @@ -524,7 +574,7 @@ struct PackPayloadBuild { entry.column = static_cast(batch.section.x); entry.modelCount = static_cast(batch.models.size()); - section_payloads[batch_index] = buildPackSectionPayloadV9(table, batch.models); + section_payloads[batch_index] = buildPackSectionPayloadV10(table, batch.models); } }); } diff --git a/scene/eu7/eu7_chunks.h b/scene/eu7/eu7_chunks.h index b984af37..b1ee94dd 100644 --- a/scene/eu7/eu7_chunks.h +++ b/scene/eu7/eu7_chunks.h @@ -59,5 +59,7 @@ constexpr std::uint32_t kChunkProt { detail::make_id4( 'P', 'R', 'O', 'T' ) }; constexpr std::uint8_t kPackSectionFormatV8 { 1 }; constexpr std::uint8_t kPackSectionFormatV9 { 2 }; +constexpr std::uint8_t kPackSectionFormatV10 { 3 }; +constexpr std::size_t kPackSectionChunkModels { 512 }; } // namespace scene::eu7 diff --git a/scene/eu7/eu7_pack_bench.cpp b/scene/eu7/eu7_pack_bench.cpp index 4f150003..34653ce2 100644 --- a/scene/eu7/eu7_pack_bench.cpp +++ b/scene/eu7/eu7_pack_bench.cpp @@ -87,7 +87,8 @@ log_pack_bench_impl( Eu7PackBench const &bench, char const *const title ) { std::vector lines { { "worker read_pack", bench.worker_read_pack_ms, - "sections=" + std::to_string( bench.worker_sections_done ) + + "chunks=" + std::to_string( bench.worker_chunks_decoded ) + + " sections=" + std::to_string( bench.worker_sections_done ) + " finalized=" + std::to_string( bench.sections_finalized ) + " models=" + std::to_string( bench.worker_models_decoded ) + " fail=" + std::to_string( bench.worker_failures ) }, diff --git a/scene/eu7/eu7_pack_bench.h b/scene/eu7/eu7_pack_bench.h index 1fa92d84..243d352e 100644 --- a/scene/eu7/eu7_pack_bench.h +++ b/scene/eu7/eu7_pack_bench.h @@ -20,6 +20,7 @@ namespace scene::eu7 { struct Eu7PackBench { // --- worker (async read PACK) --- double worker_read_pack_ms { 0.0 }; + std::uint64_t worker_chunks_decoded { 0 }; std::uint64_t worker_sections_done { 0 }; std::uint64_t sections_finalized { 0 }; std::uint64_t worker_models_decoded { 0 }; diff --git a/scene/eu7/eu7_reader.cpp b/scene/eu7/eu7_reader.cpp index c26550d5..fa624183 100644 --- a/scene/eu7/eu7_reader.cpp +++ b/scene/eu7/eu7_reader.cpp @@ -505,6 +505,41 @@ parse_pack_section_header( const std::uint8_t first_byte { peek == EOF ? std::uint8_t { 0 } : static_cast( peek ) }; + bool use_v10_header { false }; + if( first_byte == kPackSectionFormatV10 ) { + cursor.section_format = sn_utils::d_uint8( input ); + const std::uint32_t solo_total { sn_utils::ld_uint32( input ) }; + const std::uint32_t inst_total { sn_utils::ld_uint32( input ) }; + const std::uint32_t unique_mesh_count { sn_utils::ld_uint32( input ) }; + if( solo_total + inst_total == entry.model_count ) { + use_v10_header = true; + cursor.model_total = solo_total + inst_total; + cursor.solo_remaining = 0; + cursor.inst_remaining = 0; + + StringTable const strings { module.strings }; + cursor.unique_meshes.reserve( unique_mesh_count ); + for( std::uint32_t mesh_idx { 0 }; mesh_idx < unique_mesh_count; ++mesh_idx ) { + cursor.unique_meshes.push_back( + strings.resolve( sn_utils::ld_uint32( input ) ) ); + } + + cursor.chunk_count = sn_utils::ld_uint32( input ); + cursor.chunk_model_counts.resize( cursor.chunk_count ); + cursor.chunk_byte_offsets.resize( cursor.chunk_count ); + for( std::uint32_t chunk_idx { 0 }; chunk_idx < cursor.chunk_count; ++chunk_idx ) { + cursor.chunk_model_counts[ chunk_idx ] = sn_utils::ld_uint32( input ); + cursor.chunk_byte_offsets[ chunk_idx ] = sn_utils::ld_uint32( input ); + } + } + } + + if( use_v10_header ) { + cursor.models_read = 0; + cursor.header_parsed = true; + return; + } + bool use_v9_header { false }; if( first_byte == kPackSectionFormatV9 ) { cursor.section_format = sn_utils::d_uint8( input ); @@ -1037,6 +1072,129 @@ find_pack_entry_impl( Eu7Module const &module, int const row, int const column ) return std::nullopt; } +struct PackSectionReadSession { + std::string source_path; + int row { -1 }; + int column { -1 }; + Eu7PackIndexEntry entry {}; + Eu7PackSectionCursor header {}; + std::ifstream input; + bool header_ready { false }; +}; + +thread_local PackSectionReadSession g_pack_section_read_session; + +void +reset_pack_section_read_session( PackSectionReadSession &session ) { + if( session.input.is_open() ) { + session.input.close(); + } + session = {}; +} + +void +ensure_pack_file( Eu7Module const &module, PackSectionReadSession &session ) { + if( module.source_path == session.source_path && session.input.is_open() ) { + return; + } + + if( session.input.is_open() ) { + session.input.close(); + } + session.source_path = module.source_path; + session.header_ready = false; + session.row = -1; + session.column = -1; + if( false == module.source_path.empty() ) { + session.input.open( module.source_path, std::ios::binary ); + } +} + +void +ensure_section_header( + Eu7Module const &module, + int const row, + int const column, + Eu7PackIndexEntry const &entry, + PackSectionReadSession &session ) { + if( + session.header_ready && + session.row == row && + session.column == column && + session.source_path == module.source_path ) { + return; + } + + session.row = row; + session.column = column; + session.entry = entry; + parse_pack_section_header( module, session.input, entry, session.header ); + session.header_ready = true; +} + +[[nodiscard]] Eu7PackSectionChunkLoad +read_pack_section_chunk_load_impl( + Eu7Module const &module, + int const row, + int const column, + std::uint32_t const chunk_index ) { + Eu7PackSectionChunkLoad result; + if( false == module.has_pack_chunk || module.source_path.empty() ) { + return result; + } + + auto const entry { find_pack_entry_impl( module, row, column ) }; + if( false == entry.has_value() || entry->model_count == 0 ) { + return result; + } + + auto &session { g_pack_section_read_session }; + ensure_pack_file( module, session ); + if( !session.input ) { + throw std::runtime_error( + "EU7 PACK: nie mozna otworzyc \"" + module.source_path + "\"" ); + } + + ensure_section_header( module, row, column, *entry, session ); + + StringTable const strings { module.strings }; + auto const &header { session.header }; + result.chunk_count = header.chunk_count > 0 ? header.chunk_count : 1u; + + if( chunk_index >= result.chunk_count ) { + return result; + } + + result.chunk_index = chunk_index; + if( chunk_index == 0 ) { + result.unique_meshes = header.unique_meshes; + } + + Eu7PackSectionCursor chunk_cursor { header }; + if( + header.section_format == kPackSectionFormatV10 && + false == header.chunk_byte_offsets.empty() ) { + auto const section_start { + static_cast( module.pack_payload_offset + entry->pack_offset ) }; + session.input.seekg( + section_start + + static_cast( header.chunk_byte_offsets[ chunk_index ] ) ); + chunk_cursor.solo_remaining = header.chunk_model_counts[ chunk_index ]; + chunk_cursor.inst_remaining = 0; + } + else if( chunk_index > 0 ) { + return result; + } + + result.models = read_pack_models_chunk_impl( + module, + session.input, + chunk_cursor, + std::numeric_limits::max(), + strings ); + return result; +} + [[nodiscard]] Eu7PackSectionLoad read_pack_section_load_impl( Eu7Module const &module, int const row, int const column ) { Eu7PackSectionLoad result; @@ -1059,7 +1217,23 @@ read_pack_section_load_impl( Eu7Module const &module, int const row, int const c Eu7PackSectionCursor header; parse_pack_section_header( module, input, *entry, header ); - result.unique_meshes = std::move( header.unique_meshes ); + result.unique_meshes = header.unique_meshes; + + if( + header.section_format == kPackSectionFormatV10 && + header.chunk_count > 0 ) { + result.models.reserve( entry->model_count ); + for( std::uint32_t chunk_idx { 0 }; chunk_idx < header.chunk_count; ++chunk_idx ) { + auto const chunk { + read_pack_section_chunk_load_impl( module, row, column, chunk_idx ) }; + result.models.insert( + result.models.end(), + std::make_move_iterator( chunk.models.begin() ), + std::make_move_iterator( chunk.models.end() ) ); + } + return result; + } + result.models = read_pack_models_chunk_impl( module, input, header, std::numeric_limits::max(), strings ); return result; @@ -1156,4 +1330,18 @@ read_pack_section_load( Eu7Module const &module, int const row, int const column return read_pack_section_load_impl( module, row, column ); } +Eu7PackSectionChunkLoad +read_pack_section_chunk_load( + Eu7Module const &module, + int const row, + int const column, + std::uint32_t const chunk_index ) { + return read_pack_section_chunk_load_impl( module, row, column, chunk_index ); +} + +void +reset_pack_section_read_cache() { + reset_pack_section_read_session( g_pack_section_read_session ); +} + } // namespace scene::eu7 diff --git a/scene/eu7/eu7_reader.h b/scene/eu7/eu7_reader.h index bedd0ab9..5992c41c 100644 --- a/scene/eu7/eu7_reader.h +++ b/scene/eu7/eu7_reader.h @@ -31,8 +31,20 @@ find_pack_entry( Eu7Module const &Module, int Row, int Column ); [[nodiscard]] std::vector read_pack_section( Eu7Module const &Module, int Row, int Column ); -// Odczyt sekcji PACK: modele + precomputed UMES (v9). +// Odczyt sekcji PACK: modele + precomputed UMES (v9/v10). [[nodiscard]] Eu7PackSectionLoad read_pack_section_load( Eu7Module const &Module, int Row, int Column ); +// Odczyt jednego sub-chunka sekcji PACK (v10 CHNK; v9/v7 = cala sekcja przy ChunkIndex 0). +[[nodiscard]] Eu7PackSectionChunkLoad +read_pack_section_chunk_load( + Eu7Module const &Module, + int Row, + int Column, + std::uint32_t ChunkIndex ); + +// Czysci thread-local cache odczytu PACK (otwarty plik + sparsowany naglowek sekcji). +void +reset_pack_section_read_cache(); + } // namespace scene::eu7 diff --git a/scene/eu7/eu7_section_stream.cpp b/scene/eu7/eu7_section_stream.cpp index a40ae19d..34cbc6b4 100644 --- a/scene/eu7/eu7_section_stream.cpp +++ b/scene/eu7/eu7_section_stream.cpp @@ -101,6 +101,7 @@ struct PackSectionJob { int column { 0 }; std::size_t section_idx { 0 }; int priority { 0 }; + std::uint32_t next_chunk { 0 }; }; struct PackSectionReady { @@ -110,6 +111,7 @@ struct PackSectionReady { std::unique_ptr> models; std::vector unique_meshes; bool failed { false }; + bool section_final { true }; std::size_t apply_offset { 0 }; std::size_t texture_warm_offset { 0 }; std::size_t umes_cold_offset { 0 }; @@ -820,7 +822,7 @@ apply_pending_chunk( if( batch.failed ) { fail_section( batch.section_idx, batch.row, batch.column ); } - else { + else if( batch.section_final ) { finalize_section( batch ); } release_pending_buffer(); @@ -832,7 +834,9 @@ apply_pending_chunk( auto const total { batch.models->size() }; auto const offset { g_stream.pending_apply_offset }; if( offset >= total ) { - finalize_section( batch ); + if( batch.section_final ) { + finalize_section( batch ); + } release_pending_buffer(); applied_work = true; continue; @@ -983,7 +987,9 @@ apply_pending_chunk( ++chunks_done; if( g_stream.pending_apply_offset >= total ) { - finalize_section( batch ); + if( batch.section_final ) { + finalize_section( batch ); + } release_pending_buffer(); continue; } @@ -1115,12 +1121,17 @@ worker_loop( std::stop_token const stop_token ) { std::unique_ptr> models; std::vector unique_meshes; + std::uint32_t chunk_count { 1 }; { PackBenchTimer const read_timer { &Eu7PackBench::worker_read_pack_ms }; - auto section { read_pack_section_load( module, job.row, job.column ) }; - unique_meshes = std::move( section.unique_meshes ); - if( false == section.models.empty() ) { - models = std::make_unique>( std::move( section.models ) ); + auto const chunk { + read_pack_section_chunk_load( + module, job.row, job.column, job.next_chunk ) }; + chunk_count = chunk.chunk_count; + unique_meshes = std::move( chunk.unique_meshes ); + if( false == chunk.models.empty() ) { + models = std::make_unique>( + std::move( chunk.models ) ); } } @@ -1131,11 +1142,13 @@ worker_loop( std::stop_token const stop_token ) { result.models = std::move( models ); result.unique_meshes = std::move( unique_meshes ); result.failed = result.models == nullptr; + result.section_final = job.next_chunk + 1 >= chunk_count; if( result.failed ) { ErrorLog( "EU7 PACK: pusty odczyt sekcji " + std::to_string( job.row ) + "," + - std::to_string( job.column ) + " (PIDX model_count=" + + std::to_string( job.column ) + " chunk=" + + std::to_string( job.next_chunk ) + " (PIDX model_count=" + std::to_string( entry->model_count ) + ")" ); enqueue_failed_section( job ); continue; @@ -1151,7 +1164,20 @@ worker_loop( std::stop_token const stop_token ) { g_stream.ready.data.push_back( std::move( result ) ); } - pack_bench_inc( &Eu7PackBench::worker_sections_done ); + pack_bench_inc( &Eu7PackBench::worker_chunks_decoded ); + if( result.section_final ) { + pack_bench_inc( &Eu7PackBench::worker_sections_done ); + } + + if( job.next_chunk + 1 < chunk_count ) { + PackSectionJob continue_job { job }; + continue_job.next_chunk = job.next_chunk + 1; + { + std::lock_guard lock { g_stream.jobs.mutex }; + g_stream.jobs.data.push_front( continue_job ); + } + g_stream.work_cv.notify_one(); + } } catch( std::exception const &ex ) { ErrorLog( @@ -1168,6 +1194,7 @@ stop_workers() { g_stream.work_cv.notify_all(); g_stream.workers.clear(); g_stream.worker_exit = false; + reset_pack_section_read_cache(); { std::lock_guard lock { g_stream.jobs.mutex }; @@ -1581,7 +1608,7 @@ try_dequeue_ready_batch() { if( it->failed ) { fail_section( it->section_idx, it->row, it->column ); } - else { + else if( it->section_final ) { finalize_section( *it ); } it = g_stream.ready.data.erase( it ); @@ -1672,7 +1699,7 @@ try_dequeue_ready_batch() { if( batch.failed ) { fail_section( batch.section_idx, batch.row, batch.column ); } - else { + else if( batch.section_final ) { finalize_section( batch ); } return false; diff --git a/scene/eu7/eu7_types.h b/scene/eu7/eu7_types.h index 901c4abf..4acc8023 100644 --- a/scene/eu7/eu7_types.h +++ b/scene/eu7/eu7_types.h @@ -369,6 +369,9 @@ struct Eu7PackSectionCursor { std::uint8_t section_format { 0 }; bool header_parsed { false }; std::vector unique_meshes; + std::uint32_t chunk_count { 0 }; + std::vector chunk_model_counts; + std::vector chunk_byte_offsets; }; struct Eu7PackSectionLoad { @@ -376,6 +379,13 @@ struct Eu7PackSectionLoad { std::vector unique_meshes; }; +struct Eu7PackSectionChunkLoad { + std::vector models; + std::vector unique_meshes; + std::uint32_t chunk_count { 1 }; + std::uint32_t chunk_index { 0 }; +}; + struct Eu7PackCatalog { std::vector entries; std::unordered_map index_by_section;