Tune EU7 PACK streaming with v10 CHNK sub-chunks and worker read cache

Chunk PACK sections at bake time and decode them incrementally at runtime
to cut apply hitches during fast camera flight; cache parsed section headers
per worker thread to avoid reopening and reparsing on every sub-chunk read.

docs/eu7-format.md

@@ -831,6 +831,28 @@ Sekcje v7 bez nagłówka (flat) pozostają czytelne — brak UMES, runtime skanu
 
 Runtime używa UMES do cold preload meshów zamiast liniowego skanowania wszystkich instancji w sekcji.
 
+### Format sekcji — v10 CHNK (`kPackSectionFormatV10 = 3`)
+
+Rozszerzenie v9: po UMES tabela sub-chunków (domyślnie 512 modeli/chunk przy bake).
+
+```
+uint8    section_format    — musi być = 3
+uint32   solo_count
+uint32   inst_count
+uint32   unique_mesh_count
+uint32[unique_mesh_count] string_id
+
+uint32   chunk_count
+Powtórzone chunk_count razy:
+    uint32   chunk_model_count
+    uint32   chunk_byte_offset   — offset od początku sekcji (względem PIDX pack_offset)
+
+[payload chunków — solo modele, chunk_model_count rekordów każdy]
+```
+
+Runtime: worker czyta `read_pack_section_chunk_load(row, col, chunk_index)` z O(1) seek.
+Sekcje v9 (format=2) i v7 flat pozostają kompatybilne (`chunk_count` implicit = 1).
+
 ---
 
 ## 30. PROT — prototypy modeli (v8+)

eu07-parser/include/eu07/scene/binary/runtime_module.hpp

@@ -49,8 +49,10 @@ inline constexpr std::array<char, 4> kChunkPlac{'P', 'L', 'A', 'C'};
 inline constexpr std::array<char, 4> kChunkPidx{'P', 'I', 'D', 'X'};
 inline constexpr std::array<char, 4> kChunkPack{'P', 'A', 'C', 'K'};
 
-// PACK section payload: v9 = UMES (unique mesh list per 1 km section).
+// PACK section payload: v9 = UMES, v10 = UMES + CHNK (sub-chunk offset table).
 inline constexpr std::uint8_t kPackSectionFormatV9 = 2;
+inline constexpr std::uint8_t kPackSectionFormatV10 = 3;
+inline constexpr std::size_t kPackSectionChunkModels = 512;
 
 struct PackWriteStats {
     std::size_t strings_total = 0;
@@ -437,7 +439,22 @@ struct PackPayloadBuild {
     return ids;
 }
 
-[[nodiscard]] inline std::vector<char> buildPackSectionPayloadV9(
+[[nodiscard]] inline std::vector<char> buildPackModelBlob(
+    StringTable& table,
+    const std::vector<runtime::RuntimeModelInstance>& models,
+    const std::size_t begin,
+    const std::size_t end) {
+    std::ostringstream out(std::ios::binary);
+    for (std::size_t index = begin; index < end; ++index) {
+        runtime::RuntimeModelInstance worldModel = models[index];
+        worldModel.node.transform = {};
+        writeRuntimeModel(out, table, worldModel);
+    }
+    const std::string blob = out.str();
+    return {blob.begin(), blob.end()};
+}
+
+[[nodiscard]] inline std::vector<char> buildPackSectionPayloadV10(
     StringTable& table,
     const std::vector<runtime::RuntimeModelInstance>& models,
     const std::uint32_t inst_count = 0,
@@ -445,18 +462,51 @@ struct PackPayloadBuild {
     const auto mesh_ids = collectUniqueMeshIds(table, models, prototype_mesh_paths);
     const std::uint32_t solo_count = static_cast<std::uint32_t>(models.size());
 
+    std::vector<std::vector<char>> chunk_payloads;
+    chunk_payloads.reserve((models.size() + kPackSectionChunkModels - 1) / kPackSectionChunkModels);
+    for (std::size_t offset = 0; offset < models.size(); offset += kPackSectionChunkModels) {
+        const std::size_t end =
+            std::min(offset + kPackSectionChunkModels, models.size());
+        chunk_payloads.push_back(buildPackModelBlob(table, models, offset, end));
+    }
+    if (chunk_payloads.empty()) {
+        chunk_payloads.emplace_back();
+    }
+
+    const std::uint32_t chunk_count = static_cast<std::uint32_t>(chunk_payloads.size());
+    const std::uint32_t header_size =
+        1u + 4u + 4u + 4u + static_cast<std::uint32_t>(mesh_ids.size()) * 4u + 4u +
+        chunk_count * 8u;
+
+    std::vector<std::uint32_t> chunk_offsets(chunk_count);
+    std::vector<std::uint32_t> chunk_model_counts(chunk_count);
+    std::uint32_t payload_offset = header_size;
+    for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) {
+        chunk_offsets[chunk_index] = payload_offset;
+        const std::size_t model_begin =
+            static_cast<std::size_t>(chunk_index) * kPackSectionChunkModels;
+        const std::size_t model_end =
+            std::min(model_begin + kPackSectionChunkModels, models.size());
+        chunk_model_counts[chunk_index] =
+            static_cast<std::uint32_t>(model_end > model_begin ? model_end - model_begin : 0);
+        payload_offset += static_cast<std::uint32_t>(chunk_payloads[chunk_index].size());
+    }
+
     std::ostringstream packOut(std::ios::binary);
-    io::writeU8(packOut, kPackSectionFormatV9);
+    io::writeU8(packOut, kPackSectionFormatV10);
     io::writeU32(packOut, solo_count);
     io::writeU32(packOut, inst_count);
     io::writeU32(packOut, static_cast<std::uint32_t>(mesh_ids.size()));
     for (const std::uint32_t id : mesh_ids) {
         io::writeU32(packOut, id);
     }
-    for (const runtime::RuntimeModelInstance& model : models) {
+    io::writeU32(packOut, chunk_count);
-        runtime::RuntimeModelInstance worldModel = model;
+    for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) {
-        worldModel.node.transform = {};
+        io::writeU32(packOut, chunk_model_counts[chunk_index]);
-        writeRuntimeModel(packOut, table, worldModel);
+        io::writeU32(packOut, chunk_offsets[chunk_index]);
+    }
+    for (const std::vector<char>& payload : chunk_payloads) {
+        packOut.write(payload.data(), static_cast<std::streamsize>(payload.size()));
     }
     const std::string blob = packOut.str();
     return {blob.begin(), blob.end()};
@@ -490,7 +540,7 @@ struct PackPayloadBuild {
             entry.row = static_cast<std::uint16_t>(batch.section.z);
             entry.column = static_cast<std::uint16_t>(batch.section.x);
             entry.modelCount = static_cast<std::uint32_t>(batch.models.size());
-            section_payloads[batch_index] = buildPackSectionPayloadV9(table, batch.models);
+            section_payloads[batch_index] = buildPackSectionPayloadV10(table, batch.models);
         }
         std::size_t total_size = 0;
         for (const std::vector<char>& payload : section_payloads) {
@@ -524,7 +574,7 @@ struct PackPayloadBuild {
                 entry.column = static_cast<std::uint16_t>(batch.section.x);
                 entry.modelCount = static_cast<std::uint32_t>(batch.models.size());
 
-                section_payloads[batch_index] = buildPackSectionPayloadV9(table, batch.models);
+                section_payloads[batch_index] = buildPackSectionPayloadV10(table, batch.models);
             }
         });
     }

scene/eu7/eu7_chunks.h

@@ -59,5 +59,7 @@ constexpr std::uint32_t kChunkProt { detail::make_id4( 'P', 'R', 'O', 'T' ) };
 
 constexpr std::uint8_t kPackSectionFormatV8 { 1 };
 constexpr std::uint8_t kPackSectionFormatV9 { 2 };
+constexpr std::uint8_t kPackSectionFormatV10 { 3 };
+constexpr std::size_t kPackSectionChunkModels { 512 };
 
 } // namespace scene::eu7

scene/eu7/eu7_pack_bench.cpp

@@ -87,7 +87,8 @@ log_pack_bench_impl( Eu7PackBench const &bench, char const *const title ) {
 
     std::vector<BenchLine> lines {
         { "worker read_pack", bench.worker_read_pack_ms,
-            "sections=" + std::to_string( bench.worker_sections_done ) +
+            "chunks=" + std::to_string( bench.worker_chunks_decoded ) +
+            " sections=" + std::to_string( bench.worker_sections_done ) +
             " finalized=" + std::to_string( bench.sections_finalized ) +
             " models=" + std::to_string( bench.worker_models_decoded ) +
             " fail=" + std::to_string( bench.worker_failures ) },

scene/eu7/eu7_pack_bench.h

@@ -20,6 +20,7 @@ namespace scene::eu7 {
 struct Eu7PackBench {
     // --- worker (async read PACK) ---
     double worker_read_pack_ms { 0.0 };
+    std::uint64_t worker_chunks_decoded { 0 };
     std::uint64_t worker_sections_done { 0 };
     std::uint64_t sections_finalized { 0 };
     std::uint64_t worker_models_decoded { 0 };

scene/eu7/eu7_reader.cpp

@@ -505,6 +505,41 @@ parse_pack_section_header(
     const std::uint8_t first_byte {
         peek == EOF ? std::uint8_t { 0 } : static_cast<std::uint8_t>( peek ) };
 
+    bool use_v10_header { false };
+    if( first_byte == kPackSectionFormatV10 ) {
+        cursor.section_format = sn_utils::d_uint8( input );
+        const std::uint32_t solo_total { sn_utils::ld_uint32( input ) };
+        const std::uint32_t inst_total { sn_utils::ld_uint32( input ) };
+        const std::uint32_t unique_mesh_count { sn_utils::ld_uint32( input ) };
+        if( solo_total + inst_total == entry.model_count ) {
+            use_v10_header = true;
+            cursor.model_total = solo_total + inst_total;
+            cursor.solo_remaining = 0;
+            cursor.inst_remaining = 0;
+
+            StringTable const strings { module.strings };
+            cursor.unique_meshes.reserve( unique_mesh_count );
+            for( std::uint32_t mesh_idx { 0 }; mesh_idx < unique_mesh_count; ++mesh_idx ) {
+                cursor.unique_meshes.push_back(
+                    strings.resolve( sn_utils::ld_uint32( input ) ) );
+            }
+
+            cursor.chunk_count = sn_utils::ld_uint32( input );
+            cursor.chunk_model_counts.resize( cursor.chunk_count );
+            cursor.chunk_byte_offsets.resize( cursor.chunk_count );
+            for( std::uint32_t chunk_idx { 0 }; chunk_idx < cursor.chunk_count; ++chunk_idx ) {
+                cursor.chunk_model_counts[ chunk_idx ] = sn_utils::ld_uint32( input );
+                cursor.chunk_byte_offsets[ chunk_idx ] = sn_utils::ld_uint32( input );
+            }
+        }
+    }
+
+    if( use_v10_header ) {
+        cursor.models_read = 0;
+        cursor.header_parsed = true;
+        return;
+    }
+
     bool use_v9_header { false };
     if( first_byte == kPackSectionFormatV9 ) {
         cursor.section_format = sn_utils::d_uint8( input );
@@ -1037,6 +1072,129 @@ find_pack_entry_impl( Eu7Module const &module, int const row, int const column )
     return std::nullopt;
 }
 
+struct PackSectionReadSession {
+    std::string source_path;
+    int row { -1 };
+    int column { -1 };
+    Eu7PackIndexEntry entry {};
+    Eu7PackSectionCursor header {};
+    std::ifstream input;
+    bool header_ready { false };
+};
+
+thread_local PackSectionReadSession g_pack_section_read_session;
+
+void
+reset_pack_section_read_session( PackSectionReadSession &session ) {
+    if( session.input.is_open() ) {
+        session.input.close();
+    }
+    session = {};
+}
+
+void
+ensure_pack_file( Eu7Module const &module, PackSectionReadSession &session ) {
+    if( module.source_path == session.source_path && session.input.is_open() ) {
+        return;
+    }
+
+    if( session.input.is_open() ) {
+        session.input.close();
+    }
+    session.source_path = module.source_path;
+    session.header_ready = false;
+    session.row = -1;
+    session.column = -1;
+    if( false == module.source_path.empty() ) {
+        session.input.open( module.source_path, std::ios::binary );
+    }
+}
+
+void
+ensure_section_header(
+    Eu7Module const &module,
+    int const row,
+    int const column,
+    Eu7PackIndexEntry const &entry,
+    PackSectionReadSession &session ) {
+    if(
+        session.header_ready &&
+        session.row == row &&
+        session.column == column &&
+        session.source_path == module.source_path ) {
+        return;
+    }
+
+    session.row = row;
+    session.column = column;
+    session.entry = entry;
+    parse_pack_section_header( module, session.input, entry, session.header );
+    session.header_ready = true;
+}
+
+[[nodiscard]] Eu7PackSectionChunkLoad
+read_pack_section_chunk_load_impl(
+    Eu7Module const &module,
+    int const row,
+    int const column,
+    std::uint32_t const chunk_index ) {
+    Eu7PackSectionChunkLoad result;
+    if( false == module.has_pack_chunk || module.source_path.empty() ) {
+        return result;
+    }
+
+    auto const entry { find_pack_entry_impl( module, row, column ) };
+    if( false == entry.has_value() || entry->model_count == 0 ) {
+        return result;
+    }
+
+    auto &session { g_pack_section_read_session };
+    ensure_pack_file( module, session );
+    if( !session.input ) {
+        throw std::runtime_error(
+            "EU7 PACK: nie mozna otworzyc \"" + module.source_path + "\"" );
+    }
+
+    ensure_section_header( module, row, column, *entry, session );
+
+    StringTable const strings { module.strings };
+    auto const &header { session.header };
+    result.chunk_count = header.chunk_count > 0 ? header.chunk_count : 1u;
+
+    if( chunk_index >= result.chunk_count ) {
+        return result;
+    }
+
+    result.chunk_index = chunk_index;
+    if( chunk_index == 0 ) {
+        result.unique_meshes = header.unique_meshes;
+    }
+
+    Eu7PackSectionCursor chunk_cursor { header };
+    if(
+        header.section_format == kPackSectionFormatV10 &&
+        false == header.chunk_byte_offsets.empty() ) {
+        auto const section_start {
+            static_cast<std::streamoff>( module.pack_payload_offset + entry->pack_offset ) };
+        session.input.seekg(
+            section_start +
+            static_cast<std::streamoff>( header.chunk_byte_offsets[ chunk_index ] ) );
+        chunk_cursor.solo_remaining = header.chunk_model_counts[ chunk_index ];
+        chunk_cursor.inst_remaining = 0;
+    }
+    else if( chunk_index > 0 ) {
+        return result;
+    }
+
+    result.models = read_pack_models_chunk_impl(
+        module,
+        session.input,
+        chunk_cursor,
+        std::numeric_limits<std::size_t>::max(),
+        strings );
+    return result;
+}
+
 [[nodiscard]] Eu7PackSectionLoad
 read_pack_section_load_impl( Eu7Module const &module, int const row, int const column ) {
     Eu7PackSectionLoad result;
@@ -1059,7 +1217,23 @@ read_pack_section_load_impl( Eu7Module const &module, int const row, int const c
     Eu7PackSectionCursor header;
     parse_pack_section_header( module, input, *entry, header );
 
-    result.unique_meshes = std::move( header.unique_meshes );
+    result.unique_meshes = header.unique_meshes;
+
+    if(
+        header.section_format == kPackSectionFormatV10 &&
+        header.chunk_count > 0 ) {
+        result.models.reserve( entry->model_count );
+        for( std::uint32_t chunk_idx { 0 }; chunk_idx < header.chunk_count; ++chunk_idx ) {
+            auto const chunk {
+                read_pack_section_chunk_load_impl( module, row, column, chunk_idx ) };
+            result.models.insert(
+                result.models.end(),
+                std::make_move_iterator( chunk.models.begin() ),
+                std::make_move_iterator( chunk.models.end() ) );
+        }
+        return result;
+    }
+
     result.models = read_pack_models_chunk_impl(
         module, input, header, std::numeric_limits<std::size_t>::max(), strings );
     return result;
@@ -1156,4 +1330,18 @@ read_pack_section_load( Eu7Module const &module, int const row, int const column
     return read_pack_section_load_impl( module, row, column );
 }
 
+Eu7PackSectionChunkLoad
+read_pack_section_chunk_load(
+    Eu7Module const &module,
+    int const row,
+    int const column,
+    std::uint32_t const chunk_index ) {
+    return read_pack_section_chunk_load_impl( module, row, column, chunk_index );
+}
+
+void
+reset_pack_section_read_cache() {
+    reset_pack_section_read_session( g_pack_section_read_session );
+}
+
 } // namespace scene::eu7

scene/eu7/eu7_reader.h

@@ -31,8 +31,20 @@ find_pack_entry( Eu7Module const &Module, int Row, int Column );
 [[nodiscard]] std::vector<Eu7Model>
 read_pack_section( Eu7Module const &Module, int Row, int Column );
 
-// Odczyt sekcji PACK: modele + precomputed UMES (v9).
+// Odczyt sekcji PACK: modele + precomputed UMES (v9/v10).
 [[nodiscard]] Eu7PackSectionLoad
 read_pack_section_load( Eu7Module const &Module, int Row, int Column );
 
+// Odczyt jednego sub-chunka sekcji PACK (v10 CHNK; v9/v7 = cala sekcja przy ChunkIndex 0).
+[[nodiscard]] Eu7PackSectionChunkLoad
+read_pack_section_chunk_load(
+    Eu7Module const &Module,
+    int Row,
+    int Column,
+    std::uint32_t ChunkIndex );
+
+// Czysci thread-local cache odczytu PACK (otwarty plik + sparsowany naglowek sekcji).
+void
+reset_pack_section_read_cache();
+
 } // namespace scene::eu7

scene/eu7/eu7_section_stream.cpp

@@ -101,6 +101,7 @@ struct PackSectionJob {
     int column { 0 };
     std::size_t section_idx { 0 };
     int priority { 0 };
+    std::uint32_t next_chunk { 0 };
 };
 
 struct PackSectionReady {
@@ -110,6 +111,7 @@ struct PackSectionReady {
     std::unique_ptr<std::vector<Eu7Model>> models;
     std::vector<std::string> unique_meshes;
     bool failed { false };
+    bool section_final { true };
     std::size_t apply_offset { 0 };
     std::size_t texture_warm_offset { 0 };
     std::size_t umes_cold_offset { 0 };
@@ -820,7 +822,7 @@ apply_pending_chunk(
             if( batch.failed ) {
                 fail_section( batch.section_idx, batch.row, batch.column );
             }
-            else {
+            else if( batch.section_final ) {
                 finalize_section( batch );
             }
             release_pending_buffer();
@@ -832,7 +834,9 @@ apply_pending_chunk(
         auto const total { batch.models->size() };
         auto const offset { g_stream.pending_apply_offset };
         if( offset >= total ) {
-            finalize_section( batch );
+            if( batch.section_final ) {
+                finalize_section( batch );
+            }
             release_pending_buffer();
             applied_work = true;
             continue;
@@ -983,7 +987,9 @@ apply_pending_chunk(
         ++chunks_done;
 
         if( g_stream.pending_apply_offset >= total ) {
-            finalize_section( batch );
+            if( batch.section_final ) {
+                finalize_section( batch );
+            }
             release_pending_buffer();
             continue;
         }
@@ -1115,12 +1121,17 @@ worker_loop( std::stop_token const stop_token ) {
 
             std::unique_ptr<std::vector<Eu7Model>> models;
             std::vector<std::string> unique_meshes;
+            std::uint32_t chunk_count { 1 };
             {
                 PackBenchTimer const read_timer { &Eu7PackBench::worker_read_pack_ms };
-                auto section { read_pack_section_load( module, job.row, job.column ) };
+                auto const chunk {
-                unique_meshes = std::move( section.unique_meshes );
+                    read_pack_section_chunk_load(
-                if( false == section.models.empty() ) {
+                        module, job.row, job.column, job.next_chunk ) };
-                    models = std::make_unique<std::vector<Eu7Model>>( std::move( section.models ) );
+                chunk_count = chunk.chunk_count;
+                unique_meshes = std::move( chunk.unique_meshes );
+                if( false == chunk.models.empty() ) {
+                    models = std::make_unique<std::vector<Eu7Model>>(
+                        std::move( chunk.models ) );
                 }
             }
 
@@ -1131,11 +1142,13 @@ worker_loop( std::stop_token const stop_token ) {
             result.models = std::move( models );
             result.unique_meshes = std::move( unique_meshes );
             result.failed = result.models == nullptr;
+            result.section_final = job.next_chunk + 1 >= chunk_count;
 
             if( result.failed ) {
                 ErrorLog(
                     "EU7 PACK: pusty odczyt sekcji " + std::to_string( job.row ) + "," +
-                    std::to_string( job.column ) + " (PIDX model_count=" +
+                    std::to_string( job.column ) + " chunk=" +
+                    std::to_string( job.next_chunk ) + " (PIDX model_count=" +
                     std::to_string( entry->model_count ) + ")" );
                 enqueue_failed_section( job );
                 continue;
@@ -1151,7 +1164,20 @@ worker_loop( std::stop_token const stop_token ) {
                 g_stream.ready.data.push_back( std::move( result ) );
             }
 
-            pack_bench_inc( &Eu7PackBench::worker_sections_done );
+            pack_bench_inc( &Eu7PackBench::worker_chunks_decoded );
+            if( result.section_final ) {
+                pack_bench_inc( &Eu7PackBench::worker_sections_done );
+            }
+
+            if( job.next_chunk + 1 < chunk_count ) {
+                PackSectionJob continue_job { job };
+                continue_job.next_chunk = job.next_chunk + 1;
+                {
+                    std::lock_guard<std::mutex> lock { g_stream.jobs.mutex };
+                    g_stream.jobs.data.push_front( continue_job );
+                }
+                g_stream.work_cv.notify_one();
+            }
         }
         catch( std::exception const &ex ) {
             ErrorLog(
@@ -1168,6 +1194,7 @@ stop_workers() {
     g_stream.work_cv.notify_all();
     g_stream.workers.clear();
     g_stream.worker_exit = false;
+    reset_pack_section_read_cache();
 
     {
         std::lock_guard<std::mutex> lock { g_stream.jobs.mutex };
@@ -1581,7 +1608,7 @@ try_dequeue_ready_batch() {
                 if( it->failed ) {
                     fail_section( it->section_idx, it->row, it->column );
                 }
-                else {
+                else if( it->section_final ) {
                     finalize_section( *it );
                 }
                 it = g_stream.ready.data.erase( it );
@@ -1672,7 +1699,7 @@ try_dequeue_ready_batch() {
         if( batch.failed ) {
             fail_section( batch.section_idx, batch.row, batch.column );
         }
-        else {
+        else if( batch.section_final ) {
             finalize_section( batch );
         }
         return false;

scene/eu7/eu7_types.h

@@ -369,6 +369,9 @@ struct Eu7PackSectionCursor {
     std::uint8_t section_format { 0 };
     bool header_parsed { false };
     std::vector<std::string> unique_meshes;
+    std::uint32_t chunk_count { 0 };
+    std::vector<std::uint32_t> chunk_model_counts;
+    std::vector<std::uint32_t> chunk_byte_offsets;
 };
 
 struct Eu7PackSectionLoad {
@@ -376,6 +379,13 @@ struct Eu7PackSectionLoad {
     std::vector<std::string> unique_meshes;
 };
 
+struct Eu7PackSectionChunkLoad {
+    std::vector<Eu7Model> models;
+    std::vector<std::string> unique_meshes;
+    std::uint32_t chunk_count { 1 };
+    std::uint32_t chunk_index { 0 };
+};
+
 struct Eu7PackCatalog {
     std::vector<Eu7PackIndexEntry> entries;
     std::unordered_map<std::uint32_t, std::size_t> index_by_section;