eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-18 19:19:19 +02:00
Code Issues 121 Wiki Activity

Tune EU7 PACK streaming with v10 CHNK sub-chunks and worker read cache

Browse Source 
Chunk PACK sections at bake time and decode them incrementally at runtime
to cut apply hitches during fast camera flight; cache parsed section headers
per worker thread to avoid reopening and reparsing on every sub-chunk read.
This commit is contained in:
maj00r
2026-06-14 19:49:24 +02:00
parent 8c069ec5f2
commit 56d72fb25e
9 changed files with 336 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
docs/eu7-format.md
View File

@@ -831,6 +831,28 @@ Sekcje v7 bez nagłówka (flat) pozostają czytelne — brak UMES, runtime skanu
Runtime używa UMES do cold preload meshów zamiast liniowego skanowania wszystkich instancji w sekcji.
### Format sekcji — v10 CHNK (`kPackSectionFormatV10 = 3`)
Rozszerzenie v9: po UMES tabela sub-chunków (domyślnie 512 modeli/chunk przy bake).
```
uint8 section_format — musi być = 3
uint32 solo_count
uint32 inst_count
uint32 unique_mesh_count
uint32[unique_mesh_count] string_id
uint32 chunk_count
Powtórzone chunk_count razy:
uint32 chunk_model_count
uint32 chunk_byte_offset — offset od początku sekcji (względem PIDX pack_offset)
[payload chunków — solo modele, chunk_model_count rekordów każdy]
```
Runtime: worker czyta `read_pack_section_chunk_load(row, col, chunk_index)` z O(1) seek.
Sekcje v9 (format=2) i v7 flat pozostają kompatybilne (`chunk_count` implicit = 1).
---
## 30. PROT — prototypy modeli (v8+)

68
eu07-parser/include/eu07/scene/binary/runtime_module.hpp
View File

@@ -49,8 +49,10 @@ inline constexpr std::array<char, 4> kChunkPlac{'P', 'L', 'A', 'C'};
inline constexpr std::array<char, 4> kChunkPidx{'P', 'I', 'D', 'X'};
inline constexpr std::array<char, 4> kChunkPack{'P', 'A', 'C', 'K'};
// PACK section payload: v9 = UMES (unique mesh list per 1 km section).
// PACK section payload: v9 = UMES, v10 = UMES + CHNK (sub-chunk offset table).
inline constexpr std::uint8_t kPackSectionFormatV9 = 2;
inline constexpr std::uint8_t kPackSectionFormatV10 = 3;
inline constexpr std::size_t kPackSectionChunkModels = 512;
struct PackWriteStats {
std::size_t strings_total = 0;
@@ -437,7 +439,22 @@ struct PackPayloadBuild {
return ids;
}
[[nodiscard]] inline std::vector<char> buildPackSectionPayloadV9(
[[nodiscard]] inline std::vector<char> buildPackModelBlob(
StringTable& table,
const std::vector<runtime::RuntimeModelInstance>& models,
const std::size_t begin,
const std::size_t end) {
std::ostringstream out(std::ios::binary);
for (std::size_t index = begin; index < end; ++index) {
runtime::RuntimeModelInstance worldModel = models[index];
worldModel.node.transform = {};
writeRuntimeModel(out, table, worldModel);
}
const std::string blob = out.str();
return {blob.begin(), blob.end()};
}
[[nodiscard]] inline std::vector<char> buildPackSectionPayloadV10(
StringTable& table,
const std::vector<runtime::RuntimeModelInstance>& models,
const std::uint32_t inst_count = 0,
@@ -445,18 +462,51 @@ struct PackPayloadBuild {
const auto mesh_ids = collectUniqueMeshIds(table, models, prototype_mesh_paths);
const std::uint32_t solo_count = static_cast<std::uint32_t>(models.size());
std::vector<std::vector<char>> chunk_payloads;
chunk_payloads.reserve((models.size() + kPackSectionChunkModels - 1) / kPackSectionChunkModels);
for (std::size_t offset = 0; offset < models.size(); offset += kPackSectionChunkModels) {
const std::size_t end =
std::min(offset + kPackSectionChunkModels, models.size());
chunk_payloads.push_back(buildPackModelBlob(table, models, offset, end));
}
if (chunk_payloads.empty()) {
chunk_payloads.emplace_back();
}
const std::uint32_t chunk_count = static_cast<std::uint32_t>(chunk_payloads.size());
const std::uint32_t header_size =
1u + 4u + 4u + 4u + static_cast<std::uint32_t>(mesh_ids.size()) * 4u + 4u +
chunk_count * 8u;
std::vector<std::uint32_t> chunk_offsets(chunk_count);
std::vector<std::uint32_t> chunk_model_counts(chunk_count);
std::uint32_t payload_offset = header_size;
for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) {
chunk_offsets[chunk_index] = payload_offset;
const std::size_t model_begin =
static_cast<std::size_t>(chunk_index) * kPackSectionChunkModels;
const std::size_t model_end =
std::min(model_begin + kPackSectionChunkModels, models.size());
chunk_model_counts[chunk_index] =
static_cast<std::uint32_t>(model_end > model_begin ? model_end - model_begin : 0);
payload_offset += static_cast<std::uint32_t>(chunk_payloads[chunk_index].size());
}
std::ostringstream packOut(std::ios::binary);
io::writeU8(packOut, kPackSectionFormatV9);
io::writeU8(packOut, kPackSectionFormatV10);
io::writeU32(packOut, solo_count);
io::writeU32(packOut, inst_count);
io::writeU32(packOut, static_cast<std::uint32_t>(mesh_ids.size()));
for (const std::uint32_t id : mesh_ids) {
io::writeU32(packOut, id);
}
for (const runtime::RuntimeModelInstance& model : models) {
runtime::RuntimeModelInstance worldModel = model;
worldModel.node.transform = {};
writeRuntimeModel(packOut, table, worldModel);
io::writeU32(packOut, chunk_count);
for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) {
io::writeU32(packOut, chunk_model_counts[chunk_index]);
io::writeU32(packOut, chunk_offsets[chunk_index]);
}
for (const std::vector<char>& payload : chunk_payloads) {
packOut.write(payload.data(), static_cast<std::streamsize>(payload.size()));
}
const std::string blob = packOut.str();
return {blob.begin(), blob.end()};
@@ -490,7 +540,7 @@ struct PackPayloadBuild {
entry.row = static_cast<std::uint16_t>(batch.section.z);
entry.column = static_cast<std::uint16_t>(batch.section.x);
entry.modelCount = static_cast<std::uint32_t>(batch.models.size());
section_payloads[batch_index] = buildPackSectionPayloadV9(table, batch.models);
section_payloads[batch_index] = buildPackSectionPayloadV10(table, batch.models);
}
std::size_t total_size = 0;
for (const std::vector<char>& payload : section_payloads) {
@@ -524,7 +574,7 @@ struct PackPayloadBuild {
entry.column = static_cast<std::uint16_t>(batch.section.x);
entry.modelCount = static_cast<std::uint32_t>(batch.models.size());
section_payloads[batch_index] = buildPackSectionPayloadV9(table, batch.models);
section_payloads[batch_index] = buildPackSectionPayloadV10(table, batch.models);
}
});
}

2
scene/eu7/eu7_chunks.h
View File

@@ -59,5 +59,7 @@ constexpr std::uint32_t kChunkProt { detail::make_id4( 'P', 'R', 'O', 'T' ) };
constexpr std::uint8_t kPackSectionFormatV8 { 1 };
constexpr std::uint8_t kPackSectionFormatV9 { 2 };
constexpr std::uint8_t kPackSectionFormatV10 { 3 };
constexpr std::size_t kPackSectionChunkModels { 512 };
} // namespace scene::eu7

3
scene/eu7/eu7_pack_bench.cpp
View File

@@ -87,7 +87,8 @@ log_pack_bench_impl( Eu7PackBench const &bench, char const *const title ) {
std::vector<BenchLine> lines {
{ "worker read_pack", bench.worker_read_pack_ms,
"sections=" + std::to_string( bench.worker_sections_done ) +
"chunks=" + std::to_string( bench.worker_chunks_decoded ) +
" sections=" + std::to_string( bench.worker_sections_done ) +
" finalized=" + std::to_string( bench.sections_finalized ) +
" models=" + std::to_string( bench.worker_models_decoded ) +
" fail=" + std::to_string( bench.worker_failures ) },

1
scene/eu7/eu7_pack_bench.h
View File

@@ -20,6 +20,7 @@ namespace scene::eu7 {
struct Eu7PackBench {
// --- worker (async read PACK) ---
double worker_read_pack_ms { 0.0 };
std::uint64_t worker_chunks_decoded { 0 };
std::uint64_t worker_sections_done { 0 };
std::uint64_t sections_finalized { 0 };
std::uint64_t worker_models_decoded { 0 };

190
scene/eu7/eu7_reader.cpp
View File

@@ -505,6 +505,41 @@ parse_pack_section_header(
const std::uint8_t first_byte {
peek == EOF ? std::uint8_t { 0 } : static_cast<std::uint8_t>( peek ) };
bool use_v10_header { false };
if( first_byte == kPackSectionFormatV10 ) {
cursor.section_format = sn_utils::d_uint8( input );
const std::uint32_t solo_total { sn_utils::ld_uint32( input ) };
const std::uint32_t inst_total { sn_utils::ld_uint32( input ) };
const std::uint32_t unique_mesh_count { sn_utils::ld_uint32( input ) };
if( solo_total + inst_total == entry.model_count ) {
use_v10_header = true;
cursor.model_total = solo_total + inst_total;
cursor.solo_remaining = 0;
cursor.inst_remaining = 0;
StringTable const strings { module.strings };
cursor.unique_meshes.reserve( unique_mesh_count );
for( std::uint32_t mesh_idx { 0 }; mesh_idx < unique_mesh_count; ++mesh_idx ) {
cursor.unique_meshes.push_back(
strings.resolve( sn_utils::ld_uint32( input ) ) );
}
cursor.chunk_count = sn_utils::ld_uint32( input );
cursor.chunk_model_counts.resize( cursor.chunk_count );
cursor.chunk_byte_offsets.resize( cursor.chunk_count );
for( std::uint32_t chunk_idx { 0 }; chunk_idx < cursor.chunk_count; ++chunk_idx ) {
cursor.chunk_model_counts[ chunk_idx ] = sn_utils::ld_uint32( input );
cursor.chunk_byte_offsets[ chunk_idx ] = sn_utils::ld_uint32( input );
}
}
}
if( use_v10_header ) {
cursor.models_read = 0;
cursor.header_parsed = true;
return;
}
bool use_v9_header { false };
if( first_byte == kPackSectionFormatV9 ) {
cursor.section_format = sn_utils::d_uint8( input );
@@ -1037,6 +1072,129 @@ find_pack_entry_impl( Eu7Module const &module, int const row, int const column )
return std::nullopt;
}
struct PackSectionReadSession {
std::string source_path;
int row { -1 };
int column { -1 };
Eu7PackIndexEntry entry {};
Eu7PackSectionCursor header {};
std::ifstream input;
bool header_ready { false };
};
thread_local PackSectionReadSession g_pack_section_read_session;
void
reset_pack_section_read_session( PackSectionReadSession &session ) {
if( session.input.is_open() ) {
session.input.close();
}
session = {};
}
void
ensure_pack_file( Eu7Module const &module, PackSectionReadSession &session ) {
if( module.source_path == session.source_path && session.input.is_open() ) {
return;
}
if( session.input.is_open() ) {
session.input.close();
}
session.source_path = module.source_path;
session.header_ready = false;
session.row = -1;
session.column = -1;
if( false == module.source_path.empty() ) {
session.input.open( module.source_path, std::ios::binary );
}
}
void
ensure_section_header(
Eu7Module const &module,
int const row,
int const column,
Eu7PackIndexEntry const &entry,
PackSectionReadSession &session ) {
if(
session.header_ready &&
session.row == row &&
session.column == column &&
session.source_path == module.source_path ) {
return;
}
session.row = row;
session.column = column;
session.entry = entry;
parse_pack_section_header( module, session.input, entry, session.header );
session.header_ready = true;
}
[[nodiscard]] Eu7PackSectionChunkLoad
read_pack_section_chunk_load_impl(
Eu7Module const &module,
int const row,
int const column,
std::uint32_t const chunk_index ) {
Eu7PackSectionChunkLoad result;
if( false == module.has_pack_chunk || module.source_path.empty() ) {
return result;
}
auto const entry { find_pack_entry_impl( module, row, column ) };
if( false == entry.has_value() || entry->model_count == 0 ) {
return result;
}
auto &session { g_pack_section_read_session };
ensure_pack_file( module, session );
if( !session.input ) {
throw std::runtime_error(
"EU7 PACK: nie mozna otworzyc \"" + module.source_path + "\"" );
}
ensure_section_header( module, row, column, *entry, session );
StringTable const strings { module.strings };
auto const &header { session.header };
result.chunk_count = header.chunk_count > 0 ? header.chunk_count : 1u;
if( chunk_index >= result.chunk_count ) {
return result;
}
result.chunk_index = chunk_index;
if( chunk_index == 0 ) {
result.unique_meshes = header.unique_meshes;
}
Eu7PackSectionCursor chunk_cursor { header };
if(
header.section_format == kPackSectionFormatV10 &&
false == header.chunk_byte_offsets.empty() ) {
auto const section_start {
static_cast<std::streamoff>( module.pack_payload_offset + entry->pack_offset ) };
session.input.seekg(
section_start +
static_cast<std::streamoff>( header.chunk_byte_offsets[ chunk_index ] ) );
chunk_cursor.solo_remaining = header.chunk_model_counts[ chunk_index ];
chunk_cursor.inst_remaining = 0;
}
else if( chunk_index > 0 ) {
return result;
}
result.models = read_pack_models_chunk_impl(
module,
session.input,
chunk_cursor,
std::numeric_limits<std::size_t>::max(),
strings );
return result;
}
[[nodiscard]] Eu7PackSectionLoad
read_pack_section_load_impl( Eu7Module const &module, int const row, int const column ) {
Eu7PackSectionLoad result;
@@ -1059,7 +1217,23 @@ read_pack_section_load_impl( Eu7Module const &module, int const row, int const c
Eu7PackSectionCursor header;
parse_pack_section_header( module, input, *entry, header );
result.unique_meshes = std::move( header.unique_meshes );
result.unique_meshes = header.unique_meshes;
if(
header.section_format == kPackSectionFormatV10 &&
header.chunk_count > 0 ) {
result.models.reserve( entry->model_count );
for( std::uint32_t chunk_idx { 0 }; chunk_idx < header.chunk_count; ++chunk_idx ) {
auto const chunk {
read_pack_section_chunk_load_impl( module, row, column, chunk_idx ) };
result.models.insert(
result.models.end(),
std::make_move_iterator( chunk.models.begin() ),
std::make_move_iterator( chunk.models.end() ) );
}
return result;
}
result.models = read_pack_models_chunk_impl(
module, input, header, std::numeric_limits<std::size_t>::max(), strings );
return result;
@@ -1156,4 +1330,18 @@ read_pack_section_load( Eu7Module const &module, int const row, int const column
return read_pack_section_load_impl( module, row, column );
}
Eu7PackSectionChunkLoad
read_pack_section_chunk_load(
Eu7Module const &module,
int const row,
int const column,
std::uint32_t const chunk_index ) {
return read_pack_section_chunk_load_impl( module, row, column, chunk_index );
}
void
reset_pack_section_read_cache() {
reset_pack_section_read_session( g_pack_section_read_session );
}
} // namespace scene::eu7

14
scene/eu7/eu7_reader.h
View File

@@ -31,8 +31,20 @@ find_pack_entry( Eu7Module const &Module, int Row, int Column );
[[nodiscard]] std::vector<Eu7Model>
read_pack_section( Eu7Module const &Module, int Row, int Column );
// Odczyt sekcji PACK: modele + precomputed UMES (v9).
// Odczyt sekcji PACK: modele + precomputed UMES (v9/v10).
[[nodiscard]] Eu7PackSectionLoad
read_pack_section_load( Eu7Module const &Module, int Row, int Column );
// Odczyt jednego sub-chunka sekcji PACK (v10 CHNK; v9/v7 = cala sekcja przy ChunkIndex 0).
[[nodiscard]] Eu7PackSectionChunkLoad
read_pack_section_chunk_load(
Eu7Module const &Module,
int Row,
int Column,
std::uint32_t ChunkIndex );
// Czysci thread-local cache odczytu PACK (otwarty plik + sparsowany naglowek sekcji).
void
reset_pack_section_read_cache();
} // namespace scene::eu7

49
scene/eu7/eu7_section_stream.cpp
View File

@@ -101,6 +101,7 @@ struct PackSectionJob {
int column { 0 };
std::size_t section_idx { 0 };
int priority { 0 };
std::uint32_t next_chunk { 0 };
};
struct PackSectionReady {
@@ -110,6 +111,7 @@ struct PackSectionReady {
std::unique_ptr<std::vector<Eu7Model>> models;
std::vector<std::string> unique_meshes;
bool failed { false };
bool section_final { true };
std::size_t apply_offset { 0 };
std::size_t texture_warm_offset { 0 };
std::size_t umes_cold_offset { 0 };
@@ -820,7 +822,7 @@ apply_pending_chunk(
if( batch.failed ) {
fail_section( batch.section_idx, batch.row, batch.column );
}
else {
else if( batch.section_final ) {
finalize_section( batch );
}
release_pending_buffer();
@@ -832,7 +834,9 @@ apply_pending_chunk(
auto const total { batch.models->size() };
auto const offset { g_stream.pending_apply_offset };
if( offset >= total ) {
finalize_section( batch );
if( batch.section_final ) {
finalize_section( batch );
}
release_pending_buffer();
applied_work = true;
continue;
@@ -983,7 +987,9 @@ apply_pending_chunk(
++chunks_done;
if( g_stream.pending_apply_offset >= total ) {
finalize_section( batch );
if( batch.section_final ) {
finalize_section( batch );
}
release_pending_buffer();
continue;
}
@@ -1115,12 +1121,17 @@ worker_loop( std::stop_token const stop_token ) {
std::unique_ptr<std::vector<Eu7Model>> models;
std::vector<std::string> unique_meshes;
std::uint32_t chunk_count { 1 };
{
PackBenchTimer const read_timer { &Eu7PackBench::worker_read_pack_ms };
auto section { read_pack_section_load( module, job.row, job.column ) };
unique_meshes = std::move( section.unique_meshes );
if( false == section.models.empty() ) {
models = std::make_unique<std::vector<Eu7Model>>( std::move( section.models ) );
auto const chunk {
read_pack_section_chunk_load(
module, job.row, job.column, job.next_chunk ) };
chunk_count = chunk.chunk_count;
unique_meshes = std::move( chunk.unique_meshes );
if( false == chunk.models.empty() ) {
models = std::make_unique<std::vector<Eu7Model>>(
std::move( chunk.models ) );
}
}
@@ -1131,11 +1142,13 @@ worker_loop( std::stop_token const stop_token ) {
result.models = std::move( models );
result.unique_meshes = std::move( unique_meshes );
result.failed = result.models == nullptr;
result.section_final = job.next_chunk + 1 >= chunk_count;
if( result.failed ) {
ErrorLog(
"EU7 PACK: pusty odczyt sekcji " + std::to_string( job.row ) + "," +
std::to_string( job.column ) + " (PIDX model_count=" +
std::to_string( job.column ) + " chunk=" +
std::to_string( job.next_chunk ) + " (PIDX model_count=" +
std::to_string( entry->model_count ) + ")" );
enqueue_failed_section( job );
continue;
@@ -1151,7 +1164,20 @@ worker_loop( std::stop_token const stop_token ) {
g_stream.ready.data.push_back( std::move( result ) );
}
pack_bench_inc( &Eu7PackBench::worker_sections_done );
pack_bench_inc( &Eu7PackBench::worker_chunks_decoded );
if( result.section_final ) {
pack_bench_inc( &Eu7PackBench::worker_sections_done );
}
if( job.next_chunk + 1 < chunk_count ) {
PackSectionJob continue_job { job };
continue_job.next_chunk = job.next_chunk + 1;
{
std::lock_guard<std::mutex> lock { g_stream.jobs.mutex };
g_stream.jobs.data.push_front( continue_job );
}
g_stream.work_cv.notify_one();
}
}
catch( std::exception const &ex ) {
ErrorLog(
@@ -1168,6 +1194,7 @@ stop_workers() {
g_stream.work_cv.notify_all();
g_stream.workers.clear();
g_stream.worker_exit = false;
reset_pack_section_read_cache();
{
std::lock_guard<std::mutex> lock { g_stream.jobs.mutex };
@@ -1581,7 +1608,7 @@ try_dequeue_ready_batch() {
if( it->failed ) {
fail_section( it->section_idx, it->row, it->column );
}
else {
else if( it->section_final ) {
finalize_section( *it );
}
it = g_stream.ready.data.erase( it );
@@ -1672,7 +1699,7 @@ try_dequeue_ready_batch() {
if( batch.failed ) {
fail_section( batch.section_idx, batch.row, batch.column );
}
else {
else if( batch.section_final ) {
finalize_section( batch );
}
return false;

10
scene/eu7/eu7_types.h
View File

@@ -369,6 +369,9 @@ struct Eu7PackSectionCursor {
std::uint8_t section_format { 0 };
bool header_parsed { false };
std::vector<std::string> unique_meshes;
std::uint32_t chunk_count { 0 };
std::vector<std::uint32_t> chunk_model_counts;
std::vector<std::uint32_t> chunk_byte_offsets;
};
struct Eu7PackSectionLoad {
@@ -376,6 +379,13 @@ struct Eu7PackSectionLoad {
std::vector<std::string> unique_meshes;
};
struct Eu7PackSectionChunkLoad {
std::vector<Eu7Model> models;
std::vector<std::string> unique_meshes;
std::uint32_t chunk_count { 1 };
std::uint32_t chunk_index { 0 };
};
struct Eu7PackCatalog {
std::vector<Eu7PackIndexEntry> entries;
std::unordered_map<std::uint32_t, std::size_t> index_by_section;