eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-19 05:29:17 +02:00
Code Issues 121 Wiki Activity

more verbose streaming diagnostic

Browse Source
This commit is contained in:
maj00r
2026-06-14 19:32:30 +02:00
parent 0b79d5b3f4
commit 8c069ec5f2
9 changed files with 283 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
docs/eu7-format.md
View File

@@ -805,6 +805,32 @@ Powtórzone inst_count razy:
Instancje v8 są rozwijane przez `expand_prototype_instance()` łącząc dane z PROT z lokalizacją instancji.
### Format sekcji — v9 UMES (`kPackSectionFormatV9 = 2`)
Rozszerzenie v8/v7: przed payloadem modeli zapisana jest deduplikowana lista ścieżek mesh (`model_file`).
```
uint8 section_format — musi być = 2
uint32 solo_count — modele pełne (bez prototypu)
uint32 inst_count — instancje odwołujące się do PROT (0 w bake v7)
uint32 unique_mesh_count
uint32[unique_mesh_count] string_id — model_file (posortowane, unikalne; pomija puste/"notload")
Powtórzone solo_count razy:
(pełny rekord RuntimeModelInstance, transformacja zerowana)
Powtórzone inst_count razy (gdy inst_count > 0):
uint32 proto_id
Vec3 location
Vec3 angles
Vec3 scale
string_id name
```
Sekcje v7 bez nagłówka (flat) pozostają czytelne — brak UMES, runtime skanuje instancje.
Runtime używa UMES do cold preload meshów zamiast liniowego skanowania wszystkich instancji w sekcji.
---
## 30. PROT — prototypy modeli (v8+)

60
eu07-parser/include/eu07/scene/binary/runtime_module.hpp
View File

@@ -11,6 +11,7 @@
#include <eu07/scene/binary/string_table.hpp>
#include <eu07/scene/runtime/nodes.hpp>
#include <algorithm>
#include <array>
#include <atomic>
#include <cctype>
@@ -22,6 +23,7 @@
#include <iostream>
#include <sstream>
#include <stdexcept>
#include <string_view>
#include <thread>
#include <vector>
@@ -47,6 +49,9 @@ inline constexpr std::array<char, 4> kChunkPlac{'P', 'L', 'A', 'C'};
inline constexpr std::array<char, 4> kChunkPidx{'P', 'I', 'D', 'X'};
inline constexpr std::array<char, 4> kChunkPack{'P', 'A', 'C', 'K'};
// PACK section payload: v9 = UMES (unique mesh list per 1 km section).
inline constexpr std::uint8_t kPackSectionFormatV9 = 2;
struct PackWriteStats {
std::size_t strings_total = 0;
std::size_t pack_entries = 0;
@@ -397,10 +402,57 @@ struct PackPayloadBuild {
std::vector<PackSectionIndexEntry> entries;
};
[[nodiscard]] inline std::vector<char> buildPackSectionPayload(
[[nodiscard]] inline bool isLoadableMeshPath(const std::string_view path) {
return !path.empty() && path != "notload";
}
[[nodiscard]] inline std::vector<std::uint32_t> collectUniqueMeshIds(
StringTable& table,
const std::vector<runtime::RuntimeModelInstance>& models) {
const std::vector<runtime::RuntimeModelInstance>& models,
const std::vector<std::string_view>* prototype_mesh_paths = nullptr) {
std::vector<std::string> paths;
paths.reserve(models.size());
for (const runtime::RuntimeModelInstance& model : models) {
if (isLoadableMeshPath(model.modelFile)) {
paths.emplace_back(model.modelFile);
}
}
if (prototype_mesh_paths != nullptr) {
for (const std::string_view path : *prototype_mesh_paths) {
if (isLoadableMeshPath(path)) {
paths.emplace_back(path);
}
}
}
std::stable_sort(paths.begin(), paths.end());
paths.erase(std::unique(paths.begin(), paths.end()), paths.end());
std::vector<std::uint32_t> ids;
ids.reserve(paths.size());
for (const std::string& path : paths) {
ids.push_back(table.intern(path));
}
return ids;
}
[[nodiscard]] inline std::vector<char> buildPackSectionPayloadV9(
StringTable& table,
const std::vector<runtime::RuntimeModelInstance>& models,
const std::uint32_t inst_count = 0,
const std::vector<std::string_view>* prototype_mesh_paths = nullptr) {
const auto mesh_ids = collectUniqueMeshIds(table, models, prototype_mesh_paths);
const std::uint32_t solo_count = static_cast<std::uint32_t>(models.size());
std::ostringstream packOut(std::ios::binary);
io::writeU8(packOut, kPackSectionFormatV9);
io::writeU32(packOut, solo_count);
io::writeU32(packOut, inst_count);
io::writeU32(packOut, static_cast<std::uint32_t>(mesh_ids.size()));
for (const std::uint32_t id : mesh_ids) {
io::writeU32(packOut, id);
}
for (const runtime::RuntimeModelInstance& model : models) {
runtime::RuntimeModelInstance worldModel = model;
worldModel.node.transform = {};
@@ -438,7 +490,7 @@ struct PackPayloadBuild {
entry.row = static_cast<std::uint16_t>(batch.section.z);
entry.column = static_cast<std::uint16_t>(batch.section.x);
entry.modelCount = static_cast<std::uint32_t>(batch.models.size());
section_payloads[batch_index] = buildPackSectionPayload(table, batch.models);
section_payloads[batch_index] = buildPackSectionPayloadV9(table, batch.models);
}
std::size_t total_size = 0;
for (const std::vector<char>& payload : section_payloads) {
@@ -472,7 +524,7 @@ struct PackPayloadBuild {
entry.column = static_cast<std::uint16_t>(batch.section.x);
entry.modelCount = static_cast<std::uint32_t>(batch.models.size());
section_payloads[batch_index] = buildPackSectionPayload(table, batch.models);
section_payloads[batch_index] = buildPackSectionPayloadV9(table, batch.models);
}
});
}

1
scene/eu7/eu7_chunks.h
View File

@@ -58,5 +58,6 @@ constexpr std::uint32_t kChunkPack { detail::make_id4( 'P', 'A', 'C', 'K' ) };
constexpr std::uint32_t kChunkProt { detail::make_id4( 'P', 'R', 'O', 'T' ) };
constexpr std::uint8_t kPackSectionFormatV8 { 1 };
constexpr std::uint8_t kPackSectionFormatV9 { 2 };
} // namespace scene::eu7

31
scene/eu7/eu7_model_prefetch.cpp
View File

@@ -95,6 +95,26 @@ reset_pack_texture_warm_cache() {
g_session_warmed_textures.clear();
}
void
preload_pack_model_paths( std::vector<std::string> const &model_files ) {
std::unordered_set<std::string> seen;
seen.reserve( model_files.size() );
for( auto model_file : model_files ) {
if( model_file.empty() || model_file == "notload" ) {
continue;
}
replace_slashes( model_file );
if( false == seen.insert( model_file ).second ) {
continue;
}
std::lock_guard<std::mutex> lock { g_pack_mesh_load_mutex };
preload_pack_model_file( model_file );
}
}
void
preload_pack_models( std::vector<Eu7Model> const &models ) {
std::unordered_set<std::string> seen;
@@ -116,6 +136,17 @@ preload_pack_models( std::vector<Eu7Model> const &models ) {
}
}
void
preload_pack_models(
std::vector<Eu7Model> const &models,
std::vector<std::string> const &unique_meshes ) {
if( false == unique_meshes.empty() ) {
preload_pack_model_paths( unique_meshes );
return;
}
preload_pack_models( models );
}
std::size_t
warm_pack_textures_main( Eu7Model const *const models, std::size_t const count ) {
if( models == nullptr || count == 0 || GfxRenderer == nullptr ) {

6
scene/eu7/eu7_model_prefetch.h
View File

@@ -22,6 +22,12 @@ namespace scene::eu7 {
void
preload_pack_models( std::vector<Eu7Model> const &Models );
// Jak wyzej, ale iteruje precomputed UMES zamiast skanowac instancje.
void
preload_pack_models(
std::vector<Eu7Model> const &Models,
std::vector<std::string> const &UniqueMeshes );
// Main thread: Fetch_Material dla unikalnych texture_file z chunka przed apply.
// Zwraca liczbe unikalnych Fetch_Material w tym slice.
std::size_t

53
scene/eu7/eu7_reader.cpp
View File

@@ -505,11 +505,35 @@ parse_pack_section_header(
const std::uint8_t first_byte {
peek == EOF ? std::uint8_t { 0 } : static_cast<std::uint8_t>( peek ) };
if( first_byte == 2 ) {
throw std::runtime_error(
"EU7 PACK: format v9 nieobslugiwany — przebake plik .eu7" );
bool use_v9_header { false };
if( first_byte == kPackSectionFormatV9 ) {
cursor.section_format = sn_utils::d_uint8( input );
const std::uint32_t solo_total { sn_utils::ld_uint32( input ) };
const std::uint32_t inst_total { sn_utils::ld_uint32( input ) };
const std::uint32_t unique_mesh_count { sn_utils::ld_uint32( input ) };
if( solo_total + inst_total == entry.model_count ) {
use_v9_header = true;
cursor.model_total = solo_total + inst_total;
cursor.solo_remaining = solo_total;
cursor.inst_remaining = inst_total;
StringTable const strings { module.strings };
cursor.unique_meshes.reserve( unique_mesh_count );
for( std::uint32_t mesh_idx { 0 }; mesh_idx < unique_mesh_count; ++mesh_idx ) {
cursor.unique_meshes.push_back(
strings.resolve( sn_utils::ld_uint32( input ) ) );
}
}
}
if( use_v9_header ) {
cursor.models_read = 0;
cursor.header_parsed = true;
return;
}
input.seekg( section_start );
// v8 header starts with byte 1 — same as kNodeFlagHasName on flat v7 models.
// Accept v8 only when PROT exists and solo+inst counts match PIDX model_count.
bool use_v8_header { false };
@@ -1013,15 +1037,16 @@ find_pack_entry_impl( Eu7Module const &module, int const row, int const column )
return std::nullopt;
}
[[nodiscard]] std::vector<Eu7Model>
read_pack_section_impl( Eu7Module const &module, int const row, int const column ) {
[[nodiscard]] Eu7PackSectionLoad
read_pack_section_load_impl( Eu7Module const &module, int const row, int const column ) {
Eu7PackSectionLoad result;
if( false == module.has_pack_chunk || module.source_path.empty() ) {
return {};
return result;
}
auto const entry { find_pack_entry_impl( module, row, column ) };
if( false == entry.has_value() || entry->model_count == 0 ) {
return {};
return result;
}
std::ifstream input { module.source_path, std::ios::binary };
@@ -1034,8 +1059,15 @@ read_pack_section_impl( Eu7Module const &module, int const row, int const column
Eu7PackSectionCursor header;
parse_pack_section_header( module, input, *entry, header );
return read_pack_models_chunk_impl(
result.unique_meshes = std::move( header.unique_meshes );
result.models = read_pack_models_chunk_impl(
module, input, header, std::numeric_limits<std::size_t>::max(), strings );
return result;
}
[[nodiscard]] std::vector<Eu7Model>
read_pack_section_impl( Eu7Module const &module, int const row, int const column ) {
return read_pack_section_load_impl( module, row, column ).models;
}
} // namespace
@@ -1119,4 +1151,9 @@ read_pack_section( Eu7Module const &module, int const row, int const column ) {
return read_pack_section_impl( module, row, column );
}
Eu7PackSectionLoad
read_pack_section_load( Eu7Module const &module, int const row, int const column ) {
return read_pack_section_load_impl( module, row, column );
}
} // namespace scene::eu7

4
scene/eu7/eu7_reader.h
View File

@@ -31,4 +31,8 @@ find_pack_entry( Eu7Module const &Module, int Row, int Column );
[[nodiscard]] std::vector<Eu7Model>
read_pack_section( Eu7Module const &Module, int Row, int Column );
// Odczyt sekcji PACK: modele + precomputed UMES (v9).
[[nodiscard]] Eu7PackSectionLoad
read_pack_section_load( Eu7Module const &Module, int Row, int Column );
} // namespace scene::eu7

124
scene/eu7/eu7_section_stream.cpp
View File

@@ -69,8 +69,8 @@ constexpr double kReenqueueDistanceM { 500.0 };
constexpr double kCatchupReenqueueDistanceM { 40.0 };
constexpr std::size_t kHeavySectionModelThreshold { 800 };
constexpr double kUrgentApplyBudgetMs { 32.0 };
constexpr std::size_t kUrgentSliceInstances { 512 };
constexpr std::size_t kUrgentSliceColdMeshes { 20 };
constexpr std::size_t kUrgentSliceInstances { 96 };
constexpr std::size_t kUrgentSliceColdMeshes { 2 };
constexpr double kUrgentColdBudgetMs { 24.0 };
constexpr std::size_t kUrgentMaxChunksPerDrain { 8 };
constexpr double kReadyTexturePrefetchBudgetMs { 14.0 };
@@ -108,9 +108,11 @@ struct PackSectionReady {
int column { 0 };
std::size_t section_idx { 0 };
std::unique_ptr<std::vector<Eu7Model>> models;
std::vector<std::string> unique_meshes;
bool failed { false };
std::size_t apply_offset { 0 };
std::size_t texture_warm_offset { 0 };
std::size_t umes_cold_offset { 0 };
};
struct SectionStreamState {
@@ -404,13 +406,27 @@ pending_apply_is_urgent() {
adaptive_slice_instances( std::size_t const section_total, bool const urgent = false ) {
if( urgent ) {
auto limit { kUrgentSliceInstances };
auto const speed { camera_stream_speed_mps() };
if( speed > 1200.0 ) {
limit = std::min( limit, std::size_t { 64 } );
}
else if( speed > 600.0 ) {
limit = std::min( limit, std::size_t { 96 } );
}
if( section_total > 4000 ) {
limit = std::min( limit, std::size_t { 384 } );
limit = std::min( limit, std::size_t { 64 } );
}
else if( section_total > 2000 ) {
limit = std::min( limit, std::size_t { 448 } );
limit = std::min( limit, std::size_t { 96 } );
}
return limit;
auto const last_ms { pack_bench_stream().last_chunk_ms };
if( last_ms >= 24.0 ) {
limit = std::min( limit, std::size_t { 32 } );
}
else if( last_ms >= 12.0 ) {
limit = std::min( limit, std::size_t { 48 } );
}
return std::max( limit, std::size_t { 16 } );
}
auto limit { gameplay_slice_instances() };
@@ -562,6 +578,69 @@ preload_slice_cold_meshes(
return cold_loaded;
}
[[nodiscard]] std::size_t
preload_umes_cold_meshes(
std::vector<std::string> const &unique_meshes,
std::size_t const offset,
std::size_t const max_cold_meshes,
double const cold_budget_ms,
std::size_t &offset_out ) {
offset_out = offset;
if( unique_meshes.empty() || offset >= unique_meshes.size() ) {
return 0;
}
std::size_t cold_loaded { 0 };
auto const cold_started { std::chrono::steady_clock::now() };
for( std::size_t i { offset }; i < unique_meshes.size(); ++i ) {
auto model_file { unique_meshes[ i ] };
if( model_file.empty() || model_file == "notload" ) {
offset_out = i + 1;
continue;
}
replace_slashes( model_file );
if( g_stream.mesh_cache.contains( model_file ) ) {
pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit );
offset_out = i + 1;
continue;
}
if( cold_loaded >= max_cold_meshes ) {
pack_bench_inc( &Eu7PackBench::stream_cold_slice_truncated );
break;
}
if( cold_budget_ms > 0.0 && cold_loaded > 0 ) {
auto const elapsed_ms {
std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - cold_started ).count() };
if( elapsed_ms >= cold_budget_ms ) {
pack_bench_inc( &Eu7PackBench::stream_cold_slice_truncated );
break;
}
}
auto const was_global_cached { TModelsManager::IsModelCached( model_file ) };
TModel3d *mesh { nullptr };
{
PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms };
mesh = TModelsManager::GetModel( model_file, false, false );
pack_bench_inc( &Eu7PackBench::main_cold_getmodel_calls );
}
if( was_global_cached ) {
pack_bench_inc( &Eu7PackBench::stream_mesh_global_hit );
}
else {
pack_bench_inc( &Eu7PackBench::stream_mesh_disk_load );
}
if( mesh != nullptr ) {
TAnimModel::warm_instanceable_cache( mesh );
}
g_stream.mesh_cache.emplace( model_file, mesh );
++cold_loaded;
offset_out = i + 1;
}
return cold_loaded;
}
constexpr int kApplyStuckSkipFrames { 20 };
std::size_t g_apply_stuck_offset { std::numeric_limits<std::size_t>::max() };
@@ -817,17 +896,27 @@ apply_pending_chunk(
{
auto const phase_started { std::chrono::steady_clock::now() };
cold_loads = preload_slice_cold_meshes(
models_ptr + offset,
chunk_count,
cold_mesh_limit,
effective_cold_budget_ms,
chunk_count );
if( false == batch.unique_meshes.empty() ) {
cold_loads = preload_umes_cold_meshes(
batch.unique_meshes,
batch.umes_cold_offset,
cold_mesh_limit,
effective_cold_budget_ms,
batch.umes_cold_offset );
}
else {
cold_loads = preload_slice_cold_meshes(
models_ptr + offset,
chunk_count,
cold_mesh_limit,
effective_cold_budget_ms,
chunk_count );
}
cold_ms = std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - phase_started ).count();
}
pack_bench_inc( &Eu7PackBench::stream_inst_after_cold, chunk_count );
if( chunk_count < planned_inst ) {
if( batch.unique_meshes.empty() && chunk_count < planned_inst ) {
pack_bench_inc( &Eu7PackBench::stream_cold_slice_truncated );
}
if( chunk_count == 0 ) {
@@ -1025,11 +1114,13 @@ worker_loop( std::stop_token const stop_token ) {
}
std::unique_ptr<std::vector<Eu7Model>> models;
std::vector<std::string> unique_meshes;
{
PackBenchTimer const read_timer { &Eu7PackBench::worker_read_pack_ms };
auto section { read_pack_section( module, job.row, job.column ) };
if( false == section.empty() ) {
models = std::make_unique<std::vector<Eu7Model>>( std::move( section ) );
auto section { read_pack_section_load( module, job.row, job.column ) };
unique_meshes = std::move( section.unique_meshes );
if( false == section.models.empty() ) {
models = std::make_unique<std::vector<Eu7Model>>( std::move( section.models ) );
}
}
@@ -1038,6 +1129,7 @@ worker_loop( std::stop_token const stop_token ) {
result.column = job.column;
result.section_idx = job.section_idx;
result.models = std::move( models );
result.unique_meshes = std::move( unique_meshes );
result.failed = result.models == nullptr;
if( result.failed ) {
@@ -1052,7 +1144,7 @@ worker_loop( std::stop_token const stop_token ) {
pack_bench_inc(
&Eu7PackBench::worker_models_decoded, result.models->size() );
preload_pack_models( *result.models );
preload_pack_models( *result.models, result.unique_meshes );
{
std::lock_guard<std::mutex> lock { g_stream.ready.mutex };

6
scene/eu7/eu7_types.h
View File

@@ -368,6 +368,12 @@ struct Eu7PackSectionCursor {
std::uint32_t model_total { 0 };
std::uint8_t section_format { 0 };
bool header_parsed { false };
std::vector<std::string> unique_meshes;
};
struct Eu7PackSectionLoad {
std::vector<Eu7Model> models;
std::vector<std::string> unique_meshes;
};
struct Eu7PackCatalog {