eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-21 17:09:19 +02:00
Code Issues 123 Wiki Activity

Add EU7 PACK v12 PROT/INST and tune streaming apply path

Browse Source 
Introduce EU7B v8 bake with global PROT chunk and v12 section payloads
(solo MODL + compact inst records, per-chunk solo/inst tables). Reader
parses v12 with chunked decode and v11→flat fallback; fix chunk blob
layout to write solo blocks before inst records.

Streaming: add unload keep-radius hysteresis on deceleration, ring-deficit
urgent drain, higher cold-mesh/apply budgets, and smarter warm-prefix
preload. Move UMES mesh prefetch to the main thread and route PACK apply
through ensure_pack_mesh_in_session_cache to avoid GetModel races on
Global.asCurrentTexturePath; harden MdlMngr texture-path trimming.

Rebake root scenario .eu7 files to get v12; existing v7–v11 packs remain
readable.
This commit is contained in:
maj00r
2026-06-14 22:18:52 +02:00
parent 2ee5f93353
commit 2f90246907
13 changed files with 791 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
docs/eu7-format.md
View File

@@ -877,6 +877,43 @@ Powtórzone chunk_count razy:
Runtime używa UTEX do prefetch/warm tekstur bez skanowania wszystkich instancji.
### Format sekcji — v12 PROT+INST (`kPackSectionFormatV12 = 5`)
Rozszerzenie v11: sekcja dzieli modele na **solo** (pełny MODL) i **inst** (kompaktowy rekord odwołujący się do globalnego chunku PROT). Chunki zawierają mix solo+inst; w każdym chunku najpierw solo, potem inst.
```
uint8 section_format — musi być = 5
uint32 solo_count
uint32 inst_count
uint32 unique_mesh_count
uint32[unique_mesh_count] string_id
uint32 unique_texture_count
uint32[unique_texture_count] string_id
uint32 chunk_count
Powtórzone chunk_count razy:
uint32 chunk_solo_count
uint32 chunk_inst_count
uint32 chunk_byte_offset
[payload chunków — solo_count pełnych MODL + inst_count rekordów inst rozłożonych po chunkach]
```
Rekord instancji (jak v8):
```
uint32 proto_id
Vec3 location
Vec3 angles
Vec3 scale
string_id name
```
Bake v12 emituje chunk **PROT** przed **PACK** (wersja pliku 8). Runtime rozwija inst przez `expand_prototype_instance()` i `module.model_prototypes`.
Sekcje v11–v7 pozostają czytelne (fallback v12→v11→v10→flat).
---
## 30. PROT — prototypy modeli (v8+)

12
eu07-parser/include/eu07/scene/binary/common.hpp
View File

@@ -17,17 +17,22 @@ inline constexpr std::array<char, 4> kMagic{'E', 'U', '7', 'B'};
inline constexpr std::uint32_t kVersionLegacy = 1; // codec.hpp — nieobslugiwane w CLI
inline constexpr std::uint32_t kVersionRuntime = 6;
inline constexpr std::uint32_t kVersionRuntimeV7 = 7;
inline constexpr std::uint32_t kVersionRuntimeV8 = 8;
inline constexpr std::uint32_t kVersionRuntimeV5 = 5;
inline constexpr std::uint32_t kVersionRuntimeV4 = 4;
inline constexpr std::uint32_t kVersionRuntimeF32 = 3;
inline constexpr std::uint32_t kVersionRuntimeF64 = 2;
[[nodiscard]] inline bool isSupportedEu7bVersion(const std::uint32_t version) noexcept {
return version == kVersionRuntimeV7 || version == kVersionRuntime ||
version == kVersionRuntimeV5 || version == kVersionRuntimeV4;
return version == kVersionRuntimeV8 || version == kVersionRuntimeV7 ||
version == kVersionRuntime || version == kVersionRuntimeV5 ||
version == kVersionRuntimeV4;
}
[[nodiscard]] inline std::string formatVersionName(const std::uint32_t version) {
if (version == kVersionRuntimeV8) {
return "RuntimeV8";
}
if (version == kVersionRuntimeV7) {
return "RuntimeV7";
}
@@ -53,6 +58,9 @@ inline constexpr std::uint32_t kVersionRuntimeF64 = 2;
}
[[nodiscard]] inline std::string formatVersionDescription(const std::uint32_t version) {
if (version == kVersionRuntimeV8) {
return "bake v7 + PROT + PACK sekcji v12 (solo + compact inst)";
}
if (version == kVersionRuntimeV7) {
return "bake v6 + MODL w chunkach PIDX/PACK per sekcja 1km (v7)";
}

326
eu07-parser/include/eu07/scene/binary/runtime_module.hpp
View File

@@ -49,11 +49,13 @@ inline constexpr std::array<char, 4> kChunkFint{'F', 'I', 'N', 'T'};
inline constexpr std::array<char, 4> kChunkPlac{'P', 'L', 'A', 'C'};
inline constexpr std::array<char, 4> kChunkPidx{'P', 'I', 'D', 'X'};
inline constexpr std::array<char, 4> kChunkPack{'P', 'A', 'C', 'K'};
inline constexpr std::array<char, 4> kChunkProt{'P', 'R', 'O', 'T'};
// PACK section payload: v9 = UMES, v10 = UMES + CHNK, v11 = UMES + UTEX + CHNK.
// PACK section payload: v9 = UMES, v10 = UMES + CHNK, v11 = UMES + UTEX + CHNK, v12 = v11 + PROT inst.
inline constexpr std::uint8_t kPackSectionFormatV9 = 2;
inline constexpr std::uint8_t kPackSectionFormatV10 = 3;
inline constexpr std::uint8_t kPackSectionFormatV11 = 4;
inline constexpr std::uint8_t kPackSectionFormatV12 = 5;
inline constexpr std::size_t kPackSectionChunkModels = 512;
inline constexpr std::size_t kPackSectionChunkModelsHeavy = 256;
inline constexpr std::size_t kPackSectionHeavyModelThreshold = 1024;
@@ -405,6 +407,7 @@ struct PackSectionIndexEntry {
struct PackPayloadBuild {
std::vector<char> packPayload;
std::vector<char> protPayload;
std::vector<PackSectionIndexEntry> entries;
};
@@ -712,7 +715,301 @@ inline void sortPackSectionModelsForStreaming(
return {blob.begin(), blob.end()};
}
[[nodiscard]] inline PackPayloadBuild buildPackPayloadV7(
struct PackPrototypeKey {
std::string modelFile;
std::string textureFile;
double rangeSquaredMin = 0.0;
double rangeSquaredMax = 0.0;
bool visible = true;
bool isTerrain = false;
bool transition = true;
[[nodiscard]] bool operator==(const PackPrototypeKey& other) const noexcept {
return modelFile == other.modelFile && textureFile == other.textureFile &&
rangeSquaredMin == other.rangeSquaredMin &&
rangeSquaredMax == other.rangeSquaredMax && visible == other.visible &&
isTerrain == other.isTerrain && transition == other.transition;
}
};
struct PackPrototypeKeyHash {
[[nodiscard]] std::size_t operator()(const PackPrototypeKey& key) const noexcept {
std::size_t hash = std::hash<std::string>{}(key.modelFile);
hash ^= std::hash<std::string>{}(key.textureFile) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
hash ^= std::hash<double>{}(key.rangeSquaredMin) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
hash ^= std::hash<double>{}(key.rangeSquaredMax) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
hash ^= static_cast<std::size_t>(key.visible) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
hash ^= static_cast<std::size_t>(key.isTerrain) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
hash ^= static_cast<std::size_t>(key.transition) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
return hash;
}
};
[[nodiscard]] inline PackPrototypeKey makePackPrototypeKey(
const runtime::RuntimeModelInstance& model) noexcept {
PackPrototypeKey key;
key.modelFile = model.modelFile;
key.textureFile = model.textureFile;
key.rangeSquaredMin = model.node.rangeSquaredMin;
key.rangeSquaredMax = model.node.rangeSquaredMax;
key.visible = model.node.visible;
key.isTerrain = model.isTerrain;
key.transition = model.transition;
return key;
}
[[nodiscard]] inline bool modelMustBePackSolo(const runtime::RuntimeModelInstance& model) noexcept {
if (model.isTerrain) {
return true;
}
if (!model.lightStates.empty() || !model.lightColors.empty()) {
return true;
}
if (!isLoadableMeshPath(model.modelFile)) {
return true;
}
return false;
}
[[nodiscard]] inline bool modelCanBePackInstanced(const runtime::RuntimeModelInstance& model) noexcept {
return !modelMustBePackSolo(model);
}
inline void writeRuntimePrototype(
std::ostream& out,
StringTable& table,
const runtime::RuntimeModelInstance& model) {
codec::writeSlimNode(out, table, model.node, "model");
io::writeU8(out, model.isTerrain ? 1 : 0);
io::writeU8(out, model.transition ? 1 : 0);
codec::writeStringId(out, table, model.modelFile);
codec::writeStringId(out, table, model.textureFile);
io::writeU32(out, static_cast<std::uint32_t>(model.lightStates.size()));
for (float light : model.lightStates) {
io::writeF32(out, light);
}
io::writeU32(out, static_cast<std::uint32_t>(model.lightColors.size()));
for (std::uint32_t color : model.lightColors) {
io::writeU32(out, color);
}
}
[[nodiscard]] inline std::vector<char> buildProtPayload(
StringTable& table,
const std::vector<runtime::RuntimeModelInstance>& prototypes) {
std::ostringstream out(std::ios::binary);
io::writeU32(out, static_cast<std::uint32_t>(prototypes.size()));
for (const runtime::RuntimeModelInstance& proto : prototypes) {
writeRuntimePrototype(out, table, proto);
}
const std::string blob = out.str();
return {blob.begin(), blob.end()};
}
struct PackPrototypeTable {
std::vector<runtime::RuntimeModelInstance> prototypes;
std::unordered_map<PackPrototypeKey, std::uint32_t, PackPrototypeKeyHash> key_to_id;
};
[[nodiscard]] inline PackPrototypeTable collectPackPrototypes(
StringTable& table,
const std::vector<codec::ModelSectionBatch>& batches) {
std::unordered_map<PackPrototypeKey, std::uint32_t, PackPrototypeKeyHash> frequency;
frequency.reserve(1024);
for (const codec::ModelSectionBatch& batch : batches) {
for (const runtime::RuntimeModelInstance& model : batch.models) {
if (!modelCanBePackInstanced(model)) {
continue;
}
++frequency[makePackPrototypeKey(model)];
}
}
PackPrototypeTable result;
for (const codec::ModelSectionBatch& batch : batches) {
for (const runtime::RuntimeModelInstance& model : batch.models) {
if (!modelCanBePackInstanced(model)) {
continue;
}
const PackPrototypeKey key = makePackPrototypeKey(model);
const auto freq_it = frequency.find(key);
if (freq_it == frequency.end() || freq_it->second < 2) {
continue;
}
if (result.key_to_id.contains(key)) {
continue;
}
const std::uint32_t proto_id =
static_cast<std::uint32_t>(result.prototypes.size());
result.key_to_id.emplace(key, proto_id);
runtime::RuntimeModelInstance proto = model;
proto.location = {};
proto.angles = {};
proto.scale = {1.0, 1.0, 1.0};
proto.node.transform = {};
collectModelStrings(table, proto);
result.prototypes.push_back(std::move(proto));
}
}
return result;
}
enum class PackSectionEntryKind : std::uint8_t { Solo, Inst };
struct PackSectionEntry {
PackSectionEntryKind kind = PackSectionEntryKind::Solo;
std::size_t model_index = 0;
std::uint32_t proto_id = 0;
};
inline void writePackInstRecord(
std::ostream& out,
StringTable& table,
const std::uint32_t proto_id,
const runtime::RuntimeModelInstance& model) {
io::writeU32(out, proto_id);
io::writeVec3(out, model.location);
io::writeVec3(out, model.angles);
io::writeVec3(out, model.scale);
codec::writeStringId(out, table, model.node.name);
}
[[nodiscard]] inline std::vector<char> buildPackSectionChunkBlobV12(
StringTable& table,
const std::vector<runtime::RuntimeModelInstance>& models,
const std::vector<PackSectionEntry>& entries,
const std::size_t begin,
const std::size_t end) {
// Reader decodes each chunk as a solo block followed by inst records (v8 layout).
std::ostringstream out(std::ios::binary);
for (std::size_t index = begin; index < end; ++index) {
const PackSectionEntry& entry = entries[index];
if (entry.kind != PackSectionEntryKind::Solo) {
continue;
}
runtime::RuntimeModelInstance worldModel = models[entry.model_index];
worldModel.node.transform = {};
writeRuntimeModel(out, table, worldModel);
}
for (std::size_t index = begin; index < end; ++index) {
const PackSectionEntry& entry = entries[index];
if (entry.kind != PackSectionEntryKind::Inst) {
continue;
}
writePackInstRecord(out, table, entry.proto_id, models[entry.model_index]);
}
const std::string blob = out.str();
return {blob.begin(), blob.end()};
}
[[nodiscard]] inline std::vector<char> buildPackSectionPayloadV12(
StringTable& table,
const std::vector<runtime::RuntimeModelInstance>& models,
const PackPrototypeTable& prototypes) {
std::vector<PackSectionEntry> entries;
entries.reserve(models.size());
std::uint32_t solo_count = 0;
std::uint32_t inst_count = 0;
for (std::size_t model_index = 0; model_index < models.size(); ++model_index) {
const runtime::RuntimeModelInstance& model = models[model_index];
if (modelMustBePackSolo(model)) {
entries.push_back({PackSectionEntryKind::Solo, model_index, 0});
++solo_count;
continue;
}
const PackPrototypeKey key = makePackPrototypeKey(model);
const auto proto_it = prototypes.key_to_id.find(key);
if (proto_it == prototypes.key_to_id.end()) {
entries.push_back({PackSectionEntryKind::Solo, model_index, 0});
++solo_count;
continue;
}
entries.push_back({PackSectionEntryKind::Inst, model_index, proto_it->second});
++inst_count;
}
std::vector<std::string_view> prototype_mesh_paths;
prototype_mesh_paths.reserve(prototypes.prototypes.size());
for (const runtime::RuntimeModelInstance& proto : prototypes.prototypes) {
prototype_mesh_paths.emplace_back(proto.modelFile);
}
const auto mesh_ids =
collectUniqueMeshIds(table, models, &prototype_mesh_paths);
const auto texture_ids = collectUniqueTextureIds(table, models);
const std::size_t chunk_models =
models.size() > kPackSectionHeavyModelThreshold ? kPackSectionChunkModelsHeavy
: kPackSectionChunkModels;
std::vector<std::vector<char>> chunk_payloads;
std::vector<std::uint32_t> chunk_solo_counts;
std::vector<std::uint32_t> chunk_inst_counts;
chunk_payloads.reserve((entries.size() + chunk_models - 1) / chunk_models);
chunk_solo_counts.reserve(chunk_payloads.capacity());
chunk_inst_counts.reserve(chunk_payloads.capacity());
for (std::size_t offset = 0; offset < entries.size(); offset += chunk_models) {
const std::size_t end = std::min(offset + chunk_models, entries.size());
std::uint32_t chunk_solo = 0;
std::uint32_t chunk_inst = 0;
for (std::size_t index = offset; index < end; ++index) {
if (entries[index].kind == PackSectionEntryKind::Solo) {
++chunk_solo;
} else {
++chunk_inst;
}
}
chunk_solo_counts.push_back(chunk_solo);
chunk_inst_counts.push_back(chunk_inst);
chunk_payloads.push_back(
buildPackSectionChunkBlobV12(table, models, entries, offset, end));
}
if (chunk_payloads.empty()) {
chunk_payloads.emplace_back();
chunk_solo_counts.push_back(0);
chunk_inst_counts.push_back(0);
}
const std::uint32_t chunk_count = static_cast<std::uint32_t>(chunk_payloads.size());
const std::uint32_t header_size =
1u + 4u + 4u + 4u + static_cast<std::uint32_t>(mesh_ids.size()) * 4u + 4u +
static_cast<std::uint32_t>(texture_ids.size()) * 4u + 4u + chunk_count * 12u;
std::vector<std::uint32_t> chunk_offsets(chunk_count);
std::uint32_t payload_offset = header_size;
for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) {
chunk_offsets[chunk_index] = payload_offset;
payload_offset += static_cast<std::uint32_t>(chunk_payloads[chunk_index].size());
}
std::ostringstream packOut(std::ios::binary);
io::writeU8(packOut, kPackSectionFormatV12);
io::writeU32(packOut, solo_count);
io::writeU32(packOut, inst_count);
io::writeU32(packOut, static_cast<std::uint32_t>(mesh_ids.size()));
for (const std::uint32_t id : mesh_ids) {
io::writeU32(packOut, id);
}
io::writeU32(packOut, static_cast<std::uint32_t>(texture_ids.size()));
for (const std::uint32_t id : texture_ids) {
io::writeU32(packOut, id);
}
io::writeU32(packOut, chunk_count);
for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) {
io::writeU32(packOut, chunk_solo_counts[chunk_index]);
io::writeU32(packOut, chunk_inst_counts[chunk_index]);
io::writeU32(packOut, chunk_offsets[chunk_index]);
}
for (const std::vector<char>& payload : chunk_payloads) {
packOut.write(payload.data(), static_cast<std::streamsize>(payload.size()));
}
const std::string blob = packOut.str();
return {blob.begin(), blob.end()};
}
[[nodiscard]] inline PackPayloadBuild buildPackPayloadV12(
StringTable& table,
const std::vector<codec::ModelSectionBatch>& batches) {
PackPayloadBuild result;
@@ -726,6 +1023,9 @@ inline void sortPackSectionModelsForStreaming(
}
}
const PackPrototypeTable prototypes = collectPackPrototypes(table, batches);
result.protPayload = buildProtPayload(table, prototypes.prototypes);
result.entries.resize(batches.size());
std::vector<std::vector<char>> section_payloads(batches.size());
@@ -742,7 +1042,8 @@ inline void sortPackSectionModelsForStreaming(
entry.modelCount = static_cast<std::uint32_t>(batch.models.size());
std::vector<runtime::RuntimeModelInstance> section_models = batch.models;
sortPackSectionModelsForStreaming(section_models, batch.section);
section_payloads[batch_index] = buildPackSectionPayloadV11(table, section_models);
section_payloads[batch_index] =
buildPackSectionPayloadV12(table, section_models, prototypes);
}
std::size_t total_size = 0;
for (const std::vector<char>& payload : section_payloads) {
@@ -778,7 +1079,8 @@ inline void sortPackSectionModelsForStreaming(
std::vector<runtime::RuntimeModelInstance> section_models = batch.models;
sortPackSectionModelsForStreaming(section_models, batch.section);
section_payloads[batch_index] = buildPackSectionPayloadV11(table, section_models);
section_payloads[batch_index] =
buildPackSectionPayloadV12(table, section_models, prototypes);
}
});
}
@@ -804,6 +1106,12 @@ inline void sortPackSectionModelsForStreaming(
return result;
}
[[nodiscard]] inline PackPayloadBuild buildPackPayloadV7(
StringTable& table,
const std::vector<codec::ModelSectionBatch>& batches) {
return buildPackPayloadV12(table, batches);
}
[[nodiscard]] inline PackPayloadBuild buildPackPayloadV7(
StringTable& table,
const std::vector<runtime::RuntimeModelInstance>& models) {
@@ -1083,7 +1391,8 @@ inline void readPidxChunkV7(std::istream& in, std::vector<PackSectionIndexEntry>
[[nodiscard]] inline bool isKnownRuntimeChunk(const std::array<char, 4>& id) {
return id == kChunkStrs || id == kChunkIncl || id == kChunkPlac || id == kChunkTrak ||
id == kChunkTrac || id == kChunkPwrs || id == kChunkTerr || id == kChunkMesh ||
id == kChunkLine || id == kChunkModl || id == kChunkPidx || id == kChunkPack || id == kChunkMemc ||
id == kChunkLine || id == kChunkModl || id == kChunkPidx || id == kChunkPack ||
id == kChunkProt || id == kChunkMemc ||
id == kChunkLaun || id == kChunkDynm || id == kChunkSond ||
id == kChunkTrset || id == kChunkEvnt || id == kChunkFint;
}
@@ -1160,7 +1469,7 @@ inline void writeRuntimeModule(
}
out.write(kMagic.data(), 4);
io::writeU32(out, usePackModels ? kVersionRuntimeV7 : kVersionRuntime);
io::writeU32(out, usePackModels ? kVersionRuntimeV8 : kVersionRuntime);
io::writeChunkHeader(out, kChunkStrs, 8 + static_cast<std::uint32_t>(strsPayload.size()));
out.write(strsPayload.data(), static_cast<std::streamsize>(strsPayload.size()));
@@ -1193,6 +1502,11 @@ inline void writeRuntimeModule(
if (usePackModels) {
io::writeChunkHeader(out, kChunkPidx, 8 + static_cast<std::uint32_t>(pidxPayload.size()));
out.write(pidxPayload.data(), static_cast<std::streamsize>(pidxPayload.size()));
io::writeChunkHeader(
out, kChunkProt, 8 + static_cast<std::uint32_t>(packBuild.protPayload.size()));
out.write(
packBuild.protPayload.data(),
static_cast<std::streamsize>(packBuild.protPayload.size()));
io::writeChunkHeader(
out, kChunkPack, 8 + static_cast<std::uint32_t>(packBuild.packPayload.size()));
out.write(

5
model/MdlMngr.cpp
View File

@@ -86,7 +86,10 @@ TModelsManager::GetModel(std::string const &Name, bool const Dynamic, bool const
// pobieranie tekstur z katalogu, w którym jest model
// when loading vehicles the path is set by the calling routine, so we can skip it here
Global.asCurrentTexturePath += Name;
Global.asCurrentTexturePath.erase( Global.asCurrentTexturePath.rfind( '/' ) + 1 );
auto const slash_pos { Global.asCurrentTexturePath.rfind( '/' ) };
if( slash_pos != std::string::npos ) {
Global.asCurrentTexturePath.erase( slash_pos + 1 );
}
}
erase_extension( filename );
filename = ToLower( filename );

1
scene/eu7/eu7_chunks.h
View File

@@ -61,6 +61,7 @@ constexpr std::uint8_t kPackSectionFormatV8 { 1 };
constexpr std::uint8_t kPackSectionFormatV9 { 2 };
constexpr std::uint8_t kPackSectionFormatV10 { 3 };
constexpr std::uint8_t kPackSectionFormatV11 { 4 };
constexpr std::uint8_t kPackSectionFormatV12 { 5 };
constexpr std::size_t kPackSectionChunkModels { 512 };
} // namespace scene::eu7

57
scene/eu7/eu7_model_prefetch.cpp
View File

@@ -106,14 +106,9 @@ pack_texture_usable( std::string texture_file ) {
}
void
preload_pack_model_file( std::string const &model_file ) {
if( false == TModelsManager::IsModelCached( model_file ) ) {
return;
}
if( auto *const mesh { TModelsManager::GetModel( model_file, false, false ) } ) {
TAnimModel::warm_instanceable_cache( mesh );
}
preload_pack_model_file( std::string const & ) {
// Mesh warm runs on the main thread via ensure_pack_mesh_in_session_cache only.
// GetModel mutates Global.asCurrentTexturePath and must not run on PACK workers.
}
[[nodiscard]] bool
@@ -208,55 +203,17 @@ reset_pack_texture_warm_cache() {
}
void
preload_pack_model_paths( std::vector<std::string> const &model_files ) {
std::unordered_set<std::string> seen;
seen.reserve( model_files.size() );
for( auto model_file : model_files ) {
if( model_file.empty() || model_file == "notload" ) {
continue;
}
replace_slashes( model_file );
if( false == seen.insert( model_file ).second ) {
continue;
}
std::lock_guard<std::mutex> lock { g_pack_mesh_load_mutex };
preload_pack_model_file( model_file );
}
preload_pack_model_paths( std::vector<std::string> const & ) {
}
void
preload_pack_models( std::vector<Eu7Model> const &models ) {
std::unordered_set<std::string> seen;
seen.reserve( models.size() );
for( auto const &model : models ) {
auto model_file { model.model_file };
if( model_file.empty() || model_file == "notload" ) {
continue;
}
replace_slashes( model_file );
if( false == seen.insert( model_file ).second ) {
continue;
}
std::lock_guard<std::mutex> lock { g_pack_mesh_load_mutex };
preload_pack_model_file( model_file );
}
preload_pack_models( std::vector<Eu7Model> const & ) {
}
void
preload_pack_models(
std::vector<Eu7Model> const &models,
std::vector<std::string> const &unique_meshes ) {
if( false == unique_meshes.empty() ) {
preload_pack_model_paths( unique_meshes );
return;
}
preload_pack_models( models );
std::vector<Eu7Model> const &,
std::vector<std::string> const & ) {
}
std::size_t

3
scene/eu7/eu7_model_prefetch.h
View File

@@ -21,11 +21,10 @@ class TModel3d;
namespace scene::eu7 {
// Tylko warm_instanceable_cache dla juz zcache'owanych meshy (bez cold GetModel na workerze).
// Deprecated: mesh warm is main-thread only (ensure_pack_mesh_in_session_cache).
void
preload_pack_models( std::vector<Eu7Model> const &Models );
// Jak wyzej, ale iteruje precomputed UMES zamiast skanowac instancje.
void
preload_pack_models(
std::vector<Eu7Model> const &Models,

3
scene/eu7/eu7_pack_bench.cpp
View File

@@ -186,6 +186,9 @@ log_pack_bench_impl( Eu7PackBench const &bench, char const *const title ) {
" cam=" + std::to_string( bench.stream_dequeue_camera ) +
" wait_near=" + std::to_string( bench.stream_dequeue_wait_near ) +
" block_far=" + std::to_string( bench.stream_jobs_blocked_far ) +
" block_ring_in=" + std::to_string( bench.stream_jobs_blocked_ring_inner ) +
" block_ring_out=" + std::to_string( bench.stream_jobs_blocked_ring_outer ) +
" defer_dist=" + std::to_string( bench.stream_worker_deferred_distant ) +
" stuck_skip=" + std::to_string( bench.stream_apply_stuck_skip ) );
WriteLog(
" diag ready_tex_ms=" + std::to_string( static_cast<int>( bench.stream_prefetch_ready_tex_ms ) ) +

3
scene/eu7/eu7_pack_bench.h
View File

@@ -71,6 +71,9 @@ struct Eu7PackBench {
std::uint64_t stream_dequeue_camera { 0 };
std::uint64_t stream_dequeue_wait_near { 0 };
std::uint64_t stream_jobs_blocked_far { 0 };
std::uint64_t stream_jobs_blocked_ring_inner { 0 };
std::uint64_t stream_jobs_blocked_ring_outer { 0 };
std::uint64_t stream_worker_deferred_distant { 0 };
std::uint64_t stream_apply_stuck_skip { 0 };
std::uint64_t stream_sections_unloaded { 0 };
std::uint64_t stream_unload_instances { 0 };

64
scene/eu7/eu7_reader.cpp
View File

@@ -505,6 +505,52 @@ parse_pack_section_header(
const std::uint8_t first_byte {
peek == EOF ? std::uint8_t { 0 } : static_cast<std::uint8_t>( peek ) };
bool use_v12_header { false };
if( first_byte == kPackSectionFormatV12 ) {
cursor.section_format = sn_utils::d_uint8( input );
const std::uint32_t solo_total { sn_utils::ld_uint32( input ) };
const std::uint32_t inst_total { sn_utils::ld_uint32( input ) };
const std::uint32_t unique_mesh_count { sn_utils::ld_uint32( input ) };
if(
solo_total + inst_total == entry.model_count &&
( inst_total == 0 || false == module.model_prototypes.empty() ) ) {
use_v12_header = true;
cursor.model_total = solo_total + inst_total;
cursor.solo_remaining = solo_total;
cursor.inst_remaining = inst_total;
StringTable const strings { module.strings };
cursor.unique_meshes.reserve( unique_mesh_count );
for( std::uint32_t mesh_idx { 0 }; mesh_idx < unique_mesh_count; ++mesh_idx ) {
cursor.unique_meshes.push_back(
strings.resolve( sn_utils::ld_uint32( input ) ) );
}
const std::uint32_t unique_texture_count { sn_utils::ld_uint32( input ) };
cursor.unique_textures.reserve( unique_texture_count );
for( std::uint32_t tex_idx { 0 }; tex_idx < unique_texture_count; ++tex_idx ) {
cursor.unique_textures.push_back(
strings.resolve( sn_utils::ld_uint32( input ) ) );
}
cursor.chunk_count = sn_utils::ld_uint32( input );
cursor.chunk_model_counts.resize( cursor.chunk_count );
cursor.chunk_inst_counts.resize( cursor.chunk_count );
cursor.chunk_byte_offsets.resize( cursor.chunk_count );
for( std::uint32_t chunk_idx { 0 }; chunk_idx < cursor.chunk_count; ++chunk_idx ) {
cursor.chunk_model_counts[ chunk_idx ] = sn_utils::ld_uint32( input );
cursor.chunk_inst_counts[ chunk_idx ] = sn_utils::ld_uint32( input );
cursor.chunk_byte_offsets[ chunk_idx ] = sn_utils::ld_uint32( input );
}
}
}
if( use_v12_header ) {
cursor.models_read = 0;
cursor.header_parsed = true;
return;
}
bool use_v11_header { false };
if( first_byte == kPackSectionFormatV11 ) {
cursor.section_format = sn_utils::d_uint8( input );
@@ -672,7 +718,7 @@ read_pack_models_chunk_impl(
const std::uint32_t proto_id { sn_utils::ld_uint32( input ) };
if( proto_id >= module.model_prototypes.size() ) {
throw std::runtime_error(
"EU7 PACK v8: proto_id " + std::to_string( proto_id ) + " poza zakresem PROT (" +
"EU7 PACK: proto_id " + std::to_string( proto_id ) + " poza zakresem PROT (" +
std::to_string( module.model_prototypes.size() ) + ")" );
}
auto const location { read_vec3( input ) };
@@ -1216,15 +1262,22 @@ read_pack_section_chunk_load_impl(
Eu7PackSectionCursor chunk_cursor { header };
if(
( header.section_format == kPackSectionFormatV10 ||
header.section_format == kPackSectionFormatV11 ) &&
header.section_format == kPackSectionFormatV11 ||
header.section_format == kPackSectionFormatV12 ) &&
false == header.chunk_byte_offsets.empty() ) {
auto const section_start {
static_cast<std::streamoff>( module.pack_payload_offset + entry->pack_offset ) };
session.input.seekg(
section_start +
static_cast<std::streamoff>( header.chunk_byte_offsets[ chunk_index ] ) );
chunk_cursor.solo_remaining = header.chunk_model_counts[ chunk_index ];
chunk_cursor.inst_remaining = 0;
if( header.section_format == kPackSectionFormatV12 ) {
chunk_cursor.solo_remaining = header.chunk_model_counts[ chunk_index ];
chunk_cursor.inst_remaining = header.chunk_inst_counts[ chunk_index ];
}
else {
chunk_cursor.solo_remaining = header.chunk_model_counts[ chunk_index ];
chunk_cursor.inst_remaining = 0;
}
}
else if( chunk_index > 0 ) {
return result;
@@ -1266,7 +1319,8 @@ read_pack_section_load_impl( Eu7Module const &module, int const row, int const c
if(
( header.section_format == kPackSectionFormatV10 ||
header.section_format == kPackSectionFormatV11 ) &&
header.section_format == kPackSectionFormatV11 ||
header.section_format == kPackSectionFormatV12 ) &&
header.chunk_count > 0 ) {
result.models.reserve( entry->model_count );
for( std::uint32_t chunk_idx { 0 }; chunk_idx < header.chunk_count; ++chunk_idx ) {

426
scene/eu7/eu7_section_stream.cpp
View File

@@ -57,20 +57,24 @@ constexpr double kDrainBudgetMs { 12.0 };
constexpr double kLoaderDrainBudgetMs { 96.0 };
constexpr std::size_t kLoaderSectionsPerDrain { 8 };
constexpr double kGameplayApplyBudgetMs { 4.0 };
constexpr double kCatchupApplyBudgetMs { 6.0 };
constexpr double kGameplayApplyBudgetMs { 6.0 };
constexpr double kCatchupApplyBudgetMs { 14.0 };
constexpr std::size_t kGameplaySliceInstances { 96 };
constexpr std::size_t kCatchupSliceInstances { 64 };
constexpr std::size_t kCatchupSliceInstances { 96 };
constexpr std::size_t kLoaderSliceInstances { 768 };
constexpr std::size_t kGameplaySliceColdMeshes { 1 };
constexpr std::size_t kCatchupSliceColdMeshes { 2 };
constexpr std::size_t kGameplaySliceColdMeshes { 4 };
constexpr std::size_t kCatchupSliceColdMeshes { 8 };
constexpr std::size_t kLoaderSliceColdMeshes { 32 };
constexpr double kGameplayColdBudgetMs { 3.0 };
constexpr double kCatchupColdBudgetMs { 5.0 };
constexpr double kGameplayColdBudgetMs { 12.0 };
constexpr double kCatchupColdBudgetMs { 18.0 };
constexpr int kInitialBootstrapRadius { 5 };
constexpr int kStreamRadius { 14 };
constexpr int kSectionUnloadMarginKm { 2 };
constexpr int kSectionUnloadMarginKm { 3 };
constexpr int kSectionUnloadCameraGuardKm { 5 };
constexpr int kSectionUnloadMaxSpeedExtraKm { 10 };
constexpr double kSectionUnloadKeepRadiusDecayKmPerSec { 2.0 };
constexpr int kSectionUnloadTeleportJumpKm { 4 };
constexpr std::size_t kMeshCacheLruCap { 2000 };
constexpr std::size_t kNodedataCacheLruCap { 2000 };
constexpr int kMovementLookahead { 10 };
@@ -85,21 +89,35 @@ constexpr std::size_t kPackTextureWarmSlice { 8 };
constexpr double kPackTextureWarmBudgetMs { 4.0 };
constexpr double kUrgentApplyBudgetMs { 32.0 };
constexpr std::size_t kUrgentSliceInstances { 96 };
constexpr std::size_t kUrgentSliceColdMeshes { 2 };
constexpr double kUrgentColdBudgetMs { 24.0 };
constexpr std::size_t kUrgentSliceColdMeshes { 6 };
constexpr double kUrgentColdBudgetMs { 28.0 };
constexpr std::size_t kUrgentMaxChunksPerDrain { 8 };
constexpr std::size_t kRingDeficitMaxChunksPerDrain { 4 };
constexpr double kReadyTexturePrefetchBudgetMs { 14.0 };
constexpr std::size_t kReadyTexturePrefetchSlice { 256 };
constexpr int kUmesPrefetchMaxDistanceKm { 2 };
constexpr double kReadyUmesPrefetchBudgetMs { 8.0 };
constexpr std::size_t kReadyUmesPrefetchMaxMeshes { 6 };
constexpr std::size_t kUmesPrefetchJobsBacklogThreshold { 4 };
constexpr std::size_t kCatchupMaxInFlightSections { 28 };
constexpr std::size_t kCatchupMaxReadySections { 14 };
constexpr int kUrgentSectionDistanceKm { 1 };
constexpr int kFastFlightApplyMaxDistanceKm { 2 };
constexpr int kFastFlightApplyMaxDistanceKm { 3 };
constexpr int kFarApplySlowDistanceKm { 4 };
constexpr int kPreemptPendingDistanceKm { 2 };
constexpr float kRingGatedOuterThreshold { 0.35f };
constexpr float kRingGatedInnerTarget { 0.70f };
constexpr float kRingBoostOuterThreshold { 0.55f };
constexpr float kRingStrongBoostOuterThreshold { 0.75f };
constexpr std::size_t kPreemptReadyQueueThreshold { 8 };
constexpr int kWorkerContinueMaxDistanceKm { 8 };
constexpr std::size_t kFastFlightMaxFarInFlight { 3 };
constexpr double kFastFlightRingGateSpeedMps { 600.0 };
constexpr double kRingBoostInFlightSpeedMps { 600.0 };
constexpr std::size_t kRingBoostMaxInFlight { 12 };
constexpr std::size_t kRingStrongBoostMaxInFlight { 16 };
constexpr std::size_t kRingBoostMaxReady { 8 };
constexpr std::size_t kRingStrongBoostMaxReady { 10 };
constexpr double kPreemptDisableSpeedMps { 400.0 };
constexpr double kTeleportReenqueueDistanceM { 2000.0 };
constexpr float kStationaryCatchupRingThreshold { 0.70f };
constexpr double kStationaryCatchupSpeedMps { 5.0 };
@@ -157,6 +175,11 @@ struct SectionStreamState {
bool has_last_enqueue_position { false };
glm::dvec3 anchor_position {};
bool has_anchor_position { false };
float last_inner_ring { 1.f };
float last_outer_ring { 1.f };
int unload_keep_radius { 0 };
double unload_keep_radius_floor { 0.0 };
double unload_keep_radius_last_update { 0.0 };
std::unordered_map<std::string, TModel3d *> mesh_cache;
std::unordered_map<std::string, scene::node_data> nodedata_cache;
@@ -185,6 +208,15 @@ section_manhattan_sections(
int const center_row,
int const center_column );
[[nodiscard]] std::size_t
count_far_in_flight_sections( int const max_distance_km );
[[nodiscard]] bool
should_block_far_enqueue(
int const row,
int const column,
bool const lookahead_enqueue );
void
push_prioritized_job( PackSectionJob job );
@@ -195,13 +227,16 @@ reprioritize_job_queue();
enqueue_section_if_needed(
int const row,
int const column,
int const priority );
int const priority,
bool const lookahead_enqueue = false );
void
sync_stream_limits( glm::dvec3 const &world_position );
void
maybe_unload_distant_sections( int const center_row, int const center_column );
maybe_unload_distant_sections(
int const prev_center_row,
int const prev_center_column );
void
evict_unreferenced_stream_caches();
@@ -540,12 +575,67 @@ unload_pack_section( int const row, int const column ) {
}
void
maybe_unload_distant_sections( int const center_row, int const center_column ) {
if( false == g_loading_screen_dismissed || g_stream.bootstrap_active ) {
update_unload_keep_radius_hysteresis() {
if( g_stream.center_row < 0 || g_stream.center_column < 0 ) {
return;
}
auto const speed { camera_stream_speed_mps() };
auto const speed_extra {
std::min(
static_cast<int>( speed / 150.0 ),
kSectionUnloadMaxSpeedExtraKm ) };
auto const desired_keep_radius {
static_cast<double>( g_stream.radius + kSectionUnloadMarginKm + speed_extra ) };
auto const now { Timer::GetTime() };
if( g_stream.unload_keep_radius_floor <= 0.0 ) {
g_stream.unload_keep_radius_floor = desired_keep_radius;
g_stream.unload_keep_radius_last_update = now;
g_stream.unload_keep_radius = static_cast<int>( std::ceil( desired_keep_radius ) );
return;
}
auto const dt { std::max( 0.0, now - g_stream.unload_keep_radius_last_update ) };
g_stream.unload_keep_radius_last_update = now;
if( desired_keep_radius > g_stream.unload_keep_radius_floor ) {
g_stream.unload_keep_radius_floor = desired_keep_radius;
}
else if( desired_keep_radius < g_stream.unload_keep_radius_floor && dt > 0.0 ) {
g_stream.unload_keep_radius_floor = std::max(
desired_keep_radius,
g_stream.unload_keep_radius_floor - dt * kSectionUnloadKeepRadiusDecayKmPerSec );
}
g_stream.unload_keep_radius = static_cast<int>( std::ceil( g_stream.unload_keep_radius_floor ) );
}
void
maybe_unload_distant_sections(
int const prev_center_row,
int const prev_center_column ) {
if( false == g_loading_screen_dismissed || g_stream.bootstrap_active ) {
return;
}
if( g_stream.center_row < 0 || g_stream.center_column < 0 ) {
return;
}
auto const [cam_row, cam_col] { section_row_column( Global.pCamera.Pos ) };
auto const keep_radius { g_stream.unload_keep_radius };
auto const center_jump {
prev_center_row >= 0 && prev_center_column >= 0 ?
section_manhattan_sections(
g_stream.center_row,
g_stream.center_column,
prev_center_row,
prev_center_column ) :
0 };
if( center_jump >= kSectionUnloadTeleportJumpKm ) {
return;
}
auto const keep_radius { g_stream.radius + kSectionUnloadMarginKm };
std::vector<std::size_t> to_unload;
to_unload.reserve( g_stream.loaded_sections.size() );
@@ -554,7 +644,23 @@ maybe_unload_distant_sections( int const center_row, int const center_column ) {
static_cast<int>( section_idx / static_cast<std::size_t>( kRegionSideSectionCount ) ) };
auto const column {
static_cast<int>( section_idx % static_cast<std::size_t>( kRegionSideSectionCount ) ) };
if( ring_section_in_radius( row, column, center_row, center_column, keep_radius ) ) {
if(
ring_section_in_radius(
row,
column,
cam_row,
cam_col,
kSectionUnloadCameraGuardKm ) ) {
continue;
}
if(
ring_section_in_radius(
row,
column,
g_stream.center_row,
g_stream.center_column,
keep_radius ) ) {
continue;
}
to_unload.push_back( section_idx );
@@ -612,6 +718,11 @@ reset_stream_fields() {
g_stream.has_last_enqueue_position = false;
g_stream.anchor_position = {};
g_stream.has_anchor_position = false;
g_stream.last_inner_ring = 1.f;
g_stream.last_outer_ring = 1.f;
g_stream.unload_keep_radius = 0;
g_stream.unload_keep_radius_floor = 0.0;
g_stream.unload_keep_radius_last_update = 0.0;
g_stream.mesh_cache.clear();
g_stream.nodedata_cache.clear();
g_stream.mesh_lru.clear();
@@ -699,9 +810,12 @@ stream_backpressure() {
return true;
}
}
return (
g_stream.in_flight_sections.size() >= g_stream_max_in_flight ||
ready_queue_size() >= g_stream_max_ready );
if(
false == g_stream.bootstrap_active &&
g_stream.in_flight_sections.size() >= g_stream_max_in_flight ) {
return true;
}
return ready_queue_size() >= g_stream_max_ready;
}
[[nodiscard]] double
@@ -712,8 +826,16 @@ camera_stream_speed_mps() {
}
[[nodiscard]] bool pending_apply_is_urgent();
[[nodiscard]] bool stream_ring_deficit();
[[nodiscard]] double gameplay_apply_budget_ms();
[[nodiscard]] bool
stream_ring_deficit() {
return (
g_stream.last_outer_ring < kRingGatedOuterThreshold ||
g_stream.last_inner_ring < kRingGatedInnerTarget );
}
[[nodiscard]] double
ready_texture_prefetch_budget_ms() {
auto const speed { camera_stream_speed_mps() };
@@ -756,7 +878,18 @@ camera_travel_forward() {
[[nodiscard]] double
gameplay_apply_budget_ms() {
return g_stream_catchup ? kCatchupApplyBudgetMs : kGameplayApplyBudgetMs;
if( g_stream_catchup ) {
if( stream_ring_deficit() ) {
return pending_apply_is_urgent() ? kUrgentApplyBudgetMs : 16.0;
}
if(
g_stream.last_outer_ring < kRingBoostOuterThreshold &&
camera_stream_speed_mps() > kPreemptDisableSpeedMps ) {
return 12.0;
}
return kCatchupApplyBudgetMs;
}
return kGameplayApplyBudgetMs;
}
[[nodiscard]] std::size_t
@@ -802,6 +935,9 @@ pending_section_distance_km() {
[[nodiscard]] bool
pending_apply_is_urgent() {
if( stream_ring_deficit() ) {
return true;
}
auto const speed { camera_stream_speed_mps() };
if( speed <= 80.0 && false == g_stream_catchup ) {
return false;
@@ -814,18 +950,21 @@ pending_apply_is_urgent() {
[[nodiscard]] std::size_t
scene_apply_pressure_slice_cap() {
auto const applied { load_stats().pack_models };
if( applied >= 200000 ) {
return 16;
}
if( applied >= 120000 ) {
if( applied >= 300000 ) {
return 24;
}
if( applied >= 80000 ) {
if( applied >= 200000 ) {
return 32;
}
if( applied >= 40000 ) {
if( applied >= 120000 ) {
return 48;
}
if( applied >= 80000 ) {
return 64;
}
if( applied >= 40000 ) {
return 80;
}
return std::numeric_limits<std::size_t>::max();
}
@@ -914,14 +1053,18 @@ adaptive_slice_instances( std::size_t const section_total, bool const urgent = f
[[nodiscard]] std::size_t
adaptive_cold_meshes( bool const urgent = false ) {
if( urgent ) {
return kUrgentSliceColdMeshes;
auto limit {
urgent ? kUrgentSliceColdMeshes : gameplay_slice_cold_meshes() };
if( stream_ring_deficit() ) {
limit = std::max( limit, urgent ? std::size_t { 8 } : std::size_t { 6 } );
}
auto limit { gameplay_slice_cold_meshes() };
if( pack_bench_stream().last_chunk_ms >= 12.0 ) {
limit = 1;
if( pack_bench_stream().last_chunk_ms >= 40.0 ) {
limit = std::min( limit, std::size_t { 2 } );
}
return limit;
else if( pack_bench_stream().last_chunk_ms >= 24.0 ) {
limit = std::min( limit, std::max( limit / 2, std::size_t { 3 } ) );
}
return std::max( limit, std::size_t { 2 } );
}
[[nodiscard]] std::size_t
@@ -941,27 +1084,31 @@ preload_slice_cold_meshes(
std::size_t const max_cold_meshes,
double const cold_budget_ms,
std::size_t &slice_count ) {
slice_count = count;
slice_count = 0;
if( models == nullptr || count == 0 ) {
slice_count = 0;
return 0;
}
std::size_t cold_loaded { 0 };
auto const cold_started { std::chrono::steady_clock::now() };
for( std::size_t i { 0 }; i < slice_count; ++i ) {
auto model_file { models[ i ].model_file };
auto mesh_ready { [&]( std::size_t const index ) -> bool {
auto model_file { models[ index ].model_file };
if( model_file.empty() || model_file == "notload" ) {
continue;
return true;
}
replace_slashes( model_file );
if( g_stream.mesh_cache.contains( model_file ) ) {
pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit );
return true;
}
return false;
} };
std::size_t cold_loaded { 0 };
auto const cold_started { std::chrono::steady_clock::now() };
for( std::size_t i { 0 }; i < count; ++i ) {
if( mesh_ready( i ) ) {
continue;
}
if( cold_loaded >= max_cold_meshes ) {
pack_bench_inc( &Eu7PackBench::stream_cold_slice_truncated );
slice_count = i;
break;
}
if( cold_budget_ms > 0.0 && cold_loaded > 0 ) {
@@ -969,11 +1116,11 @@ preload_slice_cold_meshes(
std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - cold_started ).count() };
if( elapsed_ms >= cold_budget_ms ) {
pack_bench_inc( &Eu7PackBench::stream_cold_slice_truncated );
slice_count = i;
break;
}
}
auto model_file { models[ i ].model_file };
replace_slashes( model_file );
{
PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms };
(void)ensure_stream_pack_mesh( model_file );
@@ -982,14 +1129,22 @@ preload_slice_cold_meshes(
++cold_loaded;
}
if( slice_count == 0 && count > 0 ) {
for( std::size_t i { 0 }; i < count; ++i ) {
if( false == mesh_ready( i ) ) {
slice_count = i;
break;
}
slice_count = i + 1;
}
if( slice_count == 0 ) {
auto model_file { models[ 0 ].model_file };
if( model_file.empty() || model_file == "notload" ) {
slice_count = 1;
return 0;
return cold_loaded;
}
replace_slashes( model_file );
if( false == g_stream.mesh_cache.contains( model_file ) ) {
if( false == g_stream.mesh_cache.contains( model_file ) && cold_loaded == 0 ) {
{
PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms };
(void)ensure_stream_pack_mesh( model_file );
@@ -1118,6 +1273,16 @@ maybe_preempt_distant_pending() {
return;
}
if( ready_queue_size() >= kPreemptReadyQueueThreshold ) {
return;
}
if( camera_stream_speed_mps() > kPreemptDisableSpeedMps ) {
return;
}
if( g_stream.last_outer_ring < kRingBoostOuterThreshold ) {
return;
}
auto &batch { *g_stream.pending_apply };
if( batch.models == nullptr || batch.models->empty() ) {
return;
@@ -1275,7 +1440,8 @@ apply_pending_chunk(
auto const far_slow_drain {
cam_speed > 300.0 &&
pending_dist > kFarApplySlowDistanceKm &&
false == is_urgent };
false == is_urgent &&
false == stream_ring_deficit() };
std::size_t chunk_count { remaining };
std::size_t cold_mesh_limit { remaining };
@@ -1325,26 +1491,26 @@ apply_pending_chunk(
{
auto const phase_started { std::chrono::steady_clock::now() };
if( false == batch.unique_meshes.empty() ) {
cold_loads = preload_umes_cold_meshes(
cold_loads += preload_umes_cold_meshes(
batch.unique_meshes,
batch.umes_cold_offset,
cold_mesh_limit,
effective_cold_budget_ms,
batch.umes_cold_offset );
}
else {
cold_loads = preload_slice_cold_meshes(
models_ptr + offset,
chunk_count,
cold_mesh_limit,
effective_cold_budget_ms,
chunk_count );
}
std::size_t inst_slice_count { chunk_count };
cold_loads += preload_slice_cold_meshes(
models_ptr + offset,
chunk_count,
cold_mesh_limit,
effective_cold_budget_ms,
inst_slice_count );
chunk_count = inst_slice_count;
cold_ms = std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - phase_started ).count();
}
pack_bench_inc( &Eu7PackBench::stream_inst_after_cold, chunk_count );
if( batch.unique_meshes.empty() && chunk_count < planned_inst ) {
if( chunk_count < planned_inst ) {
pack_bench_inc( &Eu7PackBench::stream_cold_slice_truncated );
}
if( chunk_count == 0 ) {
@@ -1588,8 +1754,6 @@ worker_loop( std::stop_token const stop_token ) {
pack_bench_inc(
&Eu7PackBench::worker_models_decoded, result.models->size() );
preload_pack_models( *result.models, result.unique_meshes );
{
std::lock_guard<std::mutex> lock { g_stream.ready.mutex };
g_stream.ready.data.push_back( std::move( result ) );
@@ -1600,24 +1764,22 @@ worker_loop( std::stop_token const stop_token ) {
pack_bench_inc( &Eu7PackBench::worker_sections_done );
}
if( job.next_chunk == 0 ) {
std::size_t pending_jobs { 0 };
{
std::lock_guard<std::mutex> lock { g_stream.jobs.mutex };
pending_jobs = g_stream.jobs.data.size();
}
if( pending_jobs < kUmesPrefetchJobsBacklogThreshold ) {
prefetch_ready_queue_umes_worker(
kReadyUmesPrefetchBudgetMs, kReadyUmesPrefetchMaxMeshes );
}
}
if( job.next_chunk + 1 < chunk_count ) {
PackSectionJob continue_job { job };
continue_job.next_chunk = job.next_chunk + 1;
{
auto const dist {
section_manhattan_sections(
continue_job.row,
continue_job.column,
g_stream.center_row,
g_stream.center_column ) };
if( dist > kWorkerContinueMaxDistanceKm ) {
push_prioritized_job( continue_job );
pack_bench_inc( &Eu7PackBench::stream_worker_deferred_distant );
}
else {
std::lock_guard<std::mutex> lock { g_stream.jobs.mutex };
g_stream.jobs.data.push_front( continue_job );
g_stream.jobs.data.push_front( std::move( continue_job ) );
}
g_stream.work_cv.notify_one();
}
@@ -1735,6 +1897,71 @@ section_manhattan_sections(
return std::abs( row - center_row ) + std::abs( column - center_column );
}
[[nodiscard]] std::size_t
count_far_in_flight_sections( int const max_distance_km ) {
if( g_stream.center_row < 0 || g_stream.center_column < 0 ) {
return 0;
}
std::size_t count { 0 };
for( auto const section_idx : g_stream.in_flight_sections ) {
auto const row {
static_cast<int>( section_idx / static_cast<std::size_t>( kRegionSideSectionCount ) ) };
auto const column {
static_cast<int>( section_idx % static_cast<std::size_t>( kRegionSideSectionCount ) ) };
if(
section_manhattan_sections(
row,
column,
g_stream.center_row,
g_stream.center_column ) > max_distance_km ) {
++count;
}
}
return count;
}
[[nodiscard]] bool
should_block_far_enqueue(
int const row,
int const column,
bool const lookahead_enqueue ) {
if( g_stream.bootstrap_active || false == g_loading_screen_dismissed ) {
return false;
}
if( g_stream.center_row < 0 || g_stream.center_column < 0 ) {
return false;
}
auto const dist {
section_manhattan_sections(
row,
column,
g_stream.center_row,
g_stream.center_column ) };
if( dist <= kSectionStreamGameplayRadiusKm ) {
return false;
}
if(
lookahead_enqueue &&
g_stream.last_inner_ring < kRingGatedInnerTarget ) {
pack_bench_inc( &Eu7PackBench::stream_jobs_blocked_ring_inner );
return true;
}
auto const speed { camera_stream_speed_mps() };
if(
speed > kFastFlightRingGateSpeedMps &&
dist > kFastFlightApplyMaxDistanceKm &&
count_far_in_flight_sections( kFastFlightApplyMaxDistanceKm ) >= kFastFlightMaxFarInFlight ) {
pack_bench_inc( &Eu7PackBench::stream_jobs_blocked_far );
return true;
}
return false;
}
void
push_prioritized_job( PackSectionJob job ) {
std::lock_guard<std::mutex> lock { g_stream.jobs.mutex };
@@ -1775,7 +2002,8 @@ reprioritize_job_queue() {
enqueue_section_if_needed(
int const row,
int const column,
int const priority ) {
int const priority,
bool const lookahead_enqueue ) {
if( row < 0 || row >= kRegionSideSectionCount || column < 0 || column >= kRegionSideSectionCount ) {
return false;
}
@@ -1786,6 +2014,9 @@ enqueue_section_if_needed(
pack_bench_inc( &Eu7PackBench::stream_jobs_blocked_far );
return false;
}
if( should_block_far_enqueue( row, column, lookahead_enqueue ) ) {
return false;
}
auto const section_idx { section_index( row, column ) };
if( g_stream.loaded_sections.contains( section_idx ) ) {
@@ -1993,7 +2224,7 @@ enqueue_movement_lookahead(
auto const column { center_column + dcolumn * ( ring_radius + step ) };
auto const priority {
section_manhattan_sections( row, column, center_row, center_column ) };
if( enqueue_section_if_needed( row, column, priority ) ) {
if( enqueue_section_if_needed( row, column, priority, true ) ) {
pack_bench_inc( &Eu7PackBench::stream_lookahead_enqueue );
}
if( stream_backpressure() ) {
@@ -2481,16 +2712,35 @@ sync_stream_limits( glm::dvec3 const &world_position ) {
auto const stream_position { resolve_section_stream_position( world_position ) };
auto const cam_speed { camera_stream_speed_mps() };
auto const outer_ring {
section_stream_ring_progress( stream_position, g_stream.radius ) };
g_stream.last_inner_ring =
section_stream_ring_progress( stream_position, kSectionStreamGameplayRadiusKm );
g_stream.last_outer_ring =
section_stream_ring_progress( stream_position, g_stream.radius );
g_stream_catchup = (
cam_speed > 80.0 ||
outer_ring < 0.98f );
g_stream.last_outer_ring < 0.98f );
if( g_stream_catchup ) {
g_stream_max_in_flight = kCatchupMaxInFlightSections;
g_stream_max_ready = kCatchupMaxReadySections;
if(
cam_speed > kRingBoostInFlightSpeedMps &&
g_stream.last_outer_ring < kRingBoostOuterThreshold ) {
g_stream_max_in_flight = std::max(
g_stream_max_in_flight, kRingStrongBoostMaxInFlight );
g_stream_max_ready = std::max(
g_stream_max_ready, kRingStrongBoostMaxReady );
}
else if(
cam_speed > kRingBoostInFlightSpeedMps &&
g_stream.last_outer_ring < kRingStrongBoostOuterThreshold ) {
g_stream_max_in_flight = std::max(
g_stream_max_in_flight, kRingBoostMaxInFlight );
g_stream_max_ready = std::max(
g_stream_max_ready, kRingBoostMaxReady );
}
}
else {
g_stream_max_in_flight = kMaxInFlightSections;
@@ -2652,6 +2902,8 @@ update_section_stream( glm::dvec3 const &world_position ) {
auto const stream_position { resolve_section_stream_position( world_position ) };
auto const prev_center_row { g_stream.center_row };
auto const prev_center_column { g_stream.center_column };
auto const [center_row, center_column] { section_row_column( stream_position ) };
auto const center_moved {
center_row != g_stream.center_row || center_column != g_stream.center_column };
@@ -2667,9 +2919,11 @@ update_section_stream( glm::dvec3 const &world_position ) {
g_stream.center_row = center_row;
g_stream.center_column = center_column;
update_unload_keep_radius_hysteresis();
if( center_moved ) {
reprioritize_job_queue();
maybe_unload_distant_sections( center_row, center_column );
maybe_unload_distant_sections( prev_center_row, prev_center_column );
}
sync_stream_limits( stream_position );
@@ -2769,9 +3023,8 @@ drain_section_stream(
kSectionStreamGameplayRadiusKm ) };
if( gameplay_stream_mode() && inner_ring_ready ) {
maybe_unload_distant_sections(
g_stream.center_row,
g_stream.center_column );
prefetch_ready_queue_umes_worker(
kReadyUmesPrefetchBudgetMs, kReadyUmesPrefetchMaxMeshes );
prefetch_ready_queue_textures( ready_texture_prefetch_budget_ms() );
if( g_stream_catchup ) {
@@ -2790,7 +3043,8 @@ drain_section_stream(
adaptive_slice_instances( pending_section_total(), urgent ),
adaptive_cold_meshes( urgent ),
urgent ? kUrgentColdBudgetMs : gameplay_cold_budget_ms(),
urgent ? kUrgentMaxChunksPerDrain : 1 );
urgent ? kUrgentMaxChunksPerDrain :
( stream_ring_deficit() ? kRingDeficitMaxChunksPerDrain : 1 ) );
}
else {
pack_bench_inc( &Eu7PackBench::stream_drain_gameplay );
@@ -2799,6 +3053,8 @@ drain_section_stream(
maybe_log_stream_status( stream_position );
}
else if( gameplay_stream_mode() ) {
prefetch_ready_queue_umes_worker(
kReadyUmesPrefetchBudgetMs, kReadyUmesPrefetchMaxMeshes );
pack_bench_inc( &Eu7PackBench::stream_drain_loader );
drain_until_budget( kLoaderDrainBudgetMs );
maybe_log_stream_status( stream_position );

1
scene/eu7/eu7_types.h
View File

@@ -380,6 +380,7 @@ struct Eu7PackSectionCursor {
std::vector<std::string> unique_textures;
std::uint32_t chunk_count { 0 };
std::vector<std::uint32_t> chunk_model_counts;
std::vector<std::uint32_t> chunk_inst_counts;
std::vector<std::uint32_t> chunk_byte_offsets;
};

7
simulation/simulationstateserializer.cpp
View File

@@ -33,6 +33,7 @@ http://mozilla.org/MPL/2.0/.
#include "scene/eu7/eu7_bake.h"
#include "scene/eu7/eu7_loader.h"
#include "scene/eu7/eu7_load_stats.h"
#include "scene/eu7/eu7_model_prefetch.h"
#include "scene/eu7/eu7_pack_bench.h"
#include "scene/eu7/eu7_section.h"
#include "scene/eu7/eu7_transform.h"
@@ -545,9 +546,9 @@ state_serializer::insert_eu7_pack_models(
if( found != mesh_cache.end() ) {
mesh = found->second;
}
else {
mesh = TModelsManager::GetModel( model_file, false, false );
mesh_cache.emplace( model_file, mesh );
else if( scene::eu7::ensure_pack_mesh_in_session_cache( model_file, mesh_cache ) ) {
auto const loaded { mesh_cache.find( model_file ) };
mesh = loaded != mesh_cache.end() ? loaded->second : nullptr;
}
}
loaded = instance->LoadEu7PackWarm( mesh, texture_file );