From 2f90246907e6797077252161983842b76469a1d2 Mon Sep 17 00:00:00 2001 From: maj00r Date: Sun, 14 Jun 2026 22:18:52 +0200 Subject: [PATCH] Add EU7 PACK v12 PROT/INST and tune streaming apply path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce EU7B v8 bake with global PROT chunk and v12 section payloads (solo MODL + compact inst records, per-chunk solo/inst tables). Reader parses v12 with chunked decode and v11→flat fallback; fix chunk blob layout to write solo blocks before inst records. Streaming: add unload keep-radius hysteresis on deceleration, ring-deficit urgent drain, higher cold-mesh/apply budgets, and smarter warm-prefix preload. Move UMES mesh prefetch to the main thread and route PACK apply through ensure_pack_mesh_in_session_cache to avoid GetModel races on Global.asCurrentTexturePath; harden MdlMngr texture-path trimming. Rebake root scenario .eu7 files to get v12; existing v7–v11 packs remain readable. --- docs/eu7-format.md | 37 ++ .../include/eu07/scene/binary/common.hpp | 12 +- .../eu07/scene/binary/runtime_module.hpp | 326 +++++++++++++- model/MdlMngr.cpp | 5 +- scene/eu7/eu7_chunks.h | 1 + scene/eu7/eu7_model_prefetch.cpp | 57 +-- scene/eu7/eu7_model_prefetch.h | 3 +- scene/eu7/eu7_pack_bench.cpp | 3 + scene/eu7/eu7_pack_bench.h | 3 + scene/eu7/eu7_reader.cpp | 64 ++- scene/eu7/eu7_section_stream.cpp | 426 ++++++++++++++---- scene/eu7/eu7_types.h | 1 + simulation/simulationstateserializer.cpp | 7 +- 13 files changed, 791 insertions(+), 154 deletions(-) diff --git a/docs/eu7-format.md b/docs/eu7-format.md index cdac2c34..62bc1db8 100644 --- a/docs/eu7-format.md +++ b/docs/eu7-format.md @@ -877,6 +877,43 @@ Powtórzone chunk_count razy: Runtime używa UTEX do prefetch/warm tekstur bez skanowania wszystkich instancji. +### Format sekcji — v12 PROT+INST (`kPackSectionFormatV12 = 5`) + +Rozszerzenie v11: sekcja dzieli modele na **solo** (pełny MODL) i **inst** (kompaktowy rekord odwołujący się do globalnego chunku PROT). Chunki zawierają mix solo+inst; w każdym chunku najpierw solo, potem inst. + +``` +uint8 section_format — musi być = 5 +uint32 solo_count +uint32 inst_count +uint32 unique_mesh_count +uint32[unique_mesh_count] string_id + +uint32 unique_texture_count +uint32[unique_texture_count] string_id + +uint32 chunk_count +Powtórzone chunk_count razy: + uint32 chunk_solo_count + uint32 chunk_inst_count + uint32 chunk_byte_offset + +[payload chunków — solo_count pełnych MODL + inst_count rekordów inst rozłożonych po chunkach] +``` + +Rekord instancji (jak v8): + +``` +uint32 proto_id +Vec3 location +Vec3 angles +Vec3 scale +string_id name +``` + +Bake v12 emituje chunk **PROT** przed **PACK** (wersja pliku 8). Runtime rozwija inst przez `expand_prototype_instance()` i `module.model_prototypes`. + +Sekcje v11–v7 pozostają czytelne (fallback v12→v11→v10→flat). + --- ## 30. PROT — prototypy modeli (v8+) diff --git a/eu07-parser/include/eu07/scene/binary/common.hpp b/eu07-parser/include/eu07/scene/binary/common.hpp index 23783a05..ac9053f8 100644 --- a/eu07-parser/include/eu07/scene/binary/common.hpp +++ b/eu07-parser/include/eu07/scene/binary/common.hpp @@ -17,17 +17,22 @@ inline constexpr std::array kMagic{'E', 'U', '7', 'B'}; inline constexpr std::uint32_t kVersionLegacy = 1; // codec.hpp — nieobslugiwane w CLI inline constexpr std::uint32_t kVersionRuntime = 6; inline constexpr std::uint32_t kVersionRuntimeV7 = 7; +inline constexpr std::uint32_t kVersionRuntimeV8 = 8; inline constexpr std::uint32_t kVersionRuntimeV5 = 5; inline constexpr std::uint32_t kVersionRuntimeV4 = 4; inline constexpr std::uint32_t kVersionRuntimeF32 = 3; inline constexpr std::uint32_t kVersionRuntimeF64 = 2; [[nodiscard]] inline bool isSupportedEu7bVersion(const std::uint32_t version) noexcept { - return version == kVersionRuntimeV7 || version == kVersionRuntime || - version == kVersionRuntimeV5 || version == kVersionRuntimeV4; + return version == kVersionRuntimeV8 || version == kVersionRuntimeV7 || + version == kVersionRuntime || version == kVersionRuntimeV5 || + version == kVersionRuntimeV4; } [[nodiscard]] inline std::string formatVersionName(const std::uint32_t version) { + if (version == kVersionRuntimeV8) { + return "RuntimeV8"; + } if (version == kVersionRuntimeV7) { return "RuntimeV7"; } @@ -53,6 +58,9 @@ inline constexpr std::uint32_t kVersionRuntimeF64 = 2; } [[nodiscard]] inline std::string formatVersionDescription(const std::uint32_t version) { + if (version == kVersionRuntimeV8) { + return "bake v7 + PROT + PACK sekcji v12 (solo + compact inst)"; + } if (version == kVersionRuntimeV7) { return "bake v6 + MODL w chunkach PIDX/PACK per sekcja 1km (v7)"; } diff --git a/eu07-parser/include/eu07/scene/binary/runtime_module.hpp b/eu07-parser/include/eu07/scene/binary/runtime_module.hpp index 6496ed47..fd1a4efe 100644 --- a/eu07-parser/include/eu07/scene/binary/runtime_module.hpp +++ b/eu07-parser/include/eu07/scene/binary/runtime_module.hpp @@ -49,11 +49,13 @@ inline constexpr std::array kChunkFint{'F', 'I', 'N', 'T'}; inline constexpr std::array kChunkPlac{'P', 'L', 'A', 'C'}; inline constexpr std::array kChunkPidx{'P', 'I', 'D', 'X'}; inline constexpr std::array kChunkPack{'P', 'A', 'C', 'K'}; +inline constexpr std::array kChunkProt{'P', 'R', 'O', 'T'}; -// PACK section payload: v9 = UMES, v10 = UMES + CHNK, v11 = UMES + UTEX + CHNK. +// PACK section payload: v9 = UMES, v10 = UMES + CHNK, v11 = UMES + UTEX + CHNK, v12 = v11 + PROT inst. inline constexpr std::uint8_t kPackSectionFormatV9 = 2; inline constexpr std::uint8_t kPackSectionFormatV10 = 3; inline constexpr std::uint8_t kPackSectionFormatV11 = 4; +inline constexpr std::uint8_t kPackSectionFormatV12 = 5; inline constexpr std::size_t kPackSectionChunkModels = 512; inline constexpr std::size_t kPackSectionChunkModelsHeavy = 256; inline constexpr std::size_t kPackSectionHeavyModelThreshold = 1024; @@ -405,6 +407,7 @@ struct PackSectionIndexEntry { struct PackPayloadBuild { std::vector packPayload; + std::vector protPayload; std::vector entries; }; @@ -712,7 +715,301 @@ inline void sortPackSectionModelsForStreaming( return {blob.begin(), blob.end()}; } -[[nodiscard]] inline PackPayloadBuild buildPackPayloadV7( +struct PackPrototypeKey { + std::string modelFile; + std::string textureFile; + double rangeSquaredMin = 0.0; + double rangeSquaredMax = 0.0; + bool visible = true; + bool isTerrain = false; + bool transition = true; + + [[nodiscard]] bool operator==(const PackPrototypeKey& other) const noexcept { + return modelFile == other.modelFile && textureFile == other.textureFile && + rangeSquaredMin == other.rangeSquaredMin && + rangeSquaredMax == other.rangeSquaredMax && visible == other.visible && + isTerrain == other.isTerrain && transition == other.transition; + } +}; + +struct PackPrototypeKeyHash { + [[nodiscard]] std::size_t operator()(const PackPrototypeKey& key) const noexcept { + std::size_t hash = std::hash{}(key.modelFile); + hash ^= std::hash{}(key.textureFile) + 0x9e3779b9 + (hash << 6) + (hash >> 2); + hash ^= std::hash{}(key.rangeSquaredMin) + 0x9e3779b9 + (hash << 6) + (hash >> 2); + hash ^= std::hash{}(key.rangeSquaredMax) + 0x9e3779b9 + (hash << 6) + (hash >> 2); + hash ^= static_cast(key.visible) + 0x9e3779b9 + (hash << 6) + (hash >> 2); + hash ^= static_cast(key.isTerrain) + 0x9e3779b9 + (hash << 6) + (hash >> 2); + hash ^= static_cast(key.transition) + 0x9e3779b9 + (hash << 6) + (hash >> 2); + return hash; + } +}; + +[[nodiscard]] inline PackPrototypeKey makePackPrototypeKey( + const runtime::RuntimeModelInstance& model) noexcept { + PackPrototypeKey key; + key.modelFile = model.modelFile; + key.textureFile = model.textureFile; + key.rangeSquaredMin = model.node.rangeSquaredMin; + key.rangeSquaredMax = model.node.rangeSquaredMax; + key.visible = model.node.visible; + key.isTerrain = model.isTerrain; + key.transition = model.transition; + return key; +} + +[[nodiscard]] inline bool modelMustBePackSolo(const runtime::RuntimeModelInstance& model) noexcept { + if (model.isTerrain) { + return true; + } + if (!model.lightStates.empty() || !model.lightColors.empty()) { + return true; + } + if (!isLoadableMeshPath(model.modelFile)) { + return true; + } + return false; +} + +[[nodiscard]] inline bool modelCanBePackInstanced(const runtime::RuntimeModelInstance& model) noexcept { + return !modelMustBePackSolo(model); +} + +inline void writeRuntimePrototype( + std::ostream& out, + StringTable& table, + const runtime::RuntimeModelInstance& model) { + codec::writeSlimNode(out, table, model.node, "model"); + io::writeU8(out, model.isTerrain ? 1 : 0); + io::writeU8(out, model.transition ? 1 : 0); + codec::writeStringId(out, table, model.modelFile); + codec::writeStringId(out, table, model.textureFile); + io::writeU32(out, static_cast(model.lightStates.size())); + for (float light : model.lightStates) { + io::writeF32(out, light); + } + io::writeU32(out, static_cast(model.lightColors.size())); + for (std::uint32_t color : model.lightColors) { + io::writeU32(out, color); + } +} + +[[nodiscard]] inline std::vector buildProtPayload( + StringTable& table, + const std::vector& prototypes) { + std::ostringstream out(std::ios::binary); + io::writeU32(out, static_cast(prototypes.size())); + for (const runtime::RuntimeModelInstance& proto : prototypes) { + writeRuntimePrototype(out, table, proto); + } + const std::string blob = out.str(); + return {blob.begin(), blob.end()}; +} + +struct PackPrototypeTable { + std::vector prototypes; + std::unordered_map key_to_id; +}; + +[[nodiscard]] inline PackPrototypeTable collectPackPrototypes( + StringTable& table, + const std::vector& batches) { + std::unordered_map frequency; + frequency.reserve(1024); + + for (const codec::ModelSectionBatch& batch : batches) { + for (const runtime::RuntimeModelInstance& model : batch.models) { + if (!modelCanBePackInstanced(model)) { + continue; + } + ++frequency[makePackPrototypeKey(model)]; + } + } + + PackPrototypeTable result; + for (const codec::ModelSectionBatch& batch : batches) { + for (const runtime::RuntimeModelInstance& model : batch.models) { + if (!modelCanBePackInstanced(model)) { + continue; + } + const PackPrototypeKey key = makePackPrototypeKey(model); + const auto freq_it = frequency.find(key); + if (freq_it == frequency.end() || freq_it->second < 2) { + continue; + } + if (result.key_to_id.contains(key)) { + continue; + } + const std::uint32_t proto_id = + static_cast(result.prototypes.size()); + result.key_to_id.emplace(key, proto_id); + runtime::RuntimeModelInstance proto = model; + proto.location = {}; + proto.angles = {}; + proto.scale = {1.0, 1.0, 1.0}; + proto.node.transform = {}; + collectModelStrings(table, proto); + result.prototypes.push_back(std::move(proto)); + } + } + return result; +} + +enum class PackSectionEntryKind : std::uint8_t { Solo, Inst }; + +struct PackSectionEntry { + PackSectionEntryKind kind = PackSectionEntryKind::Solo; + std::size_t model_index = 0; + std::uint32_t proto_id = 0; +}; + +inline void writePackInstRecord( + std::ostream& out, + StringTable& table, + const std::uint32_t proto_id, + const runtime::RuntimeModelInstance& model) { + io::writeU32(out, proto_id); + io::writeVec3(out, model.location); + io::writeVec3(out, model.angles); + io::writeVec3(out, model.scale); + codec::writeStringId(out, table, model.node.name); +} + +[[nodiscard]] inline std::vector buildPackSectionChunkBlobV12( + StringTable& table, + const std::vector& models, + const std::vector& entries, + const std::size_t begin, + const std::size_t end) { + // Reader decodes each chunk as a solo block followed by inst records (v8 layout). + std::ostringstream out(std::ios::binary); + for (std::size_t index = begin; index < end; ++index) { + const PackSectionEntry& entry = entries[index]; + if (entry.kind != PackSectionEntryKind::Solo) { + continue; + } + runtime::RuntimeModelInstance worldModel = models[entry.model_index]; + worldModel.node.transform = {}; + writeRuntimeModel(out, table, worldModel); + } + for (std::size_t index = begin; index < end; ++index) { + const PackSectionEntry& entry = entries[index]; + if (entry.kind != PackSectionEntryKind::Inst) { + continue; + } + writePackInstRecord(out, table, entry.proto_id, models[entry.model_index]); + } + const std::string blob = out.str(); + return {blob.begin(), blob.end()}; +} + +[[nodiscard]] inline std::vector buildPackSectionPayloadV12( + StringTable& table, + const std::vector& models, + const PackPrototypeTable& prototypes) { + std::vector entries; + entries.reserve(models.size()); + + std::uint32_t solo_count = 0; + std::uint32_t inst_count = 0; + for (std::size_t model_index = 0; model_index < models.size(); ++model_index) { + const runtime::RuntimeModelInstance& model = models[model_index]; + if (modelMustBePackSolo(model)) { + entries.push_back({PackSectionEntryKind::Solo, model_index, 0}); + ++solo_count; + continue; + } + const PackPrototypeKey key = makePackPrototypeKey(model); + const auto proto_it = prototypes.key_to_id.find(key); + if (proto_it == prototypes.key_to_id.end()) { + entries.push_back({PackSectionEntryKind::Solo, model_index, 0}); + ++solo_count; + continue; + } + entries.push_back({PackSectionEntryKind::Inst, model_index, proto_it->second}); + ++inst_count; + } + + std::vector prototype_mesh_paths; + prototype_mesh_paths.reserve(prototypes.prototypes.size()); + for (const runtime::RuntimeModelInstance& proto : prototypes.prototypes) { + prototype_mesh_paths.emplace_back(proto.modelFile); + } + + const auto mesh_ids = + collectUniqueMeshIds(table, models, &prototype_mesh_paths); + const auto texture_ids = collectUniqueTextureIds(table, models); + const std::size_t chunk_models = + models.size() > kPackSectionHeavyModelThreshold ? kPackSectionChunkModelsHeavy + : kPackSectionChunkModels; + + std::vector> chunk_payloads; + std::vector chunk_solo_counts; + std::vector chunk_inst_counts; + chunk_payloads.reserve((entries.size() + chunk_models - 1) / chunk_models); + chunk_solo_counts.reserve(chunk_payloads.capacity()); + chunk_inst_counts.reserve(chunk_payloads.capacity()); + + for (std::size_t offset = 0; offset < entries.size(); offset += chunk_models) { + const std::size_t end = std::min(offset + chunk_models, entries.size()); + std::uint32_t chunk_solo = 0; + std::uint32_t chunk_inst = 0; + for (std::size_t index = offset; index < end; ++index) { + if (entries[index].kind == PackSectionEntryKind::Solo) { + ++chunk_solo; + } else { + ++chunk_inst; + } + } + chunk_solo_counts.push_back(chunk_solo); + chunk_inst_counts.push_back(chunk_inst); + chunk_payloads.push_back( + buildPackSectionChunkBlobV12(table, models, entries, offset, end)); + } + if (chunk_payloads.empty()) { + chunk_payloads.emplace_back(); + chunk_solo_counts.push_back(0); + chunk_inst_counts.push_back(0); + } + + const std::uint32_t chunk_count = static_cast(chunk_payloads.size()); + const std::uint32_t header_size = + 1u + 4u + 4u + 4u + static_cast(mesh_ids.size()) * 4u + 4u + + static_cast(texture_ids.size()) * 4u + 4u + chunk_count * 12u; + + std::vector chunk_offsets(chunk_count); + std::uint32_t payload_offset = header_size; + for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) { + chunk_offsets[chunk_index] = payload_offset; + payload_offset += static_cast(chunk_payloads[chunk_index].size()); + } + + std::ostringstream packOut(std::ios::binary); + io::writeU8(packOut, kPackSectionFormatV12); + io::writeU32(packOut, solo_count); + io::writeU32(packOut, inst_count); + io::writeU32(packOut, static_cast(mesh_ids.size())); + for (const std::uint32_t id : mesh_ids) { + io::writeU32(packOut, id); + } + io::writeU32(packOut, static_cast(texture_ids.size())); + for (const std::uint32_t id : texture_ids) { + io::writeU32(packOut, id); + } + io::writeU32(packOut, chunk_count); + for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) { + io::writeU32(packOut, chunk_solo_counts[chunk_index]); + io::writeU32(packOut, chunk_inst_counts[chunk_index]); + io::writeU32(packOut, chunk_offsets[chunk_index]); + } + for (const std::vector& payload : chunk_payloads) { + packOut.write(payload.data(), static_cast(payload.size())); + } + const std::string blob = packOut.str(); + return {blob.begin(), blob.end()}; +} + +[[nodiscard]] inline PackPayloadBuild buildPackPayloadV12( StringTable& table, const std::vector& batches) { PackPayloadBuild result; @@ -726,6 +1023,9 @@ inline void sortPackSectionModelsForStreaming( } } + const PackPrototypeTable prototypes = collectPackPrototypes(table, batches); + result.protPayload = buildProtPayload(table, prototypes.prototypes); + result.entries.resize(batches.size()); std::vector> section_payloads(batches.size()); @@ -742,7 +1042,8 @@ inline void sortPackSectionModelsForStreaming( entry.modelCount = static_cast(batch.models.size()); std::vector section_models = batch.models; sortPackSectionModelsForStreaming(section_models, batch.section); - section_payloads[batch_index] = buildPackSectionPayloadV11(table, section_models); + section_payloads[batch_index] = + buildPackSectionPayloadV12(table, section_models, prototypes); } std::size_t total_size = 0; for (const std::vector& payload : section_payloads) { @@ -778,7 +1079,8 @@ inline void sortPackSectionModelsForStreaming( std::vector section_models = batch.models; sortPackSectionModelsForStreaming(section_models, batch.section); - section_payloads[batch_index] = buildPackSectionPayloadV11(table, section_models); + section_payloads[batch_index] = + buildPackSectionPayloadV12(table, section_models, prototypes); } }); } @@ -804,6 +1106,12 @@ inline void sortPackSectionModelsForStreaming( return result; } +[[nodiscard]] inline PackPayloadBuild buildPackPayloadV7( + StringTable& table, + const std::vector& batches) { + return buildPackPayloadV12(table, batches); +} + [[nodiscard]] inline PackPayloadBuild buildPackPayloadV7( StringTable& table, const std::vector& models) { @@ -1083,7 +1391,8 @@ inline void readPidxChunkV7(std::istream& in, std::vector [[nodiscard]] inline bool isKnownRuntimeChunk(const std::array& id) { return id == kChunkStrs || id == kChunkIncl || id == kChunkPlac || id == kChunkTrak || id == kChunkTrac || id == kChunkPwrs || id == kChunkTerr || id == kChunkMesh || - id == kChunkLine || id == kChunkModl || id == kChunkPidx || id == kChunkPack || id == kChunkMemc || + id == kChunkLine || id == kChunkModl || id == kChunkPidx || id == kChunkPack || + id == kChunkProt || id == kChunkMemc || id == kChunkLaun || id == kChunkDynm || id == kChunkSond || id == kChunkTrset || id == kChunkEvnt || id == kChunkFint; } @@ -1160,7 +1469,7 @@ inline void writeRuntimeModule( } out.write(kMagic.data(), 4); - io::writeU32(out, usePackModels ? kVersionRuntimeV7 : kVersionRuntime); + io::writeU32(out, usePackModels ? kVersionRuntimeV8 : kVersionRuntime); io::writeChunkHeader(out, kChunkStrs, 8 + static_cast(strsPayload.size())); out.write(strsPayload.data(), static_cast(strsPayload.size())); @@ -1193,6 +1502,11 @@ inline void writeRuntimeModule( if (usePackModels) { io::writeChunkHeader(out, kChunkPidx, 8 + static_cast(pidxPayload.size())); out.write(pidxPayload.data(), static_cast(pidxPayload.size())); + io::writeChunkHeader( + out, kChunkProt, 8 + static_cast(packBuild.protPayload.size())); + out.write( + packBuild.protPayload.data(), + static_cast(packBuild.protPayload.size())); io::writeChunkHeader( out, kChunkPack, 8 + static_cast(packBuild.packPayload.size())); out.write( diff --git a/model/MdlMngr.cpp b/model/MdlMngr.cpp index c27d6f74..3b692b28 100644 --- a/model/MdlMngr.cpp +++ b/model/MdlMngr.cpp @@ -86,7 +86,10 @@ TModelsManager::GetModel(std::string const &Name, bool const Dynamic, bool const // pobieranie tekstur z katalogu, w którym jest model // when loading vehicles the path is set by the calling routine, so we can skip it here Global.asCurrentTexturePath += Name; - Global.asCurrentTexturePath.erase( Global.asCurrentTexturePath.rfind( '/' ) + 1 ); + auto const slash_pos { Global.asCurrentTexturePath.rfind( '/' ) }; + if( slash_pos != std::string::npos ) { + Global.asCurrentTexturePath.erase( slash_pos + 1 ); + } } erase_extension( filename ); filename = ToLower( filename ); diff --git a/scene/eu7/eu7_chunks.h b/scene/eu7/eu7_chunks.h index 454bf5f2..7da420cd 100644 --- a/scene/eu7/eu7_chunks.h +++ b/scene/eu7/eu7_chunks.h @@ -61,6 +61,7 @@ constexpr std::uint8_t kPackSectionFormatV8 { 1 }; constexpr std::uint8_t kPackSectionFormatV9 { 2 }; constexpr std::uint8_t kPackSectionFormatV10 { 3 }; constexpr std::uint8_t kPackSectionFormatV11 { 4 }; +constexpr std::uint8_t kPackSectionFormatV12 { 5 }; constexpr std::size_t kPackSectionChunkModels { 512 }; } // namespace scene::eu7 diff --git a/scene/eu7/eu7_model_prefetch.cpp b/scene/eu7/eu7_model_prefetch.cpp index d1467144..dd13264b 100644 --- a/scene/eu7/eu7_model_prefetch.cpp +++ b/scene/eu7/eu7_model_prefetch.cpp @@ -106,14 +106,9 @@ pack_texture_usable( std::string texture_file ) { } void -preload_pack_model_file( std::string const &model_file ) { - if( false == TModelsManager::IsModelCached( model_file ) ) { - return; - } - - if( auto *const mesh { TModelsManager::GetModel( model_file, false, false ) } ) { - TAnimModel::warm_instanceable_cache( mesh ); - } +preload_pack_model_file( std::string const & ) { + // Mesh warm runs on the main thread via ensure_pack_mesh_in_session_cache only. + // GetModel mutates Global.asCurrentTexturePath and must not run on PACK workers. } [[nodiscard]] bool @@ -208,55 +203,17 @@ reset_pack_texture_warm_cache() { } void -preload_pack_model_paths( std::vector const &model_files ) { - std::unordered_set seen; - seen.reserve( model_files.size() ); - - for( auto model_file : model_files ) { - if( model_file.empty() || model_file == "notload" ) { - continue; - } - - replace_slashes( model_file ); - if( false == seen.insert( model_file ).second ) { - continue; - } - - std::lock_guard lock { g_pack_mesh_load_mutex }; - preload_pack_model_file( model_file ); - } +preload_pack_model_paths( std::vector const & ) { } void -preload_pack_models( std::vector const &models ) { - std::unordered_set seen; - seen.reserve( models.size() ); - - for( auto const &model : models ) { - auto model_file { model.model_file }; - if( model_file.empty() || model_file == "notload" ) { - continue; - } - - replace_slashes( model_file ); - if( false == seen.insert( model_file ).second ) { - continue; - } - - std::lock_guard lock { g_pack_mesh_load_mutex }; - preload_pack_model_file( model_file ); - } +preload_pack_models( std::vector const & ) { } void preload_pack_models( - std::vector const &models, - std::vector const &unique_meshes ) { - if( false == unique_meshes.empty() ) { - preload_pack_model_paths( unique_meshes ); - return; - } - preload_pack_models( models ); + std::vector const &, + std::vector const & ) { } std::size_t diff --git a/scene/eu7/eu7_model_prefetch.h b/scene/eu7/eu7_model_prefetch.h index 5ae916fb..1381e2b1 100644 --- a/scene/eu7/eu7_model_prefetch.h +++ b/scene/eu7/eu7_model_prefetch.h @@ -21,11 +21,10 @@ class TModel3d; namespace scene::eu7 { -// Tylko warm_instanceable_cache dla juz zcache'owanych meshy (bez cold GetModel na workerze). +// Deprecated: mesh warm is main-thread only (ensure_pack_mesh_in_session_cache). void preload_pack_models( std::vector const &Models ); -// Jak wyzej, ale iteruje precomputed UMES zamiast skanowac instancje. void preload_pack_models( std::vector const &Models, diff --git a/scene/eu7/eu7_pack_bench.cpp b/scene/eu7/eu7_pack_bench.cpp index d6c9d21f..d065e58b 100644 --- a/scene/eu7/eu7_pack_bench.cpp +++ b/scene/eu7/eu7_pack_bench.cpp @@ -186,6 +186,9 @@ log_pack_bench_impl( Eu7PackBench const &bench, char const *const title ) { " cam=" + std::to_string( bench.stream_dequeue_camera ) + " wait_near=" + std::to_string( bench.stream_dequeue_wait_near ) + " block_far=" + std::to_string( bench.stream_jobs_blocked_far ) + + " block_ring_in=" + std::to_string( bench.stream_jobs_blocked_ring_inner ) + + " block_ring_out=" + std::to_string( bench.stream_jobs_blocked_ring_outer ) + + " defer_dist=" + std::to_string( bench.stream_worker_deferred_distant ) + " stuck_skip=" + std::to_string( bench.stream_apply_stuck_skip ) ); WriteLog( " diag ready_tex_ms=" + std::to_string( static_cast( bench.stream_prefetch_ready_tex_ms ) ) + diff --git a/scene/eu7/eu7_pack_bench.h b/scene/eu7/eu7_pack_bench.h index bfc7543a..48643b13 100644 --- a/scene/eu7/eu7_pack_bench.h +++ b/scene/eu7/eu7_pack_bench.h @@ -71,6 +71,9 @@ struct Eu7PackBench { std::uint64_t stream_dequeue_camera { 0 }; std::uint64_t stream_dequeue_wait_near { 0 }; std::uint64_t stream_jobs_blocked_far { 0 }; + std::uint64_t stream_jobs_blocked_ring_inner { 0 }; + std::uint64_t stream_jobs_blocked_ring_outer { 0 }; + std::uint64_t stream_worker_deferred_distant { 0 }; std::uint64_t stream_apply_stuck_skip { 0 }; std::uint64_t stream_sections_unloaded { 0 }; std::uint64_t stream_unload_instances { 0 }; diff --git a/scene/eu7/eu7_reader.cpp b/scene/eu7/eu7_reader.cpp index 765612a5..678efbf5 100644 --- a/scene/eu7/eu7_reader.cpp +++ b/scene/eu7/eu7_reader.cpp @@ -505,6 +505,52 @@ parse_pack_section_header( const std::uint8_t first_byte { peek == EOF ? std::uint8_t { 0 } : static_cast( peek ) }; + bool use_v12_header { false }; + if( first_byte == kPackSectionFormatV12 ) { + cursor.section_format = sn_utils::d_uint8( input ); + const std::uint32_t solo_total { sn_utils::ld_uint32( input ) }; + const std::uint32_t inst_total { sn_utils::ld_uint32( input ) }; + const std::uint32_t unique_mesh_count { sn_utils::ld_uint32( input ) }; + if( + solo_total + inst_total == entry.model_count && + ( inst_total == 0 || false == module.model_prototypes.empty() ) ) { + use_v12_header = true; + cursor.model_total = solo_total + inst_total; + cursor.solo_remaining = solo_total; + cursor.inst_remaining = inst_total; + + StringTable const strings { module.strings }; + cursor.unique_meshes.reserve( unique_mesh_count ); + for( std::uint32_t mesh_idx { 0 }; mesh_idx < unique_mesh_count; ++mesh_idx ) { + cursor.unique_meshes.push_back( + strings.resolve( sn_utils::ld_uint32( input ) ) ); + } + + const std::uint32_t unique_texture_count { sn_utils::ld_uint32( input ) }; + cursor.unique_textures.reserve( unique_texture_count ); + for( std::uint32_t tex_idx { 0 }; tex_idx < unique_texture_count; ++tex_idx ) { + cursor.unique_textures.push_back( + strings.resolve( sn_utils::ld_uint32( input ) ) ); + } + + cursor.chunk_count = sn_utils::ld_uint32( input ); + cursor.chunk_model_counts.resize( cursor.chunk_count ); + cursor.chunk_inst_counts.resize( cursor.chunk_count ); + cursor.chunk_byte_offsets.resize( cursor.chunk_count ); + for( std::uint32_t chunk_idx { 0 }; chunk_idx < cursor.chunk_count; ++chunk_idx ) { + cursor.chunk_model_counts[ chunk_idx ] = sn_utils::ld_uint32( input ); + cursor.chunk_inst_counts[ chunk_idx ] = sn_utils::ld_uint32( input ); + cursor.chunk_byte_offsets[ chunk_idx ] = sn_utils::ld_uint32( input ); + } + } + } + + if( use_v12_header ) { + cursor.models_read = 0; + cursor.header_parsed = true; + return; + } + bool use_v11_header { false }; if( first_byte == kPackSectionFormatV11 ) { cursor.section_format = sn_utils::d_uint8( input ); @@ -672,7 +718,7 @@ read_pack_models_chunk_impl( const std::uint32_t proto_id { sn_utils::ld_uint32( input ) }; if( proto_id >= module.model_prototypes.size() ) { throw std::runtime_error( - "EU7 PACK v8: proto_id " + std::to_string( proto_id ) + " poza zakresem PROT (" + + "EU7 PACK: proto_id " + std::to_string( proto_id ) + " poza zakresem PROT (" + std::to_string( module.model_prototypes.size() ) + ")" ); } auto const location { read_vec3( input ) }; @@ -1216,15 +1262,22 @@ read_pack_section_chunk_load_impl( Eu7PackSectionCursor chunk_cursor { header }; if( ( header.section_format == kPackSectionFormatV10 || - header.section_format == kPackSectionFormatV11 ) && + header.section_format == kPackSectionFormatV11 || + header.section_format == kPackSectionFormatV12 ) && false == header.chunk_byte_offsets.empty() ) { auto const section_start { static_cast( module.pack_payload_offset + entry->pack_offset ) }; session.input.seekg( section_start + static_cast( header.chunk_byte_offsets[ chunk_index ] ) ); - chunk_cursor.solo_remaining = header.chunk_model_counts[ chunk_index ]; - chunk_cursor.inst_remaining = 0; + if( header.section_format == kPackSectionFormatV12 ) { + chunk_cursor.solo_remaining = header.chunk_model_counts[ chunk_index ]; + chunk_cursor.inst_remaining = header.chunk_inst_counts[ chunk_index ]; + } + else { + chunk_cursor.solo_remaining = header.chunk_model_counts[ chunk_index ]; + chunk_cursor.inst_remaining = 0; + } } else if( chunk_index > 0 ) { return result; @@ -1266,7 +1319,8 @@ read_pack_section_load_impl( Eu7Module const &module, int const row, int const c if( ( header.section_format == kPackSectionFormatV10 || - header.section_format == kPackSectionFormatV11 ) && + header.section_format == kPackSectionFormatV11 || + header.section_format == kPackSectionFormatV12 ) && header.chunk_count > 0 ) { result.models.reserve( entry->model_count ); for( std::uint32_t chunk_idx { 0 }; chunk_idx < header.chunk_count; ++chunk_idx ) { diff --git a/scene/eu7/eu7_section_stream.cpp b/scene/eu7/eu7_section_stream.cpp index e1ed784d..71f5fe2e 100644 --- a/scene/eu7/eu7_section_stream.cpp +++ b/scene/eu7/eu7_section_stream.cpp @@ -57,20 +57,24 @@ constexpr double kDrainBudgetMs { 12.0 }; constexpr double kLoaderDrainBudgetMs { 96.0 }; constexpr std::size_t kLoaderSectionsPerDrain { 8 }; -constexpr double kGameplayApplyBudgetMs { 4.0 }; -constexpr double kCatchupApplyBudgetMs { 6.0 }; +constexpr double kGameplayApplyBudgetMs { 6.0 }; +constexpr double kCatchupApplyBudgetMs { 14.0 }; constexpr std::size_t kGameplaySliceInstances { 96 }; -constexpr std::size_t kCatchupSliceInstances { 64 }; +constexpr std::size_t kCatchupSliceInstances { 96 }; constexpr std::size_t kLoaderSliceInstances { 768 }; -constexpr std::size_t kGameplaySliceColdMeshes { 1 }; -constexpr std::size_t kCatchupSliceColdMeshes { 2 }; +constexpr std::size_t kGameplaySliceColdMeshes { 4 }; +constexpr std::size_t kCatchupSliceColdMeshes { 8 }; constexpr std::size_t kLoaderSliceColdMeshes { 32 }; -constexpr double kGameplayColdBudgetMs { 3.0 }; -constexpr double kCatchupColdBudgetMs { 5.0 }; +constexpr double kGameplayColdBudgetMs { 12.0 }; +constexpr double kCatchupColdBudgetMs { 18.0 }; constexpr int kInitialBootstrapRadius { 5 }; constexpr int kStreamRadius { 14 }; -constexpr int kSectionUnloadMarginKm { 2 }; +constexpr int kSectionUnloadMarginKm { 3 }; +constexpr int kSectionUnloadCameraGuardKm { 5 }; +constexpr int kSectionUnloadMaxSpeedExtraKm { 10 }; +constexpr double kSectionUnloadKeepRadiusDecayKmPerSec { 2.0 }; +constexpr int kSectionUnloadTeleportJumpKm { 4 }; constexpr std::size_t kMeshCacheLruCap { 2000 }; constexpr std::size_t kNodedataCacheLruCap { 2000 }; constexpr int kMovementLookahead { 10 }; @@ -85,21 +89,35 @@ constexpr std::size_t kPackTextureWarmSlice { 8 }; constexpr double kPackTextureWarmBudgetMs { 4.0 }; constexpr double kUrgentApplyBudgetMs { 32.0 }; constexpr std::size_t kUrgentSliceInstances { 96 }; -constexpr std::size_t kUrgentSliceColdMeshes { 2 }; -constexpr double kUrgentColdBudgetMs { 24.0 }; +constexpr std::size_t kUrgentSliceColdMeshes { 6 }; +constexpr double kUrgentColdBudgetMs { 28.0 }; constexpr std::size_t kUrgentMaxChunksPerDrain { 8 }; +constexpr std::size_t kRingDeficitMaxChunksPerDrain { 4 }; constexpr double kReadyTexturePrefetchBudgetMs { 14.0 }; constexpr std::size_t kReadyTexturePrefetchSlice { 256 }; constexpr int kUmesPrefetchMaxDistanceKm { 2 }; constexpr double kReadyUmesPrefetchBudgetMs { 8.0 }; constexpr std::size_t kReadyUmesPrefetchMaxMeshes { 6 }; -constexpr std::size_t kUmesPrefetchJobsBacklogThreshold { 4 }; constexpr std::size_t kCatchupMaxInFlightSections { 28 }; constexpr std::size_t kCatchupMaxReadySections { 14 }; constexpr int kUrgentSectionDistanceKm { 1 }; -constexpr int kFastFlightApplyMaxDistanceKm { 2 }; +constexpr int kFastFlightApplyMaxDistanceKm { 3 }; constexpr int kFarApplySlowDistanceKm { 4 }; constexpr int kPreemptPendingDistanceKm { 2 }; +constexpr float kRingGatedOuterThreshold { 0.35f }; +constexpr float kRingGatedInnerTarget { 0.70f }; +constexpr float kRingBoostOuterThreshold { 0.55f }; +constexpr float kRingStrongBoostOuterThreshold { 0.75f }; +constexpr std::size_t kPreemptReadyQueueThreshold { 8 }; +constexpr int kWorkerContinueMaxDistanceKm { 8 }; +constexpr std::size_t kFastFlightMaxFarInFlight { 3 }; +constexpr double kFastFlightRingGateSpeedMps { 600.0 }; +constexpr double kRingBoostInFlightSpeedMps { 600.0 }; +constexpr std::size_t kRingBoostMaxInFlight { 12 }; +constexpr std::size_t kRingStrongBoostMaxInFlight { 16 }; +constexpr std::size_t kRingBoostMaxReady { 8 }; +constexpr std::size_t kRingStrongBoostMaxReady { 10 }; +constexpr double kPreemptDisableSpeedMps { 400.0 }; constexpr double kTeleportReenqueueDistanceM { 2000.0 }; constexpr float kStationaryCatchupRingThreshold { 0.70f }; constexpr double kStationaryCatchupSpeedMps { 5.0 }; @@ -157,6 +175,11 @@ struct SectionStreamState { bool has_last_enqueue_position { false }; glm::dvec3 anchor_position {}; bool has_anchor_position { false }; + float last_inner_ring { 1.f }; + float last_outer_ring { 1.f }; + int unload_keep_radius { 0 }; + double unload_keep_radius_floor { 0.0 }; + double unload_keep_radius_last_update { 0.0 }; std::unordered_map mesh_cache; std::unordered_map nodedata_cache; @@ -185,6 +208,15 @@ section_manhattan_sections( int const center_row, int const center_column ); +[[nodiscard]] std::size_t +count_far_in_flight_sections( int const max_distance_km ); + +[[nodiscard]] bool +should_block_far_enqueue( + int const row, + int const column, + bool const lookahead_enqueue ); + void push_prioritized_job( PackSectionJob job ); @@ -195,13 +227,16 @@ reprioritize_job_queue(); enqueue_section_if_needed( int const row, int const column, - int const priority ); + int const priority, + bool const lookahead_enqueue = false ); void sync_stream_limits( glm::dvec3 const &world_position ); void -maybe_unload_distant_sections( int const center_row, int const center_column ); +maybe_unload_distant_sections( + int const prev_center_row, + int const prev_center_column ); void evict_unreferenced_stream_caches(); @@ -540,12 +575,67 @@ unload_pack_section( int const row, int const column ) { } void -maybe_unload_distant_sections( int const center_row, int const center_column ) { - if( false == g_loading_screen_dismissed || g_stream.bootstrap_active ) { +update_unload_keep_radius_hysteresis() { + if( g_stream.center_row < 0 || g_stream.center_column < 0 ) { + return; + } + + auto const speed { camera_stream_speed_mps() }; + auto const speed_extra { + std::min( + static_cast( speed / 150.0 ), + kSectionUnloadMaxSpeedExtraKm ) }; + auto const desired_keep_radius { + static_cast( g_stream.radius + kSectionUnloadMarginKm + speed_extra ) }; + auto const now { Timer::GetTime() }; + + if( g_stream.unload_keep_radius_floor <= 0.0 ) { + g_stream.unload_keep_radius_floor = desired_keep_radius; + g_stream.unload_keep_radius_last_update = now; + g_stream.unload_keep_radius = static_cast( std::ceil( desired_keep_radius ) ); + return; + } + + auto const dt { std::max( 0.0, now - g_stream.unload_keep_radius_last_update ) }; + g_stream.unload_keep_radius_last_update = now; + + if( desired_keep_radius > g_stream.unload_keep_radius_floor ) { + g_stream.unload_keep_radius_floor = desired_keep_radius; + } + else if( desired_keep_radius < g_stream.unload_keep_radius_floor && dt > 0.0 ) { + g_stream.unload_keep_radius_floor = std::max( + desired_keep_radius, + g_stream.unload_keep_radius_floor - dt * kSectionUnloadKeepRadiusDecayKmPerSec ); + } + g_stream.unload_keep_radius = static_cast( std::ceil( g_stream.unload_keep_radius_floor ) ); +} + +void +maybe_unload_distant_sections( + int const prev_center_row, + int const prev_center_column ) { + if( false == g_loading_screen_dismissed || g_stream.bootstrap_active ) { + return; + } + if( g_stream.center_row < 0 || g_stream.center_column < 0 ) { + return; + } + + auto const [cam_row, cam_col] { section_row_column( Global.pCamera.Pos ) }; + auto const keep_radius { g_stream.unload_keep_radius }; + + auto const center_jump { + prev_center_row >= 0 && prev_center_column >= 0 ? + section_manhattan_sections( + g_stream.center_row, + g_stream.center_column, + prev_center_row, + prev_center_column ) : + 0 }; + if( center_jump >= kSectionUnloadTeleportJumpKm ) { return; } - auto const keep_radius { g_stream.radius + kSectionUnloadMarginKm }; std::vector to_unload; to_unload.reserve( g_stream.loaded_sections.size() ); @@ -554,7 +644,23 @@ maybe_unload_distant_sections( int const center_row, int const center_column ) { static_cast( section_idx / static_cast( kRegionSideSectionCount ) ) }; auto const column { static_cast( section_idx % static_cast( kRegionSideSectionCount ) ) }; - if( ring_section_in_radius( row, column, center_row, center_column, keep_radius ) ) { + + if( + ring_section_in_radius( + row, + column, + cam_row, + cam_col, + kSectionUnloadCameraGuardKm ) ) { + continue; + } + if( + ring_section_in_radius( + row, + column, + g_stream.center_row, + g_stream.center_column, + keep_radius ) ) { continue; } to_unload.push_back( section_idx ); @@ -612,6 +718,11 @@ reset_stream_fields() { g_stream.has_last_enqueue_position = false; g_stream.anchor_position = {}; g_stream.has_anchor_position = false; + g_stream.last_inner_ring = 1.f; + g_stream.last_outer_ring = 1.f; + g_stream.unload_keep_radius = 0; + g_stream.unload_keep_radius_floor = 0.0; + g_stream.unload_keep_radius_last_update = 0.0; g_stream.mesh_cache.clear(); g_stream.nodedata_cache.clear(); g_stream.mesh_lru.clear(); @@ -699,9 +810,12 @@ stream_backpressure() { return true; } } - return ( - g_stream.in_flight_sections.size() >= g_stream_max_in_flight || - ready_queue_size() >= g_stream_max_ready ); + if( + false == g_stream.bootstrap_active && + g_stream.in_flight_sections.size() >= g_stream_max_in_flight ) { + return true; + } + return ready_queue_size() >= g_stream_max_ready; } [[nodiscard]] double @@ -712,8 +826,16 @@ camera_stream_speed_mps() { } [[nodiscard]] bool pending_apply_is_urgent(); +[[nodiscard]] bool stream_ring_deficit(); [[nodiscard]] double gameplay_apply_budget_ms(); +[[nodiscard]] bool +stream_ring_deficit() { + return ( + g_stream.last_outer_ring < kRingGatedOuterThreshold || + g_stream.last_inner_ring < kRingGatedInnerTarget ); +} + [[nodiscard]] double ready_texture_prefetch_budget_ms() { auto const speed { camera_stream_speed_mps() }; @@ -756,7 +878,18 @@ camera_travel_forward() { [[nodiscard]] double gameplay_apply_budget_ms() { - return g_stream_catchup ? kCatchupApplyBudgetMs : kGameplayApplyBudgetMs; + if( g_stream_catchup ) { + if( stream_ring_deficit() ) { + return pending_apply_is_urgent() ? kUrgentApplyBudgetMs : 16.0; + } + if( + g_stream.last_outer_ring < kRingBoostOuterThreshold && + camera_stream_speed_mps() > kPreemptDisableSpeedMps ) { + return 12.0; + } + return kCatchupApplyBudgetMs; + } + return kGameplayApplyBudgetMs; } [[nodiscard]] std::size_t @@ -802,6 +935,9 @@ pending_section_distance_km() { [[nodiscard]] bool pending_apply_is_urgent() { + if( stream_ring_deficit() ) { + return true; + } auto const speed { camera_stream_speed_mps() }; if( speed <= 80.0 && false == g_stream_catchup ) { return false; @@ -814,18 +950,21 @@ pending_apply_is_urgent() { [[nodiscard]] std::size_t scene_apply_pressure_slice_cap() { auto const applied { load_stats().pack_models }; - if( applied >= 200000 ) { - return 16; - } - if( applied >= 120000 ) { + if( applied >= 300000 ) { return 24; } - if( applied >= 80000 ) { + if( applied >= 200000 ) { return 32; } - if( applied >= 40000 ) { + if( applied >= 120000 ) { return 48; } + if( applied >= 80000 ) { + return 64; + } + if( applied >= 40000 ) { + return 80; + } return std::numeric_limits::max(); } @@ -914,14 +1053,18 @@ adaptive_slice_instances( std::size_t const section_total, bool const urgent = f [[nodiscard]] std::size_t adaptive_cold_meshes( bool const urgent = false ) { - if( urgent ) { - return kUrgentSliceColdMeshes; + auto limit { + urgent ? kUrgentSliceColdMeshes : gameplay_slice_cold_meshes() }; + if( stream_ring_deficit() ) { + limit = std::max( limit, urgent ? std::size_t { 8 } : std::size_t { 6 } ); } - auto limit { gameplay_slice_cold_meshes() }; - if( pack_bench_stream().last_chunk_ms >= 12.0 ) { - limit = 1; + if( pack_bench_stream().last_chunk_ms >= 40.0 ) { + limit = std::min( limit, std::size_t { 2 } ); } - return limit; + else if( pack_bench_stream().last_chunk_ms >= 24.0 ) { + limit = std::min( limit, std::max( limit / 2, std::size_t { 3 } ) ); + } + return std::max( limit, std::size_t { 2 } ); } [[nodiscard]] std::size_t @@ -941,27 +1084,31 @@ preload_slice_cold_meshes( std::size_t const max_cold_meshes, double const cold_budget_ms, std::size_t &slice_count ) { - slice_count = count; + slice_count = 0; if( models == nullptr || count == 0 ) { - slice_count = 0; return 0; } - std::size_t cold_loaded { 0 }; - auto const cold_started { std::chrono::steady_clock::now() }; - for( std::size_t i { 0 }; i < slice_count; ++i ) { - auto model_file { models[ i ].model_file }; + auto mesh_ready { [&]( std::size_t const index ) -> bool { + auto model_file { models[ index ].model_file }; if( model_file.empty() || model_file == "notload" ) { - continue; + return true; } replace_slashes( model_file ); if( g_stream.mesh_cache.contains( model_file ) ) { pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit ); + return true; + } + return false; + } }; + + std::size_t cold_loaded { 0 }; + auto const cold_started { std::chrono::steady_clock::now() }; + for( std::size_t i { 0 }; i < count; ++i ) { + if( mesh_ready( i ) ) { continue; } if( cold_loaded >= max_cold_meshes ) { - pack_bench_inc( &Eu7PackBench::stream_cold_slice_truncated ); - slice_count = i; break; } if( cold_budget_ms > 0.0 && cold_loaded > 0 ) { @@ -969,11 +1116,11 @@ preload_slice_cold_meshes( std::chrono::duration( std::chrono::steady_clock::now() - cold_started ).count() }; if( elapsed_ms >= cold_budget_ms ) { - pack_bench_inc( &Eu7PackBench::stream_cold_slice_truncated ); - slice_count = i; break; } } + auto model_file { models[ i ].model_file }; + replace_slashes( model_file ); { PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms }; (void)ensure_stream_pack_mesh( model_file ); @@ -982,14 +1129,22 @@ preload_slice_cold_meshes( ++cold_loaded; } - if( slice_count == 0 && count > 0 ) { + for( std::size_t i { 0 }; i < count; ++i ) { + if( false == mesh_ready( i ) ) { + slice_count = i; + break; + } + slice_count = i + 1; + } + + if( slice_count == 0 ) { auto model_file { models[ 0 ].model_file }; if( model_file.empty() || model_file == "notload" ) { slice_count = 1; - return 0; + return cold_loaded; } replace_slashes( model_file ); - if( false == g_stream.mesh_cache.contains( model_file ) ) { + if( false == g_stream.mesh_cache.contains( model_file ) && cold_loaded == 0 ) { { PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms }; (void)ensure_stream_pack_mesh( model_file ); @@ -1118,6 +1273,16 @@ maybe_preempt_distant_pending() { return; } + if( ready_queue_size() >= kPreemptReadyQueueThreshold ) { + return; + } + if( camera_stream_speed_mps() > kPreemptDisableSpeedMps ) { + return; + } + if( g_stream.last_outer_ring < kRingBoostOuterThreshold ) { + return; + } + auto &batch { *g_stream.pending_apply }; if( batch.models == nullptr || batch.models->empty() ) { return; @@ -1275,7 +1440,8 @@ apply_pending_chunk( auto const far_slow_drain { cam_speed > 300.0 && pending_dist > kFarApplySlowDistanceKm && - false == is_urgent }; + false == is_urgent && + false == stream_ring_deficit() }; std::size_t chunk_count { remaining }; std::size_t cold_mesh_limit { remaining }; @@ -1325,26 +1491,26 @@ apply_pending_chunk( { auto const phase_started { std::chrono::steady_clock::now() }; if( false == batch.unique_meshes.empty() ) { - cold_loads = preload_umes_cold_meshes( + cold_loads += preload_umes_cold_meshes( batch.unique_meshes, batch.umes_cold_offset, cold_mesh_limit, effective_cold_budget_ms, batch.umes_cold_offset ); } - else { - cold_loads = preload_slice_cold_meshes( - models_ptr + offset, - chunk_count, - cold_mesh_limit, - effective_cold_budget_ms, - chunk_count ); - } + std::size_t inst_slice_count { chunk_count }; + cold_loads += preload_slice_cold_meshes( + models_ptr + offset, + chunk_count, + cold_mesh_limit, + effective_cold_budget_ms, + inst_slice_count ); + chunk_count = inst_slice_count; cold_ms = std::chrono::duration( std::chrono::steady_clock::now() - phase_started ).count(); } pack_bench_inc( &Eu7PackBench::stream_inst_after_cold, chunk_count ); - if( batch.unique_meshes.empty() && chunk_count < planned_inst ) { + if( chunk_count < planned_inst ) { pack_bench_inc( &Eu7PackBench::stream_cold_slice_truncated ); } if( chunk_count == 0 ) { @@ -1588,8 +1754,6 @@ worker_loop( std::stop_token const stop_token ) { pack_bench_inc( &Eu7PackBench::worker_models_decoded, result.models->size() ); - preload_pack_models( *result.models, result.unique_meshes ); - { std::lock_guard lock { g_stream.ready.mutex }; g_stream.ready.data.push_back( std::move( result ) ); @@ -1600,24 +1764,22 @@ worker_loop( std::stop_token const stop_token ) { pack_bench_inc( &Eu7PackBench::worker_sections_done ); } - if( job.next_chunk == 0 ) { - std::size_t pending_jobs { 0 }; - { - std::lock_guard lock { g_stream.jobs.mutex }; - pending_jobs = g_stream.jobs.data.size(); - } - if( pending_jobs < kUmesPrefetchJobsBacklogThreshold ) { - prefetch_ready_queue_umes_worker( - kReadyUmesPrefetchBudgetMs, kReadyUmesPrefetchMaxMeshes ); - } - } - if( job.next_chunk + 1 < chunk_count ) { PackSectionJob continue_job { job }; continue_job.next_chunk = job.next_chunk + 1; - { + auto const dist { + section_manhattan_sections( + continue_job.row, + continue_job.column, + g_stream.center_row, + g_stream.center_column ) }; + if( dist > kWorkerContinueMaxDistanceKm ) { + push_prioritized_job( continue_job ); + pack_bench_inc( &Eu7PackBench::stream_worker_deferred_distant ); + } + else { std::lock_guard lock { g_stream.jobs.mutex }; - g_stream.jobs.data.push_front( continue_job ); + g_stream.jobs.data.push_front( std::move( continue_job ) ); } g_stream.work_cv.notify_one(); } @@ -1735,6 +1897,71 @@ section_manhattan_sections( return std::abs( row - center_row ) + std::abs( column - center_column ); } +[[nodiscard]] std::size_t +count_far_in_flight_sections( int const max_distance_km ) { + if( g_stream.center_row < 0 || g_stream.center_column < 0 ) { + return 0; + } + + std::size_t count { 0 }; + for( auto const section_idx : g_stream.in_flight_sections ) { + auto const row { + static_cast( section_idx / static_cast( kRegionSideSectionCount ) ) }; + auto const column { + static_cast( section_idx % static_cast( kRegionSideSectionCount ) ) }; + if( + section_manhattan_sections( + row, + column, + g_stream.center_row, + g_stream.center_column ) > max_distance_km ) { + ++count; + } + } + return count; +} + +[[nodiscard]] bool +should_block_far_enqueue( + int const row, + int const column, + bool const lookahead_enqueue ) { + if( g_stream.bootstrap_active || false == g_loading_screen_dismissed ) { + return false; + } + if( g_stream.center_row < 0 || g_stream.center_column < 0 ) { + return false; + } + + auto const dist { + section_manhattan_sections( + row, + column, + g_stream.center_row, + g_stream.center_column ) }; + if( dist <= kSectionStreamGameplayRadiusKm ) { + return false; + } + + if( + lookahead_enqueue && + g_stream.last_inner_ring < kRingGatedInnerTarget ) { + pack_bench_inc( &Eu7PackBench::stream_jobs_blocked_ring_inner ); + return true; + } + + auto const speed { camera_stream_speed_mps() }; + if( + speed > kFastFlightRingGateSpeedMps && + dist > kFastFlightApplyMaxDistanceKm && + count_far_in_flight_sections( kFastFlightApplyMaxDistanceKm ) >= kFastFlightMaxFarInFlight ) { + pack_bench_inc( &Eu7PackBench::stream_jobs_blocked_far ); + return true; + } + + return false; +} + void push_prioritized_job( PackSectionJob job ) { std::lock_guard lock { g_stream.jobs.mutex }; @@ -1775,7 +2002,8 @@ reprioritize_job_queue() { enqueue_section_if_needed( int const row, int const column, - int const priority ) { + int const priority, + bool const lookahead_enqueue ) { if( row < 0 || row >= kRegionSideSectionCount || column < 0 || column >= kRegionSideSectionCount ) { return false; } @@ -1786,6 +2014,9 @@ enqueue_section_if_needed( pack_bench_inc( &Eu7PackBench::stream_jobs_blocked_far ); return false; } + if( should_block_far_enqueue( row, column, lookahead_enqueue ) ) { + return false; + } auto const section_idx { section_index( row, column ) }; if( g_stream.loaded_sections.contains( section_idx ) ) { @@ -1993,7 +2224,7 @@ enqueue_movement_lookahead( auto const column { center_column + dcolumn * ( ring_radius + step ) }; auto const priority { section_manhattan_sections( row, column, center_row, center_column ) }; - if( enqueue_section_if_needed( row, column, priority ) ) { + if( enqueue_section_if_needed( row, column, priority, true ) ) { pack_bench_inc( &Eu7PackBench::stream_lookahead_enqueue ); } if( stream_backpressure() ) { @@ -2481,16 +2712,35 @@ sync_stream_limits( glm::dvec3 const &world_position ) { auto const stream_position { resolve_section_stream_position( world_position ) }; auto const cam_speed { camera_stream_speed_mps() }; - auto const outer_ring { - section_stream_ring_progress( stream_position, g_stream.radius ) }; + g_stream.last_inner_ring = + section_stream_ring_progress( stream_position, kSectionStreamGameplayRadiusKm ); + g_stream.last_outer_ring = + section_stream_ring_progress( stream_position, g_stream.radius ); g_stream_catchup = ( cam_speed > 80.0 || - outer_ring < 0.98f ); + g_stream.last_outer_ring < 0.98f ); if( g_stream_catchup ) { g_stream_max_in_flight = kCatchupMaxInFlightSections; g_stream_max_ready = kCatchupMaxReadySections; + + if( + cam_speed > kRingBoostInFlightSpeedMps && + g_stream.last_outer_ring < kRingBoostOuterThreshold ) { + g_stream_max_in_flight = std::max( + g_stream_max_in_flight, kRingStrongBoostMaxInFlight ); + g_stream_max_ready = std::max( + g_stream_max_ready, kRingStrongBoostMaxReady ); + } + else if( + cam_speed > kRingBoostInFlightSpeedMps && + g_stream.last_outer_ring < kRingStrongBoostOuterThreshold ) { + g_stream_max_in_flight = std::max( + g_stream_max_in_flight, kRingBoostMaxInFlight ); + g_stream_max_ready = std::max( + g_stream_max_ready, kRingBoostMaxReady ); + } } else { g_stream_max_in_flight = kMaxInFlightSections; @@ -2652,6 +2902,8 @@ update_section_stream( glm::dvec3 const &world_position ) { auto const stream_position { resolve_section_stream_position( world_position ) }; + auto const prev_center_row { g_stream.center_row }; + auto const prev_center_column { g_stream.center_column }; auto const [center_row, center_column] { section_row_column( stream_position ) }; auto const center_moved { center_row != g_stream.center_row || center_column != g_stream.center_column }; @@ -2667,9 +2919,11 @@ update_section_stream( glm::dvec3 const &world_position ) { g_stream.center_row = center_row; g_stream.center_column = center_column; + update_unload_keep_radius_hysteresis(); + if( center_moved ) { reprioritize_job_queue(); - maybe_unload_distant_sections( center_row, center_column ); + maybe_unload_distant_sections( prev_center_row, prev_center_column ); } sync_stream_limits( stream_position ); @@ -2769,9 +3023,8 @@ drain_section_stream( kSectionStreamGameplayRadiusKm ) }; if( gameplay_stream_mode() && inner_ring_ready ) { - maybe_unload_distant_sections( - g_stream.center_row, - g_stream.center_column ); + prefetch_ready_queue_umes_worker( + kReadyUmesPrefetchBudgetMs, kReadyUmesPrefetchMaxMeshes ); prefetch_ready_queue_textures( ready_texture_prefetch_budget_ms() ); if( g_stream_catchup ) { @@ -2790,7 +3043,8 @@ drain_section_stream( adaptive_slice_instances( pending_section_total(), urgent ), adaptive_cold_meshes( urgent ), urgent ? kUrgentColdBudgetMs : gameplay_cold_budget_ms(), - urgent ? kUrgentMaxChunksPerDrain : 1 ); + urgent ? kUrgentMaxChunksPerDrain : + ( stream_ring_deficit() ? kRingDeficitMaxChunksPerDrain : 1 ) ); } else { pack_bench_inc( &Eu7PackBench::stream_drain_gameplay ); @@ -2799,6 +3053,8 @@ drain_section_stream( maybe_log_stream_status( stream_position ); } else if( gameplay_stream_mode() ) { + prefetch_ready_queue_umes_worker( + kReadyUmesPrefetchBudgetMs, kReadyUmesPrefetchMaxMeshes ); pack_bench_inc( &Eu7PackBench::stream_drain_loader ); drain_until_budget( kLoaderDrainBudgetMs ); maybe_log_stream_status( stream_position ); diff --git a/scene/eu7/eu7_types.h b/scene/eu7/eu7_types.h index d8b08f3d..a7ca8974 100644 --- a/scene/eu7/eu7_types.h +++ b/scene/eu7/eu7_types.h @@ -380,6 +380,7 @@ struct Eu7PackSectionCursor { std::vector unique_textures; std::uint32_t chunk_count { 0 }; std::vector chunk_model_counts; + std::vector chunk_inst_counts; std::vector chunk_byte_offsets; }; diff --git a/simulation/simulationstateserializer.cpp b/simulation/simulationstateserializer.cpp index 555ba464..8d639d9b 100644 --- a/simulation/simulationstateserializer.cpp +++ b/simulation/simulationstateserializer.cpp @@ -33,6 +33,7 @@ http://mozilla.org/MPL/2.0/. #include "scene/eu7/eu7_bake.h" #include "scene/eu7/eu7_loader.h" #include "scene/eu7/eu7_load_stats.h" +#include "scene/eu7/eu7_model_prefetch.h" #include "scene/eu7/eu7_pack_bench.h" #include "scene/eu7/eu7_section.h" #include "scene/eu7/eu7_transform.h" @@ -545,9 +546,9 @@ state_serializer::insert_eu7_pack_models( if( found != mesh_cache.end() ) { mesh = found->second; } - else { - mesh = TModelsManager::GetModel( model_file, false, false ); - mesh_cache.emplace( model_file, mesh ); + else if( scene::eu7::ensure_pack_mesh_in_session_cache( model_file, mesh_cache ) ) { + auto const loaded { mesh_cache.find( model_file ) }; + mesh = loaded != mesh_cache.end() ? loaded->second : nullptr; } } loaded = instance->LoadEu7PackWarm( mesh, texture_file );