diff --git a/CMakeLists.txt b/CMakeLists.txt index 705b416e..865a4f84 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -156,6 +156,7 @@ set(SOURCES "scene/eu7/eu7_section.cpp" "scene/eu7/eu7_section_stream.cpp" "scene/eu7/eu7_model_prefetch.cpp" +"scene/eu7/eu7_pack_mesh_loader.cpp" "scene/eu7/eu7_reader.cpp" "scene/eu7/eu7_emit.cpp" "scene/eu7/eu7_apply.cpp" diff --git a/application/drivermode.cpp b/application/drivermode.cpp index a4ab0bae..f956bd8f 100644 --- a/application/drivermode.cpp +++ b/application/drivermode.cpp @@ -447,10 +447,6 @@ void driver_mode::enter() TDynamicObject *nPlayerTrain{((Global.local_start_vehicle != "ghostview") ? simulation::Vehicles.find(Global.local_start_vehicle) : nullptr)}; - if( scene::eu7::section_stream_active() ) { - scene::eu7::dismiss_loading_screen(); - } - Camera.Init(Global.FreeCameraInit[0], Global.FreeCameraInitAngle[0], nullptr); Global.pCamera = Camera; Global.pDebugCamera = DebugCamera; @@ -468,14 +464,6 @@ void driver_mode::enter() Camera = Global.pCamera; } scene::eu7::update_section_stream( Global.pCamera.Pos ); - for( int i { 0 }; i < 64; ++i ) { - scene::eu7::drain_section_stream(); - if( scene::eu7::section_stream_ready_around( - stream_pos, - scene::eu7::kSectionStreamBootstrapRadiusKm ) ) { - break; - } - } } if (nPlayerTrain) diff --git a/application/scenarioloadermode.cpp b/application/scenarioloadermode.cpp index c9a9cf7c..011fa4c9 100644 --- a/application/scenarioloadermode.cpp +++ b/application/scenarioloadermode.cpp @@ -60,7 +60,8 @@ bool scenarioloader_mode::update() { WriteLog( "Scenario loading time: " + std::to_string( std::chrono::duration_cast( ( std::chrono::system_clock::now() - timestart ) ).count() ) + " seconds" ); if( scene::eu7::section_stream_active() ) { - scene::eu7::preload_section_stream( 3000.0 ); + // Bounded PACK drain after scenario parse — shortens black overlay in driver mode. + scene::eu7::preload_section_stream( 8000.0 ); } // TODO: implement and use next mode cue diff --git a/docs/eu7-format.md b/docs/eu7-format.md index 62bc1db8..b17b35d9 100644 --- a/docs/eu7-format.md +++ b/docs/eu7-format.md @@ -85,8 +85,9 @@ Nie ma pola "łączny rozmiar pliku" w nagłówku. Po nagłówku następują chu | `6` | `kEu7VersionV6` | `kVersionRuntime` | Adds site transform w rekordach INCL | | `7` | `kEu7VersionV7` | `kVersionRuntimeV7` | Adds PIDX + PACK — modele scenerii per sekcja 1 km (streaming) | | `8` | `kEu7VersionV8` | — | Adds PROT (prototypy) + nowy format sekcji PACK v8 | +| `9` | `kEu7VersionV9` | `kVersionRuntimeV9` | PROT v9 (resolved_texture, pack_flags) + PACK sekcji v13 | -**Runtime maszyna-fresh obsługuje wersje 4–8.** Wersje 1–3 nie są obsługiwane przez żaden z dostępnych loaderów. +**Runtime maszyna-fresh obsługuje wersje 4–9.** Wersje 1–3 nie są obsługiwane przez żaden z dostępnych loaderów. --- @@ -1025,6 +1026,7 @@ Overlay renderera: `inst-pool:` — ile instancji zakwalifikowanych do `Render_I | v6 | Chunk INCL rozszerzony o **site transform** (pełny TransformContext per include) | | v7 | Chunki **PIDX + PACK** — modele scenerii podzielone na kafelki 1 km, asynchroniczny streaming | | v8 | Chunk **PROT** (prototypy) + nowy format sekcji PACK z `solo_count` + `inst_count` | +| v9 | **PROT v9** (resolved_texture, pack_flags, baked ranges) + **PACK v13** (mesh/tex indices, cell_id) | --- diff --git a/eu07-parser/include/eu07/scene/binary/common.hpp b/eu07-parser/include/eu07/scene/binary/common.hpp index ac9053f8..30f2320a 100644 --- a/eu07-parser/include/eu07/scene/binary/common.hpp +++ b/eu07-parser/include/eu07/scene/binary/common.hpp @@ -18,18 +18,22 @@ inline constexpr std::uint32_t kVersionLegacy = 1; // codec.hpp — nieobslugiwa inline constexpr std::uint32_t kVersionRuntime = 6; inline constexpr std::uint32_t kVersionRuntimeV7 = 7; inline constexpr std::uint32_t kVersionRuntimeV8 = 8; +inline constexpr std::uint32_t kVersionRuntimeV9 = 9; inline constexpr std::uint32_t kVersionRuntimeV5 = 5; inline constexpr std::uint32_t kVersionRuntimeV4 = 4; inline constexpr std::uint32_t kVersionRuntimeF32 = 3; inline constexpr std::uint32_t kVersionRuntimeF64 = 2; [[nodiscard]] inline bool isSupportedEu7bVersion(const std::uint32_t version) noexcept { - return version == kVersionRuntimeV8 || version == kVersionRuntimeV7 || - version == kVersionRuntime || version == kVersionRuntimeV5 || - version == kVersionRuntimeV4; + return version == kVersionRuntimeV9 || version == kVersionRuntimeV8 || + version == kVersionRuntimeV7 || version == kVersionRuntime || + version == kVersionRuntimeV5 || version == kVersionRuntimeV4; } [[nodiscard]] inline std::string formatVersionName(const std::uint32_t version) { + if (version == kVersionRuntimeV9) { + return "RuntimeV9"; + } if (version == kVersionRuntimeV8) { return "RuntimeV8"; } @@ -58,6 +62,9 @@ inline constexpr std::uint32_t kVersionRuntimeF64 = 2; } [[nodiscard]] inline std::string formatVersionDescription(const std::uint32_t version) { + if (version == kVersionRuntimeV9) { + return "bake v9 + PROT v9 + PACK sekcji v13 (indices + cell sort)"; + } if (version == kVersionRuntimeV8) { return "bake v7 + PROT + PACK sekcji v12 (solo + compact inst)"; } diff --git a/eu07-parser/include/eu07/scene/binary/runtime_module.hpp b/eu07-parser/include/eu07/scene/binary/runtime_module.hpp index fd1a4efe..2d093fe4 100644 --- a/eu07-parser/include/eu07/scene/binary/runtime_module.hpp +++ b/eu07-parser/include/eu07/scene/binary/runtime_module.hpp @@ -51,11 +51,15 @@ inline constexpr std::array kChunkPidx{'P', 'I', 'D', 'X'}; inline constexpr std::array kChunkPack{'P', 'A', 'C', 'K'}; inline constexpr std::array kChunkProt{'P', 'R', 'O', 'T'}; -// PACK section payload: v9 = UMES, v10 = UMES + CHNK, v11 = UMES + UTEX + CHNK, v12 = v11 + PROT inst. +// PACK section payload: v9 = UMES, v10 = UMES + CHNK, v11 = UMES + UTEX + CHNK, v12 = v11 + PROT inst, v13 = v12 + mesh/tex indices + cell_id. inline constexpr std::uint8_t kPackSectionFormatV9 = 2; inline constexpr std::uint8_t kPackSectionFormatV10 = 3; inline constexpr std::uint8_t kPackSectionFormatV11 = 4; inline constexpr std::uint8_t kPackSectionFormatV12 = 5; +inline constexpr std::uint8_t kPackSectionFormatV13 = 6; +inline constexpr std::uint16_t kPackIndexEmpty = 0xFFFF; +inline constexpr std::int32_t kEu07CellSize = 250; +inline constexpr std::int32_t kEu07CellsPerSection = 4; inline constexpr std::size_t kPackSectionChunkModels = 512; inline constexpr std::size_t kPackSectionChunkModelsHeavy = 256; inline constexpr std::size_t kPackSectionHeavyModelThreshold = 1024; @@ -434,6 +438,90 @@ struct PackPayloadBuild { return true; } +[[nodiscard]] inline std::string packModelTextureDir(std::string model_file) { + if (model_file.empty() || model_file == "notload") { + return {}; + } + for (char& ch : model_file) { + if (ch == '\\') { + ch = '/'; + } + } + const auto dot = model_file.rfind('.'); + if (dot != std::string::npos) { + model_file.resize(dot); + } + const auto slash_pos = model_file.rfind('/'); + if (slash_pos == std::string::npos) { + return {}; + } + return model_file.substr(0, slash_pos + 1); +} + +[[nodiscard]] inline bool packTexturePathIsRooted(const std::string_view path) { + return path.starts_with("dynamic/") || path.starts_with("textures/") || + path.starts_with("scenery/") || path.starts_with("models/"); +} + +[[nodiscard]] inline bool probePackTextureCandidate(const std::string& candidate) { + static constexpr std::array kExtensions{ + ".mat", ".dds", ".tga", ".ktx", ".png", ".bmp", ".jpg", ".tex"}; + for (const char* ext : kExtensions) { + const std::filesystem::path path(candidate + ext); + if (std::filesystem::exists(path)) { + return true; + } + } + return false; +} + +[[nodiscard]] inline std::string resolvePackTexturePath( + const std::string& model_file, + const std::string& texture_file) { + if (!isPackTexturePath(texture_file)) { + return {}; + } + + std::string normalized = texture_file; + for (char& ch : normalized) { + if (ch == '\\') { + ch = '/'; + } + } + + std::vector candidates; + candidates.reserve(4); + auto append_candidate = [&](std::string candidate) { + if (candidate.empty()) { + return; + } + for (char& ch : candidate) { + if (ch == '\\') { + ch = '/'; + } + } + if (std::find(candidates.begin(), candidates.end(), candidate) == candidates.end()) { + candidates.emplace_back(std::move(candidate)); + } + }; + + append_candidate(normalized); + const auto model_dir = packModelTextureDir(model_file); + if (!model_dir.empty()) { + append_candidate(model_dir + normalized); + } + if (!packTexturePathIsRooted(normalized)) { + append_candidate(std::string("textures/") + normalized); + } + + for (const std::string& candidate : candidates) { + if (probePackTextureCandidate(candidate)) { + return candidate; + } + } + return {}; +} + [[nodiscard]] inline runtime::Vec3 packSectionCenter( const codec::TerrSectionKey section) noexcept { runtime::Vec3 center{}; @@ -448,6 +536,21 @@ struct PackPayloadBuild { return center; } +[[nodiscard]] inline std::uint8_t computePackCellId( + const runtime::Vec3& location, + const codec::TerrSectionKey section) noexcept { + const runtime::Vec3 center = packSectionCenter(section); + const int column = static_cast(std::floor( + (location.x - (center.x - static_cast(codec::kEu07SectionSize) / 2.0)) / + static_cast(kEu07CellSize))); + const int row = static_cast(std::floor( + (location.z - (center.z - static_cast(codec::kEu07SectionSize) / 2.0)) / + static_cast(kEu07CellSize))); + const int clamped_col = std::clamp(column, 0, kEu07CellsPerSection - 1); + const int clamped_row = std::clamp(row, 0, kEu07CellsPerSection - 1); + return static_cast(clamped_row * kEu07CellsPerSection + clamped_col); +} + inline void sortPackSectionModelsForStreaming( std::vector& models, const codec::TerrSectionKey section) { @@ -579,6 +682,101 @@ inline void sortPackSectionModelsForStreaming( return ids; } +[[nodiscard]] inline std::vector collectUniqueMeshPaths( + const std::vector& models, + const std::vector* prototype_mesh_paths = nullptr) { + std::unordered_map mesh_frequency; + mesh_frequency.reserve(models.size()); + + for (const runtime::RuntimeModelInstance& model : models) { + if (isLoadableMeshPath(model.modelFile)) { + ++mesh_frequency[model.modelFile]; + } + } + + std::vector paths; + paths.reserve(mesh_frequency.size()); + for (const auto& [path, frequency] : mesh_frequency) { + if (frequency > 0) { + paths.emplace_back(path); + } + } + if (prototype_mesh_paths != nullptr) { + for (const std::string_view path : *prototype_mesh_paths) { + if (isLoadableMeshPath(path)) { + const std::string text(path); + if (!mesh_frequency.contains(text)) { + paths.emplace_back(text); + mesh_frequency.emplace(text, 0u); + } + } + } + } + + std::stable_sort( + paths.begin(), + paths.end(), + [&](const std::string& lhs, const std::string& rhs) { + const std::uint32_t lhs_rank = mesh_frequency[lhs]; + const std::uint32_t rhs_rank = mesh_frequency[rhs]; + if (lhs_rank != rhs_rank) { + return lhs_rank > rhs_rank; + } + return lhs < rhs; + }); + return paths; +} + +[[nodiscard]] inline std::vector collectUniqueTexturePaths( + const std::vector& models) { + std::unordered_map texture_frequency; + texture_frequency.reserve(models.size()); + + for (const runtime::RuntimeModelInstance& model : models) { + if (isPackTexturePath(model.textureFile)) { + ++texture_frequency[model.textureFile]; + } + } + + std::vector paths; + paths.reserve(texture_frequency.size()); + for (const auto& [path, frequency] : texture_frequency) { + if (frequency > 0) { + paths.emplace_back(path); + } + } + + std::stable_sort( + paths.begin(), + paths.end(), + [&](const std::string& lhs, const std::string& rhs) { + const std::uint32_t lhs_rank = texture_frequency[lhs]; + const std::uint32_t rhs_rank = texture_frequency[rhs]; + if (lhs_rank != rhs_rank) { + return lhs_rank > rhs_rank; + } + return lhs < rhs; + }); + return paths; +} + +[[nodiscard]] inline std::unordered_map makePackPathIndexMap( + const std::vector& paths) { + std::unordered_map index_by_path; + index_by_path.reserve(paths.size()); + for (std::size_t index = 0; index < paths.size(); ++index) { + index_by_path.emplace(paths[index], static_cast(index)); + } + return index_by_path; +} + +[[nodiscard]] inline std::uint16_t lookupPackPathIndex( + const std::unordered_map& index_by_path, + const std::string& path) { + const auto it = index_by_path.find(path); + return it == index_by_path.end() ? kPackIndexEmpty : it->second; +} + [[nodiscard]] inline std::vector buildPackModelBlob( StringTable& table, const std::vector& models, @@ -778,7 +976,8 @@ struct PackPrototypeKeyHash { inline void writeRuntimePrototype( std::ostream& out, StringTable& table, - const runtime::RuntimeModelInstance& model) { + const runtime::RuntimeModelInstance& model, + const bool emit_v9_fields = false) { codec::writeSlimNode(out, table, model.node, "model"); io::writeU8(out, model.isTerrain ? 1 : 0); io::writeU8(out, model.transition ? 1 : 0); @@ -792,15 +991,48 @@ inline void writeRuntimePrototype( for (std::uint32_t color : model.lightColors) { io::writeU32(out, color); } + if (emit_v9_fields) { + const std::string resolved = + resolvePackTexturePath(model.modelFile, model.textureFile); + codec::writeStringId(out, table, resolved); + const bool needs_full_load = + !model.lightStates.empty() || !model.lightColors.empty() || !model.transition; + std::uint8_t pack_flags = 0; + if (needs_full_load) { + pack_flags |= 1u; + } else if (!model.isTerrain) { + pack_flags |= 2u; + } + io::writeU8(out, pack_flags); + io::writeU32(out, 0u); + const float baked_range_min = + static_cast(std::sqrt(std::max(0.0, model.node.rangeSquaredMin))); + const float baked_range_max = + model.node.rangeSquaredMax >= std::numeric_limits::max() * 0.5 + ? -1.f + : static_cast(std::sqrt(model.node.rangeSquaredMax)); + io::writeF32(out, baked_range_min); + io::writeF32(out, baked_range_max); + } } [[nodiscard]] inline std::vector buildProtPayload( StringTable& table, - const std::vector& prototypes) { + const std::vector& prototypes, + const bool emit_v9_fields = false) { + if (emit_v9_fields) { + for (const runtime::RuntimeModelInstance& proto : prototypes) { + const std::string resolved = + resolvePackTexturePath(proto.modelFile, proto.textureFile); + if (!resolved.empty()) { + table.intern(resolved); + } + } + } std::ostringstream out(std::ios::binary); io::writeU32(out, static_cast(prototypes.size())); for (const runtime::RuntimeModelInstance& proto : prototypes) { - writeRuntimePrototype(out, table, proto); + writeRuntimePrototype(out, table, proto, emit_v9_fields); } const std::string blob = out.str(); return {blob.begin(), blob.end()}; @@ -861,18 +1093,47 @@ struct PackSectionEntry { PackSectionEntryKind kind = PackSectionEntryKind::Solo; std::size_t model_index = 0; std::uint32_t proto_id = 0; + std::uint8_t pack_cell_id = 0; }; +inline void writePackSoloRecordV13( + std::ostream& out, + StringTable& table, + const runtime::RuntimeModelInstance& model, + const std::unordered_map& mesh_index, + const std::unordered_map& texture_index, + const std::uint8_t pack_cell_id) { + codec::writeSlimNode(out, table, model.node, "model"); + io::writeU8(out, model.isTerrain ? 1 : 0); + io::writeU8(out, model.transition ? 1 : 0); + io::writeVec3(out, model.location); + io::writeVec3(out, model.angles); + io::writeVec3(out, model.scale); + io::writeU16(out, lookupPackPathIndex(mesh_index, model.modelFile)); + io::writeU16(out, lookupPackPathIndex(texture_index, model.textureFile)); + io::writeU32(out, static_cast(model.lightStates.size())); + for (float light : model.lightStates) { + io::writeF32(out, light); + } + io::writeU32(out, static_cast(model.lightColors.size())); + for (std::uint32_t color : model.lightColors) { + io::writeU32(out, color); + } + io::writeU8(out, pack_cell_id); +} + inline void writePackInstRecord( std::ostream& out, StringTable& table, const std::uint32_t proto_id, - const runtime::RuntimeModelInstance& model) { + const runtime::RuntimeModelInstance& model, + const std::uint8_t pack_cell_id = 0) { io::writeU32(out, proto_id); io::writeVec3(out, model.location); io::writeVec3(out, model.angles); io::writeVec3(out, model.scale); codec::writeStringId(out, table, model.node.name); + io::writeU8(out, pack_cell_id); } [[nodiscard]] inline std::vector buildPackSectionChunkBlobV12( @@ -1009,6 +1270,162 @@ inline void writePackInstRecord( return {blob.begin(), blob.end()}; } +[[nodiscard]] inline std::vector buildPackSectionChunkBlobV13( + StringTable& table, + const std::vector& models, + const std::vector& entries, + const std::size_t begin, + const std::size_t end, + const std::unordered_map& mesh_index, + const std::unordered_map& texture_index) { + std::ostringstream out(std::ios::binary); + for (std::size_t index = begin; index < end; ++index) { + const PackSectionEntry& entry = entries[index]; + if (entry.kind != PackSectionEntryKind::Solo) { + continue; + } + runtime::RuntimeModelInstance worldModel = models[entry.model_index]; + worldModel.node.transform = {}; + writePackSoloRecordV13( + out, table, worldModel, mesh_index, texture_index, entry.pack_cell_id); + } + for (std::size_t index = begin; index < end; ++index) { + const PackSectionEntry& entry = entries[index]; + if (entry.kind != PackSectionEntryKind::Inst) { + continue; + } + writePackInstRecord( + out, table, entry.proto_id, models[entry.model_index], entry.pack_cell_id); + } + const std::string blob = out.str(); + return {blob.begin(), blob.end()}; +} + +[[nodiscard]] inline std::vector buildPackSectionPayloadV13( + StringTable& table, + const std::vector& models, + const PackPrototypeTable& prototypes, + const codec::TerrSectionKey section) { + std::vector entries; + entries.reserve(models.size()); + + std::uint32_t solo_count = 0; + std::uint32_t inst_count = 0; + for (std::size_t model_index = 0; model_index < models.size(); ++model_index) { + const runtime::RuntimeModelInstance& model = models[model_index]; + PackSectionEntry entry; + entry.model_index = model_index; + entry.pack_cell_id = computePackCellId(model.location, section); + if (modelMustBePackSolo(model)) { + entry.kind = PackSectionEntryKind::Solo; + ++solo_count; + entries.push_back(entry); + continue; + } + const PackPrototypeKey key = makePackPrototypeKey(model); + const auto proto_it = prototypes.key_to_id.find(key); + if (proto_it == prototypes.key_to_id.end()) { + entry.kind = PackSectionEntryKind::Solo; + ++solo_count; + entries.push_back(entry); + continue; + } + entry.kind = PackSectionEntryKind::Inst; + entry.proto_id = proto_it->second; + ++inst_count; + entries.push_back(entry); + } + + std::stable_sort( + entries.begin(), + entries.end(), + [](const PackSectionEntry& lhs, const PackSectionEntry& rhs) { + return lhs.pack_cell_id < rhs.pack_cell_id; + }); + + std::vector prototype_mesh_paths; + prototype_mesh_paths.reserve(prototypes.prototypes.size()); + for (const runtime::RuntimeModelInstance& proto : prototypes.prototypes) { + prototype_mesh_paths.emplace_back(proto.modelFile); + } + + const auto mesh_paths = collectUniqueMeshPaths(models, &prototype_mesh_paths); + const auto texture_paths = collectUniqueTexturePaths(models); + const auto mesh_ids = collectUniqueMeshIds(table, models, &prototype_mesh_paths); + const auto texture_ids = collectUniqueTextureIds(table, models); + const auto mesh_index = makePackPathIndexMap(mesh_paths); + const auto texture_index = makePackPathIndexMap(texture_paths); + const std::size_t chunk_models = + models.size() > kPackSectionHeavyModelThreshold ? kPackSectionChunkModelsHeavy + : kPackSectionChunkModels; + + std::vector> chunk_payloads; + std::vector chunk_solo_counts; + std::vector chunk_inst_counts; + chunk_payloads.reserve((entries.size() + chunk_models - 1) / chunk_models); + chunk_solo_counts.reserve(chunk_payloads.capacity()); + chunk_inst_counts.reserve(chunk_payloads.capacity()); + + for (std::size_t offset = 0; offset < entries.size(); offset += chunk_models) { + const std::size_t end = std::min(offset + chunk_models, entries.size()); + std::uint32_t chunk_solo = 0; + std::uint32_t chunk_inst = 0; + for (std::size_t index = offset; index < end; ++index) { + if (entries[index].kind == PackSectionEntryKind::Solo) { + ++chunk_solo; + } else { + ++chunk_inst; + } + } + chunk_solo_counts.push_back(chunk_solo); + chunk_inst_counts.push_back(chunk_inst); + chunk_payloads.push_back( + buildPackSectionChunkBlobV13( + table, models, entries, offset, end, mesh_index, texture_index)); + } + if (chunk_payloads.empty()) { + chunk_payloads.emplace_back(); + chunk_solo_counts.push_back(0); + chunk_inst_counts.push_back(0); + } + + const std::uint32_t chunk_count = static_cast(chunk_payloads.size()); + const std::uint32_t header_size = + 1u + 4u + 4u + 4u + static_cast(mesh_ids.size()) * 4u + 4u + + static_cast(texture_ids.size()) * 4u + 4u + chunk_count * 12u; + + std::vector chunk_offsets(chunk_count); + std::uint32_t payload_offset = header_size; + for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) { + chunk_offsets[chunk_index] = payload_offset; + payload_offset += static_cast(chunk_payloads[chunk_index].size()); + } + + std::ostringstream packOut(std::ios::binary); + io::writeU8(packOut, kPackSectionFormatV13); + io::writeU32(packOut, solo_count); + io::writeU32(packOut, inst_count); + io::writeU32(packOut, static_cast(mesh_ids.size())); + for (const std::uint32_t id : mesh_ids) { + io::writeU32(packOut, id); + } + io::writeU32(packOut, static_cast(texture_ids.size())); + for (const std::uint32_t id : texture_ids) { + io::writeU32(packOut, id); + } + io::writeU32(packOut, chunk_count); + for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) { + io::writeU32(packOut, chunk_solo_counts[chunk_index]); + io::writeU32(packOut, chunk_inst_counts[chunk_index]); + io::writeU32(packOut, chunk_offsets[chunk_index]); + } + for (const std::vector& payload : chunk_payloads) { + packOut.write(payload.data(), static_cast(payload.size())); + } + const std::string blob = packOut.str(); + return {blob.begin(), blob.end()}; +} + [[nodiscard]] inline PackPayloadBuild buildPackPayloadV12( StringTable& table, const std::vector& batches) { @@ -1024,7 +1441,7 @@ inline void writePackInstRecord( } const PackPrototypeTable prototypes = collectPackPrototypes(table, batches); - result.protPayload = buildProtPayload(table, prototypes.prototypes); + result.protPayload = buildProtPayload(table, prototypes.prototypes, true); result.entries.resize(batches.size()); std::vector> section_payloads(batches.size()); @@ -1043,7 +1460,7 @@ inline void writePackInstRecord( std::vector section_models = batch.models; sortPackSectionModelsForStreaming(section_models, batch.section); section_payloads[batch_index] = - buildPackSectionPayloadV12(table, section_models, prototypes); + buildPackSectionPayloadV13(table, section_models, prototypes, batch.section); } std::size_t total_size = 0; for (const std::vector& payload : section_payloads) { @@ -1080,7 +1497,7 @@ inline void writePackInstRecord( std::vector section_models = batch.models; sortPackSectionModelsForStreaming(section_models, batch.section); section_payloads[batch_index] = - buildPackSectionPayloadV12(table, section_models, prototypes); + buildPackSectionPayloadV13(table, section_models, prototypes, batch.section); } }); } @@ -1469,7 +1886,7 @@ inline void writeRuntimeModule( } out.write(kMagic.data(), 4); - io::writeU32(out, usePackModels ? kVersionRuntimeV8 : kVersionRuntime); + io::writeU32(out, usePackModels ? kVersionRuntimeV9 : kVersionRuntime); io::writeChunkHeader(out, kChunkStrs, 8 + static_cast(strsPayload.size())); out.write(strsPayload.data(), static_cast(strsPayload.size())); diff --git a/model/AnimModel.cpp b/model/AnimModel.cpp index 41291c76..10ce59e2 100644 --- a/model/AnimModel.cpp +++ b/model/AnimModel.cpp @@ -14,7 +14,9 @@ http://mozilla.org/MPL/2.0/. #include "stdafx.h" #include "model/AnimModel.h" +#include "scene/eu7/eu7_model_prefetch.h" +#include #include #include "scene/eu7/eu7_pack_bench.h" @@ -390,6 +392,7 @@ TAnimModel::reset_pack_for_insert( scene::node_data const &nodedata ) { pModel = nullptr; m_eu7_pack = false; m_instanceable = false; + m_pack_cell_id = 255; m_materialdata = {}; asText.clear(); } @@ -612,10 +615,12 @@ namespace eu7_pack_cache { enum class slot_t : std::int8_t { unknown = -1, no = 0, yes = 1 }; +std::mutex g_mutex; std::unordered_map g_model; [[nodiscard]] bool lookup( TModel3d *const model, bool &instanceable ) { + std::lock_guard lock { g_mutex }; auto const it { g_model.find( model ) }; if( it == g_model.end() || it->second == slot_t::unknown ) { return false; @@ -626,6 +631,7 @@ lookup( TModel3d *const model, bool &instanceable ) { void store( TModel3d *const model, bool const instanceable ) { + std::lock_guard lock { g_mutex }; g_model[ model ] = instanceable ? slot_t::yes : slot_t::no; } @@ -634,11 +640,13 @@ warm( TModel3d *const model ) { if( model == nullptr ) { return; } + std::lock_guard lock { g_mutex }; auto const it { g_model.find( model ) }; if( it != g_model.end() && it->second != slot_t::unknown ) { return; } - store( model, false == submodel_tree_blocks_instancing( model->Root ) ); + g_model[ model ] = + false == submodel_tree_blocks_instancing( model->Root ) ? slot_t::yes : slot_t::no; } } // namespace eu7_pack_cache @@ -659,7 +667,11 @@ TAnimModel::warm_instanceable_cache( TModel3d *const model ) { bool TAnimModel::LoadEu7PackWarm( TModel3d *const mesh, - std::string const &texture_file ) { + std::string const &texture_file, + std::string const &model_file, + std::string const &resolved_texture, + std::uint32_t const textures_alpha, + bool const instanceable_hint ) { pModel = mesh; if( mesh == nullptr ) { return true; @@ -680,16 +692,27 @@ TAnimModel::LoadEu7PackWarm( false == normalized.ends_with( '/' ) && normalized.find( "tr/none" ) == std::string::npos && normalized.find( '#' ) == std::string::npos ) { - m_materialdata.assign( normalized ); - if( m_materialdata.replacable_skins[ 1 ] == null_handle ) { - scene::eu7::pack_bench_inc( &scene::eu7::Eu7PackBench::main_texture_assign_fail ); - scene::eu7::log_pack_texture_fail( normalized ); - } + (void)scene::eu7::assign_pack_texture( + m_materialdata, + model_file, + normalized, + resolved_texture, + textures_alpha ); } } + else if( textures_alpha != 0 ) { + m_materialdata.textures_alpha = static_cast( textures_alpha ); + } m_eu7_pack = true; + if( instanceable_hint ) { + ++s_classified_total; + m_instanceable = true; + ++s_instanceable_total; + return true; + } + bool cached_instanceable { false }; if( anim_instancing_detail::eu7_pack_cache::lookup( mesh, cached_instanceable ) ) { ++s_classified_total; diff --git a/model/AnimModel.h b/model/AnimModel.h index 4ae624de..a545514a 100644 --- a/model/AnimModel.h +++ b/model/AnimModel.h @@ -135,7 +135,11 @@ public: // PACK stream: mesh pointer already resolved; skips GetModel lookup. bool LoadEu7PackWarm( TModel3d *Mesh, - std::string const &texture_file ); + std::string const &texture_file, + std::string const &model_file = {}, + std::string const &resolved_texture = {}, + std::uint32_t textures_alpha = 0, + bool instanceable_hint = false ); void reset_pack_for_insert( scene::node_data const &Nodedata ); static TAnimModel * @@ -243,6 +247,7 @@ public: // computed once at end of Load() so the renderer can simply test the flag. bool m_instanceable { false }; bool m_eu7_pack { false }; + std::uint8_t m_pack_cell_id { 255 }; // helper: evaluates current state and updates m_instanceable accordingly. void update_instanceable_flag(); diff --git a/model/MdlMngr.cpp b/model/MdlMngr.cpp index 3b692b28..3540304e 100644 --- a/model/MdlMngr.cpp +++ b/model/MdlMngr.cpp @@ -21,6 +21,8 @@ http://mozilla.org/MPL/2.0/. #include "utilities/Logs.h" #include "utilities/utilities.h" +#include + // wczytanie modelu do kontenerka TModel3d * TMdlContainer::LoadModel(std::string const &Name, bool const Dynamic) { @@ -132,6 +134,53 @@ TModelsManager::IsModelCached( std::string const &Name ) { return find_in_databank( filename ).first; } +[[nodiscard]] std::string +normalize_pack_model_key( std::string name ) { + erase_extension( name ); + return ToLower( name ); +} + +std::pair +TModelsManager::TryGetCachedModel( std::string const &Name ) { + return find_in_databank( normalize_pack_model_key( Name ) ); +} + +std::string +TModelsManager::ResolveModelDiskPath( std::string Name ) { + return find_on_disk( normalize_pack_model_key( std::move( Name ) ) ); +} + +std::shared_ptr +TModelsManager::LoadModelCpuDeferred( + std::string const &DiskPath, + bool const Dynamic ) { + auto model { std::make_shared() }; + if( false == model->LoadFromFile( DiskPath, Dynamic, true ) ) { + return nullptr; + } + return model; +} + +TModel3d * +TModelsManager::PublishLoadedModel( + std::string const &VirtualName, + std::shared_ptr Model ) { + if( Model == nullptr ) { + return nullptr; + } + + auto const lookup { find_in_databank( VirtualName ) }; + if( lookup.first && lookup.second != nullptr ) { + return lookup.second; + } + + Model->Init(); + m_models.emplace_back(); + m_models.back().Model = std::move( Model ); + m_modelsmap.emplace( VirtualName, m_models.size() - 1 ); + return m_models.back().Model.get(); +} + std::pair TModelsManager::find_in_databank( std::string const &Name ) { diff --git a/model/MdlMngr.h b/model/MdlMngr.h index 03dbe169..ee39684e 100644 --- a/model/MdlMngr.h +++ b/model/MdlMngr.h @@ -10,6 +10,8 @@ http://mozilla.org/MPL/2.0/. #include "utilities/Classes.h" +#include + class TMdlContainer { friend class TModelsManager; private: @@ -24,6 +26,17 @@ public: // McZapkie: dodalem sciezke, notabene Path!=Patch :) static TModel3d *GetModel(std::string const &Name, bool const dynamic = false, bool const Logerrors = true , int uid = 0); [[nodiscard]] static bool IsModelCached( std::string const &Name ); + // EU7 PACK stream: CPU parse on workers, GPU Init on main thread. + [[nodiscard]] static std::pair + TryGetCachedModel( std::string const &Name ); + [[nodiscard]] static std::string + ResolveModelDiskPath( std::string Name ); + [[nodiscard]] static std::shared_ptr + LoadModelCpuDeferred( std::string const &DiskPath, bool const Dynamic = false ); + [[nodiscard]] static TModel3d * + PublishLoadedModel( + std::string const &VirtualName, + std::shared_ptr Model ); private: // types: diff --git a/model/Model3d.cpp b/model/Model3d.cpp index 32c757cd..781bb50b 100644 --- a/model/Model3d.cpp +++ b/model/Model3d.cpp @@ -1623,8 +1623,10 @@ glm::vec3 TSubModel::offset(float const Geometrytestoffsetthreshold) const return offset; } -bool TModel3d::LoadFromFile(std::string const &FileName, bool dynamic) -{ +bool TModel3d::LoadFromFile( + std::string const &FileName, + bool const dynamic, + bool const defer_gpu_init ) { // wczytanie modelu z pliku /* // NOTE: disabled, this work is now done by the model manager @@ -1646,20 +1648,24 @@ bool TModel3d::LoadFromFile(std::string const &FileName, bool dynamic) { WriteLog("Forced loading text model \"" + name + ".t3d\""); LoadFromTextFile(name + ".t3d", dynamic); - if (!dynamic) + if( false == defer_gpu_init && false == dynamic ) { Init(); + } } else if (FileExists(asBinary)) { LoadFromBinFile(asBinary, dynamic); asBinary = ""; - Init(); + if( false == defer_gpu_init ) { + Init(); + } } else if (FileExists(name + ".t3d")) { LoadFromTextFile(name + ".t3d", dynamic); - if (!dynamic) + if( false == defer_gpu_init && false == dynamic ) { Init(); + } } bool const result = Root ? (iSubModelsCount > 0) : false; // brak pliku albo problem z wczytaniem diff --git a/model/Model3d.h b/model/Model3d.h index 87304335..19d28114 100644 --- a/model/Model3d.h +++ b/model/Model3d.h @@ -289,7 +289,10 @@ public: void AddTo(TSubModel *tmp, TSubModel *SubModel); void LoadFromTextFile(std::string const &FileName, bool dynamic); void LoadFromBinFile(std::string const &FileName, bool dynamic); - bool LoadFromFile(std::string const &FileName, bool dynamic); + bool LoadFromFile( + std::string const &FileName, + bool dynamic, + bool defer_gpu_init = false ); TSubModel *AppendChildFromGeometry(const std::string &name, const std::string &parent, const gfx::vertex_array &vertices, const gfx::index_array &indices); void SaveToBinFile(std::string const &FileName); uint32_t Flags() const { return iFlags; }; diff --git a/rendering/opengl33renderer.cpp b/rendering/opengl33renderer.cpp index 8b112d6b..8475a905 100644 --- a/rendering/opengl33renderer.cpp +++ b/rendering/opengl33renderer.cpp @@ -18,6 +18,7 @@ http://mozilla.org/MPL/2.0/. #include "utilities/Logs.h" #include "simulation/simulationtime.h" #include "application/application.h" +#include "scene/eu7/eu7_section_stream.h" #include "model/AnimModel.h" #include "rendering/opengl33geometrybank.h" #include "rendering/screenshot.h" @@ -733,8 +734,11 @@ void opengl33_renderer::Render_pass(viewport_config &vp, rendermode const Mode) m_current_viewport = &vp; } - if ((!simulation::is_ready) || (Global.gfx_skiprendering)) - { + if( + ( !simulation::is_ready ) || ( Global.gfx_skiprendering ) || + scene::eu7::loading_screen_blocks_world( + scene::eu7::stream_loading_position(), + scene::eu7::kSectionStreamLoadingDismissRadiusKm ) ) { gl::framebuffer::unbind(); glClearColor(0.0f, 0.0f, 0.0f, 1.0f); glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); diff --git a/rendering/openglrenderer.cpp b/rendering/openglrenderer.cpp index 868e5122..5d506a33 100644 --- a/rendering/openglrenderer.cpp +++ b/rendering/openglrenderer.cpp @@ -21,6 +21,7 @@ http://mozilla.org/MPL/2.0/. #include "model/AnimModel.h" #include "world/Traction.h" #include "application/application.h" +#include "scene/eu7/eu7_section_stream.h" #include "utilities/Logs.h" #include "rendering/openglgeometrybank.h" #include "rendering/openglcolor.h" @@ -414,7 +415,11 @@ opengl_renderer::Render_pass( rendermode const Mode ) { m_colorpass = m_renderpass; - if( ( !simulation::is_ready ) || ( Global.gfx_skiprendering ) ) { + if( + ( !simulation::is_ready ) || ( Global.gfx_skiprendering ) || + scene::eu7::loading_screen_blocks_world( + scene::eu7::stream_loading_position(), + scene::eu7::kSectionStreamLoadingDismissRadiusKm ) ) { ::glClearColor( 51.0f / 255.f, 102.0f / 255.f, 85.0f / 255.f, 1.f ); // initial background Color ::glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT ); break; diff --git a/scene/eu7/eu7_chunks.h b/scene/eu7/eu7_chunks.h index 7da420cd..dee682e7 100644 --- a/scene/eu7/eu7_chunks.h +++ b/scene/eu7/eu7_chunks.h @@ -31,6 +31,7 @@ constexpr std::uint32_t kEu7VersionV5 { 5 }; constexpr std::uint32_t kEu7VersionV6 { 6 }; constexpr std::uint32_t kEu7VersionV7 { 7 }; constexpr std::uint32_t kEu7VersionV8 { 8 }; +constexpr std::uint32_t kEu7VersionV9 { 9 }; constexpr std::uint32_t kChunkStrs { detail::make_id4( 'S', 'T', 'R', 'S' ) }; constexpr std::uint32_t kChunkIncl { detail::make_id4( 'I', 'N', 'C', 'L' ) }; @@ -62,6 +63,7 @@ constexpr std::uint8_t kPackSectionFormatV9 { 2 }; constexpr std::uint8_t kPackSectionFormatV10 { 3 }; constexpr std::uint8_t kPackSectionFormatV11 { 4 }; constexpr std::uint8_t kPackSectionFormatV12 { 5 }; +constexpr std::uint8_t kPackSectionFormatV13 { 6 }; constexpr std::size_t kPackSectionChunkModels { 512 }; } // namespace scene::eu7 diff --git a/scene/eu7/eu7_loader.cpp b/scene/eu7/eu7_loader.cpp index 63f98439..caecdf98 100644 --- a/scene/eu7/eu7_loader.cpp +++ b/scene/eu7/eu7_loader.cpp @@ -553,6 +553,7 @@ load_module( std::string const &path, simulation::state_serializer &serializer ) auto const resolved { resolve_scenery_path( path ) }; auto const ok { load_module_recursive( path, serializer, g_load_session ) }; if( ok ) { + log_load_stats(); if( auto const cached { g_module_file_cache.find( resolved ) }; cached != g_module_file_cache.end() && cached->second.has_pack_chunk ) { init_section_stream( cached->second, resolved, serializer ); diff --git a/scene/eu7/eu7_model_prefetch.cpp b/scene/eu7/eu7_model_prefetch.cpp index dd13264b..2d0dd7d8 100644 --- a/scene/eu7/eu7_model_prefetch.cpp +++ b/scene/eu7/eu7_model_prefetch.cpp @@ -11,12 +11,13 @@ http://mozilla.org/MPL/2.0/. #include "scene/eu7/eu7_model_prefetch.h" #include "model/AnimModel.h" -#include "model/MdlMngr.h" #include "rendering/renderer.h" #include "scene/eu7/eu7_pack_bench.h" +#include "utilities/Globals.h" #include "utilities/utilities.h" #include "vehicle/DynObj.h" +#include #include #include #include @@ -28,11 +29,29 @@ namespace scene::eu7 { namespace { constexpr std::size_t kMaxWarmedTextures { 2048 }; +constexpr std::size_t kMaxPackMaterialAssignCache { 4096 }; -std::mutex g_pack_mesh_load_mutex; std::unordered_set g_session_warmed_textures; std::list g_session_warmed_textures_lru; std::unordered_map::iterator> g_session_warmed_textures_iters; +std::unordered_map g_pack_material_assign_cache; + +[[nodiscard]] std::string +pack_material_cache_key( + std::string const &model_file, + std::string const &texture_file ) { + return model_file + '\x1e' + texture_file; +} + +void +evict_pack_material_assign_cache_if_needed() { + while( g_pack_material_assign_cache.size() >= kMaxPackMaterialAssignCache ) { + if( g_pack_material_assign_cache.empty() ) { + break; + } + g_pack_material_assign_cache.erase( g_pack_material_assign_cache.begin() ); + } +} void touch_warmed_texture_lru( std::string const &texture_file ) { @@ -107,12 +126,119 @@ pack_texture_usable( std::string texture_file ) { void preload_pack_model_file( std::string const & ) { - // Mesh warm runs on the main thread via ensure_pack_mesh_in_session_cache only. - // GetModel mutates Global.asCurrentTexturePath and must not run on PACK workers. + // GetModel + Init must run on the main thread (Global.asCurrentTexturePath + OpenGL). } [[nodiscard]] bool -warm_one_pack_texture( std::string texture_file, std::unordered_set &seen ) { +pack_texture_path_is_rooted( std::string const &texture_file ) { + return texture_file.starts_with( "dynamic/" ) || + texture_file.starts_with( "textures/" ) || + texture_file.starts_with( "scenery/" ) || + texture_file.starts_with( "models/" ); +} + +[[nodiscard]] std::string +pack_model_texture_dir( std::string model_file ) { + if( model_file.empty() || model_file == "notload" ) { + return {}; + } + replace_slashes( model_file ); + erase_extension( model_file ); + auto const slash_pos { model_file.rfind( '/' ) }; + if( slash_pos == std::string::npos ) { + return {}; + } + return model_file.substr( 0, slash_pos + 1 ); +} + +void +append_pack_texture_candidate( + std::vector &candidates, + std::string candidate ) { + if( candidate.empty() ) { + return; + } + replace_slashes( candidate ); + if( + std::find( candidates.begin(), candidates.end(), candidate ) == + candidates.end() ) { + candidates.emplace_back( std::move( candidate ) ); + } +} + +[[nodiscard]] std::vector +pack_texture_resolve_candidates( + std::string const &model_file, + std::string texture_file ) { + std::vector candidates; + candidates.reserve( 4 ); + if( false == pack_texture_usable( texture_file ) ) { + return candidates; + } + replace_slashes( texture_file ); + append_pack_texture_candidate( candidates, texture_file ); + + auto const model_dir { pack_model_texture_dir( model_file ) }; + if( false == model_dir.empty() ) { + append_pack_texture_candidate( candidates, model_dir + texture_file ); + } + if( false == pack_texture_path_is_rooted( texture_file ) ) { + append_pack_texture_candidate( candidates, paths::textures + texture_file ); + } + return candidates; +} + +class PackTexturePathScope { +public: + explicit PackTexturePathScope( std::string const &model_file ) { + m_saved_path = Global.asCurrentTexturePath; + auto const model_dir { pack_model_texture_dir( model_file ) }; + if( false == model_dir.empty() && model_file.find( '/' ) != std::string::npos ) { + Global.asCurrentTexturePath = model_dir; + m_active = true; + } + } + + ~PackTexturePathScope() { + if( m_active ) { + Global.asCurrentTexturePath = std::move( m_saved_path ); + } + } + + PackTexturePathScope( PackTexturePathScope const & ) = delete; + PackTexturePathScope &operator=( PackTexturePathScope const & ) = delete; + +private: + std::string m_saved_path; + bool m_active { false }; +}; + +[[nodiscard]] bool +fetch_pack_texture_material( std::string const &texture_file ) { + auto const resolved { TextureTest( ToLower( texture_file ) ) }; + if( false == resolved.empty() ) { + GfxRenderer->Fetch_Material( resolved ); + return true; + } + + bool warmed { false }; + for( int skinindex { 1 }; skinindex <= 4; ++skinindex ) { + auto const multi { + TextureTest( ToLower( texture_file + "," + std::to_string( skinindex ) ) ) }; + if( multi.empty() ) { + break; + } + GfxRenderer->Fetch_Material( multi ); + warmed = true; + } + return warmed; +} + +[[nodiscard]] bool +warm_one_pack_texture( + std::string texture_file, + std::unordered_set &seen, + std::string const &model_file = {} ) { if( false == pack_texture_usable( texture_file ) ) { return false; } @@ -124,77 +250,40 @@ warm_one_pack_texture( std::string texture_file, std::unordered_set return false; } - auto const resolved { TextureTest( ToLower( texture_file ) ) }; - if( resolved.empty() ) { - bool warmed { false }; - for( int skinindex { 1 }; skinindex <= 4; ++skinindex ) { - auto const multi { - TextureTest( ToLower( texture_file + "," + std::to_string( skinindex ) ) ) }; - if( multi.empty() ) { - break; - } - GfxRenderer->Fetch_Material( multi ); - warmed = true; + PackTexturePathScope const scope { model_file }; + for( auto const &candidate : pack_texture_resolve_candidates( model_file, texture_file ) ) { + if( fetch_pack_texture_material( candidate ) ) { + remember_warmed_texture( texture_file ); + return true; } - if( false == warmed ) { - pack_bench_inc( &Eu7PackBench::main_texture_warm_miss ); - log_pack_texture_fail( texture_file ); - return false; - } - } - else { - GfxRenderer->Fetch_Material( resolved ); } - remember_warmed_texture( texture_file ); + pack_bench_inc( &Eu7PackBench::main_texture_warm_miss ); + log_pack_texture_fail( texture_file ); + return false; +} - if( false == resolved.empty() ) { - for( int skinindex { 1 }; skinindex <= 4; ++skinindex ) { - auto const multi { - TextureTest( ToLower( texture_file + "," + std::to_string( skinindex ) ) ) }; - if( multi.empty() ) { - break; - } - GfxRenderer->Fetch_Material( multi ); +[[nodiscard]] std::string +find_pack_model_file_for_texture( + Eu7Model const *const models, + std::size_t const model_count, + std::string texture_file ) { + if( models == nullptr || model_count == 0 ) { + return {}; + } + replace_slashes( texture_file ); + for( std::size_t i { 0 }; i < model_count; ++i ) { + auto candidate { models[ i ].texture_file }; + replace_slashes( candidate ); + if( candidate == texture_file ) { + return models[ i ].model_file; } } - return true; + return {}; } } // namespace -[[nodiscard]] bool -ensure_pack_mesh_in_session_cache( - std::string model_file, - std::unordered_map &session_cache ) { - if( model_file.empty() || model_file == "notload" ) { - return false; - } - replace_slashes( model_file ); - if( session_cache.contains( model_file ) ) { - pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit ); - return true; - } - - auto const was_global_cached { TModelsManager::IsModelCached( model_file ) }; - TModel3d *mesh { nullptr }; - { - std::lock_guard lock { g_pack_mesh_load_mutex }; - mesh = TModelsManager::GetModel( model_file, false, false ); - } - if( was_global_cached ) { - pack_bench_inc( &Eu7PackBench::stream_mesh_global_hit ); - } - else { - pack_bench_inc( &Eu7PackBench::stream_mesh_disk_load ); - } - if( mesh != nullptr ) { - TAnimModel::warm_instanceable_cache( mesh ); - } - session_cache.emplace( model_file, mesh ); - return true; -} - void reset_pack_texture_warm_cache() { g_session_warmed_textures.clear(); @@ -221,7 +310,9 @@ warm_pack_texture_paths_main( std::string const *const paths, std::size_t const count, double const budget_ms, - std::size_t *const processed_out ) { + std::size_t *const processed_out, + Eu7Model const *const models, + std::size_t const model_count ) { if( paths == nullptr || count == 0 || GfxRenderer == nullptr ) { if( processed_out != nullptr ) { *processed_out = 0; @@ -246,7 +337,10 @@ warm_pack_texture_paths_main( break; } } - if( warm_one_pack_texture( paths[ i ], seen ) ) { + if( warm_one_pack_texture( + paths[ i ], + seen, + find_pack_model_file_for_texture( models, model_count, paths[ i ] ) ) ) { ++warmed; } ++processed; @@ -259,7 +353,10 @@ warm_pack_texture_paths_main( } std::size_t -warm_pack_textures_main( Eu7Model const *const models, std::size_t const count ) { +warm_pack_textures_main( + Eu7Model const *const models, + std::size_t const count, + double const budget_ms ) { if( models == nullptr || count == 0 || GfxRenderer == nullptr ) { return 0; } @@ -267,13 +364,88 @@ warm_pack_textures_main( Eu7Model const *const models, std::size_t const count ) std::unordered_set seen; seen.reserve( std::min( count, std::size_t { 64 } ) ); std::size_t warmed { 0 }; + auto const started { std::chrono::steady_clock::now() }; for( std::size_t i { 0 }; i < count; ++i ) { - if( warm_one_pack_texture( models[ i ].texture_file, seen ) ) { + if( + budget_ms > 0.0 && + i > 0 ) { + auto const elapsed_ms { + std::chrono::duration( + std::chrono::steady_clock::now() - started ).count() }; + if( elapsed_ms >= budget_ms ) { + break; + } + } + if( warm_one_pack_texture( + models[ i ].texture_file, + seen, + models[ i ].model_file ) ) { ++warmed; } } return warmed; } +[[nodiscard]] bool +assign_pack_texture( + material_data &material, + std::string const &model_file, + std::string const &texture_file, + std::string const &resolved_texture, + std::uint32_t const textures_alpha ) { + if( false == pack_texture_usable( texture_file ) ) { + if( textures_alpha != 0 ) { + material.textures_alpha = static_cast( textures_alpha ); + } + return true; + } + + auto const cache_key { + resolved_texture.empty() ? + pack_material_cache_key( model_file, texture_file ) : + pack_material_cache_key( model_file, resolved_texture ) }; + auto const cached { g_pack_material_assign_cache.find( cache_key ) }; + if( + cached != g_pack_material_assign_cache.end() && + cached->second.replacable_skins[ 1 ] != null_handle ) { + material = cached->second; + if( textures_alpha != 0 ) { + material.textures_alpha = static_cast( textures_alpha ); + } + return true; + } + + if( false == resolved_texture.empty() ) { + material = {}; + material.assign( resolved_texture ); + if( material.replacable_skins[ 1 ] != null_handle ) { + if( textures_alpha != 0 ) { + material.textures_alpha = static_cast( textures_alpha ); + } + evict_pack_material_assign_cache_if_needed(); + g_pack_material_assign_cache.emplace( cache_key, material ); + return true; + } + } + + PackTexturePathScope const scope { model_file }; + for( auto const &candidate : pack_texture_resolve_candidates( model_file, texture_file ) ) { + material = {}; + material.assign( candidate ); + if( material.replacable_skins[ 1 ] != null_handle ) { + if( textures_alpha != 0 ) { + material.textures_alpha = static_cast( textures_alpha ); + } + evict_pack_material_assign_cache_if_needed(); + g_pack_material_assign_cache.emplace( cache_key, material ); + return true; + } + } + + pack_bench_inc( &Eu7PackBench::main_texture_assign_fail ); + log_pack_texture_fail( texture_file ); + return false; +} + } // namespace scene::eu7 diff --git a/scene/eu7/eu7_model_prefetch.h b/scene/eu7/eu7_model_prefetch.h index 1381e2b1..6b98c55a 100644 --- a/scene/eu7/eu7_model_prefetch.h +++ b/scene/eu7/eu7_model_prefetch.h @@ -9,19 +9,16 @@ http://mozilla.org/MPL/2.0/. #pragma once +#include "scene/eu7/eu7_pack_mesh_loader.h" #include "scene/eu7/eu7_types.h" +#include "vehicle/DynObj.h" #include #include -#include -#include #include -class TModel3d; - namespace scene::eu7 { -// Deprecated: mesh warm is main-thread only (ensure_pack_mesh_in_session_cache). void preload_pack_models( std::vector const &Models ); @@ -30,25 +27,30 @@ preload_pack_models( std::vector const &Models, std::vector const &UniqueMeshes ); -// Main thread: Fetch_Material dla unikalnych texture_file z chunka przed apply. -// Zwraca liczbe unikalnych Fetch_Material w tym slice. std::size_t warm_pack_texture_paths_main( std::string const *paths, std::size_t count, double budget_ms = 0.0, - std::size_t *processed_out = nullptr ); + std::size_t *processed_out = nullptr, + Eu7Model const *models = nullptr, + std::size_t model_count = 0 ); std::size_t -warm_pack_textures_main( Eu7Model const *models, std::size_t count ); +warm_pack_textures_main( + Eu7Model const *models, + std::size_t count, + double budget_ms = 0.0 ); void reset_pack_texture_warm_cache(); -// Thread-safe cold mesh load: session cache, then GetModel under g_pack_mesh_load_mutex. [[nodiscard]] bool -ensure_pack_mesh_in_session_cache( - std::string model_file, - std::unordered_map &session_cache ); +assign_pack_texture( + material_data &material, + std::string const &model_file, + std::string const &texture_file, + std::string const &resolved_texture = {}, + std::uint32_t textures_alpha = 0 ); } // namespace scene::eu7 diff --git a/scene/eu7/eu7_pack_bench.cpp b/scene/eu7/eu7_pack_bench.cpp index d065e58b..f3492ff0 100644 --- a/scene/eu7/eu7_pack_bench.cpp +++ b/scene/eu7/eu7_pack_bench.cpp @@ -8,6 +8,7 @@ http://mozilla.org/MPL/2.0/. */ #include "stdafx.h" +#include "scene/eu7/eu7_pack_mesh_loader.h" #include "scene/eu7/eu7_pack_bench.h" #include "utilities/Logs.h" @@ -181,6 +182,9 @@ log_pack_bench_impl( Eu7PackBench const &bench, char const *const title ) { " diag mesh: sess_hit=" + std::to_string( bench.stream_mesh_session_hit ) + " glob_hit=" + std::to_string( bench.stream_mesh_global_hit ) + " disk=" + std::to_string( bench.stream_mesh_disk_load ) + + " async_q=" + std::to_string( bench.stream_mesh_async_queued ) + + " async_r=" + std::to_string( bench.stream_mesh_async_ready ) + + " loader=" + std::to_string( bench.loader_thread_disk_loads ) + " dequeue_near=" + std::to_string( bench.stream_dequeue_near ) + " far=" + std::to_string( bench.stream_dequeue_far ) + " cam=" + std::to_string( bench.stream_dequeue_camera ) + @@ -189,7 +193,10 @@ log_pack_bench_impl( Eu7PackBench const &bench, char const *const title ) { " block_ring_in=" + std::to_string( bench.stream_jobs_blocked_ring_inner ) + " block_ring_out=" + std::to_string( bench.stream_jobs_blocked_ring_outer ) + " defer_dist=" + std::to_string( bench.stream_worker_deferred_distant ) + - " stuck_skip=" + std::to_string( bench.stream_apply_stuck_skip ) ); + " stuck_skip=" + std::to_string( bench.stream_apply_stuck_skip ) + + " apply_defer=" + std::to_string( bench.stream_apply_deferred ) + + " defer_bypass=" + std::to_string( bench.stream_apply_defer_bypass ) + + " sparse_skip=" + std::to_string( bench.stream_sparse_apply_skip ) ); WriteLog( " diag ready_tex_ms=" + std::to_string( static_cast( bench.stream_prefetch_ready_tex_ms ) ) + " ready_umes_ms=" + std::to_string( static_cast( bench.stream_prefetch_ready_umes_ms ) ) + @@ -442,6 +449,9 @@ log_pack_stream_status( " ahead=" + std::to_string( bench.stream_lookahead_enqueue ) + " urg_chunks=" + std::to_string( bench.stream_chunks_urgent ) + " cold_trunc=" + std::to_string( bench.stream_cold_slice_truncated ) + + " mesh_q=" + std::to_string( pack_mesh_loader_queue_depth() ) + + " mesh_r=" + std::to_string( pack_mesh_loader_ready_count() ) + + " mesh_d=" + std::to_string( bench.stream_mesh_async_drained ) + " preempt=" + std::to_string( bench.stream_preempt_pending ) ); } diff --git a/scene/eu7/eu7_pack_bench.h b/scene/eu7/eu7_pack_bench.h index 48643b13..85af2ba2 100644 --- a/scene/eu7/eu7_pack_bench.h +++ b/scene/eu7/eu7_pack_bench.h @@ -63,6 +63,12 @@ struct Eu7PackBench { std::uint64_t stream_mesh_session_hit { 0 }; std::uint64_t stream_mesh_global_hit { 0 }; std::uint64_t stream_mesh_disk_load { 0 }; + std::uint64_t stream_mesh_async_queued { 0 }; + std::uint64_t stream_mesh_async_ready { 0 }; + std::uint64_t stream_mesh_async_drained { 0 }; + std::uint64_t stream_mesh_async_wait_timeout { 0 }; + std::uint64_t loader_thread_disk_loads { 0 }; + double loader_thread_getmodel_ms { 0.0 }; std::uint64_t stream_chunks_urgent { 0 }; std::uint64_t stream_chunks_heavy { 0 }; std::uint64_t stream_chunks_light { 0 }; @@ -75,6 +81,9 @@ struct Eu7PackBench { std::uint64_t stream_jobs_blocked_ring_outer { 0 }; std::uint64_t stream_worker_deferred_distant { 0 }; std::uint64_t stream_apply_stuck_skip { 0 }; + std::uint64_t stream_apply_deferred { 0 }; + std::uint64_t stream_apply_defer_bypass { 0 }; + std::uint64_t stream_sparse_apply_skip { 0 }; std::uint64_t stream_sections_unloaded { 0 }; std::uint64_t stream_unload_instances { 0 }; std::uint64_t stream_mesh_cache_evictions { 0 }; diff --git a/scene/eu7/eu7_pack_mesh_loader.cpp b/scene/eu7/eu7_pack_mesh_loader.cpp new file mode 100644 index 00000000..b8866ed5 --- /dev/null +++ b/scene/eu7/eu7_pack_mesh_loader.cpp @@ -0,0 +1,501 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#include "stdafx.h" +#include "scene/eu7/eu7_pack_mesh_loader.h" + +#include "model/AnimModel.h" +#include "model/MdlMngr.h" +#include "scene/eu7/eu7_pack_bench.h" +#include "utilities/Globals.h" +#include "utilities/utilities.h" + +#include +#include +#include +#include +#include + +namespace scene::eu7 { +namespace { + +std::mutex g_pack_mesh_load_mutex; + +struct PackMeshQueueEntry { + int priority { 0 }; + std::uint64_t sequence { 0 }; + std::string model_file; +}; + +struct PackMeshQueueCompare { + [[nodiscard]] bool + operator()( PackMeshQueueEntry const &lhs, PackMeshQueueEntry const &rhs ) const { + if( lhs.priority != rhs.priority ) { + return lhs.priority > rhs.priority; + } + return lhs.sequence > rhs.sequence; + } +}; + +struct PackMeshLoader { + std::mutex mutex; + std::priority_queue< + PackMeshQueueEntry, + std::vector, + PackMeshQueueCompare> queue; + std::unordered_map queued; + std::unordered_map ready; + std::uint64_t sequence { 0 }; + bool running { false }; +}; + +PackMeshLoader g_loader; + +[[nodiscard]] std::string +pack_model_texture_dir( std::string model_file ) { + if( model_file.empty() || model_file == "notload" ) { + return {}; + } + replace_slashes( model_file ); + erase_extension( model_file ); + auto const slash_pos { model_file.rfind( '/' ) }; + if( slash_pos == std::string::npos ) { + return {}; + } + return model_file.substr( 0, slash_pos + 1 ); +} + +class PackTexturePathScope { +public: + explicit PackTexturePathScope( std::string const &model_file ) { + m_saved_path = Global.asCurrentTexturePath; + auto const model_dir { pack_model_texture_dir( model_file ) }; + if( false == model_dir.empty() && model_file.find( '/' ) != std::string::npos ) { + Global.asCurrentTexturePath = model_dir; + m_active = true; + } + } + + ~PackTexturePathScope() { + if( m_active ) { + Global.asCurrentTexturePath = std::move( m_saved_path ); + } + } + + PackTexturePathScope( PackTexturePathScope const & ) = delete; + PackTexturePathScope &operator=( PackTexturePathScope const & ) = delete; + +private: + std::string m_saved_path; + bool m_active { false }; +}; + +[[nodiscard]] bool +pack_mesh_path_valid( std::string const &model_file ) { + return false == model_file.empty() && model_file != "notload"; +} + +[[nodiscard]] bool +pack_mesh_pointer_usable( TModel3d *const mesh ) { + return mesh != nullptr; +} + +[[nodiscard]] bool +pack_mesh_cached_usable( + std::unordered_map const &session_cache, + std::string const &model_file ) { + auto const found { session_cache.find( model_file ) }; + return found != session_cache.end() && pack_mesh_pointer_usable( found->second ); +} + +[[nodiscard]] bool +pack_mesh_globally_usable( std::string model_file ) { + if( false == pack_mesh_path_valid( model_file ) ) { + return false; + } + replace_slashes( model_file ); + if( false == TModelsManager::IsModelCached( model_file ) ) { + return false; + } + + TModel3d *mesh { nullptr }; + { + std::lock_guard lock { g_pack_mesh_load_mutex }; + mesh = TModelsManager::GetModel( model_file, false, false ); + } + return pack_mesh_pointer_usable( mesh ); +} + +[[nodiscard]] bool +pop_pack_mesh_queue_entry( std::string &model_file ) { + model_file.clear(); + while( false == g_loader.queue.empty() ) { + auto entry { g_loader.queue.top() }; + g_loader.queue.pop(); + auto const found { g_loader.queued.find( entry.model_file ) }; + if( + found == g_loader.queued.end() || + found->second != entry.priority ) { + continue; + } + model_file = std::move( entry.model_file ); + return true; + } + return false; +} + +void +load_pack_mesh_on_main_thread( std::string const &model_file ) { + PackBenchTimer const load_timer { &Eu7PackBench::loader_thread_getmodel_ms }; + TModel3d *mesh { nullptr }; + { + std::lock_guard lock { g_pack_mesh_load_mutex }; + PackTexturePathScope const scope { model_file }; + mesh = TModelsManager::GetModel( model_file, false, false ); + if( pack_mesh_pointer_usable( mesh ) ) { + TAnimModel::warm_instanceable_cache( mesh ); + } + } + pack_bench_inc( &Eu7PackBench::loader_thread_disk_loads ); + + std::lock_guard lock { g_loader.mutex }; + g_loader.ready.emplace( model_file, mesh ); + g_loader.queued.erase( model_file ); +} + +[[nodiscard]] std::size_t +pump_pack_mesh_loader_main( + double const budget_ms, + std::size_t const max_loads ) { + if( false == g_loader.running ) { + return 0; + } + + auto const started { std::chrono::steady_clock::now() }; + std::size_t loaded { 0 }; + + while( loaded < max_loads ) { + if( budget_ms > 0.0 ) { + auto const elapsed_ms { + std::chrono::duration( + std::chrono::steady_clock::now() - started ).count() }; + if( elapsed_ms >= budget_ms ) { + break; + } + } + + std::string model_file; + { + std::lock_guard lock { g_loader.mutex }; + if( false == pop_pack_mesh_queue_entry( model_file ) ) { + break; + } + } + + load_pack_mesh_on_main_thread( model_file ); + ++loaded; + } + + return loaded; +} + +[[nodiscard]] bool +try_consume_loader_ready( + std::string const &model_file, + std::unordered_map &session_cache ) { + TModel3d *mesh { nullptr }; + { + std::lock_guard lock { g_loader.mutex }; + auto const found { g_loader.ready.find( model_file ) }; + if( found == g_loader.ready.end() ) { + return false; + } + mesh = found->second; + g_loader.ready.erase( found ); + } + session_cache.emplace( model_file, mesh ); + pack_bench_inc( &Eu7PackBench::stream_mesh_async_ready ); + return pack_mesh_pointer_usable( mesh ); +} + +[[nodiscard]] bool +sync_resolve_global_cached_mesh( + std::string const &model_file, + std::unordered_map &session_cache ) { + if( false == pack_mesh_globally_usable( model_file ) ) { + return false; + } + + TModel3d *mesh { nullptr }; + { + PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms }; + std::lock_guard lock { g_pack_mesh_load_mutex }; + PackTexturePathScope const scope { model_file }; + mesh = TModelsManager::GetModel( model_file, false, false ); + pack_bench_inc( &Eu7PackBench::main_cold_getmodel_calls ); + if( pack_mesh_pointer_usable( mesh ) ) { + TAnimModel::warm_instanceable_cache( mesh ); + } + } + pack_bench_inc( &Eu7PackBench::stream_mesh_global_hit ); + session_cache.emplace( model_file, mesh ); + return pack_mesh_pointer_usable( mesh ); +} + +void +enqueue_pack_mesh_load( std::string model_file, int const priority ) { + if( false == pack_mesh_path_valid( model_file ) ) { + return; + } + replace_slashes( model_file ); + + { + std::lock_guard lock { g_loader.mutex }; + if( g_loader.ready.contains( model_file ) ) { + return; + } + auto const found { g_loader.queued.find( model_file ) }; + if( found != g_loader.queued.end() ) { + if( priority >= found->second ) { + return; + } + found->second = priority; + } + else { + g_loader.queued.emplace( model_file, priority ); + } + g_loader.queue.push( + PackMeshQueueEntry { priority, g_loader.sequence++, std::move( model_file ) } ); + } + pack_bench_inc( &Eu7PackBench::stream_mesh_async_queued ); +} + +[[nodiscard]] bool +wait_for_loader_ready( + std::string const &model_file, + double const block_budget_ms ) { + auto const deadline { + block_budget_ms > 0.0 ? + std::chrono::steady_clock::now() + + std::chrono::duration_cast( + std::chrono::duration( block_budget_ms ) ) : + std::chrono::steady_clock::time_point::max() }; + + while( true ) { + { + std::lock_guard lock { g_loader.mutex }; + if( g_loader.ready.contains( model_file ) ) { + return true; + } + if( g_loader.queued.contains( model_file ) == false ) { + return false; + } + } + + if( block_budget_ms > 0.0 && std::chrono::steady_clock::now() >= deadline ) { + return false; + } + + auto remaining_ms { block_budget_ms }; + if( block_budget_ms > 0.0 ) { + remaining_ms = std::chrono::duration( + deadline - std::chrono::steady_clock::now() ).count(); + if( remaining_ms <= 0.0 ) { + return false; + } + } + + (void)pump_pack_mesh_loader_main( remaining_ms, 1 ); + } +} + +} // namespace + +void +start_pack_mesh_loader() { + stop_pack_mesh_loader(); + g_loader.running = true; +} + +void +stop_pack_mesh_loader() { + g_loader.running = false; + + std::lock_guard lock { g_loader.mutex }; + g_loader.queue = decltype( g_loader.queue )(); + g_loader.queued.clear(); + g_loader.ready.clear(); +} + +void +reset_pack_mesh_loader() { + stop_pack_mesh_loader(); +} + +void +request_pack_mesh_load( std::string const &model_file, int const priority ) { + if( false == g_loader.running ) { + return; + } + if( false == pack_mesh_path_valid( model_file ) ) { + return; + } + if( pack_mesh_globally_usable( model_file ) ) { + return; + } + enqueue_pack_mesh_load( model_file, priority ); +} + +void +request_pack_mesh_load_paths( + std::vector const &model_files, + std::size_t const max_enqueue, + int const priority ) { + if( false == g_loader.running ) { + return; + } + + std::size_t enqueued { 0 }; + for( auto const &path : model_files ) { + if( max_enqueue > 0 && enqueued >= max_enqueue ) { + break; + } + if( false == pack_mesh_path_valid( path ) ) { + continue; + } + if( pack_mesh_globally_usable( path ) ) { + continue; + } + enqueue_pack_mesh_load( path, priority ); + ++enqueued; + } +} + +[[nodiscard]] std::size_t +pump_pack_mesh_loader( + double const budget_ms, + std::size_t const max_loads ) { + return pump_pack_mesh_loader_main( budget_ms, max_loads ); +} + +[[nodiscard]] TModel3d * +session_cached_pack_mesh( + std::unordered_map const &session_cache, + std::string const &model_file ) { + auto const found { session_cache.find( model_file ) }; + return found != session_cache.end() ? found->second : nullptr; +} + +[[nodiscard]] TModel3d * +ensure_pack_mesh_in_session_cache( + std::string model_file, + std::unordered_map &session_cache, + PackMeshLoadWait const wait, + double const block_budget_ms ) { + if( false == pack_mesh_path_valid( model_file ) ) { + return nullptr; + } + replace_slashes( model_file ); + if( pack_mesh_cached_usable( session_cache, model_file ) ) { + pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit ); + return session_cached_pack_mesh( session_cache, model_file ); + } + if( session_cache.contains( model_file ) ) { + session_cache.erase( model_file ); + } + + if( try_consume_loader_ready( model_file, session_cache ) ) { + return session_cached_pack_mesh( session_cache, model_file ); + } + if( sync_resolve_global_cached_mesh( model_file, session_cache ) ) { + return session_cached_pack_mesh( session_cache, model_file ); + } + + request_pack_mesh_load( model_file ); + + if( wait == PackMeshLoadWait::BlockUntilReady ) { + if( wait_for_loader_ready( model_file, block_budget_ms ) ) { + if( try_consume_loader_ready( model_file, session_cache ) ) { + return session_cached_pack_mesh( session_cache, model_file ); + } + } + pack_bench_inc( &Eu7PackBench::stream_mesh_async_wait_timeout ); + return nullptr; + } + + return nullptr; +} + +[[nodiscard]] std::size_t +pack_mesh_loader_queue_depth() { + std::lock_guard lock { g_loader.mutex }; + return g_loader.queued.size(); +} + +[[nodiscard]] bool +try_adopt_pack_mesh_for_slice( + std::string model_file, + std::unordered_map &session_cache ) { + if( model_file.empty() || model_file == "notload" ) { + return true; + } + replace_slashes( model_file ); + if( pack_mesh_cached_usable( session_cache, model_file ) ) { + return true; + } + if( session_cache.contains( model_file ) ) { + session_cache.erase( model_file ); + } + if( try_consume_loader_ready( model_file, session_cache ) ) { + return true; + } + return sync_resolve_global_cached_mesh( model_file, session_cache ); +} + +[[nodiscard]] std::size_t +drain_pack_mesh_loader_ready( + std::unordered_map &session_cache, + std::size_t const max_drain ) { + std::vector> drained; + { + std::lock_guard lock { g_loader.mutex }; + drained.reserve( g_loader.ready.size() ); + for( auto it { g_loader.ready.begin() }; it != g_loader.ready.end(); ) { + if( max_drain > 0 && drained.size() >= max_drain ) { + break; + } + drained.emplace_back( it->first, it->second ); + it = g_loader.ready.erase( it ); + } + } + + std::size_t adopted { 0 }; + for( auto & [path, mesh] : drained ) { + session_cache.emplace( std::move( path ), mesh ); + pack_bench_inc( &Eu7PackBench::stream_mesh_async_ready ); + pack_bench_inc( &Eu7PackBench::stream_mesh_async_drained ); + if( pack_mesh_pointer_usable( mesh ) ) { + ++adopted; + } + } + return adopted; +} + +[[nodiscard]] std::size_t +pack_mesh_loader_ready_count() { + std::lock_guard lock { g_loader.mutex }; + return g_loader.ready.size(); +} + +[[nodiscard]] std::size_t +pack_mesh_loader_worker_count() { + return 0; +} + +} // namespace scene::eu7 diff --git a/scene/eu7/eu7_pack_mesh_loader.h b/scene/eu7/eu7_pack_mesh_loader.h new file mode 100644 index 00000000..fb470f90 --- /dev/null +++ b/scene/eu7/eu7_pack_mesh_loader.h @@ -0,0 +1,75 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +#include +#include +#include +#include + +class TModel3d; + +namespace scene::eu7 { + +enum class PackMeshLoadWait { + NonBlocking, + BlockUntilReady, +}; + +void +start_pack_mesh_loader(); + +void +stop_pack_mesh_loader(); + +void +reset_pack_mesh_loader(); + +void +request_pack_mesh_load( std::string const &model_file, int priority = 0 ); + +void +request_pack_mesh_load_paths( + std::vector const &model_files, + std::size_t max_enqueue = 0, + int priority = 0 ); + +[[nodiscard]] TModel3d * +ensure_pack_mesh_in_session_cache( + std::string model_file, + std::unordered_map &session_cache, + PackMeshLoadWait wait = PackMeshLoadWait::NonBlocking, + double block_budget_ms = 0.0 ); + +[[nodiscard]] bool +try_adopt_pack_mesh_for_slice( + std::string model_file, + std::unordered_map &session_cache ); + +[[nodiscard]] std::size_t +drain_pack_mesh_loader_ready( + std::unordered_map &session_cache, + std::size_t max_drain = 0 ); + +[[nodiscard]] std::size_t +pack_mesh_loader_queue_depth(); + +[[nodiscard]] std::size_t +pack_mesh_loader_ready_count(); + +[[nodiscard]] std::size_t +pack_mesh_loader_worker_count(); + +[[nodiscard]] std::size_t +pump_pack_mesh_loader( + double budget_ms = 0.0, + std::size_t max_loads = 1 ); + +} // namespace scene::eu7 diff --git a/scene/eu7/eu7_reader.cpp b/scene/eu7/eu7_reader.cpp index 678efbf5..8c063fc6 100644 --- a/scene/eu7/eu7_reader.cpp +++ b/scene/eu7/eu7_reader.cpp @@ -16,6 +16,7 @@ http://mozilla.org/MPL/2.0/. #include #include +#include #include namespace scene::eu7 { @@ -70,6 +71,80 @@ private: std::vector strings_; }; +class PackByteReader { +public: + PackByteReader( std::uint8_t const *data, std::size_t const size ) + : cur_ { data } + , end_ { data + size } {} + + [[nodiscard]] bool + enough( std::size_t const bytes ) const noexcept { + return static_cast( end_ - cur_ ) >= bytes; + } + + [[nodiscard]] std::size_t + remaining() const noexcept { + return static_cast( end_ - cur_ ); + } + + [[nodiscard]] std::uint8_t + u8() { + if( false == enough( 1 ) ) { + throw std::runtime_error( "EU7 PACK: przekroczono bufor sekcji" ); + } + return *cur_++; + } + + [[nodiscard]] std::uint16_t + u16() { + if( false == enough( 2 ) ) { + throw std::runtime_error( "EU7 PACK: przekroczono bufor sekcji" ); + } + auto const lo { cur_[ 0 ] }; + auto const hi { cur_[ 1 ] }; + cur_ += 2; + return static_cast( lo | ( static_cast( hi ) << 8u ) ); + } + + [[nodiscard]] std::uint32_t + u32() { + if( false == enough( 4 ) ) { + throw std::runtime_error( "EU7 PACK: przekroczono bufor sekcji" ); + } + std::uint32_t value { 0 }; + std::memcpy( &value, cur_, 4 ); + cur_ += 4; + return value; + } + + [[nodiscard]] std::int32_t + i32() { + return static_cast( u32() ); + } + + [[nodiscard]] float + f32() { + std::uint32_t bits { u32() }; + float value { 0.f }; + std::memcpy( &value, &bits, sizeof( value ) ); + return value; + } + + [[nodiscard]] double + f64_disk() { + return static_cast( f32() ); + } + + [[nodiscard]] glm::dvec3 + vec3() { + return { f64_disk(), f64_disk(), f64_disk() }; + } + +private: + std::uint8_t const *cur_; + std::uint8_t const *end_; +}; + [[nodiscard]] std::int16_t read_i16( std::istream &input ) { return static_cast( sn_utils::ld_uint16( input ) ); @@ -207,6 +282,97 @@ read_slim_node( std::istream &input, StringTable const &table, std::string_view return node; } +[[nodiscard]] Eu7TransformContext +read_transform_context_bytes( PackByteReader &input ) { + Eu7TransformContext transform; + const std::uint8_t origin_count { input.u8() }; + transform.origin_stack.reserve( origin_count ); + for( std::uint8_t i { 0 }; i < origin_count; ++i ) { + transform.origin_stack.push_back( input.vec3() ); + } + const std::uint8_t scale_count { input.u8() }; + transform.scale_stack.reserve( scale_count ); + for( std::uint8_t i { 0 }; i < scale_count; ++i ) { + transform.scale_stack.push_back( input.vec3() ); + } + transform.rotation = input.vec3(); + transform.group_depth = input.u8(); + return transform; +} + +[[nodiscard]] Eu7BasicNode +read_slim_node_bytes( + PackByteReader &input, + StringTable const &table, + std::string_view const implied_type ) { + Eu7BasicNode node; + node.node_type = std::string( implied_type ); + const std::uint8_t flags { input.u8() }; + if( ( flags & kNodeFlagHasName ) != 0 ) { + node.name = table.resolve( input.u32() ); + } + if( ( flags & kNodeFlagHasRangeMin ) != 0 ) { + node.range_squared_min = input.f64_disk(); + } + if( ( flags & kNodeFlagHasRangeMax ) != 0 ) { + node.range_squared_max = input.f64_disk(); + } + else { + node.range_squared_max = std::numeric_limits::max(); + } + if( ( flags & kNodeFlagHasBounds ) != 0 ) { + node.area.center = input.vec3(); + node.area.radius = input.f32(); + } + if( ( flags & kNodeFlagHasGroup ) != 0 ) { + node.group_valid = true; + node.group_handle = input.u32(); + } + if( ( flags & kNodeFlagHasTransform ) != 0 ) { + node.transform = read_transform_context_bytes( input ); + } + node.visible = ( flags & kNodeFlagNotVisible ) == 0; + return node; +} + +[[nodiscard]] Eu7Model +read_runtime_model_bytes( + PackByteReader &input, + StringTable const &table, + Eu7PackSectionCursor const *cursor = nullptr ) { + Eu7Model model; + model.node = read_slim_node_bytes( input, table, "model" ); + model.is_terrain = input.u8() != 0; + model.transition = input.u8() != 0; + model.location = input.vec3(); + model.angles = input.vec3(); + model.scale = input.vec3(); + if( + cursor != nullptr && + cursor->section_format == kPackSectionFormatV13 ) { + model.pack_mesh_index = input.u16(); + model.pack_texture_index = input.u16(); + } + else { + model.model_file = table.resolve( input.u32() ); + model.texture_file = table.resolve( input.u32() ); + } + const std::uint32_t light_count { input.u32() }; + model.light_states.resize( light_count ); + for( std::uint32_t i { 0 }; i < light_count; ++i ) { + model.light_states[ i ] = input.f32(); + } + const std::uint32_t color_count { input.u32() }; + model.light_colors.resize( color_count ); + for( std::uint32_t i { 0 }; i < color_count; ++i ) { + model.light_colors[ i ] = input.u32(); + } + if( cursor != nullptr && cursor->section_format == kPackSectionFormatV13 ) { + model.pack_cell_id = input.u8(); + } + return model; +} + [[nodiscard]] std::string read_length_string( std::istream &input ) { const std::uint32_t length { sn_utils::ld_uint32( input ) }; @@ -413,7 +579,10 @@ read_runtime_lines( std::istream &input, StringTable const &table ) { } [[nodiscard]] Eu7Model -read_runtime_model( std::istream &input, StringTable const &table ) { +read_runtime_model( + std::istream &input, + StringTable const &table, + Eu7PackSectionCursor const *cursor = nullptr ) { Eu7Model model; model.node = read_slim_node( input, table, "model" ); model.is_terrain = sn_utils::d_uint8( input ) != 0; @@ -421,8 +590,16 @@ read_runtime_model( std::istream &input, StringTable const &table ) { model.location = read_vec3( input ); model.angles = read_vec3( input ); model.scale = read_vec3( input ); - model.model_file = table.resolve( sn_utils::ld_uint32( input ) ); - model.texture_file = table.resolve( sn_utils::ld_uint32( input ) ); + if( + cursor != nullptr && + cursor->section_format == kPackSectionFormatV13 ) { + model.pack_mesh_index = sn_utils::ld_uint16( input ); + model.pack_texture_index = sn_utils::ld_uint16( input ); + } + else { + model.model_file = table.resolve( sn_utils::ld_uint32( input ) ); + model.texture_file = table.resolve( sn_utils::ld_uint32( input ) ); + } const std::uint32_t light_count { sn_utils::ld_uint32( input ) }; model.light_states.resize( light_count ); for( std::uint32_t i { 0 }; i < light_count; ++i ) { @@ -433,11 +610,17 @@ read_runtime_model( std::istream &input, StringTable const &table ) { for( std::uint32_t i { 0 }; i < color_count; ++i ) { model.light_colors[ i ] = sn_utils::ld_uint32( input ); } + if( cursor != nullptr && cursor->section_format == kPackSectionFormatV13 ) { + model.pack_cell_id = sn_utils::d_uint8( input ); + } return model; } [[nodiscard]] Eu7ModelPrototype -read_runtime_prototype( std::istream &input, StringTable const &table ) { +read_runtime_prototype( + std::istream &input, + StringTable const &table, + std::uint32_t const file_version ) { Eu7ModelPrototype proto; proto.node = read_slim_node( input, table, "model" ); proto.is_terrain = sn_utils::d_uint8( input ) != 0; @@ -454,16 +637,27 @@ read_runtime_prototype( std::istream &input, StringTable const &table ) { for( std::uint32_t i { 0 }; i < color_count; ++i ) { proto.light_colors[ i ] = sn_utils::ld_uint32( input ); } + if( file_version >= kEu7VersionV9 ) { + proto.resolved_texture = table.resolve( sn_utils::ld_uint32( input ) ); + proto.pack_flags = sn_utils::d_uint8( input ); + proto.textures_alpha = sn_utils::ld_uint32( input ); + proto.baked_range_min = sn_utils::ld_float32( input ); + proto.baked_range_max = sn_utils::ld_float32( input ); + } return proto; } void -read_prot_chunk( std::istream &input, StringTable const &strings, Eu7Module &module ) { +read_prot_chunk( + std::istream &input, + StringTable const &strings, + Eu7Module &module ) { const std::uint32_t count { sn_utils::ld_uint32( input ) }; module.model_prototypes.clear(); module.model_prototypes.reserve( count ); for( std::uint32_t i { 0 }; i < count; ++i ) { - module.model_prototypes.push_back( read_runtime_prototype( input, strings ) ); + module.model_prototypes.push_back( + read_runtime_prototype( input, strings, module.version ) ); } } @@ -483,10 +677,15 @@ expand_prototype_instance( model.scale = scale; model.model_file = proto.model_file; model.texture_file = proto.texture_file; + model.resolved_texture = proto.resolved_texture; model.light_states = proto.light_states; model.light_colors = proto.light_colors; model.transition = proto.transition; model.is_terrain = proto.is_terrain; + model.pack_flags = proto.pack_flags; + model.textures_alpha = proto.textures_alpha; + model.baked_range_min = proto.baked_range_min; + model.baked_range_max = proto.baked_range_max; return model; } @@ -506,6 +705,55 @@ parse_pack_section_header( peek == EOF ? std::uint8_t { 0 } : static_cast( peek ) }; bool use_v12_header { false }; + if( first_byte == kPackSectionFormatV13 ) { + cursor.section_format = sn_utils::d_uint8( input ); + const std::uint32_t solo_total { sn_utils::ld_uint32( input ) }; + const std::uint32_t inst_total { sn_utils::ld_uint32( input ) }; + const std::uint32_t unique_mesh_count { sn_utils::ld_uint32( input ) }; + if( + solo_total + inst_total == entry.model_count && + ( inst_total == 0 || false == module.model_prototypes.empty() ) ) { + use_v12_header = true; + cursor.model_total = solo_total + inst_total; + cursor.solo_remaining = solo_total; + cursor.inst_remaining = inst_total; + + StringTable const strings { module.strings }; + cursor.unique_meshes.reserve( unique_mesh_count ); + for( std::uint32_t mesh_idx { 0 }; mesh_idx < unique_mesh_count; ++mesh_idx ) { + cursor.unique_meshes.push_back( + strings.resolve( sn_utils::ld_uint32( input ) ) ); + } + + const std::uint32_t unique_texture_count { sn_utils::ld_uint32( input ) }; + cursor.unique_textures.reserve( unique_texture_count ); + for( std::uint32_t tex_idx { 0 }; tex_idx < unique_texture_count; ++tex_idx ) { + cursor.unique_textures.push_back( + strings.resolve( sn_utils::ld_uint32( input ) ) ); + } + + cursor.chunk_count = sn_utils::ld_uint32( input ); + cursor.chunk_model_counts.resize( cursor.chunk_count ); + cursor.chunk_inst_counts.resize( cursor.chunk_count ); + cursor.chunk_byte_offsets.resize( cursor.chunk_count ); + for( std::uint32_t chunk_idx { 0 }; chunk_idx < cursor.chunk_count; ++chunk_idx ) { + cursor.chunk_model_counts[ chunk_idx ] = sn_utils::ld_uint32( input ); + cursor.chunk_inst_counts[ chunk_idx ] = sn_utils::ld_uint32( input ); + cursor.chunk_byte_offsets[ chunk_idx ] = sn_utils::ld_uint32( input ); + } + } + if( false == use_v12_header ) { + input.seekg( section_start ); + } + } + + if( use_v12_header ) { + cursor.models_read = 0; + cursor.header_parsed = true; + return; + } + + use_v12_header = false; if( first_byte == kPackSectionFormatV12 ) { cursor.section_format = sn_utils::d_uint8( input ); const std::uint32_t solo_total { sn_utils::ld_uint32( input ) }; @@ -709,7 +957,7 @@ read_pack_models_chunk_impl( models.size() < max_count && ( cursor.solo_remaining > 0 || cursor.inst_remaining > 0 ) ) { if( cursor.solo_remaining > 0 ) { - auto model { read_runtime_model( input, strings ) }; + auto model { read_runtime_model( input, strings, &cursor ) }; model.node.transform = {}; models.push_back( std::move( model ) ); --cursor.solo_remaining; @@ -725,12 +973,75 @@ read_pack_models_chunk_impl( auto const angles { read_vec3( input ) }; auto const scale { read_vec3( input ) }; auto const name { strings.resolve( sn_utils::ld_uint32( input ) ) }; - models.push_back( expand_prototype_instance( + auto model { expand_prototype_instance( module.model_prototypes[ proto_id ], location, angles, scale, - name ) ); + name ) }; + if( cursor.section_format == kPackSectionFormatV13 ) { + model.pack_cell_id = sn_utils::d_uint8( input ); + } + models.push_back( std::move( model ) ); + --cursor.inst_remaining; + } + ++cursor.models_read; + } + + return models; +} + +[[nodiscard]] std::vector +read_pack_models_chunk_from_bytes( + Eu7Module const &module, + std::uint8_t const *data, + std::size_t const size, + Eu7PackSectionCursor &cursor, + std::size_t const max_count, + StringTable const &strings ) { + if( false == cursor.header_parsed || max_count == 0 || data == nullptr || size == 0 ) { + return {}; + } + + PackByteReader input { data, size }; + std::vector models; + auto const remaining { + std::min( + static_cast( cursor.solo_remaining ) + + static_cast( cursor.inst_remaining ), + static_cast( cursor.model_total ) ) }; + models.reserve( std::min( max_count, remaining ) ); + + while( + models.size() < max_count && + ( cursor.solo_remaining > 0 || cursor.inst_remaining > 0 ) ) { + if( cursor.solo_remaining > 0 ) { + auto model { read_runtime_model_bytes( input, strings, &cursor ) }; + model.node.transform = {}; + models.push_back( std::move( model ) ); + --cursor.solo_remaining; + } + else { + const std::uint32_t proto_id { input.u32() }; + if( proto_id >= module.model_prototypes.size() ) { + throw std::runtime_error( + "EU7 PACK: proto_id " + std::to_string( proto_id ) + " poza zakresem PROT (" + + std::to_string( module.model_prototypes.size() ) + ")" ); + } + auto const location { input.vec3() }; + auto const angles { input.vec3() }; + auto const scale { input.vec3() }; + auto const name { strings.resolve( input.u32() ) }; + auto model { expand_prototype_instance( + module.model_prototypes[ proto_id ], + location, + angles, + scale, + name ) }; + if( cursor.section_format == kPackSectionFormatV13 ) { + model.pack_cell_id = input.u8(); + } + models.push_back( std::move( model ) ); --cursor.inst_remaining; } ++cursor.models_read; @@ -1166,6 +1477,7 @@ struct PackSectionReadSession { int column { -1 }; Eu7PackIndexEntry entry {}; Eu7PackSectionCursor header {}; + std::shared_ptr path_tables; std::ifstream input; bool header_ready { false }; }; @@ -1180,6 +1492,55 @@ reset_pack_section_read_session( PackSectionReadSession &session ) { session = {}; } +[[nodiscard]] std::uint64_t +pack_section_byte_size( + Eu7Module const &module, + Eu7PackIndexEntry const &entry ) { + std::uint64_t next_offset { module.pack_catalog.pack_payload_size }; + for( auto const &candidate : module.pack_catalog.entries ) { + if( + candidate.pack_offset > entry.pack_offset && + candidate.pack_offset < next_offset ) { + next_offset = candidate.pack_offset; + } + } + return next_offset - entry.pack_offset; +} + +[[nodiscard]] std::optional +pack_chunk_byte_size( + Eu7PackSectionCursor const &header, + std::uint32_t const chunk_index, + std::uint64_t const section_byte_size ) { + if( header.chunk_byte_offsets.empty() || chunk_index >= header.chunk_count ) { + return std::nullopt; + } + auto const start { header.chunk_byte_offsets[ chunk_index ] }; + if( chunk_index + 1 < header.chunk_count ) { + return header.chunk_byte_offsets[ chunk_index + 1 ] - start; + } + if( section_byte_size > start ) { + return static_cast( section_byte_size - start ); + } + return std::nullopt; +} + +void +ensure_section_path_tables( + Eu7PackSectionCursor const &header, + PackSectionReadSession &session ) { + if( + header.unique_meshes.empty() && + header.unique_textures.empty() ) { + session.path_tables.reset(); + return; + } + auto tables { std::make_shared() }; + tables->unique_meshes = header.unique_meshes; + tables->unique_textures = header.unique_textures; + session.path_tables = std::move( tables ); +} + void ensure_pack_file( Eu7Module const &module, PackSectionReadSession &session ) { if( module.source_path == session.source_path && session.input.is_open() ) { @@ -1216,7 +1577,9 @@ ensure_section_header( session.row = row; session.column = column; session.entry = entry; + session.path_tables.reset(); parse_pack_section_header( module, session.input, entry, session.header ); + ensure_section_path_tables( session.header, session ); session.header_ready = true; } @@ -1254,23 +1617,25 @@ read_pack_section_chunk_load_impl( } result.chunk_index = chunk_index; - if( chunk_index == 0 ) { - result.unique_meshes = header.unique_meshes; - result.unique_textures = header.unique_textures; - } Eu7PackSectionCursor chunk_cursor { header }; + auto const section_byte_size { pack_section_byte_size( module, *entry ) }; + std::optional chunk_byte_size; if( ( header.section_format == kPackSectionFormatV10 || header.section_format == kPackSectionFormatV11 || - header.section_format == kPackSectionFormatV12 ) && + header.section_format == kPackSectionFormatV12 || + header.section_format == kPackSectionFormatV13 ) && false == header.chunk_byte_offsets.empty() ) { auto const section_start { static_cast( module.pack_payload_offset + entry->pack_offset ) }; + chunk_byte_size = pack_chunk_byte_size( header, chunk_index, section_byte_size ); session.input.seekg( section_start + static_cast( header.chunk_byte_offsets[ chunk_index ] ) ); - if( header.section_format == kPackSectionFormatV12 ) { + if( + header.section_format == kPackSectionFormatV12 || + header.section_format == kPackSectionFormatV13 ) { chunk_cursor.solo_remaining = header.chunk_model_counts[ chunk_index ]; chunk_cursor.inst_remaining = header.chunk_inst_counts[ chunk_index ]; } @@ -1283,12 +1648,35 @@ read_pack_section_chunk_load_impl( return result; } - result.models = read_pack_models_chunk_impl( - module, - session.input, - chunk_cursor, - std::numeric_limits::max(), - strings ); + if( chunk_byte_size.has_value() && *chunk_byte_size > 0 ) { + std::vector chunk_bytes( *chunk_byte_size ); + session.input.read( + reinterpret_cast( chunk_bytes.data() ), + static_cast( chunk_bytes.size() ) ); + if( + static_cast( session.input.gcount() ) != chunk_bytes.size() ) { + throw std::runtime_error( "EU7 PACK: niepelny odczyt chunka sekcji" ); + } + result.models = read_pack_models_chunk_from_bytes( + module, + chunk_bytes.data(), + chunk_bytes.size(), + chunk_cursor, + std::numeric_limits::max(), + strings ); + } + else { + result.models = read_pack_models_chunk_impl( + module, + session.input, + chunk_cursor, + std::numeric_limits::max(), + strings ); + } + result.path_tables = session.path_tables; + if( chunk_index == 0 && false == header.unique_textures.empty() ) { + result.unique_textures = header.unique_textures; + } return result; } @@ -1320,7 +1708,8 @@ read_pack_section_load_impl( Eu7Module const &module, int const row, int const c if( ( header.section_format == kPackSectionFormatV10 || header.section_format == kPackSectionFormatV11 || - header.section_format == kPackSectionFormatV12 ) && + header.section_format == kPackSectionFormatV12 || + header.section_format == kPackSectionFormatV13 ) && header.chunk_count > 0 ) { result.models.reserve( entry->model_count ); for( std::uint32_t chunk_idx { 0 }; chunk_idx < header.chunk_count; ++chunk_idx ) { @@ -1361,7 +1750,7 @@ is_valid_eu7b_file( std::string const &path ) { const std::uint32_t version { sn_utils::ld_uint32( input ) }; return ( version == kEu7VersionV4 || version == kEu7VersionV5 || version == kEu7VersionV6 || - version == kEu7VersionV7 || version == kEu7VersionV8 ); + version == kEu7VersionV7 || version == kEu7VersionV8 || version == kEu7VersionV9 ); } Eu7Module @@ -1380,7 +1769,7 @@ read_module( std::string const &path ) { if( module.version != kEu7VersionV4 && module.version != kEu7VersionV5 && module.version != kEu7VersionV6 && module.version != kEu7VersionV7 && - module.version != kEu7VersionV8 ) { + module.version != kEu7VersionV8 && module.version != kEu7VersionV9 ) { throw std::runtime_error( "EU7B: nieobslugiwana wersja " + std::to_string( module.version ) + " w \"" + path + "\"" ); } @@ -1444,4 +1833,58 @@ reset_pack_section_read_cache() { reset_pack_section_read_session( g_pack_section_read_session ); } +void +for_each_pack_section_unique_mesh( + Eu7Module const &module, + int const row, + int const column, + std::function const &visit ) { + if( false == module.has_pack_chunk || module.source_path.empty() ) { + return; + } + + auto const entry { find_pack_entry_impl( module, row, column ) }; + if( false == entry.has_value() || entry->model_count == 0 ) { + return; + } + + auto &session { g_pack_section_read_session }; + ensure_pack_file( module, session ); + if( !session.input ) { + return; + } + + ensure_section_header( module, row, column, *entry, session ); + for( auto const &path : session.header.unique_meshes ) { + visit( path ); + } +} + +void +for_each_pack_section_unique_texture( + Eu7Module const &module, + int const row, + int const column, + std::function const &visit ) { + if( false == module.has_pack_chunk || module.source_path.empty() ) { + return; + } + + auto const entry { find_pack_entry_impl( module, row, column ) }; + if( false == entry.has_value() || entry->model_count == 0 ) { + return; + } + + auto &session { g_pack_section_read_session }; + ensure_pack_file( module, session ); + if( !session.input ) { + return; + } + + ensure_section_header( module, row, column, *entry, session ); + for( auto const &path : session.header.unique_textures ) { + visit( path ); + } +} + } // namespace scene::eu7 diff --git a/scene/eu7/eu7_reader.h b/scene/eu7/eu7_reader.h index 5992c41c..9810ca75 100644 --- a/scene/eu7/eu7_reader.h +++ b/scene/eu7/eu7_reader.h @@ -11,7 +11,7 @@ http://mozilla.org/MPL/2.0/. #include "scene/eu7/eu7_types.h" -#include +#include #include #include @@ -47,4 +47,20 @@ read_pack_section_chunk_load( void reset_pack_section_read_cache(); +// UMES z naglowka sekcji — bez kopiowania calego wektora. +void +for_each_pack_section_unique_mesh( + Eu7Module const &Module, + int Row, + int Column, + std::function const &Visit ); + +// UTEX z naglowka sekcji (v11+) — bez skanowania instancji. +void +for_each_pack_section_unique_texture( + Eu7Module const &Module, + int Row, + int Column, + std::function const &Visit ); + } // namespace scene::eu7 diff --git a/scene/eu7/eu7_section_stream.cpp b/scene/eu7/eu7_section_stream.cpp index 71f5fe2e..a3456661 100644 --- a/scene/eu7/eu7_section_stream.cpp +++ b/scene/eu7/eu7_section_stream.cpp @@ -15,6 +15,7 @@ http://mozilla.org/MPL/2.0/. #include "scene/eu7/eu7_pack_bench.h" #include "model/AnimModel.h" #include "model/MdlMngr.h" +#include "scene/eu7/eu7_pack_mesh_loader.h" #include "scene/eu7/eu7_model_prefetch.h" #include "scene/eu7/eu7_reader.h" #include "scene/eu7/eu7_section.h" @@ -54,22 +55,35 @@ ring_section_in_radius( namespace { constexpr double kDrainBudgetMs { 12.0 }; -constexpr double kLoaderDrainBudgetMs { 96.0 }; +constexpr double kLoaderDrainBudgetMs { 256.0 }; +constexpr double kFrameDrainBudgetMs { 10.0 }; constexpr std::size_t kLoaderSectionsPerDrain { 8 }; constexpr double kGameplayApplyBudgetMs { 6.0 }; -constexpr double kCatchupApplyBudgetMs { 14.0 }; -constexpr std::size_t kGameplaySliceInstances { 96 }; -constexpr std::size_t kCatchupSliceInstances { 96 }; +constexpr double kCatchupApplyBudgetMs { 8.0 }; +constexpr std::size_t kGameplaySliceInstances { 48 }; +constexpr std::size_t kCatchupSliceInstances { 48 }; +constexpr std::size_t kFrameSliceInstances { 24 }; constexpr std::size_t kLoaderSliceInstances { 768 }; constexpr std::size_t kGameplaySliceColdMeshes { 4 }; -constexpr std::size_t kCatchupSliceColdMeshes { 8 }; +constexpr std::size_t kCatchupSliceColdMeshes { 6 }; +constexpr std::size_t kFrameSliceColdMeshes { 2 }; constexpr std::size_t kLoaderSliceColdMeshes { 32 }; -constexpr double kGameplayColdBudgetMs { 12.0 }; -constexpr double kCatchupColdBudgetMs { 18.0 }; +constexpr double kGameplayColdBudgetMs { 6.0 }; +constexpr double kCatchupColdBudgetMs { 8.0 }; +constexpr double kFrameColdBudgetMs { 4.0 }; -constexpr int kInitialBootstrapRadius { 5 }; -constexpr int kStreamRadius { 14 }; +constexpr int kInitialBootstrapEnqueueRadius { 2 }; +constexpr int kInitialBootstrapRadius { 3 }; +constexpr float kEarlyDismissRingProgress { 0.65f }; +constexpr int kStreamRadius { kSectionStreamTargetRadiusKm }; +constexpr int kMaintenanceLookaheadMax { 6 }; +constexpr double kRadiusFillApplyBudgetMs { 8.0 }; +constexpr std::size_t kRadiusFillSliceInstances { 16 }; +constexpr std::size_t kRadiusFillMaxChunksPerDrain { 2 }; +constexpr std::size_t kRadiusFillMaxInFlight { 24 }; +constexpr std::size_t kRadiusFillMaxReady { 14 }; +constexpr float kRadiusFillOuterTarget { 0.98f }; constexpr int kSectionUnloadMarginKm { 3 }; constexpr int kSectionUnloadCameraGuardKm { 5 }; constexpr int kSectionUnloadMaxSpeedExtraKm { 10 }; @@ -78,28 +92,32 @@ constexpr int kSectionUnloadTeleportJumpKm { 4 }; constexpr std::size_t kMeshCacheLruCap { 2000 }; constexpr std::size_t kNodedataCacheLruCap { 2000 }; constexpr int kMovementLookahead { 10 }; -constexpr std::size_t kMaxPackStreamWorkers { 8 }; -constexpr std::size_t kMaxInFlightSections { 12 }; -constexpr std::size_t kMaxReadySections { 6 }; +constexpr std::size_t kMaxPackStreamWorkers { 16 }; +constexpr std::size_t kMaxInFlightSections { 20 }; +constexpr std::size_t kMaxReadySections { 10 }; constexpr double kStreamStatusLogIntervalSec { 5.0 }; constexpr double kReenqueueDistanceM { 500.0 }; constexpr double kCatchupReenqueueDistanceM { 40.0 }; constexpr std::size_t kHeavySectionModelThreshold { 32 }; constexpr std::size_t kPackTextureWarmSlice { 8 }; constexpr double kPackTextureWarmBudgetMs { 4.0 }; -constexpr double kUrgentApplyBudgetMs { 32.0 }; -constexpr std::size_t kUrgentSliceInstances { 96 }; -constexpr std::size_t kUrgentSliceColdMeshes { 6 }; -constexpr double kUrgentColdBudgetMs { 28.0 }; -constexpr std::size_t kUrgentMaxChunksPerDrain { 8 }; -constexpr std::size_t kRingDeficitMaxChunksPerDrain { 4 }; +constexpr double kUrgentApplyBudgetMs { 10.0 }; +constexpr double kRingDeficitDrainBudgetMs { 8.0 }; +constexpr double kSmoothApplyTargetMs { 7.0 }; +constexpr double kSmoothApplyDeficitTargetMs { 8.0 }; +constexpr std::size_t kUrgentSliceInstances { 48 }; +constexpr std::size_t kUrgentSliceColdMeshes { 4 }; +constexpr double kUrgentColdBudgetMs { 6.0 }; +constexpr std::size_t kUrgentMaxChunksPerDrain { 2 }; +constexpr std::size_t kFrameMaxChunksPerDrain { 1 }; +constexpr std::size_t kRingDeficitMaxChunksPerDrain { 2 }; constexpr double kReadyTexturePrefetchBudgetMs { 14.0 }; constexpr std::size_t kReadyTexturePrefetchSlice { 256 }; -constexpr int kUmesPrefetchMaxDistanceKm { 2 }; -constexpr double kReadyUmesPrefetchBudgetMs { 8.0 }; -constexpr std::size_t kReadyUmesPrefetchMaxMeshes { 6 }; -constexpr std::size_t kCatchupMaxInFlightSections { 28 }; -constexpr std::size_t kCatchupMaxReadySections { 14 }; +constexpr int kUmesPrefetchMaxDistanceKm { 6 }; +constexpr double kReadyUmesPrefetchBudgetMs { 16.0 }; +constexpr std::size_t kReadyUmesPrefetchMaxMeshes { 64 }; +constexpr std::size_t kCatchupMaxInFlightSections { 32 }; +constexpr std::size_t kCatchupMaxReadySections { 16 }; constexpr int kUrgentSectionDistanceKm { 1 }; constexpr int kFastFlightApplyMaxDistanceKm { 3 }; constexpr int kFarApplySlowDistanceKm { 4 }; @@ -123,7 +141,8 @@ constexpr float kStationaryCatchupRingThreshold { 0.70f }; constexpr double kStationaryCatchupSpeedMps { 5.0 }; constexpr std::size_t kBootstrapDrainMs { 32 }; constexpr std::size_t kBootstrapTimeoutMs { 120000 }; -constexpr std::chrono::milliseconds kPresentableHoldMs { 200 }; +constexpr std::size_t kBootstrapApplyModelThreshold { 400 }; +constexpr std::chrono::milliseconds kPresentableHoldMs { 0 }; constexpr std::chrono::seconds kLoadingScreenMaxBlockSec { 90 }; std::optional g_ring_ready_since; @@ -146,13 +165,12 @@ struct PackSectionReady { int column { 0 }; std::size_t section_idx { 0 }; std::unique_ptr> models; - std::vector unique_meshes; std::vector unique_textures; + std::shared_ptr path_tables; bool failed { false }; bool section_final { true }; std::size_t apply_offset { 0 }; std::size_t texture_warm_offset { 0 }; - std::size_t umes_cold_offset { 0 }; std::size_t umes_prefetch_offset { 0 }; }; @@ -168,6 +186,7 @@ struct SectionStreamState { bool active { false }; bool bootstrap_active { false }; bool bootstrap_pending { false }; + bool radius_fill_active { false }; std::optional pending_apply; std::size_t pending_apply_offset { 0 }; @@ -259,29 +278,15 @@ release_pending_buffer(); void prefetch_ready_queue_umes_worker( double budget_ms, std::size_t max_meshes ); +[[nodiscard]] bool +gameplay_stream_mode(); + [[nodiscard]] std::size_t warm_pack_section_textures( PackSectionReady &batch, std::size_t const model_offset, std::size_t const model_count, - std::size_t const max_textures = kPackTextureWarmSlice ) { - if( false == batch.unique_textures.empty() ) { - if( batch.texture_warm_offset >= batch.unique_textures.size() ) { - return 0; - } - auto const remaining { batch.unique_textures.size() - batch.texture_warm_offset }; - auto const slice { std::min( remaining, max_textures ) }; - std::size_t processed { 0 }; - auto const warmed { - warm_pack_texture_paths_main( - batch.unique_textures.data() + batch.texture_warm_offset, - slice, - kPackTextureWarmBudgetMs, - &processed ) }; - batch.texture_warm_offset += processed; - return warmed; - } - + double const budget_ms = 0.0 ) { if( batch.models == nullptr || model_count == 0 ) { return 0; } @@ -291,8 +296,7 @@ warm_pack_section_textures( } auto const remaining { std::min( model_count, batch.models->size() - begin ) }; auto const warmed { - warm_pack_textures_main( batch.models->data() + begin, remaining ) }; - batch.texture_warm_offset = begin + remaining; + warm_pack_textures_main( batch.models->data() + begin, remaining, budget_ms ) }; return warmed; } @@ -498,15 +502,42 @@ evict_unreferenced_stream_caches() { &Eu7PackBench::stream_nodedata_cache_evictions ); } +[[nodiscard]] PackMeshLoadWait +stream_mesh_load_wait_policy() { + return gameplay_stream_mode() ? + PackMeshLoadWait::NonBlocking : + PackMeshLoadWait::BlockUntilReady; +} + [[nodiscard]] bool -ensure_stream_pack_mesh( std::string model_file ) { +pack_mesh_ready_for_slice( std::string const &model_file ) { + if( false == try_adopt_pack_mesh_for_slice( model_file, g_stream.mesh_cache ) ) { + return false; + } + auto const found { g_stream.mesh_cache.find( model_file ) }; + return found != g_stream.mesh_cache.end() && + found->second != nullptr && + TModelsManager::IsModelCached( model_file ); +} + +[[nodiscard]] bool +ensure_stream_pack_mesh( + std::string model_file, + double const block_budget_ms = 0.0 ) { if( model_file.empty() || model_file == "notload" ) { return false; } replace_slashes( model_file ); - (void)ensure_pack_mesh_in_session_cache( model_file, g_stream.mesh_cache ); - touch_string_lru( model_file, g_stream.mesh_lru, g_stream.mesh_lru_iters ); - return true; + auto *const mesh { + ensure_pack_mesh_in_session_cache( + model_file, + g_stream.mesh_cache, + stream_mesh_load_wait_policy(), + block_budget_ms ) }; + if( mesh != nullptr ) { + touch_string_lru( model_file, g_stream.mesh_lru, g_stream.mesh_lru_iters ); + } + return mesh != nullptr; } void @@ -713,6 +744,7 @@ reset_stream_fields() { g_stream.active = false; g_stream.bootstrap_active = false; g_stream.bootstrap_pending = false; + g_stream.radius_fill_active = false; g_stream.pending_apply.reset(); g_stream.pending_apply_offset = 0; g_stream.has_last_enqueue_position = false; @@ -754,7 +786,6 @@ release_pending_buffer() { if( g_stream.pending_apply->models != nullptr ) { g_stream.pending_apply->models->clear(); - g_stream.pending_apply->models->shrink_to_fit(); g_stream.pending_apply->models.reset(); } g_stream.pending_apply.reset(); @@ -825,24 +856,66 @@ camera_stream_speed_mps() { return glm::length( rotated ) * 5.0; } -[[nodiscard]] bool pending_apply_is_urgent(); [[nodiscard]] bool stream_ring_deficit(); +[[nodiscard]] bool pending_apply_is_urgent(); +[[nodiscard]] bool in_radius_fill_phase(); + +[[nodiscard]] bool +in_radius_fill_phase() { + return g_stream.radius_fill_active; +} + +[[nodiscard]] double +frame_drain_budget_ms() { + if( in_radius_fill_phase() ) { + return kRadiusFillApplyBudgetMs; + } + if( stream_ring_deficit() ) { + return kRingDeficitDrainBudgetMs; + } + auto const last_chunk { pack_bench_stream().last_chunk_ms }; + if( last_chunk >= 32.0 ) { + return 3.0; + } + if( last_chunk >= 16.0 ) { + return 5.0; + } + if( last_chunk >= 12.0 ) { + return 7.0; + } + return kFrameDrainBudgetMs; +} [[nodiscard]] double gameplay_apply_budget_ms(); [[nodiscard]] bool stream_ring_deficit() { - return ( - g_stream.last_outer_ring < kRingGatedOuterThreshold || - g_stream.last_inner_ring < kRingGatedInnerTarget ); + if( in_radius_fill_phase() ) { + return g_stream.last_outer_ring < kRadiusFillOuterTarget; + } + return g_stream.last_inner_ring < kRingGatedInnerTarget; +} + +[[nodiscard]] int +umes_prefetch_max_distance_km() { + auto const speed { camera_stream_speed_mps() }; + if( g_stream_catchup || speed > 80.0 ) { + return std::min( g_stream.radius, kWorkerContinueMaxDistanceKm ); + } + return kUmesPrefetchMaxDistanceKm; } [[nodiscard]] double ready_texture_prefetch_budget_ms() { auto const speed { camera_stream_speed_mps() }; + auto const ring_deficit { stream_ring_deficit() }; + + if( ring_deficit && speed > 200.0 ) { + return kPackTextureWarmBudgetMs; + } if( speed > 500.0 && pending_apply_is_urgent() ) { - return 0.0; + return kPackTextureWarmBudgetMs * 0.5; } double budget { kReadyTexturePrefetchBudgetMs * 0.5 }; @@ -878,18 +951,27 @@ camera_travel_forward() { [[nodiscard]] double gameplay_apply_budget_ms() { + if( in_radius_fill_phase() ) { + return kRadiusFillApplyBudgetMs; + } + double budget { kGameplayApplyBudgetMs }; if( g_stream_catchup ) { if( stream_ring_deficit() ) { - return pending_apply_is_urgent() ? kUrgentApplyBudgetMs : 16.0; + budget = pending_apply_is_urgent() ? kUrgentApplyBudgetMs : 8.0; } - if( + else if( g_stream.last_outer_ring < kRingBoostOuterThreshold && camera_stream_speed_mps() > kPreemptDisableSpeedMps ) { - return 12.0; + budget = 8.0; + } + else { + budget = kCatchupApplyBudgetMs; } - return kCatchupApplyBudgetMs; } - return kGameplayApplyBudgetMs; + if( g_stream.last_inner_ring < 0.5f ) { + budget = std::max( budget, 12.0 ); + } + return budget; } [[nodiscard]] std::size_t @@ -935,6 +1017,9 @@ pending_section_distance_km() { [[nodiscard]] bool pending_apply_is_urgent() { + if( in_radius_fill_phase() ) { + return pending_section_near_camera( kSectionStreamGameplayRadiusKm ); + } if( stream_ring_deficit() ) { return true; } @@ -947,6 +1032,46 @@ pending_apply_is_urgent() { return pending_section_near_camera( max_dist ); } +[[nodiscard]] std::size_t +apply_time_target_slice( double const target_ms, std::size_t const ceiling ) { + auto const &bench { pack_bench_stream() }; + auto effective_apply_ms { bench.last_apply_ms }; + if( bench.last_chunk_ms > effective_apply_ms && bench.last_chunk_instances > 0 ) { + effective_apply_ms = bench.last_chunk_ms; + } + if( effective_apply_ms >= 1.0 && bench.last_chunk_instances > 0 ) { + auto const scaled { static_cast( + static_cast( bench.last_chunk_instances ) * + ( target_ms / effective_apply_ms ) ) }; + return std::clamp( scaled, std::size_t { 8 }, ceiling ); + } + return std::min( ceiling, std::size_t { 24 } ); +} + +[[nodiscard]] std::size_t +gameplay_max_chunks_per_drain() { + if( in_radius_fill_phase() ) { + return kRadiusFillMaxChunksPerDrain; + } + auto const &bench { pack_bench_stream() }; + if( stream_ring_deficit() ) { + if( bench.last_chunk_ms >= 8.0 ) { + return 1; + } + return kRingDeficitMaxChunksPerDrain; + } + if( pending_apply_is_urgent() ) { + if( bench.last_chunk_ms >= 8.0 ) { + return 1; + } + return kUrgentMaxChunksPerDrain; + } + if( bench.last_chunk_ms >= 12.0 ) { + return 1; + } + return kFrameMaxChunksPerDrain; +} + [[nodiscard]] std::size_t scene_apply_pressure_slice_cap() { auto const applied { load_stats().pack_models }; @@ -970,6 +1095,8 @@ scene_apply_pressure_slice_cap() { [[nodiscard]] std::size_t adaptive_slice_instances( std::size_t const section_total, bool const urgent = false ) { + auto const ring_deficit { stream_ring_deficit() }; + if( urgent ) { auto limit { kUrgentSliceInstances }; auto const speed { camera_stream_speed_mps() }; @@ -977,29 +1104,52 @@ adaptive_slice_instances( std::size_t const section_total, bool const urgent = f limit = std::min( limit, std::size_t { 64 } ); } else if( speed > 600.0 ) { - limit = std::min( limit, std::size_t { 96 } ); - } - if( section_total > 4000 ) { - limit = std::min( limit, std::size_t { 64 } ); - } - else if( section_total > 2000 ) { - limit = std::min( limit, std::size_t { 96 } ); - } - auto const last_ms { pack_bench_stream().last_chunk_ms }; - if( last_ms >= 24.0 ) { - limit = std::min( limit, std::size_t { 32 } ); - } - else if( last_ms >= 12.0 ) { limit = std::min( limit, std::size_t { 48 } ); } - limit = std::min( limit, scene_apply_pressure_slice_cap() ); - auto const last_apply { pack_bench_stream().last_apply_ms }; - if( last_apply >= 20.0 ) { - limit = std::min( limit, std::size_t { 16 } ); + if( section_total > 4000 ) { + limit = std::min( limit, std::size_t { 48 } ); } - else if( last_apply >= 12.0 ) { - limit = std::min( limit, std::size_t { 24 } ); + else if( section_total > 2000 ) { + limit = std::min( limit, std::size_t { 48 } ); } + auto const last_ms { pack_bench_stream().last_chunk_ms }; + if( false == ring_deficit ) { + if( last_ms >= 24.0 ) { + limit = std::min( limit, std::size_t { 32 } ); + } + else if( last_ms >= 12.0 ) { + limit = std::min( limit, std::size_t { 48 } ); + } + limit = std::min( limit, scene_apply_pressure_slice_cap() ); + auto const last_apply { pack_bench_stream().last_apply_ms }; + if( last_apply >= 20.0 ) { + limit = std::min( limit, std::size_t { 16 } ); + } + else if( last_apply >= 12.0 ) { + limit = std::min( limit, std::size_t { 24 } ); + } + } + if( ring_deficit ) { + auto const target_ms { + g_stream.last_inner_ring < 0.5f ? + kSmoothApplyDeficitTargetMs : + kSmoothApplyTargetMs }; + auto const ceiling { std::size_t { 28 } }; + limit = std::min( limit, apply_time_target_slice( target_ms, ceiling ) ); + auto const last_apply { pack_bench_stream().last_apply_ms }; + if( last_apply >= 10.0 ) { + limit = std::min( + limit, + apply_time_target_slice( kSmoothApplyTargetMs * 0.6, std::size_t { 20 } ) ); + } + if( pack_bench_stream().last_chunk_ms >= 12.0 ) { + limit = std::min( limit, std::size_t { 20 } ); + } + return std::max( limit, std::size_t { 8 } ); + } + limit = std::min( + limit, + apply_time_target_slice( kSmoothApplyTargetMs, std::size_t { 48 } ) ); return std::max( limit, std::size_t { 16 } ); } @@ -1015,6 +1165,9 @@ adaptive_slice_instances( std::size_t const section_total, bool const urgent = f else { limit = std::min( limit, std::size_t { 96 } ); } + if( pack_bench_stream().last_chunk_ms >= 12.0 ) { + limit = std::min( limit, std::size_t { 32 } ); + } } if( section_total > 4000 ) { @@ -1028,25 +1181,39 @@ adaptive_slice_instances( std::size_t const section_total, bool const urgent = f } auto const last_ms { pack_bench_stream().last_chunk_ms }; - if( last_ms >= 24.0 ) { - limit = std::min( limit, std::size_t { 32 } ); - } - else if( last_ms >= 12.0 ) { - limit = std::min( limit, std::size_t { 64 } ); + if( false == ring_deficit ) { + if( last_ms >= 24.0 ) { + limit = std::min( limit, std::size_t { 32 } ); + } + else if( last_ms >= 12.0 ) { + limit = std::min( limit, std::size_t { 64 } ); + } + + auto const last_apply { pack_bench_stream().last_apply_ms }; + if( last_apply >= 24.0 ) { + limit = std::min( limit, std::size_t { 16 } ); + } + else if( last_apply >= 16.0 ) { + limit = std::min( limit, std::size_t { 24 } ); + } + else if( last_apply >= 10.0 ) { + limit = std::min( limit, std::size_t { 32 } ); + } + + limit = std::min( limit, scene_apply_pressure_slice_cap() ); } - auto const last_apply { pack_bench_stream().last_apply_ms }; - if( last_apply >= 24.0 ) { - limit = std::min( limit, std::size_t { 16 } ); + if( ring_deficit ) { + auto const target_ms { + g_stream.last_inner_ring < 0.5f ? + kSmoothApplyDeficitTargetMs : + kSmoothApplyTargetMs }; + limit = std::min( limit, apply_time_target_slice( target_ms, std::size_t { 28 } ) ); + if( pack_bench_stream().last_chunk_ms >= 12.0 ) { + limit = std::min( limit, std::size_t { 20 } ); + } + return std::max( limit, std::size_t { 8 } ); } - else if( last_apply >= 16.0 ) { - limit = std::min( limit, std::size_t { 24 } ); - } - else if( last_apply >= 10.0 ) { - limit = std::min( limit, std::size_t { 32 } ); - } - - limit = std::min( limit, scene_apply_pressure_slice_cap() ); return std::max( limit, std::size_t { 16 } ); } @@ -1056,12 +1223,12 @@ adaptive_cold_meshes( bool const urgent = false ) { auto limit { urgent ? kUrgentSliceColdMeshes : gameplay_slice_cold_meshes() }; if( stream_ring_deficit() ) { - limit = std::max( limit, urgent ? std::size_t { 8 } : std::size_t { 6 } ); + limit = std::max( limit, urgent ? std::size_t { 12 } : std::size_t { 8 } ); } - if( pack_bench_stream().last_chunk_ms >= 40.0 ) { + if( pack_bench_stream().main_chunk_cold_ms >= 12.0 ) { limit = std::min( limit, std::size_t { 2 } ); } - else if( pack_bench_stream().last_chunk_ms >= 24.0 ) { + else if( pack_bench_stream().main_chunk_cold_ms >= 6.0 ) { limit = std::min( limit, std::max( limit / 2, std::size_t { 3 } ) ); } return std::max( limit, std::size_t { 2 } ); @@ -1083,34 +1250,44 @@ preload_slice_cold_meshes( std::size_t const count, std::size_t const max_cold_meshes, double const cold_budget_ms, - std::size_t &slice_count ) { + std::size_t &slice_count, + Eu7PackSectionPathTables const *path_tables = nullptr ) { slice_count = 0; if( models == nullptr || count == 0 ) { return 0; } + (void)drain_pack_mesh_loader_ready( g_stream.mesh_cache ); + auto mesh_ready { [&]( std::size_t const index ) -> bool { - auto model_file { models[ index ].model_file }; - if( model_file.empty() || model_file == "notload" ) { + if( eu7_pack_model_needs_full_load( models[ index ] ) ) { return true; } - replace_slashes( model_file ); - if( g_stream.mesh_cache.contains( model_file ) ) { - pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit ); - return true; - } - return false; + return pack_mesh_ready_for_slice( + std::string( pack_model_mesh_path( models[ index ], path_tables ) ) ); } }; std::size_t cold_loaded { 0 }; auto const cold_started { std::chrono::steady_clock::now() }; for( std::size_t i { 0 }; i < count; ++i ) { + if( eu7_pack_model_needs_full_load( models[ i ] ) ) { + auto model_file { + std::string( pack_model_mesh_path( models[ i ], path_tables ) ) }; + if( false == model_file.empty() && model_file != "notload" ) { + replace_slashes( model_file ); + if( false == pack_mesh_ready_for_slice( model_file ) ) { + request_pack_mesh_load( model_file ); + } + } + continue; + } if( mesh_ready( i ) ) { continue; } if( cold_loaded >= max_cold_meshes ) { break; } + auto remaining_budget_ms { cold_budget_ms }; if( cold_budget_ms > 0.0 && cold_loaded > 0 ) { auto const elapsed_ms { std::chrono::duration( @@ -1118,15 +1295,19 @@ preload_slice_cold_meshes( if( elapsed_ms >= cold_budget_ms ) { break; } + remaining_budget_ms = cold_budget_ms - elapsed_ms; } - auto model_file { models[ i ].model_file }; + auto model_file { std::string( pack_model_mesh_path( models[ i ], path_tables ) ) }; replace_slashes( model_file ); - { - PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms }; - (void)ensure_stream_pack_mesh( model_file ); + if( ensure_stream_pack_mesh( model_file, remaining_budget_ms ) ) { + ++cold_loaded; pack_bench_inc( &Eu7PackBench::main_cold_getmodel_calls ); + continue; + } + request_pack_mesh_load( model_file ); + if( gameplay_stream_mode() ) { + break; } - ++cold_loaded; } for( std::size_t i { 0 }; i < count; ++i ) { @@ -1137,241 +1318,243 @@ preload_slice_cold_meshes( slice_count = i + 1; } - if( slice_count == 0 ) { - auto model_file { models[ 0 ].model_file }; + if( slice_count == 0 && count > 0 && pending_apply_is_urgent() ) { + auto model_file { + std::string( pack_model_mesh_path( models[ 0 ], path_tables ) ) }; if( model_file.empty() || model_file == "notload" ) { slice_count = 1; - return cold_loaded; } - replace_slashes( model_file ); - if( false == g_stream.mesh_cache.contains( model_file ) && cold_loaded == 0 ) { - { - PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms }; - (void)ensure_stream_pack_mesh( model_file ); - pack_bench_inc( &Eu7PackBench::main_cold_getmodel_calls ); + else { + replace_slashes( model_file ); + auto const wait_budget { std::min( cold_budget_ms, 8.0 ) }; + if( ensure_stream_pack_mesh( model_file, wait_budget ) ) { + ++cold_loaded; + } + if( mesh_ready( 0 ) ) { + slice_count = 1; } - cold_loaded = 1; } - slice_count = 1; } return cold_loaded; } -[[nodiscard]] std::size_t -preload_umes_cold_meshes( - std::vector const &unique_meshes, - std::size_t const offset, - std::size_t const max_cold_meshes, - double const cold_budget_ms, - std::size_t &offset_out ) { - offset_out = offset; - if( unique_meshes.empty() || offset >= unique_meshes.size() ) { - return 0; - } - - std::size_t cold_loaded { 0 }; - auto const cold_started { std::chrono::steady_clock::now() }; - for( std::size_t i { offset }; i < unique_meshes.size(); ++i ) { - auto model_file { unique_meshes[ i ] }; - if( model_file.empty() || model_file == "notload" ) { - offset_out = i + 1; - continue; - } - replace_slashes( model_file ); - if( g_stream.mesh_cache.contains( model_file ) ) { - pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit ); - offset_out = i + 1; - continue; - } - if( cold_loaded >= max_cold_meshes ) { - pack_bench_inc( &Eu7PackBench::stream_cold_slice_truncated ); - break; - } - if( cold_budget_ms > 0.0 && cold_loaded > 0 ) { - auto const elapsed_ms { - std::chrono::duration( - std::chrono::steady_clock::now() - cold_started ).count() }; - if( elapsed_ms >= cold_budget_ms ) { - pack_bench_inc( &Eu7PackBench::stream_cold_slice_truncated ); - break; - } - } - { - PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms }; - (void)ensure_stream_pack_mesh( model_file ); - pack_bench_inc( &Eu7PackBench::main_cold_getmodel_calls ); - } - ++cold_loaded; - offset_out = i + 1; - } - - return cold_loaded; -} - -constexpr int kApplyStuckSkipFrames { 20 }; - -std::size_t g_apply_stuck_offset { std::numeric_limits::max() }; -int g_apply_stuck_frames { 0 }; - +template void -note_apply_progress() { - g_apply_stuck_offset = std::numeric_limits::max(); - g_apply_stuck_frames = 0; +for_each_chunk_mesh_path( + PackSectionReady const &batch, + Fn const &fn ) { + if( batch.models == nullptr ) { + return; + } + + std::unordered_set seen; + seen.reserve( batch.models->size() ); + for( auto const &model : *batch.models ) { + if( eu7_pack_model_needs_full_load( model ) ) { + continue; + } + auto const model_file { + pack_model_mesh_path( model, batch.path_tables.get() ) }; + if( model_file.empty() || model_file == "notload" ) { + continue; + } + auto normalized { std::string( model_file ) }; + replace_slashes( normalized ); + if( false == seen.insert( normalized ).second ) { + continue; + } + fn( normalized ); + } } [[nodiscard]] bool -maybe_skip_stuck_apply_offset() { - if( false == g_stream.pending_apply.has_value() ) { - return false; - } +file_chunk_meshes_ready( + PackSectionReady const &batch, + std::size_t const model_offset = 0, + std::size_t const model_count = std::numeric_limits::max() ); - auto const offset { g_stream.pending_apply_offset }; - if( offset == g_apply_stuck_offset ) { - ++g_apply_stuck_frames; - } - else { - g_apply_stuck_offset = offset; - g_apply_stuck_frames = 0; - } +void +request_missing_file_chunk_meshes( + PackSectionReady const &batch, + std::size_t const model_offset = 0, + std::size_t const model_count = std::numeric_limits::max() ); - if( g_apply_stuck_frames < kApplyStuckSkipFrames ) { - return false; +[[nodiscard]] bool +has_nearer_mesh_ready_batch( int const pending_dist ) { + std::lock_guard lock { g_stream.ready.mutex }; + for( auto const &ready : g_stream.ready.data ) { + if( ready.failed || ready.models == nullptr || ready.models->empty() ) { + continue; + } + if( false == file_chunk_meshes_ready( ready ) ) { + continue; + } + if( ready.row == g_stream.center_row && ready.column == g_stream.center_column ) { + return true; + } + auto const dist { + section_manhattan_sections( + ready.row, + ready.column, + g_stream.center_row, + g_stream.center_column ) }; + if( dist + 1 < pending_dist ) { + return true; + } } - - auto const total { - g_stream.pending_apply->models != nullptr ? - g_stream.pending_apply->models->size() : - 0 }; - if( offset >= total ) { - return false; - } - - auto model_file { ( *g_stream.pending_apply->models )[ offset ].model_file }; - WriteLog( - "EU7 PACK: skip stuck model offset=" + std::to_string( offset ) + "/" + - std::to_string( total ) + " sec=" + std::to_string( g_stream.pending_apply->row ) + "," + - std::to_string( g_stream.pending_apply->column ) + - ( model_file.empty() ? "" : " file=\"" + model_file + "\"" ) ); - if( false == model_file.empty() && model_file != "notload" ) { - replace_slashes( model_file ); - touch_string_lru( model_file, g_stream.mesh_lru, g_stream.mesh_lru_iters ); - g_stream.mesh_cache.emplace( std::move( model_file ), nullptr ); - } - g_stream.pending_apply_offset = offset + 1; - note_apply_progress(); - pack_bench_inc( &Eu7PackBench::stream_apply_stuck_skip ); - return true; + return false; } void -maybe_preempt_distant_pending() { - if( - false == g_loading_screen_dismissed || - g_stream.bootstrap_active || - g_stream.bootstrap_pending || - false == g_stream.pending_apply.has_value() ) { +suspend_pending_apply_to_ready() { + if( false == g_stream.pending_apply.has_value() ) { return; } - if( ready_queue_size() >= kPreemptReadyQueueThreshold ) { - return; - } - if( camera_stream_speed_mps() > kPreemptDisableSpeedMps ) { - return; - } - if( g_stream.last_outer_ring < kRingBoostOuterThreshold ) { - return; - } + g_stream.pending_apply->apply_offset = g_stream.pending_apply_offset; + PackSectionReady suspended { std::move( *g_stream.pending_apply ) }; + g_stream.pending_apply.reset(); + g_stream.pending_apply_offset = 0; + note_apply_progress(); - auto &batch { *g_stream.pending_apply }; + { + std::lock_guard lock { g_stream.ready.mutex }; + g_stream.ready.data.push_back( std::move( suspended ) ); + } + pack_bench_inc( &Eu7PackBench::stream_reenqueue ); +} + +[[nodiscard]] bool +file_chunk_meshes_ready( + PackSectionReady const &batch, + std::size_t const model_offset, + std::size_t const model_count ) { if( batch.models == nullptr || batch.models->empty() ) { - return; - } - if( g_stream.pending_apply_offset >= batch.models->size() ) { - return; + return false; } - auto const pending_dist { + auto const end { + std::min( batch.models->size(), model_offset + model_count ) }; + if( model_offset >= end ) { + return true; + } + + bool needs_mesh { false }; + bool all_ready { true }; + std::unordered_set seen; + seen.reserve( std::min( end - model_offset, std::size_t { 64 } ) ); + for( std::size_t i { model_offset }; i < end; ++i ) { + auto const &model { ( *batch.models )[ i ] }; + if( eu7_pack_model_needs_full_load( model ) ) { + continue; + } + auto const model_file { + pack_model_mesh_path( model, batch.path_tables.get() ) }; + if( model_file.empty() || model_file == "notload" ) { + continue; + } + auto normalized { std::string( model_file ) }; + replace_slashes( normalized ); + if( false == seen.insert( normalized ).second ) { + continue; + } + needs_mesh = true; + if( false == pack_mesh_ready_for_slice( normalized ) ) { + all_ready = false; + } + } + return needs_mesh ? all_ready : true; +} + +void +request_missing_file_chunk_meshes( + PackSectionReady const &batch, + std::size_t const model_offset, + std::size_t const model_count ) { + auto const dist { section_manhattan_sections( batch.row, batch.column, g_stream.center_row, g_stream.center_column ) }; - if( pending_dist <= kUrgentSectionDistanceKm ) { - return; - } - if( pending_dist < 5 ) { + + auto const end { + std::min( batch.models->size(), model_offset + model_count ) }; + if( batch.models == nullptr || model_offset >= end ) { return; } - auto const total { batch.models->size() }; - if( - total > 0 && - g_stream.pending_apply_offset > total / 2 ) { - return; - } - - auto const speed { camera_stream_speed_mps() }; - auto const near_apply_km { - speed > 600.0 ? kFastFlightApplyMaxDistanceKm : kUrgentSectionDistanceKm }; - - bool nearer_ready { false }; - { - std::lock_guard lock { g_stream.ready.mutex }; - for( auto const &ready : g_stream.ready.data ) { - if( ready.failed || ready.models == nullptr || ready.models->empty() ) { - continue; - } - auto const ready_dist { - section_manhattan_sections( - ready.row, - ready.column, - g_stream.center_row, - g_stream.center_column ) }; - if( - ready.row == g_stream.center_row && - ready.column == g_stream.center_column ) { - nearer_ready = true; - break; - } - if( - ready_dist + 3 < pending_dist && - ready_dist <= near_apply_km ) { - nearer_ready = true; - break; - } + std::unordered_set seen; + seen.reserve( std::min( end - model_offset, std::size_t { 64 } ) ); + for( std::size_t i { model_offset }; i < end; ++i ) { + auto const &model { ( *batch.models )[ i ] }; + if( eu7_pack_model_needs_full_load( model ) ) { + continue; + } + auto const model_file { + pack_model_mesh_path( model, batch.path_tables.get() ) }; + if( model_file.empty() || model_file == "notload" ) { + continue; + } + auto normalized { std::string( model_file ) }; + replace_slashes( normalized ); + if( false == seen.insert( normalized ).second ) { + continue; + } + if( false == pack_mesh_ready_for_slice( normalized ) ) { + request_pack_mesh_load( normalized, dist ); } } +} - if( false == nearer_ready ) { - if( - pending_dist <= kPreemptPendingDistanceKm || - camera_stream_speed_mps() <= 80.0 ) { +void +block_until_file_chunk_meshes( PackSectionReady const &batch, double const block_budget_ms ) { + for_each_chunk_mesh_path( batch, [&]( std::string const &path ) { + if( pack_mesh_ready_for_slice( path ) ) { return; } + (void)ensure_stream_pack_mesh( path, block_budget_ms ); + } ); +} - auto const [cam_row, cam_col] { - section_row_column( resolve_section_stream_position( Global.pCamera.Pos ) ) }; - if( section_has_pack_models( cam_row, cam_col ) ) { - nearer_ready = true; - } - } +[[nodiscard]] bool +batch_apply_meshes_ready( + PackSectionReady const &batch, + std::size_t const offset ) { + (void)offset; + return file_chunk_meshes_ready( batch ); +} - if( false == nearer_ready ) { - return; - } +void +request_missing_batch_meshes( + PackSectionReady const &batch, + std::size_t const offset ) { + (void)offset; + request_missing_file_chunk_meshes( batch ); +} - batch.apply_offset = g_stream.pending_apply_offset; - auto suspended { std::move( batch ) }; - g_stream.pending_apply.reset(); - g_stream.pending_apply_offset = 0; +void +prefetch_pack_section_meshes( + Eu7Module const &module, + int const row, + int const column ) { + auto const dist { + section_manhattan_sections( + row, + column, + g_stream.center_row, + g_stream.center_column ) }; + for_each_pack_section_unique_mesh( + module, + row, + column, + [&]( std::string const &path ) { + request_pack_mesh_load( path, dist ); + } ); +} - { - std::lock_guard lock { g_stream.ready.mutex }; - g_stream.ready.data.push_front( std::move( suspended ) ); - } - pack_bench_inc( &Eu7PackBench::stream_preempt_pending ); - pack_bench_inc( &Eu7PackBench::stream_reenqueue ); +void +note_apply_progress() { } [[nodiscard]] bool @@ -1381,13 +1564,11 @@ apply_pending_chunk( std::size_t const max_cold_meshes, double const cold_budget_ms, std::size_t const max_chunks ) { + (void)max_cold_meshes; + (void)cold_budget_ms; if( g_stream.serializer == nullptr ) { return false; } - if( maybe_skip_stuck_apply_offset() ) { - return true; - } - maybe_preempt_distant_pending(); if( false == g_stream.pending_apply.has_value() && false == try_dequeue_ready_batch() ) { return false; } @@ -1427,53 +1608,41 @@ apply_pending_chunk( applied_work = true; continue; } - - auto const remaining { total - offset }; - auto const pending_dist { pending_section_distance_km() }; - auto const cam_speed { camera_stream_speed_mps() }; - auto const is_urgent { pending_apply_is_urgent() }; - auto const is_heavy { - false == batch.unique_meshes.empty() || - false == batch.unique_textures.empty() || - total >= kHeavySectionModelThreshold || - remaining >= kHeavySectionModelThreshold }; - auto const far_slow_drain { - cam_speed > 300.0 && - pending_dist > kFarApplySlowDistanceKm && - false == is_urgent && - false == stream_ring_deficit() }; - - std::size_t chunk_count { remaining }; - std::size_t cold_mesh_limit { remaining }; - double effective_cold_budget_ms { 0.0 }; - - if( is_heavy ) { - chunk_count = std::min( remaining, adaptive_slice_instances( total, is_urgent ) ); - cold_mesh_limit = adaptive_cold_meshes( is_urgent ); - effective_cold_budget_ms = - is_urgent ? kUrgentColdBudgetMs : gameplay_cold_budget_ms(); - if( far_slow_drain ) { - chunk_count = std::min( chunk_count, std::size_t { 32 } ); - cold_mesh_limit = std::min( cold_mesh_limit, std::size_t { 1 } ); - effective_cold_budget_ms = std::min( effective_cold_budget_ms, 8.0 ); - } - if( max_instances > 0 ) { - chunk_count = std::min( chunk_count, max_instances ); - } - if( max_cold_meshes > 0 ) { - cold_mesh_limit = std::min( cold_mesh_limit, max_cold_meshes ); - } - if( cold_budget_ms > 0.0 ) { - effective_cold_budget_ms = cold_budget_ms; + if( offset >= total ) { + if( batch.section_final ) { + finalize_section( batch ); } + release_pending_buffer(); + applied_work = true; + continue; } + auto const remaining { total - offset }; + auto const is_urgent { pending_apply_is_urgent() }; + auto const loading { false == g_loading_screen_dismissed }; + + std::size_t slice_limit { remaining }; + if( loading ) { + slice_limit = std::min( + remaining, + max_instances > 0 ? max_instances : kLoaderSliceInstances ); + } + else if( max_instances > 0 ) { + slice_limit = std::min( remaining, max_instances ); + } + else { + slice_limit = std::min( + remaining, + adaptive_slice_instances( total, is_urgent ) ); + } + + std::size_t const chunk_count { slice_limit }; auto const planned_inst { chunk_count }; pack_bench_inc( &Eu7PackBench::stream_inst_planned, planned_inst ); if( is_urgent ) { pack_bench_inc( &Eu7PackBench::stream_chunks_urgent ); } - else if( is_heavy ) { + else if( chunk_count >= kHeavySectionModelThreshold ) { pack_bench_inc( &Eu7PackBench::stream_chunks_heavy ); } else { @@ -1487,42 +1656,65 @@ apply_pending_chunk( double apply_ms { 0.0 }; std::size_t cold_loads { 0 }; std::size_t tex_fetches { 0 }; + auto const pending_dist { + section_manhattan_sections( + batch.row, + batch.column, + g_stream.center_row, + g_stream.center_column ) }; + + resolve_pack_model_paths( + batch.models->data() + offset, + chunk_count, + batch.path_tables.get() ); { auto const phase_started { std::chrono::steady_clock::now() }; - if( false == batch.unique_meshes.empty() ) { - cold_loads += preload_umes_cold_meshes( - batch.unique_meshes, - batch.umes_cold_offset, - cold_mesh_limit, - effective_cold_budget_ms, - batch.umes_cold_offset ); + auto const loading_phase { false == g_loading_screen_dismissed }; + auto const gameplay { gameplay_stream_mode() }; + (void)drain_pack_mesh_loader_ready( g_stream.mesh_cache ); + + if( false == file_chunk_meshes_ready( batch, offset, chunk_count ) ) { + pack_bench_inc( &Eu7PackBench::stream_apply_deferred ); + request_missing_file_chunk_meshes( batch, offset, chunk_count ); + + if( gameplay ) { + suspend_pending_apply_to_ready(); + return applied_work; + } + + if( has_nearer_mesh_ready_batch( pending_dist ) ) { + suspend_pending_apply_to_ready(); + return applied_work; + } + + block_until_file_chunk_meshes( batch, std::max( budget_ms, 16.0 ) ); + + if( false == file_chunk_meshes_ready( batch, offset, chunk_count ) ) { + if( has_nearer_mesh_ready_batch( pending_dist ) ) { + suspend_pending_apply_to_ready(); + } + return applied_work; + } } - std::size_t inst_slice_count { chunk_count }; - cold_loads += preload_slice_cold_meshes( - models_ptr + offset, - chunk_count, - cold_mesh_limit, - effective_cold_budget_ms, - inst_slice_count ); - chunk_count = inst_slice_count; + else if( loading_phase ) { + (void)pump_pack_mesh_loader( 8.0, 4 ); + } + cold_ms = std::chrono::duration( std::chrono::steady_clock::now() - phase_started ).count(); } pack_bench_inc( &Eu7PackBench::stream_inst_after_cold, chunk_count ); - if( chunk_count < planned_inst ) { - pack_bench_inc( &Eu7PackBench::stream_cold_slice_truncated ); - } - if( chunk_count == 0 ) { - g_stream.pending_apply_offset = offset + 1; - note_apply_progress(); - applied_work = true; - continue; - } { auto const phase_started { std::chrono::steady_clock::now() }; - tex_fetches = warm_pack_section_textures( batch, offset, chunk_count ); + if( false == gameplay_stream_mode() ) { + tex_fetches = warm_pack_section_textures( batch, offset, chunk_count ); + } + else { + tex_fetches = warm_pack_section_textures( + batch, offset, chunk_count, kPackTextureWarmBudgetMs ); + } warm_ms = std::chrono::duration( std::chrono::steady_clock::now() - phase_started ).count(); } @@ -1546,7 +1738,11 @@ apply_pending_chunk( } retain_section_mesh_keys( batch.section_idx, models_ptr, offset, chunk_count ); retain_section_nodedata_keys( batch.section_idx, models_ptr, offset, chunk_count ); - evict_unreferenced_stream_caches(); + if( + g_stream.mesh_lru.size() > kMeshCacheLruCap || + g_stream.nodedata_lru.size() > kNodedataCacheLruCap ) { + evict_unreferenced_stream_caches(); + } auto const chunk_wall_ms { std::chrono::duration( @@ -1567,15 +1763,23 @@ apply_pending_chunk( ( "/" + std::to_string( planned_inst ) ) : std::string{} ) + " urg=" + std::to_string( is_urgent ? 1 : 0 ) + - " heavy=" + std::to_string( is_heavy ? 1 : 0 ) + + " heavy=" + std::to_string( chunk_count >= kHeavySectionModelThreshold ? 1 : 0 ) + " cold=" + std::to_string( cold_loads ) + "(" + std::to_string( static_cast( cold_ms ) ) + "ms)" + " warm=" + std::to_string( tex_fetches ) + "(" + std::to_string( static_cast( warm_ms ) ) + "ms)" + " apply=" + std::to_string( static_cast( apply_ms ) ) + "ms" + + " ring=" + std::to_string( static_cast( g_stream.last_inner_ring * 100.f ) ) + "/" + + std::to_string( static_cast( g_stream.last_outer_ring * 100.f ) ) + " pending=" + std::to_string( offset + chunk_count ) + "/" + std::to_string( total ) + " sec=" + std::to_string( batch.row ) + "," + std::to_string( batch.column ) ); } load_stats().pack_models += chunk_count; + if( load_stats().pack_models == chunk_count && offset == 0 ) { + WriteLog( + "EU7 PACK: pierwszy apply CHNK inst=" + std::to_string( chunk_count ) + + " sec=" + std::to_string( batch.row ) + "," + std::to_string( batch.column ) ); + } g_stream.pending_apply_offset = offset + chunk_count; + batch.apply_offset = g_stream.pending_apply_offset; note_apply_progress(); applied_work = true; ++chunks_done; @@ -1585,7 +1789,6 @@ apply_pending_chunk( finalize_section( batch ); } release_pending_buffer(); - continue; } if( budget_ms <= 0.0 ) { @@ -1599,11 +1802,44 @@ apply_pending_chunk( pack_bench_inc( &Eu7PackBench::drain_budget_stops ); return applied_work; } + continue; } return applied_work; } +void +maybe_log_loading_stream_status() { + if( false == g_stream.active || g_loading_screen_dismissed ) { + return; + } + + static double s_last_log { 0.0 }; + auto const now { Timer::GetTime() }; + if( now - s_last_log < kStreamStatusLogIntervalSec ) { + return; + } + s_last_log = now; + + std::size_t pending_total { 0 }; + if( g_stream.pending_apply.has_value() && g_stream.pending_apply->models != nullptr ) { + pending_total = g_stream.pending_apply->models->size(); + } + std::size_t job_count { 0 }; + { + std::lock_guard lock { g_stream.jobs.mutex }; + job_count = g_stream.jobs.data.size(); + } + + WriteLog( + "EU7 PACK [loading]: pack_models=" + std::to_string( load_stats().pack_models ) + + " ready_q=" + std::to_string( ready_queue_size() ) + + " in_flight=" + std::to_string( g_stream.in_flight_sections.size() ) + + " pending=" + std::to_string( g_stream.pending_apply_offset ) + "/" + + std::to_string( pending_total ) + + " jobs=" + std::to_string( job_count ) ); +} + void maybe_log_stream_status( glm::dvec3 const &world_position ) { if( false == pack_bench_stream_phase_active() ) { @@ -1621,7 +1857,7 @@ maybe_log_stream_status( glm::dvec3 const &world_position ) { auto const inner_ring { section_stream_ring_progress( world_position, kSectionStreamGameplayRadiusKm ) }; auto const outer_ring { - section_stream_ring_progress( world_position, g_stream.radius ) }; + section_stream_ring_progress( world_position, kSectionStreamTargetRadiusKm ) }; std::size_t pending_total { 0 }; std::size_t pending_offset { g_stream.pending_apply_offset }; @@ -1707,6 +1943,11 @@ worker_loop( std::stop_token const stop_token ) { try { auto const &module { stream_module() }; + + if( job.next_chunk == 0 ) { + prefetch_pack_section_meshes( module, job.row, job.column ); + } + auto const entry { find_pack_entry( module, job.row, job.column ) }; if( false == entry.has_value() || entry->model_count == 0 ) { enqueue_failed_section( job ); @@ -1714,17 +1955,17 @@ worker_loop( std::stop_token const stop_token ) { } std::unique_ptr> models; - std::vector unique_meshes; std::vector unique_textures; + std::shared_ptr path_tables; std::uint32_t chunk_count { 1 }; { PackBenchTimer const read_timer { &Eu7PackBench::worker_read_pack_ms }; - auto const chunk { + auto chunk { read_pack_section_chunk_load( module, job.row, job.column, job.next_chunk ) }; chunk_count = chunk.chunk_count; - unique_meshes = std::move( chunk.unique_meshes ); unique_textures = std::move( chunk.unique_textures ); + path_tables = std::move( chunk.path_tables ); if( false == chunk.models.empty() ) { models = std::make_unique>( std::move( chunk.models ) ); @@ -1736,8 +1977,8 @@ worker_loop( std::stop_token const stop_token ) { result.column = job.column; result.section_idx = job.section_idx; result.models = std::move( models ); - result.unique_meshes = std::move( unique_meshes ); result.unique_textures = std::move( unique_textures ); + result.path_tables = std::move( path_tables ); result.failed = result.models == nullptr; result.section_final = job.next_chunk + 1 >= chunk_count; @@ -1754,13 +1995,14 @@ worker_loop( std::stop_token const stop_token ) { pack_bench_inc( &Eu7PackBench::worker_models_decoded, result.models->size() ); + auto const section_final { result.section_final }; { std::lock_guard lock { g_stream.ready.mutex }; g_stream.ready.data.push_back( std::move( result ) ); } pack_bench_inc( &Eu7PackBench::worker_chunks_decoded ); - if( result.section_final ) { + if( section_final ) { pack_bench_inc( &Eu7PackBench::worker_sections_done ); } @@ -1795,6 +2037,7 @@ worker_loop( std::stop_token const stop_token ) { void stop_workers() { + stop_pack_mesh_loader(); g_stream.worker_exit = true; g_stream.work_cv.notify_all(); g_stream.workers.clear(); @@ -1834,11 +2077,15 @@ start_workers() { for( std::size_t worker_idx { 0 }; worker_idx < worker_count; ++worker_idx ) { g_stream.workers.emplace_back( worker_loop ); } + start_pack_mesh_loader(); WriteLog( - "EU7 PACK: async loader started, workers=" + std::to_string( worker_count ) + - ", radius=" + std::to_string( kStreamRadius ) + - ", bootstrap=" + std::to_string( kInitialBootstrapRadius ) + "km" + + "EU7 PACK: mesh queue loader started (main-thread), stream_workers=" + + std::to_string( worker_count ) + + ", radius=" + std::to_string( kSectionStreamTargetRadiusKm ) + + "km, bootstrap_enqueue=" + std::to_string( kInitialBootstrapEnqueueRadius ) + "km" + + ", bootstrap_replenish=" + std::to_string( kInitialBootstrapRadius ) + "km" + + ", maintenance_inner=" + std::to_string( kSectionStreamGameplayRadiusKm ) + "km" + ", lookahead=" + std::to_string( kMovementLookahead ) ); } @@ -1943,6 +2190,10 @@ should_block_far_enqueue( return false; } + if( in_radius_fill_phase() ) { + return false; + } + if( lookahead_enqueue && g_stream.last_inner_ring < kRingGatedInnerTarget ) { @@ -2058,6 +2309,10 @@ enqueue_section_if_needed( g_stream.in_flight_sections.insert( section_idx ); + if( g_stream.module != nullptr ) { + prefetch_pack_section_meshes( *g_stream.module, row, column ); + } + PackSectionJob job; job.row = row; job.column = column; @@ -2143,7 +2398,81 @@ replenish_bootstrap_ring( glm::dvec3 const &world_position ) { auto const [row, column] { section_row_column( world_position ) }; g_stream.center_row = row; g_stream.center_column = column; - enqueue_sections_around( row, column, kInitialBootstrapRadius, world_position ); + + if( false == g_loading_screen_dismissed ) { + enqueue_sections_around( row, column, kInitialBootstrapEnqueueRadius, world_position ); + } + else if( in_radius_fill_phase() ) { + enqueue_sections_around( + row, column, kSectionStreamTargetRadiusKm, world_position ); + } + else { + enqueue_sections_around( + row, column, kSectionStreamGameplayRadiusKm, world_position ); + } +} + +[[nodiscard]] float +section_ring_apply_fraction( int const row, int const column ) { + auto const section_idx { section_index( row, column ) }; + + if( g_stream.loaded_sections.contains( section_idx ) ) { + return 1.0f; + } + + if( + g_stream.pending_apply.has_value() && + g_stream.pending_apply->section_idx == section_idx && + g_stream.pending_apply->models != nullptr && + false == g_stream.pending_apply->models->empty() ) { + auto const total { g_stream.pending_apply->models->size() }; + if( g_stream.pending_apply_offset >= total ) { + return 1.0f; + } + return static_cast( g_stream.pending_apply_offset ) / + static_cast( total ); + } + + { + std::lock_guard lock { g_stream.ready.mutex }; + for( auto const &batch : g_stream.ready.data ) { + if( batch.section_idx == section_idx ) { + return 0.25f; + } + } + } + + if( g_stream.in_flight_sections.contains( section_idx ) ) { + return 0.1f; + } + + return 0.f; +} + +[[nodiscard]] bool +center_section_stream_satisfied( glm::dvec3 const &world_position ) { + auto const [center_row, center_column] { section_row_column( world_position ) }; + if( false == section_has_pack_models( center_row, center_column ) ) { + return true; + } + + auto const section_idx { section_index( center_row, center_column ) }; + if( g_stream.loaded_sections.contains( section_idx ) ) { + return true; + } + + if( + g_stream.pending_apply.has_value() && + g_stream.pending_apply->section_idx == section_idx && + g_stream.pending_apply->models != nullptr && + false == g_stream.pending_apply->models->empty() ) { + auto const total { g_stream.pending_apply->models->size() }; + if( g_stream.pending_apply_offset >= total ) { + return true; + } + } + + return false; } [[nodiscard]] std::pair @@ -2273,8 +2602,6 @@ try_dequeue_ready_batch() { return true; } - PackSectionReady batch; - bool has_batch { false }; { std::lock_guard lock { g_stream.ready.mutex }; for( auto it { g_stream.ready.data.begin() }; it != g_stream.ready.data.end(); ) { @@ -2291,97 +2618,119 @@ try_dequeue_ready_batch() { ++it; } - if( false == g_stream.ready.data.empty() ) { - auto const pick_best { - [&]( int const max_distance ) -> std::deque::iterator { - auto best_it { g_stream.ready.data.end() }; - auto best_dist { std::numeric_limits::max() }; - for( auto it { g_stream.ready.data.begin() }; it != g_stream.ready.data.end(); ++it ) { - if( it->failed || it->models == nullptr || it->models->empty() ) { - continue; - } - auto const dist { - section_manhattan_sections( - it->row, - it->column, - g_stream.center_row, - g_stream.center_column ) }; - if( max_distance >= 0 && dist > max_distance ) { - continue; - } - if( dist < best_dist || ( dist == best_dist && best_it == g_stream.ready.data.end() ) ) { - best_dist = dist; - best_it = it; - } - else if( - dist == best_dist && best_it != g_stream.ready.data.end() && - it->section_idx < best_it->section_idx ) { - best_it = it; - } + if( g_stream.ready.data.empty() ) { + return false; + } + + auto const gameplay { gameplay_stream_mode() }; + + auto const pick_best { + [&]( int const max_distance, bool const require_mesh_ready ) + -> std::deque::iterator { + auto best_it { g_stream.ready.data.end() }; + auto best_dist { std::numeric_limits::max() }; + auto best_mesh_ready { false }; + for( auto it { g_stream.ready.data.begin() }; it != g_stream.ready.data.end(); ++it ) { + if( it->failed || it->models == nullptr || it->models->empty() ) { + continue; } - return best_it; - } }; - - auto const pick_camera_section { - [&]() -> std::deque::iterator { - for( auto it { g_stream.ready.data.begin() }; it != g_stream.ready.data.end(); ++it ) { - if( it->failed || it->models == nullptr || it->models->empty() ) { - continue; - } - if( - it->row == g_stream.center_row && - it->column == g_stream.center_column ) { - return it; - } + auto const dist { + section_manhattan_sections( + it->row, + it->column, + g_stream.center_row, + g_stream.center_column ) }; + if( max_distance >= 0 && dist > max_distance ) { + continue; } - return g_stream.ready.data.end(); - } }; + auto const mesh_ready { file_chunk_meshes_ready( *it ) }; + if( gameplay && require_mesh_ready && false == mesh_ready ) { + continue; + } + if( + dist < best_dist || + ( dist == best_dist && mesh_ready && false == best_mesh_ready ) || + ( dist == best_dist && best_it == g_stream.ready.data.end() ) ) { + best_dist = dist; + best_it = it; + best_mesh_ready = mesh_ready; + } + else if( + dist == best_dist && best_it != g_stream.ready.data.end() && + it->section_idx < best_it->section_idx && + mesh_ready == best_mesh_ready ) { + best_it = it; + } + } + return best_it; + } }; - auto const cam_speed { camera_stream_speed_mps() }; - auto const max_apply_dist { - cam_speed > 600.0 ? - kFastFlightApplyMaxDistanceKm : - ( cam_speed > 80.0 ? 3 : kSectionStreamGameplayRadiusKm ) }; + auto const pick_camera_section { + [&]() -> std::deque::iterator { + for( auto it { g_stream.ready.data.begin() }; it != g_stream.ready.data.end(); ++it ) { + if( it->failed || it->models == nullptr || it->models->empty() ) { + continue; + } + if( + it->row == g_stream.center_row && + it->column == g_stream.center_column ) { + if( gameplay && false == file_chunk_meshes_ready( *it ) ) { + return g_stream.ready.data.end(); + } + return it; + } + } + return g_stream.ready.data.end(); + } }; - auto best_it { pick_camera_section() }; - if( best_it != g_stream.ready.data.end() ) { - pack_bench_inc( &Eu7PackBench::stream_dequeue_camera ); - } - else { - best_it = pick_best( max_apply_dist ); - } - if( best_it == g_stream.ready.data.end() ) { - best_it = pick_best( kSectionStreamGameplayRadiusKm ); - } - if( best_it == g_stream.ready.data.end() && false == g_stream.ready.data.empty() ) { - best_it = pick_best( -1 ); - } - if( best_it == g_stream.ready.data.end() ) { - return false; - } - batch = std::move( *best_it ); - g_stream.ready.data.erase( best_it ); - has_batch = true; + auto const cam_speed { camera_stream_speed_mps() }; + auto const ring_deficit { stream_ring_deficit() }; + auto const radius_fill { in_radius_fill_phase() }; + auto max_apply_dist { + cam_speed > 600.0 ? + kFastFlightApplyMaxDistanceKm : + ( cam_speed > 80.0 ? 3 : kSectionStreamGameplayRadiusKm ) }; + if( ring_deficit && false == radius_fill ) { + max_apply_dist = cam_speed > 200.0 ? + kFastFlightApplyMaxDistanceKm : + kUrgentSectionDistanceKm; } + + auto best_it { pick_camera_section() }; + if( best_it != g_stream.ready.data.end() ) { + pack_bench_inc( &Eu7PackBench::stream_dequeue_camera ); + } + else if( radius_fill ) { + best_it = pick_best( -1, true ); + } + else { + best_it = pick_best( max_apply_dist, true ); + } + if( best_it == g_stream.ready.data.end() && false == radius_fill ) { + auto const wider_dist { + ring_deficit ? + std::max( max_apply_dist, kFastFlightApplyMaxDistanceKm ) : + kSectionStreamGameplayRadiusKm }; + best_it = pick_best( wider_dist, true ); + } + if( best_it == g_stream.ready.data.end() && false == g_stream.ready.data.empty() ) { + best_it = pick_best( -1, true ); + } + if( + best_it == g_stream.ready.data.end() && + gameplay && + false == g_stream.ready.data.empty() ) { + best_it = pick_best( -1, false ); + } + if( best_it == g_stream.ready.data.end() ) { + return false; + } + + g_stream.pending_apply.emplace( std::move( *best_it ) ); + g_stream.ready.data.erase( best_it ); } - if( false == has_batch ) { - return false; - } - - if( batch.failed || batch.models == nullptr || batch.models->empty() ) { - if( batch.failed ) { - fail_section( batch.section_idx, batch.row, batch.column ); - } - else if( batch.section_final ) { - finalize_section( batch ); - } - return false; - } - - g_stream.pending_apply = std::move( batch ); g_stream.pending_apply_offset = g_stream.pending_apply->apply_offset; - g_stream.pending_apply->apply_offset = 0; auto const dequeue_dist { section_manhattan_sections( @@ -2413,20 +2762,14 @@ prefetch_ready_queue_textures( double const budget_ms ) { { std::lock_guard lock { g_stream.ready.mutex }; for( auto it { g_stream.ready.data.begin() }; it != g_stream.ready.data.end(); ++it ) { - if( it->failed ) { + if( it->failed || it->models == nullptr || it->models->empty() ) { continue; } - if( false == it->unique_textures.empty() ) { - if( it->texture_warm_offset >= it->unique_textures.size() ) { - continue; - } - candidates.push_back( it ); - continue; - } - if( it->models == nullptr || it->models->empty() ) { - continue; - } - if( it->texture_warm_offset >= it->models->size() ) { + auto const tex_total { + false == it->unique_textures.empty() ? + it->unique_textures.size() : + it->models->size() }; + if( it->texture_warm_offset >= tex_total ) { continue; } candidates.push_back( it ); @@ -2457,38 +2800,31 @@ prefetch_ready_queue_textures( double const budget_ms ) { return lhs_dist < rhs_dist; } ); + for( auto const it : candidates ) { auto &batch { *it }; - if( false == batch.unique_textures.empty() ) { - while( batch.texture_warm_offset < batch.unique_textures.size() ) { - auto const remaining { - batch.unique_textures.size() - batch.texture_warm_offset }; - auto const slice { std::min( remaining, kReadyTexturePrefetchSlice ) }; - auto const tex_slice { std::min( slice, kPackTextureWarmSlice ) }; - std::size_t processed { 0 }; - (void)warm_pack_texture_paths_main( - batch.unique_textures.data() + batch.texture_warm_offset, - tex_slice, - kPackTextureWarmBudgetMs, - &processed ); - batch.texture_warm_offset += processed; + auto const use_utex { false == batch.unique_textures.empty() }; - auto const elapsed_ms { - std::chrono::duration( - std::chrono::steady_clock::now() - started ).count() }; - if( elapsed_ms >= budget_ms ) { - return; - } + while( true ) { + auto const tex_total { + use_utex ? batch.unique_textures.size() : batch.models->size() }; + if( batch.texture_warm_offset >= tex_total ) { + break; } - continue; - } - while( batch.texture_warm_offset < batch.models->size() ) { - auto const remaining { batch.models->size() - batch.texture_warm_offset }; + auto const remaining { tex_total - batch.texture_warm_offset }; auto const slice { std::min( remaining, kReadyTexturePrefetchSlice ) }; - (void)warm_pack_textures_main( - batch.models->data() + batch.texture_warm_offset, - slice ); + + if( use_utex ) { + (void)warm_pack_texture_paths_main( + batch.unique_textures.data() + batch.texture_warm_offset, + slice ); + } + else { + (void)warm_pack_textures_main( + batch.models->data() + batch.texture_warm_offset, + slice ); + } batch.texture_warm_offset += slice; auto const elapsed_ms { @@ -2516,10 +2852,10 @@ prefetch_ready_queue_umes_worker( double const budget_ms, std::size_t const max_ std::vector::iterator> candidates; candidates.reserve( g_stream.ready.data.size() ); for( auto it { g_stream.ready.data.begin() }; it != g_stream.ready.data.end(); ++it ) { - if( it->failed || it->unique_meshes.empty() ) { + if( it->failed || it->models == nullptr || it->models->empty() ) { continue; } - if( it->umes_prefetch_offset >= it->unique_meshes.size() ) { + if( it->umes_prefetch_offset >= it->models->size() ) { continue; } if( g_stream.center_row >= 0 && g_stream.center_column >= 0 ) { @@ -2529,7 +2865,7 @@ prefetch_ready_queue_umes_worker( double const budget_ms, std::size_t const max_ it->column, g_stream.center_row, g_stream.center_column ) }; - if( dist > kUmesPrefetchMaxDistanceKm ) { + if( dist > umes_prefetch_max_distance_km() ) { continue; } } @@ -2562,15 +2898,18 @@ prefetch_ready_queue_umes_worker( double const budget_ms, std::size_t const max_ for( auto const it : candidates ) { auto &batch { *it }; - while( batch.umes_prefetch_offset < batch.unique_meshes.size() ) { - auto model_file { batch.unique_meshes[ batch.umes_prefetch_offset ] }; - + while( batch.umes_prefetch_offset < batch.models->size() ) { + auto const &model { ( *batch.models )[ batch.umes_prefetch_offset ] }; + if( eu7_pack_model_needs_full_load( model ) ) { + ++batch.umes_prefetch_offset; + continue; + } + auto const &model_file { model.model_file }; if( model_file.empty() || model_file == "notload" ) { ++batch.umes_prefetch_offset; continue; } - replace_slashes( model_file ); - if( g_stream.mesh_cache.contains( model_file ) ) { + if( pack_mesh_ready_for_slice( model_file ) ) { pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit ); ++batch.umes_prefetch_offset; continue; @@ -2587,7 +2926,13 @@ prefetch_ready_queue_umes_worker( double const budget_ms, std::size_t const max_ } } - (void)ensure_stream_pack_mesh( model_file ); + request_pack_mesh_load( + model_file, + section_manhattan_sections( + batch.row, + batch.column, + g_stream.center_row, + g_stream.center_column ) ); ++loads; pack_bench_inc( &Eu7PackBench::stream_prefetch_ready_umes_loads ); ++batch.umes_prefetch_offset; @@ -2668,6 +3013,41 @@ drain_sections( std::size_t const max_sections ) { } } +void +drain_frame_budget( double const budget_ms ) { + if( budget_ms <= 0.0 ) { + return; + } + + auto const started { std::chrono::steady_clock::now() }; + std::size_t chunks_done { 0 }; + while( chunks_done < kFrameMaxChunksPerDrain ) { + auto const elapsed_ms { + std::chrono::duration( + std::chrono::steady_clock::now() - started ).count() }; + auto const remaining_ms { budget_ms - elapsed_ms }; + if( remaining_ms <= 0.0 ) { + break; + } + + if( false == apply_pending_chunk( + remaining_ms, + kFrameSliceInstances, + kFrameSliceColdMeshes, + kFrameColdBudgetMs, + 1 ) ) { + break; + } + ++chunks_done; + + if( + false == g_stream.pending_apply.has_value() && + ready_queue_size() == 0 ) { + break; + } + } +} + void drain_until_budget( double const budget_ms ) { if( budget_ms <= 0.0 ) { @@ -2705,8 +3085,9 @@ void sync_stream_limits( glm::dvec3 const &world_position ) { if( false == g_loading_screen_dismissed ) { g_stream_catchup = false; - g_stream_max_in_flight = 6; - g_stream_max_ready = 2; + g_stream.radius_fill_active = false; + g_stream_max_in_flight = 16; + g_stream_max_ready = 8; return; } @@ -2715,11 +3096,34 @@ sync_stream_limits( glm::dvec3 const &world_position ) { g_stream.last_inner_ring = section_stream_ring_progress( stream_position, kSectionStreamGameplayRadiusKm ); g_stream.last_outer_ring = - section_stream_ring_progress( stream_position, g_stream.radius ); + section_stream_ring_progress( stream_position, kSectionStreamTargetRadiusKm ); + + auto const was_radius_fill { g_stream.radius_fill_active }; + g_stream.radius_fill_active = false == section_stream_ready_around( + stream_position, kSectionStreamTargetRadiusKm ); + if( false == was_radius_fill && g_stream.radius_fill_active ) { + WriteLog( + "EU7 PACK: faza radius_fill — docelowy promien " + + std::to_string( kSectionStreamTargetRadiusKm ) + "km, ring=" + + std::to_string( static_cast( g_stream.last_outer_ring * 100.f ) ) + "%" ); + } + else if( was_radius_fill && false == g_stream.radius_fill_active ) { + WriteLog( + "EU7 PACK: faza maintenance — promien " + + std::to_string( kSectionStreamTargetRadiusKm ) + "km zapelniony, ring=" + + std::to_string( static_cast( g_stream.last_outer_ring * 100.f ) ) + "%" ); + } + + if( g_stream.radius_fill_active ) { + g_stream_catchup = true; + g_stream_max_in_flight = kRadiusFillMaxInFlight; + g_stream_max_ready = kRadiusFillMaxReady; + return; + } g_stream_catchup = ( cam_speed > 80.0 || - g_stream.last_outer_ring < 0.98f ); + g_stream.last_inner_ring < 0.98f ); if( g_stream_catchup ) { g_stream_max_in_flight = kCatchupMaxInFlightSections; @@ -2727,7 +3131,7 @@ sync_stream_limits( glm::dvec3 const &world_position ) { if( cam_speed > kRingBoostInFlightSpeedMps && - g_stream.last_outer_ring < kRingBoostOuterThreshold ) { + g_stream.last_inner_ring < kRingBoostOuterThreshold ) { g_stream_max_in_flight = std::max( g_stream_max_in_flight, kRingStrongBoostMaxInFlight ); g_stream_max_ready = std::max( @@ -2735,7 +3139,7 @@ sync_stream_limits( glm::dvec3 const &world_position ) { } else if( cam_speed > kRingBoostInFlightSpeedMps && - g_stream.last_outer_ring < kRingStrongBoostOuterThreshold ) { + g_stream.last_inner_ring < kRingStrongBoostOuterThreshold ) { g_stream_max_in_flight = std::max( g_stream_max_in_flight, kRingBoostMaxInFlight ); g_stream_max_ready = std::max( @@ -2810,9 +3214,33 @@ prime_section_stream( Eu7Module const &root_module ) { std::to_string( initial.z ) + ")" ); g_stream.bootstrap_active = true; - enqueue_sections_around( row, column, kInitialBootstrapRadius, initial ); + enqueue_sections_around( row, column, kInitialBootstrapEnqueueRadius, initial ); + auto const prime_started { std::chrono::steady_clock::now() }; + while( true ) { + drain_until_budget( kBootstrapDrainMs ); + if( load_stats().pack_models > 0 ) { + break; + } + auto const elapsed_ms { + std::chrono::duration_cast( + std::chrono::steady_clock::now() - prime_started ).count() }; + if( elapsed_ms >= 4000 ) { + break; + } + if( + false == g_stream.pending_apply.has_value() && + ready_queue_size() == 0 && + g_stream.in_flight_sections.empty() ) { + break; + } + } + WriteLog( + "EU7 PACK: prime drain modele=" + std::to_string( load_stats().pack_models ) + + " ready_q=" + std::to_string( ready_queue_size() ) + + " in_flight=" + std::to_string( g_stream.in_flight_sections.size() ) ); g_stream.bootstrap_active = false; g_stream.bootstrap_pending = false; + g_stream.radius_fill_active = false; } [[nodiscard]] glm::dvec3 @@ -2884,6 +3312,7 @@ bootstrap_section_stream( glm::dvec3 const &world_position ) { g_stream.bootstrap_active = false; g_stream.bootstrap_pending = false; + g_stream.radius_fill_active = false; auto const elapsed_ms { std::chrono::duration_cast( @@ -2928,34 +3357,52 @@ update_section_stream( glm::dvec3 const &world_position ) { sync_stream_limits( stream_position ); + auto const radius_fill { in_radius_fill_phase() }; auto const inner_ring_ready { section_stream_ready_around( stream_position, kSectionStreamGameplayRadiusKm ) }; auto const ring_radius { - inner_ring_ready ? g_stream.radius : kSectionStreamGameplayRadiusKm }; + radius_fill ? + kSectionStreamTargetRadiusKm : + kSectionStreamGameplayRadiusKm }; auto const cam_speed { camera_stream_speed_mps() }; auto const reenqueue_distance { - ( g_stream_catchup || cam_speed > 80.0 ) ? - std::clamp( cam_speed * 0.25, kCatchupReenqueueDistanceM, kReenqueueDistanceM ) : - kReenqueueDistanceM }; + radius_fill ? + kCatchupReenqueueDistanceM : + ( ( g_stream_catchup || cam_speed > 80.0 ) ? + std::clamp( cam_speed * 0.25, kCatchupReenqueueDistanceM, kReenqueueDistanceM ) : + kReenqueueDistanceM ) }; auto const current_section_unloaded { section_has_pack_models( center_row, center_column ) && false == g_stream.loaded_sections.contains( section_index( center_row, center_column ) ) }; + auto const ring_deficit { stream_ring_deficit() }; + auto const desperate { ring_deficit || current_section_unloaded }; + auto max_lookahead { 0 }; - if( inner_ring_ready ) { - max_lookahead = ( g_stream_catchup || cam_speed > 200.0 ) ? - kMovementLookahead : - ( cam_speed > 80.0 ? 10 : 7 ); - if( current_section_unloaded && cam_speed > 200.0 ) { - max_lookahead = std::min( max_lookahead, 4 ); + if( radius_fill ) { + max_lookahead = 0; + } + else if( inner_ring_ready || desperate ) { + if( desperate && cam_speed > 200.0 ) { + max_lookahead = cam_speed > 600.0 ? 10 : 8; } + else { + max_lookahead = ( g_stream_catchup || cam_speed > 200.0 ) ? + kMaintenanceLookaheadMax : + ( cam_speed > 80.0 ? 6 : 4 ); + } + max_lookahead = std::min( max_lookahead, kMaintenanceLookaheadMax ); } auto const travel_forward { guess_travel_forward() }; - auto const inner_ring_incomplete { current_section_unloaded || false == inner_ring_ready }; + auto const ring_incomplete { + radius_fill ? + false == section_stream_ready_around( + stream_position, kSectionStreamTargetRadiusKm ) : + ( current_section_unloaded || false == inner_ring_ready ) }; auto const should_reenqueue { center_moved || false == g_stream.has_last_enqueue_position @@ -2965,24 +3412,20 @@ update_section_stream( glm::dvec3 const &world_position ) { enqueue_section_if_needed( center_row, center_column, 0 ); } - if( should_reenqueue || inner_ring_incomplete ) { - auto const enqueue_radius { - ( current_section_unloaded && cam_speed > 400.0 ) ? - std::min( ring_radius, 3 ) : - ring_radius }; - enqueue_sections_around( center_row, center_column, enqueue_radius, stream_position ); + if( should_reenqueue || ring_incomplete ) { + enqueue_sections_around( center_row, center_column, ring_radius, stream_position ); g_stream.last_enqueue_position = stream_position; g_stream.has_last_enqueue_position = true; - if( should_reenqueue || inner_ring_incomplete ) { + if( should_reenqueue || ring_incomplete ) { pack_bench_inc( &Eu7PackBench::stream_reenqueue ); } } if( - inner_ring_ready && + false == radius_fill && + ( inner_ring_ready || desperate ) && max_lookahead > 0 && - cam_speed > 50.0 && - false == ( current_section_unloaded && cam_speed > 200.0 ) ) { + cam_speed > 50.0 ) { enqueue_movement_lookahead( center_row, center_column, @@ -3008,70 +3451,67 @@ drain_section_stream( if( false == g_stream.active ) { return; } + + (void)drain_pack_mesh_loader_ready( g_stream.mesh_cache ); + auto const stream_position { resolve_section_stream_position( g_loading_screen_dismissed ? Global.pCamera.Pos : stream_loading_position() ) }; - if( gameplay_stream_mode() && section_stream_drain_idle( stream_position ) ) { - return; + auto const loading { false == g_loading_screen_dismissed }; + + if( false == loading ) { + sync_stream_limits( stream_position ); } - auto const inner_ring_ready { - section_stream_ready_around( - stream_position, - kSectionStreamGameplayRadiusKm ) }; - - if( gameplay_stream_mode() && inner_ring_ready ) { - prefetch_ready_queue_umes_worker( - kReadyUmesPrefetchBudgetMs, kReadyUmesPrefetchMaxMeshes ); - prefetch_ready_queue_textures( ready_texture_prefetch_budget_ms() ); - - if( g_stream_catchup ) { - auto const urgent { - pending_apply_is_urgent() || - ( pending_section_distance_km() <= kFastFlightApplyMaxDistanceKm && - camera_stream_speed_mps() > 80.0 ) }; - if( urgent ) { - pack_bench_inc( &Eu7PackBench::stream_drain_catchup_urgent ); - } - else { - pack_bench_inc( &Eu7PackBench::stream_drain_catchup ); - } - drain_apply_budget( - urgent ? kUrgentApplyBudgetMs : gameplay_apply_budget_ms(), - adaptive_slice_instances( pending_section_total(), urgent ), - adaptive_cold_meshes( urgent ), - urgent ? kUrgentColdBudgetMs : gameplay_cold_budget_ms(), - urgent ? kUrgentMaxChunksPerDrain : - ( stream_ring_deficit() ? kRingDeficitMaxChunksPerDrain : 1 ) ); - } - else { - pack_bench_inc( &Eu7PackBench::stream_drain_gameplay ); - drain_until_budget( kDrainBudgetMs ); - } - maybe_log_stream_status( stream_position ); - } - else if( gameplay_stream_mode() ) { - prefetch_ready_queue_umes_worker( - kReadyUmesPrefetchBudgetMs, kReadyUmesPrefetchMaxMeshes ); - pack_bench_inc( &Eu7PackBench::stream_drain_loader ); - drain_until_budget( kLoaderDrainBudgetMs ); - maybe_log_stream_status( stream_position ); - } - else if( false == g_loading_screen_dismissed ) { - auto position { stream_loading_position() }; + if( loading ) { if( section_stream_needs_bootstrap() ) { kick_section_stream_bootstrap(); } - else if( position.x != 0.0 || position.y != 0.0 || position.z != 0.0 ) { - replenish_bootstrap_ring( position ); + else if( + stream_position.x != 0.0 || stream_position.y != 0.0 || stream_position.z != 0.0 ) { + replenish_bootstrap_ring( stream_position ); } - drain_until_budget( kLoaderDrainBudgetMs ); + (void)pump_pack_mesh_loader( 8.0, 4 ); } else { - pack_bench_inc( &Eu7PackBench::stream_drain_gameplay ); - drain_until_budget( kDrainBudgetMs ); + if( + stream_position.x != 0.0 || stream_position.y != 0.0 || stream_position.z != 0.0 ) { + replenish_bootstrap_ring( stream_position ); + } + + auto const radius_fill { in_radius_fill_phase() }; + auto const gpu_init_budget { radius_fill ? 12.0 : ( g_stream_catchup ? 14.0 : 10.0 ) }; + auto const gpu_init_loads { + radius_fill ? + std::size_t { 12 } : + ( g_stream_catchup ? std::size_t { 14 } : std::size_t { 10 } ) }; + (void)pump_pack_mesh_loader( gpu_init_budget, gpu_init_loads ); + if( false == radius_fill ) { + prefetch_ready_queue_umes_worker( + kReadyUmesPrefetchBudgetMs, + g_stream_catchup ? + kReadyUmesPrefetchMaxMeshes + kReadyUmesPrefetchMaxMeshes / 2 : + kReadyUmesPrefetchMaxMeshes ); + prefetch_ready_queue_textures( ready_texture_prefetch_budget_ms() ); + } + } + + auto const radius_fill { false == loading && in_radius_fill_phase() }; + drain_apply_budget( + loading ? kLoaderDrainBudgetMs : frame_drain_budget_ms(), + loading ? kLoaderSliceInstances : + ( radius_fill ? kRadiusFillSliceInstances : 0 ), + 0, + 0, + loading ? kLoaderSectionsPerDrain : gameplay_max_chunks_per_drain() ); + + if( loading ) { + maybe_log_loading_stream_status(); + } + else { + maybe_log_stream_status( stream_position ); } } @@ -3108,10 +3548,10 @@ kick_section_stream_bootstrap() { g_stream.center_column = column; WriteLog( - "EU7 PACK: bootstrap async " + std::to_string( kInitialBootstrapRadius ) + "km, sekcja " + + "EU7 PACK: bootstrap async " + std::to_string( kInitialBootstrapEnqueueRadius ) + "km, sekcja " + std::to_string( row ) + "," + std::to_string( column ) ); - enqueue_sections_around( row, column, kInitialBootstrapRadius, position ); + enqueue_sections_around( row, column, kInitialBootstrapEnqueueRadius, position ); g_stream.bootstrap_pending = false; } @@ -3138,7 +3578,7 @@ preload_section_stream( double const max_drain_ms ) { return; } - simulation::State.drain_deferred_eu7_trainsets( 16.0 ); + simulation::State.drain_deferred_eu7_trainsets( max_drain_ms > 0.0 ? 8.0 : 0.0 ); auto position { resolve_stream_position() }; if( position.x == 0.0 && position.y == 0.0 && position.z == 0.0 ) { @@ -3148,16 +3588,24 @@ preload_section_stream( double const max_drain_ms ) { if( section_stream_needs_bootstrap() ) { if( position.x != 0.0 || position.y != 0.0 || position.z != 0.0 ) { - bootstrap_section_stream( position ); + kick_section_stream_bootstrap(); } } else if( position.x != 0.0 || position.y != 0.0 || position.z != 0.0 ) { update_section_stream( position ); } + if( max_drain_ms <= 0.0 ) { + WriteLog( + "EU7 PACK: preload async kick, sekcji=" + + std::to_string( load_stats().pack_sections_loaded ) + + " modele=" + std::to_string( load_stats().pack_models ) ); + return; + } + auto const started { std::chrono::steady_clock::now() }; while( true ) { - drain_until_budget( kDrainBudgetMs ); + drain_until_budget( kLoaderDrainBudgetMs ); if( position.x != 0.0 || position.y != 0.0 || position.z != 0.0 ) { update_section_stream( position ); } @@ -3168,20 +3616,27 @@ preload_section_stream( double const max_drain_ms ) { if( static_cast( elapsed_ms ) >= max_drain_ms ) { break; } + if( position.x != 0.0 || position.y != 0.0 || position.z != 0.0 ) { + if( section_stream_ready_around( + position, kSectionStreamLoadingDismissRadiusKm ) ) { + break; + } + if( + center_section_stream_satisfied( position ) && + section_stream_ring_progress( + position, kSectionStreamLoadingDismissRadiusKm ) >= kEarlyDismissRingProgress ) { + break; + } + } if( g_stream.pending_apply.has_value() || ready_queue_size() > 0 || false == g_stream.in_flight_sections.empty() ) { continue; } - if( position.x != 0.0 || position.y != 0.0 || position.z != 0.0 ) { - if( section_stream_ready_around( position, kInitialBootstrapRadius ) ) { - break; - } - } - if( load_stats().pack_models >= 1500 ) { + if( load_stats().pack_models >= 400 ) { break; } - std::this_thread::sleep_for( std::chrono::milliseconds( 8 ) ); + std::this_thread::sleep_for( std::chrono::milliseconds( 4 ) ); } WriteLog( @@ -3200,40 +3655,6 @@ ring_section_in_radius( && std::abs( column - center_column ) <= radius_km; } -[[nodiscard]] float -section_ring_apply_fraction( int const row, int const column ) { - auto const section_idx { section_index( row, column ) }; - - if( g_stream.loaded_sections.contains( section_idx ) ) { - return 1.0f; - } - - if( - g_stream.pending_apply.has_value() && - g_stream.pending_apply->section_idx == section_idx && - g_stream.pending_apply->models != nullptr && - false == g_stream.pending_apply->models->empty() ) { - auto const total { g_stream.pending_apply->models->size() }; - return static_cast( g_stream.pending_apply_offset ) / - static_cast( total ); - } - - { - std::lock_guard lock { g_stream.ready.mutex }; - for( auto const &batch : g_stream.ready.data ) { - if( batch.section_idx == section_idx ) { - return 0.85f; - } - } - } - - if( g_stream.in_flight_sections.contains( section_idx ) ) { - return 0.15f; - } - - return 0.f; -} - [[nodiscard]] bool ring_has_pending_pack_work( int const center_row, @@ -3342,7 +3763,13 @@ void dismiss_loading_screen() { pack_bench_begin_stream_phase(); g_loading_screen_dismissed = true; + g_stream.radius_fill_active = true; g_ring_ready_since.reset(); + simulation::is_ready = true; + + WriteLog( + "EU7 PACK: loading screen dismissed — faza radius_fill do " + + std::to_string( kSectionStreamTargetRadiusKm ) + "km" ); if( g_stream.has_anchor_position ) { auto const &saved_camera { Global.FreeCameraInit[ 0 ] }; @@ -3366,41 +3793,43 @@ bool loading_screen_blocks_world( glm::dvec3 const &world_position, int const radius_km ) { + (void)radius_km; if( false == g_stream.active || g_loading_screen_dismissed ) { return false; } + if( load_stats().pack_models >= kBootstrapApplyModelThreshold ) { + WriteLog( + "EU7 PACK: loading screen off, pack_models=" + + std::to_string( load_stats().pack_models ) ); + dismiss_loading_screen(); + return false; + } + + auto const [center_row, center_column] { section_row_column( world_position ) }; + if( section_has_pack_models( center_row, center_column ) ) { + auto const section_idx { section_index( center_row, center_column ) }; + if( g_stream.loaded_sections.contains( section_idx ) ) { + WriteLog( "EU7 PACK: loading screen off, centrum zaladowane" ); + dismiss_loading_screen(); + return false; + } + } + if( g_loading_block_started.has_value() ) { auto const blocked_for { std::chrono::steady_clock::now() - *g_loading_block_started }; if( blocked_for >= kLoadingScreenMaxBlockSec ) { - auto const ring { - section_stream_ring_progress( world_position, radius_km ) }; ErrorLog( - "EU7 PACK: loading screen timeout — wchodzę w świat (ring=" + - std::to_string( static_cast( ring * 100.f ) ) + "%, ready=" + - std::to_string( section_stream_ready_around( world_position, radius_km ) ? 1 : 0 ) + - ", pending_apply=" + - std::to_string( g_stream.pending_apply.has_value() ? 1 : 0 ) + - ", ready_q=" + std::to_string( ready_queue_size() ) + - ", in_flight=" + std::to_string( g_stream.in_flight_sections.size() ) + ")" ); + "EU7 PACK: loading screen timeout — wchodzę w świat (pack_models=" + + std::to_string( load_stats().pack_models ) + ", ready_q=" + + std::to_string( ready_queue_size() ) + ", in_flight=" + + std::to_string( g_stream.in_flight_sections.size() ) + ")" ); dismiss_loading_screen(); return false; } } - if( - section_stream_ring_progress( world_position, radius_km ) >= 1.0f - && section_stream_ready_around( world_position, radius_km ) ) { - dismiss_loading_screen(); - return false; - } - - if( section_stream_presentable_around( world_position, radius_km ) ) { - dismiss_loading_screen(); - return false; - } - return true; } @@ -3523,7 +3952,7 @@ reset_section_stream() { bool section_stream_frame_budget_pressure() { - return g_stream_catchup || pending_apply_is_urgent(); + return g_stream_catchup || pending_apply_is_urgent() || in_radius_fill_phase(); } } // namespace scene::eu7 diff --git a/scene/eu7/eu7_section_stream.h b/scene/eu7/eu7_section_stream.h index 10fb8d15..92b63895 100644 --- a/scene/eu7/eu7_section_stream.h +++ b/scene/eu7/eu7_section_stream.h @@ -19,8 +19,15 @@ class state_serializer; namespace scene::eu7 { -constexpr int kSectionStreamBootstrapRadiusKm { 4 }; -constexpr int kSectionStreamGameplayRadiusKm { 4 }; +// Fazy streamingu PACK (eu7_section_stream.cpp): +// 1) loading_screen — minimalny bootstrap (~2–3 km) podczas ekranu ladowania; +// 2) radius_fill — po dismiss: lekki, ciagly drain az do kSectionStreamTargetRadiusKm; +// 3) maintenance — po zapelnieniu 20 km: wolny lookahead w kierunku lotu. +constexpr int kSectionStreamLoadingDismissRadiusKm { 1 }; +constexpr int kSectionStreamPresentableRadiusKm { 2 }; +constexpr int kSectionStreamBootstrapRadiusKm { 3 }; +constexpr int kSectionStreamGameplayRadiusKm { 5 }; +constexpr int kSectionStreamTargetRadiusKm { 20 }; void init_section_stream( @@ -65,15 +72,15 @@ kick_section_stream_bootstrap(); void try_bootstrap_section_stream(); -// Po zaladowaniu scenariusza: bootstrap + drain PACK przed wejsciem w jazde. +// Po zaladowaniu scenariusza: kick bootstrap + opcjonalny krotki drain (0 = tylko async). void -preload_section_stream( double MaxDrainMs = 300.0 ); +preload_section_stream( double MaxDrainMs = 0.0 ); // Wszystkie sekcje PACK w pierścieniu RadiusKm wokol pozycji wczytane i zaaplikowane. [[nodiscard]] bool section_stream_ready_around( glm::dvec3 const &WorldPosition, - int RadiusKm = kSectionStreamBootstrapRadiusKm ); + int RadiusKm = kSectionStreamTargetRadiusKm ); // ready_around stabilnie przez kilka klatek — bez migania overlay/sceny. [[nodiscard]] bool @@ -86,7 +93,7 @@ section_stream_presentable_around( [[nodiscard]] bool loading_screen_blocks_world( glm::dvec3 const &WorldPosition, - int RadiusKm = kSectionStreamBootstrapRadiusKm ); + int RadiusKm = kSectionStreamLoadingDismissRadiusKm ); [[nodiscard]] bool loading_screen_dismissed(); diff --git a/scene/eu7/eu7_types.h b/scene/eu7/eu7_types.h index a7ca8974..ecda6ae1 100644 --- a/scene/eu7/eu7_types.h +++ b/scene/eu7/eu7_types.h @@ -11,8 +11,10 @@ http://mozilla.org/MPL/2.0/. #include #include +#include #include #include +#include #include #include #include @@ -190,6 +192,8 @@ struct Eu7Lines { std::vector vertices; }; +constexpr std::uint16_t kEu7PackIndexEmpty { 0xFFFFu }; + struct Eu7Model { Eu7BasicNode node; glm::dvec3 location{}; @@ -197,18 +201,52 @@ struct Eu7Model { glm::dvec3 scale{ 1.0, 1.0, 1.0 }; std::string model_file; std::string texture_file; + std::string resolved_texture; std::vector light_states; std::vector light_colors; bool transition = true; bool is_terrain = false; + std::uint8_t pack_flags { 0 }; + std::uint32_t textures_alpha { 0 }; + float baked_range_min { -1.f }; + float baked_range_max { -1.f }; + std::uint8_t pack_cell_id { 255 }; + std::uint16_t pack_mesh_index { kEu7PackIndexEmpty }; + std::uint16_t pack_texture_index { kEu7PackIndexEmpty }; }; +constexpr std::uint8_t kEu7PackFlagNeedsFullLoad { 1u << 0 }; +constexpr std::uint8_t kEu7PackFlagInstanceableHint { 1u << 1 }; +constexpr std::uint8_t kEu7PackCellIdInvalid { 255u }; + +[[nodiscard]] inline bool +eu7_pack_model_needs_full_load( Eu7Model const &model ) { + if( model.pack_flags & kEu7PackFlagNeedsFullLoad ) { + return true; + } + if( model.pack_flags != 0 ) { + return false; + } + return false == model.light_states.empty() + || false == model.light_colors.empty() + || false == model.transition; +} + [[nodiscard]] inline std::string pack_nodedata_cache_key( Eu7Model const &model ) { - return model.model_file + '\x1f' + model.texture_file + '\x1f' - + std::to_string( model.node.range_squared_min ) + '\x1f' - + std::to_string( model.node.range_squared_max ) + '\x1f' - + ( model.is_terrain ? '1' : '0' ); + std::string key; + key.reserve( + model.model_file.size() + model.texture_file.size() + 48 ); + key.append( model.model_file ); + key.push_back( '\x1f' ); + key.append( model.texture_file ); + key.push_back( '\x1f' ); + key.append( std::to_string( model.node.range_squared_min ) ); + key.push_back( '\x1f' ); + key.append( std::to_string( model.node.range_squared_max ) ); + key.push_back( '\x1f' ); + key.push_back( model.is_terrain ? '1' : '0' ); + return key; } // Wspolna definicja modelu (EU7B v8 PROT) — bez transformacji instancji. @@ -216,10 +254,15 @@ struct Eu7ModelPrototype { Eu7BasicNode node; std::string model_file; std::string texture_file; + std::string resolved_texture; std::vector light_states; std::vector light_colors; bool transition = true; bool is_terrain = false; + std::uint8_t pack_flags { 0 }; + std::uint32_t textures_alpha { 0 }; + float baked_range_min { -1.f }; + float baked_range_max { -1.f }; }; struct Eu7MemCell { @@ -390,14 +433,85 @@ struct Eu7PackSectionLoad { std::vector unique_textures; }; -struct Eu7PackSectionChunkLoad { - std::vector models; +struct Eu7PackSectionPathTables { std::vector unique_meshes; std::vector unique_textures; +}; + +struct Eu7PackSectionChunkLoad { + std::vector models; + std::vector unique_textures; + std::shared_ptr path_tables; std::uint32_t chunk_count { 1 }; std::uint32_t chunk_index { 0 }; }; +[[nodiscard]] inline std::string_view +pack_model_mesh_path( + Eu7Model const &model, + Eu7PackSectionPathTables const *path_tables ) { + if( false == model.model_file.empty() ) { + return model.model_file; + } + if( + path_tables != nullptr && + model.pack_mesh_index != kEu7PackIndexEmpty && + model.pack_mesh_index < path_tables->unique_meshes.size() ) { + return path_tables->unique_meshes[ model.pack_mesh_index ]; + } + return {}; +} + +[[nodiscard]] inline std::string_view +pack_model_texture_path( + Eu7Model const &model, + Eu7PackSectionPathTables const *path_tables ) { + if( false == model.texture_file.empty() ) { + return model.texture_file; + } + if( + path_tables != nullptr && + model.pack_texture_index != kEu7PackIndexEmpty && + model.pack_texture_index < path_tables->unique_textures.size() ) { + return path_tables->unique_textures[ model.pack_texture_index ]; + } + return {}; +} + +inline void +resolve_pack_model_paths( + Eu7Model &model, + Eu7PackSectionPathTables const *path_tables ) { + if( path_tables == nullptr ) { + return; + } + if( model.model_file.empty() && model.pack_mesh_index != kEu7PackIndexEmpty ) { + if( model.pack_mesh_index < path_tables->unique_meshes.size() ) { + model.model_file = path_tables->unique_meshes[ model.pack_mesh_index ]; + } + model.pack_mesh_index = kEu7PackIndexEmpty; + } + if( model.texture_file.empty() && model.pack_texture_index != kEu7PackIndexEmpty ) { + if( model.pack_texture_index < path_tables->unique_textures.size() ) { + model.texture_file = path_tables->unique_textures[ model.pack_texture_index ]; + } + model.pack_texture_index = kEu7PackIndexEmpty; + } +} + +inline void +resolve_pack_model_paths( + Eu7Model *models, + std::size_t const count, + Eu7PackSectionPathTables const *path_tables ) { + if( models == nullptr || path_tables == nullptr ) { + return; + } + for( std::size_t i { 0 }; i < count; ++i ) { + resolve_pack_model_paths( models[ i ], path_tables ); + } +} + struct Eu7PackCatalog { std::vector entries; std::unordered_map index_by_section; diff --git a/scene/scene.cpp b/scene/scene.cpp index 688a59a8..486f9315 100644 --- a/scene/scene.cpp +++ b/scene/scene.cpp @@ -10,6 +10,7 @@ http://mozilla.org/MPL/2.0/. #include "stdafx.h" #include "scene/scene.h" +#include "scene/eu7/eu7_types.h" #include "simulation/simulation.h" #include "utilities/Globals.h" #include "vehicle/Camera.h" @@ -353,6 +354,21 @@ basic_cell::insert( TTraction *Traction ) { enclose_area( Traction ); } +// adds provided model instance to the section, optionally using baked PACK cell id +void +basic_section::insert( TAnimModel *Instance ) { + + auto &targetcell { + ( Instance->m_eu7_pack && Instance->m_pack_cell_id < scene::eu7::kEu7PackCellIdInvalid ) ? + m_cells[ Instance->m_pack_cell_id ] : + cell( Instance->location() ) }; + targetcell.insert( Instance ); + m_area.radius = std::max( + m_area.radius, + static_cast( + glm::length( m_area.center - targetcell.area().center ) + targetcell.area().radius ) ); +} + // adds provided model instance to the cell void basic_cell::insert( TAnimModel *Instance ) { @@ -1317,6 +1333,16 @@ basic_region::RadioStop( glm::dvec3 const &Location ) { } } +void +basic_region::insert( TAnimModel *Instance ) { + + auto const location { Instance->location() }; + if( false == point_inside( location ) ) { + return; + } + section( location ).insert( Instance ); +} + std::vector switchtrackbedtextures { "rkpd34r190-tpd1", "rkpd34r190-tpd2", @@ -1795,6 +1821,15 @@ void basic_region::create_map_geometry() if (s) s->create_map_geometry(m_map_geometrybank); } + m_map_geometry_built = true; +} + +void basic_region::ensure_map_geometry() +{ + if( m_map_geometry_built ) { + return; + } + create_map_geometry(); } void basic_region::update_poi_geometry() diff --git a/scene/scene.h b/scene/scene.h index 7b53783b..c63cb6f8 100644 --- a/scene/scene.h +++ b/scene/scene.h @@ -286,7 +286,9 @@ public: // adds provided lines to the section void insert( lines_node Lines ); - // adds provided node to the section + // adds provided model instance to the section + void + insert( TAnimModel *Instance ); template void insert( Type_ *Node ) { @@ -420,6 +422,8 @@ public: // TBD, TODO: clamp coordinates to region boundaries? return; } section( location ).insert( Node ); } + void + insert( TAnimModel *Instance ); // inserts provided node in the region and registers its ends in lookup directory template void @@ -452,6 +456,8 @@ public: sections( glm::dvec3 const &Point, float const Radius ); void create_map_geometry(); + void + ensure_map_geometry(); void update_poi_geometry(); basic_section* get_section(size_t section) @@ -472,6 +478,7 @@ public: gfx::geometrybank_handle m_map_geometrybank; gfx::geometrybank_handle m_map_poipoints; + bool m_map_geometry_built { false }; // methods // checks whether specified point is within boundaries of the region diff --git a/scene/scenenodegroups.cpp b/scene/scenenodegroups.cpp index a28b1d86..df2979f3 100644 --- a/scene/scenenodegroups.cpp +++ b/scene/scenenodegroups.cpp @@ -227,6 +227,16 @@ node_groups::update_map() } } +void +node_groups::ensure_map_index() +{ + if( m_map_index_built ) { + return; + } + update_map(); + m_map_index_built = true; +} + // returns current active group, or null_handle if group stack is empty group_handle node_groups::handle() const { diff --git a/scene/scenenodegroups.h b/scene/scenenodegroups.h index 350fa3d3..2ba171e0 100644 --- a/scene/scenenodegroups.h +++ b/scene/scenenodegroups.h @@ -37,6 +37,8 @@ public: // update minimap objects void update_map(); + void + ensure_map_index(); // returns current active group, or null_handle if group stack is empty group_handle handle() const; @@ -69,6 +71,7 @@ private: // members group_map m_groupmap; // map of established node groups std::stack m_activegroup; // helper, group to be assigned to newly created nodes + bool m_map_index_built { false }; }; extern node_groups Groups; diff --git a/simulation/simulationstateserializer.cpp b/simulation/simulationstateserializer.cpp index 8d639d9b..14af5264 100644 --- a/simulation/simulationstateserializer.cpp +++ b/simulation/simulationstateserializer.cpp @@ -32,8 +32,9 @@ http://mozilla.org/MPL/2.0/. #include "utilities/Logs.h" #include "scene/eu7/eu7_bake.h" #include "scene/eu7/eu7_loader.h" +#include "scene/eu7/eu7_types.h" #include "scene/eu7/eu7_load_stats.h" -#include "scene/eu7/eu7_model_prefetch.h" +#include "scene/eu7/eu7_pack_mesh_loader.h" #include "scene/eu7/eu7_pack_bench.h" #include "scene/eu7/eu7_section.h" #include "scene/eu7/eu7_transform.h" @@ -48,7 +49,7 @@ namespace simulation { namespace { -constexpr double kDeferTrainsetHorizDistM { 4000.0 }; +constexpr double kDeferTrainsetHorizDistM { 800.0 }; constexpr double kDeferTrainsetHorizDistSq { kDeferTrainsetHorizDistM * kDeferTrainsetHorizDistM }; constexpr double kTrainsetDrainMaxDistM { 12000.0 }; @@ -280,13 +281,6 @@ preload_unique_pack_meshes( } } -[[nodiscard]] bool -pack_model_needs_full_load( scene::eu7::Eu7Model const &model ) { - return false == model.light_states.empty() - || false == model.light_colors.empty() - || false == model.transition; -} - [[nodiscard]] scene::node_data const & pack_nodedata_cached( scene::eu7::Eu7Model const &model, @@ -303,6 +297,12 @@ pack_nodedata_cached( nodedata.name = model.node.name; nodedata.type = "model"; } + else if( model.pack_flags != 0 ) { + nodedata.range_min = static_cast( model.baked_range_min ); + nodedata.range_max = static_cast( model.baked_range_max ); + nodedata.name = model.node.name; + nodedata.type = model.node.node_type; + } else { nodedata = node_data_from_eu7( model.node ); } @@ -517,7 +517,7 @@ state_serializer::insert_eu7_pack_models( continue; } - bool const needs_full_load { pack_model_needs_full_load( model ) }; + bool const needs_full_load { scene::eu7::eu7_pack_model_needs_full_load( model ) }; bool loaded { false }; if( needs_full_load ) { scene::eu7::PackBenchTimer const load_timer { @@ -536,24 +536,29 @@ state_serializer::insert_eu7_pack_models( else { scene::eu7::PackBenchTimer const load_timer { &scene::eu7::Eu7PackBench::main_load_eu7_pack_ms }; - auto model_file { model.model_file }; - auto texture_file { model.texture_file }; - replace_slashes( model_file ); - replace_slashes( texture_file ); TModel3d *mesh { nullptr }; - if( false == model_file.empty() && model_file != "notload" ) { - auto const found { mesh_cache.find( model_file ) }; - if( found != mesh_cache.end() ) { - mesh = found->second; - } - else if( scene::eu7::ensure_pack_mesh_in_session_cache( model_file, mesh_cache ) ) { - auto const loaded { mesh_cache.find( model_file ) }; - mesh = loaded != mesh_cache.end() ? loaded->second : nullptr; - } + if( false == model.model_file.empty() && model.model_file != "notload" ) { + mesh = scene::eu7::ensure_pack_mesh_in_session_cache( + model.model_file, mesh_cache ); } - loaded = instance->LoadEu7PackWarm( mesh, texture_file ); - if( loaded ) { - scene::eu7::pack_bench_inc( &scene::eu7::Eu7PackBench::main_pack_fast_loads ); + if( + mesh == nullptr && + false == model.model_file.empty() && + model.model_file != "notload" ) { + loaded = false; + } + else { + loaded = instance->LoadEu7PackWarm( + mesh, + model.texture_file, + model.model_file, + model.resolved_texture, + model.textures_alpha, + ( model.pack_flags & scene::eu7::kEu7PackFlagInstanceableHint ) != 0 ); + if( loaded ) { + scene::eu7::pack_bench_inc( + &scene::eu7::Eu7PackBench::main_pack_fast_loads ); + } } } if( false == loaded ) { @@ -561,6 +566,10 @@ state_serializer::insert_eu7_pack_models( continue; } + if( model.pack_cell_id < scene::eu7::kEu7PackCellIdInvalid ) { + instance->m_pack_cell_id = model.pack_cell_id; + } + if( auto *const mesh { instance->Model() } ) { if( false == mesh->smoke_sources().empty() ) { for( auto const &smokesource : mesh->smoke_sources() ) { @@ -736,8 +745,7 @@ state_serializer::deserialize_continue(std::shared_ptr state scene::Groups.close(); - scene::Groups.update_map(); - Region->create_map_geometry(); + WriteLog( "Scenery: map geometry deferred (lazy on first map open)" ); return false; } diff --git a/utilities/Globals.cpp b/utilities/Globals.cpp index b7792a5c..ac71e391 100644 --- a/utilities/Globals.cpp +++ b/utilities/Globals.cpp @@ -644,6 +644,12 @@ bool global_settings::ConfigParseSimulation(cParser& Parser, const std::string& return true; } + if (token == "eu7.pack.mesh.loader.workers") + { + ParseOneClamped(Parser, eu7_pack_mesh_loader_workers, 0, 16); + return true; + } + if (token == "eu7.auto.bake") { ParseOne(Parser, eu7_auto_bake, 1, false); @@ -1607,6 +1613,7 @@ global_settings::export_as_text( std::ostream &Output ) const { export_as_text( Output, "convertmodels", iConvertModels ); export_as_text( Output, "file.binary.terrain", file_binary_terrain ); export_as_text( Output, "eu7.pack.stream.workers.percent", eu7_pack_stream_workers_percent ); + export_as_text( Output, "eu7.pack.mesh.loader.workers", eu7_pack_mesh_loader_workers ); export_as_text( Output, "eu7.auto.bake", eu7_auto_bake ); export_as_text( Output, "eu7.bake.threads", eu7_bake_threads ); export_as_text( Output, "inactivepause", bInactivePause ); diff --git a/utilities/Globals.h b/utilities/Globals.h index 2a47a798..28696cb0 100644 --- a/utilities/Globals.h +++ b/utilities/Globals.h @@ -82,7 +82,8 @@ struct global_settings { int iConvertIndexRange{ 1000 }; // range of duplicate vertex scan bool file_binary_terrain{ true }; // enable binary terrain (de)serialization bool file_binary_terrain_state{true}; - int eu7_pack_stream_workers_percent{ 75 }; // EU7 PACK async loader thread pool size (% of CPU threads) + int eu7_pack_stream_workers_percent{ 100 }; // EU7 PACK async loader thread pool size (% of CPU threads) + int eu7_pack_mesh_loader_workers{ 0 }; // 0 = main-thread only (background mesh threads unsafe with Global) bool eu7_auto_bake{ true }; // przy ladowaniu mapy: bake SCM→EU7 gdy brak lub nieaktualny .eu7 int eu7_bake_threads{ 0 }; // watki bake (0 = auto = hardware_concurrency) // logs diff --git a/widgets/map.cpp b/widgets/map.cpp index 2733c402..8ea5acac 100644 --- a/widgets/map.cpp +++ b/widgets/map.cpp @@ -8,6 +8,7 @@ #include "vehicle/Driver.h" #include "model/AnimModel.h" #include "application/application.h" +#include "scene/scenenodegroups.h" ui::map_panel::map_panel() : ui_panel(STR_C("Map"), false) { @@ -99,6 +100,9 @@ float ui::map_panel::get_vehicle_rotation() void ui::map_panel::render_map_texture(glm::mat4 transform, glm::vec2 surface_size) { + scene::Groups.ensure_map_index(); + simulation::Region->ensure_map_geometry(); + m_colored_paths.switches.clear(); m_colored_paths.occupied.clear(); m_colored_paths.future.clear();