eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-18 00:49:19 +02:00
Code Issues 121 Wiki Activity

async mesh loading

Browse Source
This commit is contained in:
maj00r
2026-06-16 01:04:37 +02:00
parent 2f90246907
commit 9574d2051b
35 changed files with 3239 additions and 859 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@@ -156,6 +156,7 @@ set(SOURCES
"scene/eu7/eu7_section.cpp"
"scene/eu7/eu7_section_stream.cpp"
"scene/eu7/eu7_model_prefetch.cpp"
"scene/eu7/eu7_pack_mesh_loader.cpp"
"scene/eu7/eu7_reader.cpp"
"scene/eu7/eu7_emit.cpp"
"scene/eu7/eu7_apply.cpp"

12
application/drivermode.cpp
View File

@@ -447,10 +447,6 @@ void driver_mode::enter()
TDynamicObject *nPlayerTrain{((Global.local_start_vehicle != "ghostview") ? simulation::Vehicles.find(Global.local_start_vehicle) : nullptr)};
if( scene::eu7::section_stream_active() ) {
scene::eu7::dismiss_loading_screen();
}
Camera.Init(Global.FreeCameraInit[0], Global.FreeCameraInitAngle[0], nullptr);
Global.pCamera = Camera;
Global.pDebugCamera = DebugCamera;
@@ -468,14 +464,6 @@ void driver_mode::enter()
Camera = Global.pCamera;
}
scene::eu7::update_section_stream( Global.pCamera.Pos );
for( int i { 0 }; i < 64; ++i ) {
scene::eu7::drain_section_stream();
if( scene::eu7::section_stream_ready_around(
stream_pos,
scene::eu7::kSectionStreamBootstrapRadiusKm ) ) {
break;
}
}
}
if (nPlayerTrain)

3
application/scenarioloadermode.cpp
View File

@@ -60,7 +60,8 @@ bool scenarioloader_mode::update() {
WriteLog( "Scenario loading time: " + std::to_string( std::chrono::duration_cast<std::chrono::seconds>( ( std::chrono::system_clock::now() - timestart ) ).count() ) + " seconds" );
if( scene::eu7::section_stream_active() ) {
scene::eu7::preload_section_stream( 3000.0 );
// Bounded PACK drain after scenario parse — shortens black overlay in driver mode.
scene::eu7::preload_section_stream( 8000.0 );
}
// TODO: implement and use next mode cue

4
docs/eu7-format.md
View File

@@ -85,8 +85,9 @@ Nie ma pola "łączny rozmiar pliku" w nagłówku. Po nagłówku następują chu
| `6` | `kEu7VersionV6` | `kVersionRuntime` | Adds site transform w rekordach INCL |
| `7` | `kEu7VersionV7` | `kVersionRuntimeV7` | Adds PIDX + PACK — modele scenerii per sekcja 1 km (streaming) |
| `8` | `kEu7VersionV8` | — | Adds PROT (prototypy) + nowy format sekcji PACK v8 |
| `9` | `kEu7VersionV9` | `kVersionRuntimeV9` | PROT v9 (resolved_texture, pack_flags) + PACK sekcji v13 |
**Runtime maszyna-fresh obsługuje wersje 4–8.** Wersje 1–3 nie są obsługiwane przez żaden z dostępnych loaderów.
**Runtime maszyna-fresh obsługuje wersje 4–9.** Wersje 1–3 nie są obsługiwane przez żaden z dostępnych loaderów.
---
@@ -1025,6 +1026,7 @@ Overlay renderera: `inst-pool:` — ile instancji zakwalifikowanych do `Render_I
| v6 | Chunk INCL rozszerzony o **site transform** (pełny TransformContext per include) |
| v7 | Chunki **PIDX + PACK** — modele scenerii podzielone na kafelki 1 km, asynchroniczny streaming |
| v8 | Chunk **PROT** (prototypy) + nowy format sekcji PACK z `solo_count` + `inst_count` |
| v9 | **PROT v9** (resolved_texture, pack_flags, baked ranges) + **PACK v13** (mesh/tex indices, cell_id) |
---

13
eu07-parser/include/eu07/scene/binary/common.hpp
View File

@@ -18,18 +18,22 @@ inline constexpr std::uint32_t kVersionLegacy = 1; // codec.hpp — nieobslugiwa
inline constexpr std::uint32_t kVersionRuntime = 6;
inline constexpr std::uint32_t kVersionRuntimeV7 = 7;
inline constexpr std::uint32_t kVersionRuntimeV8 = 8;
inline constexpr std::uint32_t kVersionRuntimeV9 = 9;
inline constexpr std::uint32_t kVersionRuntimeV5 = 5;
inline constexpr std::uint32_t kVersionRuntimeV4 = 4;
inline constexpr std::uint32_t kVersionRuntimeF32 = 3;
inline constexpr std::uint32_t kVersionRuntimeF64 = 2;
[[nodiscard]] inline bool isSupportedEu7bVersion(const std::uint32_t version) noexcept {
return version == kVersionRuntimeV8 || version == kVersionRuntimeV7 ||
version == kVersionRuntime || version == kVersionRuntimeV5 ||
version == kVersionRuntimeV4;
return version == kVersionRuntimeV9 || version == kVersionRuntimeV8 ||
version == kVersionRuntimeV7 || version == kVersionRuntime ||
version == kVersionRuntimeV5 || version == kVersionRuntimeV4;
}
[[nodiscard]] inline std::string formatVersionName(const std::uint32_t version) {
if (version == kVersionRuntimeV9) {
return "RuntimeV9";
}
if (version == kVersionRuntimeV8) {
return "RuntimeV8";
}
@@ -58,6 +62,9 @@ inline constexpr std::uint32_t kVersionRuntimeF64 = 2;
}
[[nodiscard]] inline std::string formatVersionDescription(const std::uint32_t version) {
if (version == kVersionRuntimeV9) {
return "bake v9 + PROT v9 + PACK sekcji v13 (indices + cell sort)";
}
if (version == kVersionRuntimeV8) {
return "bake v7 + PROT + PACK sekcji v12 (solo + compact inst)";
}

435
eu07-parser/include/eu07/scene/binary/runtime_module.hpp
View File

@@ -51,11 +51,15 @@ inline constexpr std::array<char, 4> kChunkPidx{'P', 'I', 'D', 'X'};
inline constexpr std::array<char, 4> kChunkPack{'P', 'A', 'C', 'K'};
inline constexpr std::array<char, 4> kChunkProt{'P', 'R', 'O', 'T'};
// PACK section payload: v9 = UMES, v10 = UMES + CHNK, v11 = UMES + UTEX + CHNK, v12 = v11 + PROT inst.
// PACK section payload: v9 = UMES, v10 = UMES + CHNK, v11 = UMES + UTEX + CHNK, v12 = v11 + PROT inst, v13 = v12 + mesh/tex indices + cell_id.
inline constexpr std::uint8_t kPackSectionFormatV9 = 2;
inline constexpr std::uint8_t kPackSectionFormatV10 = 3;
inline constexpr std::uint8_t kPackSectionFormatV11 = 4;
inline constexpr std::uint8_t kPackSectionFormatV12 = 5;
inline constexpr std::uint8_t kPackSectionFormatV13 = 6;
inline constexpr std::uint16_t kPackIndexEmpty = 0xFFFF;
inline constexpr std::int32_t kEu07CellSize = 250;
inline constexpr std::int32_t kEu07CellsPerSection = 4;
inline constexpr std::size_t kPackSectionChunkModels = 512;
inline constexpr std::size_t kPackSectionChunkModelsHeavy = 256;
inline constexpr std::size_t kPackSectionHeavyModelThreshold = 1024;
@@ -434,6 +438,90 @@ struct PackPayloadBuild {
return true;
}
[[nodiscard]] inline std::string packModelTextureDir(std::string model_file) {
if (model_file.empty() || model_file == "notload") {
return {};
}
for (char& ch : model_file) {
if (ch == '\\') {
ch = '/';
}
}
const auto dot = model_file.rfind('.');
if (dot != std::string::npos) {
model_file.resize(dot);
}
const auto slash_pos = model_file.rfind('/');
if (slash_pos == std::string::npos) {
return {};
}
return model_file.substr(0, slash_pos + 1);
}
[[nodiscard]] inline bool packTexturePathIsRooted(const std::string_view path) {
return path.starts_with("dynamic/") || path.starts_with("textures/") ||
path.starts_with("scenery/") || path.starts_with("models/");
}
[[nodiscard]] inline bool probePackTextureCandidate(const std::string& candidate) {
static constexpr std::array<const char*, 8> kExtensions{
".mat", ".dds", ".tga", ".ktx", ".png", ".bmp", ".jpg", ".tex"};
for (const char* ext : kExtensions) {
const std::filesystem::path path(candidate + ext);
if (std::filesystem::exists(path)) {
return true;
}
}
return false;
}
[[nodiscard]] inline std::string resolvePackTexturePath(
const std::string& model_file,
const std::string& texture_file) {
if (!isPackTexturePath(texture_file)) {
return {};
}
std::string normalized = texture_file;
for (char& ch : normalized) {
if (ch == '\\') {
ch = '/';
}
}
std::vector<std::string> candidates;
candidates.reserve(4);
auto append_candidate = [&](std::string candidate) {
if (candidate.empty()) {
return;
}
for (char& ch : candidate) {
if (ch == '\\') {
ch = '/';
}
}
if (std::find(candidates.begin(), candidates.end(), candidate) == candidates.end()) {
candidates.emplace_back(std::move(candidate));
}
};
append_candidate(normalized);
const auto model_dir = packModelTextureDir(model_file);
if (!model_dir.empty()) {
append_candidate(model_dir + normalized);
}
if (!packTexturePathIsRooted(normalized)) {
append_candidate(std::string("textures/") + normalized);
}
for (const std::string& candidate : candidates) {
if (probePackTextureCandidate(candidate)) {
return candidate;
}
}
return {};
}
[[nodiscard]] inline runtime::Vec3 packSectionCenter(
const codec::TerrSectionKey section) noexcept {
runtime::Vec3 center{};
@@ -448,6 +536,21 @@ struct PackPayloadBuild {
return center;
}
[[nodiscard]] inline std::uint8_t computePackCellId(
const runtime::Vec3& location,
const codec::TerrSectionKey section) noexcept {
const runtime::Vec3 center = packSectionCenter(section);
const int column = static_cast<int>(std::floor(
(location.x - (center.x - static_cast<double>(codec::kEu07SectionSize) / 2.0)) /
static_cast<double>(kEu07CellSize)));
const int row = static_cast<int>(std::floor(
(location.z - (center.z - static_cast<double>(codec::kEu07SectionSize) / 2.0)) /
static_cast<double>(kEu07CellSize)));
const int clamped_col = std::clamp(column, 0, kEu07CellsPerSection - 1);
const int clamped_row = std::clamp(row, 0, kEu07CellsPerSection - 1);
return static_cast<std::uint8_t>(clamped_row * kEu07CellsPerSection + clamped_col);
}
inline void sortPackSectionModelsForStreaming(
std::vector<runtime::RuntimeModelInstance>& models,
const codec::TerrSectionKey section) {
@@ -579,6 +682,101 @@ inline void sortPackSectionModelsForStreaming(
return ids;
}
[[nodiscard]] inline std::vector<std::string> collectUniqueMeshPaths(
const std::vector<runtime::RuntimeModelInstance>& models,
const std::vector<std::string_view>* prototype_mesh_paths = nullptr) {
std::unordered_map<std::string, std::uint32_t> mesh_frequency;
mesh_frequency.reserve(models.size());
for (const runtime::RuntimeModelInstance& model : models) {
if (isLoadableMeshPath(model.modelFile)) {
++mesh_frequency[model.modelFile];
}
}
std::vector<std::string> paths;
paths.reserve(mesh_frequency.size());
for (const auto& [path, frequency] : mesh_frequency) {
if (frequency > 0) {
paths.emplace_back(path);
}
}
if (prototype_mesh_paths != nullptr) {
for (const std::string_view path : *prototype_mesh_paths) {
if (isLoadableMeshPath(path)) {
const std::string text(path);
if (!mesh_frequency.contains(text)) {
paths.emplace_back(text);
mesh_frequency.emplace(text, 0u);
}
}
}
}
std::stable_sort(
paths.begin(),
paths.end(),
[&](const std::string& lhs, const std::string& rhs) {
const std::uint32_t lhs_rank = mesh_frequency[lhs];
const std::uint32_t rhs_rank = mesh_frequency[rhs];
if (lhs_rank != rhs_rank) {
return lhs_rank > rhs_rank;
}
return lhs < rhs;
});
return paths;
}
[[nodiscard]] inline std::vector<std::string> collectUniqueTexturePaths(
const std::vector<runtime::RuntimeModelInstance>& models) {
std::unordered_map<std::string, std::uint32_t> texture_frequency;
texture_frequency.reserve(models.size());
for (const runtime::RuntimeModelInstance& model : models) {
if (isPackTexturePath(model.textureFile)) {
++texture_frequency[model.textureFile];
}
}
std::vector<std::string> paths;
paths.reserve(texture_frequency.size());
for (const auto& [path, frequency] : texture_frequency) {
if (frequency > 0) {
paths.emplace_back(path);
}
}
std::stable_sort(
paths.begin(),
paths.end(),
[&](const std::string& lhs, const std::string& rhs) {
const std::uint32_t lhs_rank = texture_frequency[lhs];
const std::uint32_t rhs_rank = texture_frequency[rhs];
if (lhs_rank != rhs_rank) {
return lhs_rank > rhs_rank;
}
return lhs < rhs;
});
return paths;
}
[[nodiscard]] inline std::unordered_map<std::string, std::uint16_t> makePackPathIndexMap(
const std::vector<std::string>& paths) {
std::unordered_map<std::string, std::uint16_t> index_by_path;
index_by_path.reserve(paths.size());
for (std::size_t index = 0; index < paths.size(); ++index) {
index_by_path.emplace(paths[index], static_cast<std::uint16_t>(index));
}
return index_by_path;
}
[[nodiscard]] inline std::uint16_t lookupPackPathIndex(
const std::unordered_map<std::string, std::uint16_t>& index_by_path,
const std::string& path) {
const auto it = index_by_path.find(path);
return it == index_by_path.end() ? kPackIndexEmpty : it->second;
}
[[nodiscard]] inline std::vector<char> buildPackModelBlob(
StringTable& table,
const std::vector<runtime::RuntimeModelInstance>& models,
@@ -778,7 +976,8 @@ struct PackPrototypeKeyHash {
inline void writeRuntimePrototype(
std::ostream& out,
StringTable& table,
const runtime::RuntimeModelInstance& model) {
const runtime::RuntimeModelInstance& model,
const bool emit_v9_fields = false) {
codec::writeSlimNode(out, table, model.node, "model");
io::writeU8(out, model.isTerrain ? 1 : 0);
io::writeU8(out, model.transition ? 1 : 0);
@@ -792,15 +991,48 @@ inline void writeRuntimePrototype(
for (std::uint32_t color : model.lightColors) {
io::writeU32(out, color);
}
if (emit_v9_fields) {
const std::string resolved =
resolvePackTexturePath(model.modelFile, model.textureFile);
codec::writeStringId(out, table, resolved);
const bool needs_full_load =
!model.lightStates.empty() || !model.lightColors.empty() || !model.transition;
std::uint8_t pack_flags = 0;
if (needs_full_load) {
pack_flags |= 1u;
} else if (!model.isTerrain) {
pack_flags |= 2u;
}
io::writeU8(out, pack_flags);
io::writeU32(out, 0u);
const float baked_range_min =
static_cast<float>(std::sqrt(std::max(0.0, model.node.rangeSquaredMin)));
const float baked_range_max =
model.node.rangeSquaredMax >= std::numeric_limits<double>::max() * 0.5
? -1.f
: static_cast<float>(std::sqrt(model.node.rangeSquaredMax));
io::writeF32(out, baked_range_min);
io::writeF32(out, baked_range_max);
}
}
[[nodiscard]] inline std::vector<char> buildProtPayload(
StringTable& table,
const std::vector<runtime::RuntimeModelInstance>& prototypes) {
const std::vector<runtime::RuntimeModelInstance>& prototypes,
const bool emit_v9_fields = false) {
if (emit_v9_fields) {
for (const runtime::RuntimeModelInstance& proto : prototypes) {
const std::string resolved =
resolvePackTexturePath(proto.modelFile, proto.textureFile);
if (!resolved.empty()) {
table.intern(resolved);
}
}
}
std::ostringstream out(std::ios::binary);
io::writeU32(out, static_cast<std::uint32_t>(prototypes.size()));
for (const runtime::RuntimeModelInstance& proto : prototypes) {
writeRuntimePrototype(out, table, proto);
writeRuntimePrototype(out, table, proto, emit_v9_fields);
}
const std::string blob = out.str();
return {blob.begin(), blob.end()};
@@ -861,18 +1093,47 @@ struct PackSectionEntry {
PackSectionEntryKind kind = PackSectionEntryKind::Solo;
std::size_t model_index = 0;
std::uint32_t proto_id = 0;
std::uint8_t pack_cell_id = 0;
};
inline void writePackSoloRecordV13(
std::ostream& out,
StringTable& table,
const runtime::RuntimeModelInstance& model,
const std::unordered_map<std::string, std::uint16_t>& mesh_index,
const std::unordered_map<std::string, std::uint16_t>& texture_index,
const std::uint8_t pack_cell_id) {
codec::writeSlimNode(out, table, model.node, "model");
io::writeU8(out, model.isTerrain ? 1 : 0);
io::writeU8(out, model.transition ? 1 : 0);
io::writeVec3(out, model.location);
io::writeVec3(out, model.angles);
io::writeVec3(out, model.scale);
io::writeU16(out, lookupPackPathIndex(mesh_index, model.modelFile));
io::writeU16(out, lookupPackPathIndex(texture_index, model.textureFile));
io::writeU32(out, static_cast<std::uint32_t>(model.lightStates.size()));
for (float light : model.lightStates) {
io::writeF32(out, light);
}
io::writeU32(out, static_cast<std::uint32_t>(model.lightColors.size()));
for (std::uint32_t color : model.lightColors) {
io::writeU32(out, color);
}
io::writeU8(out, pack_cell_id);
}
inline void writePackInstRecord(
std::ostream& out,
StringTable& table,
const std::uint32_t proto_id,
const runtime::RuntimeModelInstance& model) {
const runtime::RuntimeModelInstance& model,
const std::uint8_t pack_cell_id = 0) {
io::writeU32(out, proto_id);
io::writeVec3(out, model.location);
io::writeVec3(out, model.angles);
io::writeVec3(out, model.scale);
codec::writeStringId(out, table, model.node.name);
io::writeU8(out, pack_cell_id);
}
[[nodiscard]] inline std::vector<char> buildPackSectionChunkBlobV12(
@@ -1009,6 +1270,162 @@ inline void writePackInstRecord(
return {blob.begin(), blob.end()};
}
[[nodiscard]] inline std::vector<char> buildPackSectionChunkBlobV13(
StringTable& table,
const std::vector<runtime::RuntimeModelInstance>& models,
const std::vector<PackSectionEntry>& entries,
const std::size_t begin,
const std::size_t end,
const std::unordered_map<std::string, std::uint16_t>& mesh_index,
const std::unordered_map<std::string, std::uint16_t>& texture_index) {
std::ostringstream out(std::ios::binary);
for (std::size_t index = begin; index < end; ++index) {
const PackSectionEntry& entry = entries[index];
if (entry.kind != PackSectionEntryKind::Solo) {
continue;
}
runtime::RuntimeModelInstance worldModel = models[entry.model_index];
worldModel.node.transform = {};
writePackSoloRecordV13(
out, table, worldModel, mesh_index, texture_index, entry.pack_cell_id);
}
for (std::size_t index = begin; index < end; ++index) {
const PackSectionEntry& entry = entries[index];
if (entry.kind != PackSectionEntryKind::Inst) {
continue;
}
writePackInstRecord(
out, table, entry.proto_id, models[entry.model_index], entry.pack_cell_id);
}
const std::string blob = out.str();
return {blob.begin(), blob.end()};
}
[[nodiscard]] inline std::vector<char> buildPackSectionPayloadV13(
StringTable& table,
const std::vector<runtime::RuntimeModelInstance>& models,
const PackPrototypeTable& prototypes,
const codec::TerrSectionKey section) {
std::vector<PackSectionEntry> entries;
entries.reserve(models.size());
std::uint32_t solo_count = 0;
std::uint32_t inst_count = 0;
for (std::size_t model_index = 0; model_index < models.size(); ++model_index) {
const runtime::RuntimeModelInstance& model = models[model_index];
PackSectionEntry entry;
entry.model_index = model_index;
entry.pack_cell_id = computePackCellId(model.location, section);
if (modelMustBePackSolo(model)) {
entry.kind = PackSectionEntryKind::Solo;
++solo_count;
entries.push_back(entry);
continue;
}
const PackPrototypeKey key = makePackPrototypeKey(model);
const auto proto_it = prototypes.key_to_id.find(key);
if (proto_it == prototypes.key_to_id.end()) {
entry.kind = PackSectionEntryKind::Solo;
++solo_count;
entries.push_back(entry);
continue;
}
entry.kind = PackSectionEntryKind::Inst;
entry.proto_id = proto_it->second;
++inst_count;
entries.push_back(entry);
}
std::stable_sort(
entries.begin(),
entries.end(),
[](const PackSectionEntry& lhs, const PackSectionEntry& rhs) {
return lhs.pack_cell_id < rhs.pack_cell_id;
});
std::vector<std::string_view> prototype_mesh_paths;
prototype_mesh_paths.reserve(prototypes.prototypes.size());
for (const runtime::RuntimeModelInstance& proto : prototypes.prototypes) {
prototype_mesh_paths.emplace_back(proto.modelFile);
}
const auto mesh_paths = collectUniqueMeshPaths(models, &prototype_mesh_paths);
const auto texture_paths = collectUniqueTexturePaths(models);
const auto mesh_ids = collectUniqueMeshIds(table, models, &prototype_mesh_paths);
const auto texture_ids = collectUniqueTextureIds(table, models);
const auto mesh_index = makePackPathIndexMap(mesh_paths);
const auto texture_index = makePackPathIndexMap(texture_paths);
const std::size_t chunk_models =
models.size() > kPackSectionHeavyModelThreshold ? kPackSectionChunkModelsHeavy
: kPackSectionChunkModels;
std::vector<std::vector<char>> chunk_payloads;
std::vector<std::uint32_t> chunk_solo_counts;
std::vector<std::uint32_t> chunk_inst_counts;
chunk_payloads.reserve((entries.size() + chunk_models - 1) / chunk_models);
chunk_solo_counts.reserve(chunk_payloads.capacity());
chunk_inst_counts.reserve(chunk_payloads.capacity());
for (std::size_t offset = 0; offset < entries.size(); offset += chunk_models) {
const std::size_t end = std::min(offset + chunk_models, entries.size());
std::uint32_t chunk_solo = 0;
std::uint32_t chunk_inst = 0;
for (std::size_t index = offset; index < end; ++index) {
if (entries[index].kind == PackSectionEntryKind::Solo) {
++chunk_solo;
} else {
++chunk_inst;
}
}
chunk_solo_counts.push_back(chunk_solo);
chunk_inst_counts.push_back(chunk_inst);
chunk_payloads.push_back(
buildPackSectionChunkBlobV13(
table, models, entries, offset, end, mesh_index, texture_index));
}
if (chunk_payloads.empty()) {
chunk_payloads.emplace_back();
chunk_solo_counts.push_back(0);
chunk_inst_counts.push_back(0);
}
const std::uint32_t chunk_count = static_cast<std::uint32_t>(chunk_payloads.size());
const std::uint32_t header_size =
1u + 4u + 4u + 4u + static_cast<std::uint32_t>(mesh_ids.size()) * 4u + 4u +
static_cast<std::uint32_t>(texture_ids.size()) * 4u + 4u + chunk_count * 12u;
std::vector<std::uint32_t> chunk_offsets(chunk_count);
std::uint32_t payload_offset = header_size;
for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) {
chunk_offsets[chunk_index] = payload_offset;
payload_offset += static_cast<std::uint32_t>(chunk_payloads[chunk_index].size());
}
std::ostringstream packOut(std::ios::binary);
io::writeU8(packOut, kPackSectionFormatV13);
io::writeU32(packOut, solo_count);
io::writeU32(packOut, inst_count);
io::writeU32(packOut, static_cast<std::uint32_t>(mesh_ids.size()));
for (const std::uint32_t id : mesh_ids) {
io::writeU32(packOut, id);
}
io::writeU32(packOut, static_cast<std::uint32_t>(texture_ids.size()));
for (const std::uint32_t id : texture_ids) {
io::writeU32(packOut, id);
}
io::writeU32(packOut, chunk_count);
for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) {
io::writeU32(packOut, chunk_solo_counts[chunk_index]);
io::writeU32(packOut, chunk_inst_counts[chunk_index]);
io::writeU32(packOut, chunk_offsets[chunk_index]);
}
for (const std::vector<char>& payload : chunk_payloads) {
packOut.write(payload.data(), static_cast<std::streamsize>(payload.size()));
}
const std::string blob = packOut.str();
return {blob.begin(), blob.end()};
}
[[nodiscard]] inline PackPayloadBuild buildPackPayloadV12(
StringTable& table,
const std::vector<codec::ModelSectionBatch>& batches) {
@@ -1024,7 +1441,7 @@ inline void writePackInstRecord(
}
const PackPrototypeTable prototypes = collectPackPrototypes(table, batches);
result.protPayload = buildProtPayload(table, prototypes.prototypes);
result.protPayload = buildProtPayload(table, prototypes.prototypes, true);
result.entries.resize(batches.size());
std::vector<std::vector<char>> section_payloads(batches.size());
@@ -1043,7 +1460,7 @@ inline void writePackInstRecord(
std::vector<runtime::RuntimeModelInstance> section_models = batch.models;
sortPackSectionModelsForStreaming(section_models, batch.section);
section_payloads[batch_index] =
buildPackSectionPayloadV12(table, section_models, prototypes);
buildPackSectionPayloadV13(table, section_models, prototypes, batch.section);
}
std::size_t total_size = 0;
for (const std::vector<char>& payload : section_payloads) {
@@ -1080,7 +1497,7 @@ inline void writePackInstRecord(
std::vector<runtime::RuntimeModelInstance> section_models = batch.models;
sortPackSectionModelsForStreaming(section_models, batch.section);
section_payloads[batch_index] =
buildPackSectionPayloadV12(table, section_models, prototypes);
buildPackSectionPayloadV13(table, section_models, prototypes, batch.section);
}
});
}
@@ -1469,7 +1886,7 @@ inline void writeRuntimeModule(
}
out.write(kMagic.data(), 4);
io::writeU32(out, usePackModels ? kVersionRuntimeV8 : kVersionRuntime);
io::writeU32(out, usePackModels ? kVersionRuntimeV9 : kVersionRuntime);
io::writeChunkHeader(out, kChunkStrs, 8 + static_cast<std::uint32_t>(strsPayload.size()));
out.write(strsPayload.data(), static_cast<std::streamsize>(strsPayload.size()));

37
model/AnimModel.cpp
View File

@@ -14,7 +14,9 @@ http://mozilla.org/MPL/2.0/.
#include "stdafx.h"
#include "model/AnimModel.h"
#include "scene/eu7/eu7_model_prefetch.h"
#include <mutex>
#include <unordered_map>
#include "scene/eu7/eu7_pack_bench.h"
@@ -390,6 +392,7 @@ TAnimModel::reset_pack_for_insert( scene::node_data const &nodedata ) {
pModel = nullptr;
m_eu7_pack = false;
m_instanceable = false;
m_pack_cell_id = 255;
m_materialdata = {};
asText.clear();
}
@@ -612,10 +615,12 @@ namespace eu7_pack_cache {
enum class slot_t : std::int8_t { unknown = -1, no = 0, yes = 1 };
std::mutex g_mutex;
std::unordered_map<TModel3d const *, slot_t> g_model;
[[nodiscard]] bool
lookup( TModel3d *const model, bool &instanceable ) {
std::lock_guard<std::mutex> lock { g_mutex };
auto const it { g_model.find( model ) };
if( it == g_model.end() || it->second == slot_t::unknown ) {
return false;
@@ -626,6 +631,7 @@ lookup( TModel3d *const model, bool &instanceable ) {
void
store( TModel3d *const model, bool const instanceable ) {
std::lock_guard<std::mutex> lock { g_mutex };
g_model[ model ] = instanceable ? slot_t::yes : slot_t::no;
}
@@ -634,11 +640,13 @@ warm( TModel3d *const model ) {
if( model == nullptr ) {
return;
}
std::lock_guard<std::mutex> lock { g_mutex };
auto const it { g_model.find( model ) };
if( it != g_model.end() && it->second != slot_t::unknown ) {
return;
}
store( model, false == submodel_tree_blocks_instancing( model->Root ) );
g_model[ model ] =
false == submodel_tree_blocks_instancing( model->Root ) ? slot_t::yes : slot_t::no;
}
} // namespace eu7_pack_cache
@@ -659,7 +667,11 @@ TAnimModel::warm_instanceable_cache( TModel3d *const model ) {
bool
TAnimModel::LoadEu7PackWarm(
TModel3d *const mesh,
std::string const &texture_file ) {
std::string const &texture_file,
std::string const &model_file,
std::string const &resolved_texture,
std::uint32_t const textures_alpha,
bool const instanceable_hint ) {
pModel = mesh;
if( mesh == nullptr ) {
return true;
@@ -680,16 +692,27 @@ TAnimModel::LoadEu7PackWarm(
false == normalized.ends_with( '/' ) &&
normalized.find( "tr/none" ) == std::string::npos &&
normalized.find( '#' ) == std::string::npos ) {
m_materialdata.assign( normalized );
if( m_materialdata.replacable_skins[ 1 ] == null_handle ) {
scene::eu7::pack_bench_inc( &scene::eu7::Eu7PackBench::main_texture_assign_fail );
scene::eu7::log_pack_texture_fail( normalized );
}
(void)scene::eu7::assign_pack_texture(
m_materialdata,
model_file,
normalized,
resolved_texture,
textures_alpha );
}
}
else if( textures_alpha != 0 ) {
m_materialdata.textures_alpha = static_cast<int>( textures_alpha );
}
m_eu7_pack = true;
if( instanceable_hint ) {
++s_classified_total;
m_instanceable = true;
++s_instanceable_total;
return true;
}
bool cached_instanceable { false };
if( anim_instancing_detail::eu7_pack_cache::lookup( mesh, cached_instanceable ) ) {
++s_classified_total;

7
model/AnimModel.h
View File

@@ -135,7 +135,11 @@ public:
// PACK stream: mesh pointer already resolved; skips GetModel lookup.
bool LoadEu7PackWarm(
TModel3d *Mesh,
std::string const &texture_file );
std::string const &texture_file,
std::string const &model_file = {},
std::string const &resolved_texture = {},
std::uint32_t textures_alpha = 0,
bool instanceable_hint = false );
void
reset_pack_for_insert( scene::node_data const &Nodedata );
static TAnimModel *
@@ -243,6 +247,7 @@ public:
// computed once at end of Load() so the renderer can simply test the flag.
bool m_instanceable { false };
bool m_eu7_pack { false };
std::uint8_t m_pack_cell_id { 255 };
// helper: evaluates current state and updates m_instanceable accordingly.
void update_instanceable_flag();

49
model/MdlMngr.cpp
View File

@@ -21,6 +21,8 @@ http://mozilla.org/MPL/2.0/.
#include "utilities/Logs.h"
#include "utilities/utilities.h"
#include <memory>
// wczytanie modelu do kontenerka
TModel3d *
TMdlContainer::LoadModel(std::string const &Name, bool const Dynamic) {
@@ -132,6 +134,53 @@ TModelsManager::IsModelCached( std::string const &Name ) {
return find_in_databank( filename ).first;
}
[[nodiscard]] std::string
normalize_pack_model_key( std::string name ) {
erase_extension( name );
return ToLower( name );
}
std::pair<bool, TModel3d *>
TModelsManager::TryGetCachedModel( std::string const &Name ) {
return find_in_databank( normalize_pack_model_key( Name ) );
}
std::string
TModelsManager::ResolveModelDiskPath( std::string Name ) {
return find_on_disk( normalize_pack_model_key( std::move( Name ) ) );
}
std::shared_ptr<TModel3d>
TModelsManager::LoadModelCpuDeferred(
std::string const &DiskPath,
bool const Dynamic ) {
auto model { std::make_shared<TModel3d>() };
if( false == model->LoadFromFile( DiskPath, Dynamic, true ) ) {
return nullptr;
}
return model;
}
TModel3d *
TModelsManager::PublishLoadedModel(
std::string const &VirtualName,
std::shared_ptr<TModel3d> Model ) {
if( Model == nullptr ) {
return nullptr;
}
auto const lookup { find_in_databank( VirtualName ) };
if( lookup.first && lookup.second != nullptr ) {
return lookup.second;
}
Model->Init();
m_models.emplace_back();
m_models.back().Model = std::move( Model );
m_modelsmap.emplace( VirtualName, m_models.size() - 1 );
return m_models.back().Model.get();
}
std::pair<bool, TModel3d *>
TModelsManager::find_in_databank( std::string const &Name ) {

13
model/MdlMngr.h
View File

@@ -10,6 +10,8 @@ http://mozilla.org/MPL/2.0/.
#include "utilities/Classes.h"
#include <memory>
class TMdlContainer {
friend class TModelsManager;
private:
@@ -24,6 +26,17 @@ public:
// McZapkie: dodalem sciezke, notabene Path!=Patch :)
static TModel3d *GetModel(std::string const &Name, bool const dynamic = false, bool const Logerrors = true , int uid = 0);
[[nodiscard]] static bool IsModelCached( std::string const &Name );
// EU7 PACK stream: CPU parse on workers, GPU Init on main thread.
[[nodiscard]] static std::pair<bool, TModel3d *>
TryGetCachedModel( std::string const &Name );
[[nodiscard]] static std::string
ResolveModelDiskPath( std::string Name );
[[nodiscard]] static std::shared_ptr<TModel3d>
LoadModelCpuDeferred( std::string const &DiskPath, bool const Dynamic = false );
[[nodiscard]] static TModel3d *
PublishLoadedModel(
std::string const &VirtualName,
std::shared_ptr<TModel3d> Model );
private:
// types:

16
model/Model3d.cpp
View File

@@ -1623,8 +1623,10 @@ glm::vec3 TSubModel::offset(float const Geometrytestoffsetthreshold) const
return offset;
}
bool TModel3d::LoadFromFile(std::string const &FileName, bool dynamic)
{
bool TModel3d::LoadFromFile(
std::string const &FileName,
bool const dynamic,
bool const defer_gpu_init ) {
// wczytanie modelu z pliku
/*
// NOTE: disabled, this work is now done by the model manager
@@ -1646,20 +1648,24 @@ bool TModel3d::LoadFromFile(std::string const &FileName, bool dynamic)
{
WriteLog("Forced loading text model \"" + name + ".t3d\"");
LoadFromTextFile(name + ".t3d", dynamic);
if (!dynamic)
if( false == defer_gpu_init && false == dynamic ) {
Init();
}
}
else if (FileExists(asBinary))
{
LoadFromBinFile(asBinary, dynamic);
asBinary = "";
Init();
if( false == defer_gpu_init ) {
Init();
}
}
else if (FileExists(name + ".t3d"))
{
LoadFromTextFile(name + ".t3d", dynamic);
if (!dynamic)
if( false == defer_gpu_init && false == dynamic ) {
Init();
}
}
bool const result = Root ? (iSubModelsCount > 0) : false; // brak pliku albo problem z wczytaniem

5
model/Model3d.h
View File

@@ -289,7 +289,10 @@ public:
void AddTo(TSubModel *tmp, TSubModel *SubModel);
void LoadFromTextFile(std::string const &FileName, bool dynamic);
void LoadFromBinFile(std::string const &FileName, bool dynamic);
bool LoadFromFile(std::string const &FileName, bool dynamic);
bool LoadFromFile(
std::string const &FileName,
bool dynamic,
bool defer_gpu_init = false );
TSubModel *AppendChildFromGeometry(const std::string &name, const std::string &parent, const gfx::vertex_array &vertices, const gfx::index_array &indices);
void SaveToBinFile(std::string const &FileName);
uint32_t Flags() const { return iFlags; };

8
rendering/opengl33renderer.cpp
View File

@@ -18,6 +18,7 @@ http://mozilla.org/MPL/2.0/.
#include "utilities/Logs.h"
#include "simulation/simulationtime.h"
#include "application/application.h"
#include "scene/eu7/eu7_section_stream.h"
#include "model/AnimModel.h"
#include "rendering/opengl33geometrybank.h"
#include "rendering/screenshot.h"
@@ -733,8 +734,11 @@ void opengl33_renderer::Render_pass(viewport_config &vp, rendermode const Mode)
m_current_viewport = &vp;
}
if ((!simulation::is_ready) || (Global.gfx_skiprendering))
{
if(
( !simulation::is_ready ) || ( Global.gfx_skiprendering ) ||
scene::eu7::loading_screen_blocks_world(
scene::eu7::stream_loading_position(),
scene::eu7::kSectionStreamLoadingDismissRadiusKm ) ) {
gl::framebuffer::unbind();
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

7
rendering/openglrenderer.cpp
View File

@@ -21,6 +21,7 @@ http://mozilla.org/MPL/2.0/.
#include "model/AnimModel.h"
#include "world/Traction.h"
#include "application/application.h"
#include "scene/eu7/eu7_section_stream.h"
#include "utilities/Logs.h"
#include "rendering/openglgeometrybank.h"
#include "rendering/openglcolor.h"
@@ -414,7 +415,11 @@ opengl_renderer::Render_pass( rendermode const Mode ) {
m_colorpass = m_renderpass;
if( ( !simulation::is_ready ) || ( Global.gfx_skiprendering ) ) {
if(
( !simulation::is_ready ) || ( Global.gfx_skiprendering ) ||
scene::eu7::loading_screen_blocks_world(
scene::eu7::stream_loading_position(),
scene::eu7::kSectionStreamLoadingDismissRadiusKm ) ) {
::glClearColor( 51.0f / 255.f, 102.0f / 255.f, 85.0f / 255.f, 1.f ); // initial background Color
::glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT );
break;

2
scene/eu7/eu7_chunks.h
View File

@@ -31,6 +31,7 @@ constexpr std::uint32_t kEu7VersionV5 { 5 };
constexpr std::uint32_t kEu7VersionV6 { 6 };
constexpr std::uint32_t kEu7VersionV7 { 7 };
constexpr std::uint32_t kEu7VersionV8 { 8 };
constexpr std::uint32_t kEu7VersionV9 { 9 };
constexpr std::uint32_t kChunkStrs { detail::make_id4( 'S', 'T', 'R', 'S' ) };
constexpr std::uint32_t kChunkIncl { detail::make_id4( 'I', 'N', 'C', 'L' ) };
@@ -62,6 +63,7 @@ constexpr std::uint8_t kPackSectionFormatV9 { 2 };
constexpr std::uint8_t kPackSectionFormatV10 { 3 };
constexpr std::uint8_t kPackSectionFormatV11 { 4 };
constexpr std::uint8_t kPackSectionFormatV12 { 5 };
constexpr std::uint8_t kPackSectionFormatV13 { 6 };
constexpr std::size_t kPackSectionChunkModels { 512 };
} // namespace scene::eu7

1
scene/eu7/eu7_loader.cpp
View File

@@ -553,6 +553,7 @@ load_module( std::string const &path, simulation::state_serializer &serializer )
auto const resolved { resolve_scenery_path( path ) };
auto const ok { load_module_recursive( path, serializer, g_load_session ) };
if( ok ) {
log_load_stats();
if( auto const cached { g_module_file_cache.find( resolved ) };
cached != g_module_file_cache.end() && cached->second.has_pack_chunk ) {
init_section_stream( cached->second, resolved, serializer );

312
scene/eu7/eu7_model_prefetch.cpp
View File

@@ -11,12 +11,13 @@ http://mozilla.org/MPL/2.0/.
#include "scene/eu7/eu7_model_prefetch.h"
#include "model/AnimModel.h"
#include "model/MdlMngr.h"
#include "rendering/renderer.h"
#include "scene/eu7/eu7_pack_bench.h"
#include "utilities/Globals.h"
#include "utilities/utilities.h"
#include "vehicle/DynObj.h"
#include <algorithm>
#include <list>
#include <mutex>
#include <string>
@@ -28,11 +29,29 @@ namespace scene::eu7 {
namespace {
constexpr std::size_t kMaxWarmedTextures { 2048 };
constexpr std::size_t kMaxPackMaterialAssignCache { 4096 };
std::mutex g_pack_mesh_load_mutex;
std::unordered_set<std::string> g_session_warmed_textures;
std::list<std::string> g_session_warmed_textures_lru;
std::unordered_map<std::string, std::list<std::string>::iterator> g_session_warmed_textures_iters;
std::unordered_map<std::string, material_data> g_pack_material_assign_cache;
[[nodiscard]] std::string
pack_material_cache_key(
std::string const &model_file,
std::string const &texture_file ) {
return model_file + '\x1e' + texture_file;
}
void
evict_pack_material_assign_cache_if_needed() {
while( g_pack_material_assign_cache.size() >= kMaxPackMaterialAssignCache ) {
if( g_pack_material_assign_cache.empty() ) {
break;
}
g_pack_material_assign_cache.erase( g_pack_material_assign_cache.begin() );
}
}
void
touch_warmed_texture_lru( std::string const &texture_file ) {
@@ -107,12 +126,119 @@ pack_texture_usable( std::string texture_file ) {
void
preload_pack_model_file( std::string const & ) {
// Mesh warm runs on the main thread via ensure_pack_mesh_in_session_cache only.
// GetModel mutates Global.asCurrentTexturePath and must not run on PACK workers.
// GetModel + Init must run on the main thread (Global.asCurrentTexturePath + OpenGL).
}
[[nodiscard]] bool
warm_one_pack_texture( std::string texture_file, std::unordered_set<std::string> &seen ) {
pack_texture_path_is_rooted( std::string const &texture_file ) {
return texture_file.starts_with( "dynamic/" ) ||
texture_file.starts_with( "textures/" ) ||
texture_file.starts_with( "scenery/" ) ||
texture_file.starts_with( "models/" );
}
[[nodiscard]] std::string
pack_model_texture_dir( std::string model_file ) {
if( model_file.empty() || model_file == "notload" ) {
return {};
}
replace_slashes( model_file );
erase_extension( model_file );
auto const slash_pos { model_file.rfind( '/' ) };
if( slash_pos == std::string::npos ) {
return {};
}
return model_file.substr( 0, slash_pos + 1 );
}
void
append_pack_texture_candidate(
std::vector<std::string> &candidates,
std::string candidate ) {
if( candidate.empty() ) {
return;
}
replace_slashes( candidate );
if(
std::find( candidates.begin(), candidates.end(), candidate ) ==
candidates.end() ) {
candidates.emplace_back( std::move( candidate ) );
}
}
[[nodiscard]] std::vector<std::string>
pack_texture_resolve_candidates(
std::string const &model_file,
std::string texture_file ) {
std::vector<std::string> candidates;
candidates.reserve( 4 );
if( false == pack_texture_usable( texture_file ) ) {
return candidates;
}
replace_slashes( texture_file );
append_pack_texture_candidate( candidates, texture_file );
auto const model_dir { pack_model_texture_dir( model_file ) };
if( false == model_dir.empty() ) {
append_pack_texture_candidate( candidates, model_dir + texture_file );
}
if( false == pack_texture_path_is_rooted( texture_file ) ) {
append_pack_texture_candidate( candidates, paths::textures + texture_file );
}
return candidates;
}
class PackTexturePathScope {
public:
explicit PackTexturePathScope( std::string const &model_file ) {
m_saved_path = Global.asCurrentTexturePath;
auto const model_dir { pack_model_texture_dir( model_file ) };
if( false == model_dir.empty() && model_file.find( '/' ) != std::string::npos ) {
Global.asCurrentTexturePath = model_dir;
m_active = true;
}
}
~PackTexturePathScope() {
if( m_active ) {
Global.asCurrentTexturePath = std::move( m_saved_path );
}
}
PackTexturePathScope( PackTexturePathScope const & ) = delete;
PackTexturePathScope &operator=( PackTexturePathScope const & ) = delete;
private:
std::string m_saved_path;
bool m_active { false };
};
[[nodiscard]] bool
fetch_pack_texture_material( std::string const &texture_file ) {
auto const resolved { TextureTest( ToLower( texture_file ) ) };
if( false == resolved.empty() ) {
GfxRenderer->Fetch_Material( resolved );
return true;
}
bool warmed { false };
for( int skinindex { 1 }; skinindex <= 4; ++skinindex ) {
auto const multi {
TextureTest( ToLower( texture_file + "," + std::to_string( skinindex ) ) ) };
if( multi.empty() ) {
break;
}
GfxRenderer->Fetch_Material( multi );
warmed = true;
}
return warmed;
}
[[nodiscard]] bool
warm_one_pack_texture(
std::string texture_file,
std::unordered_set<std::string> &seen,
std::string const &model_file = {} ) {
if( false == pack_texture_usable( texture_file ) ) {
return false;
}
@@ -124,77 +250,40 @@ warm_one_pack_texture( std::string texture_file, std::unordered_set<std::string>
return false;
}
auto const resolved { TextureTest( ToLower( texture_file ) ) };
if( resolved.empty() ) {
bool warmed { false };
for( int skinindex { 1 }; skinindex <= 4; ++skinindex ) {
auto const multi {
TextureTest( ToLower( texture_file + "," + std::to_string( skinindex ) ) ) };
if( multi.empty() ) {
break;
}
GfxRenderer->Fetch_Material( multi );
warmed = true;
PackTexturePathScope const scope { model_file };
for( auto const &candidate : pack_texture_resolve_candidates( model_file, texture_file ) ) {
if( fetch_pack_texture_material( candidate ) ) {
remember_warmed_texture( texture_file );
return true;
}
if( false == warmed ) {
pack_bench_inc( &Eu7PackBench::main_texture_warm_miss );
log_pack_texture_fail( texture_file );
return false;
}
}
else {
GfxRenderer->Fetch_Material( resolved );
}
remember_warmed_texture( texture_file );
pack_bench_inc( &Eu7PackBench::main_texture_warm_miss );
log_pack_texture_fail( texture_file );
return false;
}
if( false == resolved.empty() ) {
for( int skinindex { 1 }; skinindex <= 4; ++skinindex ) {
auto const multi {
TextureTest( ToLower( texture_file + "," + std::to_string( skinindex ) ) ) };
if( multi.empty() ) {
break;
}
GfxRenderer->Fetch_Material( multi );
[[nodiscard]] std::string
find_pack_model_file_for_texture(
Eu7Model const *const models,
std::size_t const model_count,
std::string texture_file ) {
if( models == nullptr || model_count == 0 ) {
return {};
}
replace_slashes( texture_file );
for( std::size_t i { 0 }; i < model_count; ++i ) {
auto candidate { models[ i ].texture_file };
replace_slashes( candidate );
if( candidate == texture_file ) {
return models[ i ].model_file;
}
}
return true;
return {};
}
} // namespace
[[nodiscard]] bool
ensure_pack_mesh_in_session_cache(
std::string model_file,
std::unordered_map<std::string, TModel3d *> &session_cache ) {
if( model_file.empty() || model_file == "notload" ) {
return false;
}
replace_slashes( model_file );
if( session_cache.contains( model_file ) ) {
pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit );
return true;
}
auto const was_global_cached { TModelsManager::IsModelCached( model_file ) };
TModel3d *mesh { nullptr };
{
std::lock_guard<std::mutex> lock { g_pack_mesh_load_mutex };
mesh = TModelsManager::GetModel( model_file, false, false );
}
if( was_global_cached ) {
pack_bench_inc( &Eu7PackBench::stream_mesh_global_hit );
}
else {
pack_bench_inc( &Eu7PackBench::stream_mesh_disk_load );
}
if( mesh != nullptr ) {
TAnimModel::warm_instanceable_cache( mesh );
}
session_cache.emplace( model_file, mesh );
return true;
}
void
reset_pack_texture_warm_cache() {
g_session_warmed_textures.clear();
@@ -221,7 +310,9 @@ warm_pack_texture_paths_main(
std::string const *const paths,
std::size_t const count,
double const budget_ms,
std::size_t *const processed_out ) {
std::size_t *const processed_out,
Eu7Model const *const models,
std::size_t const model_count ) {
if( paths == nullptr || count == 0 || GfxRenderer == nullptr ) {
if( processed_out != nullptr ) {
*processed_out = 0;
@@ -246,7 +337,10 @@ warm_pack_texture_paths_main(
break;
}
}
if( warm_one_pack_texture( paths[ i ], seen ) ) {
if( warm_one_pack_texture(
paths[ i ],
seen,
find_pack_model_file_for_texture( models, model_count, paths[ i ] ) ) ) {
++warmed;
}
++processed;
@@ -259,7 +353,10 @@ warm_pack_texture_paths_main(
}
std::size_t
warm_pack_textures_main( Eu7Model const *const models, std::size_t const count ) {
warm_pack_textures_main(
Eu7Model const *const models,
std::size_t const count,
double const budget_ms ) {
if( models == nullptr || count == 0 || GfxRenderer == nullptr ) {
return 0;
}
@@ -267,13 +364,88 @@ warm_pack_textures_main( Eu7Model const *const models, std::size_t const count )
std::unordered_set<std::string> seen;
seen.reserve( std::min( count, std::size_t { 64 } ) );
std::size_t warmed { 0 };
auto const started { std::chrono::steady_clock::now() };
for( std::size_t i { 0 }; i < count; ++i ) {
if( warm_one_pack_texture( models[ i ].texture_file, seen ) ) {
if(
budget_ms > 0.0 &&
i > 0 ) {
auto const elapsed_ms {
std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - started ).count() };
if( elapsed_ms >= budget_ms ) {
break;
}
}
if( warm_one_pack_texture(
models[ i ].texture_file,
seen,
models[ i ].model_file ) ) {
++warmed;
}
}
return warmed;
}
[[nodiscard]] bool
assign_pack_texture(
material_data &material,
std::string const &model_file,
std::string const &texture_file,
std::string const &resolved_texture,
std::uint32_t const textures_alpha ) {
if( false == pack_texture_usable( texture_file ) ) {
if( textures_alpha != 0 ) {
material.textures_alpha = static_cast<int>( textures_alpha );
}
return true;
}
auto const cache_key {
resolved_texture.empty() ?
pack_material_cache_key( model_file, texture_file ) :
pack_material_cache_key( model_file, resolved_texture ) };
auto const cached { g_pack_material_assign_cache.find( cache_key ) };
if(
cached != g_pack_material_assign_cache.end() &&
cached->second.replacable_skins[ 1 ] != null_handle ) {
material = cached->second;
if( textures_alpha != 0 ) {
material.textures_alpha = static_cast<int>( textures_alpha );
}
return true;
}
if( false == resolved_texture.empty() ) {
material = {};
material.assign( resolved_texture );
if( material.replacable_skins[ 1 ] != null_handle ) {
if( textures_alpha != 0 ) {
material.textures_alpha = static_cast<int>( textures_alpha );
}
evict_pack_material_assign_cache_if_needed();
g_pack_material_assign_cache.emplace( cache_key, material );
return true;
}
}
PackTexturePathScope const scope { model_file };
for( auto const &candidate : pack_texture_resolve_candidates( model_file, texture_file ) ) {
material = {};
material.assign( candidate );
if( material.replacable_skins[ 1 ] != null_handle ) {
if( textures_alpha != 0 ) {
material.textures_alpha = static_cast<int>( textures_alpha );
}
evict_pack_material_assign_cache_if_needed();
g_pack_material_assign_cache.emplace( cache_key, material );
return true;
}
}
pack_bench_inc( &Eu7PackBench::main_texture_assign_fail );
log_pack_texture_fail( texture_file );
return false;
}
} // namespace scene::eu7

28
scene/eu7/eu7_model_prefetch.h
View File

@@ -9,19 +9,16 @@ http://mozilla.org/MPL/2.0/.
#pragma once
#include "scene/eu7/eu7_pack_mesh_loader.h"
#include "scene/eu7/eu7_types.h"
#include "vehicle/DynObj.h"
#include <cstddef>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
class TModel3d;
namespace scene::eu7 {
// Deprecated: mesh warm is main-thread only (ensure_pack_mesh_in_session_cache).
void
preload_pack_models( std::vector<Eu7Model> const &Models );
@@ -30,25 +27,30 @@ preload_pack_models(
std::vector<Eu7Model> const &Models,
std::vector<std::string> const &UniqueMeshes );
// Main thread: Fetch_Material dla unikalnych texture_file z chunka przed apply.
// Zwraca liczbe unikalnych Fetch_Material w tym slice.
std::size_t
warm_pack_texture_paths_main(
std::string const *paths,
std::size_t count,
double budget_ms = 0.0,
std::size_t *processed_out = nullptr );
std::size_t *processed_out = nullptr,
Eu7Model const *models = nullptr,
std::size_t model_count = 0 );
std::size_t
warm_pack_textures_main( Eu7Model const *models, std::size_t count );
warm_pack_textures_main(
Eu7Model const *models,
std::size_t count,
double budget_ms = 0.0 );
void
reset_pack_texture_warm_cache();
// Thread-safe cold mesh load: session cache, then GetModel under g_pack_mesh_load_mutex.
[[nodiscard]] bool
ensure_pack_mesh_in_session_cache(
std::string model_file,
std::unordered_map<std::string, TModel3d *> &session_cache );
assign_pack_texture(
material_data &material,
std::string const &model_file,
std::string const &texture_file,
std::string const &resolved_texture = {},
std::uint32_t textures_alpha = 0 );
} // namespace scene::eu7

12
scene/eu7/eu7_pack_bench.cpp
View File

@@ -8,6 +8,7 @@ http://mozilla.org/MPL/2.0/.
*/
#include "stdafx.h"
#include "scene/eu7/eu7_pack_mesh_loader.h"
#include "scene/eu7/eu7_pack_bench.h"
#include "utilities/Logs.h"
@@ -181,6 +182,9 @@ log_pack_bench_impl( Eu7PackBench const &bench, char const *const title ) {
" diag mesh: sess_hit=" + std::to_string( bench.stream_mesh_session_hit ) +
" glob_hit=" + std::to_string( bench.stream_mesh_global_hit ) +
" disk=" + std::to_string( bench.stream_mesh_disk_load ) +
" async_q=" + std::to_string( bench.stream_mesh_async_queued ) +
" async_r=" + std::to_string( bench.stream_mesh_async_ready ) +
" loader=" + std::to_string( bench.loader_thread_disk_loads ) +
" dequeue_near=" + std::to_string( bench.stream_dequeue_near ) +
" far=" + std::to_string( bench.stream_dequeue_far ) +
" cam=" + std::to_string( bench.stream_dequeue_camera ) +
@@ -189,7 +193,10 @@ log_pack_bench_impl( Eu7PackBench const &bench, char const *const title ) {
" block_ring_in=" + std::to_string( bench.stream_jobs_blocked_ring_inner ) +
" block_ring_out=" + std::to_string( bench.stream_jobs_blocked_ring_outer ) +
" defer_dist=" + std::to_string( bench.stream_worker_deferred_distant ) +
" stuck_skip=" + std::to_string( bench.stream_apply_stuck_skip ) );
" stuck_skip=" + std::to_string( bench.stream_apply_stuck_skip ) +
" apply_defer=" + std::to_string( bench.stream_apply_deferred ) +
" defer_bypass=" + std::to_string( bench.stream_apply_defer_bypass ) +
" sparse_skip=" + std::to_string( bench.stream_sparse_apply_skip ) );
WriteLog(
" diag ready_tex_ms=" + std::to_string( static_cast<int>( bench.stream_prefetch_ready_tex_ms ) ) +
" ready_umes_ms=" + std::to_string( static_cast<int>( bench.stream_prefetch_ready_umes_ms ) ) +
@@ -442,6 +449,9 @@ log_pack_stream_status(
" ahead=" + std::to_string( bench.stream_lookahead_enqueue ) +
" urg_chunks=" + std::to_string( bench.stream_chunks_urgent ) +
" cold_trunc=" + std::to_string( bench.stream_cold_slice_truncated ) +
" mesh_q=" + std::to_string( pack_mesh_loader_queue_depth() ) +
" mesh_r=" + std::to_string( pack_mesh_loader_ready_count() ) +
" mesh_d=" + std::to_string( bench.stream_mesh_async_drained ) +
" preempt=" + std::to_string( bench.stream_preempt_pending ) );
}

9
scene/eu7/eu7_pack_bench.h
View File

@@ -63,6 +63,12 @@ struct Eu7PackBench {
std::uint64_t stream_mesh_session_hit { 0 };
std::uint64_t stream_mesh_global_hit { 0 };
std::uint64_t stream_mesh_disk_load { 0 };
std::uint64_t stream_mesh_async_queued { 0 };
std::uint64_t stream_mesh_async_ready { 0 };
std::uint64_t stream_mesh_async_drained { 0 };
std::uint64_t stream_mesh_async_wait_timeout { 0 };
std::uint64_t loader_thread_disk_loads { 0 };
double loader_thread_getmodel_ms { 0.0 };
std::uint64_t stream_chunks_urgent { 0 };
std::uint64_t stream_chunks_heavy { 0 };
std::uint64_t stream_chunks_light { 0 };
@@ -75,6 +81,9 @@ struct Eu7PackBench {
std::uint64_t stream_jobs_blocked_ring_outer { 0 };
std::uint64_t stream_worker_deferred_distant { 0 };
std::uint64_t stream_apply_stuck_skip { 0 };
std::uint64_t stream_apply_deferred { 0 };
std::uint64_t stream_apply_defer_bypass { 0 };
std::uint64_t stream_sparse_apply_skip { 0 };
std::uint64_t stream_sections_unloaded { 0 };
std::uint64_t stream_unload_instances { 0 };
std::uint64_t stream_mesh_cache_evictions { 0 };

501
scene/eu7/eu7_pack_mesh_loader.cpp Normal file
View File

@@ -0,0 +1,501 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#include "stdafx.h"
#include "scene/eu7/eu7_pack_mesh_loader.h"
#include "model/AnimModel.h"
#include "model/MdlMngr.h"
#include "scene/eu7/eu7_pack_bench.h"
#include "utilities/Globals.h"
#include "utilities/utilities.h"
#include <chrono>
#include <deque>
#include <mutex>
#include <queue>
#include <unordered_set>
namespace scene::eu7 {
namespace {
std::mutex g_pack_mesh_load_mutex;
struct PackMeshQueueEntry {
int priority { 0 };
std::uint64_t sequence { 0 };
std::string model_file;
};
struct PackMeshQueueCompare {
[[nodiscard]] bool
operator()( PackMeshQueueEntry const &lhs, PackMeshQueueEntry const &rhs ) const {
if( lhs.priority != rhs.priority ) {
return lhs.priority > rhs.priority;
}
return lhs.sequence > rhs.sequence;
}
};
struct PackMeshLoader {
std::mutex mutex;
std::priority_queue<
PackMeshQueueEntry,
std::vector<PackMeshQueueEntry>,
PackMeshQueueCompare> queue;
std::unordered_map<std::string, int> queued;
std::unordered_map<std::string, TModel3d *> ready;
std::uint64_t sequence { 0 };
bool running { false };
};
PackMeshLoader g_loader;
[[nodiscard]] std::string
pack_model_texture_dir( std::string model_file ) {
if( model_file.empty() || model_file == "notload" ) {
return {};
}
replace_slashes( model_file );
erase_extension( model_file );
auto const slash_pos { model_file.rfind( '/' ) };
if( slash_pos == std::string::npos ) {
return {};
}
return model_file.substr( 0, slash_pos + 1 );
}
class PackTexturePathScope {
public:
explicit PackTexturePathScope( std::string const &model_file ) {
m_saved_path = Global.asCurrentTexturePath;
auto const model_dir { pack_model_texture_dir( model_file ) };
if( false == model_dir.empty() && model_file.find( '/' ) != std::string::npos ) {
Global.asCurrentTexturePath = model_dir;
m_active = true;
}
}
~PackTexturePathScope() {
if( m_active ) {
Global.asCurrentTexturePath = std::move( m_saved_path );
}
}
PackTexturePathScope( PackTexturePathScope const & ) = delete;
PackTexturePathScope &operator=( PackTexturePathScope const & ) = delete;
private:
std::string m_saved_path;
bool m_active { false };
};
[[nodiscard]] bool
pack_mesh_path_valid( std::string const &model_file ) {
return false == model_file.empty() && model_file != "notload";
}
[[nodiscard]] bool
pack_mesh_pointer_usable( TModel3d *const mesh ) {
return mesh != nullptr;
}
[[nodiscard]] bool
pack_mesh_cached_usable(
std::unordered_map<std::string, TModel3d *> const &session_cache,
std::string const &model_file ) {
auto const found { session_cache.find( model_file ) };
return found != session_cache.end() && pack_mesh_pointer_usable( found->second );
}
[[nodiscard]] bool
pack_mesh_globally_usable( std::string model_file ) {
if( false == pack_mesh_path_valid( model_file ) ) {
return false;
}
replace_slashes( model_file );
if( false == TModelsManager::IsModelCached( model_file ) ) {
return false;
}
TModel3d *mesh { nullptr };
{
std::lock_guard<std::mutex> lock { g_pack_mesh_load_mutex };
mesh = TModelsManager::GetModel( model_file, false, false );
}
return pack_mesh_pointer_usable( mesh );
}
[[nodiscard]] bool
pop_pack_mesh_queue_entry( std::string &model_file ) {
model_file.clear();
while( false == g_loader.queue.empty() ) {
auto entry { g_loader.queue.top() };
g_loader.queue.pop();
auto const found { g_loader.queued.find( entry.model_file ) };
if(
found == g_loader.queued.end() ||
found->second != entry.priority ) {
continue;
}
model_file = std::move( entry.model_file );
return true;
}
return false;
}
void
load_pack_mesh_on_main_thread( std::string const &model_file ) {
PackBenchTimer const load_timer { &Eu7PackBench::loader_thread_getmodel_ms };
TModel3d *mesh { nullptr };
{
std::lock_guard<std::mutex> lock { g_pack_mesh_load_mutex };
PackTexturePathScope const scope { model_file };
mesh = TModelsManager::GetModel( model_file, false, false );
if( pack_mesh_pointer_usable( mesh ) ) {
TAnimModel::warm_instanceable_cache( mesh );
}
}
pack_bench_inc( &Eu7PackBench::loader_thread_disk_loads );
std::lock_guard<std::mutex> lock { g_loader.mutex };
g_loader.ready.emplace( model_file, mesh );
g_loader.queued.erase( model_file );
}
[[nodiscard]] std::size_t
pump_pack_mesh_loader_main(
double const budget_ms,
std::size_t const max_loads ) {
if( false == g_loader.running ) {
return 0;
}
auto const started { std::chrono::steady_clock::now() };
std::size_t loaded { 0 };
while( loaded < max_loads ) {
if( budget_ms > 0.0 ) {
auto const elapsed_ms {
std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - started ).count() };
if( elapsed_ms >= budget_ms ) {
break;
}
}
std::string model_file;
{
std::lock_guard<std::mutex> lock { g_loader.mutex };
if( false == pop_pack_mesh_queue_entry( model_file ) ) {
break;
}
}
load_pack_mesh_on_main_thread( model_file );
++loaded;
}
return loaded;
}
[[nodiscard]] bool
try_consume_loader_ready(
std::string const &model_file,
std::unordered_map<std::string, TModel3d *> &session_cache ) {
TModel3d *mesh { nullptr };
{
std::lock_guard<std::mutex> lock { g_loader.mutex };
auto const found { g_loader.ready.find( model_file ) };
if( found == g_loader.ready.end() ) {
return false;
}
mesh = found->second;
g_loader.ready.erase( found );
}
session_cache.emplace( model_file, mesh );
pack_bench_inc( &Eu7PackBench::stream_mesh_async_ready );
return pack_mesh_pointer_usable( mesh );
}
[[nodiscard]] bool
sync_resolve_global_cached_mesh(
std::string const &model_file,
std::unordered_map<std::string, TModel3d *> &session_cache ) {
if( false == pack_mesh_globally_usable( model_file ) ) {
return false;
}
TModel3d *mesh { nullptr };
{
PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms };
std::lock_guard<std::mutex> lock { g_pack_mesh_load_mutex };
PackTexturePathScope const scope { model_file };
mesh = TModelsManager::GetModel( model_file, false, false );
pack_bench_inc( &Eu7PackBench::main_cold_getmodel_calls );
if( pack_mesh_pointer_usable( mesh ) ) {
TAnimModel::warm_instanceable_cache( mesh );
}
}
pack_bench_inc( &Eu7PackBench::stream_mesh_global_hit );
session_cache.emplace( model_file, mesh );
return pack_mesh_pointer_usable( mesh );
}
void
enqueue_pack_mesh_load( std::string model_file, int const priority ) {
if( false == pack_mesh_path_valid( model_file ) ) {
return;
}
replace_slashes( model_file );
{
std::lock_guard<std::mutex> lock { g_loader.mutex };
if( g_loader.ready.contains( model_file ) ) {
return;
}
auto const found { g_loader.queued.find( model_file ) };
if( found != g_loader.queued.end() ) {
if( priority >= found->second ) {
return;
}
found->second = priority;
}
else {
g_loader.queued.emplace( model_file, priority );
}
g_loader.queue.push(
PackMeshQueueEntry { priority, g_loader.sequence++, std::move( model_file ) } );
}
pack_bench_inc( &Eu7PackBench::stream_mesh_async_queued );
}
[[nodiscard]] bool
wait_for_loader_ready(
std::string const &model_file,
double const block_budget_ms ) {
auto const deadline {
block_budget_ms > 0.0 ?
std::chrono::steady_clock::now() +
std::chrono::duration_cast<std::chrono::steady_clock::duration>(
std::chrono::duration<double, std::milli>( block_budget_ms ) ) :
std::chrono::steady_clock::time_point::max() };
while( true ) {
{
std::lock_guard<std::mutex> lock { g_loader.mutex };
if( g_loader.ready.contains( model_file ) ) {
return true;
}
if( g_loader.queued.contains( model_file ) == false ) {
return false;
}
}
if( block_budget_ms > 0.0 && std::chrono::steady_clock::now() >= deadline ) {
return false;
}
auto remaining_ms { block_budget_ms };
if( block_budget_ms > 0.0 ) {
remaining_ms = std::chrono::duration<double, std::milli>(
deadline - std::chrono::steady_clock::now() ).count();
if( remaining_ms <= 0.0 ) {
return false;
}
}
(void)pump_pack_mesh_loader_main( remaining_ms, 1 );
}
}
} // namespace
void
start_pack_mesh_loader() {
stop_pack_mesh_loader();
g_loader.running = true;
}
void
stop_pack_mesh_loader() {
g_loader.running = false;
std::lock_guard<std::mutex> lock { g_loader.mutex };
g_loader.queue = decltype( g_loader.queue )();
g_loader.queued.clear();
g_loader.ready.clear();
}
void
reset_pack_mesh_loader() {
stop_pack_mesh_loader();
}
void
request_pack_mesh_load( std::string const &model_file, int const priority ) {
if( false == g_loader.running ) {
return;
}
if( false == pack_mesh_path_valid( model_file ) ) {
return;
}
if( pack_mesh_globally_usable( model_file ) ) {
return;
}
enqueue_pack_mesh_load( model_file, priority );
}
void
request_pack_mesh_load_paths(
std::vector<std::string> const &model_files,
std::size_t const max_enqueue,
int const priority ) {
if( false == g_loader.running ) {
return;
}
std::size_t enqueued { 0 };
for( auto const &path : model_files ) {
if( max_enqueue > 0 && enqueued >= max_enqueue ) {
break;
}
if( false == pack_mesh_path_valid( path ) ) {
continue;
}
if( pack_mesh_globally_usable( path ) ) {
continue;
}
enqueue_pack_mesh_load( path, priority );
++enqueued;
}
}
[[nodiscard]] std::size_t
pump_pack_mesh_loader(
double const budget_ms,
std::size_t const max_loads ) {
return pump_pack_mesh_loader_main( budget_ms, max_loads );
}
[[nodiscard]] TModel3d *
session_cached_pack_mesh(
std::unordered_map<std::string, TModel3d *> const &session_cache,
std::string const &model_file ) {
auto const found { session_cache.find( model_file ) };
return found != session_cache.end() ? found->second : nullptr;
}
[[nodiscard]] TModel3d *
ensure_pack_mesh_in_session_cache(
std::string model_file,
std::unordered_map<std::string, TModel3d *> &session_cache,
PackMeshLoadWait const wait,
double const block_budget_ms ) {
if( false == pack_mesh_path_valid( model_file ) ) {
return nullptr;
}
replace_slashes( model_file );
if( pack_mesh_cached_usable( session_cache, model_file ) ) {
pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit );
return session_cached_pack_mesh( session_cache, model_file );
}
if( session_cache.contains( model_file ) ) {
session_cache.erase( model_file );
}
if( try_consume_loader_ready( model_file, session_cache ) ) {
return session_cached_pack_mesh( session_cache, model_file );
}
if( sync_resolve_global_cached_mesh( model_file, session_cache ) ) {
return session_cached_pack_mesh( session_cache, model_file );
}
request_pack_mesh_load( model_file );
if( wait == PackMeshLoadWait::BlockUntilReady ) {
if( wait_for_loader_ready( model_file, block_budget_ms ) ) {
if( try_consume_loader_ready( model_file, session_cache ) ) {
return session_cached_pack_mesh( session_cache, model_file );
}
}
pack_bench_inc( &Eu7PackBench::stream_mesh_async_wait_timeout );
return nullptr;
}
return nullptr;
}
[[nodiscard]] std::size_t
pack_mesh_loader_queue_depth() {
std::lock_guard<std::mutex> lock { g_loader.mutex };
return g_loader.queued.size();
}
[[nodiscard]] bool
try_adopt_pack_mesh_for_slice(
std::string model_file,
std::unordered_map<std::string, TModel3d *> &session_cache ) {
if( model_file.empty() || model_file == "notload" ) {
return true;
}
replace_slashes( model_file );
if( pack_mesh_cached_usable( session_cache, model_file ) ) {
return true;
}
if( session_cache.contains( model_file ) ) {
session_cache.erase( model_file );
}
if( try_consume_loader_ready( model_file, session_cache ) ) {
return true;
}
return sync_resolve_global_cached_mesh( model_file, session_cache );
}
[[nodiscard]] std::size_t
drain_pack_mesh_loader_ready(
std::unordered_map<std::string, TModel3d *> &session_cache,
std::size_t const max_drain ) {
std::vector<std::pair<std::string, TModel3d *>> drained;
{
std::lock_guard<std::mutex> lock { g_loader.mutex };
drained.reserve( g_loader.ready.size() );
for( auto it { g_loader.ready.begin() }; it != g_loader.ready.end(); ) {
if( max_drain > 0 && drained.size() >= max_drain ) {
break;
}
drained.emplace_back( it->first, it->second );
it = g_loader.ready.erase( it );
}
}
std::size_t adopted { 0 };
for( auto & [path, mesh] : drained ) {
session_cache.emplace( std::move( path ), mesh );
pack_bench_inc( &Eu7PackBench::stream_mesh_async_ready );
pack_bench_inc( &Eu7PackBench::stream_mesh_async_drained );
if( pack_mesh_pointer_usable( mesh ) ) {
++adopted;
}
}
return adopted;
}
[[nodiscard]] std::size_t
pack_mesh_loader_ready_count() {
std::lock_guard<std::mutex> lock { g_loader.mutex };
return g_loader.ready.size();
}
[[nodiscard]] std::size_t
pack_mesh_loader_worker_count() {
return 0;
}
} // namespace scene::eu7

75
scene/eu7/eu7_pack_mesh_loader.h Normal file
View File

@@ -0,0 +1,75 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#pragma once
#include <cstddef>
#include <string>
#include <unordered_map>
#include <vector>
class TModel3d;
namespace scene::eu7 {
enum class PackMeshLoadWait {
NonBlocking,
BlockUntilReady,
};
void
start_pack_mesh_loader();
void
stop_pack_mesh_loader();
void
reset_pack_mesh_loader();
void
request_pack_mesh_load( std::string const &model_file, int priority = 0 );
void
request_pack_mesh_load_paths(
std::vector<std::string> const &model_files,
std::size_t max_enqueue = 0,
int priority = 0 );
[[nodiscard]] TModel3d *
ensure_pack_mesh_in_session_cache(
std::string model_file,
std::unordered_map<std::string, TModel3d *> &session_cache,
PackMeshLoadWait wait = PackMeshLoadWait::NonBlocking,
double block_budget_ms = 0.0 );
[[nodiscard]] bool
try_adopt_pack_mesh_for_slice(
std::string model_file,
std::unordered_map<std::string, TModel3d *> &session_cache );
[[nodiscard]] std::size_t
drain_pack_mesh_loader_ready(
std::unordered_map<std::string, TModel3d *> &session_cache,
std::size_t max_drain = 0 );
[[nodiscard]] std::size_t
pack_mesh_loader_queue_depth();
[[nodiscard]] std::size_t
pack_mesh_loader_ready_count();
[[nodiscard]] std::size_t
pack_mesh_loader_worker_count();
[[nodiscard]] std::size_t
pump_pack_mesh_loader(
double budget_ms = 0.0,
std::size_t max_loads = 1 );
} // namespace scene::eu7

491
scene/eu7/eu7_reader.cpp
View File

@@ -16,6 +16,7 @@ http://mozilla.org/MPL/2.0/.
#include <fstream>
#include <limits>
#include <memory>
#include <stdexcept>
namespace scene::eu7 {
@@ -70,6 +71,80 @@ private:
std::vector<std::string> strings_;
};
class PackByteReader {
public:
PackByteReader( std::uint8_t const *data, std::size_t const size )
: cur_ { data }
, end_ { data + size } {}
[[nodiscard]] bool
enough( std::size_t const bytes ) const noexcept {
return static_cast<std::size_t>( end_ - cur_ ) >= bytes;
}
[[nodiscard]] std::size_t
remaining() const noexcept {
return static_cast<std::size_t>( end_ - cur_ );
}
[[nodiscard]] std::uint8_t
u8() {
if( false == enough( 1 ) ) {
throw std::runtime_error( "EU7 PACK: przekroczono bufor sekcji" );
}
return *cur_++;
}
[[nodiscard]] std::uint16_t
u16() {
if( false == enough( 2 ) ) {
throw std::runtime_error( "EU7 PACK: przekroczono bufor sekcji" );
}
auto const lo { cur_[ 0 ] };
auto const hi { cur_[ 1 ] };
cur_ += 2;
return static_cast<std::uint16_t>( lo | ( static_cast<std::uint16_t>( hi ) << 8u ) );
}
[[nodiscard]] std::uint32_t
u32() {
if( false == enough( 4 ) ) {
throw std::runtime_error( "EU7 PACK: przekroczono bufor sekcji" );
}
std::uint32_t value { 0 };
std::memcpy( &value, cur_, 4 );
cur_ += 4;
return value;
}
[[nodiscard]] std::int32_t
i32() {
return static_cast<std::int32_t>( u32() );
}
[[nodiscard]] float
f32() {
std::uint32_t bits { u32() };
float value { 0.f };
std::memcpy( &value, &bits, sizeof( value ) );
return value;
}
[[nodiscard]] double
f64_disk() {
return static_cast<double>( f32() );
}
[[nodiscard]] glm::dvec3
vec3() {
return { f64_disk(), f64_disk(), f64_disk() };
}
private:
std::uint8_t const *cur_;
std::uint8_t const *end_;
};
[[nodiscard]] std::int16_t
read_i16( std::istream &input ) {
return static_cast<std::int16_t>( sn_utils::ld_uint16( input ) );
@@ -207,6 +282,97 @@ read_slim_node( std::istream &input, StringTable const &table, std::string_view
return node;
}
[[nodiscard]] Eu7TransformContext
read_transform_context_bytes( PackByteReader &input ) {
Eu7TransformContext transform;
const std::uint8_t origin_count { input.u8() };
transform.origin_stack.reserve( origin_count );
for( std::uint8_t i { 0 }; i < origin_count; ++i ) {
transform.origin_stack.push_back( input.vec3() );
}
const std::uint8_t scale_count { input.u8() };
transform.scale_stack.reserve( scale_count );
for( std::uint8_t i { 0 }; i < scale_count; ++i ) {
transform.scale_stack.push_back( input.vec3() );
}
transform.rotation = input.vec3();
transform.group_depth = input.u8();
return transform;
}
[[nodiscard]] Eu7BasicNode
read_slim_node_bytes(
PackByteReader &input,
StringTable const &table,
std::string_view const implied_type ) {
Eu7BasicNode node;
node.node_type = std::string( implied_type );
const std::uint8_t flags { input.u8() };
if( ( flags & kNodeFlagHasName ) != 0 ) {
node.name = table.resolve( input.u32() );
}
if( ( flags & kNodeFlagHasRangeMin ) != 0 ) {
node.range_squared_min = input.f64_disk();
}
if( ( flags & kNodeFlagHasRangeMax ) != 0 ) {
node.range_squared_max = input.f64_disk();
}
else {
node.range_squared_max = std::numeric_limits<double>::max();
}
if( ( flags & kNodeFlagHasBounds ) != 0 ) {
node.area.center = input.vec3();
node.area.radius = input.f32();
}
if( ( flags & kNodeFlagHasGroup ) != 0 ) {
node.group_valid = true;
node.group_handle = input.u32();
}
if( ( flags & kNodeFlagHasTransform ) != 0 ) {
node.transform = read_transform_context_bytes( input );
}
node.visible = ( flags & kNodeFlagNotVisible ) == 0;
return node;
}
[[nodiscard]] Eu7Model
read_runtime_model_bytes(
PackByteReader &input,
StringTable const &table,
Eu7PackSectionCursor const *cursor = nullptr ) {
Eu7Model model;
model.node = read_slim_node_bytes( input, table, "model" );
model.is_terrain = input.u8() != 0;
model.transition = input.u8() != 0;
model.location = input.vec3();
model.angles = input.vec3();
model.scale = input.vec3();
if(
cursor != nullptr &&
cursor->section_format == kPackSectionFormatV13 ) {
model.pack_mesh_index = input.u16();
model.pack_texture_index = input.u16();
}
else {
model.model_file = table.resolve( input.u32() );
model.texture_file = table.resolve( input.u32() );
}
const std::uint32_t light_count { input.u32() };
model.light_states.resize( light_count );
for( std::uint32_t i { 0 }; i < light_count; ++i ) {
model.light_states[ i ] = input.f32();
}
const std::uint32_t color_count { input.u32() };
model.light_colors.resize( color_count );
for( std::uint32_t i { 0 }; i < color_count; ++i ) {
model.light_colors[ i ] = input.u32();
}
if( cursor != nullptr && cursor->section_format == kPackSectionFormatV13 ) {
model.pack_cell_id = input.u8();
}
return model;
}
[[nodiscard]] std::string
read_length_string( std::istream &input ) {
const std::uint32_t length { sn_utils::ld_uint32( input ) };
@@ -413,7 +579,10 @@ read_runtime_lines( std::istream &input, StringTable const &table ) {
}
[[nodiscard]] Eu7Model
read_runtime_model( std::istream &input, StringTable const &table ) {
read_runtime_model(
std::istream &input,
StringTable const &table,
Eu7PackSectionCursor const *cursor = nullptr ) {
Eu7Model model;
model.node = read_slim_node( input, table, "model" );
model.is_terrain = sn_utils::d_uint8( input ) != 0;
@@ -421,8 +590,16 @@ read_runtime_model( std::istream &input, StringTable const &table ) {
model.location = read_vec3( input );
model.angles = read_vec3( input );
model.scale = read_vec3( input );
model.model_file = table.resolve( sn_utils::ld_uint32( input ) );
model.texture_file = table.resolve( sn_utils::ld_uint32( input ) );
if(
cursor != nullptr &&
cursor->section_format == kPackSectionFormatV13 ) {
model.pack_mesh_index = sn_utils::ld_uint16( input );
model.pack_texture_index = sn_utils::ld_uint16( input );
}
else {
model.model_file = table.resolve( sn_utils::ld_uint32( input ) );
model.texture_file = table.resolve( sn_utils::ld_uint32( input ) );
}
const std::uint32_t light_count { sn_utils::ld_uint32( input ) };
model.light_states.resize( light_count );
for( std::uint32_t i { 0 }; i < light_count; ++i ) {
@@ -433,11 +610,17 @@ read_runtime_model( std::istream &input, StringTable const &table ) {
for( std::uint32_t i { 0 }; i < color_count; ++i ) {
model.light_colors[ i ] = sn_utils::ld_uint32( input );
}
if( cursor != nullptr && cursor->section_format == kPackSectionFormatV13 ) {
model.pack_cell_id = sn_utils::d_uint8( input );
}
return model;
}
[[nodiscard]] Eu7ModelPrototype
read_runtime_prototype( std::istream &input, StringTable const &table ) {
read_runtime_prototype(
std::istream &input,
StringTable const &table,
std::uint32_t const file_version ) {
Eu7ModelPrototype proto;
proto.node = read_slim_node( input, table, "model" );
proto.is_terrain = sn_utils::d_uint8( input ) != 0;
@@ -454,16 +637,27 @@ read_runtime_prototype( std::istream &input, StringTable const &table ) {
for( std::uint32_t i { 0 }; i < color_count; ++i ) {
proto.light_colors[ i ] = sn_utils::ld_uint32( input );
}
if( file_version >= kEu7VersionV9 ) {
proto.resolved_texture = table.resolve( sn_utils::ld_uint32( input ) );
proto.pack_flags = sn_utils::d_uint8( input );
proto.textures_alpha = sn_utils::ld_uint32( input );
proto.baked_range_min = sn_utils::ld_float32( input );
proto.baked_range_max = sn_utils::ld_float32( input );
}
return proto;
}
void
read_prot_chunk( std::istream &input, StringTable const &strings, Eu7Module &module ) {
read_prot_chunk(
std::istream &input,
StringTable const &strings,
Eu7Module &module ) {
const std::uint32_t count { sn_utils::ld_uint32( input ) };
module.model_prototypes.clear();
module.model_prototypes.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
module.model_prototypes.push_back( read_runtime_prototype( input, strings ) );
module.model_prototypes.push_back(
read_runtime_prototype( input, strings, module.version ) );
}
}
@@ -483,10 +677,15 @@ expand_prototype_instance(
model.scale = scale;
model.model_file = proto.model_file;
model.texture_file = proto.texture_file;
model.resolved_texture = proto.resolved_texture;
model.light_states = proto.light_states;
model.light_colors = proto.light_colors;
model.transition = proto.transition;
model.is_terrain = proto.is_terrain;
model.pack_flags = proto.pack_flags;
model.textures_alpha = proto.textures_alpha;
model.baked_range_min = proto.baked_range_min;
model.baked_range_max = proto.baked_range_max;
return model;
}
@@ -506,6 +705,55 @@ parse_pack_section_header(
peek == EOF ? std::uint8_t { 0 } : static_cast<std::uint8_t>( peek ) };
bool use_v12_header { false };
if( first_byte == kPackSectionFormatV13 ) {
cursor.section_format = sn_utils::d_uint8( input );
const std::uint32_t solo_total { sn_utils::ld_uint32( input ) };
const std::uint32_t inst_total { sn_utils::ld_uint32( input ) };
const std::uint32_t unique_mesh_count { sn_utils::ld_uint32( input ) };
if(
solo_total + inst_total == entry.model_count &&
( inst_total == 0 || false == module.model_prototypes.empty() ) ) {
use_v12_header = true;
cursor.model_total = solo_total + inst_total;
cursor.solo_remaining = solo_total;
cursor.inst_remaining = inst_total;
StringTable const strings { module.strings };
cursor.unique_meshes.reserve( unique_mesh_count );
for( std::uint32_t mesh_idx { 0 }; mesh_idx < unique_mesh_count; ++mesh_idx ) {
cursor.unique_meshes.push_back(
strings.resolve( sn_utils::ld_uint32( input ) ) );
}
const std::uint32_t unique_texture_count { sn_utils::ld_uint32( input ) };
cursor.unique_textures.reserve( unique_texture_count );
for( std::uint32_t tex_idx { 0 }; tex_idx < unique_texture_count; ++tex_idx ) {
cursor.unique_textures.push_back(
strings.resolve( sn_utils::ld_uint32( input ) ) );
}
cursor.chunk_count = sn_utils::ld_uint32( input );
cursor.chunk_model_counts.resize( cursor.chunk_count );
cursor.chunk_inst_counts.resize( cursor.chunk_count );
cursor.chunk_byte_offsets.resize( cursor.chunk_count );
for( std::uint32_t chunk_idx { 0 }; chunk_idx < cursor.chunk_count; ++chunk_idx ) {
cursor.chunk_model_counts[ chunk_idx ] = sn_utils::ld_uint32( input );
cursor.chunk_inst_counts[ chunk_idx ] = sn_utils::ld_uint32( input );
cursor.chunk_byte_offsets[ chunk_idx ] = sn_utils::ld_uint32( input );
}
}
if( false == use_v12_header ) {
input.seekg( section_start );
}
}
if( use_v12_header ) {
cursor.models_read = 0;
cursor.header_parsed = true;
return;
}
use_v12_header = false;
if( first_byte == kPackSectionFormatV12 ) {
cursor.section_format = sn_utils::d_uint8( input );
const std::uint32_t solo_total { sn_utils::ld_uint32( input ) };
@@ -709,7 +957,7 @@ read_pack_models_chunk_impl(
models.size() < max_count &&
( cursor.solo_remaining > 0 || cursor.inst_remaining > 0 ) ) {
if( cursor.solo_remaining > 0 ) {
auto model { read_runtime_model( input, strings ) };
auto model { read_runtime_model( input, strings, &cursor ) };
model.node.transform = {};
models.push_back( std::move( model ) );
--cursor.solo_remaining;
@@ -725,12 +973,75 @@ read_pack_models_chunk_impl(
auto const angles { read_vec3( input ) };
auto const scale { read_vec3( input ) };
auto const name { strings.resolve( sn_utils::ld_uint32( input ) ) };
models.push_back( expand_prototype_instance(
auto model { expand_prototype_instance(
module.model_prototypes[ proto_id ],
location,
angles,
scale,
name ) );
name ) };
if( cursor.section_format == kPackSectionFormatV13 ) {
model.pack_cell_id = sn_utils::d_uint8( input );
}
models.push_back( std::move( model ) );
--cursor.inst_remaining;
}
++cursor.models_read;
}
return models;
}
[[nodiscard]] std::vector<Eu7Model>
read_pack_models_chunk_from_bytes(
Eu7Module const &module,
std::uint8_t const *data,
std::size_t const size,
Eu7PackSectionCursor &cursor,
std::size_t const max_count,
StringTable const &strings ) {
if( false == cursor.header_parsed || max_count == 0 || data == nullptr || size == 0 ) {
return {};
}
PackByteReader input { data, size };
std::vector<Eu7Model> models;
auto const remaining {
std::min(
static_cast<std::size_t>( cursor.solo_remaining ) +
static_cast<std::size_t>( cursor.inst_remaining ),
static_cast<std::size_t>( cursor.model_total ) ) };
models.reserve( std::min( max_count, remaining ) );
while(
models.size() < max_count &&
( cursor.solo_remaining > 0 || cursor.inst_remaining > 0 ) ) {
if( cursor.solo_remaining > 0 ) {
auto model { read_runtime_model_bytes( input, strings, &cursor ) };
model.node.transform = {};
models.push_back( std::move( model ) );
--cursor.solo_remaining;
}
else {
const std::uint32_t proto_id { input.u32() };
if( proto_id >= module.model_prototypes.size() ) {
throw std::runtime_error(
"EU7 PACK: proto_id " + std::to_string( proto_id ) + " poza zakresem PROT (" +
std::to_string( module.model_prototypes.size() ) + ")" );
}
auto const location { input.vec3() };
auto const angles { input.vec3() };
auto const scale { input.vec3() };
auto const name { strings.resolve( input.u32() ) };
auto model { expand_prototype_instance(
module.model_prototypes[ proto_id ],
location,
angles,
scale,
name ) };
if( cursor.section_format == kPackSectionFormatV13 ) {
model.pack_cell_id = input.u8();
}
models.push_back( std::move( model ) );
--cursor.inst_remaining;
}
++cursor.models_read;
@@ -1166,6 +1477,7 @@ struct PackSectionReadSession {
int column { -1 };
Eu7PackIndexEntry entry {};
Eu7PackSectionCursor header {};
std::shared_ptr<Eu7PackSectionPathTables const> path_tables;
std::ifstream input;
bool header_ready { false };
};
@@ -1180,6 +1492,55 @@ reset_pack_section_read_session( PackSectionReadSession &session ) {
session = {};
}
[[nodiscard]] std::uint64_t
pack_section_byte_size(
Eu7Module const &module,
Eu7PackIndexEntry const &entry ) {
std::uint64_t next_offset { module.pack_catalog.pack_payload_size };
for( auto const &candidate : module.pack_catalog.entries ) {
if(
candidate.pack_offset > entry.pack_offset &&
candidate.pack_offset < next_offset ) {
next_offset = candidate.pack_offset;
}
}
return next_offset - entry.pack_offset;
}
[[nodiscard]] std::optional<std::uint32_t>
pack_chunk_byte_size(
Eu7PackSectionCursor const &header,
std::uint32_t const chunk_index,
std::uint64_t const section_byte_size ) {
if( header.chunk_byte_offsets.empty() || chunk_index >= header.chunk_count ) {
return std::nullopt;
}
auto const start { header.chunk_byte_offsets[ chunk_index ] };
if( chunk_index + 1 < header.chunk_count ) {
return header.chunk_byte_offsets[ chunk_index + 1 ] - start;
}
if( section_byte_size > start ) {
return static_cast<std::uint32_t>( section_byte_size - start );
}
return std::nullopt;
}
void
ensure_section_path_tables(
Eu7PackSectionCursor const &header,
PackSectionReadSession &session ) {
if(
header.unique_meshes.empty() &&
header.unique_textures.empty() ) {
session.path_tables.reset();
return;
}
auto tables { std::make_shared<Eu7PackSectionPathTables>() };
tables->unique_meshes = header.unique_meshes;
tables->unique_textures = header.unique_textures;
session.path_tables = std::move( tables );
}
void
ensure_pack_file( Eu7Module const &module, PackSectionReadSession &session ) {
if( module.source_path == session.source_path && session.input.is_open() ) {
@@ -1216,7 +1577,9 @@ ensure_section_header(
session.row = row;
session.column = column;
session.entry = entry;
session.path_tables.reset();
parse_pack_section_header( module, session.input, entry, session.header );
ensure_section_path_tables( session.header, session );
session.header_ready = true;
}
@@ -1254,23 +1617,25 @@ read_pack_section_chunk_load_impl(
}
result.chunk_index = chunk_index;
if( chunk_index == 0 ) {
result.unique_meshes = header.unique_meshes;
result.unique_textures = header.unique_textures;
}
Eu7PackSectionCursor chunk_cursor { header };
auto const section_byte_size { pack_section_byte_size( module, *entry ) };
std::optional<std::uint32_t> chunk_byte_size;
if(
( header.section_format == kPackSectionFormatV10 ||
header.section_format == kPackSectionFormatV11 ||
header.section_format == kPackSectionFormatV12 ) &&
header.section_format == kPackSectionFormatV12 ||
header.section_format == kPackSectionFormatV13 ) &&
false == header.chunk_byte_offsets.empty() ) {
auto const section_start {
static_cast<std::streamoff>( module.pack_payload_offset + entry->pack_offset ) };
chunk_byte_size = pack_chunk_byte_size( header, chunk_index, section_byte_size );
session.input.seekg(
section_start +
static_cast<std::streamoff>( header.chunk_byte_offsets[ chunk_index ] ) );
if( header.section_format == kPackSectionFormatV12 ) {
if(
header.section_format == kPackSectionFormatV12 ||
header.section_format == kPackSectionFormatV13 ) {
chunk_cursor.solo_remaining = header.chunk_model_counts[ chunk_index ];
chunk_cursor.inst_remaining = header.chunk_inst_counts[ chunk_index ];
}
@@ -1283,12 +1648,35 @@ read_pack_section_chunk_load_impl(
return result;
}
result.models = read_pack_models_chunk_impl(
module,
session.input,
chunk_cursor,
std::numeric_limits<std::size_t>::max(),
strings );
if( chunk_byte_size.has_value() && *chunk_byte_size > 0 ) {
std::vector<std::uint8_t> chunk_bytes( *chunk_byte_size );
session.input.read(
reinterpret_cast<char *>( chunk_bytes.data() ),
static_cast<std::streamsize>( chunk_bytes.size() ) );
if(
static_cast<std::size_t>( session.input.gcount() ) != chunk_bytes.size() ) {
throw std::runtime_error( "EU7 PACK: niepelny odczyt chunka sekcji" );
}
result.models = read_pack_models_chunk_from_bytes(
module,
chunk_bytes.data(),
chunk_bytes.size(),
chunk_cursor,
std::numeric_limits<std::size_t>::max(),
strings );
}
else {
result.models = read_pack_models_chunk_impl(
module,
session.input,
chunk_cursor,
std::numeric_limits<std::size_t>::max(),
strings );
}
result.path_tables = session.path_tables;
if( chunk_index == 0 && false == header.unique_textures.empty() ) {
result.unique_textures = header.unique_textures;
}
return result;
}
@@ -1320,7 +1708,8 @@ read_pack_section_load_impl( Eu7Module const &module, int const row, int const c
if(
( header.section_format == kPackSectionFormatV10 ||
header.section_format == kPackSectionFormatV11 ||
header.section_format == kPackSectionFormatV12 ) &&
header.section_format == kPackSectionFormatV12 ||
header.section_format == kPackSectionFormatV13 ) &&
header.chunk_count > 0 ) {
result.models.reserve( entry->model_count );
for( std::uint32_t chunk_idx { 0 }; chunk_idx < header.chunk_count; ++chunk_idx ) {
@@ -1361,7 +1750,7 @@ is_valid_eu7b_file( std::string const &path ) {
const std::uint32_t version { sn_utils::ld_uint32( input ) };
return (
version == kEu7VersionV4 || version == kEu7VersionV5 || version == kEu7VersionV6 ||
version == kEu7VersionV7 || version == kEu7VersionV8 );
version == kEu7VersionV7 || version == kEu7VersionV8 || version == kEu7VersionV9 );
}
Eu7Module
@@ -1380,7 +1769,7 @@ read_module( std::string const &path ) {
if(
module.version != kEu7VersionV4 && module.version != kEu7VersionV5 &&
module.version != kEu7VersionV6 && module.version != kEu7VersionV7 &&
module.version != kEu7VersionV8 ) {
module.version != kEu7VersionV8 && module.version != kEu7VersionV9 ) {
throw std::runtime_error(
"EU7B: nieobslugiwana wersja " + std::to_string( module.version ) + " w \"" + path + "\"" );
}
@@ -1444,4 +1833,58 @@ reset_pack_section_read_cache() {
reset_pack_section_read_session( g_pack_section_read_session );
}
void
for_each_pack_section_unique_mesh(
Eu7Module const &module,
int const row,
int const column,
std::function<void( std::string const & )> const &visit ) {
if( false == module.has_pack_chunk || module.source_path.empty() ) {
return;
}
auto const entry { find_pack_entry_impl( module, row, column ) };
if( false == entry.has_value() || entry->model_count == 0 ) {
return;
}
auto &session { g_pack_section_read_session };
ensure_pack_file( module, session );
if( !session.input ) {
return;
}
ensure_section_header( module, row, column, *entry, session );
for( auto const &path : session.header.unique_meshes ) {
visit( path );
}
}
void
for_each_pack_section_unique_texture(
Eu7Module const &module,
int const row,
int const column,
std::function<void( std::string const & )> const &visit ) {
if( false == module.has_pack_chunk || module.source_path.empty() ) {
return;
}
auto const entry { find_pack_entry_impl( module, row, column ) };
if( false == entry.has_value() || entry->model_count == 0 ) {
return;
}
auto &session { g_pack_section_read_session };
ensure_pack_file( module, session );
if( !session.input ) {
return;
}
ensure_section_header( module, row, column, *entry, session );
for( auto const &path : session.header.unique_textures ) {
visit( path );
}
}
} // namespace scene::eu7

18
scene/eu7/eu7_reader.h
View File

@@ -11,7 +11,7 @@ http://mozilla.org/MPL/2.0/.
#include "scene/eu7/eu7_types.h"
#include <optional>
#include <functional>
#include <string>
#include <vector>
@@ -47,4 +47,20 @@ read_pack_section_chunk_load(
void
reset_pack_section_read_cache();
// UMES z naglowka sekcji — bez kopiowania calego wektora.
void
for_each_pack_section_unique_mesh(
Eu7Module const &Module,
int Row,
int Column,
std::function<void( std::string const & )> const &Visit );
// UTEX z naglowka sekcji (v11+) — bez skanowania instancji.
void
for_each_pack_section_unique_texture(
Eu7Module const &Module,
int Row,
int Column,
std::function<void( std::string const & )> const &Visit );
} // namespace scene::eu7

1759
scene/eu7/eu7_section_stream.cpp
View File

File diff suppressed because it is too large Load Diff

19
scene/eu7/eu7_section_stream.h
View File

@@ -19,8 +19,15 @@ class state_serializer;
namespace scene::eu7 {
constexpr int kSectionStreamBootstrapRadiusKm { 4 };
constexpr int kSectionStreamGameplayRadiusKm { 4 };
// Fazy streamingu PACK (eu7_section_stream.cpp):
// 1) loading_screen — minimalny bootstrap (~2–3 km) podczas ekranu ladowania;
// 2) radius_fill — po dismiss: lekki, ciagly drain az do kSectionStreamTargetRadiusKm;
// 3) maintenance — po zapelnieniu 20 km: wolny lookahead w kierunku lotu.
constexpr int kSectionStreamLoadingDismissRadiusKm { 1 };
constexpr int kSectionStreamPresentableRadiusKm { 2 };
constexpr int kSectionStreamBootstrapRadiusKm { 3 };
constexpr int kSectionStreamGameplayRadiusKm { 5 };
constexpr int kSectionStreamTargetRadiusKm { 20 };
void
init_section_stream(
@@ -65,15 +72,15 @@ kick_section_stream_bootstrap();
void
try_bootstrap_section_stream();
// Po zaladowaniu scenariusza: bootstrap + drain PACK przed wejsciem w jazde.
// Po zaladowaniu scenariusza: kick bootstrap + opcjonalny krotki drain (0 = tylko async).
void
preload_section_stream( double MaxDrainMs = 300.0 );
preload_section_stream( double MaxDrainMs = 0.0 );
// Wszystkie sekcje PACK w pierścieniu RadiusKm wokol pozycji wczytane i zaaplikowane.
[[nodiscard]] bool
section_stream_ready_around(
glm::dvec3 const &WorldPosition,
int RadiusKm = kSectionStreamBootstrapRadiusKm );
int RadiusKm = kSectionStreamTargetRadiusKm );
// ready_around stabilnie przez kilka klatek — bez migania overlay/sceny.
[[nodiscard]] bool
@@ -86,7 +93,7 @@ section_stream_presentable_around(
[[nodiscard]] bool
loading_screen_blocks_world(
glm::dvec3 const &WorldPosition,
int RadiusKm = kSectionStreamBootstrapRadiusKm );
int RadiusKm = kSectionStreamLoadingDismissRadiusKm );
[[nodiscard]] bool
loading_screen_dismissed();

126
scene/eu7/eu7_types.h
View File

@@ -11,8 +11,10 @@ http://mozilla.org/MPL/2.0/.
#include <cstdint>
#include <limits>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <unordered_map>
#include <utility>
#include <vector>
@@ -190,6 +192,8 @@ struct Eu7Lines {
std::vector<Eu7WorldVertex> vertices;
};
constexpr std::uint16_t kEu7PackIndexEmpty { 0xFFFFu };
struct Eu7Model {
Eu7BasicNode node;
glm::dvec3 location{};
@@ -197,18 +201,52 @@ struct Eu7Model {
glm::dvec3 scale{ 1.0, 1.0, 1.0 };
std::string model_file;
std::string texture_file;
std::string resolved_texture;
std::vector<float> light_states;
std::vector<std::uint32_t> light_colors;
bool transition = true;
bool is_terrain = false;
std::uint8_t pack_flags { 0 };
std::uint32_t textures_alpha { 0 };
float baked_range_min { -1.f };
float baked_range_max { -1.f };
std::uint8_t pack_cell_id { 255 };
std::uint16_t pack_mesh_index { kEu7PackIndexEmpty };
std::uint16_t pack_texture_index { kEu7PackIndexEmpty };
};
constexpr std::uint8_t kEu7PackFlagNeedsFullLoad { 1u << 0 };
constexpr std::uint8_t kEu7PackFlagInstanceableHint { 1u << 1 };
constexpr std::uint8_t kEu7PackCellIdInvalid { 255u };
[[nodiscard]] inline bool
eu7_pack_model_needs_full_load( Eu7Model const &model ) {
if( model.pack_flags & kEu7PackFlagNeedsFullLoad ) {
return true;
}
if( model.pack_flags != 0 ) {
return false;
}
return false == model.light_states.empty()
|| false == model.light_colors.empty()
|| false == model.transition;
}
[[nodiscard]] inline std::string
pack_nodedata_cache_key( Eu7Model const &model ) {
return model.model_file + '\x1f' + model.texture_file + '\x1f'
+ std::to_string( model.node.range_squared_min ) + '\x1f'
+ std::to_string( model.node.range_squared_max ) + '\x1f'
+ ( model.is_terrain ? '1' : '0' );
std::string key;
key.reserve(
model.model_file.size() + model.texture_file.size() + 48 );
key.append( model.model_file );
key.push_back( '\x1f' );
key.append( model.texture_file );
key.push_back( '\x1f' );
key.append( std::to_string( model.node.range_squared_min ) );
key.push_back( '\x1f' );
key.append( std::to_string( model.node.range_squared_max ) );
key.push_back( '\x1f' );
key.push_back( model.is_terrain ? '1' : '0' );
return key;
}
// Wspolna definicja modelu (EU7B v8 PROT) — bez transformacji instancji.
@@ -216,10 +254,15 @@ struct Eu7ModelPrototype {
Eu7BasicNode node;
std::string model_file;
std::string texture_file;
std::string resolved_texture;
std::vector<float> light_states;
std::vector<std::uint32_t> light_colors;
bool transition = true;
bool is_terrain = false;
std::uint8_t pack_flags { 0 };
std::uint32_t textures_alpha { 0 };
float baked_range_min { -1.f };
float baked_range_max { -1.f };
};
struct Eu7MemCell {
@@ -390,14 +433,85 @@ struct Eu7PackSectionLoad {
std::vector<std::string> unique_textures;
};
struct Eu7PackSectionChunkLoad {
std::vector<Eu7Model> models;
struct Eu7PackSectionPathTables {
std::vector<std::string> unique_meshes;
std::vector<std::string> unique_textures;
};
struct Eu7PackSectionChunkLoad {
std::vector<Eu7Model> models;
std::vector<std::string> unique_textures;
std::shared_ptr<Eu7PackSectionPathTables const> path_tables;
std::uint32_t chunk_count { 1 };
std::uint32_t chunk_index { 0 };
};
[[nodiscard]] inline std::string_view
pack_model_mesh_path(
Eu7Model const &model,
Eu7PackSectionPathTables const *path_tables ) {
if( false == model.model_file.empty() ) {
return model.model_file;
}
if(
path_tables != nullptr &&
model.pack_mesh_index != kEu7PackIndexEmpty &&
model.pack_mesh_index < path_tables->unique_meshes.size() ) {
return path_tables->unique_meshes[ model.pack_mesh_index ];
}
return {};
}
[[nodiscard]] inline std::string_view
pack_model_texture_path(
Eu7Model const &model,
Eu7PackSectionPathTables const *path_tables ) {
if( false == model.texture_file.empty() ) {
return model.texture_file;
}
if(
path_tables != nullptr &&
model.pack_texture_index != kEu7PackIndexEmpty &&
model.pack_texture_index < path_tables->unique_textures.size() ) {
return path_tables->unique_textures[ model.pack_texture_index ];
}
return {};
}
inline void
resolve_pack_model_paths(
Eu7Model &model,
Eu7PackSectionPathTables const *path_tables ) {
if( path_tables == nullptr ) {
return;
}
if( model.model_file.empty() && model.pack_mesh_index != kEu7PackIndexEmpty ) {
if( model.pack_mesh_index < path_tables->unique_meshes.size() ) {
model.model_file = path_tables->unique_meshes[ model.pack_mesh_index ];
}
model.pack_mesh_index = kEu7PackIndexEmpty;
}
if( model.texture_file.empty() && model.pack_texture_index != kEu7PackIndexEmpty ) {
if( model.pack_texture_index < path_tables->unique_textures.size() ) {
model.texture_file = path_tables->unique_textures[ model.pack_texture_index ];
}
model.pack_texture_index = kEu7PackIndexEmpty;
}
}
inline void
resolve_pack_model_paths(
Eu7Model *models,
std::size_t const count,
Eu7PackSectionPathTables const *path_tables ) {
if( models == nullptr || path_tables == nullptr ) {
return;
}
for( std::size_t i { 0 }; i < count; ++i ) {
resolve_pack_model_paths( models[ i ], path_tables );
}
}
struct Eu7PackCatalog {
std::vector<Eu7PackIndexEntry> entries;
std::unordered_map<std::uint32_t, std::size_t> index_by_section;

35
scene/scene.cpp
View File

@@ -10,6 +10,7 @@ http://mozilla.org/MPL/2.0/.
#include "stdafx.h"
#include "scene/scene.h"
#include "scene/eu7/eu7_types.h"
#include "simulation/simulation.h"
#include "utilities/Globals.h"
#include "vehicle/Camera.h"
@@ -353,6 +354,21 @@ basic_cell::insert( TTraction *Traction ) {
enclose_area( Traction );
}
// adds provided model instance to the section, optionally using baked PACK cell id
void
basic_section::insert( TAnimModel *Instance ) {
auto &targetcell {
( Instance->m_eu7_pack && Instance->m_pack_cell_id < scene::eu7::kEu7PackCellIdInvalid ) ?
m_cells[ Instance->m_pack_cell_id ] :
cell( Instance->location() ) };
targetcell.insert( Instance );
m_area.radius = std::max(
m_area.radius,
static_cast<float>(
glm::length( m_area.center - targetcell.area().center ) + targetcell.area().radius ) );
}
// adds provided model instance to the cell
void
basic_cell::insert( TAnimModel *Instance ) {
@@ -1317,6 +1333,16 @@ basic_region::RadioStop( glm::dvec3 const &Location ) {
}
}
void
basic_region::insert( TAnimModel *Instance ) {
auto const location { Instance->location() };
if( false == point_inside( location ) ) {
return;
}
section( location ).insert( Instance );
}
std::vector<std::string> switchtrackbedtextures {
"rkpd34r190-tpd1",
"rkpd34r190-tpd2",
@@ -1795,6 +1821,15 @@ void basic_region::create_map_geometry()
if (s)
s->create_map_geometry(m_map_geometrybank);
}
m_map_geometry_built = true;
}
void basic_region::ensure_map_geometry()
{
if( m_map_geometry_built ) {
return;
}
create_map_geometry();
}
void basic_region::update_poi_geometry()

9
scene/scene.h
View File

@@ -286,7 +286,9 @@ public:
// adds provided lines to the section
void
insert( lines_node Lines );
// adds provided node to the section
// adds provided model instance to the section
void
insert( TAnimModel *Instance );
template <class Type_>
void
insert( Type_ *Node ) {
@@ -420,6 +422,8 @@ public:
// TBD, TODO: clamp coordinates to region boundaries?
return; }
section( location ).insert( Node ); }
void
insert( TAnimModel *Instance );
// inserts provided node in the region and registers its ends in lookup directory
template <class Type_>
void
@@ -452,6 +456,8 @@ public:
sections( glm::dvec3 const &Point, float const Radius );
void
create_map_geometry();
void
ensure_map_geometry();
void
update_poi_geometry();
basic_section* get_section(size_t section)
@@ -472,6 +478,7 @@ public:
gfx::geometrybank_handle m_map_geometrybank;
gfx::geometrybank_handle m_map_poipoints;
bool m_map_geometry_built { false };
// methods
// checks whether specified point is within boundaries of the region

10
scene/scenenodegroups.cpp
View File

@@ -227,6 +227,16 @@ node_groups::update_map()
}
}
void
node_groups::ensure_map_index()
{
if( m_map_index_built ) {
return;
}
update_map();
m_map_index_built = true;
}
// returns current active group, or null_handle if group stack is empty
group_handle
node_groups::handle() const {

3
scene/scenenodegroups.h
View File

@@ -37,6 +37,8 @@ public:
// update minimap objects
void
update_map();
void
ensure_map_index();
// returns current active group, or null_handle if group stack is empty
group_handle
handle() const;
@@ -69,6 +71,7 @@ private:
// members
group_map m_groupmap; // map of established node groups
std::stack<scene::group_handle> m_activegroup; // helper, group to be assigned to newly created nodes
bool m_map_index_built { false };
};
extern node_groups Groups;

64
simulation/simulationstateserializer.cpp
View File

@@ -32,8 +32,9 @@ http://mozilla.org/MPL/2.0/.
#include "utilities/Logs.h"
#include "scene/eu7/eu7_bake.h"
#include "scene/eu7/eu7_loader.h"
#include "scene/eu7/eu7_types.h"
#include "scene/eu7/eu7_load_stats.h"
#include "scene/eu7/eu7_model_prefetch.h"
#include "scene/eu7/eu7_pack_mesh_loader.h"
#include "scene/eu7/eu7_pack_bench.h"
#include "scene/eu7/eu7_section.h"
#include "scene/eu7/eu7_transform.h"
@@ -48,7 +49,7 @@ namespace simulation {
namespace {
constexpr double kDeferTrainsetHorizDistM { 4000.0 };
constexpr double kDeferTrainsetHorizDistM { 800.0 };
constexpr double kDeferTrainsetHorizDistSq {
kDeferTrainsetHorizDistM * kDeferTrainsetHorizDistM };
constexpr double kTrainsetDrainMaxDistM { 12000.0 };
@@ -280,13 +281,6 @@ preload_unique_pack_meshes(
}
}
[[nodiscard]] bool
pack_model_needs_full_load( scene::eu7::Eu7Model const &model ) {
return false == model.light_states.empty()
|| false == model.light_colors.empty()
|| false == model.transition;
}
[[nodiscard]] scene::node_data const &
pack_nodedata_cached(
scene::eu7::Eu7Model const &model,
@@ -303,6 +297,12 @@ pack_nodedata_cached(
nodedata.name = model.node.name;
nodedata.type = "model";
}
else if( model.pack_flags != 0 ) {
nodedata.range_min = static_cast<double>( model.baked_range_min );
nodedata.range_max = static_cast<double>( model.baked_range_max );
nodedata.name = model.node.name;
nodedata.type = model.node.node_type;
}
else {
nodedata = node_data_from_eu7( model.node );
}
@@ -517,7 +517,7 @@ state_serializer::insert_eu7_pack_models(
continue;
}
bool const needs_full_load { pack_model_needs_full_load( model ) };
bool const needs_full_load { scene::eu7::eu7_pack_model_needs_full_load( model ) };
bool loaded { false };
if( needs_full_load ) {
scene::eu7::PackBenchTimer const load_timer {
@@ -536,24 +536,29 @@ state_serializer::insert_eu7_pack_models(
else {
scene::eu7::PackBenchTimer const load_timer {
&scene::eu7::Eu7PackBench::main_load_eu7_pack_ms };
auto model_file { model.model_file };
auto texture_file { model.texture_file };
replace_slashes( model_file );
replace_slashes( texture_file );
TModel3d *mesh { nullptr };
if( false == model_file.empty() && model_file != "notload" ) {
auto const found { mesh_cache.find( model_file ) };
if( found != mesh_cache.end() ) {
mesh = found->second;
}
else if( scene::eu7::ensure_pack_mesh_in_session_cache( model_file, mesh_cache ) ) {
auto const loaded { mesh_cache.find( model_file ) };
mesh = loaded != mesh_cache.end() ? loaded->second : nullptr;
}
if( false == model.model_file.empty() && model.model_file != "notload" ) {
mesh = scene::eu7::ensure_pack_mesh_in_session_cache(
model.model_file, mesh_cache );
}
loaded = instance->LoadEu7PackWarm( mesh, texture_file );
if( loaded ) {
scene::eu7::pack_bench_inc( &scene::eu7::Eu7PackBench::main_pack_fast_loads );
if(
mesh == nullptr &&
false == model.model_file.empty() &&
model.model_file != "notload" ) {
loaded = false;
}
else {
loaded = instance->LoadEu7PackWarm(
mesh,
model.texture_file,
model.model_file,
model.resolved_texture,
model.textures_alpha,
( model.pack_flags & scene::eu7::kEu7PackFlagInstanceableHint ) != 0 );
if( loaded ) {
scene::eu7::pack_bench_inc(
&scene::eu7::Eu7PackBench::main_pack_fast_loads );
}
}
}
if( false == loaded ) {
@@ -561,6 +566,10 @@ state_serializer::insert_eu7_pack_models(
continue;
}
if( model.pack_cell_id < scene::eu7::kEu7PackCellIdInvalid ) {
instance->m_pack_cell_id = model.pack_cell_id;
}
if( auto *const mesh { instance->Model() } ) {
if( false == mesh->smoke_sources().empty() ) {
for( auto const &smokesource : mesh->smoke_sources() ) {
@@ -736,8 +745,7 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
scene::Groups.close();
scene::Groups.update_map();
Region->create_map_geometry();
WriteLog( "Scenery: map geometry deferred (lazy on first map open)" );
return false;
}

7
utilities/Globals.cpp
View File

@@ -644,6 +644,12 @@ bool global_settings::ConfigParseSimulation(cParser& Parser, const std::string&
return true;
}
if (token == "eu7.pack.mesh.loader.workers")
{
ParseOneClamped(Parser, eu7_pack_mesh_loader_workers, 0, 16);
return true;
}
if (token == "eu7.auto.bake")
{
ParseOne(Parser, eu7_auto_bake, 1, false);
@@ -1607,6 +1613,7 @@ global_settings::export_as_text( std::ostream &Output ) const {
export_as_text( Output, "convertmodels", iConvertModels );
export_as_text( Output, "file.binary.terrain", file_binary_terrain );
export_as_text( Output, "eu7.pack.stream.workers.percent", eu7_pack_stream_workers_percent );
export_as_text( Output, "eu7.pack.mesh.loader.workers", eu7_pack_mesh_loader_workers );
export_as_text( Output, "eu7.auto.bake", eu7_auto_bake );
export_as_text( Output, "eu7.bake.threads", eu7_bake_threads );
export_as_text( Output, "inactivepause", bInactivePause );

3
utilities/Globals.h
View File

@@ -82,7 +82,8 @@ struct global_settings {
int iConvertIndexRange{ 1000 }; // range of duplicate vertex scan
bool file_binary_terrain{ true }; // enable binary terrain (de)serialization
bool file_binary_terrain_state{true};
int eu7_pack_stream_workers_percent{ 75 }; // EU7 PACK async loader thread pool size (% of CPU threads)
int eu7_pack_stream_workers_percent{ 100 }; // EU7 PACK async loader thread pool size (% of CPU threads)
int eu7_pack_mesh_loader_workers{ 0 }; // 0 = main-thread only (background mesh threads unsafe with Global)
bool eu7_auto_bake{ true }; // przy ladowaniu mapy: bake SCM→EU7 gdy brak lub nieaktualny .eu7
int eu7_bake_threads{ 0 }; // watki bake (0 = auto = hardware_concurrency)
// logs

4
widgets/map.cpp
View File

@@ -8,6 +8,7 @@
#include "vehicle/Driver.h"
#include "model/AnimModel.h"
#include "application/application.h"
#include "scene/scenenodegroups.h"
ui::map_panel::map_panel() : ui_panel(STR_C("Map"), false)
{
@@ -99,6 +100,9 @@ float ui::map_panel::get_vehicle_rotation()
void ui::map_panel::render_map_texture(glm::mat4 transform, glm::vec2 surface_size)
{
scene::Groups.ensure_map_index();
simulation::Region->ensure_map_geometry();
m_colored_paths.switches.clear();
m_colored_paths.occupied.clear();
m_colored_paths.future.clear();