eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-18 01:59:19 +02:00
Code Issues 121 Wiki Activity

Add EU7 PACK v11 UTEX, section unload, and LRU streaming caches

Browse Source 
Bake deduplicated section texture lists (UTEX) and read them at runtime so
texture warm/prefetch no longer scans every instance. Evict PACK sections
outside a 16 km keep ring and cap mesh/nodedata/texture warm caches to stop
RAM and apply cost from growing without bound during long fast camera flight.

- Bake v11 (format=4): UMES + UTEX + CHNK; mesh/texture ordering by frequency
- Runtime: incremental UTEX warm, adaptive apply slices, preempt/catchup tuning
- Unload: track instances per section, Region erase, pack pool release
- LRU: mesh/nodedata (2000, ref-counted), warmed textures (2048)
- Bench/diagnostics: unload and cache eviction counters
- Fix pack instance erase in Region; resolve duplicate helper symbols
This commit is contained in:
maj00r
2026-06-14 21:22:50 +02:00
parent 56d72fb25e
commit 2ee5f93353
16 changed files with 1228 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
docs/eu7-format.md
View File

@@ -853,6 +853,30 @@ Powtórzone chunk_count razy:
Runtime: worker czyta `read_pack_section_chunk_load(row, col, chunk_index)` z O(1) seek.
Sekcje v9 (format=2) i v7 flat pozostają kompatybilne (`chunk_count` implicit = 1).
### Format sekcji — v11 UTEX (`kPackSectionFormatV11 = 4`)
Rozszerzenie v10: po UMES deduplikowana lista ścieżek tekstur (`texture_file`), posortowana po częstotliwości.
```
uint8 section_format — musi być = 4
uint32 solo_count
uint32 inst_count
uint32 unique_mesh_count
uint32[unique_mesh_count] string_id
uint32 unique_texture_count
uint32[unique_texture_count] string_id — texture_file (pomija none/make:/@/#/.e3d)
uint32 chunk_count
Powtórzone chunk_count razy:
uint32 chunk_model_count
uint32 chunk_byte_offset
[payload chunków]
```
Runtime używa UTEX do prefetch/warm tekstur bez skanowania wszystkich instancji.
---
## 30. PROT — prototypy modeli (v8+)

238
eu07-parser/include/eu07/scene/binary/runtime_module.hpp
View File

@@ -25,6 +25,7 @@
#include <stdexcept>
#include <string_view>
#include <thread>
#include <unordered_map>
#include <vector>
namespace eu07::scene::binary {
@@ -49,10 +50,13 @@ inline constexpr std::array<char, 4> kChunkPlac{'P', 'L', 'A', 'C'};
inline constexpr std::array<char, 4> kChunkPidx{'P', 'I', 'D', 'X'};
inline constexpr std::array<char, 4> kChunkPack{'P', 'A', 'C', 'K'};
// PACK section payload: v9 = UMES, v10 = UMES + CHNK (sub-chunk offset table).
// PACK section payload: v9 = UMES, v10 = UMES + CHNK, v11 = UMES + UTEX + CHNK.
inline constexpr std::uint8_t kPackSectionFormatV9 = 2;
inline constexpr std::uint8_t kPackSectionFormatV10 = 3;
inline constexpr std::uint8_t kPackSectionFormatV11 = 4;
inline constexpr std::size_t kPackSectionChunkModels = 512;
inline constexpr std::size_t kPackSectionChunkModelsHeavy = 256;
inline constexpr std::size_t kPackSectionHeavyModelThreshold = 1024;
struct PackWriteStats {
std::size_t strings_total = 0;
@@ -408,28 +412,161 @@ struct PackPayloadBuild {
return !path.empty() && path != "notload";
}
[[nodiscard]] inline bool isPackTexturePath(const std::string_view path) {
if (path.empty() || path == "none" || path.front() == '*') {
return false;
}
if (path.starts_with("make:") || path.starts_with("@") || path.starts_with("none|")) {
return false;
}
if (path.ends_with(".e3d") || path.ends_with(".t3d")) {
return false;
}
if (path == "none" || path.ends_with("/none") || path.ends_with('/')) {
return false;
}
if (path.find("tr/none") != std::string_view::npos || path.find('#') != std::string_view::npos) {
return false;
}
return true;
}
[[nodiscard]] inline runtime::Vec3 packSectionCenter(
const codec::TerrSectionKey section) noexcept {
runtime::Vec3 center{};
center.x =
(static_cast<double>(section.x) -
static_cast<double>(codec::kEu07RegionSideSectionCount) / 2.0 + 0.5) *
static_cast<double>(codec::kEu07SectionSize);
center.z =
(static_cast<double>(section.z) -
static_cast<double>(codec::kEu07RegionSideSectionCount) / 2.0 + 0.5) *
static_cast<double>(codec::kEu07SectionSize);
return center;
}
inline void sortPackSectionModelsForStreaming(
std::vector<runtime::RuntimeModelInstance>& models,
const codec::TerrSectionKey section) {
if (models.size() < 2) {
return;
}
std::unordered_map<std::string, std::uint32_t> mesh_frequency;
mesh_frequency.reserve(models.size());
for (const runtime::RuntimeModelInstance& model : models) {
if (isLoadableMeshPath(model.modelFile)) {
++mesh_frequency[model.modelFile];
}
}
const runtime::Vec3 section_center = packSectionCenter(section);
const auto mesh_rank = [&](const std::string& path) -> std::uint32_t {
const auto it = mesh_frequency.find(path);
return it == mesh_frequency.end() ? 0u : it->second;
};
const auto distance_squared = [&](const runtime::RuntimeModelInstance& model) -> double {
const double dx = model.location.x - section_center.x;
const double dz = model.location.z - section_center.z;
return dx * dx + dz * dz;
};
std::stable_sort(
models.begin(),
models.end(),
[&](const runtime::RuntimeModelInstance& lhs, const runtime::RuntimeModelInstance& rhs) {
const std::uint32_t lhs_rank = mesh_rank(lhs.modelFile);
const std::uint32_t rhs_rank = mesh_rank(rhs.modelFile);
if (lhs_rank != rhs_rank) {
return lhs_rank > rhs_rank;
}
return distance_squared(lhs) < distance_squared(rhs);
});
}
[[nodiscard]] inline std::vector<std::uint32_t> collectUniqueMeshIds(
StringTable& table,
const std::vector<runtime::RuntimeModelInstance>& models,
const std::vector<std::string_view>* prototype_mesh_paths = nullptr) {
std::vector<std::string> paths;
paths.reserve(models.size());
std::unordered_map<std::string, std::uint32_t> mesh_frequency;
mesh_frequency.reserve(models.size());
for (const runtime::RuntimeModelInstance& model : models) {
if (isLoadableMeshPath(model.modelFile)) {
paths.emplace_back(model.modelFile);
++mesh_frequency[model.modelFile];
}
}
std::vector<std::string> paths;
paths.reserve(mesh_frequency.size());
for (const auto& [path, frequency] : mesh_frequency) {
if (frequency > 0) {
paths.emplace_back(path);
}
}
if (prototype_mesh_paths != nullptr) {
for (const std::string_view path : *prototype_mesh_paths) {
if (isLoadableMeshPath(path)) {
paths.emplace_back(path);
const std::string text(path);
if (false == mesh_frequency.contains(text)) {
paths.emplace_back(text);
mesh_frequency.emplace(text, 0u);
}
}
}
}
std::stable_sort(paths.begin(), paths.end());
paths.erase(std::unique(paths.begin(), paths.end()), paths.end());
std::stable_sort(
paths.begin(),
paths.end(),
[&](const std::string& lhs, const std::string& rhs) {
const std::uint32_t lhs_rank = mesh_frequency[lhs];
const std::uint32_t rhs_rank = mesh_frequency[rhs];
if (lhs_rank != rhs_rank) {
return lhs_rank > rhs_rank;
}
return lhs < rhs;
});
std::vector<std::uint32_t> ids;
ids.reserve(paths.size());
for (const std::string& path : paths) {
ids.push_back(table.intern(path));
}
return ids;
}
[[nodiscard]] inline std::vector<std::uint32_t> collectUniqueTextureIds(
StringTable& table,
const std::vector<runtime::RuntimeModelInstance>& models) {
std::unordered_map<std::string, std::uint32_t> texture_frequency;
texture_frequency.reserve(models.size());
for (const runtime::RuntimeModelInstance& model : models) {
if (isPackTexturePath(model.textureFile)) {
++texture_frequency[model.textureFile];
}
}
std::vector<std::string> paths;
paths.reserve(texture_frequency.size());
for (const auto& [path, frequency] : texture_frequency) {
if (frequency > 0) {
paths.emplace_back(path);
}
}
std::stable_sort(
paths.begin(),
paths.end(),
[&](const std::string& lhs, const std::string& rhs) {
const std::uint32_t lhs_rank = texture_frequency[lhs];
const std::uint32_t rhs_rank = texture_frequency[rhs];
if (lhs_rank != rhs_rank) {
return lhs_rank > rhs_rank;
}
return lhs < rhs;
});
std::vector<std::uint32_t> ids;
ids.reserve(paths.size());
@@ -461,12 +598,14 @@ struct PackPayloadBuild {
const std::vector<std::string_view>* prototype_mesh_paths = nullptr) {
const auto mesh_ids = collectUniqueMeshIds(table, models, prototype_mesh_paths);
const std::uint32_t solo_count = static_cast<std::uint32_t>(models.size());
const std::size_t chunk_models =
models.size() > kPackSectionHeavyModelThreshold ? kPackSectionChunkModelsHeavy
: kPackSectionChunkModels;
std::vector<std::vector<char>> chunk_payloads;
chunk_payloads.reserve((models.size() + kPackSectionChunkModels - 1) / kPackSectionChunkModels);
for (std::size_t offset = 0; offset < models.size(); offset += kPackSectionChunkModels) {
const std::size_t end =
std::min(offset + kPackSectionChunkModels, models.size());
chunk_payloads.reserve((models.size() + chunk_models - 1) / chunk_models);
for (std::size_t offset = 0; offset < models.size(); offset += chunk_models) {
const std::size_t end = std::min(offset + chunk_models, models.size());
chunk_payloads.push_back(buildPackModelBlob(table, models, offset, end));
}
if (chunk_payloads.empty()) {
@@ -483,10 +622,8 @@ struct PackPayloadBuild {
std::uint32_t payload_offset = header_size;
for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) {
chunk_offsets[chunk_index] = payload_offset;
const std::size_t model_begin =
static_cast<std::size_t>(chunk_index) * kPackSectionChunkModels;
const std::size_t model_end =
std::min(model_begin + kPackSectionChunkModels, models.size());
const std::size_t model_begin = static_cast<std::size_t>(chunk_index) * chunk_models;
const std::size_t model_end = std::min(model_begin + chunk_models, models.size());
chunk_model_counts[chunk_index] =
static_cast<std::uint32_t>(model_end > model_begin ? model_end - model_begin : 0);
payload_offset += static_cast<std::uint32_t>(chunk_payloads[chunk_index].size());
@@ -512,6 +649,69 @@ struct PackPayloadBuild {
return {blob.begin(), blob.end()};
}
[[nodiscard]] inline std::vector<char> buildPackSectionPayloadV11(
StringTable& table,
const std::vector<runtime::RuntimeModelInstance>& models,
const std::uint32_t inst_count = 0,
const std::vector<std::string_view>* prototype_mesh_paths = nullptr) {
const auto mesh_ids = collectUniqueMeshIds(table, models, prototype_mesh_paths);
const auto texture_ids = collectUniqueTextureIds(table, models);
const std::uint32_t solo_count = static_cast<std::uint32_t>(models.size());
const std::size_t chunk_models =
models.size() > kPackSectionHeavyModelThreshold ? kPackSectionChunkModelsHeavy
: kPackSectionChunkModels;
std::vector<std::vector<char>> chunk_payloads;
chunk_payloads.reserve((models.size() + chunk_models - 1) / chunk_models);
for (std::size_t offset = 0; offset < models.size(); offset += chunk_models) {
const std::size_t end = std::min(offset + chunk_models, models.size());
chunk_payloads.push_back(buildPackModelBlob(table, models, offset, end));
}
if (chunk_payloads.empty()) {
chunk_payloads.emplace_back();
}
const std::uint32_t chunk_count = static_cast<std::uint32_t>(chunk_payloads.size());
const std::uint32_t header_size =
1u + 4u + 4u + 4u + static_cast<std::uint32_t>(mesh_ids.size()) * 4u + 4u +
static_cast<std::uint32_t>(texture_ids.size()) * 4u + 4u + chunk_count * 8u;
std::vector<std::uint32_t> chunk_offsets(chunk_count);
std::vector<std::uint32_t> chunk_model_counts(chunk_count);
std::uint32_t payload_offset = header_size;
for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) {
chunk_offsets[chunk_index] = payload_offset;
const std::size_t model_begin = static_cast<std::size_t>(chunk_index) * chunk_models;
const std::size_t model_end = std::min(model_begin + chunk_models, models.size());
chunk_model_counts[chunk_index] =
static_cast<std::uint32_t>(model_end > model_begin ? model_end - model_begin : 0);
payload_offset += static_cast<std::uint32_t>(chunk_payloads[chunk_index].size());
}
std::ostringstream packOut(std::ios::binary);
io::writeU8(packOut, kPackSectionFormatV11);
io::writeU32(packOut, solo_count);
io::writeU32(packOut, inst_count);
io::writeU32(packOut, static_cast<std::uint32_t>(mesh_ids.size()));
for (const std::uint32_t id : mesh_ids) {
io::writeU32(packOut, id);
}
io::writeU32(packOut, static_cast<std::uint32_t>(texture_ids.size()));
for (const std::uint32_t id : texture_ids) {
io::writeU32(packOut, id);
}
io::writeU32(packOut, chunk_count);
for (std::uint32_t chunk_index = 0; chunk_index < chunk_count; ++chunk_index) {
io::writeU32(packOut, chunk_model_counts[chunk_index]);
io::writeU32(packOut, chunk_offsets[chunk_index]);
}
for (const std::vector<char>& payload : chunk_payloads) {
packOut.write(payload.data(), static_cast<std::streamsize>(payload.size()));
}
const std::string blob = packOut.str();
return {blob.begin(), blob.end()};
}
[[nodiscard]] inline PackPayloadBuild buildPackPayloadV7(
StringTable& table,
const std::vector<codec::ModelSectionBatch>& batches) {
@@ -540,7 +740,9 @@ struct PackPayloadBuild {
entry.row = static_cast<std::uint16_t>(batch.section.z);
entry.column = static_cast<std::uint16_t>(batch.section.x);
entry.modelCount = static_cast<std::uint32_t>(batch.models.size());
section_payloads[batch_index] = buildPackSectionPayloadV10(table, batch.models);
std::vector<runtime::RuntimeModelInstance> section_models = batch.models;
sortPackSectionModelsForStreaming(section_models, batch.section);
section_payloads[batch_index] = buildPackSectionPayloadV11(table, section_models);
}
std::size_t total_size = 0;
for (const std::vector<char>& payload : section_payloads) {
@@ -574,7 +776,9 @@ struct PackPayloadBuild {
entry.column = static_cast<std::uint16_t>(batch.section.x);
entry.modelCount = static_cast<std::uint32_t>(batch.models.size());
section_payloads[batch_index] = buildPackSectionPayloadV10(table, batch.models);
std::vector<runtime::RuntimeModelInstance> section_models = batch.models;
sortPackSectionModelsForStreaming(section_models, batch.section);
section_payloads[batch_index] = buildPackSectionPayloadV11(table, section_models);
}
});
}

9
model/AnimModel.cpp
View File

@@ -668,13 +668,18 @@ TAnimModel::LoadEu7PackWarm(
if(
false == texture_file.empty() &&
texture_file != "none" &&
texture_file.substr( 0, 1 ) != "*" ) {
texture_file.substr( 0, 1 ) != "*" &&
false == texture_file.starts_with( "make:" ) &&
false == texture_file.starts_with( "@" ) &&
false == texture_file.starts_with( "none|" ) ) {
auto normalized { texture_file };
replace_slashes( normalized );
if(
normalized != "none" &&
false == normalized.ends_with( "/none" ) &&
normalized.find( "tr/none" ) == std::string::npos ) {
false == normalized.ends_with( '/' ) &&
normalized.find( "tr/none" ) == std::string::npos &&
normalized.find( '#' ) == std::string::npos ) {
m_materialdata.assign( normalized );
if( m_materialdata.replacable_skins[ 1 ] == null_handle ) {
scene::eu7::pack_bench_inc( &scene::eu7::Eu7PackBench::main_texture_assign_fail );

17
rendering/particles.cpp
View File

@@ -442,6 +442,23 @@ particle_manager::insert( std::string const &Sourcetemplate, TAnimModel const *N
return true;
}
void
particle_manager::erase_bound_node( TAnimModel const *const node ) {
if( node == nullptr ) {
return;
}
m_sources.erase(
std::remove_if(
std::begin( m_sources ),
std::end( m_sources ),
[node]( smoke_source const &source ) {
return source.m_ownertype == smoke_source::owner_type::node
&& source.m_owner.node == node;
} ),
std::end( m_sources ) );
}
// updates state of all owned emitters
void
particle_manager::update() {

2
rendering/particles.h
View File

@@ -181,6 +181,8 @@ public:
insert( std::string const &Sourcetemplate, TDynamicObject const *Vehicle, glm::dvec3 const Location );
bool
insert( std::string const &Sourcetemplate, TAnimModel const *Node, glm::dvec3 const Location );
void
erase_bound_node( TAnimModel const *Node );
// updates state of all owned emitters
void
update();

1
scene/eu7/eu7_chunks.h
View File

@@ -60,6 +60,7 @@ constexpr std::uint32_t kChunkProt { detail::make_id4( 'P', 'R', 'O', 'T' ) };
constexpr std::uint8_t kPackSectionFormatV8 { 1 };
constexpr std::uint8_t kPackSectionFormatV9 { 2 };
constexpr std::uint8_t kPackSectionFormatV10 { 3 };
constexpr std::uint8_t kPackSectionFormatV11 { 4 };
constexpr std::size_t kPackSectionChunkModels { 512 };
} // namespace scene::eu7

182
scene/eu7/eu7_model_prefetch.cpp
View File

@@ -17,27 +17,89 @@ http://mozilla.org/MPL/2.0/.
#include "utilities/utilities.h"
#include "vehicle/DynObj.h"
#include <list>
#include <mutex>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
namespace scene::eu7 {
namespace {
constexpr std::size_t kMaxWarmedTextures { 2048 };
std::mutex g_pack_mesh_load_mutex;
std::unordered_set<std::string> g_session_warmed_textures;
std::list<std::string> g_session_warmed_textures_lru;
std::unordered_map<std::string, std::list<std::string>::iterator> g_session_warmed_textures_iters;
void
touch_warmed_texture_lru( std::string const &texture_file ) {
auto const found { g_session_warmed_textures_iters.find( texture_file ) };
if( found == g_session_warmed_textures_iters.end() ) {
return;
}
g_session_warmed_textures_lru.splice(
g_session_warmed_textures_lru.end(),
g_session_warmed_textures_lru,
found->second );
}
void
evict_warmed_texture_lru() {
while(
g_session_warmed_textures.size() >= kMaxWarmedTextures &&
false == g_session_warmed_textures_lru.empty() ) {
auto const oldest { g_session_warmed_textures_lru.front() };
g_session_warmed_textures_lru.pop_front();
g_session_warmed_textures_iters.erase( oldest );
g_session_warmed_textures.erase( oldest );
pack_bench_inc( &Eu7PackBench::stream_texture_cache_evictions );
}
}
void
remember_warmed_texture( std::string texture_file ) {
if( g_session_warmed_textures.contains( texture_file ) ) {
touch_warmed_texture_lru( texture_file );
return;
}
evict_warmed_texture_lru();
g_session_warmed_textures.insert( texture_file );
g_session_warmed_textures_lru.push_back( texture_file );
g_session_warmed_textures_iters.emplace(
std::move( texture_file ),
std::prev( g_session_warmed_textures_lru.end() ) );
}
[[nodiscard]] bool
pack_texture_usable( std::string texture_file ) {
if( texture_file.empty() || texture_file == "none" || texture_file.front() == '*' ) {
return false;
}
replace_slashes( texture_file );
if( texture_file == "none" || texture_file.ends_with( "/none" ) ) {
if(
texture_file.starts_with( "make:" ) ||
texture_file.starts_with( "@" ) ||
texture_file.starts_with( "none|" ) ) {
return false;
}
if( texture_file.find( "tr/none" ) != std::string::npos ) {
if(
texture_file.ends_with( ".e3d" ) ||
texture_file.ends_with( ".t3d" ) ) {
return false;
}
replace_slashes( texture_file );
if(
texture_file == "none" ||
texture_file.ends_with( "/none" ) ||
texture_file.ends_with( '/' ) ) {
return false;
}
if(
texture_file.find( "tr/none" ) != std::string::npos ||
texture_file.find( '#' ) != std::string::npos ) {
return false;
}
return true;
@@ -69,30 +131,80 @@ warm_one_pack_texture( std::string texture_file, std::unordered_set<std::string>
auto const resolved { TextureTest( ToLower( texture_file ) ) };
if( resolved.empty() ) {
pack_bench_inc( &Eu7PackBench::main_texture_warm_miss );
log_pack_texture_fail( texture_file );
return false;
bool warmed { false };
for( int skinindex { 1 }; skinindex <= 4; ++skinindex ) {
auto const multi {
TextureTest( ToLower( texture_file + "," + std::to_string( skinindex ) ) ) };
if( multi.empty() ) {
break;
}
GfxRenderer->Fetch_Material( multi );
warmed = true;
}
if( false == warmed ) {
pack_bench_inc( &Eu7PackBench::main_texture_warm_miss );
log_pack_texture_fail( texture_file );
return false;
}
}
else {
GfxRenderer->Fetch_Material( resolved );
}
g_session_warmed_textures.insert( texture_file );
GfxRenderer->Fetch_Material( resolved );
remember_warmed_texture( texture_file );
for( int skinindex { 1 }; skinindex <= 4; ++skinindex ) {
auto const multi {
TextureTest( ToLower( texture_file + "," + std::to_string( skinindex ) ) ) };
if( multi.empty() ) {
break;
if( false == resolved.empty() ) {
for( int skinindex { 1 }; skinindex <= 4; ++skinindex ) {
auto const multi {
TextureTest( ToLower( texture_file + "," + std::to_string( skinindex ) ) ) };
if( multi.empty() ) {
break;
}
GfxRenderer->Fetch_Material( multi );
}
GfxRenderer->Fetch_Material( multi );
}
return true;
}
} // namespace
[[nodiscard]] bool
ensure_pack_mesh_in_session_cache(
std::string model_file,
std::unordered_map<std::string, TModel3d *> &session_cache ) {
if( model_file.empty() || model_file == "notload" ) {
return false;
}
replace_slashes( model_file );
if( session_cache.contains( model_file ) ) {
pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit );
return true;
}
auto const was_global_cached { TModelsManager::IsModelCached( model_file ) };
TModel3d *mesh { nullptr };
{
std::lock_guard<std::mutex> lock { g_pack_mesh_load_mutex };
mesh = TModelsManager::GetModel( model_file, false, false );
}
if( was_global_cached ) {
pack_bench_inc( &Eu7PackBench::stream_mesh_global_hit );
}
else {
pack_bench_inc( &Eu7PackBench::stream_mesh_disk_load );
}
if( mesh != nullptr ) {
TAnimModel::warm_instanceable_cache( mesh );
}
session_cache.emplace( model_file, mesh );
return true;
}
void
reset_pack_texture_warm_cache() {
g_session_warmed_textures.clear();
g_session_warmed_textures_lru.clear();
g_session_warmed_textures_iters.clear();
}
void
@@ -147,6 +259,48 @@ preload_pack_models(
preload_pack_models( models );
}
std::size_t
warm_pack_texture_paths_main(
std::string const *const paths,
std::size_t const count,
double const budget_ms,
std::size_t *const processed_out ) {
if( paths == nullptr || count == 0 || GfxRenderer == nullptr ) {
if( processed_out != nullptr ) {
*processed_out = 0;
}
return 0;
}
std::unordered_set<std::string> seen;
seen.reserve( std::min( count, std::size_t { 64 } ) );
std::size_t warmed { 0 };
std::size_t processed { 0 };
auto const started { std::chrono::steady_clock::now() };
for( std::size_t i { 0 }; i < count; ++i ) {
if(
budget_ms > 0.0 &&
processed > 0 ) {
auto const elapsed_ms {
std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - started ).count() };
if( elapsed_ms >= budget_ms ) {
break;
}
}
if( warm_one_pack_texture( paths[ i ], seen ) ) {
++warmed;
}
++processed;
}
if( processed_out != nullptr ) {
*processed_out = processed;
}
return warmed;
}
std::size_t
warm_pack_textures_main( Eu7Model const *const models, std::size_t const count ) {
if( models == nullptr || count == 0 || GfxRenderer == nullptr ) {

16
scene/eu7/eu7_model_prefetch.h
View File

@@ -13,9 +13,12 @@ http://mozilla.org/MPL/2.0/.
#include <cstddef>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
class TModel3d;
namespace scene::eu7 {
// Tylko warm_instanceable_cache dla juz zcache'owanych meshy (bez cold GetModel na workerze).
@@ -30,10 +33,23 @@ preload_pack_models(
// Main thread: Fetch_Material dla unikalnych texture_file z chunka przed apply.
// Zwraca liczbe unikalnych Fetch_Material w tym slice.
std::size_t
warm_pack_texture_paths_main(
std::string const *paths,
std::size_t count,
double budget_ms = 0.0,
std::size_t *processed_out = nullptr );
std::size_t
warm_pack_textures_main( Eu7Model const *models, std::size_t count );
void
reset_pack_texture_warm_cache();
// Thread-safe cold mesh load: session cache, then GetModel under g_pack_mesh_load_mutex.
[[nodiscard]] bool
ensure_pack_mesh_in_session_cache(
std::string model_file,
std::unordered_map<std::string, TModel3d *> &session_cache );
} // namespace scene::eu7

4
scene/eu7/eu7_pack_bench.cpp
View File

@@ -189,6 +189,8 @@ log_pack_bench_impl( Eu7PackBench const &bench, char const *const title ) {
" stuck_skip=" + std::to_string( bench.stream_apply_stuck_skip ) );
WriteLog(
" diag ready_tex_ms=" + std::to_string( static_cast<int>( bench.stream_prefetch_ready_tex_ms ) ) +
" ready_umes_ms=" + std::to_string( static_cast<int>( bench.stream_prefetch_ready_umes_ms ) ) +
" umes_loads=" + std::to_string( bench.stream_prefetch_ready_umes_loads ) +
" peak_pending=" + std::to_string( bench.peak_pending_total ) );
WriteLog(
" peaks: ready_q=" + std::to_string( bench.peak_ready_queue ) +
@@ -324,6 +326,7 @@ pack_bench_note_chunk(
pack_bench_inc( &Eu7PackBench::main_chunk_cold_loads, cold_meshes );
g_pack_bench.last_chunk_ms = wall_ms;
g_pack_bench.last_apply_ms = pointer_apply_ms;
g_pack_bench.last_chunk_instances = instances;
g_pack_bench.last_chunk_cold = cold_meshes;
if( wall_ms > g_pack_bench.peak_chunk_ms ) {
@@ -344,6 +347,7 @@ pack_bench_note_chunk(
auto &stream { g_pack_bench_stream };
stream.last_chunk_ms = wall_ms;
stream.last_apply_ms = pointer_apply_ms;
stream.last_chunk_instances = instances;
stream.last_chunk_cold = cold_meshes;
if( wall_ms > stream.peak_chunk_ms ) {

8
scene/eu7/eu7_pack_bench.h
View File

@@ -72,7 +72,14 @@ struct Eu7PackBench {
std::uint64_t stream_dequeue_wait_near { 0 };
std::uint64_t stream_jobs_blocked_far { 0 };
std::uint64_t stream_apply_stuck_skip { 0 };
std::uint64_t stream_sections_unloaded { 0 };
std::uint64_t stream_unload_instances { 0 };
std::uint64_t stream_mesh_cache_evictions { 0 };
std::uint64_t stream_nodedata_cache_evictions { 0 };
std::uint64_t stream_texture_cache_evictions { 0 };
double stream_prefetch_ready_tex_ms { 0.0 };
double stream_prefetch_ready_umes_ms { 0.0 };
std::uint64_t stream_prefetch_ready_umes_loads { 0 };
std::uint64_t peak_pending_total { 0 };
std::uint64_t stream_inst_planned { 0 };
std::uint64_t stream_inst_after_cold { 0 };
@@ -92,6 +99,7 @@ struct Eu7PackBench {
std::uint64_t peak_chunk_instances { 0 };
std::uint64_t peak_chunk_cold { 0 };
double last_chunk_ms { 0.0 };
double last_apply_ms { 0.0 };
std::uint64_t last_chunk_instances { 0 };
std::uint64_t last_chunk_cold { 0 };

50
scene/eu7/eu7_reader.cpp
View File

@@ -505,6 +505,48 @@ parse_pack_section_header(
const std::uint8_t first_byte {
peek == EOF ? std::uint8_t { 0 } : static_cast<std::uint8_t>( peek ) };
bool use_v11_header { false };
if( first_byte == kPackSectionFormatV11 ) {
cursor.section_format = sn_utils::d_uint8( input );
const std::uint32_t solo_total { sn_utils::ld_uint32( input ) };
const std::uint32_t inst_total { sn_utils::ld_uint32( input ) };
const std::uint32_t unique_mesh_count { sn_utils::ld_uint32( input ) };
if( solo_total + inst_total == entry.model_count ) {
use_v11_header = true;
cursor.model_total = solo_total + inst_total;
cursor.solo_remaining = 0;
cursor.inst_remaining = 0;
StringTable const strings { module.strings };
cursor.unique_meshes.reserve( unique_mesh_count );
for( std::uint32_t mesh_idx { 0 }; mesh_idx < unique_mesh_count; ++mesh_idx ) {
cursor.unique_meshes.push_back(
strings.resolve( sn_utils::ld_uint32( input ) ) );
}
const std::uint32_t unique_texture_count { sn_utils::ld_uint32( input ) };
cursor.unique_textures.reserve( unique_texture_count );
for( std::uint32_t tex_idx { 0 }; tex_idx < unique_texture_count; ++tex_idx ) {
cursor.unique_textures.push_back(
strings.resolve( sn_utils::ld_uint32( input ) ) );
}
cursor.chunk_count = sn_utils::ld_uint32( input );
cursor.chunk_model_counts.resize( cursor.chunk_count );
cursor.chunk_byte_offsets.resize( cursor.chunk_count );
for( std::uint32_t chunk_idx { 0 }; chunk_idx < cursor.chunk_count; ++chunk_idx ) {
cursor.chunk_model_counts[ chunk_idx ] = sn_utils::ld_uint32( input );
cursor.chunk_byte_offsets[ chunk_idx ] = sn_utils::ld_uint32( input );
}
}
}
if( use_v11_header ) {
cursor.models_read = 0;
cursor.header_parsed = true;
return;
}
bool use_v10_header { false };
if( first_byte == kPackSectionFormatV10 ) {
cursor.section_format = sn_utils::d_uint8( input );
@@ -1168,11 +1210,13 @@ read_pack_section_chunk_load_impl(
result.chunk_index = chunk_index;
if( chunk_index == 0 ) {
result.unique_meshes = header.unique_meshes;
result.unique_textures = header.unique_textures;
}
Eu7PackSectionCursor chunk_cursor { header };
if(
header.section_format == kPackSectionFormatV10 &&
( header.section_format == kPackSectionFormatV10 ||
header.section_format == kPackSectionFormatV11 ) &&
false == header.chunk_byte_offsets.empty() ) {
auto const section_start {
static_cast<std::streamoff>( module.pack_payload_offset + entry->pack_offset ) };
@@ -1218,9 +1262,11 @@ read_pack_section_load_impl( Eu7Module const &module, int const row, int const c
parse_pack_section_header( module, input, *entry, header );
result.unique_meshes = header.unique_meshes;
result.unique_textures = header.unique_textures;
if(
header.section_format == kPackSectionFormatV10 &&
( header.section_format == kPackSectionFormatV10 ||
header.section_format == kPackSectionFormatV11 ) &&
header.chunk_count > 0 ) {
result.models.reserve( entry->model_count );
for( std::uint32_t chunk_idx { 0 }; chunk_idx < header.chunk_count; ++chunk_idx ) {

686
scene/eu7/eu7_section_stream.cpp
View File

@@ -32,6 +32,7 @@ http://mozilla.org/MPL/2.0/.
#include <cmath>
#include <deque>
#include <limits>
#include <list>
#include <mutex>
#include <optional>
#include <stdexcept>
@@ -41,6 +42,15 @@ http://mozilla.org/MPL/2.0/.
#include <vector>
namespace scene::eu7 {
[[nodiscard]] bool
ring_section_in_radius(
int row,
int column,
int center_row,
int center_column,
int radius_km );
namespace {
constexpr double kDrainBudgetMs { 12.0 };
@@ -50,7 +60,7 @@ constexpr std::size_t kLoaderSectionsPerDrain { 8 };
constexpr double kGameplayApplyBudgetMs { 4.0 };
constexpr double kCatchupApplyBudgetMs { 6.0 };
constexpr std::size_t kGameplaySliceInstances { 96 };
constexpr std::size_t kCatchupSliceInstances { 160 };
constexpr std::size_t kCatchupSliceInstances { 64 };
constexpr std::size_t kLoaderSliceInstances { 768 };
constexpr std::size_t kGameplaySliceColdMeshes { 1 };
constexpr std::size_t kCatchupSliceColdMeshes { 2 };
@@ -60,6 +70,9 @@ constexpr double kCatchupColdBudgetMs { 5.0 };
constexpr int kInitialBootstrapRadius { 5 };
constexpr int kStreamRadius { 14 };
constexpr int kSectionUnloadMarginKm { 2 };
constexpr std::size_t kMeshCacheLruCap { 2000 };
constexpr std::size_t kNodedataCacheLruCap { 2000 };
constexpr int kMovementLookahead { 10 };
constexpr std::size_t kMaxPackStreamWorkers { 8 };
constexpr std::size_t kMaxInFlightSections { 12 };
@@ -67,7 +80,9 @@ constexpr std::size_t kMaxReadySections { 6 };
constexpr double kStreamStatusLogIntervalSec { 5.0 };
constexpr double kReenqueueDistanceM { 500.0 };
constexpr double kCatchupReenqueueDistanceM { 40.0 };
constexpr std::size_t kHeavySectionModelThreshold { 800 };
constexpr std::size_t kHeavySectionModelThreshold { 32 };
constexpr std::size_t kPackTextureWarmSlice { 8 };
constexpr double kPackTextureWarmBudgetMs { 4.0 };
constexpr double kUrgentApplyBudgetMs { 32.0 };
constexpr std::size_t kUrgentSliceInstances { 96 };
constexpr std::size_t kUrgentSliceColdMeshes { 2 };
@@ -75,6 +90,10 @@ constexpr double kUrgentColdBudgetMs { 24.0 };
constexpr std::size_t kUrgentMaxChunksPerDrain { 8 };
constexpr double kReadyTexturePrefetchBudgetMs { 14.0 };
constexpr std::size_t kReadyTexturePrefetchSlice { 256 };
constexpr int kUmesPrefetchMaxDistanceKm { 2 };
constexpr double kReadyUmesPrefetchBudgetMs { 8.0 };
constexpr std::size_t kReadyUmesPrefetchMaxMeshes { 6 };
constexpr std::size_t kUmesPrefetchJobsBacklogThreshold { 4 };
constexpr std::size_t kCatchupMaxInFlightSections { 28 };
constexpr std::size_t kCatchupMaxReadySections { 14 };
constexpr int kUrgentSectionDistanceKm { 1 };
@@ -110,11 +129,13 @@ struct PackSectionReady {
std::size_t section_idx { 0 };
std::unique_ptr<std::vector<Eu7Model>> models;
std::vector<std::string> unique_meshes;
std::vector<std::string> unique_textures;
bool failed { false };
bool section_final { true };
std::size_t apply_offset { 0 };
std::size_t texture_warm_offset { 0 };
std::size_t umes_cold_offset { 0 };
std::size_t umes_prefetch_offset { 0 };
};
struct SectionStreamState {
@@ -139,6 +160,14 @@ struct SectionStreamState {
std::unordered_map<std::string, TModel3d *> mesh_cache;
std::unordered_map<std::string, scene::node_data> nodedata_cache;
std::list<std::string> mesh_lru;
std::unordered_map<std::string, std::list<std::string>::iterator> mesh_lru_iters;
std::unordered_map<std::string, std::size_t> mesh_ref_counts;
std::unordered_map<std::size_t, std::unordered_set<std::string>> section_mesh_keys;
std::list<std::string> nodedata_lru;
std::unordered_map<std::string, std::list<std::string>::iterator> nodedata_lru_iters;
std::unordered_map<std::string, std::size_t> nodedata_ref_counts;
std::unordered_map<std::size_t, std::unordered_set<std::string>> section_nodedata_keys;
std::vector<std::jthread> workers;
std::atomic<bool> worker_exit { false };
@@ -171,6 +200,12 @@ enqueue_section_if_needed(
void
sync_stream_limits( glm::dvec3 const &world_position );
void
maybe_unload_distant_sections( int const center_row, int const center_column );
void
evict_unreferenced_stream_caches();
[[nodiscard]] bool
try_dequeue_ready_batch();
@@ -186,6 +221,46 @@ enqueue_failed_section( PackSectionJob const &job );
void
release_pending_buffer();
void
prefetch_ready_queue_umes_worker( double budget_ms, std::size_t max_meshes );
[[nodiscard]] std::size_t
warm_pack_section_textures(
PackSectionReady &batch,
std::size_t const model_offset,
std::size_t const model_count,
std::size_t const max_textures = kPackTextureWarmSlice ) {
if( false == batch.unique_textures.empty() ) {
if( batch.texture_warm_offset >= batch.unique_textures.size() ) {
return 0;
}
auto const remaining { batch.unique_textures.size() - batch.texture_warm_offset };
auto const slice { std::min( remaining, max_textures ) };
std::size_t processed { 0 };
auto const warmed {
warm_pack_texture_paths_main(
batch.unique_textures.data() + batch.texture_warm_offset,
slice,
kPackTextureWarmBudgetMs,
&processed ) };
batch.texture_warm_offset += processed;
return warmed;
}
if( batch.models == nullptr || model_count == 0 ) {
return 0;
}
auto const begin { model_offset };
if( begin >= batch.models->size() ) {
return 0;
}
auto const remaining { std::min( model_count, batch.models->size() - begin ) };
auto const warmed {
warm_pack_textures_main( batch.models->data() + begin, remaining ) };
batch.texture_warm_offset = begin + remaining;
return warmed;
}
[[nodiscard]] bool
section_has_pack_models( int const row, int const column );
@@ -216,6 +291,303 @@ stream_worker_count() {
return std::min( requested, kMaxPackStreamWorkers );
}
void
touch_string_lru(
std::string const &key,
std::list<std::string> &order,
std::unordered_map<std::string, std::list<std::string>::iterator> &iters ) {
auto const found { iters.find( key ) };
if( found == iters.end() ) {
order.push_back( key );
iters.emplace( key, std::prev( order.end() ) );
return;
}
order.splice( order.end(), order, found->second );
}
void
retain_cache_key(
std::size_t const section_idx,
std::string key,
std::unordered_map<std::size_t, std::unordered_set<std::string>> &section_keys,
std::unordered_map<std::string, std::size_t> &ref_counts,
std::list<std::string> &lru_order,
std::unordered_map<std::string, std::list<std::string>::iterator> &lru_iters ) {
if( key.empty() ) {
return;
}
auto &keys { section_keys[ section_idx ] };
if( false == keys.insert( key ).second ) {
touch_string_lru( key, lru_order, lru_iters );
return;
}
++ref_counts[ key ];
touch_string_lru( key, lru_order, lru_iters );
}
void
release_section_key_refs(
std::size_t const section_idx,
std::unordered_map<std::size_t, std::unordered_set<std::string>> &section_keys,
std::unordered_map<std::string, std::size_t> &ref_counts ) {
auto const found { section_keys.find( section_idx ) };
if( found == section_keys.end() ) {
return;
}
for( auto const &key : found->second ) {
auto ref_it { ref_counts.find( key ) };
if( ref_it == ref_counts.end() ) {
continue;
}
if( ref_it->second <= 1 ) {
ref_counts.erase( ref_it );
}
else {
--ref_it->second;
}
}
section_keys.erase( found );
}
void
evict_cache_lru_if_needed(
std::unordered_map<std::string, std::size_t> &ref_counts,
std::unordered_map<std::string, TModel3d *> *const mesh_cache,
std::unordered_map<std::string, scene::node_data> *const nodedata_cache,
std::list<std::string> &lru_order,
std::unordered_map<std::string, std::list<std::string>::iterator> &lru_iters,
std::size_t const cap,
std::uint64_t Eu7PackBench::*const eviction_counter ) {
while( lru_order.size() > cap ) {
bool evicted { false };
for( auto it { lru_order.begin() }; it != lru_order.end(); ++it ) {
auto const &key { *it };
auto const ref_it { ref_counts.find( key ) };
if( ref_it != ref_counts.end() && ref_it->second > 0 ) {
continue;
}
lru_iters.erase( key );
lru_order.erase( it );
if( mesh_cache != nullptr ) {
mesh_cache->erase( key );
}
if( nodedata_cache != nullptr ) {
nodedata_cache->erase( key );
}
pack_bench_inc( eviction_counter );
evicted = true;
break;
}
if( false == evicted ) {
break;
}
}
}
void
retain_section_mesh_keys(
std::size_t const section_idx,
Eu7Model const *const models,
std::size_t const offset,
std::size_t const count ) {
if( models == nullptr || count == 0 ) {
return;
}
for( std::size_t i { 0 }; i < count; ++i ) {
auto model_file { models[ offset + i ].model_file };
if( model_file.empty() || model_file == "notload" ) {
continue;
}
replace_slashes( model_file );
retain_cache_key(
section_idx,
model_file,
g_stream.section_mesh_keys,
g_stream.mesh_ref_counts,
g_stream.mesh_lru,
g_stream.mesh_lru_iters );
}
}
void
retain_section_nodedata_keys(
std::size_t const section_idx,
Eu7Model const *const models,
std::size_t const offset,
std::size_t const count ) {
if( models == nullptr || count == 0 ) {
return;
}
for( std::size_t i { 0 }; i < count; ++i ) {
retain_cache_key(
section_idx,
pack_nodedata_cache_key( models[ offset + i ] ),
g_stream.section_nodedata_keys,
g_stream.nodedata_ref_counts,
g_stream.nodedata_lru,
g_stream.nodedata_lru_iters );
}
}
void
release_section_cache_refs( std::size_t const section_idx ) {
release_section_key_refs(
section_idx,
g_stream.section_mesh_keys,
g_stream.mesh_ref_counts );
release_section_key_refs(
section_idx,
g_stream.section_nodedata_keys,
g_stream.nodedata_ref_counts );
evict_unreferenced_stream_caches();
}
void
evict_unreferenced_stream_caches() {
evict_cache_lru_if_needed(
g_stream.mesh_ref_counts,
&g_stream.mesh_cache,
nullptr,
g_stream.mesh_lru,
g_stream.mesh_lru_iters,
kMeshCacheLruCap,
&Eu7PackBench::stream_mesh_cache_evictions );
evict_cache_lru_if_needed(
g_stream.nodedata_ref_counts,
nullptr,
&g_stream.nodedata_cache,
g_stream.nodedata_lru,
g_stream.nodedata_lru_iters,
kNodedataCacheLruCap,
&Eu7PackBench::stream_nodedata_cache_evictions );
}
[[nodiscard]] bool
ensure_stream_pack_mesh( std::string model_file ) {
if( model_file.empty() || model_file == "notload" ) {
return false;
}
replace_slashes( model_file );
(void)ensure_pack_mesh_in_session_cache( model_file, g_stream.mesh_cache );
touch_string_lru( model_file, g_stream.mesh_lru, g_stream.mesh_lru_iters );
return true;
}
void
cancel_section_stream_work( std::size_t const section_idx ) {
{
std::lock_guard<std::mutex> lock { g_stream.jobs.mutex };
auto &jobs { g_stream.jobs.data };
jobs.erase(
std::remove_if(
jobs.begin(),
jobs.end(),
[&]( PackSectionJob const &job ) {
return job.section_idx == section_idx;
} ),
jobs.end() );
}
{
std::lock_guard<std::mutex> lock { g_stream.ready.mutex };
for( auto it { g_stream.ready.data.begin() }; it != g_stream.ready.data.end(); ) {
if( it->section_idx != section_idx ) {
++it;
continue;
}
if( it->models != nullptr ) {
it->models->clear();
it->models->shrink_to_fit();
it->models.reset();
}
it = g_stream.ready.data.erase( it );
}
}
if(
g_stream.pending_apply.has_value() &&
g_stream.pending_apply->section_idx == section_idx ) {
release_pending_buffer();
}
g_stream.in_flight_sections.erase( section_idx );
g_stream.loaded_sections.erase( section_idx );
}
[[nodiscard]] std::size_t
unload_pack_section( int const row, int const column ) {
auto const section_idx { section_index( row, column ) };
if( false == g_stream.loaded_sections.contains( section_idx ) ) {
return 0;
}
std::size_t removed { 0 };
if( g_stream.serializer != nullptr ) {
removed = g_stream.serializer->unload_eu7_pack_section( row, column );
}
release_section_cache_refs( section_idx );
cancel_section_stream_work( section_idx );
if( removed > 0 ) {
if( load_stats().pack_models >= removed ) {
load_stats().pack_models -= removed;
}
else {
load_stats().pack_models = 0;
}
pack_bench_inc( &Eu7PackBench::stream_unload_instances, removed );
}
pack_bench_inc( &Eu7PackBench::stream_sections_unloaded );
return removed;
}
void
maybe_unload_distant_sections( int const center_row, int const center_column ) {
if( false == g_loading_screen_dismissed || g_stream.bootstrap_active ) {
return;
}
auto const keep_radius { g_stream.radius + kSectionUnloadMarginKm };
std::vector<std::size_t> to_unload;
to_unload.reserve( g_stream.loaded_sections.size() );
for( auto const section_idx : g_stream.loaded_sections ) {
auto const row {
static_cast<int>( section_idx / static_cast<std::size_t>( kRegionSideSectionCount ) ) };
auto const column {
static_cast<int>( section_idx % static_cast<std::size_t>( kRegionSideSectionCount ) ) };
if( ring_section_in_radius( row, column, center_row, center_column, keep_radius ) ) {
continue;
}
to_unload.push_back( section_idx );
}
if( to_unload.empty() ) {
return;
}
static double s_last_unload_log { 0.0 };
auto const now { Timer::GetTime() };
auto const should_log {
now - s_last_unload_log >= kStreamStatusLogIntervalSec };
std::size_t total_removed { 0 };
for( auto const section_idx : to_unload ) {
auto const row {
static_cast<int>( section_idx / static_cast<std::size_t>( kRegionSideSectionCount ) ) };
auto const column {
static_cast<int>( section_idx % static_cast<std::size_t>( kRegionSideSectionCount ) ) };
total_removed += unload_pack_section( row, column );
}
if( should_log && total_removed > 0 ) {
s_last_unload_log = now;
WriteLog(
"EU7 PACK: unload " + std::to_string( to_unload.size() ) + " sekcji, inst=" +
std::to_string( total_removed ) + ", loaded=" +
std::to_string( g_stream.loaded_sections.size() ) + ", mesh_cache=" +
std::to_string( g_stream.mesh_cache.size() ) );
}
}
void
reset_stream_fields();
@@ -242,6 +614,14 @@ reset_stream_fields() {
g_stream.has_anchor_position = false;
g_stream.mesh_cache.clear();
g_stream.nodedata_cache.clear();
g_stream.mesh_lru.clear();
g_stream.mesh_lru_iters.clear();
g_stream.mesh_ref_counts.clear();
g_stream.section_mesh_keys.clear();
g_stream.nodedata_lru.clear();
g_stream.nodedata_lru_iters.clear();
g_stream.nodedata_ref_counts.clear();
g_stream.section_nodedata_keys.clear();
reset_pack_texture_warm_cache();
note_apply_progress();
}
@@ -331,6 +711,33 @@ camera_stream_speed_mps() {
return glm::length( rotated ) * 5.0;
}
[[nodiscard]] bool pending_apply_is_urgent();
[[nodiscard]] double gameplay_apply_budget_ms();
[[nodiscard]] double
ready_texture_prefetch_budget_ms() {
auto const speed { camera_stream_speed_mps() };
if(
speed > 500.0 &&
pending_apply_is_urgent() ) {
return 0.0;
}
double budget { kReadyTexturePrefetchBudgetMs * 0.5 };
if( g_stream_catchup || speed > 80.0 ) {
budget = kReadyTexturePrefetchBudgetMs;
}
if( g_stream_catchup ) {
auto const apply_budget {
pending_apply_is_urgent() ?
kUrgentApplyBudgetMs :
gameplay_apply_budget_ms() };
return std::min( budget, apply_budget * 0.25 );
}
return std::min( budget, kDrainBudgetMs * 0.35 );
}
[[nodiscard]] glm::dvec3
camera_travel_forward() {
if( glm::length2( Global.pCamera.Velocity ) > 0.01 ) {
@@ -404,6 +811,24 @@ pending_apply_is_urgent() {
return pending_section_near_camera( max_dist );
}
[[nodiscard]] std::size_t
scene_apply_pressure_slice_cap() {
auto const applied { load_stats().pack_models };
if( applied >= 200000 ) {
return 16;
}
if( applied >= 120000 ) {
return 24;
}
if( applied >= 80000 ) {
return 32;
}
if( applied >= 40000 ) {
return 48;
}
return std::numeric_limits<std::size_t>::max();
}
[[nodiscard]] std::size_t
adaptive_slice_instances( std::size_t const section_total, bool const urgent = false ) {
if( urgent ) {
@@ -428,6 +853,14 @@ adaptive_slice_instances( std::size_t const section_total, bool const urgent = f
else if( last_ms >= 12.0 ) {
limit = std::min( limit, std::size_t { 48 } );
}
limit = std::min( limit, scene_apply_pressure_slice_cap() );
auto const last_apply { pack_bench_stream().last_apply_ms };
if( last_apply >= 20.0 ) {
limit = std::min( limit, std::size_t { 16 } );
}
else if( last_apply >= 12.0 ) {
limit = std::min( limit, std::size_t { 24 } );
}
return std::max( limit, std::size_t { 16 } );
}
@@ -438,7 +871,7 @@ adaptive_slice_instances( std::size_t const section_total, bool const urgent = f
limit = std::min( limit, kCatchupSliceInstances );
}
else if( speed > 600.0 ) {
limit = std::min( limit, std::size_t { 128 } );
limit = std::min( limit, std::size_t { 48 } );
}
else {
limit = std::min( limit, std::size_t { 96 } );
@@ -463,6 +896,19 @@ adaptive_slice_instances( std::size_t const section_total, bool const urgent = f
limit = std::min( limit, std::size_t { 64 } );
}
auto const last_apply { pack_bench_stream().last_apply_ms };
if( last_apply >= 24.0 ) {
limit = std::min( limit, std::size_t { 16 } );
}
else if( last_apply >= 16.0 ) {
limit = std::min( limit, std::size_t { 24 } );
}
else if( last_apply >= 10.0 ) {
limit = std::min( limit, std::size_t { 32 } );
}
limit = std::min( limit, scene_apply_pressure_slice_cap() );
return std::max( limit, std::size_t { 16 } );
}
@@ -528,23 +974,11 @@ preload_slice_cold_meshes(
break;
}
}
auto const was_global_cached { TModelsManager::IsModelCached( model_file ) };
TModel3d *mesh { nullptr };
{
PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms };
mesh = TModelsManager::GetModel( model_file, false, false );
(void)ensure_stream_pack_mesh( model_file );
pack_bench_inc( &Eu7PackBench::main_cold_getmodel_calls );
}
if( was_global_cached ) {
pack_bench_inc( &Eu7PackBench::stream_mesh_global_hit );
}
else {
pack_bench_inc( &Eu7PackBench::stream_mesh_disk_load );
}
if( mesh != nullptr ) {
TAnimModel::warm_instanceable_cache( mesh );
}
g_stream.mesh_cache.emplace( model_file, mesh );
++cold_loaded;
}
@@ -556,23 +990,11 @@ preload_slice_cold_meshes(
}
replace_slashes( model_file );
if( false == g_stream.mesh_cache.contains( model_file ) ) {
auto const was_global_cached { TModelsManager::IsModelCached( model_file ) };
TModel3d *mesh { nullptr };
{
PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms };
mesh = TModelsManager::GetModel( model_file, false, false );
(void)ensure_stream_pack_mesh( model_file );
pack_bench_inc( &Eu7PackBench::main_cold_getmodel_calls );
}
if( was_global_cached ) {
pack_bench_inc( &Eu7PackBench::stream_mesh_global_hit );
}
else {
pack_bench_inc( &Eu7PackBench::stream_mesh_disk_load );
}
if( mesh != nullptr ) {
TAnimModel::warm_instanceable_cache( mesh );
}
g_stream.mesh_cache.emplace( model_file, mesh );
cold_loaded = 1;
}
slice_count = 1;
@@ -619,23 +1041,11 @@ preload_umes_cold_meshes(
break;
}
}
auto const was_global_cached { TModelsManager::IsModelCached( model_file ) };
TModel3d *mesh { nullptr };
{
PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms };
mesh = TModelsManager::GetModel( model_file, false, false );
(void)ensure_stream_pack_mesh( model_file );
pack_bench_inc( &Eu7PackBench::main_cold_getmodel_calls );
}
if( was_global_cached ) {
pack_bench_inc( &Eu7PackBench::stream_mesh_global_hit );
}
else {
pack_bench_inc( &Eu7PackBench::stream_mesh_disk_load );
}
if( mesh != nullptr ) {
TAnimModel::warm_instanceable_cache( mesh );
}
g_stream.mesh_cache.emplace( model_file, mesh );
++cold_loaded;
offset_out = i + 1;
}
@@ -689,6 +1099,7 @@ maybe_skip_stuck_apply_offset() {
( model_file.empty() ? "" : " file=\"" + model_file + "\"" ) );
if( false == model_file.empty() && model_file != "notload" ) {
replace_slashes( model_file );
touch_string_lru( model_file, g_stream.mesh_lru, g_stream.mesh_lru_iters );
g_stream.mesh_cache.emplace( std::move( model_file ), nullptr );
}
g_stream.pending_apply_offset = offset + 1;
@@ -724,6 +1135,16 @@ maybe_preempt_distant_pending() {
if( pending_dist <= kUrgentSectionDistanceKm ) {
return;
}
if( pending_dist < 5 ) {
return;
}
auto const total { batch.models->size() };
if(
total > 0 &&
g_stream.pending_apply_offset > total / 2 ) {
return;
}
auto const speed { camera_stream_speed_mps() };
auto const near_apply_km {
@@ -749,7 +1170,7 @@ maybe_preempt_distant_pending() {
break;
}
if(
ready_dist < pending_dist &&
ready_dist + 3 < pending_dist &&
ready_dist <= near_apply_km ) {
nearer_ready = true;
break;
@@ -847,7 +1268,10 @@ apply_pending_chunk(
auto const cam_speed { camera_stream_speed_mps() };
auto const is_urgent { pending_apply_is_urgent() };
auto const is_heavy {
total > kHeavySectionModelThreshold || remaining > kHeavySectionModelThreshold };
false == batch.unique_meshes.empty() ||
false == batch.unique_textures.empty() ||
total >= kHeavySectionModelThreshold ||
remaining >= kHeavySectionModelThreshold };
auto const far_slow_drain {
cam_speed > 300.0 &&
pending_dist > kFarApplySlowDistanceKm &&
@@ -932,7 +1356,7 @@ apply_pending_chunk(
{
auto const phase_started { std::chrono::steady_clock::now() };
tex_fetches = warm_pack_textures_main( models_ptr + offset, chunk_count );
tex_fetches = warm_pack_section_textures( batch, offset, chunk_count );
warm_ms = std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - phase_started ).count();
}
@@ -941,7 +1365,8 @@ apply_pending_chunk(
scene::scratch_data scratch;
simulation::eu7_pack_apply_session const session {
&g_stream.mesh_cache,
&g_stream.nodedata_cache };
&g_stream.nodedata_cache,
batch.section_idx };
{
auto const phase_started { std::chrono::steady_clock::now() };
g_stream.serializer->apply_eu7_pack_models(
@@ -953,6 +1378,9 @@ apply_pending_chunk(
apply_ms = std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - phase_started ).count();
}
retain_section_mesh_keys( batch.section_idx, models_ptr, offset, chunk_count );
retain_section_nodedata_keys( batch.section_idx, models_ptr, offset, chunk_count );
evict_unreferenced_stream_caches();
auto const chunk_wall_ms {
std::chrono::duration<double, std::milli>(
@@ -1121,6 +1549,7 @@ worker_loop( std::stop_token const stop_token ) {
std::unique_ptr<std::vector<Eu7Model>> models;
std::vector<std::string> unique_meshes;
std::vector<std::string> unique_textures;
std::uint32_t chunk_count { 1 };
{
PackBenchTimer const read_timer { &Eu7PackBench::worker_read_pack_ms };
@@ -1129,6 +1558,7 @@ worker_loop( std::stop_token const stop_token ) {
module, job.row, job.column, job.next_chunk ) };
chunk_count = chunk.chunk_count;
unique_meshes = std::move( chunk.unique_meshes );
unique_textures = std::move( chunk.unique_textures );
if( false == chunk.models.empty() ) {
models = std::make_unique<std::vector<Eu7Model>>(
std::move( chunk.models ) );
@@ -1141,6 +1571,7 @@ worker_loop( std::stop_token const stop_token ) {
result.section_idx = job.section_idx;
result.models = std::move( models );
result.unique_meshes = std::move( unique_meshes );
result.unique_textures = std::move( unique_textures );
result.failed = result.models == nullptr;
result.section_final = job.next_chunk + 1 >= chunk_count;
@@ -1169,6 +1600,18 @@ worker_loop( std::stop_token const stop_token ) {
pack_bench_inc( &Eu7PackBench::worker_sections_done );
}
if( job.next_chunk == 0 ) {
std::size_t pending_jobs { 0 };
{
std::lock_guard<std::mutex> lock { g_stream.jobs.mutex };
pending_jobs = g_stream.jobs.data.size();
}
if( pending_jobs < kUmesPrefetchJobsBacklogThreshold ) {
prefetch_ready_queue_umes_worker(
kReadyUmesPrefetchBudgetMs, kReadyUmesPrefetchMaxMeshes );
}
}
if( job.next_chunk + 1 < chunk_count ) {
PackSectionJob continue_job { job };
continue_job.next_chunk = job.next_chunk + 1;
@@ -1722,15 +2165,6 @@ try_dequeue_ready_batch() {
pack_bench_inc( &Eu7PackBench::stream_dequeue_far );
}
if(
g_stream.pending_apply->models != nullptr &&
g_stream.pending_apply->texture_warm_offset < g_stream.pending_apply->models->size() ) {
(void)warm_pack_textures_main(
g_stream.pending_apply->models->data() + g_stream.pending_apply->texture_warm_offset,
g_stream.pending_apply->models->size() - g_stream.pending_apply->texture_warm_offset );
g_stream.pending_apply->texture_warm_offset = g_stream.pending_apply->models->size();
}
return true;
}
@@ -1748,7 +2182,17 @@ prefetch_ready_queue_textures( double const budget_ms ) {
{
std::lock_guard<std::mutex> lock { g_stream.ready.mutex };
for( auto it { g_stream.ready.data.begin() }; it != g_stream.ready.data.end(); ++it ) {
if( it->failed || it->models == nullptr || it->models->empty() ) {
if( it->failed ) {
continue;
}
if( false == it->unique_textures.empty() ) {
if( it->texture_warm_offset >= it->unique_textures.size() ) {
continue;
}
candidates.push_back( it );
continue;
}
if( it->models == nullptr || it->models->empty() ) {
continue;
}
if( it->texture_warm_offset >= it->models->size() ) {
@@ -1784,6 +2228,30 @@ prefetch_ready_queue_textures( double const budget_ms ) {
for( auto const it : candidates ) {
auto &batch { *it };
if( false == batch.unique_textures.empty() ) {
while( batch.texture_warm_offset < batch.unique_textures.size() ) {
auto const remaining {
batch.unique_textures.size() - batch.texture_warm_offset };
auto const slice { std::min( remaining, kReadyTexturePrefetchSlice ) };
auto const tex_slice { std::min( slice, kPackTextureWarmSlice ) };
std::size_t processed { 0 };
(void)warm_pack_texture_paths_main(
batch.unique_textures.data() + batch.texture_warm_offset,
tex_slice,
kPackTextureWarmBudgetMs,
&processed );
batch.texture_warm_offset += processed;
auto const elapsed_ms {
std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - started ).count() };
if( elapsed_ms >= budget_ms ) {
return;
}
}
continue;
}
while( batch.texture_warm_offset < batch.models->size() ) {
auto const remaining { batch.models->size() - batch.texture_warm_offset };
auto const slice { std::min( remaining, kReadyTexturePrefetchSlice ) };
@@ -1802,6 +2270,100 @@ prefetch_ready_queue_textures( double const budget_ms ) {
}
}
void
prefetch_ready_queue_umes_worker( double const budget_ms, std::size_t const max_meshes ) {
if( budget_ms <= 0.0 || max_meshes == 0 ) {
return;
}
PackBenchTimer const prefetch_timer { &Eu7PackBench::stream_prefetch_ready_umes_ms };
auto const started { std::chrono::steady_clock::now() };
std::size_t loads { 0 };
std::lock_guard<std::mutex> lock { g_stream.ready.mutex };
std::vector<std::deque<PackSectionReady>::iterator> candidates;
candidates.reserve( g_stream.ready.data.size() );
for( auto it { g_stream.ready.data.begin() }; it != g_stream.ready.data.end(); ++it ) {
if( it->failed || it->unique_meshes.empty() ) {
continue;
}
if( it->umes_prefetch_offset >= it->unique_meshes.size() ) {
continue;
}
if( g_stream.center_row >= 0 && g_stream.center_column >= 0 ) {
auto const dist {
section_manhattan_sections(
it->row,
it->column,
g_stream.center_row,
g_stream.center_column ) };
if( dist > kUmesPrefetchMaxDistanceKm ) {
continue;
}
}
candidates.push_back( it );
}
if( candidates.empty() ) {
return;
}
std::sort(
candidates.begin(),
candidates.end(),
[]( std::deque<PackSectionReady>::iterator const &lhs,
std::deque<PackSectionReady>::iterator const &rhs ) {
auto const lhs_dist {
section_manhattan_sections(
lhs->row,
lhs->column,
g_stream.center_row,
g_stream.center_column ) };
auto const rhs_dist {
section_manhattan_sections(
rhs->row,
rhs->column,
g_stream.center_row,
g_stream.center_column ) };
return lhs_dist < rhs_dist;
} );
for( auto const it : candidates ) {
auto &batch { *it };
while( batch.umes_prefetch_offset < batch.unique_meshes.size() ) {
auto model_file { batch.unique_meshes[ batch.umes_prefetch_offset ] };
if( model_file.empty() || model_file == "notload" ) {
++batch.umes_prefetch_offset;
continue;
}
replace_slashes( model_file );
if( g_stream.mesh_cache.contains( model_file ) ) {
pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit );
++batch.umes_prefetch_offset;
continue;
}
if( loads >= max_meshes ) {
return;
}
if( loads > 0 ) {
auto const elapsed_ms {
std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - started ).count() };
if( elapsed_ms >= budget_ms ) {
return;
}
}
(void)ensure_stream_pack_mesh( model_file );
++loads;
pack_bench_inc( &Eu7PackBench::stream_prefetch_ready_umes_loads );
++batch.umes_prefetch_offset;
}
}
}
[[nodiscard]] bool
gameplay_stream_mode() {
return g_loading_screen_dismissed
@@ -2107,6 +2669,7 @@ update_section_stream( glm::dvec3 const &world_position ) {
if( center_moved ) {
reprioritize_job_queue();
maybe_unload_distant_sections( center_row, center_column );
}
sync_stream_limits( stream_position );
@@ -2206,10 +2769,10 @@ drain_section_stream(
kSectionStreamGameplayRadiusKm ) };
if( gameplay_stream_mode() && inner_ring_ready ) {
prefetch_ready_queue_textures(
g_stream_catchup || camera_stream_speed_mps() > 80.0 ?
kReadyTexturePrefetchBudgetMs :
kReadyTexturePrefetchBudgetMs * 0.5 );
maybe_unload_distant_sections(
g_stream.center_row,
g_stream.center_column );
prefetch_ready_queue_textures( ready_texture_prefetch_budget_ms() );
if( g_stream_catchup ) {
auto const urgent {
@@ -2689,6 +3252,9 @@ void
reset_section_stream() {
flush_pack_stream_bench();
stop_workers();
if( g_stream.serializer != nullptr ) {
g_stream.serializer->reset_eu7_pack_section_instances();
}
reset_stream_fields();
g_ring_ready_since.reset();
g_loading_block_started.reset();

11
scene/eu7/eu7_types.h
View File

@@ -203,6 +203,14 @@ struct Eu7Model {
bool is_terrain = false;
};
[[nodiscard]] inline std::string
pack_nodedata_cache_key( Eu7Model const &model ) {
return model.model_file + '\x1f' + model.texture_file + '\x1f'
+ std::to_string( model.node.range_squared_min ) + '\x1f'
+ std::to_string( model.node.range_squared_max ) + '\x1f'
+ ( model.is_terrain ? '1' : '0' );
}
// Wspolna definicja modelu (EU7B v8 PROT) — bez transformacji instancji.
struct Eu7ModelPrototype {
Eu7BasicNode node;
@@ -369,6 +377,7 @@ struct Eu7PackSectionCursor {
std::uint8_t section_format { 0 };
bool header_parsed { false };
std::vector<std::string> unique_meshes;
std::vector<std::string> unique_textures;
std::uint32_t chunk_count { 0 };
std::vector<std::uint32_t> chunk_model_counts;
std::vector<std::uint32_t> chunk_byte_offsets;
@@ -377,11 +386,13 @@ struct Eu7PackSectionCursor {
struct Eu7PackSectionLoad {
std::vector<Eu7Model> models;
std::vector<std::string> unique_meshes;
std::vector<std::string> unique_textures;
};
struct Eu7PackSectionChunkLoad {
std::vector<Eu7Model> models;
std::vector<std::string> unique_meshes;
std::vector<std::string> unique_textures;
std::uint32_t chunk_count { 1 };
std::uint32_t chunk_index { 0 };
};

27
scene/scene.cpp
View File

@@ -444,6 +444,33 @@ basic_cell::insert( TMemCell *Memorycell ) {
void
basic_cell::erase( TAnimModel *Instance ) {
if( Instance->m_eu7_pack && Instance->m_instanceable ) {
m_instancesopaque.erase(
std::remove_if(
std::begin( m_instancesopaque ), std::end( m_instancesopaque ),
[=]( TAnimModel *instance ) {
return instance == Instance; } ),
std::end( m_instancesopaque ) );
if( Instance->Model() != nullptr ) {
instance_bucket_key key;
key.pModel = Instance->Model();
auto const *mat { Instance->Material() };
if( mat != nullptr ) {
for( int i { 0 }; i < 5; ++i ) { key.skins[ i ] = mat->replacable_skins[ i ]; }
}
auto bucket { m_instancebuckets_opaque.find( key ) };
if( bucket != m_instancebuckets_opaque.end() ) {
bucket->second.erase(
std::remove( std::begin( bucket->second ), std::end( bucket->second ), Instance ),
std::end( bucket->second ) );
if( bucket->second.empty() ) {
m_instancebuckets_opaque.erase( bucket );
}
}
}
return;
}
auto const flags = Instance->Flags();
auto alpha =
( Instance->Material() != nullptr ?

47
simulation/simulationstateserializer.cpp
View File

@@ -34,6 +34,7 @@ http://mozilla.org/MPL/2.0/.
#include "scene/eu7/eu7_loader.h"
#include "scene/eu7/eu7_load_stats.h"
#include "scene/eu7/eu7_pack_bench.h"
#include "scene/eu7/eu7_section.h"
#include "scene/eu7/eu7_transform.h"
#include "world/Track.h"
#include "world/Traction.h"
@@ -285,19 +286,11 @@ pack_model_needs_full_load( scene::eu7::Eu7Model const &model ) {
|| false == model.transition;
}
[[nodiscard]] std::string
pack_nodedata_cache_key( scene::eu7::Eu7Model const &model ) {
return model.model_file + '\x1f' + model.texture_file + '\x1f'
+ std::to_string( model.node.range_squared_min ) + '\x1f'
+ std::to_string( model.node.range_squared_max ) + '\x1f'
+ ( model.is_terrain ? '1' : '0' );
}
[[nodiscard]] scene::node_data const &
pack_nodedata_cached(
scene::eu7::Eu7Model const &model,
std::unordered_map<std::string, scene::node_data> &cache ) {
auto const key { pack_nodedata_cache_key( model ) };
auto const key { scene::eu7::pack_nodedata_cache_key( model ) };
auto const found { cache.find( key ) };
if( found != cache.end() ) {
return found->second;
@@ -581,6 +574,11 @@ state_serializer::insert_eu7_pack_models(
scene::eu7::pack_bench_inc( &scene::eu7::Eu7PackBench::main_region_inserts );
scene::eu7::pack_bench_inc( &scene::eu7::Eu7PackBench::main_instances_applied );
}
if(
session != nullptr &&
session->section_idx != std::numeric_limits<std::size_t>::max() ) {
m_pack_section_instances[ session->section_idx ].push_back( instance );
}
}
}
@@ -894,6 +892,37 @@ state_serializer::apply_eu7_pack_models(
insert_eu7_pack_models( models + offset, count, scratchpad, session );
}
std::size_t
state_serializer::unload_eu7_pack_section( int const row, int const column ) {
auto const section_idx { scene::eu7::section_index( row, column ) };
auto found { m_pack_section_instances.find( section_idx ) };
if( found == m_pack_section_instances.end() || found->second.empty() ) {
m_pack_section_instances.erase( section_idx );
return 0;
}
std::size_t removed { 0 };
for( auto *instance : found->second ) {
if( instance == nullptr ) {
continue;
}
Particles.erase_bound_node( instance );
if( simulation::Region != nullptr ) {
simulation::Region->erase( instance );
}
TAnimModel::release_pack_instance( instance );
++removed;
}
m_pack_section_instances.erase( found );
return removed;
}
void
state_serializer::reset_eu7_pack_section_instances() {
m_pack_section_instances.clear();
}
void
state_serializer::apply_eu7_scene(
scene::eu7::Eu7Scene const &scene,

10
simulation/simulationstateserializer.h
View File

@@ -9,12 +9,15 @@ http://mozilla.org/MPL/2.0/.
#pragma once
#include <limits>
#include <unordered_map>
#include <vector>
#include "utilities/parser.h"
#include "scene/eu7/eu7_types.h"
#include "scene/scene.h"
class TAnimModel;
class TModel3d;
namespace simulation {
@@ -22,6 +25,7 @@ namespace simulation {
struct eu7_pack_apply_session {
std::unordered_map<std::string, TModel3d *> *mesh_cache { nullptr };
std::unordered_map<std::string, scene::node_data> *nodedata_cache { nullptr };
std::size_t section_idx { std::numeric_limits<std::size_t>::max() };
};
struct deserializer_state {
@@ -108,8 +112,14 @@ public:
// Odlozone sklady AI — ladowane w tle po wejsciu w gre.
void
drain_deferred_eu7_trainsets( double MaxMs = 12.0 );
// PACK streaming: usun instancje sekcji z Region i zwolnij pool.
[[nodiscard]] std::size_t
unload_eu7_pack_section( int Row, int Column );
void
reset_eu7_pack_section_instances();
private:
std::unordered_map<std::size_t, std::vector<TAnimModel *>> m_pack_section_instances;
struct eu7_transform_state;
void
insert_eu7_models(