eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-22 13:59:19 +02:00
Code Issues 123 Wiki Activity

async mesh loading

Browse Source
This commit is contained in:
maj00r
2026-06-16 01:04:37 +02:00
parent 2f90246907
commit 9574d2051b
35 changed files with 3239 additions and 859 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scene/eu7/eu7_chunks.h
View File

@@ -31,6 +31,7 @@ constexpr std::uint32_t kEu7VersionV5 { 5 };
constexpr std::uint32_t kEu7VersionV6 { 6 };
constexpr std::uint32_t kEu7VersionV7 { 7 };
constexpr std::uint32_t kEu7VersionV8 { 8 };
constexpr std::uint32_t kEu7VersionV9 { 9 };
constexpr std::uint32_t kChunkStrs { detail::make_id4( 'S', 'T', 'R', 'S' ) };
constexpr std::uint32_t kChunkIncl { detail::make_id4( 'I', 'N', 'C', 'L' ) };
@@ -62,6 +63,7 @@ constexpr std::uint8_t kPackSectionFormatV9 { 2 };
constexpr std::uint8_t kPackSectionFormatV10 { 3 };
constexpr std::uint8_t kPackSectionFormatV11 { 4 };
constexpr std::uint8_t kPackSectionFormatV12 { 5 };
constexpr std::uint8_t kPackSectionFormatV13 { 6 };
constexpr std::size_t kPackSectionChunkModels { 512 };
} // namespace scene::eu7

1
scene/eu7/eu7_loader.cpp
View File

@@ -553,6 +553,7 @@ load_module( std::string const &path, simulation::state_serializer &serializer )
auto const resolved { resolve_scenery_path( path ) };
auto const ok { load_module_recursive( path, serializer, g_load_session ) };
if( ok ) {
log_load_stats();
if( auto const cached { g_module_file_cache.find( resolved ) };
cached != g_module_file_cache.end() && cached->second.has_pack_chunk ) {
init_section_stream( cached->second, resolved, serializer );

312
scene/eu7/eu7_model_prefetch.cpp
View File

@@ -11,12 +11,13 @@ http://mozilla.org/MPL/2.0/.
#include "scene/eu7/eu7_model_prefetch.h"
#include "model/AnimModel.h"
#include "model/MdlMngr.h"
#include "rendering/renderer.h"
#include "scene/eu7/eu7_pack_bench.h"
#include "utilities/Globals.h"
#include "utilities/utilities.h"
#include "vehicle/DynObj.h"
#include <algorithm>
#include <list>
#include <mutex>
#include <string>
@@ -28,11 +29,29 @@ namespace scene::eu7 {
namespace {
constexpr std::size_t kMaxWarmedTextures { 2048 };
constexpr std::size_t kMaxPackMaterialAssignCache { 4096 };
std::mutex g_pack_mesh_load_mutex;
std::unordered_set<std::string> g_session_warmed_textures;
std::list<std::string> g_session_warmed_textures_lru;
std::unordered_map<std::string, std::list<std::string>::iterator> g_session_warmed_textures_iters;
std::unordered_map<std::string, material_data> g_pack_material_assign_cache;
[[nodiscard]] std::string
pack_material_cache_key(
std::string const &model_file,
std::string const &texture_file ) {
return model_file + '\x1e' + texture_file;
}
void
evict_pack_material_assign_cache_if_needed() {
while( g_pack_material_assign_cache.size() >= kMaxPackMaterialAssignCache ) {
if( g_pack_material_assign_cache.empty() ) {
break;
}
g_pack_material_assign_cache.erase( g_pack_material_assign_cache.begin() );
}
}
void
touch_warmed_texture_lru( std::string const &texture_file ) {
@@ -107,12 +126,119 @@ pack_texture_usable( std::string texture_file ) {
void
preload_pack_model_file( std::string const & ) {
// Mesh warm runs on the main thread via ensure_pack_mesh_in_session_cache only.
// GetModel mutates Global.asCurrentTexturePath and must not run on PACK workers.
// GetModel + Init must run on the main thread (Global.asCurrentTexturePath + OpenGL).
}
[[nodiscard]] bool
warm_one_pack_texture( std::string texture_file, std::unordered_set<std::string> &seen ) {
pack_texture_path_is_rooted( std::string const &texture_file ) {
return texture_file.starts_with( "dynamic/" ) ||
texture_file.starts_with( "textures/" ) ||
texture_file.starts_with( "scenery/" ) ||
texture_file.starts_with( "models/" );
}
[[nodiscard]] std::string
pack_model_texture_dir( std::string model_file ) {
if( model_file.empty() || model_file == "notload" ) {
return {};
}
replace_slashes( model_file );
erase_extension( model_file );
auto const slash_pos { model_file.rfind( '/' ) };
if( slash_pos == std::string::npos ) {
return {};
}
return model_file.substr( 0, slash_pos + 1 );
}
void
append_pack_texture_candidate(
std::vector<std::string> &candidates,
std::string candidate ) {
if( candidate.empty() ) {
return;
}
replace_slashes( candidate );
if(
std::find( candidates.begin(), candidates.end(), candidate ) ==
candidates.end() ) {
candidates.emplace_back( std::move( candidate ) );
}
}
[[nodiscard]] std::vector<std::string>
pack_texture_resolve_candidates(
std::string const &model_file,
std::string texture_file ) {
std::vector<std::string> candidates;
candidates.reserve( 4 );
if( false == pack_texture_usable( texture_file ) ) {
return candidates;
}
replace_slashes( texture_file );
append_pack_texture_candidate( candidates, texture_file );
auto const model_dir { pack_model_texture_dir( model_file ) };
if( false == model_dir.empty() ) {
append_pack_texture_candidate( candidates, model_dir + texture_file );
}
if( false == pack_texture_path_is_rooted( texture_file ) ) {
append_pack_texture_candidate( candidates, paths::textures + texture_file );
}
return candidates;
}
class PackTexturePathScope {
public:
explicit PackTexturePathScope( std::string const &model_file ) {
m_saved_path = Global.asCurrentTexturePath;
auto const model_dir { pack_model_texture_dir( model_file ) };
if( false == model_dir.empty() && model_file.find( '/' ) != std::string::npos ) {
Global.asCurrentTexturePath = model_dir;
m_active = true;
}
}
~PackTexturePathScope() {
if( m_active ) {
Global.asCurrentTexturePath = std::move( m_saved_path );
}
}
PackTexturePathScope( PackTexturePathScope const & ) = delete;
PackTexturePathScope &operator=( PackTexturePathScope const & ) = delete;
private:
std::string m_saved_path;
bool m_active { false };
};
[[nodiscard]] bool
fetch_pack_texture_material( std::string const &texture_file ) {
auto const resolved { TextureTest( ToLower( texture_file ) ) };
if( false == resolved.empty() ) {
GfxRenderer->Fetch_Material( resolved );
return true;
}
bool warmed { false };
for( int skinindex { 1 }; skinindex <= 4; ++skinindex ) {
auto const multi {
TextureTest( ToLower( texture_file + "," + std::to_string( skinindex ) ) ) };
if( multi.empty() ) {
break;
}
GfxRenderer->Fetch_Material( multi );
warmed = true;
}
return warmed;
}
[[nodiscard]] bool
warm_one_pack_texture(
std::string texture_file,
std::unordered_set<std::string> &seen,
std::string const &model_file = {} ) {
if( false == pack_texture_usable( texture_file ) ) {
return false;
}
@@ -124,77 +250,40 @@ warm_one_pack_texture( std::string texture_file, std::unordered_set<std::string>
return false;
}
auto const resolved { TextureTest( ToLower( texture_file ) ) };
if( resolved.empty() ) {
bool warmed { false };
for( int skinindex { 1 }; skinindex <= 4; ++skinindex ) {
auto const multi {
TextureTest( ToLower( texture_file + "," + std::to_string( skinindex ) ) ) };
if( multi.empty() ) {
break;
}
GfxRenderer->Fetch_Material( multi );
warmed = true;
PackTexturePathScope const scope { model_file };
for( auto const &candidate : pack_texture_resolve_candidates( model_file, texture_file ) ) {
if( fetch_pack_texture_material( candidate ) ) {
remember_warmed_texture( texture_file );
return true;
}
if( false == warmed ) {
pack_bench_inc( &Eu7PackBench::main_texture_warm_miss );
log_pack_texture_fail( texture_file );
return false;
}
}
else {
GfxRenderer->Fetch_Material( resolved );
}
remember_warmed_texture( texture_file );
pack_bench_inc( &Eu7PackBench::main_texture_warm_miss );
log_pack_texture_fail( texture_file );
return false;
}
if( false == resolved.empty() ) {
for( int skinindex { 1 }; skinindex <= 4; ++skinindex ) {
auto const multi {
TextureTest( ToLower( texture_file + "," + std::to_string( skinindex ) ) ) };
if( multi.empty() ) {
break;
}
GfxRenderer->Fetch_Material( multi );
[[nodiscard]] std::string
find_pack_model_file_for_texture(
Eu7Model const *const models,
std::size_t const model_count,
std::string texture_file ) {
if( models == nullptr || model_count == 0 ) {
return {};
}
replace_slashes( texture_file );
for( std::size_t i { 0 }; i < model_count; ++i ) {
auto candidate { models[ i ].texture_file };
replace_slashes( candidate );
if( candidate == texture_file ) {
return models[ i ].model_file;
}
}
return true;
return {};
}
} // namespace
[[nodiscard]] bool
ensure_pack_mesh_in_session_cache(
std::string model_file,
std::unordered_map<std::string, TModel3d *> &session_cache ) {
if( model_file.empty() || model_file == "notload" ) {
return false;
}
replace_slashes( model_file );
if( session_cache.contains( model_file ) ) {
pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit );
return true;
}
auto const was_global_cached { TModelsManager::IsModelCached( model_file ) };
TModel3d *mesh { nullptr };
{
std::lock_guard<std::mutex> lock { g_pack_mesh_load_mutex };
mesh = TModelsManager::GetModel( model_file, false, false );
}
if( was_global_cached ) {
pack_bench_inc( &Eu7PackBench::stream_mesh_global_hit );
}
else {
pack_bench_inc( &Eu7PackBench::stream_mesh_disk_load );
}
if( mesh != nullptr ) {
TAnimModel::warm_instanceable_cache( mesh );
}
session_cache.emplace( model_file, mesh );
return true;
}
void
reset_pack_texture_warm_cache() {
g_session_warmed_textures.clear();
@@ -221,7 +310,9 @@ warm_pack_texture_paths_main(
std::string const *const paths,
std::size_t const count,
double const budget_ms,
std::size_t *const processed_out ) {
std::size_t *const processed_out,
Eu7Model const *const models,
std::size_t const model_count ) {
if( paths == nullptr || count == 0 || GfxRenderer == nullptr ) {
if( processed_out != nullptr ) {
*processed_out = 0;
@@ -246,7 +337,10 @@ warm_pack_texture_paths_main(
break;
}
}
if( warm_one_pack_texture( paths[ i ], seen ) ) {
if( warm_one_pack_texture(
paths[ i ],
seen,
find_pack_model_file_for_texture( models, model_count, paths[ i ] ) ) ) {
++warmed;
}
++processed;
@@ -259,7 +353,10 @@ warm_pack_texture_paths_main(
}
std::size_t
warm_pack_textures_main( Eu7Model const *const models, std::size_t const count ) {
warm_pack_textures_main(
Eu7Model const *const models,
std::size_t const count,
double const budget_ms ) {
if( models == nullptr || count == 0 || GfxRenderer == nullptr ) {
return 0;
}
@@ -267,13 +364,88 @@ warm_pack_textures_main( Eu7Model const *const models, std::size_t const count )
std::unordered_set<std::string> seen;
seen.reserve( std::min( count, std::size_t { 64 } ) );
std::size_t warmed { 0 };
auto const started { std::chrono::steady_clock::now() };
for( std::size_t i { 0 }; i < count; ++i ) {
if( warm_one_pack_texture( models[ i ].texture_file, seen ) ) {
if(
budget_ms > 0.0 &&
i > 0 ) {
auto const elapsed_ms {
std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - started ).count() };
if( elapsed_ms >= budget_ms ) {
break;
}
}
if( warm_one_pack_texture(
models[ i ].texture_file,
seen,
models[ i ].model_file ) ) {
++warmed;
}
}
return warmed;
}
[[nodiscard]] bool
assign_pack_texture(
material_data &material,
std::string const &model_file,
std::string const &texture_file,
std::string const &resolved_texture,
std::uint32_t const textures_alpha ) {
if( false == pack_texture_usable( texture_file ) ) {
if( textures_alpha != 0 ) {
material.textures_alpha = static_cast<int>( textures_alpha );
}
return true;
}
auto const cache_key {
resolved_texture.empty() ?
pack_material_cache_key( model_file, texture_file ) :
pack_material_cache_key( model_file, resolved_texture ) };
auto const cached { g_pack_material_assign_cache.find( cache_key ) };
if(
cached != g_pack_material_assign_cache.end() &&
cached->second.replacable_skins[ 1 ] != null_handle ) {
material = cached->second;
if( textures_alpha != 0 ) {
material.textures_alpha = static_cast<int>( textures_alpha );
}
return true;
}
if( false == resolved_texture.empty() ) {
material = {};
material.assign( resolved_texture );
if( material.replacable_skins[ 1 ] != null_handle ) {
if( textures_alpha != 0 ) {
material.textures_alpha = static_cast<int>( textures_alpha );
}
evict_pack_material_assign_cache_if_needed();
g_pack_material_assign_cache.emplace( cache_key, material );
return true;
}
}
PackTexturePathScope const scope { model_file };
for( auto const &candidate : pack_texture_resolve_candidates( model_file, texture_file ) ) {
material = {};
material.assign( candidate );
if( material.replacable_skins[ 1 ] != null_handle ) {
if( textures_alpha != 0 ) {
material.textures_alpha = static_cast<int>( textures_alpha );
}
evict_pack_material_assign_cache_if_needed();
g_pack_material_assign_cache.emplace( cache_key, material );
return true;
}
}
pack_bench_inc( &Eu7PackBench::main_texture_assign_fail );
log_pack_texture_fail( texture_file );
return false;
}
} // namespace scene::eu7

28
scene/eu7/eu7_model_prefetch.h
View File

@@ -9,19 +9,16 @@ http://mozilla.org/MPL/2.0/.
#pragma once
#include "scene/eu7/eu7_pack_mesh_loader.h"
#include "scene/eu7/eu7_types.h"
#include "vehicle/DynObj.h"
#include <cstddef>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
class TModel3d;
namespace scene::eu7 {
// Deprecated: mesh warm is main-thread only (ensure_pack_mesh_in_session_cache).
void
preload_pack_models( std::vector<Eu7Model> const &Models );
@@ -30,25 +27,30 @@ preload_pack_models(
std::vector<Eu7Model> const &Models,
std::vector<std::string> const &UniqueMeshes );
// Main thread: Fetch_Material dla unikalnych texture_file z chunka przed apply.
// Zwraca liczbe unikalnych Fetch_Material w tym slice.
std::size_t
warm_pack_texture_paths_main(
std::string const *paths,
std::size_t count,
double budget_ms = 0.0,
std::size_t *processed_out = nullptr );
std::size_t *processed_out = nullptr,
Eu7Model const *models = nullptr,
std::size_t model_count = 0 );
std::size_t
warm_pack_textures_main( Eu7Model const *models, std::size_t count );
warm_pack_textures_main(
Eu7Model const *models,
std::size_t count,
double budget_ms = 0.0 );
void
reset_pack_texture_warm_cache();
// Thread-safe cold mesh load: session cache, then GetModel under g_pack_mesh_load_mutex.
[[nodiscard]] bool
ensure_pack_mesh_in_session_cache(
std::string model_file,
std::unordered_map<std::string, TModel3d *> &session_cache );
assign_pack_texture(
material_data &material,
std::string const &model_file,
std::string const &texture_file,
std::string const &resolved_texture = {},
std::uint32_t textures_alpha = 0 );
} // namespace scene::eu7

12
scene/eu7/eu7_pack_bench.cpp
View File

@@ -8,6 +8,7 @@ http://mozilla.org/MPL/2.0/.
*/
#include "stdafx.h"
#include "scene/eu7/eu7_pack_mesh_loader.h"
#include "scene/eu7/eu7_pack_bench.h"
#include "utilities/Logs.h"
@@ -181,6 +182,9 @@ log_pack_bench_impl( Eu7PackBench const &bench, char const *const title ) {
" diag mesh: sess_hit=" + std::to_string( bench.stream_mesh_session_hit ) +
" glob_hit=" + std::to_string( bench.stream_mesh_global_hit ) +
" disk=" + std::to_string( bench.stream_mesh_disk_load ) +
" async_q=" + std::to_string( bench.stream_mesh_async_queued ) +
" async_r=" + std::to_string( bench.stream_mesh_async_ready ) +
" loader=" + std::to_string( bench.loader_thread_disk_loads ) +
" dequeue_near=" + std::to_string( bench.stream_dequeue_near ) +
" far=" + std::to_string( bench.stream_dequeue_far ) +
" cam=" + std::to_string( bench.stream_dequeue_camera ) +
@@ -189,7 +193,10 @@ log_pack_bench_impl( Eu7PackBench const &bench, char const *const title ) {
" block_ring_in=" + std::to_string( bench.stream_jobs_blocked_ring_inner ) +
" block_ring_out=" + std::to_string( bench.stream_jobs_blocked_ring_outer ) +
" defer_dist=" + std::to_string( bench.stream_worker_deferred_distant ) +
" stuck_skip=" + std::to_string( bench.stream_apply_stuck_skip ) );
" stuck_skip=" + std::to_string( bench.stream_apply_stuck_skip ) +
" apply_defer=" + std::to_string( bench.stream_apply_deferred ) +
" defer_bypass=" + std::to_string( bench.stream_apply_defer_bypass ) +
" sparse_skip=" + std::to_string( bench.stream_sparse_apply_skip ) );
WriteLog(
" diag ready_tex_ms=" + std::to_string( static_cast<int>( bench.stream_prefetch_ready_tex_ms ) ) +
" ready_umes_ms=" + std::to_string( static_cast<int>( bench.stream_prefetch_ready_umes_ms ) ) +
@@ -442,6 +449,9 @@ log_pack_stream_status(
" ahead=" + std::to_string( bench.stream_lookahead_enqueue ) +
" urg_chunks=" + std::to_string( bench.stream_chunks_urgent ) +
" cold_trunc=" + std::to_string( bench.stream_cold_slice_truncated ) +
" mesh_q=" + std::to_string( pack_mesh_loader_queue_depth() ) +
" mesh_r=" + std::to_string( pack_mesh_loader_ready_count() ) +
" mesh_d=" + std::to_string( bench.stream_mesh_async_drained ) +
" preempt=" + std::to_string( bench.stream_preempt_pending ) );
}

9
scene/eu7/eu7_pack_bench.h
View File

@@ -63,6 +63,12 @@ struct Eu7PackBench {
std::uint64_t stream_mesh_session_hit { 0 };
std::uint64_t stream_mesh_global_hit { 0 };
std::uint64_t stream_mesh_disk_load { 0 };
std::uint64_t stream_mesh_async_queued { 0 };
std::uint64_t stream_mesh_async_ready { 0 };
std::uint64_t stream_mesh_async_drained { 0 };
std::uint64_t stream_mesh_async_wait_timeout { 0 };
std::uint64_t loader_thread_disk_loads { 0 };
double loader_thread_getmodel_ms { 0.0 };
std::uint64_t stream_chunks_urgent { 0 };
std::uint64_t stream_chunks_heavy { 0 };
std::uint64_t stream_chunks_light { 0 };
@@ -75,6 +81,9 @@ struct Eu7PackBench {
std::uint64_t stream_jobs_blocked_ring_outer { 0 };
std::uint64_t stream_worker_deferred_distant { 0 };
std::uint64_t stream_apply_stuck_skip { 0 };
std::uint64_t stream_apply_deferred { 0 };
std::uint64_t stream_apply_defer_bypass { 0 };
std::uint64_t stream_sparse_apply_skip { 0 };
std::uint64_t stream_sections_unloaded { 0 };
std::uint64_t stream_unload_instances { 0 };
std::uint64_t stream_mesh_cache_evictions { 0 };

501
scene/eu7/eu7_pack_mesh_loader.cpp Normal file
View File

@@ -0,0 +1,501 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#include "stdafx.h"
#include "scene/eu7/eu7_pack_mesh_loader.h"
#include "model/AnimModel.h"
#include "model/MdlMngr.h"
#include "scene/eu7/eu7_pack_bench.h"
#include "utilities/Globals.h"
#include "utilities/utilities.h"
#include <chrono>
#include <deque>
#include <mutex>
#include <queue>
#include <unordered_set>
namespace scene::eu7 {
namespace {
std::mutex g_pack_mesh_load_mutex;
struct PackMeshQueueEntry {
int priority { 0 };
std::uint64_t sequence { 0 };
std::string model_file;
};
struct PackMeshQueueCompare {
[[nodiscard]] bool
operator()( PackMeshQueueEntry const &lhs, PackMeshQueueEntry const &rhs ) const {
if( lhs.priority != rhs.priority ) {
return lhs.priority > rhs.priority;
}
return lhs.sequence > rhs.sequence;
}
};
struct PackMeshLoader {
std::mutex mutex;
std::priority_queue<
PackMeshQueueEntry,
std::vector<PackMeshQueueEntry>,
PackMeshQueueCompare> queue;
std::unordered_map<std::string, int> queued;
std::unordered_map<std::string, TModel3d *> ready;
std::uint64_t sequence { 0 };
bool running { false };
};
PackMeshLoader g_loader;
[[nodiscard]] std::string
pack_model_texture_dir( std::string model_file ) {
if( model_file.empty() || model_file == "notload" ) {
return {};
}
replace_slashes( model_file );
erase_extension( model_file );
auto const slash_pos { model_file.rfind( '/' ) };
if( slash_pos == std::string::npos ) {
return {};
}
return model_file.substr( 0, slash_pos + 1 );
}
class PackTexturePathScope {
public:
explicit PackTexturePathScope( std::string const &model_file ) {
m_saved_path = Global.asCurrentTexturePath;
auto const model_dir { pack_model_texture_dir( model_file ) };
if( false == model_dir.empty() && model_file.find( '/' ) != std::string::npos ) {
Global.asCurrentTexturePath = model_dir;
m_active = true;
}
}
~PackTexturePathScope() {
if( m_active ) {
Global.asCurrentTexturePath = std::move( m_saved_path );
}
}
PackTexturePathScope( PackTexturePathScope const & ) = delete;
PackTexturePathScope &operator=( PackTexturePathScope const & ) = delete;
private:
std::string m_saved_path;
bool m_active { false };
};
[[nodiscard]] bool
pack_mesh_path_valid( std::string const &model_file ) {
return false == model_file.empty() && model_file != "notload";
}
[[nodiscard]] bool
pack_mesh_pointer_usable( TModel3d *const mesh ) {
return mesh != nullptr;
}
[[nodiscard]] bool
pack_mesh_cached_usable(
std::unordered_map<std::string, TModel3d *> const &session_cache,
std::string const &model_file ) {
auto const found { session_cache.find( model_file ) };
return found != session_cache.end() && pack_mesh_pointer_usable( found->second );
}
[[nodiscard]] bool
pack_mesh_globally_usable( std::string model_file ) {
if( false == pack_mesh_path_valid( model_file ) ) {
return false;
}
replace_slashes( model_file );
if( false == TModelsManager::IsModelCached( model_file ) ) {
return false;
}
TModel3d *mesh { nullptr };
{
std::lock_guard<std::mutex> lock { g_pack_mesh_load_mutex };
mesh = TModelsManager::GetModel( model_file, false, false );
}
return pack_mesh_pointer_usable( mesh );
}
[[nodiscard]] bool
pop_pack_mesh_queue_entry( std::string &model_file ) {
model_file.clear();
while( false == g_loader.queue.empty() ) {
auto entry { g_loader.queue.top() };
g_loader.queue.pop();
auto const found { g_loader.queued.find( entry.model_file ) };
if(
found == g_loader.queued.end() ||
found->second != entry.priority ) {
continue;
}
model_file = std::move( entry.model_file );
return true;
}
return false;
}
void
load_pack_mesh_on_main_thread( std::string const &model_file ) {
PackBenchTimer const load_timer { &Eu7PackBench::loader_thread_getmodel_ms };
TModel3d *mesh { nullptr };
{
std::lock_guard<std::mutex> lock { g_pack_mesh_load_mutex };
PackTexturePathScope const scope { model_file };
mesh = TModelsManager::GetModel( model_file, false, false );
if( pack_mesh_pointer_usable( mesh ) ) {
TAnimModel::warm_instanceable_cache( mesh );
}
}
pack_bench_inc( &Eu7PackBench::loader_thread_disk_loads );
std::lock_guard<std::mutex> lock { g_loader.mutex };
g_loader.ready.emplace( model_file, mesh );
g_loader.queued.erase( model_file );
}
[[nodiscard]] std::size_t
pump_pack_mesh_loader_main(
double const budget_ms,
std::size_t const max_loads ) {
if( false == g_loader.running ) {
return 0;
}
auto const started { std::chrono::steady_clock::now() };
std::size_t loaded { 0 };
while( loaded < max_loads ) {
if( budget_ms > 0.0 ) {
auto const elapsed_ms {
std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - started ).count() };
if( elapsed_ms >= budget_ms ) {
break;
}
}
std::string model_file;
{
std::lock_guard<std::mutex> lock { g_loader.mutex };
if( false == pop_pack_mesh_queue_entry( model_file ) ) {
break;
}
}
load_pack_mesh_on_main_thread( model_file );
++loaded;
}
return loaded;
}
[[nodiscard]] bool
try_consume_loader_ready(
std::string const &model_file,
std::unordered_map<std::string, TModel3d *> &session_cache ) {
TModel3d *mesh { nullptr };
{
std::lock_guard<std::mutex> lock { g_loader.mutex };
auto const found { g_loader.ready.find( model_file ) };
if( found == g_loader.ready.end() ) {
return false;
}
mesh = found->second;
g_loader.ready.erase( found );
}
session_cache.emplace( model_file, mesh );
pack_bench_inc( &Eu7PackBench::stream_mesh_async_ready );
return pack_mesh_pointer_usable( mesh );
}
[[nodiscard]] bool
sync_resolve_global_cached_mesh(
std::string const &model_file,
std::unordered_map<std::string, TModel3d *> &session_cache ) {
if( false == pack_mesh_globally_usable( model_file ) ) {
return false;
}
TModel3d *mesh { nullptr };
{
PackBenchTimer const load_timer { &Eu7PackBench::main_cold_getmodel_ms };
std::lock_guard<std::mutex> lock { g_pack_mesh_load_mutex };
PackTexturePathScope const scope { model_file };
mesh = TModelsManager::GetModel( model_file, false, false );
pack_bench_inc( &Eu7PackBench::main_cold_getmodel_calls );
if( pack_mesh_pointer_usable( mesh ) ) {
TAnimModel::warm_instanceable_cache( mesh );
}
}
pack_bench_inc( &Eu7PackBench::stream_mesh_global_hit );
session_cache.emplace( model_file, mesh );
return pack_mesh_pointer_usable( mesh );
}
void
enqueue_pack_mesh_load( std::string model_file, int const priority ) {
if( false == pack_mesh_path_valid( model_file ) ) {
return;
}
replace_slashes( model_file );
{
std::lock_guard<std::mutex> lock { g_loader.mutex };
if( g_loader.ready.contains( model_file ) ) {
return;
}
auto const found { g_loader.queued.find( model_file ) };
if( found != g_loader.queued.end() ) {
if( priority >= found->second ) {
return;
}
found->second = priority;
}
else {
g_loader.queued.emplace( model_file, priority );
}
g_loader.queue.push(
PackMeshQueueEntry { priority, g_loader.sequence++, std::move( model_file ) } );
}
pack_bench_inc( &Eu7PackBench::stream_mesh_async_queued );
}
[[nodiscard]] bool
wait_for_loader_ready(
std::string const &model_file,
double const block_budget_ms ) {
auto const deadline {
block_budget_ms > 0.0 ?
std::chrono::steady_clock::now() +
std::chrono::duration_cast<std::chrono::steady_clock::duration>(
std::chrono::duration<double, std::milli>( block_budget_ms ) ) :
std::chrono::steady_clock::time_point::max() };
while( true ) {
{
std::lock_guard<std::mutex> lock { g_loader.mutex };
if( g_loader.ready.contains( model_file ) ) {
return true;
}
if( g_loader.queued.contains( model_file ) == false ) {
return false;
}
}
if( block_budget_ms > 0.0 && std::chrono::steady_clock::now() >= deadline ) {
return false;
}
auto remaining_ms { block_budget_ms };
if( block_budget_ms > 0.0 ) {
remaining_ms = std::chrono::duration<double, std::milli>(
deadline - std::chrono::steady_clock::now() ).count();
if( remaining_ms <= 0.0 ) {
return false;
}
}
(void)pump_pack_mesh_loader_main( remaining_ms, 1 );
}
}
} // namespace
void
start_pack_mesh_loader() {
stop_pack_mesh_loader();
g_loader.running = true;
}
void
stop_pack_mesh_loader() {
g_loader.running = false;
std::lock_guard<std::mutex> lock { g_loader.mutex };
g_loader.queue = decltype( g_loader.queue )();
g_loader.queued.clear();
g_loader.ready.clear();
}
void
reset_pack_mesh_loader() {
stop_pack_mesh_loader();
}
void
request_pack_mesh_load( std::string const &model_file, int const priority ) {
if( false == g_loader.running ) {
return;
}
if( false == pack_mesh_path_valid( model_file ) ) {
return;
}
if( pack_mesh_globally_usable( model_file ) ) {
return;
}
enqueue_pack_mesh_load( model_file, priority );
}
void
request_pack_mesh_load_paths(
std::vector<std::string> const &model_files,
std::size_t const max_enqueue,
int const priority ) {
if( false == g_loader.running ) {
return;
}
std::size_t enqueued { 0 };
for( auto const &path : model_files ) {
if( max_enqueue > 0 && enqueued >= max_enqueue ) {
break;
}
if( false == pack_mesh_path_valid( path ) ) {
continue;
}
if( pack_mesh_globally_usable( path ) ) {
continue;
}
enqueue_pack_mesh_load( path, priority );
++enqueued;
}
}
[[nodiscard]] std::size_t
pump_pack_mesh_loader(
double const budget_ms,
std::size_t const max_loads ) {
return pump_pack_mesh_loader_main( budget_ms, max_loads );
}
[[nodiscard]] TModel3d *
session_cached_pack_mesh(
std::unordered_map<std::string, TModel3d *> const &session_cache,
std::string const &model_file ) {
auto const found { session_cache.find( model_file ) };
return found != session_cache.end() ? found->second : nullptr;
}
[[nodiscard]] TModel3d *
ensure_pack_mesh_in_session_cache(
std::string model_file,
std::unordered_map<std::string, TModel3d *> &session_cache,
PackMeshLoadWait const wait,
double const block_budget_ms ) {
if( false == pack_mesh_path_valid( model_file ) ) {
return nullptr;
}
replace_slashes( model_file );
if( pack_mesh_cached_usable( session_cache, model_file ) ) {
pack_bench_inc( &Eu7PackBench::stream_mesh_session_hit );
return session_cached_pack_mesh( session_cache, model_file );
}
if( session_cache.contains( model_file ) ) {
session_cache.erase( model_file );
}
if( try_consume_loader_ready( model_file, session_cache ) ) {
return session_cached_pack_mesh( session_cache, model_file );
}
if( sync_resolve_global_cached_mesh( model_file, session_cache ) ) {
return session_cached_pack_mesh( session_cache, model_file );
}
request_pack_mesh_load( model_file );
if( wait == PackMeshLoadWait::BlockUntilReady ) {
if( wait_for_loader_ready( model_file, block_budget_ms ) ) {
if( try_consume_loader_ready( model_file, session_cache ) ) {
return session_cached_pack_mesh( session_cache, model_file );
}
}
pack_bench_inc( &Eu7PackBench::stream_mesh_async_wait_timeout );
return nullptr;
}
return nullptr;
}
[[nodiscard]] std::size_t
pack_mesh_loader_queue_depth() {
std::lock_guard<std::mutex> lock { g_loader.mutex };
return g_loader.queued.size();
}
[[nodiscard]] bool
try_adopt_pack_mesh_for_slice(
std::string model_file,
std::unordered_map<std::string, TModel3d *> &session_cache ) {
if( model_file.empty() || model_file == "notload" ) {
return true;
}
replace_slashes( model_file );
if( pack_mesh_cached_usable( session_cache, model_file ) ) {
return true;
}
if( session_cache.contains( model_file ) ) {
session_cache.erase( model_file );
}
if( try_consume_loader_ready( model_file, session_cache ) ) {
return true;
}
return sync_resolve_global_cached_mesh( model_file, session_cache );
}
[[nodiscard]] std::size_t
drain_pack_mesh_loader_ready(
std::unordered_map<std::string, TModel3d *> &session_cache,
std::size_t const max_drain ) {
std::vector<std::pair<std::string, TModel3d *>> drained;
{
std::lock_guard<std::mutex> lock { g_loader.mutex };
drained.reserve( g_loader.ready.size() );
for( auto it { g_loader.ready.begin() }; it != g_loader.ready.end(); ) {
if( max_drain > 0 && drained.size() >= max_drain ) {
break;
}
drained.emplace_back( it->first, it->second );
it = g_loader.ready.erase( it );
}
}
std::size_t adopted { 0 };
for( auto & [path, mesh] : drained ) {
session_cache.emplace( std::move( path ), mesh );
pack_bench_inc( &Eu7PackBench::stream_mesh_async_ready );
pack_bench_inc( &Eu7PackBench::stream_mesh_async_drained );
if( pack_mesh_pointer_usable( mesh ) ) {
++adopted;
}
}
return adopted;
}
[[nodiscard]] std::size_t
pack_mesh_loader_ready_count() {
std::lock_guard<std::mutex> lock { g_loader.mutex };
return g_loader.ready.size();
}
[[nodiscard]] std::size_t
pack_mesh_loader_worker_count() {
return 0;
}
} // namespace scene::eu7

75
scene/eu7/eu7_pack_mesh_loader.h Normal file
View File

@@ -0,0 +1,75 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#pragma once
#include <cstddef>
#include <string>
#include <unordered_map>
#include <vector>
class TModel3d;
namespace scene::eu7 {
enum class PackMeshLoadWait {
NonBlocking,
BlockUntilReady,
};
void
start_pack_mesh_loader();
void
stop_pack_mesh_loader();
void
reset_pack_mesh_loader();
void
request_pack_mesh_load( std::string const &model_file, int priority = 0 );
void
request_pack_mesh_load_paths(
std::vector<std::string> const &model_files,
std::size_t max_enqueue = 0,
int priority = 0 );
[[nodiscard]] TModel3d *
ensure_pack_mesh_in_session_cache(
std::string model_file,
std::unordered_map<std::string, TModel3d *> &session_cache,
PackMeshLoadWait wait = PackMeshLoadWait::NonBlocking,
double block_budget_ms = 0.0 );
[[nodiscard]] bool
try_adopt_pack_mesh_for_slice(
std::string model_file,
std::unordered_map<std::string, TModel3d *> &session_cache );
[[nodiscard]] std::size_t
drain_pack_mesh_loader_ready(
std::unordered_map<std::string, TModel3d *> &session_cache,
std::size_t max_drain = 0 );
[[nodiscard]] std::size_t
pack_mesh_loader_queue_depth();
[[nodiscard]] std::size_t
pack_mesh_loader_ready_count();
[[nodiscard]] std::size_t
pack_mesh_loader_worker_count();
[[nodiscard]] std::size_t
pump_pack_mesh_loader(
double budget_ms = 0.0,
std::size_t max_loads = 1 );
} // namespace scene::eu7

491
scene/eu7/eu7_reader.cpp
View File

@@ -16,6 +16,7 @@ http://mozilla.org/MPL/2.0/.
#include <fstream>
#include <limits>
#include <memory>
#include <stdexcept>
namespace scene::eu7 {
@@ -70,6 +71,80 @@ private:
std::vector<std::string> strings_;
};
class PackByteReader {
public:
PackByteReader( std::uint8_t const *data, std::size_t const size )
: cur_ { data }
, end_ { data + size } {}
[[nodiscard]] bool
enough( std::size_t const bytes ) const noexcept {
return static_cast<std::size_t>( end_ - cur_ ) >= bytes;
}
[[nodiscard]] std::size_t
remaining() const noexcept {
return static_cast<std::size_t>( end_ - cur_ );
}
[[nodiscard]] std::uint8_t
u8() {
if( false == enough( 1 ) ) {
throw std::runtime_error( "EU7 PACK: przekroczono bufor sekcji" );
}
return *cur_++;
}
[[nodiscard]] std::uint16_t
u16() {
if( false == enough( 2 ) ) {
throw std::runtime_error( "EU7 PACK: przekroczono bufor sekcji" );
}
auto const lo { cur_[ 0 ] };
auto const hi { cur_[ 1 ] };
cur_ += 2;
return static_cast<std::uint16_t>( lo | ( static_cast<std::uint16_t>( hi ) << 8u ) );
}
[[nodiscard]] std::uint32_t
u32() {
if( false == enough( 4 ) ) {
throw std::runtime_error( "EU7 PACK: przekroczono bufor sekcji" );
}
std::uint32_t value { 0 };
std::memcpy( &value, cur_, 4 );
cur_ += 4;
return value;
}
[[nodiscard]] std::int32_t
i32() {
return static_cast<std::int32_t>( u32() );
}
[[nodiscard]] float
f32() {
std::uint32_t bits { u32() };
float value { 0.f };
std::memcpy( &value, &bits, sizeof( value ) );
return value;
}
[[nodiscard]] double
f64_disk() {
return static_cast<double>( f32() );
}
[[nodiscard]] glm::dvec3
vec3() {
return { f64_disk(), f64_disk(), f64_disk() };
}
private:
std::uint8_t const *cur_;
std::uint8_t const *end_;
};
[[nodiscard]] std::int16_t
read_i16( std::istream &input ) {
return static_cast<std::int16_t>( sn_utils::ld_uint16( input ) );
@@ -207,6 +282,97 @@ read_slim_node( std::istream &input, StringTable const &table, std::string_view
return node;
}
[[nodiscard]] Eu7TransformContext
read_transform_context_bytes( PackByteReader &input ) {
Eu7TransformContext transform;
const std::uint8_t origin_count { input.u8() };
transform.origin_stack.reserve( origin_count );
for( std::uint8_t i { 0 }; i < origin_count; ++i ) {
transform.origin_stack.push_back( input.vec3() );
}
const std::uint8_t scale_count { input.u8() };
transform.scale_stack.reserve( scale_count );
for( std::uint8_t i { 0 }; i < scale_count; ++i ) {
transform.scale_stack.push_back( input.vec3() );
}
transform.rotation = input.vec3();
transform.group_depth = input.u8();
return transform;
}
[[nodiscard]] Eu7BasicNode
read_slim_node_bytes(
PackByteReader &input,
StringTable const &table,
std::string_view const implied_type ) {
Eu7BasicNode node;
node.node_type = std::string( implied_type );
const std::uint8_t flags { input.u8() };
if( ( flags & kNodeFlagHasName ) != 0 ) {
node.name = table.resolve( input.u32() );
}
if( ( flags & kNodeFlagHasRangeMin ) != 0 ) {
node.range_squared_min = input.f64_disk();
}
if( ( flags & kNodeFlagHasRangeMax ) != 0 ) {
node.range_squared_max = input.f64_disk();
}
else {
node.range_squared_max = std::numeric_limits<double>::max();
}
if( ( flags & kNodeFlagHasBounds ) != 0 ) {
node.area.center = input.vec3();
node.area.radius = input.f32();
}
if( ( flags & kNodeFlagHasGroup ) != 0 ) {
node.group_valid = true;
node.group_handle = input.u32();
}
if( ( flags & kNodeFlagHasTransform ) != 0 ) {
node.transform = read_transform_context_bytes( input );
}
node.visible = ( flags & kNodeFlagNotVisible ) == 0;
return node;
}
[[nodiscard]] Eu7Model
read_runtime_model_bytes(
PackByteReader &input,
StringTable const &table,
Eu7PackSectionCursor const *cursor = nullptr ) {
Eu7Model model;
model.node = read_slim_node_bytes( input, table, "model" );
model.is_terrain = input.u8() != 0;
model.transition = input.u8() != 0;
model.location = input.vec3();
model.angles = input.vec3();
model.scale = input.vec3();
if(
cursor != nullptr &&
cursor->section_format == kPackSectionFormatV13 ) {
model.pack_mesh_index = input.u16();
model.pack_texture_index = input.u16();
}
else {
model.model_file = table.resolve( input.u32() );
model.texture_file = table.resolve( input.u32() );
}
const std::uint32_t light_count { input.u32() };
model.light_states.resize( light_count );
for( std::uint32_t i { 0 }; i < light_count; ++i ) {
model.light_states[ i ] = input.f32();
}
const std::uint32_t color_count { input.u32() };
model.light_colors.resize( color_count );
for( std::uint32_t i { 0 }; i < color_count; ++i ) {
model.light_colors[ i ] = input.u32();
}
if( cursor != nullptr && cursor->section_format == kPackSectionFormatV13 ) {
model.pack_cell_id = input.u8();
}
return model;
}
[[nodiscard]] std::string
read_length_string( std::istream &input ) {
const std::uint32_t length { sn_utils::ld_uint32( input ) };
@@ -413,7 +579,10 @@ read_runtime_lines( std::istream &input, StringTable const &table ) {
}
[[nodiscard]] Eu7Model
read_runtime_model( std::istream &input, StringTable const &table ) {
read_runtime_model(
std::istream &input,
StringTable const &table,
Eu7PackSectionCursor const *cursor = nullptr ) {
Eu7Model model;
model.node = read_slim_node( input, table, "model" );
model.is_terrain = sn_utils::d_uint8( input ) != 0;
@@ -421,8 +590,16 @@ read_runtime_model( std::istream &input, StringTable const &table ) {
model.location = read_vec3( input );
model.angles = read_vec3( input );
model.scale = read_vec3( input );
model.model_file = table.resolve( sn_utils::ld_uint32( input ) );
model.texture_file = table.resolve( sn_utils::ld_uint32( input ) );
if(
cursor != nullptr &&
cursor->section_format == kPackSectionFormatV13 ) {
model.pack_mesh_index = sn_utils::ld_uint16( input );
model.pack_texture_index = sn_utils::ld_uint16( input );
}
else {
model.model_file = table.resolve( sn_utils::ld_uint32( input ) );
model.texture_file = table.resolve( sn_utils::ld_uint32( input ) );
}
const std::uint32_t light_count { sn_utils::ld_uint32( input ) };
model.light_states.resize( light_count );
for( std::uint32_t i { 0 }; i < light_count; ++i ) {
@@ -433,11 +610,17 @@ read_runtime_model( std::istream &input, StringTable const &table ) {
for( std::uint32_t i { 0 }; i < color_count; ++i ) {
model.light_colors[ i ] = sn_utils::ld_uint32( input );
}
if( cursor != nullptr && cursor->section_format == kPackSectionFormatV13 ) {
model.pack_cell_id = sn_utils::d_uint8( input );
}
return model;
}
[[nodiscard]] Eu7ModelPrototype
read_runtime_prototype( std::istream &input, StringTable const &table ) {
read_runtime_prototype(
std::istream &input,
StringTable const &table,
std::uint32_t const file_version ) {
Eu7ModelPrototype proto;
proto.node = read_slim_node( input, table, "model" );
proto.is_terrain = sn_utils::d_uint8( input ) != 0;
@@ -454,16 +637,27 @@ read_runtime_prototype( std::istream &input, StringTable const &table ) {
for( std::uint32_t i { 0 }; i < color_count; ++i ) {
proto.light_colors[ i ] = sn_utils::ld_uint32( input );
}
if( file_version >= kEu7VersionV9 ) {
proto.resolved_texture = table.resolve( sn_utils::ld_uint32( input ) );
proto.pack_flags = sn_utils::d_uint8( input );
proto.textures_alpha = sn_utils::ld_uint32( input );
proto.baked_range_min = sn_utils::ld_float32( input );
proto.baked_range_max = sn_utils::ld_float32( input );
}
return proto;
}
void
read_prot_chunk( std::istream &input, StringTable const &strings, Eu7Module &module ) {
read_prot_chunk(
std::istream &input,
StringTable const &strings,
Eu7Module &module ) {
const std::uint32_t count { sn_utils::ld_uint32( input ) };
module.model_prototypes.clear();
module.model_prototypes.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
module.model_prototypes.push_back( read_runtime_prototype( input, strings ) );
module.model_prototypes.push_back(
read_runtime_prototype( input, strings, module.version ) );
}
}
@@ -483,10 +677,15 @@ expand_prototype_instance(
model.scale = scale;
model.model_file = proto.model_file;
model.texture_file = proto.texture_file;
model.resolved_texture = proto.resolved_texture;
model.light_states = proto.light_states;
model.light_colors = proto.light_colors;
model.transition = proto.transition;
model.is_terrain = proto.is_terrain;
model.pack_flags = proto.pack_flags;
model.textures_alpha = proto.textures_alpha;
model.baked_range_min = proto.baked_range_min;
model.baked_range_max = proto.baked_range_max;
return model;
}
@@ -506,6 +705,55 @@ parse_pack_section_header(
peek == EOF ? std::uint8_t { 0 } : static_cast<std::uint8_t>( peek ) };
bool use_v12_header { false };
if( first_byte == kPackSectionFormatV13 ) {
cursor.section_format = sn_utils::d_uint8( input );
const std::uint32_t solo_total { sn_utils::ld_uint32( input ) };
const std::uint32_t inst_total { sn_utils::ld_uint32( input ) };
const std::uint32_t unique_mesh_count { sn_utils::ld_uint32( input ) };
if(
solo_total + inst_total == entry.model_count &&
( inst_total == 0 || false == module.model_prototypes.empty() ) ) {
use_v12_header = true;
cursor.model_total = solo_total + inst_total;
cursor.solo_remaining = solo_total;
cursor.inst_remaining = inst_total;
StringTable const strings { module.strings };
cursor.unique_meshes.reserve( unique_mesh_count );
for( std::uint32_t mesh_idx { 0 }; mesh_idx < unique_mesh_count; ++mesh_idx ) {
cursor.unique_meshes.push_back(
strings.resolve( sn_utils::ld_uint32( input ) ) );
}
const std::uint32_t unique_texture_count { sn_utils::ld_uint32( input ) };
cursor.unique_textures.reserve( unique_texture_count );
for( std::uint32_t tex_idx { 0 }; tex_idx < unique_texture_count; ++tex_idx ) {
cursor.unique_textures.push_back(
strings.resolve( sn_utils::ld_uint32( input ) ) );
}
cursor.chunk_count = sn_utils::ld_uint32( input );
cursor.chunk_model_counts.resize( cursor.chunk_count );
cursor.chunk_inst_counts.resize( cursor.chunk_count );
cursor.chunk_byte_offsets.resize( cursor.chunk_count );
for( std::uint32_t chunk_idx { 0 }; chunk_idx < cursor.chunk_count; ++chunk_idx ) {
cursor.chunk_model_counts[ chunk_idx ] = sn_utils::ld_uint32( input );
cursor.chunk_inst_counts[ chunk_idx ] = sn_utils::ld_uint32( input );
cursor.chunk_byte_offsets[ chunk_idx ] = sn_utils::ld_uint32( input );
}
}
if( false == use_v12_header ) {
input.seekg( section_start );
}
}
if( use_v12_header ) {
cursor.models_read = 0;
cursor.header_parsed = true;
return;
}
use_v12_header = false;
if( first_byte == kPackSectionFormatV12 ) {
cursor.section_format = sn_utils::d_uint8( input );
const std::uint32_t solo_total { sn_utils::ld_uint32( input ) };
@@ -709,7 +957,7 @@ read_pack_models_chunk_impl(
models.size() < max_count &&
( cursor.solo_remaining > 0 || cursor.inst_remaining > 0 ) ) {
if( cursor.solo_remaining > 0 ) {
auto model { read_runtime_model( input, strings ) };
auto model { read_runtime_model( input, strings, &cursor ) };
model.node.transform = {};
models.push_back( std::move( model ) );
--cursor.solo_remaining;
@@ -725,12 +973,75 @@ read_pack_models_chunk_impl(
auto const angles { read_vec3( input ) };
auto const scale { read_vec3( input ) };
auto const name { strings.resolve( sn_utils::ld_uint32( input ) ) };
models.push_back( expand_prototype_instance(
auto model { expand_prototype_instance(
module.model_prototypes[ proto_id ],
location,
angles,
scale,
name ) );
name ) };
if( cursor.section_format == kPackSectionFormatV13 ) {
model.pack_cell_id = sn_utils::d_uint8( input );
}
models.push_back( std::move( model ) );
--cursor.inst_remaining;
}
++cursor.models_read;
}
return models;
}
[[nodiscard]] std::vector<Eu7Model>
read_pack_models_chunk_from_bytes(
Eu7Module const &module,
std::uint8_t const *data,
std::size_t const size,
Eu7PackSectionCursor &cursor,
std::size_t const max_count,
StringTable const &strings ) {
if( false == cursor.header_parsed || max_count == 0 || data == nullptr || size == 0 ) {
return {};
}
PackByteReader input { data, size };
std::vector<Eu7Model> models;
auto const remaining {
std::min(
static_cast<std::size_t>( cursor.solo_remaining ) +
static_cast<std::size_t>( cursor.inst_remaining ),
static_cast<std::size_t>( cursor.model_total ) ) };
models.reserve( std::min( max_count, remaining ) );
while(
models.size() < max_count &&
( cursor.solo_remaining > 0 || cursor.inst_remaining > 0 ) ) {
if( cursor.solo_remaining > 0 ) {
auto model { read_runtime_model_bytes( input, strings, &cursor ) };
model.node.transform = {};
models.push_back( std::move( model ) );
--cursor.solo_remaining;
}
else {
const std::uint32_t proto_id { input.u32() };
if( proto_id >= module.model_prototypes.size() ) {
throw std::runtime_error(
"EU7 PACK: proto_id " + std::to_string( proto_id ) + " poza zakresem PROT (" +
std::to_string( module.model_prototypes.size() ) + ")" );
}
auto const location { input.vec3() };
auto const angles { input.vec3() };
auto const scale { input.vec3() };
auto const name { strings.resolve( input.u32() ) };
auto model { expand_prototype_instance(
module.model_prototypes[ proto_id ],
location,
angles,
scale,
name ) };
if( cursor.section_format == kPackSectionFormatV13 ) {
model.pack_cell_id = input.u8();
}
models.push_back( std::move( model ) );
--cursor.inst_remaining;
}
++cursor.models_read;
@@ -1166,6 +1477,7 @@ struct PackSectionReadSession {
int column { -1 };
Eu7PackIndexEntry entry {};
Eu7PackSectionCursor header {};
std::shared_ptr<Eu7PackSectionPathTables const> path_tables;
std::ifstream input;
bool header_ready { false };
};
@@ -1180,6 +1492,55 @@ reset_pack_section_read_session( PackSectionReadSession &session ) {
session = {};
}
[[nodiscard]] std::uint64_t
pack_section_byte_size(
Eu7Module const &module,
Eu7PackIndexEntry const &entry ) {
std::uint64_t next_offset { module.pack_catalog.pack_payload_size };
for( auto const &candidate : module.pack_catalog.entries ) {
if(
candidate.pack_offset > entry.pack_offset &&
candidate.pack_offset < next_offset ) {
next_offset = candidate.pack_offset;
}
}
return next_offset - entry.pack_offset;
}
[[nodiscard]] std::optional<std::uint32_t>
pack_chunk_byte_size(
Eu7PackSectionCursor const &header,
std::uint32_t const chunk_index,
std::uint64_t const section_byte_size ) {
if( header.chunk_byte_offsets.empty() || chunk_index >= header.chunk_count ) {
return std::nullopt;
}
auto const start { header.chunk_byte_offsets[ chunk_index ] };
if( chunk_index + 1 < header.chunk_count ) {
return header.chunk_byte_offsets[ chunk_index + 1 ] - start;
}
if( section_byte_size > start ) {
return static_cast<std::uint32_t>( section_byte_size - start );
}
return std::nullopt;
}
void
ensure_section_path_tables(
Eu7PackSectionCursor const &header,
PackSectionReadSession &session ) {
if(
header.unique_meshes.empty() &&
header.unique_textures.empty() ) {
session.path_tables.reset();
return;
}
auto tables { std::make_shared<Eu7PackSectionPathTables>() };
tables->unique_meshes = header.unique_meshes;
tables->unique_textures = header.unique_textures;
session.path_tables = std::move( tables );
}
void
ensure_pack_file( Eu7Module const &module, PackSectionReadSession &session ) {
if( module.source_path == session.source_path && session.input.is_open() ) {
@@ -1216,7 +1577,9 @@ ensure_section_header(
session.row = row;
session.column = column;
session.entry = entry;
session.path_tables.reset();
parse_pack_section_header( module, session.input, entry, session.header );
ensure_section_path_tables( session.header, session );
session.header_ready = true;
}
@@ -1254,23 +1617,25 @@ read_pack_section_chunk_load_impl(
}
result.chunk_index = chunk_index;
if( chunk_index == 0 ) {
result.unique_meshes = header.unique_meshes;
result.unique_textures = header.unique_textures;
}
Eu7PackSectionCursor chunk_cursor { header };
auto const section_byte_size { pack_section_byte_size( module, *entry ) };
std::optional<std::uint32_t> chunk_byte_size;
if(
( header.section_format == kPackSectionFormatV10 ||
header.section_format == kPackSectionFormatV11 ||
header.section_format == kPackSectionFormatV12 ) &&
header.section_format == kPackSectionFormatV12 ||
header.section_format == kPackSectionFormatV13 ) &&
false == header.chunk_byte_offsets.empty() ) {
auto const section_start {
static_cast<std::streamoff>( module.pack_payload_offset + entry->pack_offset ) };
chunk_byte_size = pack_chunk_byte_size( header, chunk_index, section_byte_size );
session.input.seekg(
section_start +
static_cast<std::streamoff>( header.chunk_byte_offsets[ chunk_index ] ) );
if( header.section_format == kPackSectionFormatV12 ) {
if(
header.section_format == kPackSectionFormatV12 ||
header.section_format == kPackSectionFormatV13 ) {
chunk_cursor.solo_remaining = header.chunk_model_counts[ chunk_index ];
chunk_cursor.inst_remaining = header.chunk_inst_counts[ chunk_index ];
}
@@ -1283,12 +1648,35 @@ read_pack_section_chunk_load_impl(
return result;
}
result.models = read_pack_models_chunk_impl(
module,
session.input,
chunk_cursor,
std::numeric_limits<std::size_t>::max(),
strings );
if( chunk_byte_size.has_value() && *chunk_byte_size > 0 ) {
std::vector<std::uint8_t> chunk_bytes( *chunk_byte_size );
session.input.read(
reinterpret_cast<char *>( chunk_bytes.data() ),
static_cast<std::streamsize>( chunk_bytes.size() ) );
if(
static_cast<std::size_t>( session.input.gcount() ) != chunk_bytes.size() ) {
throw std::runtime_error( "EU7 PACK: niepelny odczyt chunka sekcji" );
}
result.models = read_pack_models_chunk_from_bytes(
module,
chunk_bytes.data(),
chunk_bytes.size(),
chunk_cursor,
std::numeric_limits<std::size_t>::max(),
strings );
}
else {
result.models = read_pack_models_chunk_impl(
module,
session.input,
chunk_cursor,
std::numeric_limits<std::size_t>::max(),
strings );
}
result.path_tables = session.path_tables;
if( chunk_index == 0 && false == header.unique_textures.empty() ) {
result.unique_textures = header.unique_textures;
}
return result;
}
@@ -1320,7 +1708,8 @@ read_pack_section_load_impl( Eu7Module const &module, int const row, int const c
if(
( header.section_format == kPackSectionFormatV10 ||
header.section_format == kPackSectionFormatV11 ||
header.section_format == kPackSectionFormatV12 ) &&
header.section_format == kPackSectionFormatV12 ||
header.section_format == kPackSectionFormatV13 ) &&
header.chunk_count > 0 ) {
result.models.reserve( entry->model_count );
for( std::uint32_t chunk_idx { 0 }; chunk_idx < header.chunk_count; ++chunk_idx ) {
@@ -1361,7 +1750,7 @@ is_valid_eu7b_file( std::string const &path ) {
const std::uint32_t version { sn_utils::ld_uint32( input ) };
return (
version == kEu7VersionV4 || version == kEu7VersionV5 || version == kEu7VersionV6 ||
version == kEu7VersionV7 || version == kEu7VersionV8 );
version == kEu7VersionV7 || version == kEu7VersionV8 || version == kEu7VersionV9 );
}
Eu7Module
@@ -1380,7 +1769,7 @@ read_module( std::string const &path ) {
if(
module.version != kEu7VersionV4 && module.version != kEu7VersionV5 &&
module.version != kEu7VersionV6 && module.version != kEu7VersionV7 &&
module.version != kEu7VersionV8 ) {
module.version != kEu7VersionV8 && module.version != kEu7VersionV9 ) {
throw std::runtime_error(
"EU7B: nieobslugiwana wersja " + std::to_string( module.version ) + " w \"" + path + "\"" );
}
@@ -1444,4 +1833,58 @@ reset_pack_section_read_cache() {
reset_pack_section_read_session( g_pack_section_read_session );
}
void
for_each_pack_section_unique_mesh(
Eu7Module const &module,
int const row,
int const column,
std::function<void( std::string const & )> const &visit ) {
if( false == module.has_pack_chunk || module.source_path.empty() ) {
return;
}
auto const entry { find_pack_entry_impl( module, row, column ) };
if( false == entry.has_value() || entry->model_count == 0 ) {
return;
}
auto &session { g_pack_section_read_session };
ensure_pack_file( module, session );
if( !session.input ) {
return;
}
ensure_section_header( module, row, column, *entry, session );
for( auto const &path : session.header.unique_meshes ) {
visit( path );
}
}
void
for_each_pack_section_unique_texture(
Eu7Module const &module,
int const row,
int const column,
std::function<void( std::string const & )> const &visit ) {
if( false == module.has_pack_chunk || module.source_path.empty() ) {
return;
}
auto const entry { find_pack_entry_impl( module, row, column ) };
if( false == entry.has_value() || entry->model_count == 0 ) {
return;
}
auto &session { g_pack_section_read_session };
ensure_pack_file( module, session );
if( !session.input ) {
return;
}
ensure_section_header( module, row, column, *entry, session );
for( auto const &path : session.header.unique_textures ) {
visit( path );
}
}
} // namespace scene::eu7

18
scene/eu7/eu7_reader.h
View File

@@ -11,7 +11,7 @@ http://mozilla.org/MPL/2.0/.
#include "scene/eu7/eu7_types.h"
#include <optional>
#include <functional>
#include <string>
#include <vector>
@@ -47,4 +47,20 @@ read_pack_section_chunk_load(
void
reset_pack_section_read_cache();
// UMES z naglowka sekcji — bez kopiowania calego wektora.
void
for_each_pack_section_unique_mesh(
Eu7Module const &Module,
int Row,
int Column,
std::function<void( std::string const & )> const &Visit );
// UTEX z naglowka sekcji (v11+) — bez skanowania instancji.
void
for_each_pack_section_unique_texture(
Eu7Module const &Module,
int Row,
int Column,
std::function<void( std::string const & )> const &Visit );
} // namespace scene::eu7

1759
scene/eu7/eu7_section_stream.cpp
View File

File diff suppressed because it is too large Load Diff

19
scene/eu7/eu7_section_stream.h
View File

@@ -19,8 +19,15 @@ class state_serializer;
namespace scene::eu7 {
constexpr int kSectionStreamBootstrapRadiusKm { 4 };
constexpr int kSectionStreamGameplayRadiusKm { 4 };
// Fazy streamingu PACK (eu7_section_stream.cpp):
// 1) loading_screen — minimalny bootstrap (~2–3 km) podczas ekranu ladowania;
// 2) radius_fill — po dismiss: lekki, ciagly drain az do kSectionStreamTargetRadiusKm;
// 3) maintenance — po zapelnieniu 20 km: wolny lookahead w kierunku lotu.
constexpr int kSectionStreamLoadingDismissRadiusKm { 1 };
constexpr int kSectionStreamPresentableRadiusKm { 2 };
constexpr int kSectionStreamBootstrapRadiusKm { 3 };
constexpr int kSectionStreamGameplayRadiusKm { 5 };
constexpr int kSectionStreamTargetRadiusKm { 20 };
void
init_section_stream(
@@ -65,15 +72,15 @@ kick_section_stream_bootstrap();
void
try_bootstrap_section_stream();
// Po zaladowaniu scenariusza: bootstrap + drain PACK przed wejsciem w jazde.
// Po zaladowaniu scenariusza: kick bootstrap + opcjonalny krotki drain (0 = tylko async).
void
preload_section_stream( double MaxDrainMs = 300.0 );
preload_section_stream( double MaxDrainMs = 0.0 );
// Wszystkie sekcje PACK w pierścieniu RadiusKm wokol pozycji wczytane i zaaplikowane.
[[nodiscard]] bool
section_stream_ready_around(
glm::dvec3 const &WorldPosition,
int RadiusKm = kSectionStreamBootstrapRadiusKm );
int RadiusKm = kSectionStreamTargetRadiusKm );
// ready_around stabilnie przez kilka klatek — bez migania overlay/sceny.
[[nodiscard]] bool
@@ -86,7 +93,7 @@ section_stream_presentable_around(
[[nodiscard]] bool
loading_screen_blocks_world(
glm::dvec3 const &WorldPosition,
int RadiusKm = kSectionStreamBootstrapRadiusKm );
int RadiusKm = kSectionStreamLoadingDismissRadiusKm );
[[nodiscard]] bool
loading_screen_dismissed();

126
scene/eu7/eu7_types.h
View File

@@ -11,8 +11,10 @@ http://mozilla.org/MPL/2.0/.
#include <cstdint>
#include <limits>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <unordered_map>
#include <utility>
#include <vector>
@@ -190,6 +192,8 @@ struct Eu7Lines {
std::vector<Eu7WorldVertex> vertices;
};
constexpr std::uint16_t kEu7PackIndexEmpty { 0xFFFFu };
struct Eu7Model {
Eu7BasicNode node;
glm::dvec3 location{};
@@ -197,18 +201,52 @@ struct Eu7Model {
glm::dvec3 scale{ 1.0, 1.0, 1.0 };
std::string model_file;
std::string texture_file;
std::string resolved_texture;
std::vector<float> light_states;
std::vector<std::uint32_t> light_colors;
bool transition = true;
bool is_terrain = false;
std::uint8_t pack_flags { 0 };
std::uint32_t textures_alpha { 0 };
float baked_range_min { -1.f };
float baked_range_max { -1.f };
std::uint8_t pack_cell_id { 255 };
std::uint16_t pack_mesh_index { kEu7PackIndexEmpty };
std::uint16_t pack_texture_index { kEu7PackIndexEmpty };
};
constexpr std::uint8_t kEu7PackFlagNeedsFullLoad { 1u << 0 };
constexpr std::uint8_t kEu7PackFlagInstanceableHint { 1u << 1 };
constexpr std::uint8_t kEu7PackCellIdInvalid { 255u };
[[nodiscard]] inline bool
eu7_pack_model_needs_full_load( Eu7Model const &model ) {
if( model.pack_flags & kEu7PackFlagNeedsFullLoad ) {
return true;
}
if( model.pack_flags != 0 ) {
return false;
}
return false == model.light_states.empty()
|| false == model.light_colors.empty()
|| false == model.transition;
}
[[nodiscard]] inline std::string
pack_nodedata_cache_key( Eu7Model const &model ) {
return model.model_file + '\x1f' + model.texture_file + '\x1f'
+ std::to_string( model.node.range_squared_min ) + '\x1f'
+ std::to_string( model.node.range_squared_max ) + '\x1f'
+ ( model.is_terrain ? '1' : '0' );
std::string key;
key.reserve(
model.model_file.size() + model.texture_file.size() + 48 );
key.append( model.model_file );
key.push_back( '\x1f' );
key.append( model.texture_file );
key.push_back( '\x1f' );
key.append( std::to_string( model.node.range_squared_min ) );
key.push_back( '\x1f' );
key.append( std::to_string( model.node.range_squared_max ) );
key.push_back( '\x1f' );
key.push_back( model.is_terrain ? '1' : '0' );
return key;
}
// Wspolna definicja modelu (EU7B v8 PROT) — bez transformacji instancji.
@@ -216,10 +254,15 @@ struct Eu7ModelPrototype {
Eu7BasicNode node;
std::string model_file;
std::string texture_file;
std::string resolved_texture;
std::vector<float> light_states;
std::vector<std::uint32_t> light_colors;
bool transition = true;
bool is_terrain = false;
std::uint8_t pack_flags { 0 };
std::uint32_t textures_alpha { 0 };
float baked_range_min { -1.f };
float baked_range_max { -1.f };
};
struct Eu7MemCell {
@@ -390,14 +433,85 @@ struct Eu7PackSectionLoad {
std::vector<std::string> unique_textures;
};
struct Eu7PackSectionChunkLoad {
std::vector<Eu7Model> models;
struct Eu7PackSectionPathTables {
std::vector<std::string> unique_meshes;
std::vector<std::string> unique_textures;
};
struct Eu7PackSectionChunkLoad {
std::vector<Eu7Model> models;
std::vector<std::string> unique_textures;
std::shared_ptr<Eu7PackSectionPathTables const> path_tables;
std::uint32_t chunk_count { 1 };
std::uint32_t chunk_index { 0 };
};
[[nodiscard]] inline std::string_view
pack_model_mesh_path(
Eu7Model const &model,
Eu7PackSectionPathTables const *path_tables ) {
if( false == model.model_file.empty() ) {
return model.model_file;
}
if(
path_tables != nullptr &&
model.pack_mesh_index != kEu7PackIndexEmpty &&
model.pack_mesh_index < path_tables->unique_meshes.size() ) {
return path_tables->unique_meshes[ model.pack_mesh_index ];
}
return {};
}
[[nodiscard]] inline std::string_view
pack_model_texture_path(
Eu7Model const &model,
Eu7PackSectionPathTables const *path_tables ) {
if( false == model.texture_file.empty() ) {
return model.texture_file;
}
if(
path_tables != nullptr &&
model.pack_texture_index != kEu7PackIndexEmpty &&
model.pack_texture_index < path_tables->unique_textures.size() ) {
return path_tables->unique_textures[ model.pack_texture_index ];
}
return {};
}
inline void
resolve_pack_model_paths(
Eu7Model &model,
Eu7PackSectionPathTables const *path_tables ) {
if( path_tables == nullptr ) {
return;
}
if( model.model_file.empty() && model.pack_mesh_index != kEu7PackIndexEmpty ) {
if( model.pack_mesh_index < path_tables->unique_meshes.size() ) {
model.model_file = path_tables->unique_meshes[ model.pack_mesh_index ];
}
model.pack_mesh_index = kEu7PackIndexEmpty;
}
if( model.texture_file.empty() && model.pack_texture_index != kEu7PackIndexEmpty ) {
if( model.pack_texture_index < path_tables->unique_textures.size() ) {
model.texture_file = path_tables->unique_textures[ model.pack_texture_index ];
}
model.pack_texture_index = kEu7PackIndexEmpty;
}
}
inline void
resolve_pack_model_paths(
Eu7Model *models,
std::size_t const count,
Eu7PackSectionPathTables const *path_tables ) {
if( models == nullptr || path_tables == nullptr ) {
return;
}
for( std::size_t i { 0 }; i < count; ++i ) {
resolve_pack_model_paths( models[ i ], path_tables );
}
}
struct Eu7PackCatalog {
std::vector<Eu7PackIndexEntry> entries;
std::unordered_map<std::uint32_t, std::size_t> index_by_section;