mirror of
https://github.com/MaSzyna-EU07/maszyna.git
synced 2026-07-21 14:49:19 +02:00
EU7 scenery format and models streaming
This commit is contained in:
maj00r
587
scene/eu7/eu7_loader.cpp
Normal file
587
scene/eu7/eu7_loader.cpp
Normal file
@@ -0,0 +1,587 @@
|
||||
/*
|
||||
This Source Code Form is subject to the
|
||||
terms of the Mozilla Public License, v.
|
||||
2.0. If a copy of the MPL was not
|
||||
distributed with this file, You can
|
||||
obtain one at
|
||||
http://mozilla.org/MPL/2.0/.
|
||||
*/
|
||||
|
||||
#include "stdafx.h"
|
||||
#include "scene/eu7/eu7_loader.h"
|
||||
|
||||
#include "scene/eu7/eu7_apply.h"
|
||||
#include "scene/eu7/eu7_emit.h"
|
||||
#include "scene/eu7/eu7_load_stats.h"
|
||||
#include "scene/eu7/eu7_pack_bench.h"
|
||||
#include "scene/eu7/eu7_reader.h"
|
||||
#include "scene/eu7/eu7_section_stream.h"
|
||||
#include "scene/eu7/eu7_parameters.h"
|
||||
#include "scene/eu7/eu7_transform.h"
|
||||
#include "scene/scene.h"
|
||||
#include "scene/scenenode.h"
|
||||
#include "rendering/renderer.h"
|
||||
#include "simulation/simulation.h"
|
||||
#include "simulation/simulationstateserializer.h"
|
||||
#include "scene/sn_utils.h"
|
||||
#include "utilities/Globals.h"
|
||||
#include "utilities/Logs.h"
|
||||
#include "utilities/utilities.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <limits>
|
||||
#include <stdexcept>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
|
||||
namespace scene::eu7 {
|
||||
|
||||
namespace {
|
||||
|
||||
std::unordered_map<std::string, Eu7Module> g_module_file_cache;
|
||||
bool g_packed_root_active { false };
|
||||
|
||||
void
|
||||
apply_material( shape_node::shapenode_data &data, std::string material_name ) {
|
||||
replace_slashes( material_name );
|
||||
data.material = GfxRenderer->Fetch_Material( material_name );
|
||||
|
||||
auto const texturehandle {
|
||||
data.material != null_handle ?
|
||||
GfxRenderer->Material( data.material )->GetTexture( 0 ) :
|
||||
null_handle };
|
||||
auto const &texture {
|
||||
texturehandle ?
|
||||
GfxRenderer->Texture( texturehandle ) :
|
||||
*ITexture::null_texture() };
|
||||
if( texturehandle != null_handle ) {
|
||||
data.translucent = (
|
||||
contains( texture.get_name(), '@' ) && texture.get_has_alpha() );
|
||||
}
|
||||
else {
|
||||
data.translucent = false;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
finalize_shape_bounds( shape_node &shape, shape_node::shapenode_data &data ) {
|
||||
if( data.vertices.empty() ) {
|
||||
return;
|
||||
}
|
||||
data.area.center = glm::dvec3( 0.0 );
|
||||
for( auto const &vertex : data.vertices ) {
|
||||
data.area.center += vertex.position;
|
||||
}
|
||||
data.area.center /=
|
||||
static_cast<double>( data.vertices.size() );
|
||||
shape.invalidate_radius();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
shape_node
|
||||
build_shape_node( Eu7Shape const &source ) {
|
||||
shape_node shape;
|
||||
auto &data { shape.m_data };
|
||||
data.rangesquared_min = source.node.range_squared_min;
|
||||
data.rangesquared_max = source.node.range_squared_max;
|
||||
data.visible = source.node.visible;
|
||||
data.origin = source.origin;
|
||||
data.lighting.diffuse = glm::vec4(
|
||||
source.lighting.diffuse.x, source.lighting.diffuse.y,
|
||||
source.lighting.diffuse.z, source.lighting.diffuse.w );
|
||||
data.lighting.ambient = glm::vec4(
|
||||
source.lighting.ambient.x, source.lighting.ambient.y,
|
||||
source.lighting.ambient.z, source.lighting.ambient.w );
|
||||
data.lighting.specular = glm::vec4(
|
||||
source.lighting.specular.x, source.lighting.specular.y,
|
||||
source.lighting.specular.z, source.lighting.specular.w );
|
||||
if( source.translucent ) {
|
||||
data.translucent = true;
|
||||
}
|
||||
apply_material( data, source.material_path );
|
||||
data.vertices.resize( source.vertices.size() );
|
||||
for( std::size_t i { 0 }; i < source.vertices.size(); ++i ) {
|
||||
auto const &src { source.vertices[ i ] };
|
||||
world_vertex &dst { data.vertices[ i ] };
|
||||
dst.position = src.position;
|
||||
dst.normal = src.normal;
|
||||
dst.texture = glm::vec2( static_cast<float>( src.u ), static_cast<float>( src.v ) );
|
||||
}
|
||||
finalize_shape_bounds( shape, data );
|
||||
return shape;
|
||||
}
|
||||
|
||||
lines_node
|
||||
build_lines_node( Eu7Lines const &source ) {
|
||||
lines_node node;
|
||||
auto &data { node.m_data };
|
||||
data.rangesquared_min = source.node.range_squared_min;
|
||||
data.rangesquared_max = source.node.range_squared_max;
|
||||
data.visible = source.node.visible;
|
||||
data.line_width = source.line_width;
|
||||
data.lighting.diffuse = glm::vec4(
|
||||
source.lighting.diffuse.x, source.lighting.diffuse.y,
|
||||
source.lighting.diffuse.z, source.lighting.diffuse.w );
|
||||
data.lighting.ambient = glm::vec4(
|
||||
source.lighting.ambient.x, source.lighting.ambient.y,
|
||||
source.lighting.ambient.z, source.lighting.ambient.w );
|
||||
data.lighting.specular = glm::vec4(
|
||||
source.lighting.specular.x, source.lighting.specular.y,
|
||||
source.lighting.specular.z, source.lighting.specular.w );
|
||||
data.origin = source.origin;
|
||||
data.vertices.resize( source.vertices.size() );
|
||||
for( std::size_t i { 0 }; i < source.vertices.size(); ++i ) {
|
||||
data.vertices[ i ].position = source.vertices[ i ].position;
|
||||
}
|
||||
node.m_name = source.node.name;
|
||||
return node;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr std::uint32_t kMagic { MAKE_ID4( 'E', 'U', '7', 'B' ) };
|
||||
constexpr std::uint32_t kVersionV4 { 4 };
|
||||
constexpr std::uint32_t kVersionV5 { 5 };
|
||||
constexpr std::uint32_t kVersionV6 { 6 };
|
||||
constexpr std::uint32_t kVersionV7 { 7 };
|
||||
constexpr std::uint32_t kVersionV8 { 8 };
|
||||
constexpr std::uint32_t kChunkTerr { MAKE_ID4( 'T', 'E', 'R', 'R' ) };
|
||||
|
||||
std::unordered_set<std::string> g_load_session;
|
||||
|
||||
[[nodiscard]] std::string
|
||||
scenery_relative_file( std::string const &resolved ) {
|
||||
if( resolved.starts_with( Global.asCurrentSceneryPath ) ) {
|
||||
return resolved.substr( Global.asCurrentSceneryPath.size() );
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::string
|
||||
include_binary_path( Eu7Include const &include ) {
|
||||
if( !include.binary_path.empty() ) {
|
||||
return resolve_scenery_path( include.binary_path );
|
||||
}
|
||||
if( !include.source_path.empty() ) {
|
||||
return binary_path( include.source_path );
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
void
|
||||
evict_non_pack_modules_from_cache( std::string const &pack_root_resolved ) {
|
||||
for( auto it { g_module_file_cache.begin() }; it != g_module_file_cache.end(); ) {
|
||||
if( it->first != pack_root_resolved ) {
|
||||
it = g_module_file_cache.erase( it );
|
||||
}
|
||||
else {
|
||||
++it;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
load_module_recursive(
|
||||
std::string const &path,
|
||||
simulation::state_serializer &serializer,
|
||||
std::unordered_set<std::string> &loaded,
|
||||
Eu7TransformContext const &include_prefix = {},
|
||||
std::vector<std::string> const &include_parameters = {} ) {
|
||||
++load_stats().module_visits;
|
||||
|
||||
auto const resolved { resolve_scenery_path( path ) };
|
||||
auto const load_key { module_load_key( resolved, include_parameters ) };
|
||||
if( loaded.contains( load_key ) ) {
|
||||
++load_stats().module_deduped;
|
||||
return true;
|
||||
}
|
||||
if( false == probe_file( resolved ) ) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Eu7Module const *template_module { nullptr };
|
||||
Eu7Module owned_module;
|
||||
try {
|
||||
auto const cached { g_module_file_cache.find( resolved ) };
|
||||
if( cached != g_module_file_cache.end() ) {
|
||||
++load_stats().module_cache_hit;
|
||||
template_module = &cached->second;
|
||||
}
|
||||
else {
|
||||
ScopedTimer const read_timer { load_stats().read_ms };
|
||||
owned_module = read_module( resolved );
|
||||
++load_stats().module_read;
|
||||
auto const emplaced { g_module_file_cache.emplace( resolved, std::move( owned_module ) ) };
|
||||
template_module = &emplaced.first->second;
|
||||
}
|
||||
}
|
||||
catch( std::exception const &ex ) {
|
||||
ErrorLog( std::string{ "EU7: blad odczytu \"" + resolved + "\": " } + ex.what() );
|
||||
return false;
|
||||
}
|
||||
|
||||
if( g_packed_root_active && is_model_only_module( *template_module ) ) {
|
||||
loaded.insert( load_key );
|
||||
++load_stats().pack_skipped_includes;
|
||||
return true;
|
||||
}
|
||||
|
||||
loaded.insert( load_key );
|
||||
++load_stats().module_applied;
|
||||
|
||||
if(
|
||||
false == g_packed_root_active &&
|
||||
include_parameters.empty() &&
|
||||
transform_is_empty( include_prefix ) &&
|
||||
template_module->has_pack_chunk ) {
|
||||
g_packed_root_active = true;
|
||||
}
|
||||
|
||||
std::vector<Eu7Include> fallback_includes;
|
||||
for( auto const &include : template_module->includes ) {
|
||||
auto const child { include_binary_path( include ) };
|
||||
auto const ref {
|
||||
include.source_path.empty() ? include.binary_path : include.source_path };
|
||||
bool child_loaded { false };
|
||||
if( !child.empty() && should_use_binary_module( ref ) ) {
|
||||
child_loaded = load_module_recursive(
|
||||
child,
|
||||
serializer,
|
||||
loaded,
|
||||
include.site_transform,
|
||||
include.parameters );
|
||||
}
|
||||
if( child_loaded ) {
|
||||
continue;
|
||||
}
|
||||
if( !child.empty() && probe_file( child ) && false == should_use_binary_module( ref ) ) {
|
||||
WriteLog(
|
||||
"EU7: przestarzaly .eu7, fallback SCM: " + include_text_path( include ) );
|
||||
}
|
||||
else if( !child.empty() ) {
|
||||
WriteLog(
|
||||
"EU7 include niedostepny, fallback SCM: " + include_text_path( include ) );
|
||||
}
|
||||
fallback_includes.push_back( include );
|
||||
}
|
||||
|
||||
scene::scratch_data scratch;
|
||||
auto const current_relative { scenery_relative_file( resolved ) };
|
||||
if( false == fallback_includes.empty() ) {
|
||||
ScopedTimer const scm_timer { load_stats().scm_fallback_ms };
|
||||
load_stats().scm_fallback += fallback_includes.size();
|
||||
for( auto const &include : fallback_includes ) {
|
||||
serializer.deserialize_include_file(
|
||||
include_text_path( include ),
|
||||
current_relative,
|
||||
include.parameters,
|
||||
scratch );
|
||||
}
|
||||
}
|
||||
|
||||
if(
|
||||
fallback_includes.empty() &&
|
||||
is_model_only_module( *template_module ) ) {
|
||||
++load_stats().model_fast_path;
|
||||
|
||||
std::vector<Eu7Model> models { template_module->scene.models };
|
||||
{
|
||||
ScopedTimer const place_timer { load_stats().place_fast_ms };
|
||||
if( false == include_parameters.empty() ) {
|
||||
apply_include_parameters_to_models( models, include_parameters );
|
||||
}
|
||||
if( false == transform_is_empty( include_prefix ) ) {
|
||||
compose_models_with_prefix( models, include_prefix );
|
||||
}
|
||||
apply_include_placement_to_models(
|
||||
models, template_module->include_placement, include_parameters );
|
||||
}
|
||||
serializer.apply_eu7_models( models, scratch );
|
||||
}
|
||||
else {
|
||||
++load_stats().module_full_path;
|
||||
|
||||
Eu7Module module { *template_module };
|
||||
{
|
||||
ScopedTimer const place_timer { load_stats().place_full_ms };
|
||||
if( false == include_parameters.empty() ) {
|
||||
apply_include_parameters_to_scene( module.scene, include_parameters );
|
||||
}
|
||||
|
||||
if( false == transform_is_empty( include_prefix ) ) {
|
||||
compose_scene_with_include_prefix( module.scene, include_prefix );
|
||||
}
|
||||
|
||||
apply_include_placement_to_scene(
|
||||
module.scene, module.include_placement, include_parameters );
|
||||
}
|
||||
|
||||
apply_module( module, serializer, scratch );
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool
|
||||
file_has_terr_chunk( std::string const &path ) {
|
||||
std::ifstream input { path, std::ios::binary };
|
||||
if( !input ) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if( sn_utils::ld_uint32( input ) != kMagic ) {
|
||||
return false;
|
||||
}
|
||||
const std::uint32_t version { sn_utils::ld_uint32( input ) };
|
||||
if(
|
||||
version != kVersionV4 && version != kVersionV5 && version != kVersionV6 &&
|
||||
version != kVersionV7 && version != kVersionV8 ) {
|
||||
return false;
|
||||
}
|
||||
|
||||
while( input.peek() != EOF ) {
|
||||
const std::uint32_t chunk_type { sn_utils::ld_uint32( input ) };
|
||||
const std::uint32_t chunk_size { sn_utils::ld_uint32( input ) };
|
||||
if( chunk_size < 8 ) {
|
||||
return false;
|
||||
}
|
||||
if( chunk_type == kChunkTerr ) {
|
||||
return true;
|
||||
}
|
||||
input.seekg( static_cast<std::streamoff>( chunk_size - 8 ), std::ios::cur );
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string
|
||||
resolve_scenery_path( std::string const &reference ) {
|
||||
auto path { reference };
|
||||
while( false == path.empty() && path[ 0 ] == '$' ) {
|
||||
path.erase( 0, 1 );
|
||||
}
|
||||
replace_slashes( path );
|
||||
if( path.starts_with( Global.asCurrentSceneryPath ) ) {
|
||||
return path;
|
||||
}
|
||||
return Global.asCurrentSceneryPath + path;
|
||||
}
|
||||
|
||||
std::string
|
||||
binary_path( std::string const &reference ) {
|
||||
auto path { resolve_scenery_path( reference ) };
|
||||
if( path.ends_with( ".scm" ) || path.ends_with( ".sbt" ) || path.ends_with( ".inc" ) ||
|
||||
path.ends_with( ".scn" ) ) {
|
||||
path.replace( path.size() - 4, 4, ".eu7" );
|
||||
}
|
||||
else if( false == path.ends_with( ".eu7" ) ) {
|
||||
erase_extension( path );
|
||||
path += ".eu7";
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
std::string
|
||||
resolve_parser_include_path(
|
||||
std::string const &parser_path,
|
||||
std::string const ¤t_file,
|
||||
std::string const &include_reference ) {
|
||||
auto reference { include_reference };
|
||||
while( false == reference.empty() && reference[ 0 ] == '$' ) {
|
||||
reference.erase( 0, 1 );
|
||||
}
|
||||
replace_slashes( reference );
|
||||
if( reference.starts_with( Global.asCurrentSceneryPath ) ) {
|
||||
return reference;
|
||||
}
|
||||
if( reference.find( '/' ) != std::string::npos ) {
|
||||
return parser_path + reference;
|
||||
}
|
||||
if( false == current_file.empty() ) {
|
||||
auto const slash { current_file.find_last_of( '/' ) };
|
||||
if( slash != std::string::npos ) {
|
||||
return parser_path + current_file.substr( 0, slash + 1 ) + reference;
|
||||
}
|
||||
}
|
||||
return parser_path + reference;
|
||||
}
|
||||
|
||||
std::string
|
||||
include_eu7_path(
|
||||
std::string const &parser_path,
|
||||
std::string const ¤t_file,
|
||||
std::string const &include_reference ) {
|
||||
return binary_path( resolve_parser_include_path( parser_path, current_file, include_reference ) );
|
||||
}
|
||||
|
||||
std::string
|
||||
terrain_binary_path( std::string const &terrain_reference ) {
|
||||
return binary_path( terrain_reference );
|
||||
}
|
||||
|
||||
bool
|
||||
probe_file( std::string const &path ) {
|
||||
return FileExists( path ) && is_valid_eu7b_file( path );
|
||||
}
|
||||
|
||||
bool
|
||||
probe_baked_scenario( std::string const &scenario_file ) {
|
||||
return probe_file( resolve_scenery_path( binary_path( scenario_file ) ) );
|
||||
}
|
||||
|
||||
bool
|
||||
is_text_module_extension( std::string const &path ) {
|
||||
return path.ends_with( ".scn" ) || path.ends_with( ".scm" ) ||
|
||||
path.ends_with( ".inc" ) || path.ends_with( ".sbt" );
|
||||
}
|
||||
|
||||
std::string
|
||||
text_source_path( std::string const &reference ) {
|
||||
auto const resolved { resolve_scenery_path( reference ) };
|
||||
if( is_text_module_extension( resolved ) ) {
|
||||
return resolved;
|
||||
}
|
||||
if( resolved.ends_with( ".eu7" ) ) {
|
||||
auto stem { resolved };
|
||||
stem.erase( stem.size() - 4 );
|
||||
for( auto const *ext : { ".scm", ".scn", ".inc", ".sbt" } ) {
|
||||
auto const candidate { stem + ext };
|
||||
if( FileExists( candidate ) ) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
bool
|
||||
text_module_is_newer_than_binary( std::string const &reference ) {
|
||||
auto const text { text_source_path( reference ) };
|
||||
if( text.empty() || false == FileExists( text ) ) {
|
||||
return false;
|
||||
}
|
||||
auto const eu7 { binary_path( reference ) };
|
||||
if( false == FileExists( eu7 ) ) {
|
||||
return true;
|
||||
}
|
||||
return last_modified( text ) > last_modified( eu7 );
|
||||
}
|
||||
|
||||
bool
|
||||
should_use_binary_module( std::string const &reference ) {
|
||||
auto const eu7 { binary_path( reference ) };
|
||||
return probe_file( eu7 ) && false == text_module_is_newer_than_binary( reference );
|
||||
}
|
||||
|
||||
bool
|
||||
probe_terrain_file( std::string const &path ) {
|
||||
return FileExists( path ) && file_has_terr_chunk( path );
|
||||
}
|
||||
|
||||
bool
|
||||
is_scenario_terrain( std::string const &scenario_file ) {
|
||||
auto stem { scenario_file };
|
||||
while( false == stem.empty() && stem[ 0 ] == '$' ) {
|
||||
stem.erase( 0, 1 );
|
||||
}
|
||||
erase_extension( stem );
|
||||
return probe_terrain_file( resolve_scenery_path( stem + ".eu7" ) );
|
||||
}
|
||||
|
||||
bool
|
||||
try_load_scenario_terrain( basic_region ®ion, std::string const &scenario_file ) {
|
||||
auto stem { scenario_file };
|
||||
while( false == stem.empty() && stem[ 0 ] == '$' ) {
|
||||
stem.erase( 0, 1 );
|
||||
}
|
||||
erase_extension( stem );
|
||||
return load_terrain( region, resolve_scenery_path( stem + ".eu7" ) );
|
||||
}
|
||||
|
||||
void
|
||||
insert_terrain_shapes( basic_region ®ion, Eu7Module const &module ) {
|
||||
scene::scratch_data scratch;
|
||||
std::size_t shape_count { 0 };
|
||||
std::size_t vertex_count { 0 };
|
||||
for( auto const &source : module.scene.terrain_shapes ) {
|
||||
shape_node shape { build_shape_node( source ) };
|
||||
vertex_count += source.vertices.size();
|
||||
region.insert( shape, scratch, false );
|
||||
++shape_count;
|
||||
}
|
||||
if( shape_count > 0 ) {
|
||||
WriteLog(
|
||||
"EU7 terrain shapes: count=" + std::to_string( shape_count ) +
|
||||
" vertices=" + std::to_string( vertex_count ) );
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
load_terrain( basic_region ®ion, std::string const &path ) {
|
||||
if( false == probe_terrain_file( path ) ) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
auto const module { read_module( path ) };
|
||||
if( module.scene.terrain_shapes.empty() ) {
|
||||
ErrorLog( "EU7: brak chunka TERR w \"" + path + "\"" );
|
||||
return false;
|
||||
}
|
||||
insert_terrain_shapes( region, module );
|
||||
WriteLog( "EU7 terrain loaded: " + path );
|
||||
return true;
|
||||
}
|
||||
catch( std::exception const &ex ) {
|
||||
ErrorLog( std::string{ "EU7: blad terenu \"" + path + "\": " } + ex.what() );
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
begin_load_session() {
|
||||
g_load_session.clear();
|
||||
g_packed_root_active = false;
|
||||
reset_section_stream();
|
||||
g_module_file_cache.clear();
|
||||
reset_load_stats();
|
||||
reset_pack_bench();
|
||||
}
|
||||
|
||||
bool
|
||||
is_module_loaded( std::string const &path ) {
|
||||
return g_load_session.contains( resolve_scenery_path( path ) );
|
||||
}
|
||||
|
||||
bool
|
||||
pack_scenery_active() {
|
||||
return g_packed_root_active;
|
||||
}
|
||||
|
||||
bool
|
||||
load_module( std::string const &path, simulation::state_serializer &serializer ) {
|
||||
if( simulation::Region == nullptr ) {
|
||||
ErrorLog( "EU7: Region nie jest zainicjalizowany" );
|
||||
return false;
|
||||
}
|
||||
auto const resolved { resolve_scenery_path( path ) };
|
||||
auto const ok { load_module_recursive( path, serializer, g_load_session ) };
|
||||
if( ok ) {
|
||||
if( auto const cached { g_module_file_cache.find( resolved ) };
|
||||
cached != g_module_file_cache.end() && cached->second.has_pack_chunk ) {
|
||||
init_section_stream( cached->second, resolved, serializer );
|
||||
prime_section_stream( cached->second );
|
||||
evict_non_pack_modules_from_cache( resolved );
|
||||
}
|
||||
else {
|
||||
evict_non_pack_modules_from_cache( {} );
|
||||
}
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
} // namespace scene::eu7
|
||||
Reference in New Issue
Block a user