eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-19 02:09:17 +02:00
Code Issues 121 Wiki Activity

Skip pure-visual leaf includes in the infrastructure pass

Browse Source 
On a million-instance scenery (tomaszewo) the infra pass was spending ~24s
reopening pure-visual leaf twins (grass.incb etc.) just to skip their content --
~200k cParser constructions, because every flora include reopens the same leaf.

Twin header now flags whether a file has any infrastructure node or include
(format bumped to v10). A pure-visual leaf (flora .incb: triangles + transform
directives only) has it clear, so the infra pass skips opening it: the first open
of each file caches the verdict, later opens are dropped before construction.

Result on tomaszewo: infra 55s -> 31s, getToken 1.06M -> 89k. Also adds a load
profiler (per-type build time, dispatch time, getToken count) behind WriteLog so
the next bottleneck is measured, not guessed (it's now the 25s of decorative
vehicle media loading).
This commit is contained in:
maj00r
2026-06-25 00:53:24 +02:00
parent 4a9bfdc0aa
commit ef4e99a582
6 changed files with 114 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
scene/scenerybinary.cpp
View File

@@ -160,6 +160,7 @@ scenery_binary_writer::add_number( double Value ) {
void
scenery_binary_writer::add_include( std::vector<std::string> const &Fileexpr, std::vector<std::string> const &Params ) {
m_has_include = true; // file references other files -> the infra pass must still process it
auto &out = sink();
write_varint( out, TAG_INCLUDE );
write_varint( out, Fileexpr.size() );
@@ -183,6 +184,7 @@ scenery_binary_writer::end_node( bool Visual, bool Haspos, double X, double Y, d
m_innode = false;
auto const body = m_nodebuf.str();
write_varint( m_entries, TAG_NODE );
if( false == Visual ) { m_has_infra = true; } // infrastructure node -> infra pass must process it
auto const cls =
( false == Visual ) ? NODECLASS_INFRA :
( Haspos ) ? NODECLASS_VISUAL_POS :
@@ -206,7 +208,9 @@ scenery_binary_writer::write( std::ostream &Output, scenery_file_kind Kind ) con
// header: magic + kind + flags
sn_utils::ls_uint32( Output, SCENERYBINARY_MAGIC );
sn_utils::s_uint8( Output, static_cast<std::uint8_t>( Kind ) );
sn_utils::ls_uint32( Output, 0 ); // flags, reserved
// flags bit0: infra-relevant (has an infrastructure node or an include). a pure-visual leaf
// has it clear, so the infrastructure pass skips opening it entirely.
sn_utils::ls_uint32( Output, ( m_has_infra || m_has_include ) ? 1u : 0u );
// string table: count, then each string as varint(length) + raw bytes (so the
// reader can take views into the buffer without scanning for terminators)
@@ -240,7 +244,7 @@ scenery_binary_reader::open( std::string_view Buffer ) {
return false;
}
m_kind = static_cast<scenery_file_kind>( static_cast<std::uint8_t>( *cursor++ ) );
read_u32le( cursor, end ); // flags, reserved
m_infra_relevant = ( ( read_u32le( cursor, end ) & 1u ) != 0u ); // flags bit0
// string table: views into the buffer (no copies)
auto tablesize = read_varint( cursor, end );

10
scene/scenerybinary.h
View File

@@ -64,7 +64,7 @@ namespace scene {
// in the visual pass, swallowed the following endorigin -> the origin stack accumulated
// and flung terrain/models across the map. bumping invalidates those bad twins.
// bumping the version invalidates older twins so they are recompiled rather than misread.
constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 9 ) };
constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 10 ) };
// which entries a reader serves in a given load pass; nodes outside the requested class
// are skipped (directives/includes are always served, to keep transform/group state)
@@ -139,6 +139,8 @@ private:
std::ostringstream m_nodebuf; // current node's entries (buffered)
bool m_innode { false }; // currently between begin/end_node
std::size_t m_count { 0 }; // number of entries
bool m_has_infra { false }; // emitted any infrastructure node
bool m_has_include { false }; // emitted any include directive
};
// parses a binary twin held in a memory buffer and streams its entries on demand.
@@ -151,6 +153,11 @@ public:
bool open( std::string_view Buffer );
scenery_file_kind kind() const { return m_kind; }
// true if this file has any infrastructure node or include directive. a pure-visual leaf
// (e.g. a flora .incb: triangles + transform directives only) returns false, so the
// infrastructure pass can skip opening it entirely instead of traversing it to drop its
// visual content -- which on a million-instance scenery was ~200k wasted reopens.
bool infra_relevant() const { return m_infra_relevant; }
// selects which nodes are served vs skipped (default: all). may be changed between
// reads (e.g. to drive separate eager/visual passes over a re-opened twin).
void set_pass( scenery_load_pass Pass ) { m_pass = Pass; }
@@ -190,6 +197,7 @@ private:
double m_noderange { -1.0 }; // range_max of the current node (v9 model marker)
std::size_t m_nodeoffset { 0 }; // byte offset of the current node's marker (for seek_node)
bool m_nodehaspos { false }; // whether the current node's marker carried a position
bool m_infra_relevant { true }; // header flag: has infrastructure node or include
std::ptrdiff_t m_size { 0 }; // entry section byte length
scenery_load_pass m_pass { scenery_load_pass::all };
scenery_file_kind m_kind { scenery_file_kind::scn };

47
simulation/simulationstateserializer.cpp
View File

@@ -46,6 +46,41 @@ constexpr double STREAM_RADIUS { 2000.0 };
// a too-small budget is a tiny duty cycle, so streaming a million flora instances drags.
constexpr int VISUAL_BUDGET_MS { 24 };
// --- load profiler: where the load time actually goes, so we optimise the real bottleneck ---
struct load_profile {
std::unordered_map<std::string, double> typetime; // seconds building each node type (inside deserialize_node)
std::unordered_map<std::string, long> typecount; // how many of each node type
std::unordered_map<std::string, double> dispatchtime; // seconds per top-level token (node/event/trainset/...)
long tokens { 0 }; // top-level getToken() dispatches (gauges the cParser scan cost)
double finalize { 0 }; // create_map_geometry + InitInstanceEvents
void reset() { typetime.clear(); typecount.clear(); dispatchtime.clear(); tokens = 0; finalize = 0; }
static void dump( std::unordered_map<std::string, double> const &M, char const *Tag, std::unordered_map<std::string, long> const *C ) {
std::vector<std::pair<std::string, double>> v( M.begin(), M.end() );
std::sort( v.begin(), v.end(), []( auto const &a, auto const &b ) { return a.second > b.second; } );
for( auto const &p : v ) {
if( p.second < 0.05 ) { break; }
WriteLog( std::string( " " ) + Tag + " " + p.first + ": " + std::to_string( p.second ) + "s"
+ ( C ? " x" + std::to_string( ( *const_cast<std::unordered_map<std::string, long>*>( C ) )[ p.first ] ) : "" ) );
}
}
void log( char const *Phase ) {
double dtotal = finalize; for( auto const &p : dispatchtime ) { dtotal += p.second; }
WriteLog( "=== load profile [" + std::string( Phase ) + "]: dispatch " + std::to_string( dtotal )
+ "s, getToken " + std::to_string( tokens ) + ", finalize " + std::to_string( finalize ) + "s ===" );
dump( dispatchtime, "[top]", nullptr );
dump( typetime, "[node]", &typecount );
}
};
load_profile g_profile;
// RAII: add the elapsed time to Acc on scope exit (handles deserialize_node's many returns)
struct scoped_accum {
std::chrono::steady_clock::time_point t0 { std::chrono::steady_clock::now() };
double &acc;
explicit scoped_accum( double &Acc ) : acc( Acc ) {}
~scoped_accum() { acc += std::chrono::duration<double>( std::chrono::steady_clock::now() - t0 ).count(); }
};
// fills Tobuild with the region-section indices within STREAM_RADIUS of Eye that are not yet
// in Built. mirrors basic_region::section() indexing (clamped to the grid). returns count.
std::size_t wanted_sections( glm::dvec3 const &Eye, std::unordered_set<int> const &Built, std::unordered_set<int> &Tobuild ) {
@@ -78,6 +113,7 @@ std::shared_ptr<deserializer_state>
state_serializer::deserialize_begin( std::string const &Scenariofile ) {
crashreport_add_info("scenario", Scenariofile);
cParser::clearInfraSkipCache(); // fresh per-load cache of pure-visual leaf includes
// TODO: move initialization to separate routine so we can reuse it
SafeDelete( Region );
@@ -275,6 +311,7 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
auto timelast { std::chrono::steady_clock::now() };
std::string token { Input.getToken<std::string>() };
while( false == token.empty() ) {
++g_profile.tokens; // profile: gauge the cParser scan cost
if( state->visualphase ) {
auto const skip = visualskip.find( token );
@@ -318,6 +355,7 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
auto lookup = state->functionmap.find( token );
if( lookup != state->functionmap.end() ) {
scoped_accum const dispatchguard { g_profile.dispatchtime[ token ] };
lookup->second();
}
else {
@@ -343,6 +381,7 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
// group stack is left open on purpose in section mode -- later cycles keep inserting into it
// (update_map reads the persistent group map, not the stack, so it works either way).
auto const finalize = [ & ]( bool const Closegroups ) {
scoped_accum const fg { g_profile.finalize };
if( true == Closegroups ) { scene::Groups.close(); }
scene::Groups.update_map();
Region->create_map_geometry();
@@ -359,6 +398,8 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
&& ( true == Input.restartReplay( scene::scenery_load_pass::visual ) ) ) {
state->visualphase = true;
resettransform();
g_profile.log( "infrastructure" );
g_profile.reset(); // measure the visual phase separately
WriteLog( "Progressive visual load: infrastructure ready, streaming visuals from the driver" );
return false; // infrastructure ready -> go to driver; visuals continue there
}
@@ -380,6 +421,8 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
std::size_t refs = 0; for( auto const &s : state->index ) { refs += s.second.size(); }
WriteLog( "Progressive visual load: spawn ready (" + std::to_string( simulation::Instances.sequence().size() )
+ " instances), " + std::to_string( refs ) + " nodes indexed across " + std::to_string( state->index.size() ) + " sections" );
g_profile.log( "visual first pass" );
g_profile.reset();
}
}
return true; // keep streaming alive; sections the camera enters are served from the index
@@ -388,6 +431,7 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
// build-all (no camera centre, e.g. ghostview): everything was built in this single pass.
finalize( /*Closegroups*/ true );
state->done = true;
g_profile.log( "visual build-all" );
return false;
}
@@ -706,6 +750,9 @@ state_serializer::deserialize_node( cParser &Input, scene::scratch_data &Scratch
>> nodedata.name
>> nodedata.type;
if( nodedata.name == "none" ) { nodedata.name.clear(); }
// profile: attribute this node's build time to its type (see g_profile log at pass boundaries)
scoped_accum const profileguard { g_profile.typetime[ nodedata.type ] };
++g_profile.typecount[ nodedata.type ];
// type-based deserialization. not elegant but it'll do
if( nodedata.type == "dynamic" ) {

32
utilities/parser.cpp
View File

@@ -621,6 +621,14 @@ void cParser::processInclude(cParser& srcParser, bool ToLower) {
startIncludeDirect(std::move(pick), std::move(params));
}
std::map<std::string, bool> cParser::s_infraskip;
bool cParser::infraRelevant() const {
return (m_replay && m_reader) ? m_reader->infra_relevant() : true;
}
void cParser::clearInfraSkipCache() { s_infraskip.clear(); }
void cParser::startIncludeDirect(std::string includefile, std::vector<std::string> Params) {
const bool allowTraction =
@@ -632,6 +640,19 @@ void cParser::startIncludeDirect(std::string includefile, std::vector<std::strin
return;
}
// infrastructure pass: don't open a pure-visual leaf (e.g. a flora .incb) just to skip its
// content -- on a million-instance scenery that was ~200k wasted reopens. the first open of
// each file records whether it's infra-relevant; later opens of the same file are skipped.
bool const infrapass = (m_replaypass == scene::scenery_load_pass::infrastructure);
std::string skipkey;
if (infrapass) {
skipkey = mPath + includefile;
auto const it = s_infraskip.find(skipkey);
if ((it != s_infraskip.end()) && (true == it->second)) {
return; // known pure-visual leaf
}
}
if (Global.ParserLogIncludes) {
// WriteLog("including: " + includefile);
}
@@ -645,6 +666,17 @@ void cParser::startIncludeDirect(std::string includefile, std::vector<std::strin
// consistently (e.g. visuals-only on the second pass)
mIncludeParser->setReplayPass(m_replaypass);
// infrastructure pass: a pure-visual leaf twin has nothing to do here. record it (so the next
// of its thousands of reopens is skipped before even opening) and drop it now.
if (infrapass && mIncludeParser->m_replay) {
bool const relevant = mIncludeParser->infraRelevant();
s_infraskip[skipkey] = (false == relevant);
if (false == relevant) {
mIncludeParser = nullptr;
return;
}
}
// pass filtering (infra eager / visual deferred) relies on the per-node markers that
// only a binary twin carries. an un-baked (text) include has no markers, so it can't
// be split: it is loaded in full during the infrastructure pass. on the visual pass we

8
utilities/parser.h
View File

@@ -57,6 +57,12 @@ class cParser //: public std::stringstream
// true when this (top-level) file is served from a binary twin, i.e. a second
// (visual) pass via restartReplay() is possible. false for a text/compile load.
bool isReplaying() const { return m_replay; }
// true if this twin has any infrastructure node or include (so the infra pass must process
// it); false for a pure-visual leaf (a flora .incb), which the infra pass can skip opening.
bool infraRelevant() const;
// clears the per-load cache of which include files are pure-visual leaves (call at the start
// of a scenario load so a rebake that changed a file's class isn't masked by a stale entry).
static void clearInfraSkipCache();
// skips the rest of the node currently being replayed in O(1) (jump over its v6 marker
// span), delegating to the active include child that is actually serving it. returns
// false if the skip can't be done here (e.g. a text include with no binary twin), so the
@@ -212,6 +218,8 @@ class cParser //: public std::stringstream
// --- binary scenery twin support ---
// last token produced by readTokenFromStream was (partly) quoted -> case preserved
bool m_lastquoted { false };
// per-load cache: include path -> true if it's a pure-visual leaf the infra pass skips opening
static std::map<std::string, bool> s_infraskip;
// replay: this file is served from its binary twin instead of text
bool m_replay { false };
bool m_replayexhausted { false }; // all twin entries consumed (for inline eof())

13
vehicle/DynObj.cpp
View File

@@ -2017,8 +2017,13 @@ TDynamicObject::Init(std::string Name, // nazwa pojazdu, np. "EU07-424"
*/
// utworzenie parametrów fizyki
MoverParameters = new TMoverParameters(iDirection ? fVel : -fVel, Type_Name, asName, Cab);
// --- profile vehicle load (LoadFIZ physics parse vs LoadMMediaFile model/textures) ---
static double s_fiztime = 0.0, s_mediatime = 0.0; static long s_vehcount = 0;
auto const s_t0 = std::chrono::steady_clock::now();
// McZapkie: TypeName musi byc nazwą CHK/MMD pojazdu
if (!MoverParameters->LoadFIZ(asBaseDir))
bool const s_fizok = MoverParameters->LoadFIZ(asBaseDir);
s_fiztime += std::chrono::duration<double>( std::chrono::steady_clock::now() - s_t0 ).count();
if (!s_fizok)
{ // jak wczytanie CHK się nie uda, to błąd
if (ConversionError == 666)
ErrorLog( "Bad vehicle: failed to locate definition file \"" + BaseDir + "/" + Type_Name + ".fiz" + "\"" );
@@ -2347,7 +2352,13 @@ TDynamicObject::Init(std::string Name, // nazwa pojazdu, np. "EU07-424"
*/
// wczytywanie z pliku nazwatypu.mmd, w tym model
erase_extension( asReplacableSkin );
auto const s_t1 = std::chrono::steady_clock::now();
LoadMMediaFile(Type_Name, asReplacableSkin);
s_mediatime += std::chrono::duration<double>( std::chrono::steady_clock::now() - s_t1 ).count();
if( ( ++s_vehcount % 500 ) == 0 ) {
WriteLog( "VEH PROFILE @" + std::to_string( s_vehcount ) + ": LoadFIZ " + std::to_string( s_fiztime )
+ "s, LoadMMediaFile " + std::to_string( s_mediatime ) + "s" );
}
// McZapkie-100402: wyszukiwanie submodeli sprzegów
btCoupler1.Init( "coupler1", mdModel ); // false - ma być wyłączony
btCoupler2.Init( "coupler2", mdModel );