diff --git a/scene/scenerybinary.cpp b/scene/scenerybinary.cpp index 38d5817e..858b3862 100644 --- a/scene/scenerybinary.cpp +++ b/scene/scenerybinary.cpp @@ -160,6 +160,7 @@ scenery_binary_writer::add_number( double Value ) { void scenery_binary_writer::add_include( std::vector const &Fileexpr, std::vector const &Params ) { + m_has_include = true; // file references other files -> the infra pass must still process it auto &out = sink(); write_varint( out, TAG_INCLUDE ); write_varint( out, Fileexpr.size() ); @@ -183,6 +184,7 @@ scenery_binary_writer::end_node( bool Visual, bool Haspos, double X, double Y, d m_innode = false; auto const body = m_nodebuf.str(); write_varint( m_entries, TAG_NODE ); + if( false == Visual ) { m_has_infra = true; } // infrastructure node -> infra pass must process it auto const cls = ( false == Visual ) ? NODECLASS_INFRA : ( Haspos ) ? NODECLASS_VISUAL_POS : @@ -206,7 +208,9 @@ scenery_binary_writer::write( std::ostream &Output, scenery_file_kind Kind ) con // header: magic + kind + flags sn_utils::ls_uint32( Output, SCENERYBINARY_MAGIC ); sn_utils::s_uint8( Output, static_cast( Kind ) ); - sn_utils::ls_uint32( Output, 0 ); // flags, reserved + // flags bit0: infra-relevant (has an infrastructure node or an include). a pure-visual leaf + // has it clear, so the infrastructure pass skips opening it entirely. + sn_utils::ls_uint32( Output, ( m_has_infra || m_has_include ) ? 1u : 0u ); // string table: count, then each string as varint(length) + raw bytes (so the // reader can take views into the buffer without scanning for terminators) @@ -240,7 +244,7 @@ scenery_binary_reader::open( std::string_view Buffer ) { return false; } m_kind = static_cast( static_cast( *cursor++ ) ); - read_u32le( cursor, end ); // flags, reserved + m_infra_relevant = ( ( read_u32le( cursor, end ) & 1u ) != 0u ); // flags bit0 // string table: views into the buffer (no copies) auto tablesize = read_varint( cursor, end ); diff --git a/scene/scenerybinary.h b/scene/scenerybinary.h index eff04827..2e55a35f 100644 --- a/scene/scenerybinary.h +++ b/scene/scenerybinary.h @@ -64,7 +64,7 @@ namespace scene { // in the visual pass, swallowed the following endorigin -> the origin stack accumulated // and flung terrain/models across the map. bumping invalidates those bad twins. // bumping the version invalidates older twins so they are recompiled rather than misread. -constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 9 ) }; +constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 10 ) }; // which entries a reader serves in a given load pass; nodes outside the requested class // are skipped (directives/includes are always served, to keep transform/group state) @@ -139,6 +139,8 @@ private: std::ostringstream m_nodebuf; // current node's entries (buffered) bool m_innode { false }; // currently between begin/end_node std::size_t m_count { 0 }; // number of entries + bool m_has_infra { false }; // emitted any infrastructure node + bool m_has_include { false }; // emitted any include directive }; // parses a binary twin held in a memory buffer and streams its entries on demand. @@ -151,6 +153,11 @@ public: bool open( std::string_view Buffer ); scenery_file_kind kind() const { return m_kind; } + // true if this file has any infrastructure node or include directive. a pure-visual leaf + // (e.g. a flora .incb: triangles + transform directives only) returns false, so the + // infrastructure pass can skip opening it entirely instead of traversing it to drop its + // visual content -- which on a million-instance scenery was ~200k wasted reopens. + bool infra_relevant() const { return m_infra_relevant; } // selects which nodes are served vs skipped (default: all). may be changed between // reads (e.g. to drive separate eager/visual passes over a re-opened twin). void set_pass( scenery_load_pass Pass ) { m_pass = Pass; } @@ -190,6 +197,7 @@ private: double m_noderange { -1.0 }; // range_max of the current node (v9 model marker) std::size_t m_nodeoffset { 0 }; // byte offset of the current node's marker (for seek_node) bool m_nodehaspos { false }; // whether the current node's marker carried a position + bool m_infra_relevant { true }; // header flag: has infrastructure node or include std::ptrdiff_t m_size { 0 }; // entry section byte length scenery_load_pass m_pass { scenery_load_pass::all }; scenery_file_kind m_kind { scenery_file_kind::scn }; diff --git a/simulation/simulationstateserializer.cpp b/simulation/simulationstateserializer.cpp index 7f8474db..1eb410f9 100644 --- a/simulation/simulationstateserializer.cpp +++ b/simulation/simulationstateserializer.cpp @@ -46,6 +46,41 @@ constexpr double STREAM_RADIUS { 2000.0 }; // a too-small budget is a tiny duty cycle, so streaming a million flora instances drags. constexpr int VISUAL_BUDGET_MS { 24 }; +// --- load profiler: where the load time actually goes, so we optimise the real bottleneck --- +struct load_profile { + std::unordered_map typetime; // seconds building each node type (inside deserialize_node) + std::unordered_map typecount; // how many of each node type + std::unordered_map dispatchtime; // seconds per top-level token (node/event/trainset/...) + long tokens { 0 }; // top-level getToken() dispatches (gauges the cParser scan cost) + double finalize { 0 }; // create_map_geometry + InitInstanceEvents + void reset() { typetime.clear(); typecount.clear(); dispatchtime.clear(); tokens = 0; finalize = 0; } + static void dump( std::unordered_map const &M, char const *Tag, std::unordered_map const *C ) { + std::vector> v( M.begin(), M.end() ); + std::sort( v.begin(), v.end(), []( auto const &a, auto const &b ) { return a.second > b.second; } ); + for( auto const &p : v ) { + if( p.second < 0.05 ) { break; } + WriteLog( std::string( " " ) + Tag + " " + p.first + ": " + std::to_string( p.second ) + "s" + + ( C ? " x" + std::to_string( ( *const_cast*>( C ) )[ p.first ] ) : "" ) ); + } + } + void log( char const *Phase ) { + double dtotal = finalize; for( auto const &p : dispatchtime ) { dtotal += p.second; } + WriteLog( "=== load profile [" + std::string( Phase ) + "]: dispatch " + std::to_string( dtotal ) + + "s, getToken " + std::to_string( tokens ) + ", finalize " + std::to_string( finalize ) + "s ===" ); + dump( dispatchtime, "[top]", nullptr ); + dump( typetime, "[node]", &typecount ); + } +}; +load_profile g_profile; + +// RAII: add the elapsed time to Acc on scope exit (handles deserialize_node's many returns) +struct scoped_accum { + std::chrono::steady_clock::time_point t0 { std::chrono::steady_clock::now() }; + double &acc; + explicit scoped_accum( double &Acc ) : acc( Acc ) {} + ~scoped_accum() { acc += std::chrono::duration( std::chrono::steady_clock::now() - t0 ).count(); } +}; + // fills Tobuild with the region-section indices within STREAM_RADIUS of Eye that are not yet // in Built. mirrors basic_region::section() indexing (clamped to the grid). returns count. std::size_t wanted_sections( glm::dvec3 const &Eye, std::unordered_set const &Built, std::unordered_set &Tobuild ) { @@ -78,6 +113,7 @@ std::shared_ptr state_serializer::deserialize_begin( std::string const &Scenariofile ) { crashreport_add_info("scenario", Scenariofile); + cParser::clearInfraSkipCache(); // fresh per-load cache of pure-visual leaf includes // TODO: move initialization to separate routine so we can reuse it SafeDelete( Region ); @@ -275,6 +311,7 @@ state_serializer::deserialize_continue(std::shared_ptr state auto timelast { std::chrono::steady_clock::now() }; std::string token { Input.getToken() }; while( false == token.empty() ) { + ++g_profile.tokens; // profile: gauge the cParser scan cost if( state->visualphase ) { auto const skip = visualskip.find( token ); @@ -318,6 +355,7 @@ state_serializer::deserialize_continue(std::shared_ptr state auto lookup = state->functionmap.find( token ); if( lookup != state->functionmap.end() ) { + scoped_accum const dispatchguard { g_profile.dispatchtime[ token ] }; lookup->second(); } else { @@ -343,6 +381,7 @@ state_serializer::deserialize_continue(std::shared_ptr state // group stack is left open on purpose in section mode -- later cycles keep inserting into it // (update_map reads the persistent group map, not the stack, so it works either way). auto const finalize = [ & ]( bool const Closegroups ) { + scoped_accum const fg { g_profile.finalize }; if( true == Closegroups ) { scene::Groups.close(); } scene::Groups.update_map(); Region->create_map_geometry(); @@ -359,6 +398,8 @@ state_serializer::deserialize_continue(std::shared_ptr state && ( true == Input.restartReplay( scene::scenery_load_pass::visual ) ) ) { state->visualphase = true; resettransform(); + g_profile.log( "infrastructure" ); + g_profile.reset(); // measure the visual phase separately WriteLog( "Progressive visual load: infrastructure ready, streaming visuals from the driver" ); return false; // infrastructure ready -> go to driver; visuals continue there } @@ -380,6 +421,8 @@ state_serializer::deserialize_continue(std::shared_ptr state std::size_t refs = 0; for( auto const &s : state->index ) { refs += s.second.size(); } WriteLog( "Progressive visual load: spawn ready (" + std::to_string( simulation::Instances.sequence().size() ) + " instances), " + std::to_string( refs ) + " nodes indexed across " + std::to_string( state->index.size() ) + " sections" ); + g_profile.log( "visual first pass" ); + g_profile.reset(); } } return true; // keep streaming alive; sections the camera enters are served from the index @@ -388,6 +431,7 @@ state_serializer::deserialize_continue(std::shared_ptr state // build-all (no camera centre, e.g. ghostview): everything was built in this single pass. finalize( /*Closegroups*/ true ); state->done = true; + g_profile.log( "visual build-all" ); return false; } @@ -706,6 +750,9 @@ state_serializer::deserialize_node( cParser &Input, scene::scratch_data &Scratch >> nodedata.name >> nodedata.type; if( nodedata.name == "none" ) { nodedata.name.clear(); } + // profile: attribute this node's build time to its type (see g_profile log at pass boundaries) + scoped_accum const profileguard { g_profile.typetime[ nodedata.type ] }; + ++g_profile.typecount[ nodedata.type ]; // type-based deserialization. not elegant but it'll do if( nodedata.type == "dynamic" ) { diff --git a/utilities/parser.cpp b/utilities/parser.cpp index d213b194..20957aa4 100644 --- a/utilities/parser.cpp +++ b/utilities/parser.cpp @@ -621,6 +621,14 @@ void cParser::processInclude(cParser& srcParser, bool ToLower) { startIncludeDirect(std::move(pick), std::move(params)); } +std::map cParser::s_infraskip; + +bool cParser::infraRelevant() const { + return (m_replay && m_reader) ? m_reader->infra_relevant() : true; +} + +void cParser::clearInfraSkipCache() { s_infraskip.clear(); } + void cParser::startIncludeDirect(std::string includefile, std::vector Params) { const bool allowTraction = @@ -632,6 +640,19 @@ void cParser::startIncludeDirect(std::string includefile, std::vectorsecond)) { + return; // known pure-visual leaf + } + } + if (Global.ParserLogIncludes) { // WriteLog("including: " + includefile); } @@ -645,6 +666,17 @@ void cParser::startIncludeDirect(std::string includefile, std::vectorsetReplayPass(m_replaypass); + // infrastructure pass: a pure-visual leaf twin has nothing to do here. record it (so the next + // of its thousands of reopens is skipped before even opening) and drop it now. + if (infrapass && mIncludeParser->m_replay) { + bool const relevant = mIncludeParser->infraRelevant(); + s_infraskip[skipkey] = (false == relevant); + if (false == relevant) { + mIncludeParser = nullptr; + return; + } + } + // pass filtering (infra eager / visual deferred) relies on the per-node markers that // only a binary twin carries. an un-baked (text) include has no markers, so it can't // be split: it is loaded in full during the infrastructure pass. on the visual pass we diff --git a/utilities/parser.h b/utilities/parser.h index 53fa9dfa..9594adef 100644 --- a/utilities/parser.h +++ b/utilities/parser.h @@ -57,6 +57,12 @@ class cParser //: public std::stringstream // true when this (top-level) file is served from a binary twin, i.e. a second // (visual) pass via restartReplay() is possible. false for a text/compile load. bool isReplaying() const { return m_replay; } + // true if this twin has any infrastructure node or include (so the infra pass must process + // it); false for a pure-visual leaf (a flora .incb), which the infra pass can skip opening. + bool infraRelevant() const; + // clears the per-load cache of which include files are pure-visual leaves (call at the start + // of a scenario load so a rebake that changed a file's class isn't masked by a stale entry). + static void clearInfraSkipCache(); // skips the rest of the node currently being replayed in O(1) (jump over its v6 marker // span), delegating to the active include child that is actually serving it. returns // false if the skip can't be done here (e.g. a text include with no binary twin), so the @@ -212,6 +218,8 @@ class cParser //: public std::stringstream // --- binary scenery twin support --- // last token produced by readTokenFromStream was (partly) quoted -> case preserved bool m_lastquoted { false }; + // per-load cache: include path -> true if it's a pure-visual leaf the infra pass skips opening + static std::map s_infraskip; // replay: this file is served from its binary twin instead of text bool m_replay { false }; bool m_replayexhausted { false }; // all twin entries consumed (for inline eof()) diff --git a/vehicle/DynObj.cpp b/vehicle/DynObj.cpp index 262005ba..0fbaaa69 100644 --- a/vehicle/DynObj.cpp +++ b/vehicle/DynObj.cpp @@ -2017,8 +2017,13 @@ TDynamicObject::Init(std::string Name, // nazwa pojazdu, np. "EU07-424" */ // utworzenie parametrów fizyki MoverParameters = new TMoverParameters(iDirection ? fVel : -fVel, Type_Name, asName, Cab); + // --- profile vehicle load (LoadFIZ physics parse vs LoadMMediaFile model/textures) --- + static double s_fiztime = 0.0, s_mediatime = 0.0; static long s_vehcount = 0; + auto const s_t0 = std::chrono::steady_clock::now(); // McZapkie: TypeName musi byc nazwą CHK/MMD pojazdu - if (!MoverParameters->LoadFIZ(asBaseDir)) + bool const s_fizok = MoverParameters->LoadFIZ(asBaseDir); + s_fiztime += std::chrono::duration( std::chrono::steady_clock::now() - s_t0 ).count(); + if (!s_fizok) { // jak wczytanie CHK się nie uda, to błąd if (ConversionError == 666) ErrorLog( "Bad vehicle: failed to locate definition file \"" + BaseDir + "/" + Type_Name + ".fiz" + "\"" ); @@ -2347,7 +2352,13 @@ TDynamicObject::Init(std::string Name, // nazwa pojazdu, np. "EU07-424" */ // wczytywanie z pliku nazwatypu.mmd, w tym model erase_extension( asReplacableSkin ); + auto const s_t1 = std::chrono::steady_clock::now(); LoadMMediaFile(Type_Name, asReplacableSkin); + s_mediatime += std::chrono::duration( std::chrono::steady_clock::now() - s_t1 ).count(); + if( ( ++s_vehcount % 500 ) == 0 ) { + WriteLog( "VEH PROFILE @" + std::to_string( s_vehcount ) + ": LoadFIZ " + std::to_string( s_fiztime ) + + "s, LoadMMediaFile " + std::to_string( s_mediatime ) + "s" ); + } // McZapkie-100402: wyszukiwanie submodeli sprzegów btCoupler1.Init( "coupler1", mdModel ); // false - ma być wyłączony btCoupler2.Init( "coupler2", mdModel );