diff --git a/scene/scenerybinary.cpp b/scene/scenerybinary.cpp index 86329f68..38d5817e 100644 --- a/scene/scenerybinary.cpp +++ b/scene/scenerybinary.cpp @@ -178,7 +178,7 @@ scenery_binary_writer::begin_node() { } void -scenery_binary_writer::end_node( bool Visual, bool Haspos, double X, double Y, double Z ) { +scenery_binary_writer::end_node( bool Visual, bool Haspos, double X, double Y, double Z, double Range ) { if( false == m_innode ) { return; } m_innode = false; auto const body = m_nodebuf.str(); @@ -189,12 +189,13 @@ scenery_binary_writer::end_node( bool Visual, bool Haspos, double X, double Y, d NODECLASS_VISUAL; write_varint( m_entries, cls ); write_varint( m_entries, body.size() ); - // a visual model node stores its local position right after the span, so the reader can - // hand it to the camera-ring load without the node body being decoded + // a visual model node stores its local position + visibility range right after the span, so + // the reader can hand them to the streaming load without the node body being decoded if( cls == NODECLASS_VISUAL_POS ) { sn_utils::ls_float32( m_entries, static_cast( X ) ); sn_utils::ls_float32( m_entries, static_cast( Y ) ); sn_utils::ls_float32( m_entries, static_cast( Z ) ); + sn_utils::ls_float32( m_entries, static_cast( Range ) ); } m_entries.write( body.data(), static_cast( body.size() ) ); } @@ -271,9 +272,13 @@ scenery_binary_reader::next( scenery_entry_view &Out ) { std::uint64_t tag = 0; for( ;; ) { if( m_cursor >= m_end ) { return false; } + char const *markerstart = m_cursor; // where this entry's marker begins (for seek_node) head = read_varint( m_cursor, m_end ); tag = head & TAG_MASK; if( tag != TAG_NODE ) { break; } + // record the served node's marker offset so the section-index streamer can seek back to + // it and rebuild the node on demand without re-scanning the whole twin + m_nodeoffset = static_cast( markerstart - m_begin ); auto const cls = read_varint( m_cursor, m_end ); auto const span = read_varint( m_cursor, m_end ); bool const isvisual = ( cls == NODECLASS_VISUAL ) || ( cls == NODECLASS_VISUAL_POS ); @@ -283,6 +288,7 @@ scenery_binary_reader::next( scenery_entry_view &Out ) { m_nodepos[ 0 ] = static_cast( read_f32le( m_cursor, m_end ) ); m_nodepos[ 1 ] = static_cast( read_f32le( m_cursor, m_end ) ); m_nodepos[ 2 ] = static_cast( read_f32le( m_cursor, m_end ) ); + m_noderange = static_cast( read_f32le( m_cursor, m_end ) ); } bool const process = ( m_pass == scenery_load_pass::all ) diff --git a/scene/scenerybinary.h b/scene/scenerybinary.h index 7cb9ef94..eff04827 100644 --- a/scene/scenerybinary.h +++ b/scene/scenerybinary.h @@ -64,7 +64,7 @@ namespace scene { // in the visual pass, swallowed the following endorigin -> the origin stack accumulated // and flung terrain/models across the map. bumping invalidates those bad twins. // bumping the version invalidates older twins so they are recompiled rather than misread. -constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 8 ) }; +constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 9 ) }; // which entries a reader serves in a given load pass; nodes outside the requested class // are skipped (directives/includes are always served, to keep transform/group state) @@ -124,7 +124,7 @@ public: // Haspos/X/Y/Z give a model node's local position so the camera-ring load can skip it // without reading its tokens. void begin_node(); - void end_node( bool Visual, bool Haspos = false, double X = 0.0, double Y = 0.0, double Z = 0.0 ); + void end_node( bool Visual, bool Haspos = false, double X = 0.0, double Y = 0.0, double Z = 0.0, double Range = -1.0 ); std::size_t entry_count() const { return m_count; } // serializes header + string table + encoded entries. returns false on stream failure. bool write( std::ostream &Output, scenery_file_kind Kind ) const; @@ -162,11 +162,20 @@ public: // node; returns false (no-op) otherwise. used by the camera-ring visual load to drop a // node outside the current distance ring once its position has been read. bool skip_to_node_end() { if( m_nodeend == nullptr ) { return false; } m_cursor = m_nodeend; m_nodeend = nullptr; return true; } - // local position of the node currently being served, if its marker carried one (visual - // model nodes, v7+). returns false for shapes / infrastructure / older twins. - bool node_position( double &X, double &Y, double &Z ) const { + // local position + visibility range (range_max) of the node currently being served, if its + // marker carried one (visual model nodes, v8+). returns false for shapes / infrastructure / + // older twins. Range lets the streamer build far-but-large-range models (range_max beyond + // the stream radius) eagerly instead of dropping them when their section is out of range. + bool node_position( double &X, double &Y, double &Z, double &Range ) const { if( false == m_nodehaspos ) { return false; } - X = m_nodepos[ 0 ]; Y = m_nodepos[ 1 ]; Z = m_nodepos[ 2 ]; return true; } + X = m_nodepos[ 0 ]; Y = m_nodepos[ 1 ]; Z = m_nodepos[ 2 ]; Range = m_noderange; return true; } + // byte offset of the node currently being served, for the section-index streamer to seek back + std::size_t node_offset() const { return m_nodeoffset; } + // reposition at a node's marker (recorded via node_offset()) so the next next() re-serves it; + // used to rebuild a section's nodes on demand without re-scanning the twin + void seek_node( std::size_t Offset ) { + m_cursor = ( m_begin + Offset <= m_end ) ? m_begin + Offset : m_end; + m_nodeend = nullptr; m_nodehaspos = false; } bool exhausted() const { return m_cursor >= m_end; } // fraction of bytes consumed so far, 0..100, for the loading bar int progress() const { return ( m_size == 0 ? 100 : static_cast( ( m_cursor - m_begin ) * 100 / m_size ) ); } @@ -177,7 +186,9 @@ private: char const *m_cursor { nullptr }; char const *m_end { nullptr }; char const *m_nodeend { nullptr }; // end of the node currently being served (for skip_to_node_end) - double m_nodepos[ 3 ] { 0.0, 0.0, 0.0 }; // local position of the current node (v7 model marker) + double m_nodepos[ 3 ] { 0.0, 0.0, 0.0 }; // local position of the current node (v8 model marker) + double m_noderange { -1.0 }; // range_max of the current node (v9 model marker) + std::size_t m_nodeoffset { 0 }; // byte offset of the current node's marker (for seek_node) bool m_nodehaspos { false }; // whether the current node's marker carried a position std::ptrdiff_t m_size { 0 }; // entry section byte length scenery_load_pass m_pass { scenery_load_pass::all }; diff --git a/simulation/simulationstateserializer.cpp b/simulation/simulationstateserializer.cpp index 92e31750..7f8474db 100644 --- a/simulation/simulationstateserializer.cpp +++ b/simulation/simulationstateserializer.cpp @@ -37,25 +37,43 @@ http://mozilla.org/MPL/2.0/. namespace simulation { namespace { -// camera-distance rings for nearest-first visual streaming: the squared outer radius of -// each ring. the visual pass is replayed once per ring (nearest first), building only the -// nodes whose squared distance to the camera falls in [inner, outer); the last ring is -// unbounded so every remaining node is built exactly once. -constexpr double RING_OUTER2[] = { - 500.0 * 500.0, - 1500.0 * 1500.0, - 4000.0 * 4000.0, - std::numeric_limits::infinity() }; -constexpr int RING_COUNT { static_cast( sizeof( RING_OUTER2 ) / sizeof( RING_OUTER2[ 0 ] ) ) }; -inline int ring_lastindex() { return RING_COUNT - 1; } -inline double ring_min2( int const K ) { return ( K <= 0 ? 0.0 : RING_OUTER2[ K - 1 ] ); } -inline double ring_max2( int const K ) { return RING_OUTER2[ ( K < RING_COUNT ? K : RING_COUNT - 1 ) ]; } +// camera-following visual streaming: visual nodes are built only for region sections within +// this radius (m) of the camera, and more are built as the camera moves into new sections. +// should comfortably cover the model render range so nothing visibly pops in at the edge. +constexpr double STREAM_RADIUS { 2000.0 }; // per-frame time budget (ms) the driver spends streaming visual nodes. larger = the // surroundings fill in faster but the frame it runs on is longer; on a heavy scene (low fps) // a too-small budget is a tiny duty cycle, so streaming a million flora instances drags. constexpr int VISUAL_BUDGET_MS { 24 }; + +// fills Tobuild with the region-section indices within STREAM_RADIUS of Eye that are not yet +// in Built. mirrors basic_region::section() indexing (clamped to the grid). returns count. +std::size_t wanted_sections( glm::dvec3 const &Eye, std::unordered_set const &Built, std::unordered_set &Tobuild ) { + Tobuild.clear(); + int const N { scene::EU07_REGIONSIDESECTIONCOUNT }; + int const ccol { static_cast( std::floor( Eye.x / scene::EU07_SECTIONSIZE + N / 2 ) ) }; + int const crow { static_cast( std::floor( Eye.z / scene::EU07_SECTIONSIZE + N / 2 ) ) }; + int const span { static_cast( std::ceil( STREAM_RADIUS / scene::EU07_SECTIONSIZE ) ) }; + for( int r = crow - span; r <= crow + span; ++r ) { + for( int c = ccol - span; c <= ccol + span; ++c ) { + int const idx { std::clamp( r, 0, N - 1 ) * N + std::clamp( c, 0, N - 1 ) }; + if( 0 == Built.count( idx ) ) { Tobuild.insert( idx ); } + } + } + return Tobuild.size(); +} } // anonymous namespace +// region-section index enclosing a world position (row-major, clamped) -- matches +// basic_region::section() so a node buckets into the same section it inserts into. +int +state_serializer::section_index( glm::dvec3 const &World ) { + int const N { scene::EU07_REGIONSIDESECTIONCOUNT }; + int const col { static_cast( std::floor( World.x / scene::EU07_SECTIONSIZE + N / 2 ) ) }; + int const row { static_cast( std::floor( World.z / scene::EU07_SECTIONSIZE + N / 2 ) ) }; + return std::clamp( row, 0, N - 1 ) * N + std::clamp( col, 0, N - 1 ); +} + std::shared_ptr state_serializer::deserialize_begin( std::string const &Scenariofile ) { @@ -139,16 +157,30 @@ state_serializer::deserialize_continue(std::shared_ptr state cParser &Input = state->input; scene::scratch_data &Scratchpad = state->scratchpad; - // mirror the camera-ring state so deserialize_model()/deserialize_node() can ring-test - // each node by distance: in the visual phase the twin is replayed once per ring and only - // the nodes in the current ring are built, the rest skipped in O(1) + // reset the transform stack before each replay pass. the directives (origin/rotate/scale) + // are replayed in order, so resetting here reproduces the single-pass placement exactly; + // without it an unbalanced origin left on the stack would be applied again and shift nodes. + auto const resettransform = [ &Scratchpad ]() { + // a well-formed pass ends with a balanced (empty) transform stack; a leftover means an + // origin/scale was pushed but never popped -- e.g. a node whose binary marker span + // over-ran its terminator and skipped the following endorigin. warn rather than let it + // silently accumulate into the next pass. + if( false == Scratchpad.location.offset.empty() ) { + WriteLog( "Bad scenery: " + std::to_string( Scratchpad.location.offset.size() ) + " unbalanced origin(s) left on the stack at end of a load pass" ); + } + Scratchpad.location.offset = {}; + Scratchpad.location.scale = {}; + Scratchpad.location.rotation = glm::vec3{}; }; + + // mirror the visual-streaming state so deserialize_model()/deserialize_node() can decide + // whether a node belongs to the section set being built this cycle (or, in ringall, build + // everything). inactive (builds everything) outside the visual phase. m_ringactive = state->visualphase; if( true == m_ringactive ) { if( false == state->ringeye_valid ) { - // the visual streaming builds nodes nearest-this-point first, so it must be the - // player's spawn. the camera isn't positioned during load (especially in ghostview), - // so prefer the player vehicle's position; fall back to the camera, then the - // scenery's first camera directive. wait a few frames if nothing is available yet. + // the camera centre decides spawn-area-first streaming; the camera isn't positioned + // during load (especially ghostview), so prefer the player vehicle, then the camera, + // then the scenery's first camera directive. wait a few frames if nothing is ready. auto const iszero = []( glm::dvec3 const &V ) { return ( V.x == 0.0 ) && ( V.y == 0.0 ) && ( V.z == 0.0 ); }; glm::dvec3 eye = Global.pCamera.Pos; char const *src = "camera"; @@ -156,6 +188,12 @@ state_serializer::deserialize_continue(std::shared_ptr state auto *player = simulation::Vehicles.find( Global.local_start_vehicle ); if( player != nullptr ) { eye = player->GetPosition(); src = "player vehicle"; } } + if( ( true == iszero( eye ) ) && ( false == simulation::Vehicles.sequence().empty() ) ) { + // no designated player (e.g. ghostview), but the scenery has consists -- centre on + // the first one; it sits on the network, near where the action is. + eye = simulation::Vehicles.sequence().front()->GetPosition(); + src = "first vehicle"; + } if( true == iszero( eye ) ) { eye = Global.FreeCameraInit[ 0 ]; src = "camera directive"; } if( ( true == iszero( eye ) ) && ( state->ringeye_waits < 120 ) ) { ++state->ringeye_waits; @@ -163,20 +201,55 @@ state_serializer::deserialize_continue(std::shared_ptr state } state->ringeye = eye; state->ringeye_valid = true; - // no spawn/camera to centre on (e.g. ghostview): nearest-first is meaningless, so - // build every visual node in a single budgeted pass instead of partitioning into - // distance rings (which would dump everything in the far ring and stream it last). + // no spawn/camera to centre on (e.g. ghostview at the origin): camera-following is + // meaningless, so build every visual node in one pass. otherwise stream by section. state->ringall = iszero( eye ); + state->sectionmode = ( false == state->ringall ); WriteLog( std::string( "Progressive visual load: " ) + ( state->ringall ? "no camera centre -- building all visual nodes in one pass" : - "ring centre at " + std::to_string( eye.x ) + " " + std::to_string( eye.y ) + " " + std::to_string( eye.z ) + " (from " + src + ")" ) ); + "streaming sections within " + std::to_string( static_cast( STREAM_RADIUS ) ) + "m of the camera (from " + src + ")" ) ); } m_ringall = state->ringall; - m_ringindex = state->ringindex; + m_sectionmode = state->sectionmode; + m_shapes_built = state->shapes_built; m_ringeye = state->ringeye; - m_ringmin2 = ring_min2( state->ringindex ); - m_ringmax2 = ring_max2( state->ringindex ); + m_tobuild = &state->tobuild; + + // section streaming. the first pass replays the whole twin once, indexing every deferred + // node under its section while building the spawn area. afterwards (state->indexed) the + // sections the camera moves into are rebuilt by seeking straight to their nodes -- no more + // whole-twin re-scans, which is what was tanking fps / dragging on a million-node scenery. + if( true == state->sectionmode ) { + if( true == state->indexed ) { + if( true == state->tobuild.empty() ) { + if( 0 == wanted_sections( Global.pCamera.Pos, state->built, state->tobuild ) ) { + return true; // surroundings already built; stay alive for camera moves + } + } + m_rebuilding = true; + m_state = state.get(); + auto streamstart { std::chrono::steady_clock::now() }; + while( false == state->tobuild.empty() ) { + int const sec = *state->tobuild.begin(); + state->tobuild.erase( state->tobuild.begin() ); + rebuild_section( *state, sec ); + state->built.insert( sec ); + if( std::chrono::duration_cast( std::chrono::steady_clock::now() - streamstart ).count() >= VISUAL_BUDGET_MS ) { break; } + } + m_rebuilding = false; + return true; + } + // first pass: index every deferred node while building the spawn area + m_indexing = true; + m_state = state.get(); + if( false == state->pass_active ) { + wanted_sections( Global.pCamera.Pos, state->built, state->tobuild ); + Input.restartReplay( scene::scenery_load_pass::visual ); + resettransform(); + state->pass_active = true; + } + } } // stateful directives that build objects/lists; on the visual (second) pass they are @@ -195,7 +268,10 @@ state_serializer::deserialize_continue(std::shared_ptr state { "terrain", "endterrain" }, }; - // deserialize content from the provided input + // deserialize content from the provided input. modest budget while streaming visuals in the + // driver (rendering is live, so a big slice would tank fps), generous budget while the loading + // screen is up (infrastructure pass, nothing rendering yet). + int const budget { state->visualphase ? VISUAL_BUDGET_MS : 200 }; auto timelast { std::chrono::steady_clock::now() }; std::string token { Input.getToken() }; while( false == token.empty() ) { @@ -208,21 +284,28 @@ state_serializer::deserialize_continue(std::shared_ptr state token = Input.getToken(); continue; } - // fast camera-ring skip: a visual model node carries its local position in its v7 - // marker, so we can distance-test it and drop it in O(1) -- without deserialize_node - // decoding any of its tokens. this is what keeps the per-ring replay cheap when a - // scenery has a million flora instances. (shapes/older twins have no marker - // position; they fall through to deserialize_node, which ring-tests them itself.) - if( ( false == m_ringall ) && ( token == "node" ) ) { - double x, y, z; - if( true == Input.currentNodePosition( x, y, z ) ) { + // fast section skip: a visual model node carries its local position in its v7 + // marker, so we can section-test it and drop it in O(1) -- without deserialize_node + // decoding any of its tokens. this is what keeps the per-cycle replay cheap when a + // scenery has a million flora instances. (shapes/older twins have no marker position; + // they fall through to deserialize_node, which section-tests them itself.) + if( ( true == m_sectionmode ) && ( token == "node" ) ) { + double x, y, z, range; + if( true == Input.currentNodePosition( x, y, z, range ) ) { auto const world { transform( glm::dvec3{ x, y, z }, Scratchpad ) }; - auto const d { world - m_ringeye }; - auto const d2 { d.x * d.x + d.y * d.y + d.z * d.z }; - if( ( ( d2 < m_ringmin2 ) || ( d2 >= m_ringmax2 ) ) + // a model visible from beyond the stream radius (large/unlimited range_max) is + // built once, up front, regardless of section -- otherwise it would vanish at + // distance. the rest are indexed and built when their section comes into range. + bool const eager { ( range < 0.0 ) || ( range > STREAM_RADIUS ) }; + if( ( true == m_indexing ) && ( false == eager ) ) { capture_node( Input, Scratchpad, world ); } + bool const wanted { + eager ? + ( false == m_shapes_built ) : + ( 0 != m_tobuild->count( section_index( world ) ) ) }; + if( ( false == wanted ) && ( true == Input.skipReplayNode() ) ) { auto timenow = std::chrono::steady_clock::now(); - if( std::chrono::duration_cast( timenow - timelast ).count() >= VISUAL_BUDGET_MS ) { + if( std::chrono::duration_cast( timenow - timelast ).count() >= budget ) { Application.set_progress( Input.getProgress(), Input.getFullProgress() ); return true; } @@ -242,9 +325,6 @@ state_serializer::deserialize_continue(std::shared_ptr state } auto timenow = std::chrono::steady_clock::now(); - // per-frame budget while streaming visuals in the driver (kept modest to limit - // stutter), generous budget while the loading screen is up (infrastructure pass) - auto const budget = ( state->visualphase ? VISUAL_BUDGET_MS : 200 ); if( std::chrono::duration_cast( timenow - timelast ).count() >= budget ) { Application.set_progress( Input.getProgress(), Input.getFullProgress() ); return true; @@ -258,67 +338,116 @@ state_serializer::deserialize_continue(std::shared_ptr state deserialize_firstinit( Input, Scratchpad ); } - // helper: reset the transform stack before each replay pass. the directives - // (origin/rotate/scale) are replayed in order, so resetting here reproduces the - // single-pass placement exactly; without it an unbalanced origin left on the stack would - // be applied again and shift every deferred visual node ("terrain dumped beside the tracks"). - auto const resettransform = [ &Scratchpad ]() { - // a well-formed pass ends with a balanced (empty) transform stack; a leftover means - // an origin/scale was pushed but never popped -- e.g. a node whose binary marker span - // over-ran its terminator and skipped the following endorigin. warn rather than let it - // silently accumulate into the next pass. - if( false == Scratchpad.location.offset.empty() ) { - WriteLog( "Bad scenery: " + std::to_string( Scratchpad.location.offset.size() ) + " unbalanced origin(s) left on the stack at end of a load pass" ); - } - Scratchpad.location.offset = {}; - Scratchpad.location.scale = {}; - Scratchpad.location.rotation = glm::vec3{}; }; + // helper: make the scenario playable / persist the twin. the map, instance-bound events and + // twin flush are done once, after the first cycle (or the single build-all pass). the active + // group stack is left open on purpose in section mode -- later cycles keep inserting into it + // (update_map reads the persistent group map, not the stack, so it works either way). + auto const finalize = [ & ]( bool const Closegroups ) { + if( true == Closegroups ) { scene::Groups.close(); } + scene::Groups.update_map(); + Region->create_map_geometry(); + simulation::Events.InitInstanceEvents(); + Input.flushBinaryTwin(); + scene::scenerybinary_wait_all(); }; // first (infrastructure) pass finished: the scenario is now playable (tracks, events, - // signals, the player train are all loaded). hand control back so the loader can switch - // to the driver; the visual nodes load progressively from the driver, replayed once per - // camera-distance ring (nearest first). the ring centre is sampled later, on the first - // driver pass (the camera isn't positioned here yet). only possible when replaying a binary - // twin -- a text/compile load did everything in one pass (restartReplay returns false). + // signals, the player train are all loaded). hand control back so the loader can switch to + // the driver; the visual nodes stream in from the driver. the camera centre / mode are + // resolved later, on the first driver pass (the camera isn't positioned here yet). only + // possible when replaying a binary twin -- a text/compile load did everything in one pass. if( ( false == state->visualphase ) && ( true == Input.restartReplay( scene::scenery_load_pass::visual ) ) ) { state->visualphase = true; - state->ringindex = 0; resettransform(); - WriteLog( "Progressive visual load: streaming deferred nodes nearest-camera first (" + std::to_string( RING_COUNT ) + " rings)" ); + WriteLog( "Progressive visual load: infrastructure ready, streaming visuals from the driver" ); return false; // infrastructure ready -> go to driver; visuals continue there } - // a ring pass finished: advance to the next (farther) ring and replay again, until the - // outermost ring has been built. skipped in build-all mode, which builds everything in one pass. - if( ( true == state->visualphase ) - && ( false == state->ringall ) - && ( state->ringindex < ring_lastindex() ) - && ( true == Input.restartReplay( scene::scenery_load_pass::visual ) ) ) { - ++state->ringindex; - resettransform(); - return true; // more rings to build + // section streaming: a build cycle's replay pass just finished. mark its sections built so + // they aren't rebuilt, finalize once after the first cycle, and stay alive so the next call + // can pick up sections the camera has since moved into. the load never reports "done". + if( ( true == state->visualphase ) && ( true == state->sectionmode ) ) { + if( true == state->pass_active ) { + state->built.insert( std::begin( state->tobuild ), std::end( state->tobuild ) ); + state->tobuild.clear(); + state->pass_active = false; + state->shapes_built = true; // explicit shapes + eager models are built in the first pass + m_indexing = false; + if( false == state->initial_done ) { + finalize( /*Closegroups*/ false ); // keep groups open for later cycles + state->initial_done = true; + state->indexed = true; // the first pass has indexed every deferred node by section + std::size_t refs = 0; for( auto const &s : state->index ) { refs += s.second.size(); } + WriteLog( "Progressive visual load: spawn ready (" + std::to_string( simulation::Instances.sequence().size() ) + + " instances), " + std::to_string( refs ) + " nodes indexed across " + std::to_string( state->index.size() ) + " sections" ); + } + } + return true; // keep streaming alive; sections the camera enters are served from the index } - scene::Groups.close(); + // build-all (no camera centre, e.g. ghostview): everything was built in this single pass. + finalize( /*Closegroups*/ true ); + state->done = true; + return false; +} - scene::Groups.update_map(); - Region->create_map_geometry(); +int +state_serializer::twin_id( deserializer_state &State, std::string const &File, std::string const &Path ) { + std::string key = Path + "|" + File; + auto const it = State.twinids.find( key ); + if( it != State.twinids.end() ) { return it->second; } + int const id = static_cast( State.twins.size() ); + State.twins.emplace_back( File, Path ); + State.twinids.emplace( std::move( key ), id ); + return id; +} - // all nodes (including visual model instances) are now loaded, so initialise the - // events that bind to model instances; in a single-pass (text/compile) load nothing - // was deferred, but InitEvents() still skipped them, so do it here too - simulation::Events.InitInstanceEvents(); +void +state_serializer::capture_node( cParser &Input, scene::scratch_data const &Scratchpad, glm::dvec3 const &World ) { + // record where the node lives and the context it needs, so rebuild_section() can place it + // identically later without re-scanning the twin. only small-range nodes are indexed; large/ + // unlimited-range ("eager") ones are built once up front and never streamed again. + if( m_state == nullptr ) { return; } + visual_ref ref; + ref.twin = twin_id( *m_state, Input.currentReplayFile(), Input.currentReplayPath() ); + ref.offset = Input.currentReplayOffset(); + ref.has_offset = ( false == Scratchpad.location.offset.empty() ); + if( true == ref.has_offset ) { ref.t_offset = Scratchpad.location.offset.top(); } + ref.has_scale = ( false == Scratchpad.location.scale.empty() ); + if( true == ref.has_scale ) { ref.t_scale = Scratchpad.location.scale.top(); } + ref.t_rotation = Scratchpad.location.rotation; + ref.params = Input.currentReplayParams(); + m_state->index[ section_index( World ) ].emplace_back( std::move( ref ) ); +} - // loading finished: flush the top-level scenario's binary twin now rather than - // waiting for the parser to be destroyed (the loader keeps the state around) - Input.flushBinaryTwin(); - // wait out any background twin writes (includes) so they are complete and logged - // before we report the scenario as loaded - scene::scenerybinary_wait_all(); - - state->done = true; - return false; +void +state_serializer::rebuild_section( deserializer_state &State, int Section ) { + auto const it = State.index.find( Section ); + if( it == State.index.end() ) { return; } + scene::scratch_data &Scratchpad = State.scratchpad; + for( auto &ref : it->second ) { + // reuse one parser per twin across the whole stream (re-opening an .inc is not free) + auto pit = State.rebuild_parsers.find( ref.twin ); + if( pit == State.rebuild_parsers.end() ) { + auto const &tw = State.twins[ ref.twin ]; + pit = State.rebuild_parsers.emplace( + ref.twin, + std::make_unique( tw.first, cParser::buffer_FILE, tw.second, Global.bLoadTraction ) ).first; + } + cParser &cp = *pit->second; + cp.seekReplayNode( ref.offset ); + cp.setReplayParams( ref.params ); + // restore the transform context captured for this node + Scratchpad.location.offset = {}; + if( true == ref.has_offset ) { Scratchpad.location.offset.push( ref.t_offset ); } + Scratchpad.location.scale = {}; + if( true == ref.has_scale ) { Scratchpad.location.scale.push( ref.t_scale ); } + Scratchpad.location.rotation = ref.t_rotation; + auto const tok = cp.getToken(); + if( tok == "node" ) { deserialize_node( cp, Scratchpad ); } + } + // the section is built; release its index entries + std::vector().swap( it->second ); } void @@ -703,12 +832,23 @@ state_serializer::deserialize_node( cParser &Input, scene::scratch_data &Scratch || ( nodedata.type == "triangle_strip" ) || ( nodedata.type == "triangle_fan" ) ) { - // explicit shapes have no single position to ring-test by, so build them only in the - // nearest ring pass (ring 0) and skip them in O(1) on the farther passes (build-all - // mode is a single pass, so they always build there) - if( ( true == m_ringactive ) && ( false == m_ringall ) && ( m_ringindex > 0 ) ) { - if( false == Input.skipReplayNode() ) { skip_until( Input, "endtri" ); } - return; + // origin-placed shapes (e.g. flora includes) carry their world position in the active + // origin, so they section-stream like models: indexed on the first pass, rebuilt per + // section after. absolute shapes (terrain, no origin) have no single position -> built + // once in the first pass. (m_rebuilding: chosen from the index -> build unconditionally.) + if( ( true == m_sectionmode ) && ( false == m_rebuilding ) && ( nullptr != m_tobuild ) ) { + if( false == Scratchpad.location.offset.empty() ) { + glm::dvec3 const world { Scratchpad.location.offset.top() }; + if( true == m_indexing ) { capture_node( Input, Scratchpad, world ); } + if( 0 == m_tobuild->count( section_index( world ) ) ) { + if( false == Input.skipReplayNode() ) { skip_until( Input, "endtri" ); } + return; + } + } + else if( true == m_shapes_built ) { + if( false == Input.skipReplayNode() ) { skip_until( Input, "endtri" ); } + return; + } } auto const skip { @@ -734,10 +874,20 @@ state_serializer::deserialize_node( cParser &Input, scene::scratch_data &Scratch || ( nodedata.type == "line_strip" ) || ( nodedata.type == "line_loop" ) ) { - // see the triangles branch: explicit shapes build in ring 0 only (or always, build-all) - if( ( true == m_ringactive ) && ( false == m_ringall ) && ( m_ringindex > 0 ) ) { - if( false == Input.skipReplayNode() ) { skip_until( Input, "endline" ); } - return; + // see the triangles branch: origin-placed lines section-stream, absolute ones build once. + if( ( true == m_sectionmode ) && ( false == m_rebuilding ) && ( nullptr != m_tobuild ) ) { + if( false == Scratchpad.location.offset.empty() ) { + glm::dvec3 const world { Scratchpad.location.offset.top() }; + if( true == m_indexing ) { capture_node( Input, Scratchpad, world ); } + if( 0 == m_tobuild->count( section_index( world ) ) ) { + if( false == Input.skipReplayNode() ) { skip_until( Input, "endline" ); } + return; + } + } + else if( true == m_shapes_built ) { + if( false == Input.skipReplayNode() ) { skip_until( Input, "endline" ); } + return; + } } simulation::Region->insert( @@ -1085,21 +1235,28 @@ state_serializer::deserialize_model( cParser &Input, scene::scratch_data &Scratc >> location.z >> rotation.y; - // camera-ring visual streaming: build this model only if it falls in the ring currently - // being streamed; otherwise skip the rest of its body in O(1) and let a later (farther) - // ring pass pick it up. covers terrain models (range_min<0) too -- they also have X Y Z. - // most out-of-ring models are already dropped O(1) at the dispatch loop via their v7 - // marker; this is the fallback for nodes that reached here (in-ring, or no marker pos). - // use the marker position when present so this ring decision matches the dispatch one - // exactly -- otherwise a node near a ring boundary could be dropped by both adjacent rings. - if( ( true == m_ringactive ) && ( false == m_ringall ) ) { - glm::dvec3 ringlocal { location }; - double mx, my, mz; - if( true == Input.currentNodePosition( mx, my, mz ) ) { ringlocal = glm::dvec3{ mx, my, mz }; } - auto const world { transform( ringlocal, Scratchpad ) }; - auto const d { world - m_ringeye }; - auto const d2 { d.x * d.x + d.y * d.y + d.z * d.z }; - if( ( d2 < m_ringmin2 ) || ( d2 >= m_ringmax2 ) ) { + // camera-following visual streaming: build this model only if its region section is in the + // set being built this cycle; otherwise skip the rest of its body in O(1) and let a later + // cycle (once the camera is near) pick it up. covers terrain models (range_min<0) too -- they + // also have X Y Z. most out-of-range models are already dropped O(1) at the dispatch loop via + // their v7 marker; this is the fallback for nodes that reached here (in-range, or no marker). + // use the marker position when present so this decision matches the dispatch one exactly. + // m_rebuilding: this node was chosen from the section index, so build it unconditionally. + if( ( true == m_sectionmode ) && ( false == m_rebuilding ) && ( nullptr != m_tobuild ) ) { + // models visible from beyond the stream radius build once (first cycle); the rest build + // when their section is in range. mirrors the dispatch-loop fast path exactly. + bool const eager { ( Nodedata.range_max < 0.0 ) || ( Nodedata.range_max > STREAM_RADIUS ) }; + bool wanted; + if( true == eager ) { + wanted = ( false == m_shapes_built ); + } + else { + glm::dvec3 modellocal { location }; + double mx, my, mz, mr; + if( true == Input.currentNodePosition( mx, my, mz, mr ) ) { modellocal = glm::dvec3{ mx, my, mz }; } + wanted = ( 0 != m_tobuild->count( section_index( transform( modellocal, Scratchpad ) ) ) ); + } + if( false == wanted ) { if( false == Input.skipReplayNode() ) { skip_until( Input, "endmodel" ); } return nullptr; } diff --git a/simulation/simulationstateserializer.h b/simulation/simulationstateserializer.h index c3aad5ce..4b563923 100644 --- a/simulation/simulationstateserializer.h +++ b/simulation/simulationstateserializer.h @@ -14,6 +14,22 @@ http://mozilla.org/MPL/2.0/. namespace simulation { +// a deferred visual node recorded during the first (indexing) replay pass, so it can be rebuilt +// on demand when its region section enters camera range -- without re-scanning the whole twin. +// holds where to find the node (which twin + byte offset of its marker) and the context needed to +// place it identically (the transform active at that point, and the include parameters its "(pN)" +// tokens resolve against). +struct visual_ref { + int twin { -1 }; // index into deserializer_state::twins (file+path to re-open) + std::size_t offset { 0 }; // byte offset of the node's marker within that twin + glm::dvec3 t_offset { 0.0 }; + glm::vec3 t_rotation { 0.f }; + glm::vec3 t_scale { 1.f }; + bool has_offset { false }; + bool has_scale { false }; + std::vector params; // include parameters in effect (empty for a direct node) +}; + struct deserializer_state { std::string scenariofile; cParser input; @@ -27,18 +43,32 @@ struct deserializer_state { bool visualphase { false }; // set once the whole load (both passes / single text pass) has fully finished bool done { false }; - // camera-ring visual streaming: the visual phase replays the twin once per distance - // ring (nearest first), building only the nodes whose distance to ringeye falls in the - // current ring and O(1)-skipping the rest. ringeye is sampled once when the visual phase - // starts so the ring partition is stable across passes. - int ringindex { 0 }; + // camera-following visual streaming. once infrastructure is up, visual nodes (3d models, + // terrain shapes) stream in only for the region sections currently within STREAM_RADIUS of + // the camera; as the camera moves into new sections they are built too. nothing is unloaded. + // the twin is replayed once per "build cycle" (a set of newly-wanted sections), building + // only nodes whose section is in that set and O(1)-skipping the rest by their v7 marker. glm::dvec3 ringeye { 0.0 }; - // ringeye must be sampled from the camera *after* control passes to the driver (the - // loader hasn't positioned the camera yet -- sampling there put every node in the far - // ring and built models last / seemingly never). sampled lazily on the first driver pass. + // the camera centre is only meaningful once control reaches the driver (the loader hasn't + // positioned the camera yet). sampled lazily on the first driver pass; if still unusable + // (e.g. ghostview at the origin) we fall back to building everything in one pass (ringall). bool ringeye_valid { false }; int ringeye_waits { 0 }; // frames spent waiting for the camera to be positioned bool ringall { false }; // no usable camera centre -> build all visual nodes in one pass + bool sectionmode { false }; // usable camera centre -> stream sections within range, follow camera + bool shapes_built { false }; // first cycle done: explicit shapes + large-range (eager) models built + bool initial_done { false }; // the first cycle finished -> scenario finalised (map/events/twin) + bool pass_active { false }; // a section build cycle's replay pass is currently in progress + std::unordered_set built; // region-section indices already streamed in + std::unordered_set tobuild; // section indices targeted by the current/next cycle + // section index, built during the first replay pass: every deferred visual node is recorded + // under its region section, so later sections rebuild by seeking straight to their nodes + // instead of replaying (re-scanning) the whole million-node twin every cycle. + bool indexed { false }; // first pass finished -> the index below is complete + std::vector> twins; // (file, path) to re-open, interned + std::unordered_map twinids; // "path|file" -> index into twins + std::unordered_map> index; // section -> deferred nodes there + std::unordered_map> rebuild_parsers; // one reused parser per twin deserializer_state(std::string const &File, cParser::buffertype const Type, const std::string &Path, bool const Loadtraction) : scenariofile(File), input(File, Type, Path, Loadtraction) { } @@ -105,16 +135,33 @@ private: // transforms provided location by specifed rotation and offset glm::dvec3 transform( glm::dvec3 Location, scene::scratch_data const &Scratchpad ); void export_nodes_to_stream( std::ostream &, bool Dirty ) const; + // region-section index (row-major, clamped to the grid) enclosing a world position -- + // matches basic_region::section()'s indexing, used to bucket visual nodes for streaming. + static int section_index( glm::dvec3 const &World ); + // record the visual node currently being replayed (twin/offset/transform/params) under its + // section in the index, so it can be rebuilt later without re-scanning the twin. + void capture_node( cParser &Input, scene::scratch_data const &Scratchpad, glm::dvec3 const &World ); + // rebuild every indexed node of one section by seeking straight to it (no twin re-scan). + void rebuild_section( deserializer_state &State, int Section ); + // interns a (file, path) pair into State.twins, returning its index + static int twin_id( deserializer_state &State, std::string const &File, std::string const &Path ); // members - // camera-ring visual streaming state, mirrored from deserializer_state each - // deserialize_continue() call so deserialize_model()/deserialize_node() can ring-test a - // node by distance. inactive (builds everything) outside the ring/visual phase. + // camera-following visual streaming state, mirrored from deserializer_state each + // deserialize_continue() call so deserialize_model()/deserialize_node() can decide whether + // a node's section is in the current build set. inactive (builds everything) outside the + // visual phase, or in ringall (no camera centre) where every node is built in one pass. bool m_ringactive { false }; - bool m_ringall { false }; // no camera centre available -> build every node in one pass - int m_ringindex { 0 }; + bool m_ringall { false }; // no camera centre available -> build every node in one pass + bool m_sectionmode { false }; // stream by camera-range sections + bool m_shapes_built { false }; // first cycle done: shapes + large-range (eager) models built (skip them) glm::dvec3 m_ringeye { 0.0 }; - double m_ringmin2 { 0.0 }; - double m_ringmax2 { 0.0 }; + std::unordered_set const *m_tobuild { nullptr }; // section indices to build this cycle + // first replay pass records each deferred node into the index (m_indexing); later cycles + // rebuild sections straight from it (m_rebuilding bypasses the section test so the chosen + // node always builds). m_state gives deserialize_model()/node() access for capture. + bool m_indexing { false }; + bool m_rebuilding { false }; + deserializer_state *m_state { nullptr }; }; } // simulation diff --git a/utilities/parser.cpp b/utilities/parser.cpp index 5589a6f8..d213b194 100644 --- a/utilities/parser.cpp +++ b/utilities/parser.cpp @@ -319,7 +319,7 @@ void cParser::bakeFinishNode() // flush a node still open at end-of-file or one whose type was unrecognized if (m_bakenode_active && m_writer) { - m_writer->end_node(m_bakenode_visual, m_bakenode_haspos, m_bakenode_pos[0], m_bakenode_pos[1], m_bakenode_pos[2]); + m_writer->end_node(m_bakenode_visual, m_bakenode_haspos, m_bakenode_pos[0], m_bakenode_pos[1], m_bakenode_pos[2], m_bakenode_rangemax); } m_bakenode_active = false; } @@ -817,22 +817,48 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break) if (m_bakenode_active) { ++m_bakenode_count; + if (m_bakenode_count == 2) + { + // 2nd node entry is range_max (visibility range); kept in the model marker + // so the streamer builds far-but-large-range models eagerly. -1 (unlimited) + // when it isn't a plain number (e.g. an unresolved parameter). + double rmax; + m_bakenode_rangemax = (sniffNumber(rawtoken, rmax) ? rmax : -1.0); + } if ((m_bakenode_count == 5) && m_bakenode_end.empty()) { // 5th node entry is the type token (node, range_max, range_min, name, type) classifyNodeType(lowered, m_bakenode_visual, m_bakenode_end); // a model node's entries 6,7,8 are its local X Y Z -- record them in the // marker so the camera-ring load can skip the node without reading its body - m_bakenode_haspos = (lowered == "model"); + { + bool const isshape = + (lowered == "triangles") || (lowered == "triangle_strip") || (lowered == "triangle_fan") + || (lowered == "lines") || (lowered == "line_strip") || (lowered == "line_loop"); + m_bakenode_haspos = (lowered == "model") || isshape; + m_bakenode_shape = isshape; + m_bakenode_posstate = 0; + m_bakenode_posidx = 0; + } } - else if (m_bakenode_haspos && (m_bakenode_count >= 6) && (m_bakenode_count <= 8)) + else if (m_bakenode_haspos && (false == m_bakenode_shape) && (m_bakenode_count >= 6) && (m_bakenode_count <= 8)) { m_bakenode_pos[m_bakenode_count - 6] = value; } - else if ((false == m_bakenode_end.empty()) && (lowered == m_bakenode_end)) + else if (m_bakenode_shape && (m_bakenode_posstate < 3)) + { + // walk a shape to its first vertex: 0=before material -> if "material" enter the + // block (1), else it is a shortcut material (start seeking numbers, 2); 1=skip until + // "endmaterial"; 2=skip the texture string, then take the 3 vertex numbers. + double sv; + if (m_bakenode_posstate == 0) { m_bakenode_posstate = (lowered == "material") ? 1 : 2; } + else if (m_bakenode_posstate == 1) { if (lowered == "endmaterial") m_bakenode_posstate = 2; } + else if (sniffNumber(rawtoken, sv)) { m_bakenode_pos[m_bakenode_posidx++] = sv; if (m_bakenode_posidx == 3) m_bakenode_posstate = 3; } + } + else if ((false == m_bakenode_end.empty()) && (lowered == m_bakenode_end)) { // terminator captured: close the node - m_writer->end_node(m_bakenode_visual, m_bakenode_haspos, m_bakenode_pos[0], m_bakenode_pos[1], m_bakenode_pos[2]); + m_writer->end_node(m_bakenode_visual, m_bakenode_haspos, m_bakenode_pos[0], m_bakenode_pos[1], m_bakenode_pos[2], m_bakenode_rangemax); m_bakenode_active = false; } } @@ -924,11 +950,54 @@ bool cParser::skipReplayNode() return false; } -bool cParser::currentNodePosition(double &X, double &Y, double &Z) +bool cParser::currentNodePosition(double &X, double &Y, double &Z, double &Range) { // delegate to the deepest active include child (it serves the current node), like skip - if (mIncludeParser) { return mIncludeParser->currentNodePosition(X, Y, Z); } - return (m_replay && m_reader && m_reader->node_position(X, Y, Z)); + if (mIncludeParser) { return mIncludeParser->currentNodePosition(X, Y, Z, Range); } + return (m_replay && m_reader && m_reader->node_position(X, Y, Z, Range)); +} + +std::string cParser::currentReplayFile() +{ + if (mIncludeParser) { return mIncludeParser->currentReplayFile(); } + return mFile; +} + +std::string cParser::currentReplayPath() +{ + if (mIncludeParser) { return mIncludeParser->currentReplayPath(); } + return mPath; +} + +std::size_t cParser::currentReplayOffset() +{ + if (mIncludeParser) { return mIncludeParser->currentReplayOffset(); } + return (m_reader ? m_reader->node_offset() : 0); +} + +std::vector cParser::currentReplayParams() +{ + if (mIncludeParser) { return mIncludeParser->currentReplayParams(); } + return parameters; +} + +void cParser::seekReplayNode(std::size_t Offset) +{ + // rebuild serves a single self-contained node, so drop any open include child and rewind + // this twin's reader to the node's marker; the next getToken() decodes it + mIncludeParser = nullptr; + tokens.clear(); + if (m_replay && m_reader) + { + m_reader->set_pass(scene::scenery_load_pass::all); + m_reader->seek_node(Offset); + m_replayexhausted = false; + } +} + +void cParser::setReplayParams(std::vector Params) +{ + parameters = std::move(Params); } std::vector cParser::readParameters(cParser &Input) diff --git a/utilities/parser.h b/utilities/parser.h index 3ee7f902..53fa9dfa 100644 --- a/utilities/parser.h +++ b/utilities/parser.h @@ -65,7 +65,22 @@ class cParser //: public std::stringstream // local position of the node about to be replayed (the "node" token just read), if its // v7 marker carried one (visual model nodes). lets the camera-ring load distance-test and // skip a node without decoding its body. returns false for shapes / non-replay / older twins. - bool currentNodePosition( double &X, double &Y, double &Z ); + bool currentNodePosition( double &X, double &Y, double &Z, double &Range ); + // --- section-index streaming: capture enough to rebuild the node about to be replayed + // (the deepest active include serves it) without re-scanning the whole twin later --- + // source file + base path of the twin the current node lives in (to re-open it) + std::string currentReplayFile(); + std::string currentReplayPath(); + // byte offset of the current node's marker within that twin (to seek back to it) + std::size_t currentReplayOffset(); + // the include parameters in effect for the current node (its "(pN)" tokens resolve against + // these); empty for a node read directly from a parameterless file + std::vector currentReplayParams(); + // reposition this (replay) parser at a node recorded via currentReplayOffset() and serve it + // next; drops any open include child. used to rebuild one indexed node on demand. + void seekReplayNode( std::size_t Offset ); + // set the include parameters used to resolve the next node's "(pN)" tokens during rebuild + void setReplayParams( std::vector Params ); // methods: template cParser & @@ -222,6 +237,13 @@ class cParser //: public std::stringstream // camera-ring load can distance-test it without decoding the node body bool m_bakenode_haspos { false }; double m_bakenode_pos[ 3 ] { 0.0, 0.0, 0.0 }; + double m_bakenode_rangemax { -1.0 }; // range_max (entry 2), stored in the model marker + // explicit shape (triangles/lines) position extraction: a shape has no fixed-index position + // like a model, so walk past its material to its first vertex and use that. posstate: + // 0=before material, 1=inside material...endmaterial, 2=seeking the 3 vertex numbers, 3=done. + bool m_bakenode_shape { false }; + int m_bakenode_posstate { 0 }; + int m_bakenode_posidx { 0 }; // flushes a node still open at end-of-file (or unknown type), so its buffer is written void bakeFinishNode(); };