eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-18 00:49:19 +02:00
Code Issues 121 Wiki Activity

Section-index visual streaming (no per-cycle twin re-scan)

Browse Source 
Replaces the camera-distance ring passes with section-following streaming and a
persistent section index, so a million-node scenery no longer re-scans the whole
twin every time the camera moves into new ground.

- v9 node marker also stores range_max; a model visible from beyond the stream
  radius (or unlimited) is built once up front, the rest stream by section so
  distant landmarks/traction/buildings don't pop out while flora stays local.
- Reader gains node_offset()/seek_node(); cParser exposes the deepest twin's
  file/path/offset/params and seekReplayNode/setReplayParams to rebuild one node.
- First visual pass indexes every deferred node (models via the dispatch fast
  path, origin-placed flora/shapes in deserialize_node) under its region section
  while building the spawn area; later cycles rebuild only the newly-wanted
  sections by seeking straight to their nodes -- O(visible), not O(whole twin).
- Build-all fallback retained for ghostview with no camera centre.

Known: absolute terrain triangles (no origin) still build in the first pass; only
origin-placed content section-streams. Untested in-game (format bump needs a
rebake).
This commit is contained in:
maj00r
2026-06-24 23:56:53 +02:00
parent 16b39a21b4
commit 4a9bfdc0aa
6 changed files with 461 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
scene/scenerybinary.cpp
View File

@@ -178,7 +178,7 @@ scenery_binary_writer::begin_node() {
}
void
scenery_binary_writer::end_node( bool Visual, bool Haspos, double X, double Y, double Z ) {
scenery_binary_writer::end_node( bool Visual, bool Haspos, double X, double Y, double Z, double Range ) {
if( false == m_innode ) { return; }
m_innode = false;
auto const body = m_nodebuf.str();
@@ -189,12 +189,13 @@ scenery_binary_writer::end_node( bool Visual, bool Haspos, double X, double Y, d
NODECLASS_VISUAL;
write_varint( m_entries, cls );
write_varint( m_entries, body.size() );
// a visual model node stores its local position right after the span, so the reader can
// hand it to the camera-ring load without the node body being decoded
// a visual model node stores its local position + visibility range right after the span, so
// the reader can hand them to the streaming load without the node body being decoded
if( cls == NODECLASS_VISUAL_POS ) {
sn_utils::ls_float32( m_entries, static_cast<float>( X ) );
sn_utils::ls_float32( m_entries, static_cast<float>( Y ) );
sn_utils::ls_float32( m_entries, static_cast<float>( Z ) );
sn_utils::ls_float32( m_entries, static_cast<float>( Range ) );
}
m_entries.write( body.data(), static_cast<std::streamsize>( body.size() ) );
}
@@ -271,9 +272,13 @@ scenery_binary_reader::next( scenery_entry_view &Out ) {
std::uint64_t tag = 0;
for( ;; ) {
if( m_cursor >= m_end ) { return false; }
char const *markerstart = m_cursor; // where this entry's marker begins (for seek_node)
head = read_varint( m_cursor, m_end );
tag = head & TAG_MASK;
if( tag != TAG_NODE ) { break; }
// record the served node's marker offset so the section-index streamer can seek back to
// it and rebuild the node on demand without re-scanning the whole twin
m_nodeoffset = static_cast<std::size_t>( markerstart - m_begin );
auto const cls = read_varint( m_cursor, m_end );
auto const span = read_varint( m_cursor, m_end );
bool const isvisual = ( cls == NODECLASS_VISUAL ) || ( cls == NODECLASS_VISUAL_POS );
@@ -283,6 +288,7 @@ scenery_binary_reader::next( scenery_entry_view &Out ) {
m_nodepos[ 0 ] = static_cast<double>( read_f32le( m_cursor, m_end ) );
m_nodepos[ 1 ] = static_cast<double>( read_f32le( m_cursor, m_end ) );
m_nodepos[ 2 ] = static_cast<double>( read_f32le( m_cursor, m_end ) );
m_noderange = static_cast<double>( read_f32le( m_cursor, m_end ) );
}
bool const process =
( m_pass == scenery_load_pass::all )

25
scene/scenerybinary.h
View File

@@ -64,7 +64,7 @@ namespace scene {
// in the visual pass, swallowed the following endorigin -> the origin stack accumulated
// and flung terrain/models across the map. bumping invalidates those bad twins.
// bumping the version invalidates older twins so they are recompiled rather than misread.
constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 8 ) };
constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 9 ) };
// which entries a reader serves in a given load pass; nodes outside the requested class
// are skipped (directives/includes are always served, to keep transform/group state)
@@ -124,7 +124,7 @@ public:
// Haspos/X/Y/Z give a model node's local position so the camera-ring load can skip it
// without reading its tokens.
void begin_node();
void end_node( bool Visual, bool Haspos = false, double X = 0.0, double Y = 0.0, double Z = 0.0 );
void end_node( bool Visual, bool Haspos = false, double X = 0.0, double Y = 0.0, double Z = 0.0, double Range = -1.0 );
std::size_t entry_count() const { return m_count; }
// serializes header + string table + encoded entries. returns false on stream failure.
bool write( std::ostream &Output, scenery_file_kind Kind ) const;
@@ -162,11 +162,20 @@ public:
// node; returns false (no-op) otherwise. used by the camera-ring visual load to drop a
// node outside the current distance ring once its position has been read.
bool skip_to_node_end() { if( m_nodeend == nullptr ) { return false; } m_cursor = m_nodeend; m_nodeend = nullptr; return true; }
// local position of the node currently being served, if its marker carried one (visual
// model nodes, v7+). returns false for shapes / infrastructure / older twins.
bool node_position( double &X, double &Y, double &Z ) const {
// local position + visibility range (range_max) of the node currently being served, if its
// marker carried one (visual model nodes, v8+). returns false for shapes / infrastructure /
// older twins. Range lets the streamer build far-but-large-range models (range_max beyond
// the stream radius) eagerly instead of dropping them when their section is out of range.
bool node_position( double &X, double &Y, double &Z, double &Range ) const {
if( false == m_nodehaspos ) { return false; }
X = m_nodepos[ 0 ]; Y = m_nodepos[ 1 ]; Z = m_nodepos[ 2 ]; return true; }
X = m_nodepos[ 0 ]; Y = m_nodepos[ 1 ]; Z = m_nodepos[ 2 ]; Range = m_noderange; return true; }
// byte offset of the node currently being served, for the section-index streamer to seek back
std::size_t node_offset() const { return m_nodeoffset; }
// reposition at a node's marker (recorded via node_offset()) so the next next() re-serves it;
// used to rebuild a section's nodes on demand without re-scanning the twin
void seek_node( std::size_t Offset ) {
m_cursor = ( m_begin + Offset <= m_end ) ? m_begin + Offset : m_end;
m_nodeend = nullptr; m_nodehaspos = false; }
bool exhausted() const { return m_cursor >= m_end; }
// fraction of bytes consumed so far, 0..100, for the loading bar
int progress() const { return ( m_size == 0 ? 100 : static_cast<int>( ( m_cursor - m_begin ) * 100 / m_size ) ); }
@@ -177,7 +186,9 @@ private:
char const *m_cursor { nullptr };
char const *m_end { nullptr };
char const *m_nodeend { nullptr }; // end of the node currently being served (for skip_to_node_end)
double m_nodepos[ 3 ] { 0.0, 0.0, 0.0 }; // local position of the current node (v7 model marker)
double m_nodepos[ 3 ] { 0.0, 0.0, 0.0 }; // local position of the current node (v8 model marker)
double m_noderange { -1.0 }; // range_max of the current node (v9 model marker)
std::size_t m_nodeoffset { 0 }; // byte offset of the current node's marker (for seek_node)
bool m_nodehaspos { false }; // whether the current node's marker carried a position
std::ptrdiff_t m_size { 0 }; // entry section byte length
scenery_load_pass m_pass { scenery_load_pass::all };

387
simulation/simulationstateserializer.cpp
View File

@@ -37,25 +37,43 @@ http://mozilla.org/MPL/2.0/.
namespace simulation {
namespace {
// camera-distance rings for nearest-first visual streaming: the squared outer radius of
// each ring. the visual pass is replayed once per ring (nearest first), building only the
// nodes whose squared distance to the camera falls in [inner, outer); the last ring is
// unbounded so every remaining node is built exactly once.
constexpr double RING_OUTER2[] = {
500.0 * 500.0,
1500.0 * 1500.0,
4000.0 * 4000.0,
std::numeric_limits<double>::infinity() };
constexpr int RING_COUNT { static_cast<int>( sizeof( RING_OUTER2 ) / sizeof( RING_OUTER2[ 0 ] ) ) };
inline int ring_lastindex() { return RING_COUNT - 1; }
inline double ring_min2( int const K ) { return ( K <= 0 ? 0.0 : RING_OUTER2[ K - 1 ] ); }
inline double ring_max2( int const K ) { return RING_OUTER2[ ( K < RING_COUNT ? K : RING_COUNT - 1 ) ]; }
// camera-following visual streaming: visual nodes are built only for region sections within
// this radius (m) of the camera, and more are built as the camera moves into new sections.
// should comfortably cover the model render range so nothing visibly pops in at the edge.
constexpr double STREAM_RADIUS { 2000.0 };
// per-frame time budget (ms) the driver spends streaming visual nodes. larger = the
// surroundings fill in faster but the frame it runs on is longer; on a heavy scene (low fps)
// a too-small budget is a tiny duty cycle, so streaming a million flora instances drags.
constexpr int VISUAL_BUDGET_MS { 24 };
// fills Tobuild with the region-section indices within STREAM_RADIUS of Eye that are not yet
// in Built. mirrors basic_region::section() indexing (clamped to the grid). returns count.
std::size_t wanted_sections( glm::dvec3 const &Eye, std::unordered_set<int> const &Built, std::unordered_set<int> &Tobuild ) {
Tobuild.clear();
int const N { scene::EU07_REGIONSIDESECTIONCOUNT };
int const ccol { static_cast<int>( std::floor( Eye.x / scene::EU07_SECTIONSIZE + N / 2 ) ) };
int const crow { static_cast<int>( std::floor( Eye.z / scene::EU07_SECTIONSIZE + N / 2 ) ) };
int const span { static_cast<int>( std::ceil( STREAM_RADIUS / scene::EU07_SECTIONSIZE ) ) };
for( int r = crow - span; r <= crow + span; ++r ) {
for( int c = ccol - span; c <= ccol + span; ++c ) {
int const idx { std::clamp( r, 0, N - 1 ) * N + std::clamp( c, 0, N - 1 ) };
if( 0 == Built.count( idx ) ) { Tobuild.insert( idx ); }
}
}
return Tobuild.size();
}
} // anonymous namespace
// region-section index enclosing a world position (row-major, clamped) -- matches
// basic_region::section() so a node buckets into the same section it inserts into.
int
state_serializer::section_index( glm::dvec3 const &World ) {
int const N { scene::EU07_REGIONSIDESECTIONCOUNT };
int const col { static_cast<int>( std::floor( World.x / scene::EU07_SECTIONSIZE + N / 2 ) ) };
int const row { static_cast<int>( std::floor( World.z / scene::EU07_SECTIONSIZE + N / 2 ) ) };
return std::clamp( row, 0, N - 1 ) * N + std::clamp( col, 0, N - 1 );
}
std::shared_ptr<deserializer_state>
state_serializer::deserialize_begin( std::string const &Scenariofile ) {
@@ -139,16 +157,30 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
cParser &Input = state->input;
scene::scratch_data &Scratchpad = state->scratchpad;
// mirror the camera-ring state so deserialize_model()/deserialize_node() can ring-test
// each node by distance: in the visual phase the twin is replayed once per ring and only
// the nodes in the current ring are built, the rest skipped in O(1)
// reset the transform stack before each replay pass. the directives (origin/rotate/scale)
// are replayed in order, so resetting here reproduces the single-pass placement exactly;
// without it an unbalanced origin left on the stack would be applied again and shift nodes.
auto const resettransform = [ &Scratchpad ]() {
// a well-formed pass ends with a balanced (empty) transform stack; a leftover means an
// origin/scale was pushed but never popped -- e.g. a node whose binary marker span
// over-ran its terminator and skipped the following endorigin. warn rather than let it
// silently accumulate into the next pass.
if( false == Scratchpad.location.offset.empty() ) {
WriteLog( "Bad scenery: " + std::to_string( Scratchpad.location.offset.size() ) + " unbalanced origin(s) left on the stack at end of a load pass" );
}
Scratchpad.location.offset = {};
Scratchpad.location.scale = {};
Scratchpad.location.rotation = glm::vec3{}; };
// mirror the visual-streaming state so deserialize_model()/deserialize_node() can decide
// whether a node belongs to the section set being built this cycle (or, in ringall, build
// everything). inactive (builds everything) outside the visual phase.
m_ringactive = state->visualphase;
if( true == m_ringactive ) {
if( false == state->ringeye_valid ) {
// the visual streaming builds nodes nearest-this-point first, so it must be the
// player's spawn. the camera isn't positioned during load (especially in ghostview),
// so prefer the player vehicle's position; fall back to the camera, then the
// scenery's first camera directive. wait a few frames if nothing is available yet.
// the camera centre decides spawn-area-first streaming; the camera isn't positioned
// during load (especially ghostview), so prefer the player vehicle, then the camera,
// then the scenery's first camera directive. wait a few frames if nothing is ready.
auto const iszero = []( glm::dvec3 const &V ) { return ( V.x == 0.0 ) && ( V.y == 0.0 ) && ( V.z == 0.0 ); };
glm::dvec3 eye = Global.pCamera.Pos;
char const *src = "camera";
@@ -156,6 +188,12 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
auto *player = simulation::Vehicles.find( Global.local_start_vehicle );
if( player != nullptr ) { eye = player->GetPosition(); src = "player vehicle"; }
}
if( ( true == iszero( eye ) ) && ( false == simulation::Vehicles.sequence().empty() ) ) {
// no designated player (e.g. ghostview), but the scenery has consists -- centre on
// the first one; it sits on the network, near where the action is.
eye = simulation::Vehicles.sequence().front()->GetPosition();
src = "first vehicle";
}
if( true == iszero( eye ) ) { eye = Global.FreeCameraInit[ 0 ]; src = "camera directive"; }
if( ( true == iszero( eye ) ) && ( state->ringeye_waits < 120 ) ) {
++state->ringeye_waits;
@@ -163,20 +201,55 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
}
state->ringeye = eye;
state->ringeye_valid = true;
// no spawn/camera to centre on (e.g. ghostview): nearest-first is meaningless, so
// build every visual node in a single budgeted pass instead of partitioning into
// distance rings (which would dump everything in the far ring and stream it last).
// no spawn/camera to centre on (e.g. ghostview at the origin): camera-following is
// meaningless, so build every visual node in one pass. otherwise stream by section.
state->ringall = iszero( eye );
state->sectionmode = ( false == state->ringall );
WriteLog( std::string( "Progressive visual load: " )
+ ( state->ringall ?
"no camera centre -- building all visual nodes in one pass" :
"ring centre at " + std::to_string( eye.x ) + " " + std::to_string( eye.y ) + " " + std::to_string( eye.z ) + " (from " + src + ")" ) );
"streaming sections within " + std::to_string( static_cast<int>( STREAM_RADIUS ) ) + "m of the camera (from " + src + ")" ) );
}
m_ringall = state->ringall;
m_ringindex = state->ringindex;
m_sectionmode = state->sectionmode;
m_shapes_built = state->shapes_built;
m_ringeye = state->ringeye;
m_ringmin2 = ring_min2( state->ringindex );
m_ringmax2 = ring_max2( state->ringindex );
m_tobuild = &state->tobuild;
// section streaming. the first pass replays the whole twin once, indexing every deferred
// node under its section while building the spawn area. afterwards (state->indexed) the
// sections the camera moves into are rebuilt by seeking straight to their nodes -- no more
// whole-twin re-scans, which is what was tanking fps / dragging on a million-node scenery.
if( true == state->sectionmode ) {
if( true == state->indexed ) {
if( true == state->tobuild.empty() ) {
if( 0 == wanted_sections( Global.pCamera.Pos, state->built, state->tobuild ) ) {
return true; // surroundings already built; stay alive for camera moves
}
}
m_rebuilding = true;
m_state = state.get();
auto streamstart { std::chrono::steady_clock::now() };
while( false == state->tobuild.empty() ) {
int const sec = *state->tobuild.begin();
state->tobuild.erase( state->tobuild.begin() );
rebuild_section( *state, sec );
state->built.insert( sec );
if( std::chrono::duration_cast<std::chrono::milliseconds>( std::chrono::steady_clock::now() - streamstart ).count() >= VISUAL_BUDGET_MS ) { break; }
}
m_rebuilding = false;
return true;
}
// first pass: index every deferred node while building the spawn area
m_indexing = true;
m_state = state.get();
if( false == state->pass_active ) {
wanted_sections( Global.pCamera.Pos, state->built, state->tobuild );
Input.restartReplay( scene::scenery_load_pass::visual );
resettransform();
state->pass_active = true;
}
}
}
// stateful directives that build objects/lists; on the visual (second) pass they are
@@ -195,7 +268,10 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
{ "terrain", "endterrain" },
};
// deserialize content from the provided input
// deserialize content from the provided input. modest budget while streaming visuals in the
// driver (rendering is live, so a big slice would tank fps), generous budget while the loading
// screen is up (infrastructure pass, nothing rendering yet).
int const budget { state->visualphase ? VISUAL_BUDGET_MS : 200 };
auto timelast { std::chrono::steady_clock::now() };
std::string token { Input.getToken<std::string>() };
while( false == token.empty() ) {
@@ -208,21 +284,28 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
token = Input.getToken<std::string>();
continue;
}
// fast camera-ring skip: a visual model node carries its local position in its v7
// marker, so we can distance-test it and drop it in O(1) -- without deserialize_node
// decoding any of its tokens. this is what keeps the per-ring replay cheap when a
// scenery has a million flora instances. (shapes/older twins have no marker
// position; they fall through to deserialize_node, which ring-tests them itself.)
if( ( false == m_ringall ) && ( token == "node" ) ) {
double x, y, z;
if( true == Input.currentNodePosition( x, y, z ) ) {
// fast section skip: a visual model node carries its local position in its v7
// marker, so we can section-test it and drop it in O(1) -- without deserialize_node
// decoding any of its tokens. this is what keeps the per-cycle replay cheap when a
// scenery has a million flora instances. (shapes/older twins have no marker position;
// they fall through to deserialize_node, which section-tests them itself.)
if( ( true == m_sectionmode ) && ( token == "node" ) ) {
double x, y, z, range;
if( true == Input.currentNodePosition( x, y, z, range ) ) {
auto const world { transform( glm::dvec3{ x, y, z }, Scratchpad ) };
auto const d { world - m_ringeye };
auto const d2 { d.x * d.x + d.y * d.y + d.z * d.z };
if( ( ( d2 < m_ringmin2 ) || ( d2 >= m_ringmax2 ) )
// a model visible from beyond the stream radius (large/unlimited range_max) is
// built once, up front, regardless of section -- otherwise it would vanish at
// distance. the rest are indexed and built when their section comes into range.
bool const eager { ( range < 0.0 ) || ( range > STREAM_RADIUS ) };
if( ( true == m_indexing ) && ( false == eager ) ) { capture_node( Input, Scratchpad, world ); }
bool const wanted {
eager ?
( false == m_shapes_built ) :
( 0 != m_tobuild->count( section_index( world ) ) ) };
if( ( false == wanted )
&& ( true == Input.skipReplayNode() ) ) {
auto timenow = std::chrono::steady_clock::now();
if( std::chrono::duration_cast<std::chrono::milliseconds>( timenow - timelast ).count() >= VISUAL_BUDGET_MS ) {
if( std::chrono::duration_cast<std::chrono::milliseconds>( timenow - timelast ).count() >= budget ) {
Application.set_progress( Input.getProgress(), Input.getFullProgress() );
return true;
}
@@ -242,9 +325,6 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
}
auto timenow = std::chrono::steady_clock::now();
// per-frame budget while streaming visuals in the driver (kept modest to limit
// stutter), generous budget while the loading screen is up (infrastructure pass)
auto const budget = ( state->visualphase ? VISUAL_BUDGET_MS : 200 );
if( std::chrono::duration_cast<std::chrono::milliseconds>( timenow - timelast ).count() >= budget ) {
Application.set_progress( Input.getProgress(), Input.getFullProgress() );
return true;
@@ -258,67 +338,116 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
deserialize_firstinit( Input, Scratchpad );
}
// helper: reset the transform stack before each replay pass. the directives
// (origin/rotate/scale) are replayed in order, so resetting here reproduces the
// single-pass placement exactly; without it an unbalanced origin left on the stack would
// be applied again and shift every deferred visual node ("terrain dumped beside the tracks").
auto const resettransform = [ &Scratchpad ]() {
// a well-formed pass ends with a balanced (empty) transform stack; a leftover means
// an origin/scale was pushed but never popped -- e.g. a node whose binary marker span
// over-ran its terminator and skipped the following endorigin. warn rather than let it
// silently accumulate into the next pass.
if( false == Scratchpad.location.offset.empty() ) {
WriteLog( "Bad scenery: " + std::to_string( Scratchpad.location.offset.size() ) + " unbalanced origin(s) left on the stack at end of a load pass" );
}
Scratchpad.location.offset = {};
Scratchpad.location.scale = {};
Scratchpad.location.rotation = glm::vec3{}; };
// helper: make the scenario playable / persist the twin. the map, instance-bound events and
// twin flush are done once, after the first cycle (or the single build-all pass). the active
// group stack is left open on purpose in section mode -- later cycles keep inserting into it
// (update_map reads the persistent group map, not the stack, so it works either way).
auto const finalize = [ & ]( bool const Closegroups ) {
if( true == Closegroups ) { scene::Groups.close(); }
scene::Groups.update_map();
Region->create_map_geometry();
simulation::Events.InitInstanceEvents();
Input.flushBinaryTwin();
scene::scenerybinary_wait_all(); };
// first (infrastructure) pass finished: the scenario is now playable (tracks, events,
// signals, the player train are all loaded). hand control back so the loader can switch
// to the driver; the visual nodes load progressively from the driver, replayed once per
// camera-distance ring (nearest first). the ring centre is sampled later, on the first
// driver pass (the camera isn't positioned here yet). only possible when replaying a binary
// twin -- a text/compile load did everything in one pass (restartReplay returns false).
// signals, the player train are all loaded). hand control back so the loader can switch to
// the driver; the visual nodes stream in from the driver. the camera centre / mode are
// resolved later, on the first driver pass (the camera isn't positioned here yet). only
// possible when replaying a binary twin -- a text/compile load did everything in one pass.
if( ( false == state->visualphase )
&& ( true == Input.restartReplay( scene::scenery_load_pass::visual ) ) ) {
state->visualphase = true;
state->ringindex = 0;
resettransform();
WriteLog( "Progressive visual load: streaming deferred nodes nearest-camera first (" + std::to_string( RING_COUNT ) + " rings)" );
WriteLog( "Progressive visual load: infrastructure ready, streaming visuals from the driver" );
return false; // infrastructure ready -> go to driver; visuals continue there
}
// a ring pass finished: advance to the next (farther) ring and replay again, until the
// outermost ring has been built. skipped in build-all mode, which builds everything in one pass.
if( ( true == state->visualphase )
&& ( false == state->ringall )
&& ( state->ringindex < ring_lastindex() )
&& ( true == Input.restartReplay( scene::scenery_load_pass::visual ) ) ) {
++state->ringindex;
resettransform();
return true; // more rings to build
// section streaming: a build cycle's replay pass just finished. mark its sections built so
// they aren't rebuilt, finalize once after the first cycle, and stay alive so the next call
// can pick up sections the camera has since moved into. the load never reports "done".
if( ( true == state->visualphase ) && ( true == state->sectionmode ) ) {
if( true == state->pass_active ) {
state->built.insert( std::begin( state->tobuild ), std::end( state->tobuild ) );
state->tobuild.clear();
state->pass_active = false;
state->shapes_built = true; // explicit shapes + eager models are built in the first pass
m_indexing = false;
if( false == state->initial_done ) {
finalize( /*Closegroups*/ false ); // keep groups open for later cycles
state->initial_done = true;
state->indexed = true; // the first pass has indexed every deferred node by section
std::size_t refs = 0; for( auto const &s : state->index ) { refs += s.second.size(); }
WriteLog( "Progressive visual load: spawn ready (" + std::to_string( simulation::Instances.sequence().size() )
+ " instances), " + std::to_string( refs ) + " nodes indexed across " + std::to_string( state->index.size() ) + " sections" );
}
}
return true; // keep streaming alive; sections the camera enters are served from the index
}
scene::Groups.close();
// build-all (no camera centre, e.g. ghostview): everything was built in this single pass.
finalize( /*Closegroups*/ true );
state->done = true;
return false;
}
scene::Groups.update_map();
Region->create_map_geometry();
int
state_serializer::twin_id( deserializer_state &State, std::string const &File, std::string const &Path ) {
std::string key = Path + "|" + File;
auto const it = State.twinids.find( key );
if( it != State.twinids.end() ) { return it->second; }
int const id = static_cast<int>( State.twins.size() );
State.twins.emplace_back( File, Path );
State.twinids.emplace( std::move( key ), id );
return id;
}
// all nodes (including visual model instances) are now loaded, so initialise the
// events that bind to model instances; in a single-pass (text/compile) load nothing
// was deferred, but InitEvents() still skipped them, so do it here too
simulation::Events.InitInstanceEvents();
void
state_serializer::capture_node( cParser &Input, scene::scratch_data const &Scratchpad, glm::dvec3 const &World ) {
// record where the node lives and the context it needs, so rebuild_section() can place it
// identically later without re-scanning the twin. only small-range nodes are indexed; large/
// unlimited-range ("eager") ones are built once up front and never streamed again.
if( m_state == nullptr ) { return; }
visual_ref ref;
ref.twin = twin_id( *m_state, Input.currentReplayFile(), Input.currentReplayPath() );
ref.offset = Input.currentReplayOffset();
ref.has_offset = ( false == Scratchpad.location.offset.empty() );
if( true == ref.has_offset ) { ref.t_offset = Scratchpad.location.offset.top(); }
ref.has_scale = ( false == Scratchpad.location.scale.empty() );
if( true == ref.has_scale ) { ref.t_scale = Scratchpad.location.scale.top(); }
ref.t_rotation = Scratchpad.location.rotation;
ref.params = Input.currentReplayParams();
m_state->index[ section_index( World ) ].emplace_back( std::move( ref ) );
}
// loading finished: flush the top-level scenario's binary twin now rather than
// waiting for the parser to be destroyed (the loader keeps the state around)
Input.flushBinaryTwin();
// wait out any background twin writes (includes) so they are complete and logged
// before we report the scenario as loaded
scene::scenerybinary_wait_all();
state->done = true;
return false;
void
state_serializer::rebuild_section( deserializer_state &State, int Section ) {
auto const it = State.index.find( Section );
if( it == State.index.end() ) { return; }
scene::scratch_data &Scratchpad = State.scratchpad;
for( auto &ref : it->second ) {
// reuse one parser per twin across the whole stream (re-opening an .inc is not free)
auto pit = State.rebuild_parsers.find( ref.twin );
if( pit == State.rebuild_parsers.end() ) {
auto const &tw = State.twins[ ref.twin ];
pit = State.rebuild_parsers.emplace(
ref.twin,
std::make_unique<cParser>( tw.first, cParser::buffer_FILE, tw.second, Global.bLoadTraction ) ).first;
}
cParser &cp = *pit->second;
cp.seekReplayNode( ref.offset );
cp.setReplayParams( ref.params );
// restore the transform context captured for this node
Scratchpad.location.offset = {};
if( true == ref.has_offset ) { Scratchpad.location.offset.push( ref.t_offset ); }
Scratchpad.location.scale = {};
if( true == ref.has_scale ) { Scratchpad.location.scale.push( ref.t_scale ); }
Scratchpad.location.rotation = ref.t_rotation;
auto const tok = cp.getToken<std::string>();
if( tok == "node" ) { deserialize_node( cp, Scratchpad ); }
}
// the section is built; release its index entries
std::vector<visual_ref>().swap( it->second );
}
void
@@ -703,12 +832,23 @@ state_serializer::deserialize_node( cParser &Input, scene::scratch_data &Scratch
|| ( nodedata.type == "triangle_strip" )
|| ( nodedata.type == "triangle_fan" ) ) {
// explicit shapes have no single position to ring-test by, so build them only in the
// nearest ring pass (ring 0) and skip them in O(1) on the farther passes (build-all
// mode is a single pass, so they always build there)
if( ( true == m_ringactive ) && ( false == m_ringall ) && ( m_ringindex > 0 ) ) {
if( false == Input.skipReplayNode() ) { skip_until( Input, "endtri" ); }
return;
// origin-placed shapes (e.g. flora includes) carry their world position in the active
// origin, so they section-stream like models: indexed on the first pass, rebuilt per
// section after. absolute shapes (terrain, no origin) have no single position -> built
// once in the first pass. (m_rebuilding: chosen from the index -> build unconditionally.)
if( ( true == m_sectionmode ) && ( false == m_rebuilding ) && ( nullptr != m_tobuild ) ) {
if( false == Scratchpad.location.offset.empty() ) {
glm::dvec3 const world { Scratchpad.location.offset.top() };
if( true == m_indexing ) { capture_node( Input, Scratchpad, world ); }
if( 0 == m_tobuild->count( section_index( world ) ) ) {
if( false == Input.skipReplayNode() ) { skip_until( Input, "endtri" ); }
return;
}
}
else if( true == m_shapes_built ) {
if( false == Input.skipReplayNode() ) { skip_until( Input, "endtri" ); }
return;
}
}
auto const skip {
@@ -734,10 +874,20 @@ state_serializer::deserialize_node( cParser &Input, scene::scratch_data &Scratch
|| ( nodedata.type == "line_strip" )
|| ( nodedata.type == "line_loop" ) ) {
// see the triangles branch: explicit shapes build in ring 0 only (or always, build-all)
if( ( true == m_ringactive ) && ( false == m_ringall ) && ( m_ringindex > 0 ) ) {
if( false == Input.skipReplayNode() ) { skip_until( Input, "endline" ); }
return;
// see the triangles branch: origin-placed lines section-stream, absolute ones build once.
if( ( true == m_sectionmode ) && ( false == m_rebuilding ) && ( nullptr != m_tobuild ) ) {
if( false == Scratchpad.location.offset.empty() ) {
glm::dvec3 const world { Scratchpad.location.offset.top() };
if( true == m_indexing ) { capture_node( Input, Scratchpad, world ); }
if( 0 == m_tobuild->count( section_index( world ) ) ) {
if( false == Input.skipReplayNode() ) { skip_until( Input, "endline" ); }
return;
}
}
else if( true == m_shapes_built ) {
if( false == Input.skipReplayNode() ) { skip_until( Input, "endline" ); }
return;
}
}
simulation::Region->insert(
@@ -1085,21 +1235,28 @@ state_serializer::deserialize_model( cParser &Input, scene::scratch_data &Scratc
>> location.z
>> rotation.y;
// camera-ring visual streaming: build this model only if it falls in the ring currently
// being streamed; otherwise skip the rest of its body in O(1) and let a later (farther)
// ring pass pick it up. covers terrain models (range_min<0) too -- they also have X Y Z.
// most out-of-ring models are already dropped O(1) at the dispatch loop via their v7
// marker; this is the fallback for nodes that reached here (in-ring, or no marker pos).
// use the marker position when present so this ring decision matches the dispatch one
// exactly -- otherwise a node near a ring boundary could be dropped by both adjacent rings.
if( ( true == m_ringactive ) && ( false == m_ringall ) ) {
glm::dvec3 ringlocal { location };
double mx, my, mz;
if( true == Input.currentNodePosition( mx, my, mz ) ) { ringlocal = glm::dvec3{ mx, my, mz }; }
auto const world { transform( ringlocal, Scratchpad ) };
auto const d { world - m_ringeye };
auto const d2 { d.x * d.x + d.y * d.y + d.z * d.z };
if( ( d2 < m_ringmin2 ) || ( d2 >= m_ringmax2 ) ) {
// camera-following visual streaming: build this model only if its region section is in the
// set being built this cycle; otherwise skip the rest of its body in O(1) and let a later
// cycle (once the camera is near) pick it up. covers terrain models (range_min<0) too -- they
// also have X Y Z. most out-of-range models are already dropped O(1) at the dispatch loop via
// their v7 marker; this is the fallback for nodes that reached here (in-range, or no marker).
// use the marker position when present so this decision matches the dispatch one exactly.
// m_rebuilding: this node was chosen from the section index, so build it unconditionally.
if( ( true == m_sectionmode ) && ( false == m_rebuilding ) && ( nullptr != m_tobuild ) ) {
// models visible from beyond the stream radius build once (first cycle); the rest build
// when their section is in range. mirrors the dispatch-loop fast path exactly.
bool const eager { ( Nodedata.range_max < 0.0 ) || ( Nodedata.range_max > STREAM_RADIUS ) };
bool wanted;
if( true == eager ) {
wanted = ( false == m_shapes_built );
}
else {
glm::dvec3 modellocal { location };
double mx, my, mz, mr;
if( true == Input.currentNodePosition( mx, my, mz, mr ) ) { modellocal = glm::dvec3{ mx, my, mz }; }
wanted = ( 0 != m_tobuild->count( section_index( transform( modellocal, Scratchpad ) ) ) );
}
if( false == wanted ) {
if( false == Input.skipReplayNode() ) { skip_until( Input, "endmodel" ); }
return nullptr;
}

77
simulation/simulationstateserializer.h
View File

@@ -14,6 +14,22 @@ http://mozilla.org/MPL/2.0/.
namespace simulation {
// a deferred visual node recorded during the first (indexing) replay pass, so it can be rebuilt
// on demand when its region section enters camera range -- without re-scanning the whole twin.
// holds where to find the node (which twin + byte offset of its marker) and the context needed to
// place it identically (the transform active at that point, and the include parameters its "(pN)"
// tokens resolve against).
struct visual_ref {
int twin { -1 }; // index into deserializer_state::twins (file+path to re-open)
std::size_t offset { 0 }; // byte offset of the node's marker within that twin
glm::dvec3 t_offset { 0.0 };
glm::vec3 t_rotation { 0.f };
glm::vec3 t_scale { 1.f };
bool has_offset { false };
bool has_scale { false };
std::vector<std::string> params; // include parameters in effect (empty for a direct node)
};
struct deserializer_state {
std::string scenariofile;
cParser input;
@@ -27,18 +43,32 @@ struct deserializer_state {
bool visualphase { false };
// set once the whole load (both passes / single text pass) has fully finished
bool done { false };
// camera-ring visual streaming: the visual phase replays the twin once per distance
// ring (nearest first), building only the nodes whose distance to ringeye falls in the
// current ring and O(1)-skipping the rest. ringeye is sampled once when the visual phase
// starts so the ring partition is stable across passes.
int ringindex { 0 };
// camera-following visual streaming. once infrastructure is up, visual nodes (3d models,
// terrain shapes) stream in only for the region sections currently within STREAM_RADIUS of
// the camera; as the camera moves into new sections they are built too. nothing is unloaded.
// the twin is replayed once per "build cycle" (a set of newly-wanted sections), building
// only nodes whose section is in that set and O(1)-skipping the rest by their v7 marker.
glm::dvec3 ringeye { 0.0 };
// ringeye must be sampled from the camera *after* control passes to the driver (the
// loader hasn't positioned the camera yet -- sampling there put every node in the far
// ring and built models last / seemingly never). sampled lazily on the first driver pass.
// the camera centre is only meaningful once control reaches the driver (the loader hasn't
// positioned the camera yet). sampled lazily on the first driver pass; if still unusable
// (e.g. ghostview at the origin) we fall back to building everything in one pass (ringall).
bool ringeye_valid { false };
int ringeye_waits { 0 }; // frames spent waiting for the camera to be positioned
bool ringall { false }; // no usable camera centre -> build all visual nodes in one pass
bool sectionmode { false }; // usable camera centre -> stream sections within range, follow camera
bool shapes_built { false }; // first cycle done: explicit shapes + large-range (eager) models built
bool initial_done { false }; // the first cycle finished -> scenario finalised (map/events/twin)
bool pass_active { false }; // a section build cycle's replay pass is currently in progress
std::unordered_set<int> built; // region-section indices already streamed in
std::unordered_set<int> tobuild; // section indices targeted by the current/next cycle
// section index, built during the first replay pass: every deferred visual node is recorded
// under its region section, so later sections rebuild by seeking straight to their nodes
// instead of replaying (re-scanning) the whole million-node twin every cycle.
bool indexed { false }; // first pass finished -> the index below is complete
std::vector<std::pair<std::string, std::string>> twins; // (file, path) to re-open, interned
std::unordered_map<std::string, int> twinids; // "path|file" -> index into twins
std::unordered_map<int, std::vector<visual_ref>> index; // section -> deferred nodes there
std::unordered_map<int, std::unique_ptr<cParser>> rebuild_parsers; // one reused parser per twin
deserializer_state(std::string const &File, cParser::buffertype const Type, const std::string &Path, bool const Loadtraction)
: scenariofile(File), input(File, Type, Path, Loadtraction) { }
@@ -105,16 +135,33 @@ private:
// transforms provided location by specifed rotation and offset
glm::dvec3 transform( glm::dvec3 Location, scene::scratch_data const &Scratchpad );
void export_nodes_to_stream( std::ostream &, bool Dirty ) const;
// region-section index (row-major, clamped to the grid) enclosing a world position --
// matches basic_region::section()'s indexing, used to bucket visual nodes for streaming.
static int section_index( glm::dvec3 const &World );
// record the visual node currently being replayed (twin/offset/transform/params) under its
// section in the index, so it can be rebuilt later without re-scanning the twin.
void capture_node( cParser &Input, scene::scratch_data const &Scratchpad, glm::dvec3 const &World );
// rebuild every indexed node of one section by seeking straight to it (no twin re-scan).
void rebuild_section( deserializer_state &State, int Section );
// interns a (file, path) pair into State.twins, returning its index
static int twin_id( deserializer_state &State, std::string const &File, std::string const &Path );
// members
// camera-ring visual streaming state, mirrored from deserializer_state each
// deserialize_continue() call so deserialize_model()/deserialize_node() can ring-test a
// node by distance. inactive (builds everything) outside the ring/visual phase.
// camera-following visual streaming state, mirrored from deserializer_state each
// deserialize_continue() call so deserialize_model()/deserialize_node() can decide whether
// a node's section is in the current build set. inactive (builds everything) outside the
// visual phase, or in ringall (no camera centre) where every node is built in one pass.
bool m_ringactive { false };
bool m_ringall { false }; // no camera centre available -> build every node in one pass
int m_ringindex { 0 };
bool m_ringall { false }; // no camera centre available -> build every node in one pass
bool m_sectionmode { false }; // stream by camera-range sections
bool m_shapes_built { false }; // first cycle done: shapes + large-range (eager) models built (skip them)
glm::dvec3 m_ringeye { 0.0 };
double m_ringmin2 { 0.0 };
double m_ringmax2 { 0.0 };
std::unordered_set<int> const *m_tobuild { nullptr }; // section indices to build this cycle
// first replay pass records each deferred node into the index (m_indexing); later cycles
// rebuild sections straight from it (m_rebuilding bypasses the section test so the chosen
// node always builds). m_state gives deserialize_model()/node() access for capture.
bool m_indexing { false };
bool m_rebuilding { false };
deserializer_state *m_state { nullptr };
};
} // simulation

85
utilities/parser.cpp
View File

@@ -319,7 +319,7 @@ void cParser::bakeFinishNode()
// flush a node still open at end-of-file or one whose type was unrecognized
if (m_bakenode_active && m_writer)
{
m_writer->end_node(m_bakenode_visual, m_bakenode_haspos, m_bakenode_pos[0], m_bakenode_pos[1], m_bakenode_pos[2]);
m_writer->end_node(m_bakenode_visual, m_bakenode_haspos, m_bakenode_pos[0], m_bakenode_pos[1], m_bakenode_pos[2], m_bakenode_rangemax);
}
m_bakenode_active = false;
}
@@ -817,22 +817,48 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break)
if (m_bakenode_active)
{
++m_bakenode_count;
if (m_bakenode_count == 2)
{
// 2nd node entry is range_max (visibility range); kept in the model marker
// so the streamer builds far-but-large-range models eagerly. -1 (unlimited)
// when it isn't a plain number (e.g. an unresolved parameter).
double rmax;
m_bakenode_rangemax = (sniffNumber(rawtoken, rmax) ? rmax : -1.0);
}
if ((m_bakenode_count == 5) && m_bakenode_end.empty())
{
// 5th node entry is the type token (node, range_max, range_min, name, type)
classifyNodeType(lowered, m_bakenode_visual, m_bakenode_end);
// a model node's entries 6,7,8 are its local X Y Z -- record them in the
// marker so the camera-ring load can skip the node without reading its body
m_bakenode_haspos = (lowered == "model");
{
bool const isshape =
(lowered == "triangles") || (lowered == "triangle_strip") || (lowered == "triangle_fan")
|| (lowered == "lines") || (lowered == "line_strip") || (lowered == "line_loop");
m_bakenode_haspos = (lowered == "model") || isshape;
m_bakenode_shape = isshape;
m_bakenode_posstate = 0;
m_bakenode_posidx = 0;
}
}
else if (m_bakenode_haspos && (m_bakenode_count >= 6) && (m_bakenode_count <= 8))
else if (m_bakenode_haspos && (false == m_bakenode_shape) && (m_bakenode_count >= 6) && (m_bakenode_count <= 8))
{
m_bakenode_pos[m_bakenode_count - 6] = value;
}
else if ((false == m_bakenode_end.empty()) && (lowered == m_bakenode_end))
else if (m_bakenode_shape && (m_bakenode_posstate < 3))
{
// walk a shape to its first vertex: 0=before material -> if "material" enter the
// block (1), else it is a shortcut material (start seeking numbers, 2); 1=skip until
// "endmaterial"; 2=skip the texture string, then take the 3 vertex numbers.
double sv;
if (m_bakenode_posstate == 0) { m_bakenode_posstate = (lowered == "material") ? 1 : 2; }
else if (m_bakenode_posstate == 1) { if (lowered == "endmaterial") m_bakenode_posstate = 2; }
else if (sniffNumber(rawtoken, sv)) { m_bakenode_pos[m_bakenode_posidx++] = sv; if (m_bakenode_posidx == 3) m_bakenode_posstate = 3; }
}
else if ((false == m_bakenode_end.empty()) && (lowered == m_bakenode_end))
{
// terminator captured: close the node
m_writer->end_node(m_bakenode_visual, m_bakenode_haspos, m_bakenode_pos[0], m_bakenode_pos[1], m_bakenode_pos[2]);
m_writer->end_node(m_bakenode_visual, m_bakenode_haspos, m_bakenode_pos[0], m_bakenode_pos[1], m_bakenode_pos[2], m_bakenode_rangemax);
m_bakenode_active = false;
}
}
@@ -924,11 +950,54 @@ bool cParser::skipReplayNode()
return false;
}
bool cParser::currentNodePosition(double &X, double &Y, double &Z)
bool cParser::currentNodePosition(double &X, double &Y, double &Z, double &Range)
{
// delegate to the deepest active include child (it serves the current node), like skip
if (mIncludeParser) { return mIncludeParser->currentNodePosition(X, Y, Z); }
return (m_replay && m_reader && m_reader->node_position(X, Y, Z));
if (mIncludeParser) { return mIncludeParser->currentNodePosition(X, Y, Z, Range); }
return (m_replay && m_reader && m_reader->node_position(X, Y, Z, Range));
}
std::string cParser::currentReplayFile()
{
if (mIncludeParser) { return mIncludeParser->currentReplayFile(); }
return mFile;
}
std::string cParser::currentReplayPath()
{
if (mIncludeParser) { return mIncludeParser->currentReplayPath(); }
return mPath;
}
std::size_t cParser::currentReplayOffset()
{
if (mIncludeParser) { return mIncludeParser->currentReplayOffset(); }
return (m_reader ? m_reader->node_offset() : 0);
}
std::vector<std::string> cParser::currentReplayParams()
{
if (mIncludeParser) { return mIncludeParser->currentReplayParams(); }
return parameters;
}
void cParser::seekReplayNode(std::size_t Offset)
{
// rebuild serves a single self-contained node, so drop any open include child and rewind
// this twin's reader to the node's marker; the next getToken() decodes it
mIncludeParser = nullptr;
tokens.clear();
if (m_replay && m_reader)
{
m_reader->set_pass(scene::scenery_load_pass::all);
m_reader->seek_node(Offset);
m_replayexhausted = false;
}
}
void cParser::setReplayParams(std::vector<std::string> Params)
{
parameters = std::move(Params);
}
std::vector<std::string> cParser::readParameters(cParser &Input)

24
utilities/parser.h
View File

@@ -65,7 +65,22 @@ class cParser //: public std::stringstream
// local position of the node about to be replayed (the "node" token just read), if its
// v7 marker carried one (visual model nodes). lets the camera-ring load distance-test and
// skip a node without decoding its body. returns false for shapes / non-replay / older twins.
bool currentNodePosition( double &X, double &Y, double &Z );
bool currentNodePosition( double &X, double &Y, double &Z, double &Range );
// --- section-index streaming: capture enough to rebuild the node about to be replayed
// (the deepest active include serves it) without re-scanning the whole twin later ---
// source file + base path of the twin the current node lives in (to re-open it)
std::string currentReplayFile();
std::string currentReplayPath();
// byte offset of the current node's marker within that twin (to seek back to it)
std::size_t currentReplayOffset();
// the include parameters in effect for the current node (its "(pN)" tokens resolve against
// these); empty for a node read directly from a parameterless file
std::vector<std::string> currentReplayParams();
// reposition this (replay) parser at a node recorded via currentReplayOffset() and serve it
// next; drops any open include child. used to rebuild one indexed node on demand.
void seekReplayNode( std::size_t Offset );
// set the include parameters used to resolve the next node's "(pN)" tokens during rebuild
void setReplayParams( std::vector<std::string> Params );
// methods:
template <typename Type_>
cParser &
@@ -222,6 +237,13 @@ class cParser //: public std::stringstream
// camera-ring load can distance-test it without decoding the node body
bool m_bakenode_haspos { false };
double m_bakenode_pos[ 3 ] { 0.0, 0.0, 0.0 };
double m_bakenode_rangemax { -1.0 }; // range_max (entry 2), stored in the model marker
// explicit shape (triangles/lines) position extraction: a shape has no fixed-index position
// like a model, so walk past its material to its first vertex and use that. posstate:
// 0=before material, 1=inside material...endmaterial, 2=seeking the 3 vertex numbers, 3=done.
bool m_bakenode_shape { false };
int m_bakenode_posstate { 0 };
int m_bakenode_posidx { 0 };
// flushes a node still open at end-of-file (or unknown type), so its buffer is written
void bakeFinishNode();
};