eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-18 00:49:19 +02:00
Code Issues 121 Wiki Activity

Replace per-node text capture with camera-distance ring multi-pass

Browse Source 
The previous nearest-first build captured every deferred visual node as text into a
sorted vector, which does not scale: one tomaszewo flora file alone holds 440k model
nodes (the scenery has 1M+), so the capture ran the process to ~7 GB and its
enumeration never finished.

Stream the visual nodes in camera-distance rings instead, with no per-node capture
(O(1) memory). The visual pass replays the twin once per ring (nearest first); a node
is built only when its squared distance to the camera -- sampled once when the visual
phase starts, so the partition is stable across passes -- falls in the current ring,
otherwise the rest of its body is skipped in O(1) by jumping over the v6 marker span.
Each node is therefore built exactly once, in roughly nearest-first order, through the
normal node path (instancing buckets unchanged). Explicit triangles/lines shapes have
no single position to ring-test by, so they build in the nearest ring pass only.

Reader gains skip_to_node_end() (remembers the served node's end and jumps the cursor
there); cParser::skipReplayNode() delegates it down the active include child.

Verified: td.scn builds 4 rings, no duplicates, no unexpected tokens, complete ~1s
after the infrastructure pass. tomaszewo stays memory-bounded (no OOM, no duplicates)
where the capture approach previously hung.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
maj00r
2026-06-23 17:41:46 +02:00
parent 1160bfecac
commit d7a3e1310a
6 changed files with 127 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
scene/scenerybinary.cpp
View File

@@ -273,6 +273,13 @@ scenery_binary_reader::next( scenery_entry_view &Out ) {
m_cursor += static_cast<std::ptrdiff_t>( span );
if( m_cursor > m_end ) { m_cursor = m_end; }
}
else {
// remember where this node ends so the consumer can bail out of it in O(1)
// (skip_to_node_end) after peeking just its first few entries -- used by the
// camera-ring visual load to drop a node that's outside the current distance ring
m_nodeend = m_cursor + static_cast<std::ptrdiff_t>( span );
if( m_nodeend > m_end ) { m_nodeend = m_end; }
}
// when processing, fall through: the loop re-reads and decodes the node's first entry
}

6
scene/scenerybinary.h
View File

@@ -148,6 +148,11 @@ public:
// decodes the next served entry into Out, skipping node markers / out-of-pass nodes;
// returns false once all entries are consumed
bool next( scenery_entry_view &Out );
// jumps the cursor to the end of the node currently being served, so the rest of its
// body is skipped in O(1). valid right after next() handed out an entry belonging to a
// node; returns false (no-op) otherwise. used by the camera-ring visual load to drop a
// node outside the current distance ring once its position has been read.
bool skip_to_node_end() { if( m_nodeend == nullptr ) { return false; } m_cursor = m_nodeend; m_nodeend = nullptr; return true; }
bool exhausted() const { return m_cursor >= m_end; }
// fraction of bytes consumed so far, 0..100, for the loading bar
int progress() const { return ( m_size == 0 ? 100 : static_cast<int>( ( m_cursor - m_begin ) * 100 / m_size ) ); }
@@ -157,6 +162,7 @@ private:
char const *m_begin { nullptr }; // start of the entry section
char const *m_cursor { nullptr };
char const *m_end { nullptr };
char const *m_nodeend { nullptr }; // end of the node currently being served (for skip_to_node_end)
std::ptrdiff_t m_size { 0 }; // entry section byte length
scenery_load_pass m_pass { scenery_load_pass::all };
scenery_file_kind m_kind { scenery_file_kind::scn };

289
simulation/simulationstateserializer.cpp
View File

@@ -32,9 +32,26 @@ http://mozilla.org/MPL/2.0/.
#include <algorithm>
#include <cctype>
#include <cstdlib>
#include <limits>
namespace simulation {
namespace {
// camera-distance rings for nearest-first visual streaming: the squared outer radius of
// each ring. the visual pass is replayed once per ring (nearest first), building only the
// nodes whose squared distance to the camera falls in [inner, outer); the last ring is
// unbounded so every remaining node is built exactly once.
constexpr double RING_OUTER2[] = {
500.0 * 500.0,
1500.0 * 1500.0,
4000.0 * 4000.0,
std::numeric_limits<double>::infinity() };
constexpr int RING_COUNT { static_cast<int>( sizeof( RING_OUTER2 ) / sizeof( RING_OUTER2[ 0 ] ) ) };
inline int ring_lastindex() { return RING_COUNT - 1; }
inline double ring_min2( int const K ) { return ( K <= 0 ? 0.0 : RING_OUTER2[ K - 1 ] ); }
inline double ring_max2( int const K ) { return RING_OUTER2[ ( K < RING_COUNT ? K : RING_COUNT - 1 ) ]; }
} // anonymous namespace
std::shared_ptr<deserializer_state>
state_serializer::deserialize_begin( std::string const &Scenariofile ) {
@@ -118,10 +135,15 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
cParser &Input = state->input;
scene::scratch_data &Scratchpad = state->scratchpad;
// camera-ordered build phase: the visual replay has been enumerated into records and
// sorted; build them nearest-camera first, a budgeted slice per call.
if( state->enumdone ) {
return build_visual_records( state );
// mirror the camera-ring state so deserialize_model()/deserialize_node() can ring-test
// each node by distance: in the visual phase the twin is replayed once per ring and only
// the nodes in the current ring are built, the rest skipped in O(1)
m_ringactive = state->visualphase;
if( true == m_ringactive ) {
m_ringindex = state->ringindex;
m_ringeye = state->ringeye;
m_ringmin2 = ring_min2( state->ringindex );
m_ringmax2 = ring_max2( state->ringindex );
}
// stateful directives that build objects/lists; on the visual (second) pass they are
@@ -153,18 +175,6 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
token = Input.getToken<std::string>();
continue;
}
if( ( true == state->enumerate ) && ( token == "node" ) ) {
// capture the visual node (text + transform/group snapshot) for later,
// camera-ordered building, instead of building it in file order now
enumerate_visual_node( *state );
auto timenow = std::chrono::steady_clock::now();
if( std::chrono::duration_cast<std::chrono::milliseconds>( timenow - timelast ).count() >= 8 ) {
Application.set_progress( Input.getProgress(), Input.getFullProgress() );
return true;
}
token = Input.getToken<std::string>();
continue;
}
}
auto lookup = state->functionmap.find( token );
@@ -187,53 +197,46 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
token = Input.getToken<std::string>();
}
if( ( true == state->visualphase ) && ( true == state->enumerate ) ) {
// visual replay exhausted: order the captured nodes by distance to the camera so
// the player's surroundings stream in first (terrain shapes ahead of models, so the
// ground appears before the props on it), then switch to the build phase.
auto const eye { Global.pCamera.Pos };
for( auto &rec : state->records ) {
auto const d { rec.worldpos - eye };
rec.sortkey =
( rec.isshape ?
-1.0 : // shapes (terrain) first, regardless of distance
( d.x * d.x + d.y * d.y + d.z * d.z ) );
}
std::stable_sort(
std::begin( state->records ), std::end( state->records ),
[]( visual_record const &L, visual_record const &R ) {
return L.sortkey < R.sortkey; } );
WriteLog( "Progressive visual load: " + std::to_string( state->records.size() ) + " deferred nodes enumerated, building nearest-camera first" );
state->enumerate = false;
state->enumdone = true;
return true; // proceed to the build phase
}
if( false == Scratchpad.initialized ) {
// manually perform scenario initialization
deserialize_firstinit( Input, Scratchpad );
}
// helper: reset the transform stack before each replay pass. the directives
// (origin/rotate/scale) are replayed in order, so resetting here reproduces the
// single-pass placement exactly; without it an unbalanced origin left on the stack would
// be applied again and shift every deferred visual node ("terrain dumped beside the tracks").
auto const resettransform = [ &Scratchpad ]() {
Scratchpad.location.offset = {};
Scratchpad.location.scale = {};
Scratchpad.location.rotation = glm::vec3{}; };
// first (infrastructure) pass finished: the scenario is now playable (tracks, events,
// signals, the player train are all loaded). hand control back so the loader can switch
// to the driver; the visual nodes load progressively from the driver via a second pass
// over the same twin. only possible when replaying a binary twin -- a text/compile load
// did everything in one pass (restartReplay returns false).
// to the driver; the visual nodes load progressively from the driver, replayed once per
// camera-distance ring (nearest first). the camera is sampled once here so the ring
// partition stays stable across passes. only possible when replaying a binary twin -- a
// text/compile load did everything in one pass (restartReplay returns false).
if( ( false == state->visualphase )
&& ( true == Input.restartReplay( scene::scenery_load_pass::visual ) ) ) {
state->visualphase = true;
state->enumerate = true; // capture deferred visual nodes, build them camera-ordered
// rebuild the transform state from scratch for the visual pass: the directives
// (origin/rotate/scale) are replayed in order, so resetting here reproduces the
// single-pass placement exactly. without this, an unbalanced origin left on the
// stack by the infrastructure pass would be applied a second time and shift every
// deferred visual node ("terrain dumped next to the tracks").
Scratchpad.location.offset = {};
Scratchpad.location.scale = {};
Scratchpad.location.rotation = glm::vec3{};
state->ringindex = 0;
state->ringeye = Global.pCamera.Pos;
resettransform();
WriteLog( "Progressive visual load: streaming deferred nodes nearest-camera first (" + std::to_string( RING_COUNT ) + " rings)" );
return false; // infrastructure ready -> go to driver; visuals continue there
}
// a ring pass finished: advance to the next (farther) ring and replay again, until the
// outermost ring has been built
if( ( true == state->visualphase )
&& ( state->ringindex < ring_lastindex() )
&& ( true == Input.restartReplay( scene::scenery_load_pass::visual ) ) ) {
++state->ringindex;
resettransform();
return true; // more rings to build
}
scene::Groups.close();
scene::Groups.update_map();
@@ -255,166 +258,6 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
return false;
}
// captures one visual node (the "node" token already consumed) into a record for later,
// camera-ordered building: stores the node's verbatim tokens plus the transform/group
// context it was read under. numbers come through cParser losslessly, so the rebuild is
// exact. only model/triangles/lines reach the visual pass.
void
state_serializer::enumerate_visual_node( deserializer_state &State ) {
cParser &Input = State.input;
scene::scratch_data &Scratchpad = State.scratchpad;
// node header, original case preserved (model file paths are case-sensitive at replay)
auto const smax { Input.getToken<std::string>( false ) };
auto const smin { Input.getToken<std::string>( false ) };
auto const sname { Input.getToken<std::string>( false ) };
auto const stype { Input.getToken<std::string>( false ) };
auto lower = []( std::string s ) {
for( auto &c : s ) { c = static_cast<char>( std::tolower( static_cast<unsigned char>( c ) ) ); }
return s; };
auto const typelc { lower( stype ) };
std::string endtok;
bool ismodel { false };
bool istriangles { false };
if( typelc == "model" ) {
endtok = "endmodel"; ismodel = true;
}
else if( ( typelc == "triangles" ) || ( typelc == "triangle_strip" ) || ( typelc == "triangle_fan" ) ) {
endtok = "endtri"; istriangles = true;
}
else if( ( typelc == "lines" ) || ( typelc == "line_strip" ) || ( typelc == "line_loop" ) ) {
endtok = "endline";
}
// mirror deserialize_node's switch-trackbed skip here, while the source file name is
// still known (Input.Name() is the live include during replay; the per-record rebuild
// parser has no name). without this, fixed trackbed geometry from scenery/zwr*.inc that
// the in-order load drops would be rebuilt and duplicate the procedural trackbeds.
bool skipnode { false };
if( ( true == istriangles ) && ( true == Global.CreateSwitchTrackbeds ) ) {
auto const name { Input.Name() };
skipnode =
( name.size() >= 15 )
&& name.starts_with( "scenery/zwr" )
&& name.ends_with( ".inc" );
}
// appends one token to the rebuilt node text, re-quoting it if it carries a token break
// (a quoted source token such as a name/path with spaces arrives here unquoted; without
// re-quoting it the rebuild parser would split it into several tokens)
auto appendtok = []( std::string &Text, std::string const &Token ) {
bool const needsquote {
Token.empty()
|| ( Token.find_first_of( "\n\r\t ;" ) != std::string::npos ) };
Text.push_back( ' ' );
if( true == needsquote ) { Text.push_back( '\"' ); }
Text.append( Token );
if( true == needsquote ) { Text.push_back( '\"' ); } };
std::string text;
text.reserve( 96 );
text.append( "node" );
appendtok( text, smax );
appendtok( text, smin );
appendtok( text, sname );
appendtok( text, stype );
glm::dvec3 localpos { 0.0 };
int posread { 0 };
// read the rest of the node verbatim until its terminator (or the first end* token if the
// type is unrecognised), capturing a model's local position (the first 3 numbers)
for( ;; ) {
auto const tok { Input.getToken<std::string>( false ) };
if( tok.empty() ) { break; } // safety: input ran out
appendtok( text, tok );
if( ( true == ismodel ) && ( posread < 3 ) ) {
localpos[ posread ] = std::atof( tok.c_str() );
++posread;
}
auto const toklc { lower( tok ) };
if( false == endtok.empty() ) {
if( toklc == endtok ) { break; }
}
else if( ( toklc.size() >= 3 ) && ( 0 == toklc.compare( 0, 3, "end" ) ) ) {
break;
}
}
if( true == skipnode ) {
// node consumed from the stream above; intentionally not recorded (matches the
// in-order load, which skips this geometry in favour of procedural trackbeds)
return;
}
visual_record rec;
rec.text = std::move( text );
rec.group = scene::Groups.handle();
rec.has_offset = ( false == Scratchpad.location.offset.empty() );
if( true == rec.has_offset ) { rec.offset = Scratchpad.location.offset.top(); }
rec.has_scale = ( false == Scratchpad.location.scale.empty() );
if( true == rec.has_scale ) { rec.scale = Scratchpad.location.scale.top(); }
rec.rotation = Scratchpad.location.rotation;
rec.isshape = ( false == ismodel );
if( true == ismodel ) {
rec.worldpos = transform( localpos, Scratchpad );
}
State.records.emplace_back( std::move( rec ) );
}
// builds the enumerated visual records in (already-sorted) nearest-camera-first order, a
// budgeted slice per call. each node is rebuilt through the normal node path with its
// captured transform/group restored, so placement, grouping and the per-cell instance
// buckets come out identical to an in-order load.
bool
state_serializer::build_visual_records( std::shared_ptr<deserializer_state> state ) {
scene::scratch_data &Scratchpad = state->scratchpad;
auto timelast { std::chrono::steady_clock::now() };
while( state->buildcursor < state->records.size() ) {
auto &rec = state->records[ state->buildcursor ];
// restore the transform context captured for this node
Scratchpad.location.offset = {};
if( true == rec.has_offset ) { Scratchpad.location.offset.push( rec.offset ); }
Scratchpad.location.scale = {};
if( true == rec.has_scale ) { Scratchpad.location.scale.push( rec.scale ); }
Scratchpad.location.rotation = rec.rotation;
// re-attach to the node's original group and build via the normal node path (this
// routes instanceable models into their per-cell instance bucket automatically)
scene::Groups.push_active( rec.group );
{
cParser nodeparser( rec.text, cParser::buffer_TEXT );
nodeparser.getToken<std::string>(); // consume "node"
deserialize_node( nodeparser, Scratchpad );
}
scene::Groups.pop_active();
std::string().swap( rec.text ); // release the captured text as we go
++state->buildcursor;
auto timenow = std::chrono::steady_clock::now();
if( std::chrono::duration_cast<std::chrono::milliseconds>( timenow - timelast ).count() >= 8 ) {
return true; // yield; resume next frame
}
}
// every deferred visual node is now built: finalise the scenario
scene::Groups.close();
scene::Groups.update_map();
Region->create_map_geometry();
// the visual model instances now exist, so initialise the events that bind to them
// (lights/animation/texture/visible) -- these were deferred during the infrastructure
// pass when their target models hadn't been streamed in yet
simulation::Events.InitInstanceEvents();
state->input.flushBinaryTwin();
scene::scenerybinary_wait_all();
std::vector<visual_record>().swap( state->records );
state->done = true;
return false;
}
void
state_serializer::deserialize_isolated( cParser &Input, scene::scratch_data &Scratchpad ) {
// first parameter specifies name of parent piece...
@@ -797,6 +640,13 @@ state_serializer::deserialize_node( cParser &Input, scene::scratch_data &Scratch
|| ( nodedata.type == "triangle_strip" )
|| ( nodedata.type == "triangle_fan" ) ) {
// explicit shapes have no single position to ring-test by, so build them only in the
// nearest ring pass (ring 0) and skip them in O(1) on the farther passes
if( ( true == m_ringactive ) && ( m_ringindex > 0 ) ) {
if( false == Input.skipReplayNode() ) { skip_until( Input, "endtri" ); }
return;
}
auto const skip {
// crude way to detect fixed switch trackbed geometry
( ( true == Global.CreateSwitchTrackbeds )
@@ -820,6 +670,12 @@ state_serializer::deserialize_node( cParser &Input, scene::scratch_data &Scratch
|| ( nodedata.type == "line_strip" )
|| ( nodedata.type == "line_loop" ) ) {
// see the triangles branch: explicit shapes build in ring 0 only
if( ( true == m_ringactive ) && ( m_ringindex > 0 ) ) {
if( false == Input.skipReplayNode() ) { skip_until( Input, "endline" ); }
return;
}
simulation::Region->insert(
scene::lines_node().import(
Input, nodedata ),
@@ -1165,6 +1021,19 @@ state_serializer::deserialize_model( cParser &Input, scene::scratch_data &Scratc
>> location.z
>> rotation.y;
// camera-ring visual streaming: build this model only if it falls in the ring currently
// being streamed; otherwise skip the rest of its body in O(1) and let a later (farther)
// ring pass pick it up. covers terrain models (range_min<0) too -- they also have X Y Z.
if( true == m_ringactive ) {
auto const world { transform( location, Scratchpad ) };
auto const d { world - m_ringeye };
auto const d2 { d.x * d.x + d.y * d.y + d.z * d.z };
if( ( d2 < m_ringmin2 ) || ( d2 >= m_ringmax2 ) ) {
if( false == Input.skipReplayNode() ) { skip_until( Input, "endmodel" ); }
return nullptr;
}
}
auto *instance = new TAnimModel( Nodedata );
instance->Angles( Scratchpad.location.rotation + rotation ); // dostosowanie do pochylania linii
// pick up the scale active at this point in the scenario stream — outer

44
simulation/simulationstateserializer.h
View File

@@ -14,24 +14,6 @@ http://mozilla.org/MPL/2.0/.
namespace simulation {
// a deferred visual node captured during the visual enumeration pass, to be (re)built
// later in camera-distance order. holds the node's fully resolved tokens as text -- numbers
// come through cParser losslessly (std::to_chars shortest round-trip), so rebuilding from
// this text reproduces the exact same node -- plus a snapshot of the transform/group context
// it was read under, so the out-of-order rebuild places and groups it identically.
struct visual_record {
std::string text; // "node <range> <range> <name> <type> ... end<type>"
glm::dvec3 offset { 0.0 }; // top of the origin stack at capture (identity if none)
glm::vec3 scale { 1.f }; // top of the scale stack at capture (identity if none)
glm::vec3 rotation { 0.f }; // active rotation at capture
bool has_offset { false };
bool has_scale { false };
scene::group_handle group {}; // group the node was read under
glm::dvec3 worldpos { 0.0 }; // transformed position (for models; sort key source)
bool isshape { false }; // terrain shape/lines -> load before models (ground first)
double sortkey { 0.0 }; // squared distance to camera; lower = built sooner
};
struct deserializer_state {
std::string scenariofile;
cParser input;
@@ -45,13 +27,12 @@ struct deserializer_state {
bool visualphase { false };
// set once the whole load (both passes / single text pass) has fully finished
bool done { false };
// visual phase sub-state: while enumerate is true the visual pass captures deferred
// nodes into records instead of building them; once the replay is exhausted they are
// sorted by camera distance and built in that order (enumdone).
bool enumerate { false };
bool enumdone { false };
std::vector<visual_record> records;
std::size_t buildcursor { 0 };
// camera-ring visual streaming: the visual phase replays the twin once per distance
// ring (nearest first), building only the nodes whose distance to ringeye falls in the
// current ring and O(1)-skipping the rest. ringeye is sampled once when the visual phase
// starts so the ring partition is stable across passes.
int ringindex { 0 };
glm::dvec3 ringeye { 0.0 };
deserializer_state(std::string const &File, cParser::buffertype const Type, const std::string &Path, bool const Loadtraction)
: scenariofile(File), input(File, Type, Path, Loadtraction) { }
@@ -93,10 +74,6 @@ private:
void deserialize_endgroup( cParser &Input, scene::scratch_data &Scratchpad );
void deserialize_light( cParser &Input, scene::scratch_data &Scratchpad );
void deserialize_node( cParser &Input, scene::scratch_data &Scratchpad );
// visual streaming (camera-ordered deferred load): capture one visual node (already
// past its "node" token) into a record; build budgeted records in sorted order.
void enumerate_visual_node( deserializer_state &State );
bool build_visual_records( std::shared_ptr<deserializer_state> State );
void deserialize_origin( cParser &Input, scene::scratch_data &Scratchpad );
void deserialize_endorigin( cParser &Input, scene::scratch_data &Scratchpad );
void deserialize_scale( cParser &Input, scene::scratch_data &Scratchpad );
@@ -122,6 +99,15 @@ private:
// transforms provided location by specifed rotation and offset
glm::dvec3 transform( glm::dvec3 Location, scene::scratch_data const &Scratchpad );
void export_nodes_to_stream( std::ostream &, bool Dirty ) const;
// members
// camera-ring visual streaming state, mirrored from deserializer_state each
// deserialize_continue() call so deserialize_model()/deserialize_node() can ring-test a
// node by distance. inactive (builds everything) outside the ring/visual phase.
bool m_ringactive { false };
int m_ringindex { 0 };
glm::dvec3 m_ringeye { 0.0 };
double m_ringmin2 { 0.0 };
double m_ringmax2 { 0.0 };
};
} // simulation

15
utilities/parser.cpp
View File

@@ -888,6 +888,21 @@ void cParser::readReplayToken(std::string &out, bool ToLower, const char *Break)
out.clear();
}
bool cParser::skipReplayNode()
{
// a node's entries are contiguous within a single file, served by the deepest active
// include child -- delegate down to whichever parser is actually replaying it
if (mIncludeParser) { return mIncludeParser->skipReplayNode(); }
if (m_replay && m_reader && m_reader->skip_to_node_end())
{
// any tokens already pulled into the lookahead deque belong to the node we just
// skipped past (or earlier) -- drop them so the next read starts at the next node
tokens.clear();
return true;
}
return false;
}
std::vector<std::string> cParser::readParameters(cParser &Input)
{

5
utilities/parser.h
View File

@@ -57,6 +57,11 @@ class cParser //: public std::stringstream
// true when this (top-level) file is served from a binary twin, i.e. a second
// (visual) pass via restartReplay() is possible. false for a text/compile load.
bool isReplaying() const { return m_replay; }
// skips the rest of the node currently being replayed in O(1) (jump over its v6 marker
// span), delegating to the active include child that is actually serving it. returns
// false if the skip can't be done here (e.g. a text include with no binary twin), so the
// caller can fall back to a token-by-token skip. used by the camera-ring visual load.
bool skipReplayNode();
// methods:
template <typename Type_>
cParser &