Store model position in the v7 node marker for O(1) ring skipping

The camera-ring visual load replays the twin once per distance ring, and each pass
had to read every node's header + X Y Z tokens (~8 tokens through the cParser pipeline)
just to decide whether the node is in the current ring -- expensive when a scenery has
a million flora instances.

Bump the twin format to v7: a visual model node's marker now carries its local position
(3 f32), captured at bake time from the node's first three numbers. The reader hands the
position out (scenery_binary_reader::node_position), and the deserializer ring-tests a
model and skips it in O(1) over the marker span, straight from the dispatch loop, without
deserialize_node decoding any of its tokens. deserialize_model uses the same marker
position for its own ring test so the two decisions agree exactly (a node near a ring
boundary can't be dropped by both adjacent rings). Shapes/older twins have no marker
position and fall back to the token-based test. The per-frame visual budget is factored
into VISUAL_BUDGET_MS.

Verified: td.scn rebakes to v7, no duplicates, no unexpected tokens, completes ~1s.
tomaszewo (1M+ flora) stays memory-bounded with no duplicates; full streaming is still
paced by walking every node per ring (the remaining cost is the replay pipeline itself,
which a spatial section index would address).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

scene/scenerybinary.cpp

@@ -98,8 +98,9 @@ enum : std::uint64_t {
 
 // node class stored in the TAG_NODE marker
 enum : std::uint64_t {
-    NODECLASS_INFRA  = 0,
-    NODECLASS_VISUAL = 1,
+    NODECLASS_INFRA      = 0,
+    NODECLASS_VISUAL     = 1,
+    NODECLASS_VISUAL_POS = 2, // visual node whose marker is followed by 3 f32 local position
 };
 
 // writes a numeric value in the most compact lossless-enough form: integral values as a
@@ -177,13 +178,24 @@ scenery_binary_writer::begin_node() {
 }
 
 void
-scenery_binary_writer::end_node( bool Visual ) {
+scenery_binary_writer::end_node( bool Visual, bool Haspos, double X, double Y, double Z ) {
     if( false == m_innode ) { return; }
     m_innode = false;
     auto const body = m_nodebuf.str();
     write_varint( m_entries, TAG_NODE );
-    write_varint( m_entries, Visual ? NODECLASS_VISUAL : NODECLASS_INFRA );
+    auto const cls =
+        ( false == Visual ) ? NODECLASS_INFRA :
+        ( Haspos          ) ? NODECLASS_VISUAL_POS :
+                              NODECLASS_VISUAL;
+    write_varint( m_entries, cls );
     write_varint( m_entries, body.size() );
+    // a visual model node stores its local position right after the span, so the reader can
+    // hand it to the camera-ring load without the node body being decoded
+    if( cls == NODECLASS_VISUAL_POS ) {
+        sn_utils::ls_float32( m_entries, static_cast<float>( X ) );
+        sn_utils::ls_float32( m_entries, static_cast<float>( Y ) );
+        sn_utils::ls_float32( m_entries, static_cast<float>( Z ) );
+    }
     m_entries.write( body.data(), static_cast<std::streamsize>( body.size() ) );
 }
 
@@ -264,14 +276,23 @@ scenery_binary_reader::next( scenery_entry_view &Out ) {
         if( tag != TAG_NODE ) { break; }
         auto const cls = read_varint( m_cursor, m_end );
         auto const span = read_varint( m_cursor, m_end );
+        bool const isvisual = ( cls == NODECLASS_VISUAL ) || ( cls == NODECLASS_VISUAL_POS );
+        // a visual model marker carries the node's local position right after the span
+        m_nodehaspos = ( cls == NODECLASS_VISUAL_POS );
+        if( true == m_nodehaspos ) {
+            m_nodepos[ 0 ] = static_cast<double>( read_f32le( m_cursor, m_end ) );
+            m_nodepos[ 1 ] = static_cast<double>( read_f32le( m_cursor, m_end ) );
+            m_nodepos[ 2 ] = static_cast<double>( read_f32le( m_cursor, m_end ) );
+        }
         bool const process =
             ( m_pass == scenery_load_pass::all )
          || ( ( m_pass == scenery_load_pass::infrastructure ) && ( cls == NODECLASS_INFRA ) )
-         || ( ( m_pass == scenery_load_pass::visual ) && ( cls == NODECLASS_VISUAL ) );
+         || ( ( m_pass == scenery_load_pass::visual ) && ( true == isvisual ) );
         if( false == process ) {
             // skip the whole node body
             m_cursor += static_cast<std::ptrdiff_t>( span );
             if( m_cursor > m_end ) { m_cursor = m_end; }
+            m_nodehaspos = false;
         }
         else {
             // remember where this node ends so the consumer can bail out of it in O(1)

scene/scenerybinary.h

@@ -56,8 +56,11 @@ namespace scene {
 // v6: top-level nodes wrapped in a marker carrying their class (infrastructure/visual)
 //     and byte span, so the reader can serve or skip a whole node per load pass
 //     (enables progressive loading: infrastructure eager, visuals deferred).
+// v7: a visual model node's marker also carries the model's local position (3 f32), so the
+//     camera-ring visual load can distance-test and skip a node in O(1) without reading any
+//     of its tokens -- the difference between scanning 1M flora nodes per ring and not.
 // bumping the version invalidates older twins so they are recompiled rather than misread.
-constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 6 ) };
+constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 7 ) };
 
 // which entries a reader serves in a given load pass; nodes outside the requested class
 // are skipped (directives/includes are always served, to keep transform/group state)
@@ -113,9 +116,11 @@ public:
     void add_include( std::vector<std::string> const &Fileexpr, std::vector<std::string> const &Params );
     // wrap a top-level node so the reader can serve/skip it per pass. between begin_node()
     // and end_node() the add_*() entries are buffered; end_node() emits a marker (class +
-    // byte span) followed by the buffered entries.
+    // byte span [+ local position for visual models]) followed by the buffered entries.
+    // Haspos/X/Y/Z give a model node's local position so the camera-ring load can skip it
+    // without reading its tokens.
     void begin_node();
-    void end_node( bool Visual );
+    void end_node( bool Visual, bool Haspos = false, double X = 0.0, double Y = 0.0, double Z = 0.0 );
     std::size_t entry_count() const { return m_count; }
     // serializes header + string table + encoded entries. returns false on stream failure.
     bool write( std::ostream &Output, scenery_file_kind Kind ) const;
@@ -153,6 +158,11 @@ public:
     // node; returns false (no-op) otherwise. used by the camera-ring visual load to drop a
     // node outside the current distance ring once its position has been read.
     bool skip_to_node_end() { if( m_nodeend == nullptr ) { return false; } m_cursor = m_nodeend; m_nodeend = nullptr; return true; }
+    // local position of the node currently being served, if its marker carried one (visual
+    // model nodes, v7+). returns false for shapes / infrastructure / older twins.
+    bool node_position( double &X, double &Y, double &Z ) const {
+        if( false == m_nodehaspos ) { return false; }
+        X = m_nodepos[ 0 ]; Y = m_nodepos[ 1 ]; Z = m_nodepos[ 2 ]; return true; }
     bool exhausted() const { return m_cursor >= m_end; }
     // fraction of bytes consumed so far, 0..100, for the loading bar
     int progress() const { return ( m_size == 0 ? 100 : static_cast<int>( ( m_cursor - m_begin ) * 100 / m_size ) ); }
@@ -163,6 +173,8 @@ private:
     char const *m_cursor { nullptr };
     char const *m_end { nullptr };
     char const *m_nodeend { nullptr }; // end of the node currently being served (for skip_to_node_end)
+    double m_nodepos[ 3 ] { 0.0, 0.0, 0.0 }; // local position of the current node (v7 model marker)
+    bool m_nodehaspos { false };            // whether the current node's marker carried a position
     std::ptrdiff_t m_size { 0 };      // entry section byte length
     scenery_load_pass m_pass { scenery_load_pass::all };
     scenery_file_kind m_kind { scenery_file_kind::scn };

simulation/simulationstateserializer.cpp

@@ -50,6 +50,10 @@ constexpr int RING_COUNT { static_cast<int>( sizeof( RING_OUTER2 ) / sizeof( RIN
 inline int ring_lastindex() { return RING_COUNT - 1; }
 inline double ring_min2( int const K ) { return ( K <= 0 ? 0.0 : RING_OUTER2[ K - 1 ] ); }
 inline double ring_max2( int const K ) { return RING_OUTER2[ ( K < RING_COUNT ? K : RING_COUNT - 1 ) ]; }
+// per-frame time budget (ms) the driver spends streaming visual nodes. larger = the
+// surroundings fill in faster but the frame it runs on is longer; on a heavy scene (low fps)
+// a too-small budget is a tiny duty cycle, so streaming a million flora instances drags.
+constexpr int VISUAL_BUDGET_MS { 16 };
 } // anonymous namespace
 
 std::shared_ptr<deserializer_state>
@@ -175,6 +179,29 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
                 token = Input.getToken<std::string>();
                 continue;
             }
+            // fast camera-ring skip: a visual model node carries its local position in its v7
+            // marker, so we can distance-test it and drop it in O(1) -- without deserialize_node
+            // decoding any of its tokens. this is what keeps the per-ring replay cheap when a
+            // scenery has a million flora instances. (shapes/older twins have no marker
+            // position; they fall through to deserialize_node, which ring-tests them itself.)
+            if( token == "node" ) {
+                double x, y, z;
+                if( true == Input.currentNodePosition( x, y, z ) ) {
+                    auto const world { transform( glm::dvec3{ x, y, z }, Scratchpad ) };
+                    auto const d { world - m_ringeye };
+                    auto const d2 { d.x * d.x + d.y * d.y + d.z * d.z };
+                    if( ( ( d2 < m_ringmin2 ) || ( d2 >= m_ringmax2 ) )
+                     && ( true == Input.skipReplayNode() ) ) {
+                        auto timenow = std::chrono::steady_clock::now();
+                        if( std::chrono::duration_cast<std::chrono::milliseconds>( timenow - timelast ).count() >= VISUAL_BUDGET_MS ) {
+                            Application.set_progress( Input.getProgress(), Input.getFullProgress() );
+                            return true;
+                        }
+                        token = Input.getToken<std::string>();
+                        continue;
+                    }
+                }
+            }
         }
 
 		auto lookup = state->functionmap.find( token );
@@ -186,9 +213,9 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
         }
 
 		auto timenow = std::chrono::steady_clock::now();
-        // small per-frame budget while streaming visuals in the driver (avoid stutter),
-        // generous budget while the loading screen is up (infrastructure pass)
-        auto const budget = ( state->visualphase ? 8 : 200 );
+        // per-frame budget while streaming visuals in the driver (kept modest to limit
+        // stutter), generous budget while the loading screen is up (infrastructure pass)
+        auto const budget = ( state->visualphase ? VISUAL_BUDGET_MS : 200 );
         if( std::chrono::duration_cast<std::chrono::milliseconds>( timenow - timelast ).count() >= budget ) {
             Application.set_progress( Input.getProgress(), Input.getFullProgress() );
 			return true;
@@ -1024,8 +1051,15 @@ state_serializer::deserialize_model( cParser &Input, scene::scratch_data &Scratc
     // camera-ring visual streaming: build this model only if it falls in the ring currently
     // being streamed; otherwise skip the rest of its body in O(1) and let a later (farther)
     // ring pass pick it up. covers terrain models (range_min<0) too -- they also have X Y Z.
+    // most out-of-ring models are already dropped O(1) at the dispatch loop via their v7
+    // marker; this is the fallback for nodes that reached here (in-ring, or no marker pos).
+    // use the marker position when present so this ring decision matches the dispatch one
+    // exactly -- otherwise a node near a ring boundary could be dropped by both adjacent rings.
     if( true == m_ringactive ) {
-        auto const world { transform( location, Scratchpad ) };
+        glm::dvec3 ringlocal { location };
+        double mx, my, mz;
+        if( true == Input.currentNodePosition( mx, my, mz ) ) { ringlocal = glm::dvec3{ mx, my, mz }; }
+        auto const world { transform( ringlocal, Scratchpad ) };
         auto const d { world - m_ringeye };
         auto const d2 { d.x * d.x + d.y * d.y + d.z * d.z };
         if( ( d2 < m_ringmin2 ) || ( d2 >= m_ringmax2 ) ) {

utilities/parser.cpp

@@ -319,7 +319,7 @@ void cParser::bakeFinishNode()
 	// flush a node still open at end-of-file or one whose type was unrecognized
 	if (m_bakenode_active && m_writer)
 	{
-		m_writer->end_node(m_bakenode_visual);
+		m_writer->end_node(m_bakenode_visual, m_bakenode_haspos, m_bakenode_pos[0], m_bakenode_pos[1], m_bakenode_pos[2]);
 	}
 	m_bakenode_active = false;
 }
@@ -774,6 +774,7 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break)
 				m_bakenode_active = true;
 				m_bakenode_count = 0;
 				m_bakenode_visual = false;
+				m_bakenode_haspos = false;
 				m_bakenode_end.clear();
 			}
 			else if (m_bakenode_active && m_bakenode_end.empty() && (lowered == "node"))
@@ -785,6 +786,7 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break)
 				m_bakenode_active = true;
 				m_bakenode_count = 0;
 				m_bakenode_visual = false;
+				m_bakenode_haspos = false;
 				m_bakenode_end.clear();
 			}
 
@@ -807,11 +809,18 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break)
 				{
 					// 5th node entry is the type token (node, range_max, range_min, name, type)
 					classifyNodeType(lowered, m_bakenode_visual, m_bakenode_end);
+					// a model node's entries 6,7,8 are its local X Y Z -- record them in the
+					// marker so the camera-ring load can skip the node without reading its body
+					m_bakenode_haspos = (lowered == "model");
+				}
+				else if (m_bakenode_haspos && (m_bakenode_count >= 6) && (m_bakenode_count <= 8))
+				{
+					m_bakenode_pos[m_bakenode_count - 6] = value;
 				}
 				else if ((false == m_bakenode_end.empty()) && (lowered == m_bakenode_end))
 				{
 					// terminator captured: close the node
-					m_writer->end_node(m_bakenode_visual);
+					m_writer->end_node(m_bakenode_visual, m_bakenode_haspos, m_bakenode_pos[0], m_bakenode_pos[1], m_bakenode_pos[2]);
 					m_bakenode_active = false;
 				}
 			}
@@ -903,6 +912,13 @@ bool cParser::skipReplayNode()
 	return false;
 }
 
+bool cParser::currentNodePosition(double &X, double &Y, double &Z)
+{
+	// delegate to the deepest active include child (it serves the current node), like skip
+	if (mIncludeParser) { return mIncludeParser->currentNodePosition(X, Y, Z); }
+	return (m_replay && m_reader && m_reader->node_position(X, Y, Z));
+}
+
 std::vector<std::string> cParser::readParameters(cParser &Input)
 {
 

utilities/parser.h

@@ -62,6 +62,10 @@ class cParser //: public std::stringstream
     // false if the skip can't be done here (e.g. a text include with no binary twin), so the
     // caller can fall back to a token-by-token skip. used by the camera-ring visual load.
     bool skipReplayNode();
+    // local position of the node about to be replayed (the "node" token just read), if its
+    // v7 marker carried one (visual model nodes). lets the camera-ring load distance-test and
+    // skip a node without decoding its body. returns false for shapes / non-replay / older twins.
+    bool currentNodePosition( double &X, double &Y, double &Z );
     // methods:
     template <typename Type_>
     cParser &
@@ -214,6 +218,10 @@ class cParser //: public std::stringstream
     int m_bakenode_count { 0 };       // entries captured since "node" (1=node,...,5=type)
     bool m_bakenode_visual { false };
     std::string m_bakenode_end;       // terminator token; empty until type classified
+    // v7: a model node's local position (entries 6,7,8 = X,Y,Z), stored in its marker so the
+    // camera-ring load can distance-test it without decoding the node body
+    bool m_bakenode_haspos { false };
+    double m_bakenode_pos[ 3 ] { 0.0, 0.0, 0.0 };
     // flushes a node still open at end-of-file (or unknown type), so its buffer is written
     void bakeFinishNode();
 };