Add v6 node-class markers (Stage 2 foundation for progressive loading)

Each top-level node is wrapped in a marker carrying its class (infrastructure vs
visual) and byte span, so the reader can serve or skip a whole node per load pass
without knowing its terminator token. The bake recognizes nodes by the "node"
keyword + type token (map type->terminator confirmed against the deserializers)
and classifies them; the writer buffers a node's entries and emits the marker.
The reader gains a pass selector (all / infrastructure / visual) and skips nodes
not in the pass by advancing past their byte span.

This is the foundation only: the default pass is "all", so loading is identical
to v5 (markers are transparent) -- verified by loading td.scn to the same point.
The eager/visual split is wired up by the upcoming loader/driver integration.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

scene/scenerybinary.cpp

@@ -91,10 +91,17 @@ enum : std::uint64_t {
     TAG_F32     = 3, // followed by 4 little-endian bytes
     TAG_F64     = 4, // followed by 8 little-endian bytes
     TAG_QTOKEN  = 5, // quoted token; head >> 3 == interned string index
+    TAG_NODE    = 6, // node marker: followed by varint(class) + varint(byte span)
     TAG_BITS    = 3,
     TAG_MASK    = 0x7,
 };
 
+// node class stored in the TAG_NODE marker
+enum : std::uint64_t {
+    NODECLASS_INFRA  = 0,
+    NODECLASS_VISUAL = 1,
+};
+
 // writes a numeric value in the most compact lossless-enough form: integral values as a
 // zig-zag varint, otherwise f32 when it represents the value with negligible error,
 // otherwise full f64.
@@ -132,29 +139,54 @@ scenery_binary_writer::intern( std::string const &Text ) {
     return index;
 }
 
+std::ostream &
+scenery_binary_writer::sink() {
+    return ( m_innode ? m_nodebuf : m_entries );
+}
+
 void
 scenery_binary_writer::add_token( std::string const &Token, bool Quoted ) {
     auto const tag = ( Quoted ? TAG_QTOKEN : TAG_TOKEN );
-    write_varint( m_entries, ( static_cast<std::uint64_t>( intern( Token ) ) << TAG_BITS ) | tag );
+    write_varint( sink(), ( static_cast<std::uint64_t>( intern( Token ) ) << TAG_BITS ) | tag );
     ++m_count;
 }
 
 void
 scenery_binary_writer::add_number( double Value ) {
-    write_number( m_entries, Value );
+    write_number( sink(), Value );
     ++m_count;
 }
 
 void
 scenery_binary_writer::add_include( std::vector<std::string> const &Fileexpr, std::vector<std::string> const &Params ) {
-    write_varint( m_entries, TAG_INCLUDE );
+    auto &out = sink();
-    write_varint( m_entries, Fileexpr.size() );
+    write_varint( out, TAG_INCLUDE );
-    for( auto const &token : Fileexpr ) { write_varint( m_entries, intern( token ) ); }
+    write_varint( out, Fileexpr.size() );
-    write_varint( m_entries, Params.size() );
+    for( auto const &token : Fileexpr ) { write_varint( out, intern( token ) ); }
-    for( auto const &param : Params ) { write_varint( m_entries, intern( param ) ); }
+    write_varint( out, Params.size() );
+    for( auto const &param : Params ) { write_varint( out, intern( param ) ); }
     ++m_count;
 }
 
+void
+scenery_binary_writer::begin_node() {
+    // start buffering this node's entries; end_node() emits the marker + buffered bytes
+    m_nodebuf.str( std::string() );
+    m_nodebuf.clear();
+    m_innode = true;
+}
+
+void
+scenery_binary_writer::end_node( bool Visual ) {
+    if( false == m_innode ) { return; }
+    m_innode = false;
+    auto const body = m_nodebuf.str();
+    write_varint( m_entries, TAG_NODE );
+    write_varint( m_entries, Visual ? NODECLASS_VISUAL : NODECLASS_INFRA );
+    write_varint( m_entries, body.size() );
+    m_entries.write( body.data(), static_cast<std::streamsize>( body.size() ) );
+}
+
 bool
 scenery_binary_writer::write( std::ostream &Output, scenery_file_kind Kind ) const {
 
@@ -171,8 +203,8 @@ scenery_binary_writer::write( std::ostream &Output, scenery_file_kind Kind ) con
         Output.write( text.data(), static_cast<std::streamsize>( text.size() ) );
     }
 
-    // entries: count, then the pre-encoded entry bytes (heads + payloads) verbatim
+    // entries: the pre-encoded entry bytes (heads + payloads + node markers) verbatim,
-    write_varint( Output, m_count );
+    // running to end-of-file (the reader streams until the buffer is exhausted)
     auto const encoded = m_entries.str();
     Output.write( encoded.data(), static_cast<std::streamsize>( encoded.size() ) );
 
@@ -183,8 +215,8 @@ bool
 scenery_binary_reader::open( std::string_view Buffer ) {
 
     m_table.clear();
-    m_cursor = m_end = nullptr;
+    m_begin = m_cursor = m_end = nullptr;
-    m_remaining = m_total = 0;
+    m_size = 0;
 
     char const *cursor = Buffer.data();
     char const *const end = Buffer.data() + Buffer.size();
@@ -207,28 +239,46 @@ scenery_binary_reader::open( std::string_view Buffer ) {
         cursor += len;
     }
 
-    m_total = m_remaining = static_cast<std::uint32_t>( read_varint( cursor, end ) );
+    // entries run from here to end-of-buffer
-    m_cursor = cursor;
+    m_begin = m_cursor = cursor;
     m_end = end;
+    m_size = end - cursor;
     return true;
 }
 
 bool
 scenery_binary_reader::next( scenery_entry_view &Out ) {
 
-    if( ( m_remaining == 0 ) || ( m_cursor >= m_end ) ) { return false; }
-    --m_remaining;
-
-    Out.fileexpr.clear();
-    Out.params.clear();
-
-    auto const head = read_varint( m_cursor, m_end );
-    auto const tag = head & TAG_MASK;
-
     auto const resolve = [ this ]( std::uint64_t index ) -> std::string_view {
         return ( index < m_table.size() ) ? m_table[ static_cast<std::size_t>( index ) ] : std::string_view();
     };
 
+    // skip node markers (and whole nodes not belonging to the current pass) until a
+    // servable entry is reached
+    std::uint64_t head = 0;
+    std::uint64_t tag = 0;
+    for( ;; ) {
+        if( m_cursor >= m_end ) { return false; }
+        head = read_varint( m_cursor, m_end );
+        tag = head & TAG_MASK;
+        if( tag != TAG_NODE ) { break; }
+        auto const cls = read_varint( m_cursor, m_end );
+        auto const span = read_varint( m_cursor, m_end );
+        bool const process =
+            ( m_pass == scenery_load_pass::all )
+         || ( ( m_pass == scenery_load_pass::infrastructure ) && ( cls == NODECLASS_INFRA ) )
+         || ( ( m_pass == scenery_load_pass::visual ) && ( cls == NODECLASS_VISUAL ) );
+        if( false == process ) {
+            // skip the whole node body
+            m_cursor += static_cast<std::ptrdiff_t>( span );
+            if( m_cursor > m_end ) { m_cursor = m_end; }
+        }
+        // when processing, fall through: the loop re-reads and decodes the node's first entry
+    }
+
+    Out.fileexpr.clear();
+    Out.params.clear();
+
     switch( tag ) {
         case TAG_INCLUDE: {
             Out.type = scenery_entry_type::include;

scene/scenerybinary.h

@@ -53,8 +53,19 @@ namespace scene {
 // v4: buffer-streamed reader, packed entry tags, zig-zag varint integers.
 // v5: tokens stored in original case (lower-cased per consumer at replay) with a quoted
 //     flag, so baking is grammar-independent (enables the headless/standalone baker).
+// v6: top-level nodes wrapped in a marker carrying their class (infrastructure/visual)
+//     and byte span, so the reader can serve or skip a whole node per load pass
+//     (enables progressive loading: infrastructure eager, visuals deferred).
 // bumping the version invalidates older twins so they are recompiled rather than misread.
-constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 5 ) };
+constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 6 ) };
+
+// which entries a reader serves in a given load pass; nodes outside the requested class
+// are skipped (directives/includes are always served, to keep transform/group state)
+enum class scenery_load_pass : std::uint8_t {
+    all = 0,            // everything (single-pass load, == pre-v6 behaviour)
+    infrastructure = 1, // directives + infrastructure nodes; visual nodes skipped
+    visual = 2,         // directives + visual nodes; infrastructure nodes skipped
+};
 
 // file extension of the binary twin, derived from source kind
 std::string const SCENERYBINARY_EXT_SCN { ".scnb" };
@@ -100,16 +111,24 @@ public:
     void add_number( double Value );
     // Fileexpr is the verbatim filename expression (single token or random set)
     void add_include( std::vector<std::string> const &Fileexpr, std::vector<std::string> const &Params );
+    // wrap a top-level node so the reader can serve/skip it per pass. between begin_node()
+    // and end_node() the add_*() entries are buffered; end_node() emits a marker (class +
+    // byte span) followed by the buffered entries.
+    void begin_node();
+    void end_node( bool Visual );
     std::size_t entry_count() const { return m_count; }
     // serializes header + string table + encoded entries. returns false on stream failure.
     bool write( std::ostream &Output, scenery_file_kind Kind ) const;
 
 private:
     std::uint32_t intern( std::string const &Text );
+    std::ostream &sink(); // current entry sink: node buffer while in a node, else m_entries
 
     std::unordered_map<std::string, std::uint32_t> m_lookup; // string -> table index
     std::vector<std::string> m_table;                        // interned strings, in order
     std::ostringstream m_entries;                            // encoded entry bytes
+    std::ostringstream m_nodebuf;                            // current node's entries (buffered)
+    bool m_innode { false };                                 // currently between begin/end_node
     std::size_t m_count { 0 };                               // number of entries
 };
 
@@ -123,18 +142,23 @@ public:
     bool open( std::string_view Buffer );
 
     scenery_file_kind kind() const { return m_kind; }
-    // decodes the next entry into Out; returns false once all entries are consumed
+    // selects which nodes are served vs skipped (default: all). may be changed between
+    // reads (e.g. to drive separate eager/visual passes over a re-opened twin).
+    void set_pass( scenery_load_pass Pass ) { m_pass = Pass; }
+    // decodes the next served entry into Out, skipping node markers / out-of-pass nodes;
+    // returns false once all entries are consumed
     bool next( scenery_entry_view &Out );
-    bool exhausted() const { return m_remaining == 0; }
+    bool exhausted() const { return m_cursor >= m_end; }
-    // fraction of entries consumed so far, 0..100, for the loading bar
+    // fraction of bytes consumed so far, 0..100, for the loading bar
-    int progress() const { return ( m_total == 0 ? 100 : static_cast<int>( ( m_total - m_remaining ) * 100 / m_total ) ); }
+    int progress() const { return ( m_size == 0 ? 100 : static_cast<int>( ( m_cursor - m_begin ) * 100 / m_size ) ); }
 
 private:
     std::vector<std::string_view> m_table;
+    char const *m_begin { nullptr }; // start of the entry section
     char const *m_cursor { nullptr };
     char const *m_end { nullptr };
-    std::uint32_t m_remaining { 0 };
+    std::ptrdiff_t m_size { 0 };      // entry section byte length
-    std::uint32_t m_total { 0 };
+    scenery_load_pass m_pass { scenery_load_pass::all };
     scenery_file_kind m_kind { scenery_file_kind::scn };
 };
 

utilities/parser.cpp

@@ -90,6 +90,34 @@ inline std::string formatNumber(double Value)
 	return std::string(buffer, result.ptr);
 }
 
+// classifies a scenery node by its `type` token (lower-cased) for v6 markers: sets Visual
+// and the node's terminator token. returns false for an unrecognized type. mapping
+// confirmed against the node deserializers.
+inline bool classifyNodeType(std::string const &Type, bool &Visual, std::string &Endtoken)
+{
+	struct entry { char const *type; bool visual; char const *end; };
+	static const entry table[] = {
+		{ "model",               true,  "endmodel" },
+		{ "triangles",           true,  "endtri" },
+		{ "triangle_strip",      true,  "endtri" },
+		{ "triangle_fan",        true,  "endtri" },
+		{ "lines",               true,  "endline" },
+		{ "line_strip",          true,  "endline" },
+		{ "line_loop",           true,  "endline" },
+		{ "track",               false, "endtrack" },
+		{ "traction",            false, "endtraction" },
+		{ "tractionpowersource", false, "end" },
+		{ "memcell",             false, "endmemcell" },
+		{ "eventlauncher",       false, "endevent" },
+		{ "sound",               false, "endsound" },
+		{ "dynamic",             false, "enddynamic" },
+	};
+	for (auto const &e : table) {
+		if (Type == e.type) { Visual = e.visual; Endtoken = e.end; return true; }
+	}
+	return false;
+}
+
 inline bool endsWithLower(const std::string &s, const char *suffix)
 {
 	const std::string suf(suffix);
@@ -286,12 +314,23 @@ cParser::~cParser()
 	}
 }
 
+void cParser::bakeFinishNode()
+{
+	// flush a node still open at end-of-file or one whose type was unrecognized
+	if (m_bakenode_active && m_writer)
+	{
+		m_writer->end_node(m_bakenode_visual);
+	}
+	m_bakenode_active = false;
+}
+
 std::vector<std::string> cParser::bakeFile()
 {
 	// drain the file: every token is captured into the twin, include directives are
 	// recorded (their candidate filenames collected) but not opened
 	std::string token;
 	do { token = getToken<std::string>(); } while (false == token.empty());
+	bakeFinishNode();
 
 	// write the twin synchronously -- the parallel baker already runs one file per
 	// worker thread, so there is no point handing this to the async pool
@@ -319,6 +358,7 @@ void cParser::flushBinaryTwin()
 		// source not fully consumed (aborted parse): don't leave a truncated twin
 		return;
 	}
+	bakeFinishNode(); // close a node still open at end-of-file
 	m_twinwritten = true;
 
 	// hand the finished writer to the background pool; serialization and file I/O then
@@ -698,6 +738,30 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break)
 		if (m_compiling && m_writer && fromOwn
 		 && (false == m_capturesuppress) && (false == rawtoken.empty()))
 		{
+			// v6: wrap each top-level node in a marker so the reader can serve/skip it per
+			// load pass. a node starts at the "node" keyword and ends at its type-specific
+			// terminator; nodes are never nested.
+			std::string const lowered = ::ToLower(rawtoken); // ::-qualified; the param `ToLower` shadows it
+			if ((false == m_bakenode_active) && (lowered == "node"))
+			{
+				m_writer->begin_node();
+				m_bakenode_active = true;
+				m_bakenode_count = 0;
+				m_bakenode_visual = false;
+				m_bakenode_end.clear();
+			}
+			else if (m_bakenode_active && m_bakenode_end.empty() && (lowered == "node"))
+			{
+				// previous node had an unrecognized type (no terminator found): close it
+				// just before this new node starts
+				bakeFinishNode();
+				m_writer->begin_node();
+				m_bakenode_active = true;
+				m_bakenode_count = 0;
+				m_bakenode_visual = false;
+				m_bakenode_end.clear();
+			}
+
 			// numeric tokens -> typed values; quoted strings preserve case at replay;
 			// other strings (names, paths, keywords, "(pN)") may be lower-cased per consumer
 			double value = 0.0;
@@ -709,6 +773,22 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break)
 			{
 				m_writer->add_token(rawtoken, rawquoted);
 			}
+
+			if (m_bakenode_active)
+			{
+				++m_bakenode_count;
+				if ((m_bakenode_count == 5) && m_bakenode_end.empty())
+				{
+					// 5th node entry is the type token (node, range_max, range_min, name, type)
+					classifyNodeType(lowered, m_bakenode_visual, m_bakenode_end);
+				}
+				else if ((false == m_bakenode_end.empty()) && (lowered == m_bakenode_end))
+				{
+					// terminator captured: close the node
+					m_writer->end_node(m_bakenode_visual);
+					m_bakenode_active = false;
+				}
+			}
 		}
 		return;
 	}

utilities/parser.h

@@ -192,6 +192,13 @@ class cParser //: public std::stringstream
     // standalone/parallel bake: capture this file only, collect its include filenames
     bool m_bakeonly { false };
     std::vector<std::string> m_bakeincludes;
+    // v6 node-marker tracking during bake (wrap each top-level node for pass selection)
+    bool m_bakenode_active { false };
+    int m_bakenode_count { 0 };       // entries captured since "node" (1=node,...,5=type)
+    bool m_bakenode_visual { false };
+    std::string m_bakenode_end;       // terminator token; empty until type classified
+    // flushes a node still open at end-of-file (or unknown type), so its buffer is written
+    void bakeFinishNode();
 };