From c337866c58016c299eda8582482805dd1956b438 Mon Sep 17 00:00:00 2001 From: maj00r Date: Mon, 22 Jun 2026 20:09:09 +0200 Subject: [PATCH] Add v6 node-class markers (Stage 2 foundation for progressive loading) Each top-level node is wrapped in a marker carrying its class (infrastructure vs visual) and byte span, so the reader can serve or skip a whole node per load pass without knowing its terminator token. The bake recognizes nodes by the "node" keyword + type token (map type->terminator confirmed against the deserializers) and classifies them; the writer buffers a node's entries and emits the marker. The reader gains a pass selector (all / infrastructure / visual) and skips nodes not in the pass by advancing past their byte span. This is the foundation only: the default pass is "all", so loading is identical to v5 (markers are transparent) -- verified by loading td.scn to the same point. The eager/visual split is wired up by the upcoming loader/driver integration. Co-Authored-By: Claude Opus 4.8 --- scene/scenerybinary.cpp | 94 +++++++++++++++++++++++++++++++---------- scene/scenerybinary.h | 38 ++++++++++++++--- utilities/parser.cpp | 80 +++++++++++++++++++++++++++++++++++ utilities/parser.h | 7 +++ 4 files changed, 190 insertions(+), 29 deletions(-) diff --git a/scene/scenerybinary.cpp b/scene/scenerybinary.cpp index dc7027ff..348cbc56 100644 --- a/scene/scenerybinary.cpp +++ b/scene/scenerybinary.cpp @@ -91,10 +91,17 @@ enum : std::uint64_t { TAG_F32 = 3, // followed by 4 little-endian bytes TAG_F64 = 4, // followed by 8 little-endian bytes TAG_QTOKEN = 5, // quoted token; head >> 3 == interned string index + TAG_NODE = 6, // node marker: followed by varint(class) + varint(byte span) TAG_BITS = 3, TAG_MASK = 0x7, }; +// node class stored in the TAG_NODE marker +enum : std::uint64_t { + NODECLASS_INFRA = 0, + NODECLASS_VISUAL = 1, +}; + // writes a numeric value in the most compact lossless-enough form: integral values as a // zig-zag varint, otherwise f32 when it represents the value with negligible error, // otherwise full f64. @@ -132,29 +139,54 @@ scenery_binary_writer::intern( std::string const &Text ) { return index; } +std::ostream & +scenery_binary_writer::sink() { + return ( m_innode ? m_nodebuf : m_entries ); +} + void scenery_binary_writer::add_token( std::string const &Token, bool Quoted ) { auto const tag = ( Quoted ? TAG_QTOKEN : TAG_TOKEN ); - write_varint( m_entries, ( static_cast( intern( Token ) ) << TAG_BITS ) | tag ); + write_varint( sink(), ( static_cast( intern( Token ) ) << TAG_BITS ) | tag ); ++m_count; } void scenery_binary_writer::add_number( double Value ) { - write_number( m_entries, Value ); + write_number( sink(), Value ); ++m_count; } void scenery_binary_writer::add_include( std::vector const &Fileexpr, std::vector const &Params ) { - write_varint( m_entries, TAG_INCLUDE ); - write_varint( m_entries, Fileexpr.size() ); - for( auto const &token : Fileexpr ) { write_varint( m_entries, intern( token ) ); } - write_varint( m_entries, Params.size() ); - for( auto const ¶m : Params ) { write_varint( m_entries, intern( param ) ); } + auto &out = sink(); + write_varint( out, TAG_INCLUDE ); + write_varint( out, Fileexpr.size() ); + for( auto const &token : Fileexpr ) { write_varint( out, intern( token ) ); } + write_varint( out, Params.size() ); + for( auto const ¶m : Params ) { write_varint( out, intern( param ) ); } ++m_count; } +void +scenery_binary_writer::begin_node() { + // start buffering this node's entries; end_node() emits the marker + buffered bytes + m_nodebuf.str( std::string() ); + m_nodebuf.clear(); + m_innode = true; +} + +void +scenery_binary_writer::end_node( bool Visual ) { + if( false == m_innode ) { return; } + m_innode = false; + auto const body = m_nodebuf.str(); + write_varint( m_entries, TAG_NODE ); + write_varint( m_entries, Visual ? NODECLASS_VISUAL : NODECLASS_INFRA ); + write_varint( m_entries, body.size() ); + m_entries.write( body.data(), static_cast( body.size() ) ); +} + bool scenery_binary_writer::write( std::ostream &Output, scenery_file_kind Kind ) const { @@ -171,8 +203,8 @@ scenery_binary_writer::write( std::ostream &Output, scenery_file_kind Kind ) con Output.write( text.data(), static_cast( text.size() ) ); } - // entries: count, then the pre-encoded entry bytes (heads + payloads) verbatim - write_varint( Output, m_count ); + // entries: the pre-encoded entry bytes (heads + payloads + node markers) verbatim, + // running to end-of-file (the reader streams until the buffer is exhausted) auto const encoded = m_entries.str(); Output.write( encoded.data(), static_cast( encoded.size() ) ); @@ -183,8 +215,8 @@ bool scenery_binary_reader::open( std::string_view Buffer ) { m_table.clear(); - m_cursor = m_end = nullptr; - m_remaining = m_total = 0; + m_begin = m_cursor = m_end = nullptr; + m_size = 0; char const *cursor = Buffer.data(); char const *const end = Buffer.data() + Buffer.size(); @@ -207,28 +239,46 @@ scenery_binary_reader::open( std::string_view Buffer ) { cursor += len; } - m_total = m_remaining = static_cast( read_varint( cursor, end ) ); - m_cursor = cursor; + // entries run from here to end-of-buffer + m_begin = m_cursor = cursor; m_end = end; + m_size = end - cursor; return true; } bool scenery_binary_reader::next( scenery_entry_view &Out ) { - if( ( m_remaining == 0 ) || ( m_cursor >= m_end ) ) { return false; } - --m_remaining; - - Out.fileexpr.clear(); - Out.params.clear(); - - auto const head = read_varint( m_cursor, m_end ); - auto const tag = head & TAG_MASK; - auto const resolve = [ this ]( std::uint64_t index ) -> std::string_view { return ( index < m_table.size() ) ? m_table[ static_cast( index ) ] : std::string_view(); }; + // skip node markers (and whole nodes not belonging to the current pass) until a + // servable entry is reached + std::uint64_t head = 0; + std::uint64_t tag = 0; + for( ;; ) { + if( m_cursor >= m_end ) { return false; } + head = read_varint( m_cursor, m_end ); + tag = head & TAG_MASK; + if( tag != TAG_NODE ) { break; } + auto const cls = read_varint( m_cursor, m_end ); + auto const span = read_varint( m_cursor, m_end ); + bool const process = + ( m_pass == scenery_load_pass::all ) + || ( ( m_pass == scenery_load_pass::infrastructure ) && ( cls == NODECLASS_INFRA ) ) + || ( ( m_pass == scenery_load_pass::visual ) && ( cls == NODECLASS_VISUAL ) ); + if( false == process ) { + // skip the whole node body + m_cursor += static_cast( span ); + if( m_cursor > m_end ) { m_cursor = m_end; } + } + // when processing, fall through: the loop re-reads and decodes the node's first entry + } + + Out.fileexpr.clear(); + Out.params.clear(); + switch( tag ) { case TAG_INCLUDE: { Out.type = scenery_entry_type::include; diff --git a/scene/scenerybinary.h b/scene/scenerybinary.h index 33b1435e..2cd40b6e 100644 --- a/scene/scenerybinary.h +++ b/scene/scenerybinary.h @@ -53,8 +53,19 @@ namespace scene { // v4: buffer-streamed reader, packed entry tags, zig-zag varint integers. // v5: tokens stored in original case (lower-cased per consumer at replay) with a quoted // flag, so baking is grammar-independent (enables the headless/standalone baker). +// v6: top-level nodes wrapped in a marker carrying their class (infrastructure/visual) +// and byte span, so the reader can serve or skip a whole node per load pass +// (enables progressive loading: infrastructure eager, visuals deferred). // bumping the version invalidates older twins so they are recompiled rather than misread. -constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 5 ) }; +constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 6 ) }; + +// which entries a reader serves in a given load pass; nodes outside the requested class +// are skipped (directives/includes are always served, to keep transform/group state) +enum class scenery_load_pass : std::uint8_t { + all = 0, // everything (single-pass load, == pre-v6 behaviour) + infrastructure = 1, // directives + infrastructure nodes; visual nodes skipped + visual = 2, // directives + visual nodes; infrastructure nodes skipped +}; // file extension of the binary twin, derived from source kind std::string const SCENERYBINARY_EXT_SCN { ".scnb" }; @@ -100,16 +111,24 @@ public: void add_number( double Value ); // Fileexpr is the verbatim filename expression (single token or random set) void add_include( std::vector const &Fileexpr, std::vector const &Params ); + // wrap a top-level node so the reader can serve/skip it per pass. between begin_node() + // and end_node() the add_*() entries are buffered; end_node() emits a marker (class + + // byte span) followed by the buffered entries. + void begin_node(); + void end_node( bool Visual ); std::size_t entry_count() const { return m_count; } // serializes header + string table + encoded entries. returns false on stream failure. bool write( std::ostream &Output, scenery_file_kind Kind ) const; private: std::uint32_t intern( std::string const &Text ); + std::ostream &sink(); // current entry sink: node buffer while in a node, else m_entries std::unordered_map m_lookup; // string -> table index std::vector m_table; // interned strings, in order std::ostringstream m_entries; // encoded entry bytes + std::ostringstream m_nodebuf; // current node's entries (buffered) + bool m_innode { false }; // currently between begin/end_node std::size_t m_count { 0 }; // number of entries }; @@ -123,18 +142,23 @@ public: bool open( std::string_view Buffer ); scenery_file_kind kind() const { return m_kind; } - // decodes the next entry into Out; returns false once all entries are consumed + // selects which nodes are served vs skipped (default: all). may be changed between + // reads (e.g. to drive separate eager/visual passes over a re-opened twin). + void set_pass( scenery_load_pass Pass ) { m_pass = Pass; } + // decodes the next served entry into Out, skipping node markers / out-of-pass nodes; + // returns false once all entries are consumed bool next( scenery_entry_view &Out ); - bool exhausted() const { return m_remaining == 0; } - // fraction of entries consumed so far, 0..100, for the loading bar - int progress() const { return ( m_total == 0 ? 100 : static_cast( ( m_total - m_remaining ) * 100 / m_total ) ); } + bool exhausted() const { return m_cursor >= m_end; } + // fraction of bytes consumed so far, 0..100, for the loading bar + int progress() const { return ( m_size == 0 ? 100 : static_cast( ( m_cursor - m_begin ) * 100 / m_size ) ); } private: std::vector m_table; + char const *m_begin { nullptr }; // start of the entry section char const *m_cursor { nullptr }; char const *m_end { nullptr }; - std::uint32_t m_remaining { 0 }; - std::uint32_t m_total { 0 }; + std::ptrdiff_t m_size { 0 }; // entry section byte length + scenery_load_pass m_pass { scenery_load_pass::all }; scenery_file_kind m_kind { scenery_file_kind::scn }; }; diff --git a/utilities/parser.cpp b/utilities/parser.cpp index ba9f0238..741de367 100644 --- a/utilities/parser.cpp +++ b/utilities/parser.cpp @@ -90,6 +90,34 @@ inline std::string formatNumber(double Value) return std::string(buffer, result.ptr); } +// classifies a scenery node by its `type` token (lower-cased) for v6 markers: sets Visual +// and the node's terminator token. returns false for an unrecognized type. mapping +// confirmed against the node deserializers. +inline bool classifyNodeType(std::string const &Type, bool &Visual, std::string &Endtoken) +{ + struct entry { char const *type; bool visual; char const *end; }; + static const entry table[] = { + { "model", true, "endmodel" }, + { "triangles", true, "endtri" }, + { "triangle_strip", true, "endtri" }, + { "triangle_fan", true, "endtri" }, + { "lines", true, "endline" }, + { "line_strip", true, "endline" }, + { "line_loop", true, "endline" }, + { "track", false, "endtrack" }, + { "traction", false, "endtraction" }, + { "tractionpowersource", false, "end" }, + { "memcell", false, "endmemcell" }, + { "eventlauncher", false, "endevent" }, + { "sound", false, "endsound" }, + { "dynamic", false, "enddynamic" }, + }; + for (auto const &e : table) { + if (Type == e.type) { Visual = e.visual; Endtoken = e.end; return true; } + } + return false; +} + inline bool endsWithLower(const std::string &s, const char *suffix) { const std::string suf(suffix); @@ -286,12 +314,23 @@ cParser::~cParser() } } +void cParser::bakeFinishNode() +{ + // flush a node still open at end-of-file or one whose type was unrecognized + if (m_bakenode_active && m_writer) + { + m_writer->end_node(m_bakenode_visual); + } + m_bakenode_active = false; +} + std::vector cParser::bakeFile() { // drain the file: every token is captured into the twin, include directives are // recorded (their candidate filenames collected) but not opened std::string token; do { token = getToken(); } while (false == token.empty()); + bakeFinishNode(); // write the twin synchronously -- the parallel baker already runs one file per // worker thread, so there is no point handing this to the async pool @@ -319,6 +358,7 @@ void cParser::flushBinaryTwin() // source not fully consumed (aborted parse): don't leave a truncated twin return; } + bakeFinishNode(); // close a node still open at end-of-file m_twinwritten = true; // hand the finished writer to the background pool; serialization and file I/O then @@ -698,6 +738,30 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break) if (m_compiling && m_writer && fromOwn && (false == m_capturesuppress) && (false == rawtoken.empty())) { + // v6: wrap each top-level node in a marker so the reader can serve/skip it per + // load pass. a node starts at the "node" keyword and ends at its type-specific + // terminator; nodes are never nested. + std::string const lowered = ::ToLower(rawtoken); // ::-qualified; the param `ToLower` shadows it + if ((false == m_bakenode_active) && (lowered == "node")) + { + m_writer->begin_node(); + m_bakenode_active = true; + m_bakenode_count = 0; + m_bakenode_visual = false; + m_bakenode_end.clear(); + } + else if (m_bakenode_active && m_bakenode_end.empty() && (lowered == "node")) + { + // previous node had an unrecognized type (no terminator found): close it + // just before this new node starts + bakeFinishNode(); + m_writer->begin_node(); + m_bakenode_active = true; + m_bakenode_count = 0; + m_bakenode_visual = false; + m_bakenode_end.clear(); + } + // numeric tokens -> typed values; quoted strings preserve case at replay; // other strings (names, paths, keywords, "(pN)") may be lower-cased per consumer double value = 0.0; @@ -709,6 +773,22 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break) { m_writer->add_token(rawtoken, rawquoted); } + + if (m_bakenode_active) + { + ++m_bakenode_count; + if ((m_bakenode_count == 5) && m_bakenode_end.empty()) + { + // 5th node entry is the type token (node, range_max, range_min, name, type) + classifyNodeType(lowered, m_bakenode_visual, m_bakenode_end); + } + else if ((false == m_bakenode_end.empty()) && (lowered == m_bakenode_end)) + { + // terminator captured: close the node + m_writer->end_node(m_bakenode_visual); + m_bakenode_active = false; + } + } } return; } diff --git a/utilities/parser.h b/utilities/parser.h index 607781a7..b6be4c70 100644 --- a/utilities/parser.h +++ b/utilities/parser.h @@ -192,6 +192,13 @@ class cParser //: public std::stringstream // standalone/parallel bake: capture this file only, collect its include filenames bool m_bakeonly { false }; std::vector m_bakeincludes; + // v6 node-marker tracking during bake (wrap each top-level node for pass selection) + bool m_bakenode_active { false }; + int m_bakenode_count { 0 }; // entries captured since "node" (1=node,...,5=type) + bool m_bakenode_visual { false }; + std::string m_bakenode_end; // terminator token; empty until type classified + // flushes a node still open at end-of-file (or unknown type), so its buffer is written + void bakeFinishNode(); };