eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-18 01:59:19 +02:00
Code Issues 121 Wiki Activity

Add v6 node-class markers (Stage 2 foundation for progressive loading)

Browse Source 
Each top-level node is wrapped in a marker carrying its class (infrastructure vs
visual) and byte span, so the reader can serve or skip a whole node per load pass
without knowing its terminator token. The bake recognizes nodes by the "node"
keyword + type token (map type->terminator confirmed against the deserializers)
and classifies them; the writer buffers a node's entries and emits the marker.
The reader gains a pass selector (all / infrastructure / visual) and skips nodes
not in the pass by advancing past their byte span.

This is the foundation only: the default pass is "all", so loading is identical
to v5 (markers are transparent) -- verified by loading td.scn to the same point.
The eager/visual split is wired up by the upcoming loader/driver integration.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
maj00r
2026-06-22 20:09:09 +02:00
parent 8a91559895
commit c337866c58
4 changed files with 190 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
scene/scenerybinary.cpp
View File

@@ -91,10 +91,17 @@ enum : std::uint64_t {
TAG_F32 = 3, // followed by 4 little-endian bytes
TAG_F64 = 4, // followed by 8 little-endian bytes
TAG_QTOKEN = 5, // quoted token; head >> 3 == interned string index
TAG_NODE = 6, // node marker: followed by varint(class) + varint(byte span)
TAG_BITS = 3,
TAG_MASK = 0x7,
};
// node class stored in the TAG_NODE marker
enum : std::uint64_t {
NODECLASS_INFRA = 0,
NODECLASS_VISUAL = 1,
};
// writes a numeric value in the most compact lossless-enough form: integral values as a
// zig-zag varint, otherwise f32 when it represents the value with negligible error,
// otherwise full f64.
@@ -132,29 +139,54 @@ scenery_binary_writer::intern( std::string const &Text ) {
return index;
}
std::ostream &
scenery_binary_writer::sink() {
return ( m_innode ? m_nodebuf : m_entries );
}
void
scenery_binary_writer::add_token( std::string const &Token, bool Quoted ) {
auto const tag = ( Quoted ? TAG_QTOKEN : TAG_TOKEN );
write_varint( m_entries, ( static_cast<std::uint64_t>( intern( Token ) ) << TAG_BITS ) | tag );
write_varint( sink(), ( static_cast<std::uint64_t>( intern( Token ) ) << TAG_BITS ) | tag );
++m_count;
}
void
scenery_binary_writer::add_number( double Value ) {
write_number( m_entries, Value );
write_number( sink(), Value );
++m_count;
}
void
scenery_binary_writer::add_include( std::vector<std::string> const &Fileexpr, std::vector<std::string> const &Params ) {
write_varint( m_entries, TAG_INCLUDE );
write_varint( m_entries, Fileexpr.size() );
for( auto const &token : Fileexpr ) { write_varint( m_entries, intern( token ) ); }
write_varint( m_entries, Params.size() );
for( auto const &param : Params ) { write_varint( m_entries, intern( param ) ); }
auto &out = sink();
write_varint( out, TAG_INCLUDE );
write_varint( out, Fileexpr.size() );
for( auto const &token : Fileexpr ) { write_varint( out, intern( token ) ); }
write_varint( out, Params.size() );
for( auto const &param : Params ) { write_varint( out, intern( param ) ); }
++m_count;
}
void
scenery_binary_writer::begin_node() {
// start buffering this node's entries; end_node() emits the marker + buffered bytes
m_nodebuf.str( std::string() );
m_nodebuf.clear();
m_innode = true;
}
void
scenery_binary_writer::end_node( bool Visual ) {
if( false == m_innode ) { return; }
m_innode = false;
auto const body = m_nodebuf.str();
write_varint( m_entries, TAG_NODE );
write_varint( m_entries, Visual ? NODECLASS_VISUAL : NODECLASS_INFRA );
write_varint( m_entries, body.size() );
m_entries.write( body.data(), static_cast<std::streamsize>( body.size() ) );
}
bool
scenery_binary_writer::write( std::ostream &Output, scenery_file_kind Kind ) const {
@@ -171,8 +203,8 @@ scenery_binary_writer::write( std::ostream &Output, scenery_file_kind Kind ) con
Output.write( text.data(), static_cast<std::streamsize>( text.size() ) );
}
// entries: count, then the pre-encoded entry bytes (heads + payloads) verbatim
write_varint( Output, m_count );
// entries: the pre-encoded entry bytes (heads + payloads + node markers) verbatim,
// running to end-of-file (the reader streams until the buffer is exhausted)
auto const encoded = m_entries.str();
Output.write( encoded.data(), static_cast<std::streamsize>( encoded.size() ) );
@@ -183,8 +215,8 @@ bool
scenery_binary_reader::open( std::string_view Buffer ) {
m_table.clear();
m_cursor = m_end = nullptr;
m_remaining = m_total = 0;
m_begin = m_cursor = m_end = nullptr;
m_size = 0;
char const *cursor = Buffer.data();
char const *const end = Buffer.data() + Buffer.size();
@@ -207,28 +239,46 @@ scenery_binary_reader::open( std::string_view Buffer ) {
cursor += len;
}
m_total = m_remaining = static_cast<std::uint32_t>( read_varint( cursor, end ) );
m_cursor = cursor;
// entries run from here to end-of-buffer
m_begin = m_cursor = cursor;
m_end = end;
m_size = end - cursor;
return true;
}
bool
scenery_binary_reader::next( scenery_entry_view &Out ) {
if( ( m_remaining == 0 ) || ( m_cursor >= m_end ) ) { return false; }
--m_remaining;
Out.fileexpr.clear();
Out.params.clear();
auto const head = read_varint( m_cursor, m_end );
auto const tag = head & TAG_MASK;
auto const resolve = [ this ]( std::uint64_t index ) -> std::string_view {
return ( index < m_table.size() ) ? m_table[ static_cast<std::size_t>( index ) ] : std::string_view();
};
// skip node markers (and whole nodes not belonging to the current pass) until a
// servable entry is reached
std::uint64_t head = 0;
std::uint64_t tag = 0;
for( ;; ) {
if( m_cursor >= m_end ) { return false; }
head = read_varint( m_cursor, m_end );
tag = head & TAG_MASK;
if( tag != TAG_NODE ) { break; }
auto const cls = read_varint( m_cursor, m_end );
auto const span = read_varint( m_cursor, m_end );
bool const process =
( m_pass == scenery_load_pass::all )
|| ( ( m_pass == scenery_load_pass::infrastructure ) && ( cls == NODECLASS_INFRA ) )
|| ( ( m_pass == scenery_load_pass::visual ) && ( cls == NODECLASS_VISUAL ) );
if( false == process ) {
// skip the whole node body
m_cursor += static_cast<std::ptrdiff_t>( span );
if( m_cursor > m_end ) { m_cursor = m_end; }
}
// when processing, fall through: the loop re-reads and decodes the node's first entry
}
Out.fileexpr.clear();
Out.params.clear();
switch( tag ) {
case TAG_INCLUDE: {
Out.type = scenery_entry_type::include;

38
scene/scenerybinary.h
View File

@@ -53,8 +53,19 @@ namespace scene {
// v4: buffer-streamed reader, packed entry tags, zig-zag varint integers.
// v5: tokens stored in original case (lower-cased per consumer at replay) with a quoted
// flag, so baking is grammar-independent (enables the headless/standalone baker).
// v6: top-level nodes wrapped in a marker carrying their class (infrastructure/visual)
// and byte span, so the reader can serve or skip a whole node per load pass
// (enables progressive loading: infrastructure eager, visuals deferred).
// bumping the version invalidates older twins so they are recompiled rather than misread.
constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 5 ) };
constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 6 ) };
// which entries a reader serves in a given load pass; nodes outside the requested class
// are skipped (directives/includes are always served, to keep transform/group state)
enum class scenery_load_pass : std::uint8_t {
all = 0, // everything (single-pass load, == pre-v6 behaviour)
infrastructure = 1, // directives + infrastructure nodes; visual nodes skipped
visual = 2, // directives + visual nodes; infrastructure nodes skipped
};
// file extension of the binary twin, derived from source kind
std::string const SCENERYBINARY_EXT_SCN { ".scnb" };
@@ -100,16 +111,24 @@ public:
void add_number( double Value );
// Fileexpr is the verbatim filename expression (single token or random set)
void add_include( std::vector<std::string> const &Fileexpr, std::vector<std::string> const &Params );
// wrap a top-level node so the reader can serve/skip it per pass. between begin_node()
// and end_node() the add_*() entries are buffered; end_node() emits a marker (class +
// byte span) followed by the buffered entries.
void begin_node();
void end_node( bool Visual );
std::size_t entry_count() const { return m_count; }
// serializes header + string table + encoded entries. returns false on stream failure.
bool write( std::ostream &Output, scenery_file_kind Kind ) const;
private:
std::uint32_t intern( std::string const &Text );
std::ostream &sink(); // current entry sink: node buffer while in a node, else m_entries
std::unordered_map<std::string, std::uint32_t> m_lookup; // string -> table index
std::vector<std::string> m_table; // interned strings, in order
std::ostringstream m_entries; // encoded entry bytes
std::ostringstream m_nodebuf; // current node's entries (buffered)
bool m_innode { false }; // currently between begin/end_node
std::size_t m_count { 0 }; // number of entries
};
@@ -123,18 +142,23 @@ public:
bool open( std::string_view Buffer );
scenery_file_kind kind() const { return m_kind; }
// decodes the next entry into Out; returns false once all entries are consumed
// selects which nodes are served vs skipped (default: all). may be changed between
// reads (e.g. to drive separate eager/visual passes over a re-opened twin).
void set_pass( scenery_load_pass Pass ) { m_pass = Pass; }
// decodes the next served entry into Out, skipping node markers / out-of-pass nodes;
// returns false once all entries are consumed
bool next( scenery_entry_view &Out );
bool exhausted() const { return m_remaining == 0; }
// fraction of entries consumed so far, 0..100, for the loading bar
int progress() const { return ( m_total == 0 ? 100 : static_cast<int>( ( m_total - m_remaining ) * 100 / m_total ) ); }
bool exhausted() const { return m_cursor >= m_end; }
// fraction of bytes consumed so far, 0..100, for the loading bar
int progress() const { return ( m_size == 0 ? 100 : static_cast<int>( ( m_cursor - m_begin ) * 100 / m_size ) ); }
private:
std::vector<std::string_view> m_table;
char const *m_begin { nullptr }; // start of the entry section
char const *m_cursor { nullptr };
char const *m_end { nullptr };
std::uint32_t m_remaining { 0 };
std::uint32_t m_total { 0 };
std::ptrdiff_t m_size { 0 }; // entry section byte length
scenery_load_pass m_pass { scenery_load_pass::all };
scenery_file_kind m_kind { scenery_file_kind::scn };
};

80
utilities/parser.cpp
View File

@@ -90,6 +90,34 @@ inline std::string formatNumber(double Value)
return std::string(buffer, result.ptr);
}
// classifies a scenery node by its `type` token (lower-cased) for v6 markers: sets Visual
// and the node's terminator token. returns false for an unrecognized type. mapping
// confirmed against the node deserializers.
inline bool classifyNodeType(std::string const &Type, bool &Visual, std::string &Endtoken)
{
struct entry { char const *type; bool visual; char const *end; };
static const entry table[] = {
{ "model", true, "endmodel" },
{ "triangles", true, "endtri" },
{ "triangle_strip", true, "endtri" },
{ "triangle_fan", true, "endtri" },
{ "lines", true, "endline" },
{ "line_strip", true, "endline" },
{ "line_loop", true, "endline" },
{ "track", false, "endtrack" },
{ "traction", false, "endtraction" },
{ "tractionpowersource", false, "end" },
{ "memcell", false, "endmemcell" },
{ "eventlauncher", false, "endevent" },
{ "sound", false, "endsound" },
{ "dynamic", false, "enddynamic" },
};
for (auto const &e : table) {
if (Type == e.type) { Visual = e.visual; Endtoken = e.end; return true; }
}
return false;
}
inline bool endsWithLower(const std::string &s, const char *suffix)
{
const std::string suf(suffix);
@@ -286,12 +314,23 @@ cParser::~cParser()
}
}
void cParser::bakeFinishNode()
{
// flush a node still open at end-of-file or one whose type was unrecognized
if (m_bakenode_active && m_writer)
{
m_writer->end_node(m_bakenode_visual);
}
m_bakenode_active = false;
}
std::vector<std::string> cParser::bakeFile()
{
// drain the file: every token is captured into the twin, include directives are
// recorded (their candidate filenames collected) but not opened
std::string token;
do { token = getToken<std::string>(); } while (false == token.empty());
bakeFinishNode();
// write the twin synchronously -- the parallel baker already runs one file per
// worker thread, so there is no point handing this to the async pool
@@ -319,6 +358,7 @@ void cParser::flushBinaryTwin()
// source not fully consumed (aborted parse): don't leave a truncated twin
return;
}
bakeFinishNode(); // close a node still open at end-of-file
m_twinwritten = true;
// hand the finished writer to the background pool; serialization and file I/O then
@@ -698,6 +738,30 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break)
if (m_compiling && m_writer && fromOwn
&& (false == m_capturesuppress) && (false == rawtoken.empty()))
{
// v6: wrap each top-level node in a marker so the reader can serve/skip it per
// load pass. a node starts at the "node" keyword and ends at its type-specific
// terminator; nodes are never nested.
std::string const lowered = ::ToLower(rawtoken); // ::-qualified; the param `ToLower` shadows it
if ((false == m_bakenode_active) && (lowered == "node"))
{
m_writer->begin_node();
m_bakenode_active = true;
m_bakenode_count = 0;
m_bakenode_visual = false;
m_bakenode_end.clear();
}
else if (m_bakenode_active && m_bakenode_end.empty() && (lowered == "node"))
{
// previous node had an unrecognized type (no terminator found): close it
// just before this new node starts
bakeFinishNode();
m_writer->begin_node();
m_bakenode_active = true;
m_bakenode_count = 0;
m_bakenode_visual = false;
m_bakenode_end.clear();
}
// numeric tokens -> typed values; quoted strings preserve case at replay;
// other strings (names, paths, keywords, "(pN)") may be lower-cased per consumer
double value = 0.0;
@@ -709,6 +773,22 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break)
{
m_writer->add_token(rawtoken, rawquoted);
}
if (m_bakenode_active)
{
++m_bakenode_count;
if ((m_bakenode_count == 5) && m_bakenode_end.empty())
{
// 5th node entry is the type token (node, range_max, range_min, name, type)
classifyNodeType(lowered, m_bakenode_visual, m_bakenode_end);
}
else if ((false == m_bakenode_end.empty()) && (lowered == m_bakenode_end))
{
// terminator captured: close the node
m_writer->end_node(m_bakenode_visual);
m_bakenode_active = false;
}
}
}
return;
}

7
utilities/parser.h
View File

@@ -192,6 +192,13 @@ class cParser //: public std::stringstream
// standalone/parallel bake: capture this file only, collect its include filenames
bool m_bakeonly { false };
std::vector<std::string> m_bakeincludes;
// v6 node-marker tracking during bake (wrap each top-level node for pass selection)
bool m_bakenode_active { false };
int m_bakenode_count { 0 }; // entries captured since "node" (1=node,...,5=type)
bool m_bakenode_visual { false };
std::string m_bakenode_end; // terminator token; empty until type classified
// flushes a node still open at end-of-file (or unknown type), so its buffer is written
void bakeFinishNode();
};