eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-22 12:49:18 +02:00
Code Issues 123 Wiki Activity

Add v6 node-class markers (Stage 2 foundation for progressive loading)

Browse Source 
Each top-level node is wrapped in a marker carrying its class (infrastructure vs
visual) and byte span, so the reader can serve or skip a whole node per load pass
without knowing its terminator token. The bake recognizes nodes by the "node"
keyword + type token (map type->terminator confirmed against the deserializers)
and classifies them; the writer buffers a node's entries and emits the marker.
The reader gains a pass selector (all / infrastructure / visual) and skips nodes
not in the pass by advancing past their byte span.

This is the foundation only: the default pass is "all", so loading is identical
to v5 (markers are transparent) -- verified by loading td.scn to the same point.
The eager/visual split is wired up by the upcoming loader/driver integration.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
maj00r
2026-06-22 20:09:09 +02:00
parent 8a91559895
commit c337866c58
4 changed files with 190 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
scene/scenerybinary.cpp
View File

@@ -91,10 +91,17 @@ enum : std::uint64_t {
TAG_F32 = 3, // followed by 4 little-endian bytes
TAG_F64 = 4, // followed by 8 little-endian bytes
TAG_QTOKEN = 5, // quoted token; head >> 3 == interned string index
TAG_NODE = 6, // node marker: followed by varint(class) + varint(byte span)
TAG_BITS = 3,
TAG_MASK = 0x7,
};
// node class stored in the TAG_NODE marker
enum : std::uint64_t {
NODECLASS_INFRA = 0,
NODECLASS_VISUAL = 1,
};
// writes a numeric value in the most compact lossless-enough form: integral values as a
// zig-zag varint, otherwise f32 when it represents the value with negligible error,
// otherwise full f64.
@@ -132,29 +139,54 @@ scenery_binary_writer::intern( std::string const &Text ) {
return index;
}
std::ostream &
scenery_binary_writer::sink() {
return ( m_innode ? m_nodebuf : m_entries );
}
void
scenery_binary_writer::add_token( std::string const &Token, bool Quoted ) {
auto const tag = ( Quoted ? TAG_QTOKEN : TAG_TOKEN );
write_varint( m_entries, ( static_cast<std::uint64_t>( intern( Token ) ) << TAG_BITS ) | tag );
write_varint( sink(), ( static_cast<std::uint64_t>( intern( Token ) ) << TAG_BITS ) | tag );
++m_count;
}
void
scenery_binary_writer::add_number( double Value ) {
write_number( m_entries, Value );
write_number( sink(), Value );
++m_count;
}
void
scenery_binary_writer::add_include( std::vector<std::string> const &Fileexpr, std::vector<std::string> const &Params ) {
write_varint( m_entries, TAG_INCLUDE );
write_varint( m_entries, Fileexpr.size() );
for( auto const &token : Fileexpr ) { write_varint( m_entries, intern( token ) ); }
write_varint( m_entries, Params.size() );
for( auto const &param : Params ) { write_varint( m_entries, intern( param ) ); }
auto &out = sink();
write_varint( out, TAG_INCLUDE );
write_varint( out, Fileexpr.size() );
for( auto const &token : Fileexpr ) { write_varint( out, intern( token ) ); }
write_varint( out, Params.size() );
for( auto const &param : Params ) { write_varint( out, intern( param ) ); }
++m_count;
}
void
scenery_binary_writer::begin_node() {
// start buffering this node's entries; end_node() emits the marker + buffered bytes
m_nodebuf.str( std::string() );
m_nodebuf.clear();
m_innode = true;
}
void
scenery_binary_writer::end_node( bool Visual ) {
if( false == m_innode ) { return; }
m_innode = false;
auto const body = m_nodebuf.str();
write_varint( m_entries, TAG_NODE );
write_varint( m_entries, Visual ? NODECLASS_VISUAL : NODECLASS_INFRA );
write_varint( m_entries, body.size() );
m_entries.write( body.data(), static_cast<std::streamsize>( body.size() ) );
}
bool
scenery_binary_writer::write( std::ostream &Output, scenery_file_kind Kind ) const {
@@ -171,8 +203,8 @@ scenery_binary_writer::write( std::ostream &Output, scenery_file_kind Kind ) con
Output.write( text.data(), static_cast<std::streamsize>( text.size() ) );
}
// entries: count, then the pre-encoded entry bytes (heads + payloads) verbatim
write_varint( Output, m_count );
// entries: the pre-encoded entry bytes (heads + payloads + node markers) verbatim,
// running to end-of-file (the reader streams until the buffer is exhausted)
auto const encoded = m_entries.str();
Output.write( encoded.data(), static_cast<std::streamsize>( encoded.size() ) );
@@ -183,8 +215,8 @@ bool
scenery_binary_reader::open( std::string_view Buffer ) {
m_table.clear();
m_cursor = m_end = nullptr;
m_remaining = m_total = 0;
m_begin = m_cursor = m_end = nullptr;
m_size = 0;
char const *cursor = Buffer.data();
char const *const end = Buffer.data() + Buffer.size();
@@ -207,28 +239,46 @@ scenery_binary_reader::open( std::string_view Buffer ) {
cursor += len;
}
m_total = m_remaining = static_cast<std::uint32_t>( read_varint( cursor, end ) );
m_cursor = cursor;
// entries run from here to end-of-buffer
m_begin = m_cursor = cursor;
m_end = end;
m_size = end - cursor;
return true;
}
bool
scenery_binary_reader::next( scenery_entry_view &Out ) {
if( ( m_remaining == 0 ) || ( m_cursor >= m_end ) ) { return false; }
--m_remaining;
Out.fileexpr.clear();
Out.params.clear();
auto const head = read_varint( m_cursor, m_end );
auto const tag = head & TAG_MASK;
auto const resolve = [ this ]( std::uint64_t index ) -> std::string_view {
return ( index < m_table.size() ) ? m_table[ static_cast<std::size_t>( index ) ] : std::string_view();
};
// skip node markers (and whole nodes not belonging to the current pass) until a
// servable entry is reached
std::uint64_t head = 0;
std::uint64_t tag = 0;
for( ;; ) {
if( m_cursor >= m_end ) { return false; }
head = read_varint( m_cursor, m_end );
tag = head & TAG_MASK;
if( tag != TAG_NODE ) { break; }
auto const cls = read_varint( m_cursor, m_end );
auto const span = read_varint( m_cursor, m_end );
bool const process =
( m_pass == scenery_load_pass::all )
|| ( ( m_pass == scenery_load_pass::infrastructure ) && ( cls == NODECLASS_INFRA ) )
|| ( ( m_pass == scenery_load_pass::visual ) && ( cls == NODECLASS_VISUAL ) );
if( false == process ) {
// skip the whole node body
m_cursor += static_cast<std::ptrdiff_t>( span );
if( m_cursor > m_end ) { m_cursor = m_end; }
}
// when processing, fall through: the loop re-reads and decodes the node's first entry
}
Out.fileexpr.clear();
Out.params.clear();
switch( tag ) {
case TAG_INCLUDE: {
Out.type = scenery_entry_type::include;

38
scene/scenerybinary.h
View File

@@ -53,8 +53,19 @@ namespace scene {
// v4: buffer-streamed reader, packed entry tags, zig-zag varint integers.
// v5: tokens stored in original case (lower-cased per consumer at replay) with a quoted
// flag, so baking is grammar-independent (enables the headless/standalone baker).
// v6: top-level nodes wrapped in a marker carrying their class (infrastructure/visual)
// and byte span, so the reader can serve or skip a whole node per load pass
// (enables progressive loading: infrastructure eager, visuals deferred).
// bumping the version invalidates older twins so they are recompiled rather than misread.
constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 5 ) };
constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 6 ) };
// which entries a reader serves in a given load pass; nodes outside the requested class
// are skipped (directives/includes are always served, to keep transform/group state)
enum class scenery_load_pass : std::uint8_t {
all = 0, // everything (single-pass load, == pre-v6 behaviour)
infrastructure = 1, // directives + infrastructure nodes; visual nodes skipped
visual = 2, // directives + visual nodes; infrastructure nodes skipped
};
// file extension of the binary twin, derived from source kind
std::string const SCENERYBINARY_EXT_SCN { ".scnb" };
@@ -100,16 +111,24 @@ public:
void add_number( double Value );
// Fileexpr is the verbatim filename expression (single token or random set)
void add_include( std::vector<std::string> const &Fileexpr, std::vector<std::string> const &Params );
// wrap a top-level node so the reader can serve/skip it per pass. between begin_node()
// and end_node() the add_*() entries are buffered; end_node() emits a marker (class +
// byte span) followed by the buffered entries.
void begin_node();
void end_node( bool Visual );
std::size_t entry_count() const { return m_count; }
// serializes header + string table + encoded entries. returns false on stream failure.
bool write( std::ostream &Output, scenery_file_kind Kind ) const;
private:
std::uint32_t intern( std::string const &Text );
std::ostream &sink(); // current entry sink: node buffer while in a node, else m_entries
std::unordered_map<std::string, std::uint32_t> m_lookup; // string -> table index
std::vector<std::string> m_table; // interned strings, in order
std::ostringstream m_entries; // encoded entry bytes
std::ostringstream m_nodebuf; // current node's entries (buffered)
bool m_innode { false }; // currently between begin/end_node
std::size_t m_count { 0 }; // number of entries
};
@@ -123,18 +142,23 @@ public:
bool open( std::string_view Buffer );
scenery_file_kind kind() const { return m_kind; }
// decodes the next entry into Out; returns false once all entries are consumed
// selects which nodes are served vs skipped (default: all). may be changed between
// reads (e.g. to drive separate eager/visual passes over a re-opened twin).
void set_pass( scenery_load_pass Pass ) { m_pass = Pass; }
// decodes the next served entry into Out, skipping node markers / out-of-pass nodes;
// returns false once all entries are consumed
bool next( scenery_entry_view &Out );
bool exhausted() const { return m_remaining == 0; }
// fraction of entries consumed so far, 0..100, for the loading bar
int progress() const { return ( m_total == 0 ? 100 : static_cast<int>( ( m_total - m_remaining ) * 100 / m_total ) ); }
bool exhausted() const { return m_cursor >= m_end; }
// fraction of bytes consumed so far, 0..100, for the loading bar
int progress() const { return ( m_size == 0 ? 100 : static_cast<int>( ( m_cursor - m_begin ) * 100 / m_size ) ); }
private:
std::vector<std::string_view> m_table;
char const *m_begin { nullptr }; // start of the entry section
char const *m_cursor { nullptr };
char const *m_end { nullptr };
std::uint32_t m_remaining { 0 };
std::uint32_t m_total { 0 };
std::ptrdiff_t m_size { 0 }; // entry section byte length
scenery_load_pass m_pass { scenery_load_pass::all };
scenery_file_kind m_kind { scenery_file_kind::scn };
};