eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-18 03:09:18 +02:00
Code Issues 121 Wiki Activity

Improve binary scenery format (v5): streaming reader, smaller files, headless parallel bake

Browse Source 
- Reader is buffer-based and streaming: the twin is read once and parsed by
  pointer, entries decoded on demand (no per-byte stream, no vector-of-entry
  materialisation), string tokens served as views into the buffer.
- Format v5 is markedly smaller without compression: string interning with
  varint indices, entry type packed into the head varint, integers as zig-zag
  varint and fractional numbers as f32 (f64 only when needed). Tokens are stored
  in original case with a quoted flag and lower-cased per consumer at replay, so
  capture is grammar-independent.
- Writer encodes entries incrementally into a compact buffer instead of holding
  a struct per token, keeping memory bounded when baking huge files in parallel.
- New headless bake mode (-bake) precompiles a scenario and all its includes into
  twins on a thread pool, with no window/renderer/scene; each file is tokenised in
  isolation and baked exactly once.
- Fix stack overflow on files with long runs of consecutive includes by handling
  include directives iteratively instead of recursively (also hardens the normal
  text load).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
maj00r
2026-06-22 18:49:09 +02:00
parent 65fd1c10f3
commit eec5986056
5 changed files with 573 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
EU07.cpp
View File

@@ -20,6 +20,7 @@ Stele, firleju, szociu, hunter, ZiomalCl, OLI_EU and others
#include "application/application.h"
#include "utilities/Logs.h"
#include "scene/scenerybinary.h"
#include <cstdlib>
#ifdef WITHDUMPGEN
#ifdef _WIN32
@@ -98,6 +99,12 @@ int main(int argc, char *argv[])
int dynamic = std::stoi(std::string(argv[5]));
export_e3d_standalone(in, out, flags, dynamic);
}
// headless bake: precompile a scenery (and all its includes) into binary twins with
// no window, renderer or simulation. usage: eu07 -bake <sceneryfile relative to scenery/>
else if (argc >= 3 && std::string(argv[1]) == "-bake")
{
scene::scenerybinary_bake_headless(argv[2], Global.asCurrentSceneryPath, Global.bLoadTraction);
}
else
{
try

299
scene/scenerybinary.cpp
View File

@@ -21,6 +21,9 @@ http://mozilla.org/MPL/2.0/.
#include <queue>
#include <functional>
#include <fstream>
#include <string_view>
#include <cmath>
#include <cstring>
namespace scene {
@@ -37,173 +40,229 @@ void write_varint( std::ostream &Output, std::uint64_t Value ) {
} while( Value != 0 );
}
std::uint64_t read_varint( std::istream &Input ) {
// zig-zag maps small-magnitude signed integers to small unsigned varints
std::uint64_t zigzag_encode( std::int64_t Value ) {
return ( static_cast<std::uint64_t>( Value ) << 1 ) ^ static_cast<std::uint64_t>( Value >> 63 );
}
std::int64_t zigzag_decode( std::uint64_t Value ) {
return static_cast<std::int64_t>( ( Value >> 1 ) ^ ( ~( Value & 1 ) + 1 ) );
}
// buffer cursor reads (little-endian, matching sn_utils), bounds-checked: on overrun the
// cursor is parked at the end and zero is returned, so a truncated twin decodes to empty
// rather than reading out of bounds.
std::uint64_t read_varint( char const *&Cursor, char const *End ) {
std::uint64_t value = 0;
int shift = 0;
while( shift < 64 ) {
int const c = Input.get();
if( c == std::char_traits<char>::eof() ) { break; }
value |= ( static_cast<std::uint64_t>( c & 0x7F ) << shift );
if( ( c & 0x80 ) == 0 ) { break; }
while( ( Cursor < End ) && ( shift < 64 ) ) {
std::uint8_t const byte = static_cast<std::uint8_t>( *Cursor++ );
value |= ( static_cast<std::uint64_t>( byte & 0x7F ) << shift );
if( ( byte & 0x80 ) == 0 ) { break; }
shift += 7;
}
return value;
}
std::uint32_t read_u32le( char const *&Cursor, char const *End ) {
if( End - Cursor < 4 ) { Cursor = End; return 0; }
auto const *b = reinterpret_cast<std::uint8_t const *>( Cursor );
Cursor += 4;
return ( std::uint32_t( b[ 0 ] ) ) | ( std::uint32_t( b[ 1 ] ) << 8 )
| ( std::uint32_t( b[ 2 ] ) << 16 ) | ( std::uint32_t( b[ 3 ] ) << 24 );
}
float read_f32le( char const *&Cursor, char const *End ) {
if( End - Cursor < 4 ) { Cursor = End; return 0.f; }
std::uint32_t const v = read_u32le( Cursor, End );
float f; std::memcpy( &f, &v, 4 ); return f;
}
double read_f64le( char const *&Cursor, char const *End ) {
if( End - Cursor < 8 ) { Cursor = End; return 0.0; }
auto const *b = reinterpret_cast<std::uint8_t const *>( Cursor );
Cursor += 8;
std::uint64_t v = 0;
for( int i = 0; i < 8; ++i ) { v |= ( static_cast<std::uint64_t>( b[ i ] ) << ( 8 * i ) ); }
double d; std::memcpy( &d, &v, 8 ); return d;
}
// builds the per-file string table while interning, returning each string's index
class string_interner {
public:
std::uint32_t intern( std::string const &Text ) {
auto const it = m_lookup.find( Text );
if( it != m_lookup.end() ) { return it->second; }
auto const index = static_cast<std::uint32_t>( m_table.size() );
m_lookup.emplace( Text, index );
m_table.emplace_back( Text );
return index;
}
std::vector<std::string> const &table() const { return m_table; }
private:
std::unordered_map<std::string, std::uint32_t> m_lookup;
std::vector<std::string> m_table;
// on-disk entry tag, packed into the low 3 bits of the per-entry head varint
enum : std::uint64_t {
TAG_TOKEN = 0, // head >> 3 == interned string index
TAG_INCLUDE = 1,
TAG_INT = 2, // followed by a zig-zag varint
TAG_F32 = 3, // followed by 4 little-endian bytes
TAG_F64 = 4, // followed by 8 little-endian bytes
TAG_QTOKEN = 5, // quoted token; head >> 3 == interned string index
TAG_BITS = 3,
TAG_MASK = 0x7,
};
// writes a numeric value in the most compact lossless-enough form: integral values as a
// zig-zag varint, otherwise f32 when it represents the value with negligible error,
// otherwise full f64.
void write_number( std::ostream &Output, double Value ) {
double integral = 0.0;
if( ( std::modf( Value, &integral ) == 0.0 )
&& ( Value >= -9.007199254740992e15 ) && ( Value <= 9.007199254740992e15 ) ) {
write_varint( Output, TAG_INT );
write_varint( Output, zigzag_encode( static_cast<std::int64_t>( Value ) ) );
return;
}
float const f = static_cast<float>( Value );
bool const f32ok = std::isfinite( f )
&& ( ( static_cast<double>( f ) == Value )
|| ( std::abs( static_cast<double>( f ) - Value ) <= 1e-6 * std::abs( Value ) ) );
if( f32ok ) {
write_varint( Output, TAG_F32 );
sn_utils::ls_float32( Output, f );
}
else {
write_varint( Output, TAG_F64 );
sn_utils::ls_float64( Output, Value );
}
}
} // anonymous namespace
std::uint32_t
scenery_binary_writer::intern( std::string const &Text ) {
auto const it = m_lookup.find( Text );
if( it != m_lookup.end() ) { return it->second; }
auto const index = static_cast<std::uint32_t>( m_table.size() );
m_lookup.emplace( Text, index );
m_table.emplace_back( Text );
return index;
}
void
scenery_binary_writer::add_token( std::string Token ) {
scenery_entry entry;
entry.type = scenery_entry_type::token;
entry.text = std::move( Token );
m_entries.emplace_back( std::move( entry ) );
scenery_binary_writer::add_token( std::string const &Token, bool Quoted ) {
auto const tag = ( Quoted ? TAG_QTOKEN : TAG_TOKEN );
write_varint( m_entries, ( static_cast<std::uint64_t>( intern( Token ) ) << TAG_BITS ) | tag );
++m_count;
}
void
scenery_binary_writer::add_number( double Value ) {
scenery_entry entry;
entry.type = scenery_entry_type::number;
entry.number = Value;
m_entries.emplace_back( std::move( entry ) );
write_number( m_entries, Value );
++m_count;
}
void
scenery_binary_writer::add_include( std::vector<std::string> Fileexpr, std::vector<std::string> Params ) {
scenery_entry entry;
entry.type = scenery_entry_type::include;
entry.fileexpr = std::move( Fileexpr );
entry.params = std::move( Params );
m_entries.emplace_back( std::move( entry ) );
scenery_binary_writer::add_include( std::vector<std::string> const &Fileexpr, std::vector<std::string> const &Params ) {
write_varint( m_entries, TAG_INCLUDE );
write_varint( m_entries, Fileexpr.size() );
for( auto const &token : Fileexpr ) { write_varint( m_entries, intern( token ) ); }
write_varint( m_entries, Params.size() );
for( auto const &param : Params ) { write_varint( m_entries, intern( param ) ); }
++m_count;
}
bool
scenery_binary_writer::write( std::ostream &Output, scenery_file_kind Kind ) const {
// first pass: intern every string the entries reference so keywords/paths that
// repeat are stored once and referenced by index
string_interner interner;
std::vector<std::uint32_t> tokenindex; // index per token entry, in order
std::vector<std::vector<std::uint32_t>> fileexpridx, paramidx; // per include entry
for( auto const &entry : m_entries ) {
if( entry.type == scenery_entry_type::token ) {
tokenindex.emplace_back( interner.intern( entry.text ) );
}
else if( entry.type == scenery_entry_type::include ) {
std::vector<std::uint32_t> fe, pe;
for( auto const &token : entry.fileexpr ) { fe.emplace_back( interner.intern( token ) ); }
for( auto const &param : entry.params ) { pe.emplace_back( interner.intern( param ) ); }
fileexpridx.emplace_back( std::move( fe ) );
paramidx.emplace_back( std::move( pe ) );
}
}
// header: magic + kind + flags
sn_utils::ls_uint32( Output, SCENERYBINARY_MAGIC );
sn_utils::s_uint8( Output, static_cast<std::uint8_t>( Kind ) );
sn_utils::ls_uint32( Output, 0 ); // flags, reserved
// string table
auto const &table = interner.table();
sn_utils::ls_uint32( Output, static_cast<std::uint32_t>( table.size() ) );
for( auto const &text : table ) {
sn_utils::s_str( Output, text );
// string table: count, then each string as varint(length) + raw bytes (so the
// reader can take views into the buffer without scanning for terminators)
write_varint( Output, m_table.size() );
for( auto const &text : m_table ) {
write_varint( Output, text.size() );
Output.write( text.data(), static_cast<std::streamsize>( text.size() ) );
}
// entries
sn_utils::ls_uint32( Output, static_cast<std::uint32_t>( m_entries.size() ) );
std::size_t tokencursor = 0, includecursor = 0;
for( auto const &entry : m_entries ) {
sn_utils::s_uint8( Output, static_cast<std::uint8_t>( entry.type ) );
if( entry.type == scenery_entry_type::include ) {
auto const &fe = fileexpridx[ includecursor ];
auto const &pe = paramidx[ includecursor ];
++includecursor;
write_varint( Output, fe.size() );
for( auto idx : fe ) { write_varint( Output, idx ); }
write_varint( Output, pe.size() );
for( auto idx : pe ) { write_varint( Output, idx ); }
}
else if( entry.type == scenery_entry_type::number ) {
sn_utils::ls_float64( Output, entry.number );
}
else {
write_varint( Output, tokenindex[ tokencursor++ ] );
}
}
// entries: count, then the pre-encoded entry bytes (heads + payloads) verbatim
write_varint( Output, m_count );
auto const encoded = m_entries.str();
Output.write( encoded.data(), static_cast<std::streamsize>( encoded.size() ) );
return ( false == Output.fail() );
}
bool
scenery_binary_reader::open( std::istream &Input ) {
scenery_binary_reader::open( std::string_view Buffer ) {
m_entries.clear();
m_table.clear();
m_cursor = m_end = nullptr;
m_remaining = m_total = 0;
auto const magic { sn_utils::ld_uint32( Input ) };
if( magic != SCENERYBINARY_MAGIC ) {
char const *cursor = Buffer.data();
char const *const end = Buffer.data() + Buffer.size();
if( end - cursor < 9 ) { return false; } // magic(4) + kind(1) + flags(4)
if( read_u32le( cursor, end ) != SCENERYBINARY_MAGIC ) {
// unrecognized type or incompatible version
return false;
}
m_kind = static_cast<scenery_file_kind>( sn_utils::d_uint8( Input ) );
sn_utils::ld_uint32( Input ); // flags, reserved
m_kind = static_cast<scenery_file_kind>( static_cast<std::uint8_t>( *cursor++ ) );
read_u32le( cursor, end ); // flags, reserved
// string table
std::vector<std::string> table;
auto tablesize { sn_utils::ld_uint32( Input ) };
table.reserve( tablesize );
while( tablesize-- ) {
table.emplace_back( sn_utils::d_str( Input ) );
// string table: views into the buffer (no copies)
auto tablesize = read_varint( cursor, end );
m_table.reserve( tablesize );
while( ( tablesize-- != 0 ) && ( cursor < end ) ) {
auto const len = read_varint( cursor, end );
if( static_cast<std::uint64_t>( end - cursor ) < len ) { cursor = end; break; }
m_table.emplace_back( cursor, static_cast<std::size_t>( len ) );
cursor += len;
}
// resolves an interned index back to its string, guarding against corruption
auto const resolve = [ & ]( std::uint64_t index ) -> std::string {
return ( index < table.size() ) ? table[ static_cast<std::size_t>( index ) ] : std::string();
m_total = m_remaining = static_cast<std::uint32_t>( read_varint( cursor, end ) );
m_cursor = cursor;
m_end = end;
return true;
}
bool
scenery_binary_reader::next( scenery_entry_view &Out ) {
if( ( m_remaining == 0 ) || ( m_cursor >= m_end ) ) { return false; }
--m_remaining;
Out.fileexpr.clear();
Out.params.clear();
auto const head = read_varint( m_cursor, m_end );
auto const tag = head & TAG_MASK;
auto const resolve = [ this ]( std::uint64_t index ) -> std::string_view {
return ( index < m_table.size() ) ? m_table[ static_cast<std::size_t>( index ) ] : std::string_view();
};
auto count { sn_utils::ld_uint32( Input ) };
m_entries.reserve( count );
while( count-- ) {
scenery_entry entry;
entry.type = static_cast<scenery_entry_type>( sn_utils::d_uint8( Input ) );
if( entry.type == scenery_entry_type::include ) {
auto expr { read_varint( Input ) };
entry.fileexpr.reserve( expr );
while( expr-- ) {
entry.fileexpr.emplace_back( resolve( read_varint( Input ) ) );
}
auto params { read_varint( Input ) };
entry.params.reserve( params );
while( params-- ) {
entry.params.emplace_back( resolve( read_varint( Input ) ) );
}
switch( tag ) {
case TAG_INCLUDE: {
Out.type = scenery_entry_type::include;
auto fe = read_varint( m_cursor, m_end );
Out.fileexpr.reserve( fe );
while( ( fe-- != 0 ) && ( m_cursor < m_end ) ) { Out.fileexpr.emplace_back( resolve( read_varint( m_cursor, m_end ) ) ); }
auto pe = read_varint( m_cursor, m_end );
Out.params.reserve( pe );
while( ( pe-- != 0 ) && ( m_cursor < m_end ) ) { Out.params.emplace_back( resolve( read_varint( m_cursor, m_end ) ) ); }
break;
}
else if( entry.type == scenery_entry_type::number ) {
entry.number = sn_utils::ld_float64( Input );
}
else {
entry.text = resolve( read_varint( Input ) );
}
m_entries.emplace_back( std::move( entry ) );
case TAG_INT:
Out.type = scenery_entry_type::number;
Out.number = static_cast<double>( zigzag_decode( read_varint( m_cursor, m_end ) ) );
break;
case TAG_F32:
Out.type = scenery_entry_type::number;
Out.number = static_cast<double>( read_f32le( m_cursor, m_end ) );
break;
case TAG_F64:
Out.type = scenery_entry_type::number;
Out.number = read_f64le( m_cursor, m_end );
break;
case TAG_QTOKEN:
Out.type = scenery_entry_type::qtoken;
Out.text = resolve( head >> TAG_BITS );
break;
case TAG_TOKEN:
default:
Out.type = scenery_entry_type::token;
Out.text = resolve( head >> TAG_BITS );
break;
}
return ( false == Input.fail() );
return true;
}
namespace {

147
scene/scenerybinary.h
View File

@@ -11,46 +11,50 @@ http://mozilla.org/MPL/2.0/.
#include <cstdint>
#include <string>
#include <string_view>
#include <vector>
#include <iosfwd>
#include <memory>
#include <sstream>
#include <unordered_map>
#include "utilities/utilities.h" // MAKE_ID4
// Binary, modular scenery container ("eu7" format, version 1).
// Binary, modular scenery container ("eu7" format).
//
// Replaces the legacy text formats (.scn / .scm / .inc) and the old terrain-only
// .sbt blob. The mapping is one-to-one and modular: every text file compiles to
// exactly one binary twin and includes are preserved as references, so the binary
// twin of one file points to the binary twins of the files it includes.
// route.scn -> route.scnb
// piece.inc -> piece.incb
// mesh.scm -> mesh.scmb
// Replaces the legacy text formats (.scn / .scm / .inc / .ctr) and the old terrain-only
// .sbt blob. The mapping is one-to-one and modular: every text file compiles to exactly
// one binary twin and includes are preserved as references, so the binary twin of one
// file points to the binary twins of the files it includes.
// route.scn -> route.scnb piece.inc -> piece.incb
// mesh.scm -> mesh.scmb events.ctr -> events.ctrb
//
// A binary twin stores the source file's own, fully resolved content (comments
// stripped, parameters and random sets resolved) as an ordered list of entries.
// Numeric tokens are stored as 8-byte IEEE doubles (so coordinates, ranges etc. are
// genuinely binary, not ASCII). All string tokens (keywords like "node"/"endmodel",
// names, paths) are interned into a per-file string table and referenced from the
// entries by a small varint index, so a keyword that occurs thousands of times is
// stored once. An "include" entry marks the spot where the source file pulled in
// another file, carrying the (interned) include filename expression and parameters.
// A twin stores the source file's own, fully resolved content (comments stripped,
// parameters/random sets resolved) as an ordered list of entries:
// * string tokens (keywords like "node"/"endmodel", names, paths) are interned into a
// per-file table and referenced by a varint index, so a keyword that occurs
// thousands of times is stored once;
// * numeric tokens are stored typed -- integral values as a zig-zag varint (1-2 bytes
// for the many small ints), fractional values as an 8-byte IEEE double;
// * an "include" entry carries the (interned) filename expression and parameters.
// Each entry's type is packed into the low bits of a single varint together with the
// token index, so the common case (a token) costs just that one varint.
//
// The binary is consumed transparently at the cParser file layer: opening a file
// whose binary twin exists serves tokens from the twin instead of tokenizing text,
// and an include entry triggers the same include machinery as the text path (which
// recursively prefers the included file's own binary twin). The scenery
// deserializer is unaware of the format and needs no changes; the file's identity
// (cParser::Name()), node grouping per .inc and parameter substitution are all
// preserved exactly as in the text path.
// Reading is buffer-based and streaming: the whole twin is read into memory once and
// parsed by pointer (no per-byte stream calls), and entries are decoded on demand
// (no intermediate materialisation), with string tokens served as views into the
// buffer. The binary is consumed transparently at the cParser file layer; the scenery
// deserializer is unaware of the format. cParser::Name(), per-.inc node grouping and
// parameter substitution are preserved exactly as in the text path.
namespace scene {
// "eu7" + format version, little-endian via MAKE_ID4 ('e','u','7',<ver>).
// v2 introduced typed numeric entries (f64); v3 interns string tokens into a per-file
// table referenced by varint indices. bumping the version invalidates older twins so
// they are recompiled rather than misread.
constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 3 ) };
// v4: buffer-streamed reader, packed entry tags, zig-zag varint integers.
// v5: tokens stored in original case (lower-cased per consumer at replay) with a quoted
// flag, so baking is grammar-independent (enables the headless/standalone baker).
// bumping the version invalidates older twins so they are recompiled rather than misread.
constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 5 ) };
// file extension of the binary twin, derived from source kind
std::string const SCENERYBINARY_EXT_SCN { ".scnb" };
@@ -65,72 +69,91 @@ enum class scenery_file_kind : std::uint8_t {
scm = 2,
};
// kind of a stored entry
// logical kind of an entry as seen by the consumer (the on-disk integer/float split is
// an encoding detail hidden inside the reader/writer)
enum class scenery_entry_type : std::uint8_t {
token = 0, // a non-numeric resolved token (name/path/keyword), stored as a string
token = 0, // a non-numeric token (name/path/keyword); lower-cased per consumer at replay
include = 1, // an include directive: pulls in another file with parameters
number = 2, // a numeric token, stored as an 8-byte IEEE double
number = 2, // a numeric token
qtoken = 3, // a quoted token; case preserved verbatim at replay
};
// a single ordered element of a file's resolved content
struct scenery_entry {
// a decoded entry handed to the consumer during streaming reads; string fields are
// views into the reader's buffer and are valid until the next read / reader destruction
struct scenery_entry_view {
scenery_entry_type type { scenery_entry_type::token };
// token value (token entries only)
std::string text;
// numeric value (number entries only)
double number { 0.0 };
// include only: the raw filename expression as written in the source, i.e. either
// a single filename token or a random set "[ a b c ]" (including the brackets).
// kept verbatim so the random choice is re-evaluated on every replay rather than
// frozen at compile time.
std::vector<std::string> fileexpr;
// include only: directive parameters
std::vector<std::string> params;
std::string_view text; // token
double number { 0.0 }; // number
std::vector<std::string_view> fileexpr; // include
std::vector<std::string_view> params; // include
};
// accumulates a file's entries and writes them, with a header, to a stream
// accumulates a file's content and serializes it. entries are encoded incrementally into
// a compact byte buffer as they are added (strings interned on the fly), so even a huge
// source file costs only its interned table plus a few bytes per token -- not a heavy
// struct per token -- keeping parallel baking within memory.
class scenery_binary_writer {
public:
void add_token( std::string Token );
// stores a numeric token as a typed double
// Quoted marks the token as a quoted span (its case is preserved verbatim at replay)
void add_token( std::string const &Token, bool Quoted = false );
void add_number( double Value );
// Fileexpr is the verbatim filename expression (single token or random set)
void add_include( std::vector<std::string> Fileexpr, std::vector<std::string> Params );
std::size_t entry_count() const { return m_entries.size(); }
// serializes header + entries to the stream. returns false on stream failure.
void add_include( std::vector<std::string> const &Fileexpr, std::vector<std::string> const &Params );
std::size_t entry_count() const { return m_count; }
// serializes header + string table + encoded entries. returns false on stream failure.
bool write( std::ostream &Output, scenery_file_kind Kind ) const;
private:
std::vector<scenery_entry> m_entries;
std::uint32_t intern( std::string const &Text );
std::unordered_map<std::string, std::uint32_t> m_lookup; // string -> table index
std::vector<std::string> m_table; // interned strings, in order
std::ostringstream m_entries; // encoded entry bytes
std::size_t m_count { 0 }; // number of entries
};
// reads a binary scenery file fully into memory and exposes its entries in order
// parses a binary twin held in a memory buffer and streams its entries on demand.
// the buffer must outlive the reader (the cParser owns both).
class scenery_binary_reader {
public:
// validates magic/version and loads every entry. returns false if the stream is
// not a valid current-version binary scenery file.
bool open( std::istream &Input );
// validates magic/version and indexes the string table; positions at the first
// entry. returns false if Buffer is not a valid current-version twin.
bool open( std::string_view Buffer );
scenery_file_kind kind() const { return m_kind; }
std::vector<scenery_entry> const &entries() const { return m_entries; }
// decodes the next entry into Out; returns false once all entries are consumed
bool next( scenery_entry_view &Out );
bool exhausted() const { return m_remaining == 0; }
// fraction of entries consumed so far, 0..100, for the loading bar
int progress() const { return ( m_total == 0 ? 100 : static_cast<int>( ( m_total - m_remaining ) * 100 / m_total ) ); }
private:
std::vector<std::string_view> m_table;
char const *m_cursor { nullptr };
char const *m_end { nullptr };
std::uint32_t m_remaining { 0 };
std::uint32_t m_total { 0 };
scenery_file_kind m_kind { scenery_file_kind::scn };
std::vector<scenery_entry> m_entries;
};
// maps a source scenery filename to the extension of its binary twin
std::string scenerybinary_extension_for( std::string const &Sourcefile );
// Asynchronous baking: serializing and writing a twin is pure, self-contained work
// (it only needs the writer's collected entries), so it is offloaded to a small thread
// pool and overlaps with the rest of scene construction instead of blocking the load.
// Takes ownership of the writer.
// Asynchronous baking: serializing and writing a twin is self-contained work, so it is
// offloaded to a small thread pool and overlaps with scene construction. Takes ownership
// of the writer.
void scenerybinary_write_async( std::unique_ptr<scenery_binary_writer> Writer, std::string Path, scenery_file_kind Kind );
// Blocks until every queued async write has finished, then emits their log lines on the
// calling (main) thread. Call once the scenario has finished loading.
// Blocks until every queued async write has finished, logging results on the calling
// (main) thread. Call once the scenario has finished loading.
void scenerybinary_wait_all();
// Headless bake: tokenizes the scenario file and, recursively, every file it includes,
// writing each one's binary twin -- with no scene construction, renderer or window. Used
// by the "-bake" command line mode to precompile a scenery offline. Fresh twins are left
// untouched; only missing/stale ones are (re)compiled. Returns false if the file can't be
// opened.
bool scenerybinary_bake_headless( std::string const &Scenariofile, std::string const &Path, bool Loadtraction );
} // namespace scene

357
utilities/parser.cpp
View File

@@ -19,6 +19,12 @@ http://mozilla.org/MPL/2.0/.
#include <charconv>
#include <cmath>
#include <filesystem>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <deque>
#include <atomic>
#include <unordered_set>
/*
MaSzyna EU07 locomotive simulator parser
@@ -128,8 +134,8 @@ inline bool twinIsFresh(const std::string &sourcefull, const std::string &twinfu
} // namespace
// constructors
cParser::cParser(std::string const &Stream, buffertype const Type, std::string Path, bool const Loadtraction, std::vector<std::string> Parameters, bool allowRandom)
: allowRandomIncludes(allowRandom), LoadTraction(Loadtraction), mPath(Path)
cParser::cParser(std::string const &Stream, buffertype const Type, std::string Path, bool const Loadtraction, std::vector<std::string> Parameters, bool allowRandom, bool BakeOnly)
: allowRandomIncludes(allowRandom), LoadTraction(Loadtraction), mPath(Path), m_bakeonly(BakeOnly)
{
// store to calculate sub-sequent includes from relative path
if (Type == buffertype::buffer_FILE)
@@ -141,6 +147,26 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P
{
case buffer_FILE:
{
// bake-only: always compile this one file's twin from text, never touch scene
// groups and never replay; includes are recorded but not opened (the parallel
// baker compiles them separately)
if (m_bakeonly)
{
Path.append(Stream);
mStream = std::make_shared<std::ifstream>(Path, std::ios_base::binary);
if (false == mStream->fail())
{
m_compiling = true;
std::string twinrel = Stream;
erase_extension(twinrel);
twinrel += scene::scenerybinary_extension_for(Stream);
m_binarytwinpath = mPath + twinrel;
m_binarykind = sceneryKind(Stream);
m_writer = std::make_unique<scene::scenery_binary_writer>();
}
break;
}
// content of *.inc files is grouped together (same for text and binary replay)
if (endsWithLower(Stream, ".inc"))
{
@@ -161,18 +187,28 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P
std::string const sourcefull = mPath + Stream;
std::ifstream probe(twinfull, std::ios_base::binary);
scene::scenery_binary_reader reader;
if (probe.good() && twinIsFresh(sourcefull, twinfull) && reader.open(probe))
bool replaying = false;
if (probe.good() && twinIsFresh(sourcefull, twinfull))
{
// replay: serve tokens from the twin, no text tokenization at all
m_entries = reader.entries();
m_entrycount = m_entries.size();
m_replay = true;
m_entryindex = 0;
mStream = std::make_shared<std::istringstream>(std::string());
opened = true;
// slurp the whole twin once; the reader parses it by pointer and serves
// entries on demand (string tokens as views into this buffer)
std::ostringstream slurp;
slurp << probe.rdbuf();
m_twinbuf = std::move(slurp).str();
m_reader = std::make_unique<scene::scenery_binary_reader>();
if (m_reader->open(m_twinbuf))
{
m_replay = true;
mStream = std::make_shared<std::istringstream>(std::string());
replaying = true;
opened = true;
}
else
{
m_reader.reset();
}
}
else
if (false == replaying)
{
Path.append(Stream);
mStream = std::make_shared<std::ifstream>(Path, std::ios_base::binary);
@@ -205,17 +241,28 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P
break;
}
}
// calculate stream size
// slurp the whole source into memory and tokenize from the buffer; reading char
// by char from the stream (get()/peek()) goes through the virtual streambuf on
// every character, which dominates parse time on large sceneries.
if (mStream)
{
if (true == mStream->fail())
{
ErrorLog("Failed to open file \"" + Path + "\"");
// bake-only parsers run on worker threads; the logger is not thread-safe, so
// stay silent here (the driver checks ok() and simply skips a file it couldn't
// open). missing includes still surface during the normal text/replay load.
if (false == m_bakeonly)
{
ErrorLog("Failed to open file \"" + Path + "\"");
}
}
else
else if (false == m_replay)
{
mSize = mStream->rdbuf()->pubseekoff(0, std::ios_base::end);
mStream->rdbuf()->pubseekoff(0, std::ios_base::beg);
std::ostringstream slurp;
slurp << mStream->rdbuf();
m_buffer = std::move(slurp).str();
m_bufferpos = 0;
mSize = static_cast<std::streamoff>(m_buffer.size());
mLine = 1;
}
}
@@ -239,6 +286,26 @@ cParser::~cParser()
}
}
std::vector<std::string> cParser::bakeFile()
{
// drain the file: every token is captured into the twin, include directives are
// recorded (their candidate filenames collected) but not opened
std::string token;
do { token = getToken<std::string>(); } while (false == token.empty());
// write the twin synchronously -- the parallel baker already runs one file per
// worker thread, so there is no point handing this to the async pool
if (m_compiling && m_writer && (false == m_twinwritten) && (m_bufferpos >= m_buffer.size()))
{
m_twinwritten = true;
std::ofstream output(m_binarytwinpath, std::ios_base::binary);
// errors are intentionally not logged here (worker thread); a missing/short twin
// is simply recompiled on the next normal load
(void)(output.good() && m_writer->write(output, m_binarykind));
}
return std::move(m_bakeincludes);
}
void cParser::flushBinaryTwin()
{
// write the twin only once, only when compiling, and only if the source text was
@@ -247,8 +314,9 @@ void cParser::flushBinaryTwin()
{
return;
}
if ((nullptr == mStream) || (false == mStream->eof()))
if (m_bufferpos < m_buffer.size())
{
// source not fully consumed (aborted parse): don't leave a truncated twin
return;
}
m_twinwritten = true;
@@ -365,6 +433,13 @@ bool cParser::getTokens(unsigned int Count, bool ToLower, const char *Break)
std::string cParser::readTokenFromStream(bool ToLower, const char *Break)
{
// the token is produced in its ORIGINAL case; lower-casing (when requested) is applied
// to the consumer copy in readToken. this keeps the binary twin's captured tokens
// case-faithful regardless of how a given read is cased, which is what lets the
// headless/standalone bake tokenize correctly without knowing the grammar.
(void)ToLower;
m_lastquoted = false;
std::string token;
token.reserve(64);
@@ -372,9 +447,9 @@ std::string cParser::readTokenFromStream(bool ToLower, const char *Break)
char c = 0;
while (token.empty() && mStream->peek() != EOF) {
while (mStream->peek() != EOF) { // idk why but with mStream->get(c) not all cars are loaded
c = static_cast<char>(mStream->get());
while (token.empty() && m_bufferpos < m_buffer.size()) {
while (m_bufferpos < m_buffer.size()) {
c = m_buffer[m_bufferpos++];
if (c == '\n') {
++mLine;
}
@@ -387,10 +462,10 @@ std::string cParser::readTokenFromStream(bool ToLower, const char *Break)
continue;
}
if (ToLower) c = toLowerChar(c);
token.push_back(c);
if (findQuotes(token)) {
m_lastquoted = true; // came from a quoted span; never lower-cased
continue; // glue quoted content
}
if (skipComments && trimComments(token)) {
@@ -411,7 +486,7 @@ void cParser::stripFirstTokenBOM(std::string& token, bool ToLower, const char* B
}
// if first "token" was standalone BOM, read the next real token (avoid recursion)
while (token.empty() && mStream->peek() != EOF) {
while (token.empty() && m_bufferpos < m_buffer.size()) {
readToken(token, ToLower, Break);
// readToken will not re-enter BOM stripping because mFirstToken is now false
break;
@@ -487,6 +562,20 @@ void cParser::processInclude(cParser& srcParser, bool ToLower) {
m_writer->add_include(fileexpr, params);
}
if (m_bakeonly) {
// standalone/parallel bake: don't open the child here. record every candidate
// filename (a random set may list several) so the baker can compile each twin
// independently on its own worker thread.
for (auto const &token : fileexpr) {
if ((token != "[") && (token != "]") && (false == token.empty())) {
std::string candidate = token;
replace_slashes(candidate);
m_bakeincludes.emplace_back(std::move(candidate));
}
}
return;
}
// open the include for the live load with a freshly evaluated filename
replace_slashes(pick);
startIncludeDirect(std::move(pick), std::move(params));
@@ -520,11 +609,13 @@ void cParser::startIncludeDirect(std::string includefile, std::vector<std::strin
}
bool cParser::handleIncludeIfPresent(std::string& token, bool ToLower, const char* Break) {
// token-mode include: token == "include"
// token-mode include: token == "include". NOTE: we only process the directive here
// and report it; readToken loops to fetch the next token. fetching it here (the old
// behaviour) recursed once per consecutive include, overflowing the stack on files
// with long runs of includes (e.g. signaling .scm) -- especially in bake-only mode
// where the child isn't opened to break the chain.
if (expandIncludes && token == "include") {
processInclude(*this, ToLower);
// after processing include, return next token from current parser
readToken(token, ToLower, Break);
return true;
}
@@ -532,7 +623,6 @@ bool cParser::handleIncludeIfPresent(std::string& token, bool ToLower, const cha
if ((std::strcmp(Break, "\n\r") == 0) && token.compare(0, 7, "include") == 0) {
cParser includeparser(token.substr(7));
processInclude(includeparser, ToLower);
readToken(token, ToLower, Break);
return true;
}
@@ -548,54 +638,79 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break)
return;
}
bool fromOwn;
if (mIncludeParser)
// include directives are handled iteratively: a run of consecutive includes loops
// here instead of recursing through readToken (which overflowed the stack on files
// with hundreds of back-to-back include lines).
for (;;)
{
mIncludeParser->readToken(out, ToLower, Break);
if (out.empty())
bool fromOwn;
bool quoted = false;
if (mIncludeParser)
{
mIncludeParser->readToken(out, ToLower, Break);
if (out.empty())
{
mIncludeParser = nullptr;
out = readTokenFromStream(ToLower, Break);
quoted = m_lastquoted;
fromOwn = true;
}
else
{
fromOwn = false;
}
}
else
{
mIncludeParser = nullptr;
out = readTokenFromStream(ToLower, Break);
quoted = m_lastquoted;
fromOwn = true;
}
else
stripFirstTokenBOM(out, ToLower, Break);
// snapshot the original-case token (BOM stripped, before substitution) for the
// twin, then produce the consumer copy: only unquoted tokens are lower-cased
// (quoted spans keep their case, matching the legacy tokenizer)
std::string rawtoken;
bool rawquoted = false;
if (fromOwn)
{
fromOwn = false;
rawtoken = out;
rawquoted = quoted;
if (ToLower && (false == quoted))
{
for (auto &ch : out) { ch = toLowerChar(ch); }
}
}
}
else
{
out = readTokenFromStream(ToLower, Break);
fromOwn = true;
}
stripFirstTokenBOM(out, ToLower, Break);
substituteParameters(out, ToLower);
// snapshot the raw token (BOM stripped, parameters NOT yet substituted) for the
// binary twin, so parameterised includes can be re-substituted at replay time
std::string const rawtoken = out;
substituteParameters(out, ToLower);
bool const wasInclude = handleIncludeIfPresent(out, ToLower, Break);
// capture this file's own content into its twin. include directive tokens
// (the keyword/filename/parameters) are excluded via wasInclude/suppression,
// and child-include tokens (fromOwn == false) belong to the child's own twin.
if (m_compiling && m_writer && fromOwn && (false == wasInclude)
&& (false == m_capturesuppress) && (false == rawtoken.empty()))
{
// store numeric tokens as typed doubles (genuinely binary), everything else
// (names, paths, keywords, "(pN)" placeholders) as strings
double value = 0.0;
if (sniffNumber(rawtoken, value))
if (handleIncludeIfPresent(out, ToLower, Break))
{
m_writer->add_number(value);
// the include directive was consumed; fetch the next token
continue;
}
else
// capture this file's own content into its twin. include directive tokens are
// excluded (handled above); child-include tokens (fromOwn == false) belong to the
// child's own twin.
if (m_compiling && m_writer && fromOwn
&& (false == m_capturesuppress) && (false == rawtoken.empty()))
{
m_writer->add_token(rawtoken);
// numeric tokens -> typed values; quoted strings preserve case at replay;
// other strings (names, paths, keywords, "(pN)") may be lower-cased per consumer
double value = 0.0;
if ((false == rawquoted) && sniffNumber(rawtoken, value))
{
m_writer->add_number(value);
}
else
{
m_writer->add_token(rawtoken, rawquoted);
}
}
return;
}
}
@@ -612,13 +727,22 @@ void cParser::readReplayToken(std::string &out, bool ToLower, const char *Break)
mIncludeParser = nullptr;
}
while (m_entryindex < m_entries.size())
scene::scenery_entry_view entry;
while (m_reader && m_reader->next(entry))
{
scene::scenery_entry const &entry = m_entries[m_entryindex++];
if (entry.type == scene::scenery_entry_type::token)
{
out = entry.text;
// re-apply this file's include parameters, mirroring the text path
// stored in original case: lower-case per the consumer (unquoted tokens only),
// then re-apply this file's include parameters, mirroring the text path
out.assign(entry.text);
if (ToLower) { for (auto &ch : out) { ch = toLowerChar(ch); } }
substituteParameters(out, ToLower);
return;
}
if (entry.type == scene::scenery_entry_type::qtoken)
{
// quoted token: case preserved verbatim
out.assign(entry.text);
substituteParameters(out, ToLower);
return;
}
@@ -632,10 +756,17 @@ void cParser::readReplayToken(std::string &out, bool ToLower, const char *Break)
// include entry: re-evaluate the filename expression (re-randomizing any
// random set), then enter the child (its own twin or text) and serve its
// tokens; an empty/skipped child just advances to the next entry
std::vector<std::string> fileexpr;
fileexpr.reserve(entry.fileexpr.size());
for (auto const sv : entry.fileexpr) { fileexpr.emplace_back(sv); }
std::vector<std::string> params;
params.reserve(entry.params.size());
for (auto const sv : entry.params) { params.emplace_back(sv); }
std::size_t pos = 0;
std::string includefile = resolve_random_set(entry.fileexpr, pos);
std::string includefile = resolve_random_set(fileexpr, pos);
replace_slashes(includefile);
startIncludeDirect(std::move(includefile), entry.params);
startIncludeDirect(std::move(includefile), std::move(params));
if (mIncludeParser)
{
mIncludeParser->readToken(out, ToLower, Break);
@@ -647,6 +778,7 @@ void cParser::readReplayToken(std::string &out, bool ToLower, const char *Break)
}
}
m_replayexhausted = true;
out.clear();
}
@@ -665,12 +797,13 @@ std::vector<std::string> cParser::readParameters(cParser &Input)
}
std::string cParser::readQuotes(char const Quote)
{ // read the stream until specified char or stream end
{ // read the buffer until specified char or end
std::string token;
char c{0};
bool escaped = false;
while (mStream->get(c))
while (m_bufferpos < m_buffer.size())
{ // get all chars until the quote mark
c = m_buffer[m_bufferpos++];
if (escaped)
{
escaped = false;
@@ -699,8 +832,9 @@ void cParser::skipComment(std::string const &Endmark)
std::string input;
char c{0};
auto const endmarksize = Endmark.size();
while (mStream->get(c))
while (m_bufferpos < m_buffer.size())
{
c = m_buffer[m_bufferpos++];
if (c == '\n')
{
// update line counter
@@ -767,15 +901,13 @@ int cParser::getProgress() const
{
if (m_replay)
{
return ( m_entries.empty()
? 100
: static_cast<int>(m_entryindex * 100 / m_entries.size()) );
return ( m_reader ? m_reader->progress() : 100 );
}
if (mSize <= 0)
if (m_buffer.empty())
{
return 100;
}
return static_cast<int>(mStream->rdbuf()->pubseekoff(0, std::ios_base::cur) * 100 / mSize);
return static_cast<int>(m_bufferpos * 100 / m_buffer.size());
}
int cParser::getFullProgress() const
@@ -847,3 +979,76 @@ int cParser::LineMain() const
{
return mIncludeParser ? -1 : mLine;
}
namespace scene
{
// Headless bake: drains the scenario through cParser so the existing include machinery
// tokenizes and (re)compiles every reachable file's binary twin, with no deserializer,
// scene, renderer or window involved. Because tokens are captured in original case
// (v5 format), draining with a uniform case is correct.
bool scenerybinary_bake_headless(std::string const &Scenariofile, std::string const &Path, bool Loadtraction)
{
// parallel bake: discover the file graph by tokenizing each file in isolation (no
// scene state, no includes opened -- just their references collected) and compile
// each twin on a worker thread. each file is baked exactly once (deduped), so twin
// writes never race.
std::mutex mutex;
std::condition_variable cv;
std::unordered_set<std::string> visited;
std::deque<std::string> queue;
std::size_t active = 0;
std::atomic<std::size_t> baked { 0 };
auto const enqueue = [ & ]( std::string const &File ) {
// only scenery component files get twins; dedupe so shared includes bake once
if ( ( false == isSceneryFile( File ) ) || File.empty() || ( File[ 0 ] == '$' ) ) { return; }
if ( visited.insert( File ).second ) { queue.push_back( File ); }
};
{ std::lock_guard<std::mutex> lock( mutex ); enqueue( Scenariofile ); }
unsigned const workercount = std::max( 2u, std::min( 16u, std::thread::hardware_concurrency() ) );
auto const worker = [ & ] {
for ( ;; )
{
std::string file;
{
std::unique_lock<std::mutex> lock( mutex );
cv.wait( lock, [ & ] { return ( false == queue.empty() ) || ( active == 0 ); } );
if ( queue.empty() )
{
if ( active == 0 ) { cv.notify_all(); return; }
continue;
}
file = std::move( queue.front() );
queue.pop_front();
++active;
}
std::vector<std::string> includes;
{
cParser parser( file, cParser::buffer_FILE, Path, Loadtraction, std::vector<std::string>(), false, /*BakeOnly*/ true );
if ( parser.ok() )
{
includes = parser.bakeFile();
++baked;
}
}
{
std::unique_lock<std::mutex> lock( mutex );
for ( auto const &inc : includes ) { enqueue( inc ); }
--active;
cv.notify_all();
}
}
};
std::vector<std::thread> workers;
workers.reserve( workercount );
for ( unsigned i = 0; i < workercount; ++i ) { workers.emplace_back( worker ); }
for ( auto &thread : workers ) { thread.join(); }
WriteLog( "Bake: compiled " + std::to_string( baked.load() ) + " binary scenery twins from \"" + Scenariofile + "\"" );
return ( baked.load() > 0 );
}
} // namespace scene

29
utilities/parser.h
View File

@@ -22,8 +22,8 @@ http://mozilla.org/MPL/2.0/.
// before cParser is defined.
namespace scene {
enum class scenery_file_kind : std::uint8_t;
struct scenery_entry;
class scenery_binary_writer;
class scenery_binary_reader;
}
/////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -39,9 +39,13 @@ class cParser //: public std::stringstream
buffer_TEXT
};
// constructors:
cParser(std::string const &Stream, buffertype const Type = buffer_TEXT, std::string Path = "", bool const Loadtraction = true, std::vector<std::string> Parameters = std::vector<std::string>(), bool allowRandom = false );
cParser(std::string const &Stream, buffertype const Type = buffer_TEXT, std::string Path = "", bool const Loadtraction = true, std::vector<std::string> Parameters = std::vector<std::string>(), bool allowRandom = false, bool BakeOnly = false );
// destructor:
virtual ~cParser();
// bake-only: tokenize this file into its binary twin without opening includes or
// touching scene state, and return the (relative) filenames it includes so a parallel
// baker can compile each of those twins on its own thread. requires BakeOnly ctor.
std::vector<std::string> bakeFile();
// methods:
template <typename Type_>
cParser &
@@ -70,9 +74,11 @@ class cParser //: public std::stringstream
// in replay mode there is no character stream; exhaustion is reaching the
// end of the twin's entries with no include still being drained
if( m_replay ) {
return ( m_entryindex >= m_entrycount ) && ( !mIncludeParser );
return m_replayexhausted && ( !mIncludeParser );
}
return mStream->eof(); };
// text is tokenized from an in-memory buffer; eof is the read cursor
// reaching its end
return ( m_bufferpos >= m_buffer.size() ); };
inline
bool
ok() {
@@ -145,7 +151,9 @@ class cParser //: public std::stringstream
// members:
bool m_autoclear { true }; // unretrieved tokens are discarded when another read command is issued (legacy behaviour)
bool LoadTraction { true }; // load traction?
std::shared_ptr<std::istream> mStream; // relevant kind of buffer is attached on creation.
std::shared_ptr<std::istream> mStream; // attached on creation; kept for open/fail state.
std::string m_buffer; // whole source text read into memory; tokenized from here
std::size_t m_bufferpos { 0 }; // read cursor into m_buffer
std::string mFile; // name of the open file, if any
std::string mPath; // path to open stream, for relative path lookups.
std::streamoff mSize { 0 }; // size of open stream, for progress report.
@@ -160,11 +168,13 @@ class cParser //: public std::stringstream
std::vector<std::string> parameters; // parameter list for included file.
std::deque<std::string> tokens;
// --- binary scenery twin support ---
// last token produced by readTokenFromStream was (partly) quoted -> case preserved
bool m_lastquoted { false };
// replay: this file is served from its binary twin instead of text
bool m_replay { false };
std::vector<scene::scenery_entry> m_entries; // entries loaded from the twin (replay)
std::size_t m_entryindex { 0 }; // read cursor into m_entries
std::size_t m_entrycount { 0 }; // m_entries.size(), cached for inline eof()
bool m_replayexhausted { false }; // all twin entries consumed (for inline eof())
std::string m_twinbuf; // whole twin file held in memory; the reader views into it
std::unique_ptr<scene::scenery_binary_reader> m_reader; // streams entries from m_twinbuf
// compile: this file is being captured into a binary twin alongside the text read
bool m_compiling { false };
std::unique_ptr<scene::scenery_binary_writer> m_writer;
@@ -172,6 +182,9 @@ class cParser //: public std::stringstream
scene::scenery_file_kind m_binarykind {}; // value-initialised (== scenery_file_kind::scn)
bool m_capturesuppress { false }; // suppress token capture (include directive internals)
bool m_twinwritten { false }; // guards against writing the twin more than once
// standalone/parallel bake: capture this file only, collect its include filenames
bool m_bakeonly { false };
std::vector<std::string> m_bakeincludes;
};