diff --git a/EU07.cpp b/EU07.cpp index e1be8584..3c6a77d7 100644 --- a/EU07.cpp +++ b/EU07.cpp @@ -20,6 +20,7 @@ Stele, firleju, szociu, hunter, ZiomalCl, OLI_EU and others #include "application/application.h" #include "utilities/Logs.h" +#include "scene/scenerybinary.h" #include #ifdef WITHDUMPGEN #ifdef _WIN32 @@ -98,6 +99,12 @@ int main(int argc, char *argv[]) int dynamic = std::stoi(std::string(argv[5])); export_e3d_standalone(in, out, flags, dynamic); } + // headless bake: precompile a scenery (and all its includes) into binary twins with + // no window, renderer or simulation. usage: eu07 -bake + else if (argc >= 3 && std::string(argv[1]) == "-bake") + { + scene::scenerybinary_bake_headless(argv[2], Global.asCurrentSceneryPath, Global.bLoadTraction); + } else { try diff --git a/scene/scenerybinary.cpp b/scene/scenerybinary.cpp index 66209fb4..dc7027ff 100644 --- a/scene/scenerybinary.cpp +++ b/scene/scenerybinary.cpp @@ -21,6 +21,9 @@ http://mozilla.org/MPL/2.0/. #include #include #include +#include +#include +#include namespace scene { @@ -37,173 +40,229 @@ void write_varint( std::ostream &Output, std::uint64_t Value ) { } while( Value != 0 ); } -std::uint64_t read_varint( std::istream &Input ) { +// zig-zag maps small-magnitude signed integers to small unsigned varints +std::uint64_t zigzag_encode( std::int64_t Value ) { + return ( static_cast( Value ) << 1 ) ^ static_cast( Value >> 63 ); +} +std::int64_t zigzag_decode( std::uint64_t Value ) { + return static_cast( ( Value >> 1 ) ^ ( ~( Value & 1 ) + 1 ) ); +} + +// buffer cursor reads (little-endian, matching sn_utils), bounds-checked: on overrun the +// cursor is parked at the end and zero is returned, so a truncated twin decodes to empty +// rather than reading out of bounds. +std::uint64_t read_varint( char const *&Cursor, char const *End ) { std::uint64_t value = 0; int shift = 0; - while( shift < 64 ) { - int const c = Input.get(); - if( c == std::char_traits::eof() ) { break; } - value |= ( static_cast( c & 0x7F ) << shift ); - if( ( c & 0x80 ) == 0 ) { break; } + while( ( Cursor < End ) && ( shift < 64 ) ) { + std::uint8_t const byte = static_cast( *Cursor++ ); + value |= ( static_cast( byte & 0x7F ) << shift ); + if( ( byte & 0x80 ) == 0 ) { break; } shift += 7; } return value; } +std::uint32_t read_u32le( char const *&Cursor, char const *End ) { + if( End - Cursor < 4 ) { Cursor = End; return 0; } + auto const *b = reinterpret_cast( Cursor ); + Cursor += 4; + return ( std::uint32_t( b[ 0 ] ) ) | ( std::uint32_t( b[ 1 ] ) << 8 ) + | ( std::uint32_t( b[ 2 ] ) << 16 ) | ( std::uint32_t( b[ 3 ] ) << 24 ); +} +float read_f32le( char const *&Cursor, char const *End ) { + if( End - Cursor < 4 ) { Cursor = End; return 0.f; } + std::uint32_t const v = read_u32le( Cursor, End ); + float f; std::memcpy( &f, &v, 4 ); return f; +} +double read_f64le( char const *&Cursor, char const *End ) { + if( End - Cursor < 8 ) { Cursor = End; return 0.0; } + auto const *b = reinterpret_cast( Cursor ); + Cursor += 8; + std::uint64_t v = 0; + for( int i = 0; i < 8; ++i ) { v |= ( static_cast( b[ i ] ) << ( 8 * i ) ); } + double d; std::memcpy( &d, &v, 8 ); return d; +} -// builds the per-file string table while interning, returning each string's index -class string_interner { -public: - std::uint32_t intern( std::string const &Text ) { - auto const it = m_lookup.find( Text ); - if( it != m_lookup.end() ) { return it->second; } - auto const index = static_cast( m_table.size() ); - m_lookup.emplace( Text, index ); - m_table.emplace_back( Text ); - return index; - } - std::vector const &table() const { return m_table; } -private: - std::unordered_map m_lookup; - std::vector m_table; +// on-disk entry tag, packed into the low 3 bits of the per-entry head varint +enum : std::uint64_t { + TAG_TOKEN = 0, // head >> 3 == interned string index + TAG_INCLUDE = 1, + TAG_INT = 2, // followed by a zig-zag varint + TAG_F32 = 3, // followed by 4 little-endian bytes + TAG_F64 = 4, // followed by 8 little-endian bytes + TAG_QTOKEN = 5, // quoted token; head >> 3 == interned string index + TAG_BITS = 3, + TAG_MASK = 0x7, }; +// writes a numeric value in the most compact lossless-enough form: integral values as a +// zig-zag varint, otherwise f32 when it represents the value with negligible error, +// otherwise full f64. +void write_number( std::ostream &Output, double Value ) { + double integral = 0.0; + if( ( std::modf( Value, &integral ) == 0.0 ) + && ( Value >= -9.007199254740992e15 ) && ( Value <= 9.007199254740992e15 ) ) { + write_varint( Output, TAG_INT ); + write_varint( Output, zigzag_encode( static_cast( Value ) ) ); + return; + } + float const f = static_cast( Value ); + bool const f32ok = std::isfinite( f ) + && ( ( static_cast( f ) == Value ) + || ( std::abs( static_cast( f ) - Value ) <= 1e-6 * std::abs( Value ) ) ); + if( f32ok ) { + write_varint( Output, TAG_F32 ); + sn_utils::ls_float32( Output, f ); + } + else { + write_varint( Output, TAG_F64 ); + sn_utils::ls_float64( Output, Value ); + } +} + } // anonymous namespace +std::uint32_t +scenery_binary_writer::intern( std::string const &Text ) { + auto const it = m_lookup.find( Text ); + if( it != m_lookup.end() ) { return it->second; } + auto const index = static_cast( m_table.size() ); + m_lookup.emplace( Text, index ); + m_table.emplace_back( Text ); + return index; +} + void -scenery_binary_writer::add_token( std::string Token ) { - scenery_entry entry; - entry.type = scenery_entry_type::token; - entry.text = std::move( Token ); - m_entries.emplace_back( std::move( entry ) ); +scenery_binary_writer::add_token( std::string const &Token, bool Quoted ) { + auto const tag = ( Quoted ? TAG_QTOKEN : TAG_TOKEN ); + write_varint( m_entries, ( static_cast( intern( Token ) ) << TAG_BITS ) | tag ); + ++m_count; } void scenery_binary_writer::add_number( double Value ) { - scenery_entry entry; - entry.type = scenery_entry_type::number; - entry.number = Value; - m_entries.emplace_back( std::move( entry ) ); + write_number( m_entries, Value ); + ++m_count; } void -scenery_binary_writer::add_include( std::vector Fileexpr, std::vector Params ) { - scenery_entry entry; - entry.type = scenery_entry_type::include; - entry.fileexpr = std::move( Fileexpr ); - entry.params = std::move( Params ); - m_entries.emplace_back( std::move( entry ) ); +scenery_binary_writer::add_include( std::vector const &Fileexpr, std::vector const &Params ) { + write_varint( m_entries, TAG_INCLUDE ); + write_varint( m_entries, Fileexpr.size() ); + for( auto const &token : Fileexpr ) { write_varint( m_entries, intern( token ) ); } + write_varint( m_entries, Params.size() ); + for( auto const ¶m : Params ) { write_varint( m_entries, intern( param ) ); } + ++m_count; } bool scenery_binary_writer::write( std::ostream &Output, scenery_file_kind Kind ) const { - // first pass: intern every string the entries reference so keywords/paths that - // repeat are stored once and referenced by index - string_interner interner; - std::vector tokenindex; // index per token entry, in order - std::vector> fileexpridx, paramidx; // per include entry - for( auto const &entry : m_entries ) { - if( entry.type == scenery_entry_type::token ) { - tokenindex.emplace_back( interner.intern( entry.text ) ); - } - else if( entry.type == scenery_entry_type::include ) { - std::vector fe, pe; - for( auto const &token : entry.fileexpr ) { fe.emplace_back( interner.intern( token ) ); } - for( auto const ¶m : entry.params ) { pe.emplace_back( interner.intern( param ) ); } - fileexpridx.emplace_back( std::move( fe ) ); - paramidx.emplace_back( std::move( pe ) ); - } - } - // header: magic + kind + flags sn_utils::ls_uint32( Output, SCENERYBINARY_MAGIC ); sn_utils::s_uint8( Output, static_cast( Kind ) ); sn_utils::ls_uint32( Output, 0 ); // flags, reserved - // string table - auto const &table = interner.table(); - sn_utils::ls_uint32( Output, static_cast( table.size() ) ); - for( auto const &text : table ) { - sn_utils::s_str( Output, text ); + // string table: count, then each string as varint(length) + raw bytes (so the + // reader can take views into the buffer without scanning for terminators) + write_varint( Output, m_table.size() ); + for( auto const &text : m_table ) { + write_varint( Output, text.size() ); + Output.write( text.data(), static_cast( text.size() ) ); } - // entries - sn_utils::ls_uint32( Output, static_cast( m_entries.size() ) ); - std::size_t tokencursor = 0, includecursor = 0; - for( auto const &entry : m_entries ) { - sn_utils::s_uint8( Output, static_cast( entry.type ) ); - if( entry.type == scenery_entry_type::include ) { - auto const &fe = fileexpridx[ includecursor ]; - auto const &pe = paramidx[ includecursor ]; - ++includecursor; - write_varint( Output, fe.size() ); - for( auto idx : fe ) { write_varint( Output, idx ); } - write_varint( Output, pe.size() ); - for( auto idx : pe ) { write_varint( Output, idx ); } - } - else if( entry.type == scenery_entry_type::number ) { - sn_utils::ls_float64( Output, entry.number ); - } - else { - write_varint( Output, tokenindex[ tokencursor++ ] ); - } - } + // entries: count, then the pre-encoded entry bytes (heads + payloads) verbatim + write_varint( Output, m_count ); + auto const encoded = m_entries.str(); + Output.write( encoded.data(), static_cast( encoded.size() ) ); return ( false == Output.fail() ); } bool -scenery_binary_reader::open( std::istream &Input ) { +scenery_binary_reader::open( std::string_view Buffer ) { - m_entries.clear(); + m_table.clear(); + m_cursor = m_end = nullptr; + m_remaining = m_total = 0; - auto const magic { sn_utils::ld_uint32( Input ) }; - if( magic != SCENERYBINARY_MAGIC ) { + char const *cursor = Buffer.data(); + char const *const end = Buffer.data() + Buffer.size(); + + if( end - cursor < 9 ) { return false; } // magic(4) + kind(1) + flags(4) + if( read_u32le( cursor, end ) != SCENERYBINARY_MAGIC ) { // unrecognized type or incompatible version return false; } - m_kind = static_cast( sn_utils::d_uint8( Input ) ); - sn_utils::ld_uint32( Input ); // flags, reserved + m_kind = static_cast( static_cast( *cursor++ ) ); + read_u32le( cursor, end ); // flags, reserved - // string table - std::vector table; - auto tablesize { sn_utils::ld_uint32( Input ) }; - table.reserve( tablesize ); - while( tablesize-- ) { - table.emplace_back( sn_utils::d_str( Input ) ); + // string table: views into the buffer (no copies) + auto tablesize = read_varint( cursor, end ); + m_table.reserve( tablesize ); + while( ( tablesize-- != 0 ) && ( cursor < end ) ) { + auto const len = read_varint( cursor, end ); + if( static_cast( end - cursor ) < len ) { cursor = end; break; } + m_table.emplace_back( cursor, static_cast( len ) ); + cursor += len; } - // resolves an interned index back to its string, guarding against corruption - auto const resolve = [ & ]( std::uint64_t index ) -> std::string { - return ( index < table.size() ) ? table[ static_cast( index ) ] : std::string(); + + m_total = m_remaining = static_cast( read_varint( cursor, end ) ); + m_cursor = cursor; + m_end = end; + return true; +} + +bool +scenery_binary_reader::next( scenery_entry_view &Out ) { + + if( ( m_remaining == 0 ) || ( m_cursor >= m_end ) ) { return false; } + --m_remaining; + + Out.fileexpr.clear(); + Out.params.clear(); + + auto const head = read_varint( m_cursor, m_end ); + auto const tag = head & TAG_MASK; + + auto const resolve = [ this ]( std::uint64_t index ) -> std::string_view { + return ( index < m_table.size() ) ? m_table[ static_cast( index ) ] : std::string_view(); }; - auto count { sn_utils::ld_uint32( Input ) }; - m_entries.reserve( count ); - while( count-- ) { - scenery_entry entry; - entry.type = static_cast( sn_utils::d_uint8( Input ) ); - if( entry.type == scenery_entry_type::include ) { - auto expr { read_varint( Input ) }; - entry.fileexpr.reserve( expr ); - while( expr-- ) { - entry.fileexpr.emplace_back( resolve( read_varint( Input ) ) ); - } - auto params { read_varint( Input ) }; - entry.params.reserve( params ); - while( params-- ) { - entry.params.emplace_back( resolve( read_varint( Input ) ) ); - } + switch( tag ) { + case TAG_INCLUDE: { + Out.type = scenery_entry_type::include; + auto fe = read_varint( m_cursor, m_end ); + Out.fileexpr.reserve( fe ); + while( ( fe-- != 0 ) && ( m_cursor < m_end ) ) { Out.fileexpr.emplace_back( resolve( read_varint( m_cursor, m_end ) ) ); } + auto pe = read_varint( m_cursor, m_end ); + Out.params.reserve( pe ); + while( ( pe-- != 0 ) && ( m_cursor < m_end ) ) { Out.params.emplace_back( resolve( read_varint( m_cursor, m_end ) ) ); } + break; } - else if( entry.type == scenery_entry_type::number ) { - entry.number = sn_utils::ld_float64( Input ); - } - else { - entry.text = resolve( read_varint( Input ) ); - } - m_entries.emplace_back( std::move( entry ) ); + case TAG_INT: + Out.type = scenery_entry_type::number; + Out.number = static_cast( zigzag_decode( read_varint( m_cursor, m_end ) ) ); + break; + case TAG_F32: + Out.type = scenery_entry_type::number; + Out.number = static_cast( read_f32le( m_cursor, m_end ) ); + break; + case TAG_F64: + Out.type = scenery_entry_type::number; + Out.number = read_f64le( m_cursor, m_end ); + break; + case TAG_QTOKEN: + Out.type = scenery_entry_type::qtoken; + Out.text = resolve( head >> TAG_BITS ); + break; + case TAG_TOKEN: + default: + Out.type = scenery_entry_type::token; + Out.text = resolve( head >> TAG_BITS ); + break; } - - return ( false == Input.fail() ); + return true; } namespace { diff --git a/scene/scenerybinary.h b/scene/scenerybinary.h index dd81c3a3..33b1435e 100644 --- a/scene/scenerybinary.h +++ b/scene/scenerybinary.h @@ -11,46 +11,50 @@ http://mozilla.org/MPL/2.0/. #include #include +#include #include #include #include +#include +#include #include "utilities/utilities.h" // MAKE_ID4 -// Binary, modular scenery container ("eu7" format, version 1). +// Binary, modular scenery container ("eu7" format). // -// Replaces the legacy text formats (.scn / .scm / .inc) and the old terrain-only -// .sbt blob. The mapping is one-to-one and modular: every text file compiles to -// exactly one binary twin and includes are preserved as references, so the binary -// twin of one file points to the binary twins of the files it includes. -// route.scn -> route.scnb -// piece.inc -> piece.incb -// mesh.scm -> mesh.scmb +// Replaces the legacy text formats (.scn / .scm / .inc / .ctr) and the old terrain-only +// .sbt blob. The mapping is one-to-one and modular: every text file compiles to exactly +// one binary twin and includes are preserved as references, so the binary twin of one +// file points to the binary twins of the files it includes. +// route.scn -> route.scnb piece.inc -> piece.incb +// mesh.scm -> mesh.scmb events.ctr -> events.ctrb // -// A binary twin stores the source file's own, fully resolved content (comments -// stripped, parameters and random sets resolved) as an ordered list of entries. -// Numeric tokens are stored as 8-byte IEEE doubles (so coordinates, ranges etc. are -// genuinely binary, not ASCII). All string tokens (keywords like "node"/"endmodel", -// names, paths) are interned into a per-file string table and referenced from the -// entries by a small varint index, so a keyword that occurs thousands of times is -// stored once. An "include" entry marks the spot where the source file pulled in -// another file, carrying the (interned) include filename expression and parameters. +// A twin stores the source file's own, fully resolved content (comments stripped, +// parameters/random sets resolved) as an ordered list of entries: +// * string tokens (keywords like "node"/"endmodel", names, paths) are interned into a +// per-file table and referenced by a varint index, so a keyword that occurs +// thousands of times is stored once; +// * numeric tokens are stored typed -- integral values as a zig-zag varint (1-2 bytes +// for the many small ints), fractional values as an 8-byte IEEE double; +// * an "include" entry carries the (interned) filename expression and parameters. +// Each entry's type is packed into the low bits of a single varint together with the +// token index, so the common case (a token) costs just that one varint. // -// The binary is consumed transparently at the cParser file layer: opening a file -// whose binary twin exists serves tokens from the twin instead of tokenizing text, -// and an include entry triggers the same include machinery as the text path (which -// recursively prefers the included file's own binary twin). The scenery -// deserializer is unaware of the format and needs no changes; the file's identity -// (cParser::Name()), node grouping per .inc and parameter substitution are all -// preserved exactly as in the text path. +// Reading is buffer-based and streaming: the whole twin is read into memory once and +// parsed by pointer (no per-byte stream calls), and entries are decoded on demand +// (no intermediate materialisation), with string tokens served as views into the +// buffer. The binary is consumed transparently at the cParser file layer; the scenery +// deserializer is unaware of the format. cParser::Name(), per-.inc node grouping and +// parameter substitution are preserved exactly as in the text path. namespace scene { // "eu7" + format version, little-endian via MAKE_ID4 ('e','u','7',). -// v2 introduced typed numeric entries (f64); v3 interns string tokens into a per-file -// table referenced by varint indices. bumping the version invalidates older twins so -// they are recompiled rather than misread. -constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 3 ) }; +// v4: buffer-streamed reader, packed entry tags, zig-zag varint integers. +// v5: tokens stored in original case (lower-cased per consumer at replay) with a quoted +// flag, so baking is grammar-independent (enables the headless/standalone baker). +// bumping the version invalidates older twins so they are recompiled rather than misread. +constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 5 ) }; // file extension of the binary twin, derived from source kind std::string const SCENERYBINARY_EXT_SCN { ".scnb" }; @@ -65,72 +69,91 @@ enum class scenery_file_kind : std::uint8_t { scm = 2, }; -// kind of a stored entry +// logical kind of an entry as seen by the consumer (the on-disk integer/float split is +// an encoding detail hidden inside the reader/writer) enum class scenery_entry_type : std::uint8_t { - token = 0, // a non-numeric resolved token (name/path/keyword), stored as a string + token = 0, // a non-numeric token (name/path/keyword); lower-cased per consumer at replay include = 1, // an include directive: pulls in another file with parameters - number = 2, // a numeric token, stored as an 8-byte IEEE double + number = 2, // a numeric token + qtoken = 3, // a quoted token; case preserved verbatim at replay }; -// a single ordered element of a file's resolved content -struct scenery_entry { +// a decoded entry handed to the consumer during streaming reads; string fields are +// views into the reader's buffer and are valid until the next read / reader destruction +struct scenery_entry_view { scenery_entry_type type { scenery_entry_type::token }; - // token value (token entries only) - std::string text; - // numeric value (number entries only) - double number { 0.0 }; - // include only: the raw filename expression as written in the source, i.e. either - // a single filename token or a random set "[ a b c ]" (including the brackets). - // kept verbatim so the random choice is re-evaluated on every replay rather than - // frozen at compile time. - std::vector fileexpr; - // include only: directive parameters - std::vector params; + std::string_view text; // token + double number { 0.0 }; // number + std::vector fileexpr; // include + std::vector params; // include }; -// accumulates a file's entries and writes them, with a header, to a stream +// accumulates a file's content and serializes it. entries are encoded incrementally into +// a compact byte buffer as they are added (strings interned on the fly), so even a huge +// source file costs only its interned table plus a few bytes per token -- not a heavy +// struct per token -- keeping parallel baking within memory. class scenery_binary_writer { public: - void add_token( std::string Token ); - // stores a numeric token as a typed double + // Quoted marks the token as a quoted span (its case is preserved verbatim at replay) + void add_token( std::string const &Token, bool Quoted = false ); void add_number( double Value ); // Fileexpr is the verbatim filename expression (single token or random set) - void add_include( std::vector Fileexpr, std::vector Params ); - std::size_t entry_count() const { return m_entries.size(); } - // serializes header + entries to the stream. returns false on stream failure. + void add_include( std::vector const &Fileexpr, std::vector const &Params ); + std::size_t entry_count() const { return m_count; } + // serializes header + string table + encoded entries. returns false on stream failure. bool write( std::ostream &Output, scenery_file_kind Kind ) const; private: - std::vector m_entries; + std::uint32_t intern( std::string const &Text ); + + std::unordered_map m_lookup; // string -> table index + std::vector m_table; // interned strings, in order + std::ostringstream m_entries; // encoded entry bytes + std::size_t m_count { 0 }; // number of entries }; -// reads a binary scenery file fully into memory and exposes its entries in order +// parses a binary twin held in a memory buffer and streams its entries on demand. +// the buffer must outlive the reader (the cParser owns both). class scenery_binary_reader { public: - // validates magic/version and loads every entry. returns false if the stream is - // not a valid current-version binary scenery file. - bool open( std::istream &Input ); + // validates magic/version and indexes the string table; positions at the first + // entry. returns false if Buffer is not a valid current-version twin. + bool open( std::string_view Buffer ); scenery_file_kind kind() const { return m_kind; } - std::vector const &entries() const { return m_entries; } + // decodes the next entry into Out; returns false once all entries are consumed + bool next( scenery_entry_view &Out ); + bool exhausted() const { return m_remaining == 0; } + // fraction of entries consumed so far, 0..100, for the loading bar + int progress() const { return ( m_total == 0 ? 100 : static_cast( ( m_total - m_remaining ) * 100 / m_total ) ); } private: + std::vector m_table; + char const *m_cursor { nullptr }; + char const *m_end { nullptr }; + std::uint32_t m_remaining { 0 }; + std::uint32_t m_total { 0 }; scenery_file_kind m_kind { scenery_file_kind::scn }; - std::vector m_entries; }; // maps a source scenery filename to the extension of its binary twin std::string scenerybinary_extension_for( std::string const &Sourcefile ); -// Asynchronous baking: serializing and writing a twin is pure, self-contained work -// (it only needs the writer's collected entries), so it is offloaded to a small thread -// pool and overlaps with the rest of scene construction instead of blocking the load. -// Takes ownership of the writer. +// Asynchronous baking: serializing and writing a twin is self-contained work, so it is +// offloaded to a small thread pool and overlaps with scene construction. Takes ownership +// of the writer. void scenerybinary_write_async( std::unique_ptr Writer, std::string Path, scenery_file_kind Kind ); -// Blocks until every queued async write has finished, then emits their log lines on the -// calling (main) thread. Call once the scenario has finished loading. +// Blocks until every queued async write has finished, logging results on the calling +// (main) thread. Call once the scenario has finished loading. void scenerybinary_wait_all(); +// Headless bake: tokenizes the scenario file and, recursively, every file it includes, +// writing each one's binary twin -- with no scene construction, renderer or window. Used +// by the "-bake" command line mode to precompile a scenery offline. Fresh twins are left +// untouched; only missing/stale ones are (re)compiled. Returns false if the file can't be +// opened. +bool scenerybinary_bake_headless( std::string const &Scenariofile, std::string const &Path, bool Loadtraction ); + } // namespace scene diff --git a/utilities/parser.cpp b/utilities/parser.cpp index 96625518..ba9f0238 100644 --- a/utilities/parser.cpp +++ b/utilities/parser.cpp @@ -19,6 +19,12 @@ http://mozilla.org/MPL/2.0/. #include #include #include +#include +#include +#include +#include +#include +#include /* MaSzyna EU07 locomotive simulator parser @@ -128,8 +134,8 @@ inline bool twinIsFresh(const std::string &sourcefull, const std::string &twinfu } // namespace // constructors -cParser::cParser(std::string const &Stream, buffertype const Type, std::string Path, bool const Loadtraction, std::vector Parameters, bool allowRandom) - : allowRandomIncludes(allowRandom), LoadTraction(Loadtraction), mPath(Path) +cParser::cParser(std::string const &Stream, buffertype const Type, std::string Path, bool const Loadtraction, std::vector Parameters, bool allowRandom, bool BakeOnly) + : allowRandomIncludes(allowRandom), LoadTraction(Loadtraction), mPath(Path), m_bakeonly(BakeOnly) { // store to calculate sub-sequent includes from relative path if (Type == buffertype::buffer_FILE) @@ -141,6 +147,26 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P { case buffer_FILE: { + // bake-only: always compile this one file's twin from text, never touch scene + // groups and never replay; includes are recorded but not opened (the parallel + // baker compiles them separately) + if (m_bakeonly) + { + Path.append(Stream); + mStream = std::make_shared(Path, std::ios_base::binary); + if (false == mStream->fail()) + { + m_compiling = true; + std::string twinrel = Stream; + erase_extension(twinrel); + twinrel += scene::scenerybinary_extension_for(Stream); + m_binarytwinpath = mPath + twinrel; + m_binarykind = sceneryKind(Stream); + m_writer = std::make_unique(); + } + break; + } + // content of *.inc files is grouped together (same for text and binary replay) if (endsWithLower(Stream, ".inc")) { @@ -161,18 +187,28 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P std::string const sourcefull = mPath + Stream; std::ifstream probe(twinfull, std::ios_base::binary); - scene::scenery_binary_reader reader; - if (probe.good() && twinIsFresh(sourcefull, twinfull) && reader.open(probe)) + bool replaying = false; + if (probe.good() && twinIsFresh(sourcefull, twinfull)) { - // replay: serve tokens from the twin, no text tokenization at all - m_entries = reader.entries(); - m_entrycount = m_entries.size(); - m_replay = true; - m_entryindex = 0; - mStream = std::make_shared(std::string()); - opened = true; + // slurp the whole twin once; the reader parses it by pointer and serves + // entries on demand (string tokens as views into this buffer) + std::ostringstream slurp; + slurp << probe.rdbuf(); + m_twinbuf = std::move(slurp).str(); + m_reader = std::make_unique(); + if (m_reader->open(m_twinbuf)) + { + m_replay = true; + mStream = std::make_shared(std::string()); + replaying = true; + opened = true; + } + else + { + m_reader.reset(); + } } - else + if (false == replaying) { Path.append(Stream); mStream = std::make_shared(Path, std::ios_base::binary); @@ -205,17 +241,28 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P break; } } - // calculate stream size + // slurp the whole source into memory and tokenize from the buffer; reading char + // by char from the stream (get()/peek()) goes through the virtual streambuf on + // every character, which dominates parse time on large sceneries. if (mStream) { if (true == mStream->fail()) { - ErrorLog("Failed to open file \"" + Path + "\""); + // bake-only parsers run on worker threads; the logger is not thread-safe, so + // stay silent here (the driver checks ok() and simply skips a file it couldn't + // open). missing includes still surface during the normal text/replay load. + if (false == m_bakeonly) + { + ErrorLog("Failed to open file \"" + Path + "\""); + } } - else + else if (false == m_replay) { - mSize = mStream->rdbuf()->pubseekoff(0, std::ios_base::end); - mStream->rdbuf()->pubseekoff(0, std::ios_base::beg); + std::ostringstream slurp; + slurp << mStream->rdbuf(); + m_buffer = std::move(slurp).str(); + m_bufferpos = 0; + mSize = static_cast(m_buffer.size()); mLine = 1; } } @@ -239,6 +286,26 @@ cParser::~cParser() } } +std::vector cParser::bakeFile() +{ + // drain the file: every token is captured into the twin, include directives are + // recorded (their candidate filenames collected) but not opened + std::string token; + do { token = getToken(); } while (false == token.empty()); + + // write the twin synchronously -- the parallel baker already runs one file per + // worker thread, so there is no point handing this to the async pool + if (m_compiling && m_writer && (false == m_twinwritten) && (m_bufferpos >= m_buffer.size())) + { + m_twinwritten = true; + std::ofstream output(m_binarytwinpath, std::ios_base::binary); + // errors are intentionally not logged here (worker thread); a missing/short twin + // is simply recompiled on the next normal load + (void)(output.good() && m_writer->write(output, m_binarykind)); + } + return std::move(m_bakeincludes); +} + void cParser::flushBinaryTwin() { // write the twin only once, only when compiling, and only if the source text was @@ -247,8 +314,9 @@ void cParser::flushBinaryTwin() { return; } - if ((nullptr == mStream) || (false == mStream->eof())) + if (m_bufferpos < m_buffer.size()) { + // source not fully consumed (aborted parse): don't leave a truncated twin return; } m_twinwritten = true; @@ -365,6 +433,13 @@ bool cParser::getTokens(unsigned int Count, bool ToLower, const char *Break) std::string cParser::readTokenFromStream(bool ToLower, const char *Break) { + // the token is produced in its ORIGINAL case; lower-casing (when requested) is applied + // to the consumer copy in readToken. this keeps the binary twin's captured tokens + // case-faithful regardless of how a given read is cased, which is what lets the + // headless/standalone bake tokenize correctly without knowing the grammar. + (void)ToLower; + m_lastquoted = false; + std::string token; token.reserve(64); @@ -372,9 +447,9 @@ std::string cParser::readTokenFromStream(bool ToLower, const char *Break) char c = 0; - while (token.empty() && mStream->peek() != EOF) { - while (mStream->peek() != EOF) { // idk why but with mStream->get(c) not all cars are loaded - c = static_cast(mStream->get()); + while (token.empty() && m_bufferpos < m_buffer.size()) { + while (m_bufferpos < m_buffer.size()) { + c = m_buffer[m_bufferpos++]; if (c == '\n') { ++mLine; } @@ -387,10 +462,10 @@ std::string cParser::readTokenFromStream(bool ToLower, const char *Break) continue; } - if (ToLower) c = toLowerChar(c); token.push_back(c); if (findQuotes(token)) { + m_lastquoted = true; // came from a quoted span; never lower-cased continue; // glue quoted content } if (skipComments && trimComments(token)) { @@ -411,7 +486,7 @@ void cParser::stripFirstTokenBOM(std::string& token, bool ToLower, const char* B } // if first "token" was standalone BOM, read the next real token (avoid recursion) - while (token.empty() && mStream->peek() != EOF) { + while (token.empty() && m_bufferpos < m_buffer.size()) { readToken(token, ToLower, Break); // readToken will not re-enter BOM stripping because mFirstToken is now false break; @@ -487,6 +562,20 @@ void cParser::processInclude(cParser& srcParser, bool ToLower) { m_writer->add_include(fileexpr, params); } + if (m_bakeonly) { + // standalone/parallel bake: don't open the child here. record every candidate + // filename (a random set may list several) so the baker can compile each twin + // independently on its own worker thread. + for (auto const &token : fileexpr) { + if ((token != "[") && (token != "]") && (false == token.empty())) { + std::string candidate = token; + replace_slashes(candidate); + m_bakeincludes.emplace_back(std::move(candidate)); + } + } + return; + } + // open the include for the live load with a freshly evaluated filename replace_slashes(pick); startIncludeDirect(std::move(pick), std::move(params)); @@ -520,11 +609,13 @@ void cParser::startIncludeDirect(std::string includefile, std::vectorreadToken(out, ToLower, Break); - if (out.empty()) + bool fromOwn; + bool quoted = false; + if (mIncludeParser) + { + mIncludeParser->readToken(out, ToLower, Break); + if (out.empty()) + { + mIncludeParser = nullptr; + out = readTokenFromStream(ToLower, Break); + quoted = m_lastquoted; + fromOwn = true; + } + else + { + fromOwn = false; + } + } + else { - mIncludeParser = nullptr; out = readTokenFromStream(ToLower, Break); + quoted = m_lastquoted; fromOwn = true; } - else + + stripFirstTokenBOM(out, ToLower, Break); + + // snapshot the original-case token (BOM stripped, before substitution) for the + // twin, then produce the consumer copy: only unquoted tokens are lower-cased + // (quoted spans keep their case, matching the legacy tokenizer) + std::string rawtoken; + bool rawquoted = false; + if (fromOwn) { - fromOwn = false; + rawtoken = out; + rawquoted = quoted; + if (ToLower && (false == quoted)) + { + for (auto &ch : out) { ch = toLowerChar(ch); } + } } - } - else - { - out = readTokenFromStream(ToLower, Break); - fromOwn = true; - } - stripFirstTokenBOM(out, ToLower, Break); + substituteParameters(out, ToLower); - // snapshot the raw token (BOM stripped, parameters NOT yet substituted) for the - // binary twin, so parameterised includes can be re-substituted at replay time - std::string const rawtoken = out; - - substituteParameters(out, ToLower); - - bool const wasInclude = handleIncludeIfPresent(out, ToLower, Break); - - // capture this file's own content into its twin. include directive tokens - // (the keyword/filename/parameters) are excluded via wasInclude/suppression, - // and child-include tokens (fromOwn == false) belong to the child's own twin. - if (m_compiling && m_writer && fromOwn && (false == wasInclude) - && (false == m_capturesuppress) && (false == rawtoken.empty())) - { - // store numeric tokens as typed doubles (genuinely binary), everything else - // (names, paths, keywords, "(pN)" placeholders) as strings - double value = 0.0; - if (sniffNumber(rawtoken, value)) + if (handleIncludeIfPresent(out, ToLower, Break)) { - m_writer->add_number(value); + // the include directive was consumed; fetch the next token + continue; } - else + + // capture this file's own content into its twin. include directive tokens are + // excluded (handled above); child-include tokens (fromOwn == false) belong to the + // child's own twin. + if (m_compiling && m_writer && fromOwn + && (false == m_capturesuppress) && (false == rawtoken.empty())) { - m_writer->add_token(rawtoken); + // numeric tokens -> typed values; quoted strings preserve case at replay; + // other strings (names, paths, keywords, "(pN)") may be lower-cased per consumer + double value = 0.0; + if ((false == rawquoted) && sniffNumber(rawtoken, value)) + { + m_writer->add_number(value); + } + else + { + m_writer->add_token(rawtoken, rawquoted); + } } + return; } } @@ -612,13 +727,22 @@ void cParser::readReplayToken(std::string &out, bool ToLower, const char *Break) mIncludeParser = nullptr; } - while (m_entryindex < m_entries.size()) + scene::scenery_entry_view entry; + while (m_reader && m_reader->next(entry)) { - scene::scenery_entry const &entry = m_entries[m_entryindex++]; if (entry.type == scene::scenery_entry_type::token) { - out = entry.text; - // re-apply this file's include parameters, mirroring the text path + // stored in original case: lower-case per the consumer (unquoted tokens only), + // then re-apply this file's include parameters, mirroring the text path + out.assign(entry.text); + if (ToLower) { for (auto &ch : out) { ch = toLowerChar(ch); } } + substituteParameters(out, ToLower); + return; + } + if (entry.type == scene::scenery_entry_type::qtoken) + { + // quoted token: case preserved verbatim + out.assign(entry.text); substituteParameters(out, ToLower); return; } @@ -632,10 +756,17 @@ void cParser::readReplayToken(std::string &out, bool ToLower, const char *Break) // include entry: re-evaluate the filename expression (re-randomizing any // random set), then enter the child (its own twin or text) and serve its // tokens; an empty/skipped child just advances to the next entry + std::vector fileexpr; + fileexpr.reserve(entry.fileexpr.size()); + for (auto const sv : entry.fileexpr) { fileexpr.emplace_back(sv); } + std::vector params; + params.reserve(entry.params.size()); + for (auto const sv : entry.params) { params.emplace_back(sv); } + std::size_t pos = 0; - std::string includefile = resolve_random_set(entry.fileexpr, pos); + std::string includefile = resolve_random_set(fileexpr, pos); replace_slashes(includefile); - startIncludeDirect(std::move(includefile), entry.params); + startIncludeDirect(std::move(includefile), std::move(params)); if (mIncludeParser) { mIncludeParser->readToken(out, ToLower, Break); @@ -647,6 +778,7 @@ void cParser::readReplayToken(std::string &out, bool ToLower, const char *Break) } } + m_replayexhausted = true; out.clear(); } @@ -665,12 +797,13 @@ std::vector cParser::readParameters(cParser &Input) } std::string cParser::readQuotes(char const Quote) -{ // read the stream until specified char or stream end +{ // read the buffer until specified char or end std::string token; char c{0}; bool escaped = false; - while (mStream->get(c)) + while (m_bufferpos < m_buffer.size()) { // get all chars until the quote mark + c = m_buffer[m_bufferpos++]; if (escaped) { escaped = false; @@ -699,8 +832,9 @@ void cParser::skipComment(std::string const &Endmark) std::string input; char c{0}; auto const endmarksize = Endmark.size(); - while (mStream->get(c)) + while (m_bufferpos < m_buffer.size()) { + c = m_buffer[m_bufferpos++]; if (c == '\n') { // update line counter @@ -767,15 +901,13 @@ int cParser::getProgress() const { if (m_replay) { - return ( m_entries.empty() - ? 100 - : static_cast(m_entryindex * 100 / m_entries.size()) ); + return ( m_reader ? m_reader->progress() : 100 ); } - if (mSize <= 0) + if (m_buffer.empty()) { return 100; } - return static_cast(mStream->rdbuf()->pubseekoff(0, std::ios_base::cur) * 100 / mSize); + return static_cast(m_bufferpos * 100 / m_buffer.size()); } int cParser::getFullProgress() const @@ -847,3 +979,76 @@ int cParser::LineMain() const { return mIncludeParser ? -1 : mLine; } + +namespace scene +{ +// Headless bake: drains the scenario through cParser so the existing include machinery +// tokenizes and (re)compiles every reachable file's binary twin, with no deserializer, +// scene, renderer or window involved. Because tokens are captured in original case +// (v5 format), draining with a uniform case is correct. +bool scenerybinary_bake_headless(std::string const &Scenariofile, std::string const &Path, bool Loadtraction) +{ + // parallel bake: discover the file graph by tokenizing each file in isolation (no + // scene state, no includes opened -- just their references collected) and compile + // each twin on a worker thread. each file is baked exactly once (deduped), so twin + // writes never race. + std::mutex mutex; + std::condition_variable cv; + std::unordered_set visited; + std::deque queue; + std::size_t active = 0; + std::atomic baked { 0 }; + + auto const enqueue = [ & ]( std::string const &File ) { + // only scenery component files get twins; dedupe so shared includes bake once + if ( ( false == isSceneryFile( File ) ) || File.empty() || ( File[ 0 ] == '$' ) ) { return; } + if ( visited.insert( File ).second ) { queue.push_back( File ); } + }; + { std::lock_guard lock( mutex ); enqueue( Scenariofile ); } + + unsigned const workercount = std::max( 2u, std::min( 16u, std::thread::hardware_concurrency() ) ); + auto const worker = [ & ] { + for ( ;; ) + { + std::string file; + { + std::unique_lock lock( mutex ); + cv.wait( lock, [ & ] { return ( false == queue.empty() ) || ( active == 0 ); } ); + if ( queue.empty() ) + { + if ( active == 0 ) { cv.notify_all(); return; } + continue; + } + file = std::move( queue.front() ); + queue.pop_front(); + ++active; + } + + std::vector includes; + { + cParser parser( file, cParser::buffer_FILE, Path, Loadtraction, std::vector(), false, /*BakeOnly*/ true ); + if ( parser.ok() ) + { + includes = parser.bakeFile(); + ++baked; + } + } + + { + std::unique_lock lock( mutex ); + for ( auto const &inc : includes ) { enqueue( inc ); } + --active; + cv.notify_all(); + } + } + }; + + std::vector workers; + workers.reserve( workercount ); + for ( unsigned i = 0; i < workercount; ++i ) { workers.emplace_back( worker ); } + for ( auto &thread : workers ) { thread.join(); } + + WriteLog( "Bake: compiled " + std::to_string( baked.load() ) + " binary scenery twins from \"" + Scenariofile + "\"" ); + return ( baked.load() > 0 ); +} +} // namespace scene diff --git a/utilities/parser.h b/utilities/parser.h index cf5da7ea..4b102aab 100644 --- a/utilities/parser.h +++ b/utilities/parser.h @@ -22,8 +22,8 @@ http://mozilla.org/MPL/2.0/. // before cParser is defined. namespace scene { enum class scenery_file_kind : std::uint8_t; - struct scenery_entry; class scenery_binary_writer; + class scenery_binary_reader; } ///////////////////////////////////////////////////////////////////////////////////////////////////// @@ -39,9 +39,13 @@ class cParser //: public std::stringstream buffer_TEXT }; // constructors: - cParser(std::string const &Stream, buffertype const Type = buffer_TEXT, std::string Path = "", bool const Loadtraction = true, std::vector Parameters = std::vector(), bool allowRandom = false ); + cParser(std::string const &Stream, buffertype const Type = buffer_TEXT, std::string Path = "", bool const Loadtraction = true, std::vector Parameters = std::vector(), bool allowRandom = false, bool BakeOnly = false ); // destructor: virtual ~cParser(); + // bake-only: tokenize this file into its binary twin without opening includes or + // touching scene state, and return the (relative) filenames it includes so a parallel + // baker can compile each of those twins on its own thread. requires BakeOnly ctor. + std::vector bakeFile(); // methods: template cParser & @@ -70,9 +74,11 @@ class cParser //: public std::stringstream // in replay mode there is no character stream; exhaustion is reaching the // end of the twin's entries with no include still being drained if( m_replay ) { - return ( m_entryindex >= m_entrycount ) && ( !mIncludeParser ); + return m_replayexhausted && ( !mIncludeParser ); } - return mStream->eof(); }; + // text is tokenized from an in-memory buffer; eof is the read cursor + // reaching its end + return ( m_bufferpos >= m_buffer.size() ); }; inline bool ok() { @@ -145,7 +151,9 @@ class cParser //: public std::stringstream // members: bool m_autoclear { true }; // unretrieved tokens are discarded when another read command is issued (legacy behaviour) bool LoadTraction { true }; // load traction? - std::shared_ptr mStream; // relevant kind of buffer is attached on creation. + std::shared_ptr mStream; // attached on creation; kept for open/fail state. + std::string m_buffer; // whole source text read into memory; tokenized from here + std::size_t m_bufferpos { 0 }; // read cursor into m_buffer std::string mFile; // name of the open file, if any std::string mPath; // path to open stream, for relative path lookups. std::streamoff mSize { 0 }; // size of open stream, for progress report. @@ -160,11 +168,13 @@ class cParser //: public std::stringstream std::vector parameters; // parameter list for included file. std::deque tokens; // --- binary scenery twin support --- + // last token produced by readTokenFromStream was (partly) quoted -> case preserved + bool m_lastquoted { false }; // replay: this file is served from its binary twin instead of text bool m_replay { false }; - std::vector m_entries; // entries loaded from the twin (replay) - std::size_t m_entryindex { 0 }; // read cursor into m_entries - std::size_t m_entrycount { 0 }; // m_entries.size(), cached for inline eof() + bool m_replayexhausted { false }; // all twin entries consumed (for inline eof()) + std::string m_twinbuf; // whole twin file held in memory; the reader views into it + std::unique_ptr m_reader; // streams entries from m_twinbuf // compile: this file is being captured into a binary twin alongside the text read bool m_compiling { false }; std::unique_ptr m_writer; @@ -172,6 +182,9 @@ class cParser //: public std::stringstream scene::scenery_file_kind m_binarykind {}; // value-initialised (== scenery_file_kind::scn) bool m_capturesuppress { false }; // suppress token capture (include directive internals) bool m_twinwritten { false }; // guards against writing the twin more than once + // standalone/parallel bake: capture this file only, collect its include filenames + bool m_bakeonly { false }; + std::vector m_bakeincludes; };