diff --git a/CMakeLists.txt b/CMakeLists.txt index 85007284..88e0dfc8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -149,6 +149,7 @@ set(SOURCES "input/messaging.cpp" "scene/scene.cpp" "scene/scenenode.cpp" +"scene/scenerybinary.cpp" "simulation/simulation.cpp" "model/vertex.cpp" "audio/audio.cpp" diff --git a/scene/scene.cpp b/scene/scene.cpp index 0675fe3b..32e56b9e 100644 --- a/scene/scene.cpp +++ b/scene/scene.cpp @@ -24,9 +24,6 @@ http://mozilla.org/MPL/2.0/. namespace scene { -std::string const EU07_FILEEXTENSION_REGION { ".sbt" }; -std::uint32_t const EU07_FILEHEADER { MAKE_ID4( 'E','U','0','7' ) }; -std::uint32_t const EU07_FILEVERSION_REGION { MAKE_ID4( 'S', 'B', 'T', '2' ) }; std::map Hierarchy; // potentially activates event handler with the same name as provided node, and within handler activation range @@ -1105,123 +1102,6 @@ basic_region::update_traction( TDynamicObject *Vehicle, int const Pantographinde } } -// checks whether specified file is a valid region data file -bool -basic_region::is_scene( std::string const &Scenariofile ) const { - - auto filename { Scenariofile }; - while( filename[ 0 ] == '$' ) { - // trim leading $ char rainsted utility may add to the base name for modified .scn files - filename.erase( 0, 1 ); - } - erase_extension( filename ); - filename = Global.asCurrentSceneryPath + filename; - filename += EU07_FILEEXTENSION_REGION; - - if( false == FileExists( filename ) ) { - return false; - } - // file type and version check - std::ifstream input( filename, std::ios::binary ); - - uint32_t headermain{ sn_utils::ld_uint32( input ) }; - uint32_t headertype{ sn_utils::ld_uint32( input ) }; - - if( ( headermain != EU07_FILEHEADER - || ( headertype != EU07_FILEVERSION_REGION ) ) ) { - // wrong file type - return false; - } - - return true; -} - -// stores content of the class in file with specified name -void -basic_region::serialize( std::string const &Scenariofile ) const { - - auto filename { Scenariofile }; - while( filename[ 0 ] == '$' ) { - // trim leading $ char rainsted utility may add to the base name for modified .scn files - filename.erase( 0, 1 ); - } - erase_extension( filename ); - filename = Global.asCurrentSceneryPath + filename; - filename += EU07_FILEEXTENSION_REGION; - - std::ofstream output { filename, std::ios::binary }; - - // region file version 1 - // header: EU07SBT + version (0-255) - sn_utils::ls_uint32( output, EU07_FILEHEADER ); - sn_utils::ls_uint32( output, EU07_FILEVERSION_REGION ); - // sections - // TBD, TODO: build table of sections and file offsets, if we postpone section loading until they're within range - std::uint32_t sectioncount { 0 }; - for( auto *section : m_sections ) { - if( section != nullptr ) { - ++sectioncount; - } - } - // section count, followed by section data - sn_utils::ls_uint32( output, sectioncount ); - std::uint32_t sectionindex { 0 }; - for( auto *section : m_sections ) { - // section data: section index, followed by length of section data, followed by section data - if( section != nullptr ) { - sn_utils::ls_uint32( output, sectionindex ); - section->serialize( output ); } - ++sectionindex; - } -} - -// restores content of the class from file with specified name. returns: true on success, false otherwise -bool -basic_region::deserialize( std::string const &Scenariofile ) { - - auto filename { Scenariofile }; - while( filename[ 0 ] == '$' ) { - // trim leading $ char rainsted utility may add to the base name for modified .scn files - filename.erase( 0, 1 ); - } - erase_extension( filename ); - filename = Global.asCurrentSceneryPath + filename; - filename += EU07_FILEEXTENSION_REGION; - - if( false == FileExists( filename ) ) { - Global.file_binary_terrain_state = false; - return false; - } - // region file version 1 - // file type and version check - std::ifstream input( filename, std::ios::binary ); - - uint32_t headermain { sn_utils::ld_uint32( input ) }; - uint32_t headertype { sn_utils::ld_uint32( input ) }; - - if( ( headermain != EU07_FILEHEADER - || ( headertype != EU07_FILEVERSION_REGION ) ) ) { - // wrong file type - WriteLog( "Bad file: \"" + filename + "\" is of either unrecognized type or version" ); - return false; - } - // sections - // TBD, TODO: build table of sections and file offsets, if we postpone section loading until they're within range - // section count - auto sectioncount { sn_utils::ld_uint32( input ) }; - while( sectioncount-- ) { - // section index, followed by section data size, followed by section data - auto const sectionindex { sn_utils::ld_uint32( input ) }; - auto const sectionsize { sn_utils::ld_uint32( input ) }; - if( m_sections[ sectionindex ] == nullptr ) { - m_sections[ sectionindex ] = new basic_section(); - } - m_sections[ sectionindex ]->deserialize( input ); - } - - return true; -} - // sends content of the class in legacy (text) format to provided stream void basic_region::export_as_text( std::ostream &Output ) const { diff --git a/scene/scene.h b/scene/scene.h index 7b53783b..ef16fcd9 100644 --- a/scene/scene.h +++ b/scene/scene.h @@ -37,12 +37,6 @@ int constexpr EU07_REGIONSIDESECTIONCOUNT = 500; // number of sections along a s struct scratch_data { - struct binary_data { - - bool terrain{ false }; - bool terrain_included{false}; - } binary; - struct location_data { std::stack offset; @@ -68,7 +62,6 @@ struct scratch_data { } trainset; std::string name; - std::string terrain_name; bool initialized { false }; bool time_initialized { false }; @@ -386,15 +379,6 @@ public: // legacy method, updates sounds around camera void update_sounds(); - // checks whether specified file is a valid region data file - bool - is_scene( std::string const &Scenariofile ) const; - // stores content of the class in file with specified name - void - serialize( std::string const &Scenariofile ) const; - // restores content of the class from file with specified name. returns: true on success, false otherwise - bool - deserialize( std::string const &Scenariofile ); // sends content of the class in legacy (text) format to provided stream void export_as_text( std::ostream &Output ) const; diff --git a/scene/scenerybinary.cpp b/scene/scenerybinary.cpp new file mode 100644 index 00000000..66209fb4 --- /dev/null +++ b/scene/scenerybinary.cpp @@ -0,0 +1,317 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#include "stdafx.h" +#include "scene/scenerybinary.h" + +#include "scene/sn_utils.h" +#include "utilities/utilities.h" // ToLower +#include "utilities/Logs.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace scene { + +namespace { + +// LEB128 unsigned varint: 1 byte for values < 128 (covers most string-table indices), +// keeping keyword/index references compact. +void write_varint( std::ostream &Output, std::uint64_t Value ) { + do { + std::uint8_t byte = Value & 0x7F; + Value >>= 7; + if( Value != 0 ) { byte |= 0x80; } + Output.put( static_cast( byte ) ); + } while( Value != 0 ); +} + +std::uint64_t read_varint( std::istream &Input ) { + std::uint64_t value = 0; + int shift = 0; + while( shift < 64 ) { + int const c = Input.get(); + if( c == std::char_traits::eof() ) { break; } + value |= ( static_cast( c & 0x7F ) << shift ); + if( ( c & 0x80 ) == 0 ) { break; } + shift += 7; + } + return value; +} + +// builds the per-file string table while interning, returning each string's index +class string_interner { +public: + std::uint32_t intern( std::string const &Text ) { + auto const it = m_lookup.find( Text ); + if( it != m_lookup.end() ) { return it->second; } + auto const index = static_cast( m_table.size() ); + m_lookup.emplace( Text, index ); + m_table.emplace_back( Text ); + return index; + } + std::vector const &table() const { return m_table; } +private: + std::unordered_map m_lookup; + std::vector m_table; +}; + +} // anonymous namespace + +void +scenery_binary_writer::add_token( std::string Token ) { + scenery_entry entry; + entry.type = scenery_entry_type::token; + entry.text = std::move( Token ); + m_entries.emplace_back( std::move( entry ) ); +} + +void +scenery_binary_writer::add_number( double Value ) { + scenery_entry entry; + entry.type = scenery_entry_type::number; + entry.number = Value; + m_entries.emplace_back( std::move( entry ) ); +} + +void +scenery_binary_writer::add_include( std::vector Fileexpr, std::vector Params ) { + scenery_entry entry; + entry.type = scenery_entry_type::include; + entry.fileexpr = std::move( Fileexpr ); + entry.params = std::move( Params ); + m_entries.emplace_back( std::move( entry ) ); +} + +bool +scenery_binary_writer::write( std::ostream &Output, scenery_file_kind Kind ) const { + + // first pass: intern every string the entries reference so keywords/paths that + // repeat are stored once and referenced by index + string_interner interner; + std::vector tokenindex; // index per token entry, in order + std::vector> fileexpridx, paramidx; // per include entry + for( auto const &entry : m_entries ) { + if( entry.type == scenery_entry_type::token ) { + tokenindex.emplace_back( interner.intern( entry.text ) ); + } + else if( entry.type == scenery_entry_type::include ) { + std::vector fe, pe; + for( auto const &token : entry.fileexpr ) { fe.emplace_back( interner.intern( token ) ); } + for( auto const ¶m : entry.params ) { pe.emplace_back( interner.intern( param ) ); } + fileexpridx.emplace_back( std::move( fe ) ); + paramidx.emplace_back( std::move( pe ) ); + } + } + + // header: magic + kind + flags + sn_utils::ls_uint32( Output, SCENERYBINARY_MAGIC ); + sn_utils::s_uint8( Output, static_cast( Kind ) ); + sn_utils::ls_uint32( Output, 0 ); // flags, reserved + + // string table + auto const &table = interner.table(); + sn_utils::ls_uint32( Output, static_cast( table.size() ) ); + for( auto const &text : table ) { + sn_utils::s_str( Output, text ); + } + + // entries + sn_utils::ls_uint32( Output, static_cast( m_entries.size() ) ); + std::size_t tokencursor = 0, includecursor = 0; + for( auto const &entry : m_entries ) { + sn_utils::s_uint8( Output, static_cast( entry.type ) ); + if( entry.type == scenery_entry_type::include ) { + auto const &fe = fileexpridx[ includecursor ]; + auto const &pe = paramidx[ includecursor ]; + ++includecursor; + write_varint( Output, fe.size() ); + for( auto idx : fe ) { write_varint( Output, idx ); } + write_varint( Output, pe.size() ); + for( auto idx : pe ) { write_varint( Output, idx ); } + } + else if( entry.type == scenery_entry_type::number ) { + sn_utils::ls_float64( Output, entry.number ); + } + else { + write_varint( Output, tokenindex[ tokencursor++ ] ); + } + } + + return ( false == Output.fail() ); +} + +bool +scenery_binary_reader::open( std::istream &Input ) { + + m_entries.clear(); + + auto const magic { sn_utils::ld_uint32( Input ) }; + if( magic != SCENERYBINARY_MAGIC ) { + // unrecognized type or incompatible version + return false; + } + m_kind = static_cast( sn_utils::d_uint8( Input ) ); + sn_utils::ld_uint32( Input ); // flags, reserved + + // string table + std::vector table; + auto tablesize { sn_utils::ld_uint32( Input ) }; + table.reserve( tablesize ); + while( tablesize-- ) { + table.emplace_back( sn_utils::d_str( Input ) ); + } + // resolves an interned index back to its string, guarding against corruption + auto const resolve = [ & ]( std::uint64_t index ) -> std::string { + return ( index < table.size() ) ? table[ static_cast( index ) ] : std::string(); + }; + + auto count { sn_utils::ld_uint32( Input ) }; + m_entries.reserve( count ); + while( count-- ) { + scenery_entry entry; + entry.type = static_cast( sn_utils::d_uint8( Input ) ); + if( entry.type == scenery_entry_type::include ) { + auto expr { read_varint( Input ) }; + entry.fileexpr.reserve( expr ); + while( expr-- ) { + entry.fileexpr.emplace_back( resolve( read_varint( Input ) ) ); + } + auto params { read_varint( Input ) }; + entry.params.reserve( params ); + while( params-- ) { + entry.params.emplace_back( resolve( read_varint( Input ) ) ); + } + } + else if( entry.type == scenery_entry_type::number ) { + entry.number = sn_utils::ld_float64( Input ); + } + else { + entry.text = resolve( read_varint( Input ) ); + } + m_entries.emplace_back( std::move( entry ) ); + } + + return ( false == Input.fail() ); +} + +namespace { + +// minimal bounded thread pool for offloading twin serialization/writing +class bake_pool { +public: + bake_pool() { + unsigned const workers = std::max( 2u, std::min( 8u, std::thread::hardware_concurrency() ) ); + for( unsigned i = 0; i < workers; ++i ) { + m_threads.emplace_back( [ this ] { run(); } ); + } + } + ~bake_pool() { + { std::unique_lock lock( m_mutex ); m_stop = true; } + m_wake.notify_all(); + for( auto &thread : m_threads ) { if( thread.joinable() ) { thread.join(); } } + } + void enqueue( std::function Task ) { + { std::unique_lock lock( m_mutex ); m_tasks.emplace( std::move( Task ) ); ++m_pending; } + m_wake.notify_one(); + } + void wait_idle() { + std::unique_lock lock( m_mutex ); + m_idle.wait( lock, [ this ] { return m_pending == 0; } ); + } +private: + void run() { + for( ;; ) { + std::function task; + { + std::unique_lock lock( m_mutex ); + m_wake.wait( lock, [ this ] { return m_stop || ( false == m_tasks.empty() ); } ); + if( m_stop && m_tasks.empty() ) { return; } + task = std::move( m_tasks.front() ); + m_tasks.pop(); + } + task(); + { + std::unique_lock lock( m_mutex ); + if( --m_pending == 0 ) { m_idle.notify_all(); } + } + } + } + std::vector m_threads; + std::queue> m_tasks; + std::mutex m_mutex; + std::condition_variable m_wake; + std::condition_variable m_idle; + bool m_stop { false }; + std::size_t m_pending { 0 }; +}; + +bake_pool &pool() { + static bake_pool instance; + return instance; +} + +// results recorded by worker threads, drained and logged on the main thread +std::mutex g_resultmutex; +std::vector g_resultlog; // success lines, in completion order +std::vector g_resultfail; // failure paths + +} // anonymous namespace + +void +scenerybinary_write_async( std::unique_ptr Writer, std::string Path, scenery_file_kind Kind ) { + // std::function requires a copyable target, so move the writer into a shared_ptr + std::shared_ptr writer { std::move( Writer ) }; + pool().enqueue( [ writer, path = std::move( Path ), Kind ] { + std::ofstream output( path, std::ios::binary ); + bool const ok = output.good() && writer->write( output, Kind ); + output.flush(); + std::lock_guard lock( g_resultmutex ); + if( ok ) { + g_resultlog.emplace_back( "Compiled binary scenery: " + path + " (" + std::to_string( writer->entry_count() ) + " entries)" ); + } + else { + g_resultfail.emplace_back( path ); + } + } ); +} + +void +scenerybinary_wait_all() { + pool().wait_idle(); + std::lock_guard lock( g_resultmutex ); + for( auto const &line : g_resultlog ) { WriteLog( line ); } + for( auto const &path : g_resultfail ) { ErrorLog( "Failed to write binary scenery \"" + path + "\"" ); } + g_resultlog.clear(); + g_resultfail.clear(); +} + +std::string +scenerybinary_extension_for( std::string const &Sourcefile ) { + + auto const lower { ToLower( Sourcefile ) }; + if( lower.size() >= 4 && lower.compare( lower.size() - 4, 4, ".inc" ) == 0 ) { + return SCENERYBINARY_EXT_INC; + } + if( lower.size() >= 4 && lower.compare( lower.size() - 4, 4, ".scm" ) == 0 ) { + return SCENERYBINARY_EXT_SCM; + } + if( lower.size() >= 4 && lower.compare( lower.size() - 4, 4, ".ctr" ) == 0 ) { + return SCENERYBINARY_EXT_CTR; + } + // default: treat as a top-level scenario file (.scn) + return SCENERYBINARY_EXT_SCN; +} + +} // namespace scene diff --git a/scene/scenerybinary.h b/scene/scenerybinary.h new file mode 100644 index 00000000..dd81c3a3 --- /dev/null +++ b/scene/scenerybinary.h @@ -0,0 +1,136 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +#include +#include +#include +#include +#include + +#include "utilities/utilities.h" // MAKE_ID4 + +// Binary, modular scenery container ("eu7" format, version 1). +// +// Replaces the legacy text formats (.scn / .scm / .inc) and the old terrain-only +// .sbt blob. The mapping is one-to-one and modular: every text file compiles to +// exactly one binary twin and includes are preserved as references, so the binary +// twin of one file points to the binary twins of the files it includes. +// route.scn -> route.scnb +// piece.inc -> piece.incb +// mesh.scm -> mesh.scmb +// +// A binary twin stores the source file's own, fully resolved content (comments +// stripped, parameters and random sets resolved) as an ordered list of entries. +// Numeric tokens are stored as 8-byte IEEE doubles (so coordinates, ranges etc. are +// genuinely binary, not ASCII). All string tokens (keywords like "node"/"endmodel", +// names, paths) are interned into a per-file string table and referenced from the +// entries by a small varint index, so a keyword that occurs thousands of times is +// stored once. An "include" entry marks the spot where the source file pulled in +// another file, carrying the (interned) include filename expression and parameters. +// +// The binary is consumed transparently at the cParser file layer: opening a file +// whose binary twin exists serves tokens from the twin instead of tokenizing text, +// and an include entry triggers the same include machinery as the text path (which +// recursively prefers the included file's own binary twin). The scenery +// deserializer is unaware of the format and needs no changes; the file's identity +// (cParser::Name()), node grouping per .inc and parameter substitution are all +// preserved exactly as in the text path. + +namespace scene { + +// "eu7" + format version, little-endian via MAKE_ID4 ('e','u','7',). +// v2 introduced typed numeric entries (f64); v3 interns string tokens into a per-file +// table referenced by varint indices. bumping the version invalidates older twins so +// they are recompiled rather than misread. +constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 3 ) }; + +// file extension of the binary twin, derived from source kind +std::string const SCENERYBINARY_EXT_SCN { ".scnb" }; +std::string const SCENERYBINARY_EXT_INC { ".incb" }; +std::string const SCENERYBINARY_EXT_SCM { ".scmb" }; +std::string const SCENERYBINARY_EXT_CTR { ".ctrb" }; + +// which text format the binary file was compiled from +enum class scenery_file_kind : std::uint8_t { + scn = 0, + inc = 1, + scm = 2, +}; + +// kind of a stored entry +enum class scenery_entry_type : std::uint8_t { + token = 0, // a non-numeric resolved token (name/path/keyword), stored as a string + include = 1, // an include directive: pulls in another file with parameters + number = 2, // a numeric token, stored as an 8-byte IEEE double +}; + +// a single ordered element of a file's resolved content +struct scenery_entry { + scenery_entry_type type { scenery_entry_type::token }; + // token value (token entries only) + std::string text; + // numeric value (number entries only) + double number { 0.0 }; + // include only: the raw filename expression as written in the source, i.e. either + // a single filename token or a random set "[ a b c ]" (including the brackets). + // kept verbatim so the random choice is re-evaluated on every replay rather than + // frozen at compile time. + std::vector fileexpr; + // include only: directive parameters + std::vector params; +}; + +// accumulates a file's entries and writes them, with a header, to a stream +class scenery_binary_writer { + +public: + void add_token( std::string Token ); + // stores a numeric token as a typed double + void add_number( double Value ); + // Fileexpr is the verbatim filename expression (single token or random set) + void add_include( std::vector Fileexpr, std::vector Params ); + std::size_t entry_count() const { return m_entries.size(); } + // serializes header + entries to the stream. returns false on stream failure. + bool write( std::ostream &Output, scenery_file_kind Kind ) const; + +private: + std::vector m_entries; +}; + +// reads a binary scenery file fully into memory and exposes its entries in order +class scenery_binary_reader { + +public: + // validates magic/version and loads every entry. returns false if the stream is + // not a valid current-version binary scenery file. + bool open( std::istream &Input ); + + scenery_file_kind kind() const { return m_kind; } + std::vector const &entries() const { return m_entries; } + +private: + scenery_file_kind m_kind { scenery_file_kind::scn }; + std::vector m_entries; +}; + +// maps a source scenery filename to the extension of its binary twin +std::string scenerybinary_extension_for( std::string const &Sourcefile ); + +// Asynchronous baking: serializing and writing a twin is pure, self-contained work +// (it only needs the writer's collected entries), so it is offloaded to a small thread +// pool and overlaps with the rest of scene construction instead of blocking the load. +// Takes ownership of the writer. +void scenerybinary_write_async( std::unique_ptr Writer, std::string Path, scenery_file_kind Kind ); +// Blocks until every queued async write has finished, then emits their log lines on the +// calling (main) thread. Call once the scenario has finished loading. +void scenerybinary_wait_all(); + +} // namespace scene diff --git a/simulation/simulationstateserializer.cpp b/simulation/simulationstateserializer.cpp index 7277e338..166cf1a6 100644 --- a/simulation/simulationstateserializer.cpp +++ b/simulation/simulationstateserializer.cpp @@ -16,6 +16,7 @@ http://mozilla.org/MPL/2.0/. #include "simulation/simulationsounds.h" #include "simulation/simulationenvironment.h" #include "scene/scenenodegroups.h" +#include "scene/scenerybinary.h" #include "rendering/particles.h" #include "world/Event.h" #include "world/MemCell.h" @@ -41,30 +42,13 @@ state_serializer::deserialize_begin( std::string const &Scenariofile ) { simulation::State.init_scripting_interface(); - // NOTE: for the time being import from text format is a given, since we don't have full binary serialization - std::shared_ptr state = - std::make_shared(Scenariofile, cParser::buffer_FILE, Global.asCurrentSceneryPath, Global.bLoadTraction); - - // TODO: check first for presence of serialized binary files - // if this fails, fall back on the legacy text format - state->scratchpad.name = Scenariofile; - if( ( true == Global.file_binary_terrain ) - && ( Scenariofile != "$.scn" ) ) { - // compilation to binary file isn't supported for rainsted-created overrides - // NOTE: we postpone actual loading of the scene until we process time, season and weather data - state->scratchpad.binary.terrain = Region->is_scene( Scenariofile ) ; - } - - if (false != state->scratchpad.binary.terrain) - { - Global.file_binary_terrain_state = true; - WriteLog("Default SBT present"); - } - else - { - Global.file_binary_terrain_state = false; - WriteLog("Default SBT absent"); - } + // open the scenario file. binary scenery twins (.scnb/.incb/.scmb) are handled + // transparently inside cParser: if a twin exists it is replayed instead of the + // text, otherwise the text is parsed and a twin compiled alongside it. + std::shared_ptr state = + std::make_shared( Scenariofile, cParser::buffer_FILE, Global.asCurrentSceneryPath, Global.bLoadTraction ); + state->scenariofile = Scenariofile; + state->scratchpad.name = Scenariofile; scene::Groups.create(); if( false == state->input.ok() ) @@ -152,13 +136,12 @@ state_serializer::deserialize_continue(std::shared_ptr state scene::Groups.update_map(); Region->create_map_geometry(); - if( ( true == Global.file_binary_terrain ) - && ( false == state->scratchpad.binary.terrain ) - && ( state->scenariofile != "$.scn" ) ) { - // if we didn't find usable binary version of the scenario files, create them now for future use - // as long as the scenario file wasn't rainsted-created base file override - Region->serialize( state->scenariofile ); - } + // loading finished: flush the top-level scenario's binary twin now rather than + // waiting for the parser to be destroyed (the loader keeps the state around) + Input.flushBinaryTwin(); + // wait out any background twin writes (includes) so they are complete and logged + // before we report the scenario as loaded + scene::scenerybinary_wait_all(); return false; } @@ -361,16 +344,6 @@ state_serializer::deserialize_firstinit( cParser &Input, scene::scratch_data &Sc if( true == Scratchpad.initialized ) { return; } - if( true == Scratchpad.binary.terrain ) { - // at this stage it should be safe to import terrain from the binary scene file - // TBD: postpone loading furter and only load required blocks during the simulation? - if (false == Scratchpad.binary.terrain_included) - { - Region->deserialize(Scratchpad.name); - } - - } - simulation::Paths.InitTracks(); simulation::Traction.InitTraction(); simulation::Events.InitEvents(); @@ -500,37 +473,30 @@ state_serializer::deserialize_node( cParser &Input, scene::scratch_data &Scratch else if( nodedata.type == "model" ) { if( nodedata.range_min < 0.0 ) { - // 3d terrain - if( false == Scratchpad.binary.terrain ) { - // if we're loading data from text .scn file convert and import - auto *instance = deserialize_model( Input, Scratchpad, nodedata ); - // model import can potentially fail - if( instance == nullptr ) { return; } - // go through submodels, and import them as shapes - auto const cellcount = instance->TerrainCount() + 1; // zliczenie submodeli - for( auto i = 1; i < cellcount; ++i ) { - auto *submodel = instance->TerrainSquare( i - 1 ); + // 3d terrain: convert the model's submodels into region shapes + auto *instance = deserialize_model( Input, Scratchpad, nodedata ); + // model import can potentially fail + if( instance == nullptr ) { return; } + // go through submodels, and import them as shapes + auto const cellcount = instance->TerrainCount() + 1; // zliczenie submodeli + for( auto i = 1; i < cellcount; ++i ) { + auto *submodel = instance->TerrainSquare( i - 1 ); + simulation::Region->insert( + scene::shape_node().convert( submodel ), + Scratchpad, + false ); + // if there's more than one group of triangles in the cell they're held as children of the primary submodel + submodel = submodel->ChildGet(); + while( submodel != nullptr ) { simulation::Region->insert( scene::shape_node().convert( submodel ), Scratchpad, false ); - // if there's more than one group of triangles in the cell they're held as children of the primary submodel - submodel = submodel->ChildGet(); - while( submodel != nullptr ) { - simulation::Region->insert( - scene::shape_node().convert( submodel ), - Scratchpad, - false ); - submodel = submodel->NextGet(); - } + submodel = submodel->NextGet(); } - // with the import done we can get rid of the source model - delete instance; - } - else { - // if binary terrain file was present, we already have this data - skip_until( Input, "endmodel" ); } + // with the import done we can get rid of the source model + delete instance; } else { // regular instance of 3d mesh @@ -563,10 +529,8 @@ state_serializer::deserialize_node( cParser &Input, scene::scratch_data &Scratch || ( nodedata.type == "triangle_fan" ) ) { auto const skip { - // all shapes will be loaded from the binary version of the file - ( true == Scratchpad.binary.terrain ) // crude way to detect fixed switch trackbed geometry - || ( ( true == Global.CreateSwitchTrackbeds ) + ( ( true == Global.CreateSwitchTrackbeds ) && ( Input.Name().size() >= 15 ) && Input.Name().starts_with("scenery/zwr") && Input.Name().ends_with(".inc") ) }; @@ -587,17 +551,10 @@ state_serializer::deserialize_node( cParser &Input, scene::scratch_data &Scratch || ( nodedata.type == "line_strip" ) || ( nodedata.type == "line_loop" ) ) { - if( false == Scratchpad.binary.terrain ) { - - simulation::Region->insert( - scene::lines_node().import( - Input, nodedata ), - Scratchpad ); - } - else { - // all lines were already loaded from the binary version of the file - skip_until( Input, "endline" ); - } + simulation::Region->insert( + scene::lines_node().import( + Input, nodedata ), + Scratchpad ); } else if( nodedata.type == "memcell" ) { @@ -785,22 +742,9 @@ state_serializer::deserialize_trainset( cParser &Input, scene::scratch_data &Scr void state_serializer::deserialize_terrain(cParser &Input, scene::scratch_data &Scratchpad) { - std::string line; - Input.getTokens(1); - Input >> line; - if (Global.file_binary_terrain && line.ends_with(".sbt")) - { - Scratchpad.binary.terrain = Region->is_scene(line); - Global.file_binary_terrain_state = true; - Scratchpad.binary.terrain_included = true; - Scratchpad.terrain_name = line; - WriteLog("Included SBT file: " + line); - Region->deserialize(Scratchpad.terrain_name); - - } - - skip_until(Input, "endterrain"); - + // legacy directive; the SBT terrain blob has been retired and terrain now loads + // as ordinary scenery content, so the block is simply consumed. + skip_until(Input, "endterrain"); } void diff --git a/utilities/Globals.cpp b/utilities/Globals.cpp index 0dba2041..1e9913a3 100644 --- a/utilities/Globals.cpp +++ b/utilities/Globals.cpp @@ -632,9 +632,9 @@ bool global_settings::ConfigParseSimulation(cParser& Parser, const std::string& return true; } - if (token == "file.binary.terrain") + if (token == "file.binary.scenery") { - ParseOne(Parser, file_binary_terrain, 1, false); + ParseOne(Parser, file_binary_scenery, 1, false); return true; } @@ -1587,7 +1587,7 @@ global_settings::export_as_text( std::ostream &Output ) const { export_as_text( Output, "multisampling", iMultisampling ); export_as_text( Output, "latitude", fLatitudeDeg ); export_as_text( Output, "convertmodels", iConvertModels ); - export_as_text( Output, "file.binary.terrain", file_binary_terrain ); + export_as_text( Output, "file.binary.scenery", file_binary_scenery ); export_as_text( Output, "inactivepause", bInactivePause ); export_as_text( Output, "slowmotion", iSlowMotionMask ); export_as_text( Output, "hideconsole", bHideConsole ); diff --git a/utilities/Globals.h b/utilities/Globals.h index a21a191e..11d6b15f 100644 --- a/utilities/Globals.h +++ b/utilities/Globals.h @@ -80,8 +80,7 @@ struct global_settings { std::string local_start_vehicle{ "EU07-424" }; int iConvertModels{ 0 }; // tworzenie plików binarnych int iConvertIndexRange{ 1000 }; // range of duplicate vertex scan - bool file_binary_terrain{ true }; // enable binary terrain (de)serialization - bool file_binary_terrain_state{true}; + bool file_binary_scenery{ true }; // enable binary scenery twins (.scnb/.incb/.scmb) // logs bool priorityLoadText3D{false}; // ladowanie T3D priorytetowo int iWriteLogEnabled{ 3 }; // maska bitowa: 1-zapis do pliku, 2-okienko, 4-nazwy torów diff --git a/utilities/parser.cpp b/utilities/parser.cpp index 68656c5d..96625518 100644 --- a/utilities/parser.cpp +++ b/utilities/parser.cpp @@ -10,8 +10,15 @@ http://mozilla.org/MPL/2.0/. #include "stdafx.h" #include "utilities/parser.h" #include "utilities/Logs.h" +#include "utilities/utilities.h" +#include "utilities/Globals.h" #include "scene/scenenodegroups.h" +#include "scene/scenerybinary.h" + +#include +#include +#include /* MaSzyna EU07 locomotive simulator parser @@ -46,6 +53,78 @@ inline bool startsWithBOM(const std::string &s) && static_cast(s[1]) == 0xBB && static_cast(s[2]) == 0xBF; } + +// true if the whole token is a finite decimal number; on success Value is set. +// used to store numeric tokens as typed doubles in the binary twin instead of text. +inline bool sniffNumber(const std::string &token, double &Value) +{ + if (token.empty()) + { + return false; + } + const char *first = token.data(); + const char *last = token.data() + token.size(); + double value = 0.0; + auto const result = std::from_chars(first, last, value); + // require the entire token to be consumed and the value to be finite (reject + // "inf"/"nan"/overflow, and identifiers like "12abc" or "1.2.3") + if ((result.ec != std::errc()) || (result.ptr != last) || (false == std::isfinite(value))) + { + return false; + } + return (Value = value, true); +} + +// shortest representation of a double that round-trips back to the same value, +// used when serving a numeric twin entry to the (text-oriented) deserializer +inline std::string formatNumber(double Value) +{ + char buffer[32]; + auto const result = std::to_chars(buffer, buffer + sizeof(buffer), Value); + return std::string(buffer, result.ptr); +} + +inline bool endsWithLower(const std::string &s, const char *suffix) +{ + const std::string suf(suffix); + return s.size() >= suf.size() && ToLower(s.substr(s.size() - suf.size())) == suf; +} + +// scenery source files (and only these) get binary twins +inline bool isSceneryFile(const std::string &name) +{ + // text scenery component files share one syntax (per wiki: SCN/SCM/CTR/INC). + // .eu7 is intentionally excluded: it is not a documented text format (it can be an + // editor/binary file), so tokenizing it into a twin would be wrong. + return endsWithLower(name, ".scn") || endsWithLower(name, ".inc") + || endsWithLower(name, ".scm") || endsWithLower(name, ".ctr"); +} + +inline scene::scenery_file_kind sceneryKind(const std::string &name) +{ + if (endsWithLower(name, ".inc")) return scene::scenery_file_kind::inc; + if (endsWithLower(name, ".scm")) return scene::scenery_file_kind::scm; + return scene::scenery_file_kind::scn; +} + +// the twin may be replayed only if it is at least as new as its source text, so that +// editing a scenery file forces a recompile instead of replaying a stale twin +inline bool twinIsFresh(const std::string &sourcefull, const std::string &twinfull) +{ + std::error_code ec; + auto const sourcetime = std::filesystem::last_write_time(sourcefull, ec); + if (ec) + { + // can't read the source time (e.g. packed/missing): trust the twin + return true; + } + auto const twintime = std::filesystem::last_write_time(twinfull, ec); + if (ec) + { + return false; + } + return (twintime >= sourcetime); +} } // namespace // constructors @@ -62,14 +141,58 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P { case buffer_FILE: { - Path.append(Stream); - mStream = std::make_shared(Path, std::ios_base::binary); - // content of *.inc files is potentially grouped together - if ((Stream.size() >= 4) && (ToLower(Stream.substr(Stream.size() - 4)) == ".inc")) + // content of *.inc files is grouped together (same for text and binary replay) + if (endsWithLower(Stream, ".inc")) { mIncFile = true; scene::Groups.create(); } + + bool opened = false; + // scenery files (.scn/.inc/.scm) are backed by a binary twin: replay it if + // present, otherwise parse the text and compile a twin alongside it. + // rainsted-created '$' overrides are always parsed from text. + if (Global.file_binary_scenery && isSceneryFile(Stream) && (false == Stream.empty()) && (Stream[0] != '$')) + { + std::string twinrel = Stream; + erase_extension(twinrel); + twinrel += scene::scenerybinary_extension_for(Stream); + std::string const twinfull = mPath + twinrel; + std::string const sourcefull = mPath + Stream; + + std::ifstream probe(twinfull, std::ios_base::binary); + scene::scenery_binary_reader reader; + if (probe.good() && twinIsFresh(sourcefull, twinfull) && reader.open(probe)) + { + // replay: serve tokens from the twin, no text tokenization at all + m_entries = reader.entries(); + m_entrycount = m_entries.size(); + m_replay = true; + m_entryindex = 0; + mStream = std::make_shared(std::string()); + opened = true; + } + else + { + Path.append(Stream); + mStream = std::make_shared(Path, std::ios_base::binary); + if (false == mStream->fail()) + { + // no usable twin: compile one while parsing the text + m_compiling = true; + m_binarytwinpath = twinfull; + m_binarykind = sceneryKind(Stream); + m_writer = std::make_unique(); + } + opened = true; + } + } + if (false == opened) + { + // non-scenery file, or binary scenery disabled: plain text parse + Path.append(Stream); + mStream = std::make_shared(Path, std::ios_base::binary); + } break; } case buffer_TEXT: @@ -106,6 +229,8 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P // destructor cParser::~cParser() { + // fallback flush in case the twin wasn't explicitly finalized + flushBinaryTwin(); if (true == mIncFile) { @@ -114,6 +239,26 @@ cParser::~cParser() } } +void cParser::flushBinaryTwin() +{ + // write the twin only once, only when compiling, and only if the source text was + // fully consumed -- an aborted parse must not leave a truncated twin to be replayed + if ((false == m_compiling) || (false == static_cast(m_writer)) || m_twinwritten) + { + return; + } + if ((nullptr == mStream) || (false == mStream->eof())) + { + return; + } + m_twinwritten = true; + + // hand the finished writer to the background pool; serialization and file I/O then + // overlap with the rest of the scene build instead of blocking it. ownership of the + // writer transfers to the task, so this cParser no longer touches it. + scene::scenerybinary_write_async(std::move(m_writer), m_binarytwinpath, m_binarykind); +} + template <> glm::vec3 cParser::getToken(bool const ToLower, char const *Break) { // NOTE: this specialization ignores default arguments @@ -313,40 +458,63 @@ void cParser::skipIncludeBlock() { } while (t != "end" && !t.empty()); } -void cParser::startIncludeFromParser(cParser& srcParser, bool ToLower, std::string includefile) { - replace_slashes(includefile); +void cParser::processInclude(cParser& srcParser, bool ToLower) { + // the filename expression and parameters are part of the directive, not file + // content, so keep them out of this file's own captured token stream + bool const prevsuppress = m_capturesuppress; + m_capturesuppress = true; + + std::vector fileexpr; + std::string pick; + if (allowRandomIncludes) { + // capture the verbatim expression (incl. random-set brackets) and pick one + pick = deserialize_random_set_capture(srcParser, fileexpr); + } + else { + std::string filename; + srcParser.readToken(filename, ToLower); + std::replace(filename.begin(), filename.end(), '\\', '/'); + fileexpr.emplace_back(filename); + pick = filename; + } + // consume the directive's parameter list (up to "end") + std::vector params = readParameters(srcParser); + + m_capturesuppress = prevsuppress; + + // record the include reference verbatim so replay can re-randomize the choice + if (m_compiling && m_writer) { + m_writer->add_include(fileexpr, params); + } + + // open the include for the live load with a freshly evaluated filename + replace_slashes(pick); + startIncludeDirect(std::move(pick), std::move(params)); +} + +void cParser::startIncludeDirect(std::string includefile, std::vector Params) { const bool allowTraction = (true == LoadTraction) || ((false == contains(includefile, "tr/")) && (false == contains(includefile, "tra/"))); if (!allowTraction) { - // skip include block until "end" (original behavior in token-mode include) - skipIncludeBlock(); - return; - } - - const bool isTerrain = contains(includefile, "_ter.scm"); - if (isTerrain && true == Global.file_binary_terrain_state) { - WriteLog("SBT found, ignoring: " + includefile); - readParameters(srcParser); // preserve original side-effect: still consume parameters + // traction loading disabled: the include is simply not opened return; } if (Global.ParserLogIncludes) { - if (isTerrain) WriteLog("including terrain: " + includefile); - else { - // WriteLog("including: " + includefile); - } + // WriteLog("including: " + includefile); } mIncludeParser = std::make_shared( - includefile, /*buffer_FILE*/ static_cast(/*buffer_FILE*/ 0), mPath, LoadTraction, readParameters(srcParser) + includefile, buffer_FILE, mPath, LoadTraction, std::move(Params) ); mIncludeParser->allowRandomIncludes = allowRandomIncludes; mIncludeParser->autoclear(m_autoclear); - if (mIncludeParser->mSize <= 0) { + // a binary-twin replay child reports mSize 0 but is still valid + if (mIncludeParser->mSize <= 0 && (false == mIncludeParser->m_replay)) { ErrorLog("Bad include: can't open file \"" + includefile + "\""); } } @@ -354,14 +522,7 @@ void cParser::startIncludeFromParser(cParser& srcParser, bool ToLower, std::stri bool cParser::handleIncludeIfPresent(std::string& token, bool ToLower, const char* Break) { // token-mode include: token == "include" if (expandIncludes && token == "include") { - std::string includefile; - if (allowRandomIncludes) - includefile = deserialize_random_set(*this); - else - readToken(includefile, ToLower); - - startIncludeFromParser(*this, ToLower, std::move(includefile)); - + processInclude(*this, ToLower); // after processing include, return next token from current parser readToken(token, ToLower, Break); return true; @@ -370,14 +531,7 @@ bool cParser::handleIncludeIfPresent(std::string& token, bool ToLower, const cha // line-mode HACK: Break == "\n\r" and line begins with "include" if ((std::strcmp(Break, "\n\r") == 0) && token.compare(0, 7, "include") == 0) { cParser includeparser(token.substr(7)); - std::string includefile; - if (allowRandomIncludes) - includefile = deserialize_random_set(includeparser); - else - includeparser.readToken(includefile, ToLower); - - startIncludeFromParser(includeparser, ToLower, std::move(includefile)); - + processInclude(includeparser, ToLower); readToken(token, ToLower, Break); return true; } @@ -387,6 +541,14 @@ bool cParser::handleIncludeIfPresent(std::string& token, bool ToLower, const cha void cParser::readToken(std::string &out, bool ToLower, const char *Break) { + if (m_replay) + { + // served from the binary twin; includes are entered transparently + readReplayToken(out, ToLower, Break); + return; + } + + bool fromOwn; if (mIncludeParser) { mIncludeParser->readToken(out, ToLower, Break); @@ -394,18 +556,98 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break) { mIncludeParser = nullptr; out = readTokenFromStream(ToLower, Break); + fromOwn = true; + } + else + { + fromOwn = false; } } else { out = readTokenFromStream(ToLower, Break); + fromOwn = true; } stripFirstTokenBOM(out, ToLower, Break); + // snapshot the raw token (BOM stripped, parameters NOT yet substituted) for the + // binary twin, so parameterised includes can be re-substituted at replay time + std::string const rawtoken = out; + substituteParameters(out, ToLower); - handleIncludeIfPresent(out, ToLower, Break); + bool const wasInclude = handleIncludeIfPresent(out, ToLower, Break); + + // capture this file's own content into its twin. include directive tokens + // (the keyword/filename/parameters) are excluded via wasInclude/suppression, + // and child-include tokens (fromOwn == false) belong to the child's own twin. + if (m_compiling && m_writer && fromOwn && (false == wasInclude) + && (false == m_capturesuppress) && (false == rawtoken.empty())) + { + // store numeric tokens as typed doubles (genuinely binary), everything else + // (names, paths, keywords, "(pN)" placeholders) as strings + double value = 0.0; + if (sniffNumber(rawtoken, value)) + { + m_writer->add_number(value); + } + else + { + m_writer->add_token(rawtoken); + } + } +} + +void cParser::readReplayToken(std::string &out, bool ToLower, const char *Break) +{ + // drain an active child include first, exactly like the text path + if (mIncludeParser) + { + mIncludeParser->readToken(out, ToLower, Break); + if (false == out.empty()) + { + return; + } + mIncludeParser = nullptr; + } + + while (m_entryindex < m_entries.size()) + { + scene::scenery_entry const &entry = m_entries[m_entryindex++]; + if (entry.type == scene::scenery_entry_type::token) + { + out = entry.text; + // re-apply this file's include parameters, mirroring the text path + substituteParameters(out, ToLower); + return; + } + if (entry.type == scene::scenery_entry_type::number) + { + // typed numeric entry: hand back its shortest round-tripping text form + // (no parameter substitution applies to a literal number) + out = formatNumber(entry.number); + return; + } + // include entry: re-evaluate the filename expression (re-randomizing any + // random set), then enter the child (its own twin or text) and serve its + // tokens; an empty/skipped child just advances to the next entry + std::size_t pos = 0; + std::string includefile = resolve_random_set(entry.fileexpr, pos); + replace_slashes(includefile); + startIncludeDirect(std::move(includefile), entry.params); + if (mIncludeParser) + { + mIncludeParser->readToken(out, ToLower, Break); + if (false == out.empty()) + { + return; + } + mIncludeParser = nullptr; + } + } + + out.clear(); } std::vector cParser::readParameters(cParser &Input) @@ -523,6 +765,16 @@ void cParser::injectString(const std::string &str) int cParser::getProgress() const { + if (m_replay) + { + return ( m_entries.empty() + ? 100 + : static_cast(m_entryindex * 100 / m_entries.size()) ); + } + if (mSize <= 0) + { + return 100; + } return static_cast(mStream->rdbuf()->pubseekoff(0, std::ios_base::cur) * 100 / mSize); } diff --git a/utilities/parser.h b/utilities/parser.h index 6bac77e7..cf5da7ea 100644 --- a/utilities/parser.h +++ b/utilities/parser.h @@ -14,6 +14,17 @@ http://mozilla.org/MPL/2.0/. #include #include #include +#include +#include + +// binary scenery twin support (full definitions in scene/scenerybinary.h, included +// from parser.cpp). only forward declarations here to avoid pulling utilities.h in +// before cParser is defined. +namespace scene { + enum class scenery_file_kind : std::uint8_t; + struct scenery_entry; + class scenery_binary_writer; +} ///////////////////////////////////////////////////////////////////////////////////////////////////// // cParser -- generic class for parsing text data, either from file or provided string @@ -56,6 +67,11 @@ class cParser //: public std::stringstream inline bool eof() { + // in replay mode there is no character stream; exhaustion is reaching the + // end of the twin's entries with no include still being drained + if( m_replay ) { + return ( m_entryindex >= m_entrycount ) && ( !mIncludeParser ); + } return mStream->eof(); }; inline bool @@ -84,6 +100,11 @@ class cParser //: public std::stringstream // inject string as internal include void injectString(const std::string &str); + // writes the compiled binary twin to disk if this parser was compiling one and the + // source was fully consumed. safe to call multiple times; the destructor also calls + // it as a fallback. used to flush the top-level twin promptly once loading finishes. + void flushBinaryTwin(); + // returns percentage of file processed so far int getProgress() const; int getFullProgress() const; @@ -102,8 +123,17 @@ class cParser //: public std::stringstream bool skipComments = true; private: - void startIncludeFromParser(cParser &srcParser, bool ToLower, std::string includefile); + // reads an include directive (filename expression + parameters) from srcParser, + // records it for the binary twin when compiling, and opens the include with a + // freshly evaluated filename (re-randomizing any random set). + void processInclude(cParser &srcParser, bool ToLower); + // opens an include directly from a (filename, parameters) pair, as reconstructed + // from a binary twin's include entry, minus the stream-driven parameter parsing. + void startIncludeDirect(std::string includefile, std::vector Params); bool handleIncludeIfPresent(std::string &token, bool ToLower, const char *Break); + // serves the next token when replaying a binary twin, transparently entering + // child includes when an include entry is reached. + void readReplayToken(std::string &out, bool ToLower, const char *Break); // methods: void readToken(std::string& out, bool ToLower = true, const char *Break = "\n\r\t ;"); static std::vector readParameters( cParser &Input ); @@ -129,6 +159,19 @@ class cParser //: public std::stringstream std::shared_ptr mIncludeParser; // child class to handle include directives. std::vector parameters; // parameter list for included file. std::deque tokens; + // --- binary scenery twin support --- + // replay: this file is served from its binary twin instead of text + bool m_replay { false }; + std::vector m_entries; // entries loaded from the twin (replay) + std::size_t m_entryindex { 0 }; // read cursor into m_entries + std::size_t m_entrycount { 0 }; // m_entries.size(), cached for inline eof() + // compile: this file is being captured into a binary twin alongside the text read + bool m_compiling { false }; + std::unique_ptr m_writer; + std::string m_binarytwinpath; // destination path for the compiled twin + scene::scenery_file_kind m_binarykind {}; // value-initialised (== scenery_file_kind::scn) + bool m_capturesuppress { false }; // suppress token capture (include directive internals) + bool m_twinwritten { false }; // guards against writing the twin more than once }; diff --git a/utilities/utilities.cpp b/utilities/utilities.cpp index 6ee5ffe6..072b93fc 100644 --- a/utilities/utilities.cpp +++ b/utilities/utilities.cpp @@ -471,4 +471,54 @@ std::string deserialize_random_set(cParser &Input, char const *Break) // shouldn't ever get here but, eh return ""; } +} + +std::string deserialize_random_set_capture(cParser &Input, std::vector &Raw, char const *Break) +{ + auto token{Input.getToken(true, Break)}; + std::replace(token.begin(), token.end(), '\\', '/'); + Raw.emplace_back(token); // record the raw token verbatim (incl. brackets / "") + if (token != "[") + { + // simple case, single token + return token; + } + // random set: recurse for entries, recording every token, until the closing ']' + std::vector tokens; + std::string entry; + while (((entry = deserialize_random_set_capture(Input, Raw, Break)) != "") && (entry != "]")) + { + tokens.emplace_back(entry); + } + if (false == tokens.empty()) + { + std::shuffle(std::begin(tokens), std::end(tokens), Global.random_engine); + return tokens.front(); + } + return ""; +} + +std::string resolve_random_set(std::vector const &Raw, std::size_t &Pos) +{ + if (Pos >= Raw.size()) + { + return ""; + } + auto token{Raw[Pos++]}; + if (token != "[") + { + return token; + } + std::vector tokens; + std::string entry; + while (((entry = resolve_random_set(Raw, Pos)) != "") && (entry != "]")) + { + tokens.emplace_back(entry); + } + if (false == tokens.empty()) + { + std::shuffle(std::begin(tokens), std::end(tokens), Global.random_engine); + return tokens.front(); + } + return ""; } \ No newline at end of file diff --git a/utilities/utilities.h b/utilities/utilities.h index 5097173f..b94c7ce7 100644 --- a/utilities/utilities.h +++ b/utilities/utilities.h @@ -348,6 +348,15 @@ glm::dvec3 LoadPoint(class cParser &Input); // extracts a group of tokens from provided data stream std::string deserialize_random_set(cParser &Input, char const *Break = "\n\r\t ;"); +// like deserialize_random_set, but additionally records every raw token it consumes +// (including any random-set brackets) into Raw, so the same expression can be replayed +// and re-randomized later. returns the randomly chosen entry, as deserialize_random_set. +std::string deserialize_random_set_capture(cParser &Input, std::vector &Raw, char const *Break = "\n\r\t ;"); + +// re-evaluates a raw random-set expression captured by deserialize_random_set_capture, +// returning a fresh random choice. Pos is advanced through Raw. +std::string resolve_random_set(std::vector const &Raw, std::size_t &Pos); + // extracts a group of pairs from provided data stream // NOTE: expects no more than single pair per line template void deserialize_map(MapType_ &Map, cParser &Input)