eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-18 00:49:19 +02:00
Code Issues 121 Wiki Activity

Add binary scenery format (eu7 v3) with async baking, replacing SBT

Browse Source 
Text scenery component files (.scn/.inc/.scm/.ctr) are compiled into
per-file binary twins (.scnb/.incb/.scmb/.ctrb), handled transparently
at the cParser layer: a fresh twin (mtime-checked, version-matched) is
replayed instead of re-tokenizing text; otherwise the text is parsed and
a twin is compiled alongside it for next time.

Format details:
- per-file string interning: keywords/paths stored once, referenced by
  varint index (so node/endmodel/... are not repeated as text)
- numeric tokens stored as 8-byte IEEE doubles, not ASCII
- includes kept as references with parameters; random sets stored verbatim
  and re-evaluated on every load (choice not frozen at compile time)
- twin writing offloaded to a bounded thread pool so baking overlaps
  scene construction instead of blocking the load

The legacy terrain-only .sbt path is removed; terrain now loads as
ordinary scenery content.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
maj00r
2026-06-21 21:37:36 +02:00
parent 1aac9b4e76
commit 65fd1c10f3
12 changed files with 891 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@@ -149,6 +149,7 @@ set(SOURCES
"input/messaging.cpp"
"scene/scene.cpp"
"scene/scenenode.cpp"
"scene/scenerybinary.cpp"
"simulation/simulation.cpp"
"model/vertex.cpp"
"audio/audio.cpp"

120
scene/scene.cpp
View File

@@ -24,9 +24,6 @@ http://mozilla.org/MPL/2.0/.
namespace scene {
std::string const EU07_FILEEXTENSION_REGION { ".sbt" };
std::uint32_t const EU07_FILEHEADER { MAKE_ID4( 'E','U','0','7' ) };
std::uint32_t const EU07_FILEVERSION_REGION { MAKE_ID4( 'S', 'B', 'T', '2' ) };
std::map<std::string, basic_node *> Hierarchy;
// potentially activates event handler with the same name as provided node, and within handler activation range
@@ -1105,123 +1102,6 @@ basic_region::update_traction( TDynamicObject *Vehicle, int const Pantographinde
}
}
// checks whether specified file is a valid region data file
bool
basic_region::is_scene( std::string const &Scenariofile ) const {
auto filename { Scenariofile };
while( filename[ 0 ] == '$' ) {
// trim leading $ char rainsted utility may add to the base name for modified .scn files
filename.erase( 0, 1 );
}
erase_extension( filename );
filename = Global.asCurrentSceneryPath + filename;
filename += EU07_FILEEXTENSION_REGION;
if( false == FileExists( filename ) ) {
return false;
}
// file type and version check
std::ifstream input( filename, std::ios::binary );
uint32_t headermain{ sn_utils::ld_uint32( input ) };
uint32_t headertype{ sn_utils::ld_uint32( input ) };
if( ( headermain != EU07_FILEHEADER
|| ( headertype != EU07_FILEVERSION_REGION ) ) ) {
// wrong file type
return false;
}
return true;
}
// stores content of the class in file with specified name
void
basic_region::serialize( std::string const &Scenariofile ) const {
auto filename { Scenariofile };
while( filename[ 0 ] == '$' ) {
// trim leading $ char rainsted utility may add to the base name for modified .scn files
filename.erase( 0, 1 );
}
erase_extension( filename );
filename = Global.asCurrentSceneryPath + filename;
filename += EU07_FILEEXTENSION_REGION;
std::ofstream output { filename, std::ios::binary };
// region file version 1
// header: EU07SBT + version (0-255)
sn_utils::ls_uint32( output, EU07_FILEHEADER );
sn_utils::ls_uint32( output, EU07_FILEVERSION_REGION );
// sections
// TBD, TODO: build table of sections and file offsets, if we postpone section loading until they're within range
std::uint32_t sectioncount { 0 };
for( auto *section : m_sections ) {
if( section != nullptr ) {
++sectioncount;
}
}
// section count, followed by section data
sn_utils::ls_uint32( output, sectioncount );
std::uint32_t sectionindex { 0 };
for( auto *section : m_sections ) {
// section data: section index, followed by length of section data, followed by section data
if( section != nullptr ) {
sn_utils::ls_uint32( output, sectionindex );
section->serialize( output ); }
++sectionindex;
}
}
// restores content of the class from file with specified name. returns: true on success, false otherwise
bool
basic_region::deserialize( std::string const &Scenariofile ) {
auto filename { Scenariofile };
while( filename[ 0 ] == '$' ) {
// trim leading $ char rainsted utility may add to the base name for modified .scn files
filename.erase( 0, 1 );
}
erase_extension( filename );
filename = Global.asCurrentSceneryPath + filename;
filename += EU07_FILEEXTENSION_REGION;
if( false == FileExists( filename ) ) {
Global.file_binary_terrain_state = false;
return false;
}
// region file version 1
// file type and version check
std::ifstream input( filename, std::ios::binary );
uint32_t headermain { sn_utils::ld_uint32( input ) };
uint32_t headertype { sn_utils::ld_uint32( input ) };
if( ( headermain != EU07_FILEHEADER
|| ( headertype != EU07_FILEVERSION_REGION ) ) ) {
// wrong file type
WriteLog( "Bad file: \"" + filename + "\" is of either unrecognized type or version" );
return false;
}
// sections
// TBD, TODO: build table of sections and file offsets, if we postpone section loading until they're within range
// section count
auto sectioncount { sn_utils::ld_uint32( input ) };
while( sectioncount-- ) {
// section index, followed by section data size, followed by section data
auto const sectionindex { sn_utils::ld_uint32( input ) };
auto const sectionsize { sn_utils::ld_uint32( input ) };
if( m_sections[ sectionindex ] == nullptr ) {
m_sections[ sectionindex ] = new basic_section();
}
m_sections[ sectionindex ]->deserialize( input );
}
return true;
}
// sends content of the class in legacy (text) format to provided stream
void
basic_region::export_as_text( std::ostream &Output ) const {

16
scene/scene.h
View File

@@ -37,12 +37,6 @@ int constexpr EU07_REGIONSIDESECTIONCOUNT = 500; // number of sections along a s
struct scratch_data {
struct binary_data {
bool terrain{ false };
bool terrain_included{false};
} binary;
struct location_data {
std::stack<glm::dvec3> offset;
@@ -68,7 +62,6 @@ struct scratch_data {
} trainset;
std::string name;
std::string terrain_name;
bool initialized { false };
bool time_initialized { false };
@@ -386,15 +379,6 @@ public:
// legacy method, updates sounds around camera
void
update_sounds();
// checks whether specified file is a valid region data file
bool
is_scene( std::string const &Scenariofile ) const;
// stores content of the class in file with specified name
void
serialize( std::string const &Scenariofile ) const;
// restores content of the class from file with specified name. returns: true on success, false otherwise
bool
deserialize( std::string const &Scenariofile );
// sends content of the class in legacy (text) format to provided stream
void
export_as_text( std::ostream &Output ) const;

317
scene/scenerybinary.cpp Normal file
View File

@@ -0,0 +1,317 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#include "stdafx.h"
#include "scene/scenerybinary.h"
#include "scene/sn_utils.h"
#include "utilities/utilities.h" // ToLower
#include "utilities/Logs.h"
#include <unordered_map>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <queue>
#include <functional>
#include <fstream>
namespace scene {
namespace {
// LEB128 unsigned varint: 1 byte for values < 128 (covers most string-table indices),
// keeping keyword/index references compact.
void write_varint( std::ostream &Output, std::uint64_t Value ) {
do {
std::uint8_t byte = Value & 0x7F;
Value >>= 7;
if( Value != 0 ) { byte |= 0x80; }
Output.put( static_cast<char>( byte ) );
} while( Value != 0 );
}
std::uint64_t read_varint( std::istream &Input ) {
std::uint64_t value = 0;
int shift = 0;
while( shift < 64 ) {
int const c = Input.get();
if( c == std::char_traits<char>::eof() ) { break; }
value |= ( static_cast<std::uint64_t>( c & 0x7F ) << shift );
if( ( c & 0x80 ) == 0 ) { break; }
shift += 7;
}
return value;
}
// builds the per-file string table while interning, returning each string's index
class string_interner {
public:
std::uint32_t intern( std::string const &Text ) {
auto const it = m_lookup.find( Text );
if( it != m_lookup.end() ) { return it->second; }
auto const index = static_cast<std::uint32_t>( m_table.size() );
m_lookup.emplace( Text, index );
m_table.emplace_back( Text );
return index;
}
std::vector<std::string> const &table() const { return m_table; }
private:
std::unordered_map<std::string, std::uint32_t> m_lookup;
std::vector<std::string> m_table;
};
} // anonymous namespace
void
scenery_binary_writer::add_token( std::string Token ) {
scenery_entry entry;
entry.type = scenery_entry_type::token;
entry.text = std::move( Token );
m_entries.emplace_back( std::move( entry ) );
}
void
scenery_binary_writer::add_number( double Value ) {
scenery_entry entry;
entry.type = scenery_entry_type::number;
entry.number = Value;
m_entries.emplace_back( std::move( entry ) );
}
void
scenery_binary_writer::add_include( std::vector<std::string> Fileexpr, std::vector<std::string> Params ) {
scenery_entry entry;
entry.type = scenery_entry_type::include;
entry.fileexpr = std::move( Fileexpr );
entry.params = std::move( Params );
m_entries.emplace_back( std::move( entry ) );
}
bool
scenery_binary_writer::write( std::ostream &Output, scenery_file_kind Kind ) const {
// first pass: intern every string the entries reference so keywords/paths that
// repeat are stored once and referenced by index
string_interner interner;
std::vector<std::uint32_t> tokenindex; // index per token entry, in order
std::vector<std::vector<std::uint32_t>> fileexpridx, paramidx; // per include entry
for( auto const &entry : m_entries ) {
if( entry.type == scenery_entry_type::token ) {
tokenindex.emplace_back( interner.intern( entry.text ) );
}
else if( entry.type == scenery_entry_type::include ) {
std::vector<std::uint32_t> fe, pe;
for( auto const &token : entry.fileexpr ) { fe.emplace_back( interner.intern( token ) ); }
for( auto const &param : entry.params ) { pe.emplace_back( interner.intern( param ) ); }
fileexpridx.emplace_back( std::move( fe ) );
paramidx.emplace_back( std::move( pe ) );
}
}
// header: magic + kind + flags
sn_utils::ls_uint32( Output, SCENERYBINARY_MAGIC );
sn_utils::s_uint8( Output, static_cast<std::uint8_t>( Kind ) );
sn_utils::ls_uint32( Output, 0 ); // flags, reserved
// string table
auto const &table = interner.table();
sn_utils::ls_uint32( Output, static_cast<std::uint32_t>( table.size() ) );
for( auto const &text : table ) {
sn_utils::s_str( Output, text );
}
// entries
sn_utils::ls_uint32( Output, static_cast<std::uint32_t>( m_entries.size() ) );
std::size_t tokencursor = 0, includecursor = 0;
for( auto const &entry : m_entries ) {
sn_utils::s_uint8( Output, static_cast<std::uint8_t>( entry.type ) );
if( entry.type == scenery_entry_type::include ) {
auto const &fe = fileexpridx[ includecursor ];
auto const &pe = paramidx[ includecursor ];
++includecursor;
write_varint( Output, fe.size() );
for( auto idx : fe ) { write_varint( Output, idx ); }
write_varint( Output, pe.size() );
for( auto idx : pe ) { write_varint( Output, idx ); }
}
else if( entry.type == scenery_entry_type::number ) {
sn_utils::ls_float64( Output, entry.number );
}
else {
write_varint( Output, tokenindex[ tokencursor++ ] );
}
}
return ( false == Output.fail() );
}
bool
scenery_binary_reader::open( std::istream &Input ) {
m_entries.clear();
auto const magic { sn_utils::ld_uint32( Input ) };
if( magic != SCENERYBINARY_MAGIC ) {
// unrecognized type or incompatible version
return false;
}
m_kind = static_cast<scenery_file_kind>( sn_utils::d_uint8( Input ) );
sn_utils::ld_uint32( Input ); // flags, reserved
// string table
std::vector<std::string> table;
auto tablesize { sn_utils::ld_uint32( Input ) };
table.reserve( tablesize );
while( tablesize-- ) {
table.emplace_back( sn_utils::d_str( Input ) );
}
// resolves an interned index back to its string, guarding against corruption
auto const resolve = [ & ]( std::uint64_t index ) -> std::string {
return ( index < table.size() ) ? table[ static_cast<std::size_t>( index ) ] : std::string();
};
auto count { sn_utils::ld_uint32( Input ) };
m_entries.reserve( count );
while( count-- ) {
scenery_entry entry;
entry.type = static_cast<scenery_entry_type>( sn_utils::d_uint8( Input ) );
if( entry.type == scenery_entry_type::include ) {
auto expr { read_varint( Input ) };
entry.fileexpr.reserve( expr );
while( expr-- ) {
entry.fileexpr.emplace_back( resolve( read_varint( Input ) ) );
}
auto params { read_varint( Input ) };
entry.params.reserve( params );
while( params-- ) {
entry.params.emplace_back( resolve( read_varint( Input ) ) );
}
}
else if( entry.type == scenery_entry_type::number ) {
entry.number = sn_utils::ld_float64( Input );
}
else {
entry.text = resolve( read_varint( Input ) );
}
m_entries.emplace_back( std::move( entry ) );
}
return ( false == Input.fail() );
}
namespace {
// minimal bounded thread pool for offloading twin serialization/writing
class bake_pool {
public:
bake_pool() {
unsigned const workers = std::max( 2u, std::min( 8u, std::thread::hardware_concurrency() ) );
for( unsigned i = 0; i < workers; ++i ) {
m_threads.emplace_back( [ this ] { run(); } );
}
}
~bake_pool() {
{ std::unique_lock<std::mutex> lock( m_mutex ); m_stop = true; }
m_wake.notify_all();
for( auto &thread : m_threads ) { if( thread.joinable() ) { thread.join(); } }
}
void enqueue( std::function<void()> Task ) {
{ std::unique_lock<std::mutex> lock( m_mutex ); m_tasks.emplace( std::move( Task ) ); ++m_pending; }
m_wake.notify_one();
}
void wait_idle() {
std::unique_lock<std::mutex> lock( m_mutex );
m_idle.wait( lock, [ this ] { return m_pending == 0; } );
}
private:
void run() {
for( ;; ) {
std::function<void()> task;
{
std::unique_lock<std::mutex> lock( m_mutex );
m_wake.wait( lock, [ this ] { return m_stop || ( false == m_tasks.empty() ); } );
if( m_stop && m_tasks.empty() ) { return; }
task = std::move( m_tasks.front() );
m_tasks.pop();
}
task();
{
std::unique_lock<std::mutex> lock( m_mutex );
if( --m_pending == 0 ) { m_idle.notify_all(); }
}
}
}
std::vector<std::thread> m_threads;
std::queue<std::function<void()>> m_tasks;
std::mutex m_mutex;
std::condition_variable m_wake;
std::condition_variable m_idle;
bool m_stop { false };
std::size_t m_pending { 0 };
};
bake_pool &pool() {
static bake_pool instance;
return instance;
}
// results recorded by worker threads, drained and logged on the main thread
std::mutex g_resultmutex;
std::vector<std::string> g_resultlog; // success lines, in completion order
std::vector<std::string> g_resultfail; // failure paths
} // anonymous namespace
void
scenerybinary_write_async( std::unique_ptr<scenery_binary_writer> Writer, std::string Path, scenery_file_kind Kind ) {
// std::function requires a copyable target, so move the writer into a shared_ptr
std::shared_ptr<scenery_binary_writer> writer { std::move( Writer ) };
pool().enqueue( [ writer, path = std::move( Path ), Kind ] {
std::ofstream output( path, std::ios::binary );
bool const ok = output.good() && writer->write( output, Kind );
output.flush();
std::lock_guard<std::mutex> lock( g_resultmutex );
if( ok ) {
g_resultlog.emplace_back( "Compiled binary scenery: " + path + " (" + std::to_string( writer->entry_count() ) + " entries)" );
}
else {
g_resultfail.emplace_back( path );
}
} );
}
void
scenerybinary_wait_all() {
pool().wait_idle();
std::lock_guard<std::mutex> lock( g_resultmutex );
for( auto const &line : g_resultlog ) { WriteLog( line ); }
for( auto const &path : g_resultfail ) { ErrorLog( "Failed to write binary scenery \"" + path + "\"" ); }
g_resultlog.clear();
g_resultfail.clear();
}
std::string
scenerybinary_extension_for( std::string const &Sourcefile ) {
auto const lower { ToLower( Sourcefile ) };
if( lower.size() >= 4 && lower.compare( lower.size() - 4, 4, ".inc" ) == 0 ) {
return SCENERYBINARY_EXT_INC;
}
if( lower.size() >= 4 && lower.compare( lower.size() - 4, 4, ".scm" ) == 0 ) {
return SCENERYBINARY_EXT_SCM;
}
if( lower.size() >= 4 && lower.compare( lower.size() - 4, 4, ".ctr" ) == 0 ) {
return SCENERYBINARY_EXT_CTR;
}
// default: treat as a top-level scenario file (.scn)
return SCENERYBINARY_EXT_SCN;
}
} // namespace scene

136
scene/scenerybinary.h Normal file
View File

@@ -0,0 +1,136 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#pragma once
#include <cstdint>
#include <string>
#include <vector>
#include <iosfwd>
#include <memory>
#include "utilities/utilities.h" // MAKE_ID4
// Binary, modular scenery container ("eu7" format, version 1).
//
// Replaces the legacy text formats (.scn / .scm / .inc) and the old terrain-only
// .sbt blob. The mapping is one-to-one and modular: every text file compiles to
// exactly one binary twin and includes are preserved as references, so the binary
// twin of one file points to the binary twins of the files it includes.
// route.scn -> route.scnb
// piece.inc -> piece.incb
// mesh.scm -> mesh.scmb
//
// A binary twin stores the source file's own, fully resolved content (comments
// stripped, parameters and random sets resolved) as an ordered list of entries.
// Numeric tokens are stored as 8-byte IEEE doubles (so coordinates, ranges etc. are
// genuinely binary, not ASCII). All string tokens (keywords like "node"/"endmodel",
// names, paths) are interned into a per-file string table and referenced from the
// entries by a small varint index, so a keyword that occurs thousands of times is
// stored once. An "include" entry marks the spot where the source file pulled in
// another file, carrying the (interned) include filename expression and parameters.
//
// The binary is consumed transparently at the cParser file layer: opening a file
// whose binary twin exists serves tokens from the twin instead of tokenizing text,
// and an include entry triggers the same include machinery as the text path (which
// recursively prefers the included file's own binary twin). The scenery
// deserializer is unaware of the format and needs no changes; the file's identity
// (cParser::Name()), node grouping per .inc and parameter substitution are all
// preserved exactly as in the text path.
namespace scene {
// "eu7" + format version, little-endian via MAKE_ID4 ('e','u','7',<ver>).
// v2 introduced typed numeric entries (f64); v3 interns string tokens into a per-file
// table referenced by varint indices. bumping the version invalidates older twins so
// they are recompiled rather than misread.
constexpr std::uint32_t SCENERYBINARY_MAGIC { MAKE_ID4( 'e', 'u', '7', 3 ) };
// file extension of the binary twin, derived from source kind
std::string const SCENERYBINARY_EXT_SCN { ".scnb" };
std::string const SCENERYBINARY_EXT_INC { ".incb" };
std::string const SCENERYBINARY_EXT_SCM { ".scmb" };
std::string const SCENERYBINARY_EXT_CTR { ".ctrb" };
// which text format the binary file was compiled from
enum class scenery_file_kind : std::uint8_t {
scn = 0,
inc = 1,
scm = 2,
};
// kind of a stored entry
enum class scenery_entry_type : std::uint8_t {
token = 0, // a non-numeric resolved token (name/path/keyword), stored as a string
include = 1, // an include directive: pulls in another file with parameters
number = 2, // a numeric token, stored as an 8-byte IEEE double
};
// a single ordered element of a file's resolved content
struct scenery_entry {
scenery_entry_type type { scenery_entry_type::token };
// token value (token entries only)
std::string text;
// numeric value (number entries only)
double number { 0.0 };
// include only: the raw filename expression as written in the source, i.e. either
// a single filename token or a random set "[ a b c ]" (including the brackets).
// kept verbatim so the random choice is re-evaluated on every replay rather than
// frozen at compile time.
std::vector<std::string> fileexpr;
// include only: directive parameters
std::vector<std::string> params;
};
// accumulates a file's entries and writes them, with a header, to a stream
class scenery_binary_writer {
public:
void add_token( std::string Token );
// stores a numeric token as a typed double
void add_number( double Value );
// Fileexpr is the verbatim filename expression (single token or random set)
void add_include( std::vector<std::string> Fileexpr, std::vector<std::string> Params );
std::size_t entry_count() const { return m_entries.size(); }
// serializes header + entries to the stream. returns false on stream failure.
bool write( std::ostream &Output, scenery_file_kind Kind ) const;
private:
std::vector<scenery_entry> m_entries;
};
// reads a binary scenery file fully into memory and exposes its entries in order
class scenery_binary_reader {
public:
// validates magic/version and loads every entry. returns false if the stream is
// not a valid current-version binary scenery file.
bool open( std::istream &Input );
scenery_file_kind kind() const { return m_kind; }
std::vector<scenery_entry> const &entries() const { return m_entries; }
private:
scenery_file_kind m_kind { scenery_file_kind::scn };
std::vector<scenery_entry> m_entries;
};
// maps a source scenery filename to the extension of its binary twin
std::string scenerybinary_extension_for( std::string const &Sourcefile );
// Asynchronous baking: serializing and writing a twin is pure, self-contained work
// (it only needs the writer's collected entries), so it is offloaded to a small thread
// pool and overlaps with the rest of scene construction instead of blocking the load.
// Takes ownership of the writer.
void scenerybinary_write_async( std::unique_ptr<scenery_binary_writer> Writer, std::string Path, scenery_file_kind Kind );
// Blocks until every queued async write has finished, then emits their log lines on the
// calling (main) thread. Call once the scenario has finished loading.
void scenerybinary_wait_all();
} // namespace scene

136
simulation/simulationstateserializer.cpp
View File

@@ -16,6 +16,7 @@ http://mozilla.org/MPL/2.0/.
#include "simulation/simulationsounds.h"
#include "simulation/simulationenvironment.h"
#include "scene/scenenodegroups.h"
#include "scene/scenerybinary.h"
#include "rendering/particles.h"
#include "world/Event.h"
#include "world/MemCell.h"
@@ -41,30 +42,13 @@ state_serializer::deserialize_begin( std::string const &Scenariofile ) {
simulation::State.init_scripting_interface();
// NOTE: for the time being import from text format is a given, since we don't have full binary serialization
std::shared_ptr<deserializer_state> state =
std::make_shared<deserializer_state>(Scenariofile, cParser::buffer_FILE, Global.asCurrentSceneryPath, Global.bLoadTraction);
// TODO: check first for presence of serialized binary files
// if this fails, fall back on the legacy text format
state->scratchpad.name = Scenariofile;
if( ( true == Global.file_binary_terrain )
&& ( Scenariofile != "$.scn" ) ) {
// compilation to binary file isn't supported for rainsted-created overrides
// NOTE: we postpone actual loading of the scene until we process time, season and weather data
state->scratchpad.binary.terrain = Region->is_scene( Scenariofile ) ;
}
if (false != state->scratchpad.binary.terrain)
{
Global.file_binary_terrain_state = true;
WriteLog("Default SBT present");
}
else
{
Global.file_binary_terrain_state = false;
WriteLog("Default SBT absent");
}
// open the scenario file. binary scenery twins (.scnb/.incb/.scmb) are handled
// transparently inside cParser: if a twin exists it is replayed instead of the
// text, otherwise the text is parsed and a twin compiled alongside it.
std::shared_ptr<deserializer_state> state =
std::make_shared<deserializer_state>( Scenariofile, cParser::buffer_FILE, Global.asCurrentSceneryPath, Global.bLoadTraction );
state->scenariofile = Scenariofile;
state->scratchpad.name = Scenariofile;
scene::Groups.create();
if( false == state->input.ok() )
@@ -152,13 +136,12 @@ state_serializer::deserialize_continue(std::shared_ptr<deserializer_state> state
scene::Groups.update_map();
Region->create_map_geometry();
if( ( true == Global.file_binary_terrain )
&& ( false == state->scratchpad.binary.terrain )
&& ( state->scenariofile != "$.scn" ) ) {
// if we didn't find usable binary version of the scenario files, create them now for future use
// as long as the scenario file wasn't rainsted-created base file override
Region->serialize( state->scenariofile );
}
// loading finished: flush the top-level scenario's binary twin now rather than
// waiting for the parser to be destroyed (the loader keeps the state around)
Input.flushBinaryTwin();
// wait out any background twin writes (includes) so they are complete and logged
// before we report the scenario as loaded
scene::scenerybinary_wait_all();
return false;
}
@@ -361,16 +344,6 @@ state_serializer::deserialize_firstinit( cParser &Input, scene::scratch_data &Sc
if( true == Scratchpad.initialized ) { return; }
if( true == Scratchpad.binary.terrain ) {
// at this stage it should be safe to import terrain from the binary scene file
// TBD: postpone loading furter and only load required blocks during the simulation?
if (false == Scratchpad.binary.terrain_included)
{
Region->deserialize(Scratchpad.name);
}
}
simulation::Paths.InitTracks();
simulation::Traction.InitTraction();
simulation::Events.InitEvents();
@@ -500,37 +473,30 @@ state_serializer::deserialize_node( cParser &Input, scene::scratch_data &Scratch
else if( nodedata.type == "model" ) {
if( nodedata.range_min < 0.0 ) {
// 3d terrain
if( false == Scratchpad.binary.terrain ) {
// if we're loading data from text .scn file convert and import
auto *instance = deserialize_model( Input, Scratchpad, nodedata );
// model import can potentially fail
if( instance == nullptr ) { return; }
// go through submodels, and import them as shapes
auto const cellcount = instance->TerrainCount() + 1; // zliczenie submodeli
for( auto i = 1; i < cellcount; ++i ) {
auto *submodel = instance->TerrainSquare( i - 1 );
// 3d terrain: convert the model's submodels into region shapes
auto *instance = deserialize_model( Input, Scratchpad, nodedata );
// model import can potentially fail
if( instance == nullptr ) { return; }
// go through submodels, and import them as shapes
auto const cellcount = instance->TerrainCount() + 1; // zliczenie submodeli
for( auto i = 1; i < cellcount; ++i ) {
auto *submodel = instance->TerrainSquare( i - 1 );
simulation::Region->insert(
scene::shape_node().convert( submodel ),
Scratchpad,
false );
// if there's more than one group of triangles in the cell they're held as children of the primary submodel
submodel = submodel->ChildGet();
while( submodel != nullptr ) {
simulation::Region->insert(
scene::shape_node().convert( submodel ),
Scratchpad,
false );
// if there's more than one group of triangles in the cell they're held as children of the primary submodel
submodel = submodel->ChildGet();
while( submodel != nullptr ) {
simulation::Region->insert(
scene::shape_node().convert( submodel ),
Scratchpad,
false );
submodel = submodel->NextGet();
}
submodel = submodel->NextGet();
}
// with the import done we can get rid of the source model
delete instance;
}
else {
// if binary terrain file was present, we already have this data
skip_until( Input, "endmodel" );
}
// with the import done we can get rid of the source model
delete instance;
}
else {
// regular instance of 3d mesh
@@ -563,10 +529,8 @@ state_serializer::deserialize_node( cParser &Input, scene::scratch_data &Scratch
|| ( nodedata.type == "triangle_fan" ) ) {
auto const skip {
// all shapes will be loaded from the binary version of the file
( true == Scratchpad.binary.terrain )
// crude way to detect fixed switch trackbed geometry
|| ( ( true == Global.CreateSwitchTrackbeds )
( ( true == Global.CreateSwitchTrackbeds )
&& ( Input.Name().size() >= 15 )
&& Input.Name().starts_with("scenery/zwr")
&& Input.Name().ends_with(".inc") ) };
@@ -587,17 +551,10 @@ state_serializer::deserialize_node( cParser &Input, scene::scratch_data &Scratch
|| ( nodedata.type == "line_strip" )
|| ( nodedata.type == "line_loop" ) ) {
if( false == Scratchpad.binary.terrain ) {
simulation::Region->insert(
scene::lines_node().import(
Input, nodedata ),
Scratchpad );
}
else {
// all lines were already loaded from the binary version of the file
skip_until( Input, "endline" );
}
simulation::Region->insert(
scene::lines_node().import(
Input, nodedata ),
Scratchpad );
}
else if( nodedata.type == "memcell" ) {
@@ -785,22 +742,9 @@ state_serializer::deserialize_trainset( cParser &Input, scene::scratch_data &Scr
void
state_serializer::deserialize_terrain(cParser &Input, scene::scratch_data &Scratchpad)
{
std::string line;
Input.getTokens(1);
Input >> line;
if (Global.file_binary_terrain && line.ends_with(".sbt"))
{
Scratchpad.binary.terrain = Region->is_scene(line);
Global.file_binary_terrain_state = true;
Scratchpad.binary.terrain_included = true;
Scratchpad.terrain_name = line;
WriteLog("Included SBT file: " + line);
Region->deserialize(Scratchpad.terrain_name);
}
skip_until(Input, "endterrain");
// legacy directive; the SBT terrain blob has been retired and terrain now loads
// as ordinary scenery content, so the block is simply consumed.
skip_until(Input, "endterrain");
}
void

6
utilities/Globals.cpp
View File

@@ -632,9 +632,9 @@ bool global_settings::ConfigParseSimulation(cParser& Parser, const std::string&
return true;
}
if (token == "file.binary.terrain")
if (token == "file.binary.scenery")
{
ParseOne(Parser, file_binary_terrain, 1, false);
ParseOne(Parser, file_binary_scenery, 1, false);
return true;
}
@@ -1587,7 +1587,7 @@ global_settings::export_as_text( std::ostream &Output ) const {
export_as_text( Output, "multisampling", iMultisampling );
export_as_text( Output, "latitude", fLatitudeDeg );
export_as_text( Output, "convertmodels", iConvertModels );
export_as_text( Output, "file.binary.terrain", file_binary_terrain );
export_as_text( Output, "file.binary.scenery", file_binary_scenery );
export_as_text( Output, "inactivepause", bInactivePause );
export_as_text( Output, "slowmotion", iSlowMotionMask );
export_as_text( Output, "hideconsole", bHideConsole );

3
utilities/Globals.h
View File

@@ -80,8 +80,7 @@ struct global_settings {
std::string local_start_vehicle{ "EU07-424" };
int iConvertModels{ 0 }; // tworzenie plików binarnych
int iConvertIndexRange{ 1000 }; // range of duplicate vertex scan
bool file_binary_terrain{ true }; // enable binary terrain (de)serialization
bool file_binary_terrain_state{true};
bool file_binary_scenery{ true }; // enable binary scenery twins (.scnb/.incb/.scmb)
// logs
bool priorityLoadText3D{false}; // ladowanie T3D priorytetowo
int iWriteLogEnabled{ 3 }; // maska bitowa: 1-zapis do pliku, 2-okienko, 4-nazwy torów

328
utilities/parser.cpp
View File

@@ -10,8 +10,15 @@ http://mozilla.org/MPL/2.0/.
#include "stdafx.h"
#include "utilities/parser.h"
#include "utilities/Logs.h"
#include "utilities/utilities.h"
#include "utilities/Globals.h"
#include "scene/scenenodegroups.h"
#include "scene/scenerybinary.h"
#include <charconv>
#include <cmath>
#include <filesystem>
/*
MaSzyna EU07 locomotive simulator parser
@@ -46,6 +53,78 @@ inline bool startsWithBOM(const std::string &s)
&& static_cast<unsigned char>(s[1]) == 0xBB
&& static_cast<unsigned char>(s[2]) == 0xBF;
}
// true if the whole token is a finite decimal number; on success Value is set.
// used to store numeric tokens as typed doubles in the binary twin instead of text.
inline bool sniffNumber(const std::string &token, double &Value)
{
if (token.empty())
{
return false;
}
const char *first = token.data();
const char *last = token.data() + token.size();
double value = 0.0;
auto const result = std::from_chars(first, last, value);
// require the entire token to be consumed and the value to be finite (reject
// "inf"/"nan"/overflow, and identifiers like "12abc" or "1.2.3")
if ((result.ec != std::errc()) || (result.ptr != last) || (false == std::isfinite(value)))
{
return false;
}
return (Value = value, true);
}
// shortest representation of a double that round-trips back to the same value,
// used when serving a numeric twin entry to the (text-oriented) deserializer
inline std::string formatNumber(double Value)
{
char buffer[32];
auto const result = std::to_chars(buffer, buffer + sizeof(buffer), Value);
return std::string(buffer, result.ptr);
}
inline bool endsWithLower(const std::string &s, const char *suffix)
{
const std::string suf(suffix);
return s.size() >= suf.size() && ToLower(s.substr(s.size() - suf.size())) == suf;
}
// scenery source files (and only these) get binary twins
inline bool isSceneryFile(const std::string &name)
{
// text scenery component files share one syntax (per wiki: SCN/SCM/CTR/INC).
// .eu7 is intentionally excluded: it is not a documented text format (it can be an
// editor/binary file), so tokenizing it into a twin would be wrong.
return endsWithLower(name, ".scn") || endsWithLower(name, ".inc")
|| endsWithLower(name, ".scm") || endsWithLower(name, ".ctr");
}
inline scene::scenery_file_kind sceneryKind(const std::string &name)
{
if (endsWithLower(name, ".inc")) return scene::scenery_file_kind::inc;
if (endsWithLower(name, ".scm")) return scene::scenery_file_kind::scm;
return scene::scenery_file_kind::scn;
}
// the twin may be replayed only if it is at least as new as its source text, so that
// editing a scenery file forces a recompile instead of replaying a stale twin
inline bool twinIsFresh(const std::string &sourcefull, const std::string &twinfull)
{
std::error_code ec;
auto const sourcetime = std::filesystem::last_write_time(sourcefull, ec);
if (ec)
{
// can't read the source time (e.g. packed/missing): trust the twin
return true;
}
auto const twintime = std::filesystem::last_write_time(twinfull, ec);
if (ec)
{
return false;
}
return (twintime >= sourcetime);
}
} // namespace
// constructors
@@ -62,14 +141,58 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P
{
case buffer_FILE:
{
Path.append(Stream);
mStream = std::make_shared<std::ifstream>(Path, std::ios_base::binary);
// content of *.inc files is potentially grouped together
if ((Stream.size() >= 4) && (ToLower(Stream.substr(Stream.size() - 4)) == ".inc"))
// content of *.inc files is grouped together (same for text and binary replay)
if (endsWithLower(Stream, ".inc"))
{
mIncFile = true;
scene::Groups.create();
}
bool opened = false;
// scenery files (.scn/.inc/.scm) are backed by a binary twin: replay it if
// present, otherwise parse the text and compile a twin alongside it.
// rainsted-created '$' overrides are always parsed from text.
if (Global.file_binary_scenery && isSceneryFile(Stream) && (false == Stream.empty()) && (Stream[0] != '$'))
{
std::string twinrel = Stream;
erase_extension(twinrel);
twinrel += scene::scenerybinary_extension_for(Stream);
std::string const twinfull = mPath + twinrel;
std::string const sourcefull = mPath + Stream;
std::ifstream probe(twinfull, std::ios_base::binary);
scene::scenery_binary_reader reader;
if (probe.good() && twinIsFresh(sourcefull, twinfull) && reader.open(probe))
{
// replay: serve tokens from the twin, no text tokenization at all
m_entries = reader.entries();
m_entrycount = m_entries.size();
m_replay = true;
m_entryindex = 0;
mStream = std::make_shared<std::istringstream>(std::string());
opened = true;
}
else
{
Path.append(Stream);
mStream = std::make_shared<std::ifstream>(Path, std::ios_base::binary);
if (false == mStream->fail())
{
// no usable twin: compile one while parsing the text
m_compiling = true;
m_binarytwinpath = twinfull;
m_binarykind = sceneryKind(Stream);
m_writer = std::make_unique<scene::scenery_binary_writer>();
}
opened = true;
}
}
if (false == opened)
{
// non-scenery file, or binary scenery disabled: plain text parse
Path.append(Stream);
mStream = std::make_shared<std::ifstream>(Path, std::ios_base::binary);
}
break;
}
case buffer_TEXT:
@@ -106,6 +229,8 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P
// destructor
cParser::~cParser()
{
// fallback flush in case the twin wasn't explicitly finalized
flushBinaryTwin();
if (true == mIncFile)
{
@@ -114,6 +239,26 @@ cParser::~cParser()
}
}
void cParser::flushBinaryTwin()
{
// write the twin only once, only when compiling, and only if the source text was
// fully consumed -- an aborted parse must not leave a truncated twin to be replayed
if ((false == m_compiling) || (false == static_cast<bool>(m_writer)) || m_twinwritten)
{
return;
}
if ((nullptr == mStream) || (false == mStream->eof()))
{
return;
}
m_twinwritten = true;
// hand the finished writer to the background pool; serialization and file I/O then
// overlap with the rest of the scene build instead of blocking it. ownership of the
// writer transfers to the task, so this cParser no longer touches it.
scene::scenerybinary_write_async(std::move(m_writer), m_binarytwinpath, m_binarykind);
}
template <> glm::vec3 cParser::getToken(bool const ToLower, char const *Break)
{
// NOTE: this specialization ignores default arguments
@@ -313,40 +458,63 @@ void cParser::skipIncludeBlock() {
} while (t != "end" && !t.empty());
}
void cParser::startIncludeFromParser(cParser& srcParser, bool ToLower, std::string includefile) {
replace_slashes(includefile);
void cParser::processInclude(cParser& srcParser, bool ToLower) {
// the filename expression and parameters are part of the directive, not file
// content, so keep them out of this file's own captured token stream
bool const prevsuppress = m_capturesuppress;
m_capturesuppress = true;
std::vector<std::string> fileexpr;
std::string pick;
if (allowRandomIncludes) {
// capture the verbatim expression (incl. random-set brackets) and pick one
pick = deserialize_random_set_capture(srcParser, fileexpr);
}
else {
std::string filename;
srcParser.readToken(filename, ToLower);
std::replace(filename.begin(), filename.end(), '\\', '/');
fileexpr.emplace_back(filename);
pick = filename;
}
// consume the directive's parameter list (up to "end")
std::vector<std::string> params = readParameters(srcParser);
m_capturesuppress = prevsuppress;
// record the include reference verbatim so replay can re-randomize the choice
if (m_compiling && m_writer) {
m_writer->add_include(fileexpr, params);
}
// open the include for the live load with a freshly evaluated filename
replace_slashes(pick);
startIncludeDirect(std::move(pick), std::move(params));
}
void cParser::startIncludeDirect(std::string includefile, std::vector<std::string> Params) {
const bool allowTraction =
(true == LoadTraction) ||
((false == contains(includefile, "tr/")) && (false == contains(includefile, "tra/")));
if (!allowTraction) {
// skip include block until "end" (original behavior in token-mode include)
skipIncludeBlock();
return;
}
const bool isTerrain = contains(includefile, "_ter.scm");
if (isTerrain && true == Global.file_binary_terrain_state) {
WriteLog("SBT found, ignoring: " + includefile);
readParameters(srcParser); // preserve original side-effect: still consume parameters
// traction loading disabled: the include is simply not opened
return;
}
if (Global.ParserLogIncludes) {
if (isTerrain) WriteLog("including terrain: " + includefile);
else {
// WriteLog("including: " + includefile);
}
// WriteLog("including: " + includefile);
}
mIncludeParser = std::make_shared<cParser>(
includefile, /*buffer_FILE*/ static_cast<buffertype>(/*buffer_FILE*/ 0), mPath, LoadTraction, readParameters(srcParser)
includefile, buffer_FILE, mPath, LoadTraction, std::move(Params)
);
mIncludeParser->allowRandomIncludes = allowRandomIncludes;
mIncludeParser->autoclear(m_autoclear);
if (mIncludeParser->mSize <= 0) {
// a binary-twin replay child reports mSize 0 but is still valid
if (mIncludeParser->mSize <= 0 && (false == mIncludeParser->m_replay)) {
ErrorLog("Bad include: can't open file \"" + includefile + "\"");
}
}
@@ -354,14 +522,7 @@ void cParser::startIncludeFromParser(cParser& srcParser, bool ToLower, std::stri
bool cParser::handleIncludeIfPresent(std::string& token, bool ToLower, const char* Break) {
// token-mode include: token == "include"
if (expandIncludes && token == "include") {
std::string includefile;
if (allowRandomIncludes)
includefile = deserialize_random_set(*this);
else
readToken(includefile, ToLower);
startIncludeFromParser(*this, ToLower, std::move(includefile));
processInclude(*this, ToLower);
// after processing include, return next token from current parser
readToken(token, ToLower, Break);
return true;
@@ -370,14 +531,7 @@ bool cParser::handleIncludeIfPresent(std::string& token, bool ToLower, const cha
// line-mode HACK: Break == "\n\r" and line begins with "include"
if ((std::strcmp(Break, "\n\r") == 0) && token.compare(0, 7, "include") == 0) {
cParser includeparser(token.substr(7));
std::string includefile;
if (allowRandomIncludes)
includefile = deserialize_random_set(includeparser);
else
includeparser.readToken(includefile, ToLower);
startIncludeFromParser(includeparser, ToLower, std::move(includefile));
processInclude(includeparser, ToLower);
readToken(token, ToLower, Break);
return true;
}
@@ -387,6 +541,14 @@ bool cParser::handleIncludeIfPresent(std::string& token, bool ToLower, const cha
void cParser::readToken(std::string &out, bool ToLower, const char *Break)
{
if (m_replay)
{
// served from the binary twin; includes are entered transparently
readReplayToken(out, ToLower, Break);
return;
}
bool fromOwn;
if (mIncludeParser)
{
mIncludeParser->readToken(out, ToLower, Break);
@@ -394,18 +556,98 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break)
{
mIncludeParser = nullptr;
out = readTokenFromStream(ToLower, Break);
fromOwn = true;
}
else
{
fromOwn = false;
}
}
else
{
out = readTokenFromStream(ToLower, Break);
fromOwn = true;
}
stripFirstTokenBOM(out, ToLower, Break);
// snapshot the raw token (BOM stripped, parameters NOT yet substituted) for the
// binary twin, so parameterised includes can be re-substituted at replay time
std::string const rawtoken = out;
substituteParameters(out, ToLower);
handleIncludeIfPresent(out, ToLower, Break);
bool const wasInclude = handleIncludeIfPresent(out, ToLower, Break);
// capture this file's own content into its twin. include directive tokens
// (the keyword/filename/parameters) are excluded via wasInclude/suppression,
// and child-include tokens (fromOwn == false) belong to the child's own twin.
if (m_compiling && m_writer && fromOwn && (false == wasInclude)
&& (false == m_capturesuppress) && (false == rawtoken.empty()))
{
// store numeric tokens as typed doubles (genuinely binary), everything else
// (names, paths, keywords, "(pN)" placeholders) as strings
double value = 0.0;
if (sniffNumber(rawtoken, value))
{
m_writer->add_number(value);
}
else
{
m_writer->add_token(rawtoken);
}
}
}
void cParser::readReplayToken(std::string &out, bool ToLower, const char *Break)
{
// drain an active child include first, exactly like the text path
if (mIncludeParser)
{
mIncludeParser->readToken(out, ToLower, Break);
if (false == out.empty())
{
return;
}
mIncludeParser = nullptr;
}
while (m_entryindex < m_entries.size())
{
scene::scenery_entry const &entry = m_entries[m_entryindex++];
if (entry.type == scene::scenery_entry_type::token)
{
out = entry.text;
// re-apply this file's include parameters, mirroring the text path
substituteParameters(out, ToLower);
return;
}
if (entry.type == scene::scenery_entry_type::number)
{
// typed numeric entry: hand back its shortest round-tripping text form
// (no parameter substitution applies to a literal number)
out = formatNumber(entry.number);
return;
}
// include entry: re-evaluate the filename expression (re-randomizing any
// random set), then enter the child (its own twin or text) and serve its
// tokens; an empty/skipped child just advances to the next entry
std::size_t pos = 0;
std::string includefile = resolve_random_set(entry.fileexpr, pos);
replace_slashes(includefile);
startIncludeDirect(std::move(includefile), entry.params);
if (mIncludeParser)
{
mIncludeParser->readToken(out, ToLower, Break);
if (false == out.empty())
{
return;
}
mIncludeParser = nullptr;
}
}
out.clear();
}
std::vector<std::string> cParser::readParameters(cParser &Input)
@@ -523,6 +765,16 @@ void cParser::injectString(const std::string &str)
int cParser::getProgress() const
{
if (m_replay)
{
return ( m_entries.empty()
? 100
: static_cast<int>(m_entryindex * 100 / m_entries.size()) );
}
if (mSize <= 0)
{
return 100;
}
return static_cast<int>(mStream->rdbuf()->pubseekoff(0, std::ios_base::cur) * 100 / mSize);
}

45
utilities/parser.h
View File

@@ -14,6 +14,17 @@ http://mozilla.org/MPL/2.0/.
#include <fstream>
#include <vector>
#include <map>
#include <memory>
#include <cstdint>
// binary scenery twin support (full definitions in scene/scenerybinary.h, included
// from parser.cpp). only forward declarations here to avoid pulling utilities.h in
// before cParser is defined.
namespace scene {
enum class scenery_file_kind : std::uint8_t;
struct scenery_entry;
class scenery_binary_writer;
}
/////////////////////////////////////////////////////////////////////////////////////////////////////
// cParser -- generic class for parsing text data, either from file or provided string
@@ -56,6 +67,11 @@ class cParser //: public std::stringstream
inline
bool
eof() {
// in replay mode there is no character stream; exhaustion is reaching the
// end of the twin's entries with no include still being drained
if( m_replay ) {
return ( m_entryindex >= m_entrycount ) && ( !mIncludeParser );
}
return mStream->eof(); };
inline
bool
@@ -84,6 +100,11 @@ class cParser //: public std::stringstream
// inject string as internal include
void injectString(const std::string &str);
// writes the compiled binary twin to disk if this parser was compiling one and the
// source was fully consumed. safe to call multiple times; the destructor also calls
// it as a fallback. used to flush the top-level twin promptly once loading finishes.
void flushBinaryTwin();
// returns percentage of file processed so far
int getProgress() const;
int getFullProgress() const;
@@ -102,8 +123,17 @@ class cParser //: public std::stringstream
bool skipComments = true;
private:
void startIncludeFromParser(cParser &srcParser, bool ToLower, std::string includefile);
// reads an include directive (filename expression + parameters) from srcParser,
// records it for the binary twin when compiling, and opens the include with a
// freshly evaluated filename (re-randomizing any random set).
void processInclude(cParser &srcParser, bool ToLower);
// opens an include directly from a (filename, parameters) pair, as reconstructed
// from a binary twin's include entry, minus the stream-driven parameter parsing.
void startIncludeDirect(std::string includefile, std::vector<std::string> Params);
bool handleIncludeIfPresent(std::string &token, bool ToLower, const char *Break);
// serves the next token when replaying a binary twin, transparently entering
// child includes when an include entry is reached.
void readReplayToken(std::string &out, bool ToLower, const char *Break);
// methods:
void readToken(std::string& out, bool ToLower = true, const char *Break = "\n\r\t ;");
static std::vector<std::string> readParameters( cParser &Input );
@@ -129,6 +159,19 @@ class cParser //: public std::stringstream
std::shared_ptr<cParser> mIncludeParser; // child class to handle include directives.
std::vector<std::string> parameters; // parameter list for included file.
std::deque<std::string> tokens;
// --- binary scenery twin support ---
// replay: this file is served from its binary twin instead of text
bool m_replay { false };
std::vector<scene::scenery_entry> m_entries; // entries loaded from the twin (replay)
std::size_t m_entryindex { 0 }; // read cursor into m_entries
std::size_t m_entrycount { 0 }; // m_entries.size(), cached for inline eof()
// compile: this file is being captured into a binary twin alongside the text read
bool m_compiling { false };
std::unique_ptr<scene::scenery_binary_writer> m_writer;
std::string m_binarytwinpath; // destination path for the compiled twin
scene::scenery_file_kind m_binarykind {}; // value-initialised (== scenery_file_kind::scn)
bool m_capturesuppress { false }; // suppress token capture (include directive internals)
bool m_twinwritten { false }; // guards against writing the twin more than once
};

50
utilities/utilities.cpp
View File

@@ -471,4 +471,54 @@ std::string deserialize_random_set(cParser &Input, char const *Break)
// shouldn't ever get here but, eh
return "";
}
}
std::string deserialize_random_set_capture(cParser &Input, std::vector<std::string> &Raw, char const *Break)
{
auto token{Input.getToken<std::string>(true, Break)};
std::replace(token.begin(), token.end(), '\\', '/');
Raw.emplace_back(token); // record the raw token verbatim (incl. brackets / "")
if (token != "[")
{
// simple case, single token
return token;
}
// random set: recurse for entries, recording every token, until the closing ']'
std::vector<std::string> tokens;
std::string entry;
while (((entry = deserialize_random_set_capture(Input, Raw, Break)) != "") && (entry != "]"))
{
tokens.emplace_back(entry);
}
if (false == tokens.empty())
{
std::shuffle(std::begin(tokens), std::end(tokens), Global.random_engine);
return tokens.front();
}
return "";
}
std::string resolve_random_set(std::vector<std::string> const &Raw, std::size_t &Pos)
{
if (Pos >= Raw.size())
{
return "";
}
auto token{Raw[Pos++]};
if (token != "[")
{
return token;
}
std::vector<std::string> tokens;
std::string entry;
while (((entry = resolve_random_set(Raw, Pos)) != "") && (entry != "]"))
{
tokens.emplace_back(entry);
}
if (false == tokens.empty())
{
std::shuffle(std::begin(tokens), std::end(tokens), Global.random_engine);
return tokens.front();
}
return "";
}

9
utilities/utilities.h
View File

@@ -348,6 +348,15 @@ glm::dvec3 LoadPoint(class cParser &Input);
// extracts a group of tokens from provided data stream
std::string deserialize_random_set(cParser &Input, char const *Break = "\n\r\t ;");
// like deserialize_random_set, but additionally records every raw token it consumes
// (including any random-set brackets) into Raw, so the same expression can be replayed
// and re-randomized later. returns the randomly chosen entry, as deserialize_random_set.
std::string deserialize_random_set_capture(cParser &Input, std::vector<std::string> &Raw, char const *Break = "\n\r\t ;");
// re-evaluates a raw random-set expression captured by deserialize_random_set_capture,
// returning a fresh random choice. Pos is advanced through Raw.
std::string resolve_random_set(std::vector<std::string> const &Raw, std::size_t &Pos);
// extracts a group of <key, value> pairs from provided data stream
// NOTE: expects no more than single pair per line
template <typename MapType_> void deserialize_map(MapType_ &Map, cParser &Input)