eu07/maszyna
17
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-11-03 17:50:21 +01:00
Code Issues 156 Wiki Activity

Merge branch 'master' into fix/sonar-reliability-security

Browse Source
This commit is contained in:
jerrrrycho
2026-10-11 12:33:07 +02:00
committed by GitHub
parent b6f8cf669e c26041f2e7
commit 1e6b236c2f
179 changed files with 172242 additions and 16421 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
utilities/Classes.h
View File

@@ -43,6 +43,14 @@ class vehicle_table;
class train_table;
struct light_array;
class particle_manager;
class road_node; // kawałek drogi, z którego powstają pasy ruchu
class road_table;
class sweep_node; // model ułożony wzdłuż krzywej: wygięty albo powtarzany
class sweep_table;
class junction_node;
class junction_table;
class roadpoint_node; // przejazd kolejowy albo miejsce, w którym pojazdy drogowe się pojawiają lub znikają
class roadpoint_table;
struct dictionary_source;
class trainset_desc;
class scenery_desc;
@@ -56,6 +64,8 @@ namespace scene {
struct node_data;
class basic_node;
using group_handle = std::size_t;
using layer_handle = std::uint16_t;
using instance_handle = std::uint32_t;
}
namespace Mtable

38
utilities/Globals.cpp
View File

@@ -187,6 +187,12 @@ bool global_settings::ConfigParseAudio(cParser& Parser, const std::string& token
return true;
}
if (token == "sound.asyncload")
{
ParseOne(Parser, AudioAsyncLoad);
return true;
}
if (token == "sound.volume.vehicle")
{
ParseOneClamped(Parser, VehicleVolume, 0.f, 1.f);
@@ -630,6 +636,15 @@ bool global_settings::ConfigParseSimulation(cParser& Parser, const std::string&
return true;
}
if (token == "file.binary.terrain.checksum")
{
// off, simple or full; anything else leaves the setting as it was
std::string mode;
ParseOne(Parser, mode, 1, true);
file_binary_terrain_checksum = (mode == "off" ? 0 : mode == "simple" ? 1 : mode == "full" ? 2 : file_binary_terrain_checksum);
return true;
}
if (token == "inactivepause")
{
ParseOne(Parser, bInactivePause);
@@ -769,6 +784,12 @@ bool global_settings::ConfigParseUI(cParser& Parser, const std::string& token)
return true;
}
if (token == "ui.viewports")
{
ParseOne(Parser, ui_viewports, 1);
return true;
}
if (token == "gui.defaultwindows")
{
ParseOne(Parser, gui_defaultwindows, 1);
@@ -1338,6 +1359,18 @@ global_settings::ConfigParse_gfx( cParser &Parser, std::string_view const Token
Parser.getTokens();
Parser >> ResourceMove;
}
else if (Token == "gfx.textures.streaming")
{
Parser.getTokens();
Parser >> gfx_texture_streaming;
}
else if (Token == "gfx.textures.releasedistance")
{
// multiplier of the draw range, 0 turns the release off. textures within the draw range are never released
Parser.getTokens();
Parser >> gfx_texture_releasedistance;
gfx_texture_releasedistance = (gfx_texture_releasedistance > 0.f ? std::max(gfx_texture_releasedistance, 1.f) : 0.f);
}
else if (Token == "gfx.reflections.framerate")
{
auto const updatespersecond{std::abs(Parser.getToken<double>())};
@@ -1521,6 +1554,7 @@ global_settings::export_as_text( std::ostream &Output ) const {
export_as_text( Output, "sound.volume.vehicle", VehicleVolume );
export_as_text( Output, "sound.volume.positional", EnvironmentPositionalVolume );
export_as_text( Output, "sound.volume.ambient", EnvironmentAmbientVolume );
export_as_text( Output, "sound.asyncload", AudioAsyncLoad );
export_as_text( Output, "physicslog", WriteLogFlag );
export_as_text( Output, "fullphysics", FullPhysics );
export_as_text( Output, "debuglog", iWriteLogEnabled );
@@ -1575,6 +1609,8 @@ global_settings::export_as_text( std::ostream &Output ) const {
export_as_text( Output, "createswitchtrackbeds", CreateSwitchTrackbeds );
export_as_text( Output, "gfx.resource.sweep", ResourceSweep );
export_as_text( Output, "gfx.resource.move", ResourceMove );
export_as_text( Output, "gfx.textures.streaming", gfx_texture_streaming );
export_as_text( Output, "gfx.textures.releasedistance", gfx_texture_releasedistance );
export_as_text( Output, "gfx.reflections.framerate", 1.0 / reflectiontune.update_interval );
export_as_text( Output, "gfx.reflections.fidelity", reflectiontune.fidelity );
export_as_text( Output, "timespeed", fTimeSpeed );
@@ -1582,6 +1618,7 @@ global_settings::export_as_text( std::ostream &Output ) const {
export_as_text( Output, "latitude", fLatitudeDeg );
export_as_text( Output, "convertmodels", iConvertModels );
export_as_text( Output, "file.binary.terrain", file_binary_terrain );
export_as_text( Output, "file.binary.terrain.checksum", std::string( file_binary_terrain_checksum == 0 ? "off" : file_binary_terrain_checksum == 1 ? "simple" : "full" ) );
export_as_text( Output, "inactivepause", bInactivePause );
export_as_text( Output, "slowmotion", iSlowMotionMask );
export_as_text( Output, "hideconsole", bHideConsole );
@@ -1639,6 +1676,7 @@ global_settings::export_as_text( std::ostream &Output ) const {
<< UITextColor.g * 255 << " "
<< UITextColor.b * 255 << "\n";
export_as_text( Output, "ui.bg.opacity", UIBgOpacity );
export_as_text( Output, "ui.viewports", ui_viewports );
export_as_text( Output, "input.gamepad", InputGamepad );
#ifdef WITH_UART
if( uart_conf.enable ) {

11
utilities/Globals.h
View File

@@ -84,10 +84,15 @@ struct global_settings {
std::string szDefaultExt{ szTexturesDDS };
std::string SceneryFile;
std::string local_start_vehicle{ "EU07-424" };
std::string editor_enter_vehicle; // vehicle placed in the editor, to be entered when the driving goes on
bool editor_session{ false }; // scenery was opened for editing (-edit): loader keeps track of scenery layers and hands over to the editor
bool editor_tracks{ false }; // F3 track workspace: draw and pick tracks without the scenery draw-range cap
int iConvertModels{ 0 }; // tworzenie plików binarnych
int iConvertIndexRange{ 1000 }; // range of duplicate vertex scan
bool file_binary_terrain{ true }; // enable binary terrain (de)serialization
int file_binary_terrain_checksum{ 1 }; // check of a binary terrain file (.btf) against its text: 0 = none, 1 = length of the text, 2 = length and checksum
bool file_binary_terrain_state{true};
int file_binary_terrain_skipped{0}; // number of terrain includes left out by the parser during the current scenery load
// logs
bool priorityLoadText3D{false}; // ladowanie T3D priorytetowo
int iWriteLogEnabled{ 3 }; // maska bitowa: 1-zapis do pliku, 2-okienko, 4-nazwy torów
@@ -167,6 +172,8 @@ struct global_settings {
bool bUseVBO{ true }; // czy jest VBO w karcie graficznej (czy użyć)
float AnisotropicFiltering{ 8.f }; // requested level of anisotropic filtering. TODO: move it to renderer object
float FieldOfView{ 45.f }; // vertical field of view for the camera. TODO: move it to the renderer
bool EditorOrtho{ false };
float EditorOrthoExtent{ 150.f };
GLint iMaxTextureSize{ 4096 }; // maksymalny rozmiar tekstury
GLint iMaxCabTextureSize{ 4096 }; // largest allowed texture in vehicle cab
int iMultisampling{ 2 }; // tryb antyaliasingu: 0=brak,1=2px,2=4px,3=8px,4=16px
@@ -176,6 +183,8 @@ struct global_settings {
float SmokeFidelity{ 1.f }; // determines amount of generated smoke particles
bool ResourceSweep{ true }; // gfx resource garbage collection
bool ResourceMove{ false }; // gfx resources are moved between cpu and gpu side instead of sending a copy
bool gfx_texture_streaming{ false }; // data of dds textures is read by worker threads, the rest of the loading doesn't wait for it
float gfx_texture_releasedistance{ 0.f }; // gl textures of vehicles farther than this many draw ranges are released. 0 = off
bool compress_tex{ true }; // all textures are compressed on gpu side
std::string asSky{ "1" };
float fFpsAverage{ 0.f }; // oczekiwana wartosć FPS
@@ -191,6 +200,7 @@ struct global_settings {
float EnvironmentAmbientVolume{ 1.0f };
int audio_max_sources = 30;
std::string AudioRenderer;
bool AudioAsyncLoad{ false }; // sound files are decoded by worker threads
// input
float fMouseXScale{ 1.5f };
float fMouseYScale{ 0.2f };
@@ -297,6 +307,7 @@ struct global_settings {
float ui_fontsize = 13.0f;
float ui_scale = 1.0f;
bool ui_viewports = true; // panels of the ui can be dragged out of the simulator window, into windows of their own
float map_highlight_distance = 3000.0f;

30
utilities/Names.h
View File

@@ -11,6 +11,7 @@ http://mozilla.org/MPL/2.0/.
#include <unordered_map>
#include <string>
#include <algorithm>
template <typename Type_>
class basic_table {
@@ -31,7 +32,7 @@ public:
}
auto const itemhandle { m_items.size() - 1 };
// add item name to the map
auto mapping = m_itemmap.emplace( itemname, itemhandle );
auto mapping = m_itemmap.emplace( std::move( itemname ), itemhandle );
if( true == mapping.second ) {
return true;
}
@@ -42,6 +43,33 @@ public:
{
return insert(Item, Item->name());
}
// changes the name specified item is found by; empty text or "none" leaves the item without one.
// NOTE: the name held by the item itself is for the caller to change afterwards.
// returns: false if the name is in use by another item or the item isn't in the collection, true otherwise
bool
rename( Type_ const *Item, std::string const &Name ) {
auto const named { false == Name.empty() && Name != "none" };
if( named ) {
auto const taken { m_itemmap.find( Name ) };
if( taken != m_itemmap.end()
&& m_items[ taken->second ] != nullptr
&& m_items[ taken->second ] != Item ) {
return false; } }
// the current name leads to the item unless the item has none, or shares it with an item added later
auto itemhandle { m_items.size() };
auto const current { m_itemmap.find( Item->name() ) };
if( current != m_itemmap.end()
&& m_items[ current->second ] == Item ) {
itemhandle = current->second;
m_itemmap.erase( current ); }
else {
auto const lookup { std::find( m_items.begin(), m_items.end(), Item ) };
if( lookup == m_items.end() ) {
return false; }
itemhandle = static_cast<std::size_t>( std::distance( m_items.begin(), lookup ) ); }
if( named ) {
m_itemmap[ Name ] = itemhandle; }
return true; }
void purge (std::string const &Name)
{
auto lookup = m_itemmap.find( Name );

399
utilities/parser.cpp
View File

@@ -12,6 +12,7 @@ http://mozilla.org/MPL/2.0/.
#include "utilities/Logs.h"
#include "scene/scenenodegroups.h"
#include "scene/scenelayers.h"
/*
MaSzyna EU07 locomotive simulator parser
@@ -24,24 +25,14 @@ http://mozilla.org/MPL/2.0/.
namespace
{
inline std::array<bool, 256> makeBreakTable(const char *brk)
{
std::array<bool, 256> arr{};
for (unsigned char c : std::string_view(brk ? brk : ""))
{
arr[c] = true;
}
return arr;
}
inline char toLowerChar(char c)
{
// Only fold ASCII letters. Bytes >= 0x80 belong to multibyte UTF-8
// sequences and must be passed through untouched, otherwise a non-"C"
// global locale could remap them and corrupt UTF-8 encoded text.
const unsigned char uc = static_cast<unsigned char>(c);
if (uc < 0x80)
return static_cast<char>(std::tolower(uc));
// The remaining ASCII characters have no lower case form, and don't need the call.
if (c >= 'A' && c <= 'Z')
return static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
return c;
}
@@ -69,7 +60,28 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P
case buffer_FILE:
{
Path.append(Stream);
mStream = std::make_shared<std::ifstream>(Path, std::ios_base::binary);
// the file is read in one go, tokens are then extracted straight from the memory
std::ifstream file(Path, std::ios_base::binary);
if (file.fail())
{
mFailBit = true;
}
else
{
auto const size{file.rdbuf()->pubseekoff(0, std::ios_base::end)};
file.rdbuf()->pubseekoff(0, std::ios_base::beg);
if (size > 0 && static_cast<std::uintmax_t>(size) < mBuffer.max_size())
{
mBuffer.resize(static_cast<std::size_t>(size));
file.read(mBuffer.data(), static_cast<std::streamsize>(mBuffer.size()));
mBuffer.resize(static_cast<std::size_t>(std::max<std::streamsize>(0, file.gcount())));
}
else if (size != 0)
{
// not a file which can be read, e.g. a directory, which some systems allow to open
mFailBit = true;
}
}
// content of *.inc files is potentially grouped together
if (Stream.size() >= 4 && ToLower(Stream.substr(Stream.size() - 4)) == ".inc")
{
@@ -80,7 +92,7 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P
}
case buffer_TEXT:
{
mStream = std::make_shared<std::istringstream>(Stream);
mBuffer = Stream;
break;
}
default:
@@ -89,18 +101,14 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P
}
}
// calculate stream size
if (mStream)
if (true == mFailBit)
{
if (true == mStream->fail())
{
ErrorLog("Failed to open file \"" + Path + "\"");
}
else
{
mSize = mStream->rdbuf()->pubseekoff(0, std::ios_base::end);
mStream->rdbuf()->pubseekoff(0, std::ios_base::beg);
mLine = 1;
}
ErrorLog("Failed to open file \"" + Path + "\"");
}
else
{
mSize = static_cast<std::streamoff>(mBuffer.size());
mLine = 1;
}
// set parameter set if one was provided
if (false == Parameters.empty())
@@ -118,6 +126,15 @@ cParser::~cParser()
// wrap up the node group holding content of processed file
scene::Groups.close();
}
if (true == mLayerFile)
{
// content of the processed file ended, following nodes belong to the parent layer again
scene::Layers.close();
}
if (true == mInstanceFile)
{
scene::Layers.instance_end();
}
}
template <> glm::vec3 cParser::getToken(bool const ToLower, char const *Break)
@@ -137,7 +154,7 @@ template <> cParser &cParser::operator>>(std::string &Right)
return *this;
}
Right = this->tokens.front();
Right = std::move(this->tokens.front());
this->tokens.pop_front();
return *this;
@@ -157,6 +174,25 @@ template <> cParser &cParser::operator>>(bool &Right)
return *this;
}
template <> std::string cParser::getToken<std::string>(bool const ToLower, const char *Break)
{
// does what the generic version would, minus the trip of the token through the queue
if (true == m_autoclear)
{
tokens.clear();
}
std::string token;
if (false == tokens.empty())
{
*this >> token;
}
else
{
readToken(token, ToLower, Break);
}
return token;
}
template <> bool cParser::getToken<bool>(bool const ToLower, const char *Break)
{
@@ -224,30 +260,147 @@ bool cParser::getTokens(unsigned int Count, bool ToLower, const char *Break)
return true;
}
std::string cParser::readTokenFromStream(bool ToLower, const char *Break)
// mimics std::istream::peek() != EOF of the stream the parser used to read from
bool cParser::hasChar()
{
std::string token;
token.reserve(64);
if (mEofBit || mFailBit)
{
// an attempt to read from a stream which isn't in good state marks it as failed
mFailBit = true;
return false;
}
if (static_cast<std::size_t>(mPosition) >= mBuffer.size())
{
mEofBit = true;
return false;
}
return true;
}
const auto breakTable = makeBreakTable(Break);
// mimics std::istream::get(char&). NOTE: doesn't advance the read position, the caller does
bool cParser::getChar(char &Char)
{
if (mEofBit || mFailBit)
{
mFailBit = true;
return false;
}
if (static_cast<std::size_t>(mPosition) >= mBuffer.size())
{
// failed extraction at the end of data sets both flags
mEofBit = true;
mFailBit = true;
return false;
}
Char = mBuffer[static_cast<std::size_t>(mPosition)];
return true;
}
// prepares character lookup for specified set of token separators
void cParser::updateCharClasses(char const *Break)
{
if (Break == nullptr)
{
Break = "";
}
if (mCharClassesValid && std::strcmp(mCharClassesBreak.c_str(), Break) == 0)
{
// the set of separators rarely changes from one token to another
return;
}
mCharClassesBreak = Break;
mCharClasses.fill(0);
for (unsigned char const c : mCharClassesBreak)
{
mCharClasses[c] |= char_break;
}
// characters which need more than being added to the token:
// line ends are counted,
mCharClasses[static_cast<unsigned char>('\n')] |= char_special;
// quotes glue the text they enclose into a single token,
mCharClasses[static_cast<unsigned char>('\"')] |= char_special;
// and a comment can start only once the last character of its opening mark arrives
for (auto const &comment : mComments)
{
for (auto character{0}; character < 256; ++character)
{
// NOTE: the mark is compared with the token, which can hold the character converted to lower case
if (comment.first.empty() || comment.first.back() == static_cast<char>(character) || comment.first.back() == toLowerChar(static_cast<char>(character)))
{
mCharClasses[character] |= char_special;
}
}
}
mCharClassesValid = true;
}
void cParser::readTokenFromStream(std::string &token, bool ToLower, const char *Break)
{
token.clear();
updateCharClasses(Break);
char c = 0;
bool separated = false; // the token was ended by a separator, which isn't a part of it
while (token.empty() && mStream->peek() != EOF) {
while (mStream->peek() != EOF) { // idk why but with mStream->get(c) not all cars are loaded
c = static_cast<char>(mStream->get());
while (token.empty() && hasChar()) {
while (hasChar()) {
auto const *const data{mBuffer.data()};
auto const size{mBuffer.size()};
auto const begin{static_cast<std::size_t>(mPosition)};
if (mCharClasses[static_cast<unsigned char>(data[begin])] == 0)
{
// run of regular characters, which only need to be added to the token
auto end{begin + 1};
while (end < size && mCharClasses[static_cast<unsigned char>(data[end])] == 0)
{
++end;
}
if (token.empty())
{
mTokenBegin = mPosition;
}
auto const converted{token.size()};
token.append(data + begin, end - begin);
if (ToLower)
{
for (auto index{converted}; index < token.size(); ++index)
{
token[index] = toLowerChar(token[index]);
}
}
mPosition = static_cast<std::streamoff>(end);
continue;
}
c = data[begin];
++mPosition;
if (c == '\n') {
++mLine;
}
const unsigned char uc = static_cast<unsigned char>(c);
if (breakTable[uc]) {
if (mCharClasses[uc] & char_break) {
// separator ends token (or continues skipping if token empty)
if (!token.empty())
if (!token.empty()) {
separated = true;
break;
}
// the rest of a run of separators is skipped right away
auto next{begin + 1};
while (next < size && (mCharClasses[static_cast<unsigned char>(data[next])] & char_break)) {
if (data[next] == '\n') {
++mLine;
}
++next;
}
mPosition = static_cast<std::streamoff>(next);
continue;
}
if (token.empty()) {
mTokenBegin = mPosition - 1;
}
if (ToLower) c = toLowerChar(c);
token.push_back(c);
@@ -259,8 +412,8 @@ std::string cParser::readTokenFromStream(bool ToLower, const char *Break)
}
}
}
return token;
// NOTE: comment glued to the end of a token is counted as a part of it
mTokenEnd = separated ? mPosition - 1 : mPosition;
}
void cParser::stripFirstTokenBOM(std::string& token, bool ToLower, const char* Break) {
@@ -269,6 +422,7 @@ void cParser::stripFirstTokenBOM(std::string& token, bool ToLower, const char* B
if (startsWithBOM(token)) {
token.erase(0, 3);
mTokenBegin += 3;
}
// if first "token" was standalone BOM, read the next real token (avoid recursion)
@@ -331,10 +485,22 @@ void cParser::startIncludeFromParser(cParser& srcParser, bool ToLower, std::stri
return;
}
// layers are named by their files relative to the scenery folder; the terrain files (.txtf) come from includes made up by
// the loader, relative to the folder of the simulator
auto layername{mPath + includefile};
layername = (layername.starts_with(Global.asCurrentSceneryPath) ? layername.substr(Global.asCurrentSceneryPath.size()) : includefile);
const bool isTerrain = contains(includefile, "_ter.scm");
if (isTerrain && true == Global.file_binary_terrain_state) {
WriteLog("SBT found, ignoring: " + includefile);
readParameters(srcParser); // preserve original side-effect: still consume parameters
++Global.file_binary_terrain_skipped;
if (sceneryLayers)
{
// the file is still a part of the scenery, even though its content comes from the binary terrain file
scene::Layers.layer(scene::Layers.open(layername, include_site())).binary = true;
scene::Layers.close();
}
return;
}
@@ -354,11 +520,51 @@ void cParser::startIncludeFromParser(cParser& srcParser, bool ToLower, std::stri
if (mIncludeParser->mSize <= 0) {
ErrorLog("Bad include: can't open file \"" + includefile + "\"");
}
if (sceneryLayers)
{
mIncludeParser->sceneryLayers = true;
// content of an included scenery file forms a layer of its own. *.inc files are node templates
// reused all over the scenery, so their content stays in the layer of the file which includes them
if (mIncludeParser->mIncFile)
{
scene::Layers.template_used(includefile);
// include of a template made directly by a layer file is handled by the editor as a whole. templates
// included by that template are a part of it
auto const site{include_site()};
if (false == site.fixed && scene::Layers.instance() == 0 && mIncludeParser->mSize > 0)
{
scene::Layers.instance_begin(includefile, site.span);
mIncludeParser->mInstanceFile = true;
}
}
else if (mIncludeParser->mSize > 0)
{
auto site{include_site()};
site.parameters = false == mIncludeParser->parameters.empty();
scene::Layers.open(layername, site);
mIncludeParser->mLayerFile = true;
}
}
}
// describes the include directive being processed, for the scenery layer bookkeeping
scene::include_site cParser::include_site() const
{
scene::include_site site;
// the directive can be rewritten on scenery save only if we know where it is, and it sits in a layer file
site.fixed = mIncludeBegin < 0 || mFile.empty() || mIncFile;
if (false == site.fixed)
{
site.span = {mIncludeBegin, mTokenEnd};
}
return site;
}
bool cParser::handleIncludeIfPresent(std::string& token, bool ToLower, const char* Break) {
// token-mode include: token == "include"
if (expandIncludes && token == "include") {
mIncludeBegin = mTokenBegin;
std::string includefile;
if (allowRandomIncludes)
includefile = deserialize_random_set(*this);
@@ -374,6 +580,7 @@ bool cParser::handleIncludeIfPresent(std::string& token, bool ToLower, const cha
// line-mode HACK: Break == "\n\r" and line begins with "include"
if (std::strcmp(Break, "\n\r") == 0 && token.compare(0, 7, "include") == 0) {
mIncludeBegin = -1; // the directive is parsed out of a line of text, its exact location isn't known
cParser includeparser(token.substr(7));
std::string includefile;
if (allowRandomIncludes)
@@ -398,19 +605,30 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break)
if (out.empty())
{
mIncludeParser = nullptr;
out = readTokenFromStream(ToLower, Break);
readTokenFromStream(out, ToLower, Break);
}
}
else
{
out = readTokenFromStream(ToLower, Break);
readTokenFromStream(out, ToLower, Break);
}
stripFirstTokenBOM(out, ToLower, Break);
// NOTE: the checks repeat what the called methods test first, to skip the calls for vast majority of the tokens
if (mFirstToken)
{
stripFirstTokenBOM(out, ToLower, Break);
}
substituteParameters(out, ToLower);
if (false == parameters.empty())
{
substituteParameters(out, ToLower);
}
handleIncludeIfPresent(out, ToLower, Break);
if (out.size() >= 7 && out[0] == 'i')
{
// nothing else can be an include directive
handleIncludeIfPresent(out, ToLower, Break);
}
}
std::vector<std::string> cParser::readParameters(cParser &Input)
@@ -432,8 +650,9 @@ std::string cParser::readQuotes(char const Quote)
std::string token;
char c{0};
bool escaped = false;
while (mStream->get(c))
while (getChar(c))
{ // get all chars until the quote mark
++mPosition;
if (escaped)
{
escaped = false;
@@ -459,24 +678,23 @@ std::string cParser::readQuotes(char const Quote)
void cParser::skipComment(std::string const &Endmark)
{ // pobieranie znaków aż do znalezienia znacznika końca
std::string input;
char c{0};
auto const endmarksize = Endmark.size();
while (mStream->get(c))
if (false == getChar(c))
{
if (c == '\n')
{
// update line counter
++mLine;
}
input += c;
if (input == Endmark) // szukanie znacznika końca
break;
if (input.size() >= endmarksize)
{
// keep the read text short, to avoid pointless string re-allocations on longer comments
input = input.substr(1);
}
return;
}
auto const begin{static_cast<std::size_t>(mPosition)};
// szukanie znacznika końca
// NOTE: with nothing to look for the comment runs until the end of data
auto const found{Endmark.empty() ? std::string::npos : mBuffer.find(Endmark, begin)};
auto const end{found != std::string::npos ? found + Endmark.size() : mBuffer.size()};
// update line counter
mLine += static_cast<std::size_t>(std::count(mBuffer.data() + begin, mBuffer.data() + end, '\n'));
mPosition = static_cast<std::streamoff>(end);
if (found == std::string::npos)
{
// the data ended before the comment did; record the attempt to read past the end
getChar(c);
}
return;
}
@@ -523,12 +741,13 @@ void cParser::injectString(const std::string &str)
{
mIncludeParser = std::make_shared<cParser>(str, buffer_TEXT, "", LoadTraction, std::vector<std::string>(), allowRandomIncludes);
mIncludeParser->autoclear(m_autoclear);
mIncludeParser->sceneryLayers = sceneryLayers;
}
}
int cParser::getProgress() const
{
return static_cast<int>(mStream->rdbuf()->pubseekoff(0, std::ios_base::cur) * 100 / mSize);
return (mSize > 0 ? static_cast<int>(mPosition * 100 / mSize) : 100);
}
int cParser::getFullProgress() const
@@ -566,6 +785,8 @@ void cParser::addCommentStyle(std::string const &Commentstart, std::string const
{
mComments.insert(commentmap::value_type(Commentstart, Commentend));
// the character lookup depends on the comment marks
mCharClassesValid = false;
}
// returns name of currently open file, or empty string for text type stream
@@ -600,3 +821,59 @@ int cParser::LineMain() const
{
return mIncludeParser ? -1 : mLine;
}
std::streamoff cParser::TokenBegin() const
{
return mIncludeParser ? mIncludeParser->TokenBegin() : mTokenBegin;
}
std::streamoff cParser::TokenEnd() const
{
return mIncludeParser ? mIncludeParser->TokenEnd() : mTokenEnd;
}
bool cParser::InLayerFile() const
{
return mIncludeParser ? mIncludeParser->InLayerFile() : (false == mFile.empty() && false == mIncFile);
}
std::string_view cParser::remainingText() const
{
if (false == tokens.empty())
{
return {};
}
// each parser on the way gets to change the tokens of the file being read
auto const *source{this};
while (true)
{
if (false == source->parameters.empty() || false == source->skipComments || source->mComments.size() != 2)
{
return {};
}
if (source->mIncludeParser == nullptr)
{
break;
}
source = source->mIncludeParser.get();
}
// NOTE: the first token of a file is left to the regular code, which deals with the byte order mark
if (source->mFirstToken || source->mEofBit || source->mFailBit || static_cast<std::size_t>(source->mPosition) >= source->mBuffer.size())
{
return {};
}
return std::string_view{source->mBuffer}.substr(static_cast<std::size_t>(source->mPosition));
}
void cParser::skipText(std::size_t const Count, std::size_t const Lines, std::size_t const Tokenlength)
{
if (mIncludeParser)
{
mIncludeParser->skipText(Count, Lines, Tokenlength);
return;
}
mPosition = static_cast<std::streamoff>(std::min(static_cast<std::size_t>(mPosition) + Count, mBuffer.size()));
mLine += Lines;
mTokenEnd = mPosition;
mTokenBegin = mPosition - static_cast<std::streamoff>(std::min(Tokenlength, static_cast<std::size_t>(mPosition)));
}

131
utilities/parser.h
View File

@@ -10,10 +10,19 @@ http://mozilla.org/MPL/2.0/.
#pragma once
#include <string>
#include <string_view>
#include <sstream>
#include <fstream>
#include <vector>
#include <map>
#include <array>
#include <charconv>
#include <type_traits>
namespace scene
{
struct include_site;
}
/////////////////////////////////////////////////////////////////////////////////////////////////////
// cParser -- generic class for parsing text data, either from file or provided string
@@ -56,11 +65,11 @@ class cParser //: public std::stringstream
inline
bool
eof() {
return mStream->eof(); };
return mEofBit; };
inline
bool
ok() {
return !mStream->fail(); };
return !mFailBit; };
cParser &
autoclear( bool const Autoclear );
inline
@@ -69,7 +78,7 @@ class cParser //: public std::stringstream
return m_autoclear; }
bool
getTokens( unsigned int Count = 1, bool ToLower = true, char const *Break = "\n\r\t ;" );
std::string readTokenFromStream(bool ToLower, const char *Break);
void readTokenFromStream(std::string &token, bool ToLower, const char *Break);
void stripFirstTokenBOM(std::string &token, bool ToLower, const char *Break);
void substituteParameters(std::string &token, bool ToLower);
void skipIncludeBlock();
@@ -94,12 +103,58 @@ class cParser //: public std::stringstream
std::size_t Line() const;
// returns number of currently processed line in main file, -1 if inside include
int LineMain() const;
// location of the most recently read token in its source file, as byte offsets
std::streamoff TokenBegin() const;
std::streamoff TokenEnd() const;
// true if the most recently read token came straight from a scenery file which isn't an *.inc node template
bool InLayerFile() const;
// text which follows the most recently read token, for the readers which take entries of a known kind straight from
// the text instead of token by token. empty if the tokens can't be taken that way: some wait to be picked up, there
// are parameters of an include to put in them, or comment marks other than the standard ones are in use
std::string_view remainingText() const;
// moves past specified amount of the text returned by remainingText(): Lines is the number of line breaks in it,
// Tokenlength the length of the token it ends with
void skipText(std::size_t Count, std::size_t Lines, std::size_t Tokenlength);
// converts text holding a plain number without involving a stream. returns: true on success, false if the text needs regular treatment
template <typename Type_>
static bool parseNumber( std::string_view Token, Type_ &Output );
bool expandIncludes = true;
bool allowRandomIncludes = false;
bool skipComments = true;
bool sceneryLayers = false; // included scenery files are registered as scenery layers (scenery opened for editing)
private:
// types:
// retrieved tokens waiting for pick up. a vector with moving head instead of a deque,
// as the latter can allocate memory for each stored item
struct token_queue {
std::vector<std::string> items;
std::size_t head { 0 };
bool empty() const {
return head == items.size(); }
std::size_t size() const {
return items.size() - head; }
std::string & front() {
return items[ head ]; }
std::string const & front() const {
return items[ head ]; }
void emplace_back( std::string &&Token ) {
items.emplace_back( std::move( Token ) ); }
void pop_front() {
++head;
if( head == items.size() ) {
clear(); }
else if( head >= 64 && head * 2 >= items.size() ) {
// keep the queue from growing without end if it never gets fully emptied
items.erase( items.begin(), items.begin() + static_cast<std::ptrdiff_t>( head ) );
head = 0; } }
void clear() {
items.clear();
head = 0; }
};
void startIncludeFromParser(cParser &srcParser, bool ToLower, std::string includefile);
scene::include_site include_site() const;
bool handleIncludeIfPresent(std::string &token, bool ToLower, const char *Break);
// methods:
void readToken(std::string& out, bool ToLower = true, const char *Break = "\n\r\t ;");
@@ -109,15 +164,34 @@ class cParser //: public std::stringstream
bool findQuotes( std::string &String );
bool trimComments( std::string &String );
std::size_t count();
// input access. these mimic peek() and get() of the stream the parser used to read from,
// including their effect on the state reported by eof() and ok()
bool hasChar();
bool getChar( char &Char );
void updateCharClasses( char const *Break );
// members:
bool m_autoclear { true }; // unretrieved tokens are discarded when another read command is issued (legacy behaviour)
bool LoadTraction { true }; // load traction?
std::shared_ptr<std::istream> mStream; // relevant kind of buffer is attached on creation.
std::string mBuffer; // content of the open file or the provided text, loaded on creation.
bool mEofBit { false }; // input state, maintained the way a stream would do it
bool mFailBit { false };
enum charclass : unsigned char {
char_break = 1, // separates tokens
char_special = 2 }; // can't be simply added to the token: line end, quote, or potential end of a comment mark
std::array<unsigned char, 256> mCharClasses {}; // lookup built for the most recently used set of token separators
std::string mCharClassesBreak; // set of separators the lookup was built for
bool mCharClassesValid { false };
std::string mFile; // name of the open file, if any
std::string mPath; // path to open stream, for relative path lookups.
std::streamoff mSize { 0 }; // size of open stream, for progress report.
std::size_t mLine { 0 }; // currently processed line
bool mIncFile { false }; // the parser is processing an *.inc file
bool mLayerFile { false }; // the parser is processing a file registered as scenery layer
bool mInstanceFile { false }; // the parser is processing a template whose include is registered with the scenery layers
std::streamoff mPosition { 0 }; // amount of bytes read from the stream so far
std::streamoff mTokenBegin { 0 }; // location of the most recently read token
std::streamoff mTokenEnd { 0 };
std::streamoff mIncludeBegin { -1 }; // location of the include directive being processed, -1 if unknown
bool mFirstToken { true }; // processing first token in the current file; helper used when checking for utf bom
typedef std::map<std::string, std::string> commentmap;
commentmap mComments {
@@ -125,7 +199,7 @@ class cParser //: public std::stringstream
commentmap::value_type( "//", "\n" ) };
std::shared_ptr<cParser> mIncludeParser; // child class to handle include directives.
std::vector<std::string> parameters; // parameter list for included file.
std::deque<std::string> tokens;
token_queue tokens;
};
@@ -134,14 +208,53 @@ glm::vec3
cParser::getToken( bool const ToLower, const char *Break );
template<typename Type_>
bool
cParser::parseNumber( std::string_view const Token, Type_ &Output ) {
using type = std::remove_cv_t<Type_>;
if constexpr(
#ifdef __cpp_lib_to_chars
std::is_floating_point_v<type> ||
#endif
( std::is_integral_v<type>
&& false == std::is_same_v<type, bool>
&& false == std::is_same_v<type, char>
&& false == std::is_same_v<type, signed char>
&& false == std::is_same_v<type, unsigned char>
&& false == std::is_same_v<type, wchar_t>
&& false == std::is_same_v<type, char8_t>
&& false == std::is_same_v<type, char16_t>
&& false == std::is_same_v<type, char32_t> ) ) {
auto const *first { Token.data() };
auto const *last { first + Token.size() };
// only text which is a number from start to end is converted here. anything else, including forms
// the stream extraction doesn't recognize (inf, nan), is left for the stream to make sense of
auto const *digits { ( first != last && *first == '-' ) ? first + 1 : first };
if( digits == last ) { return false; }
if( ( *digits < '0' || *digits > '9' ) && *digits != '.' ) { return false; }
type value {};
auto const result { std::from_chars( first, last, value ) };
if( result.ec != std::errc() || result.ptr != last ) { return false; }
Output = value;
return true;
}
else {
return false;
}
}
template<typename Type_>
cParser&
cParser::operator>>( Type_ &Right ) {
if( true == this->tokens.empty() ) { return *this; }
std::stringstream converter( this->tokens.front() );
converter >> Right;
if( false == parseNumber( this->tokens.front(), Right ) ) {
std::stringstream converter( this->tokens.front() );
converter >> Right;
}
this->tokens.pop_front();
return *this;
@@ -158,3 +271,7 @@ cParser::operator>>( bool &Right );
template<>
bool
cParser::getToken<bool>( bool const ToLower, const char *Break );
template<>
std::string
cParser::getToken<std::string>( bool const ToLower, const char *Break );

5
utilities/translation.cpp
View File

@@ -30,6 +30,11 @@ void locale::init()
while (parse_translation(stream));
// texts of the scenery editor, kept apart from the translation of the simulator
std::fstream editor("lang/editor_" + Global.asLang + ".po", std::ios_base::in | std::ios_base::binary);
if (editor.is_open())
while (parse_translation(editor));
WriteLog("translation: " + std::to_string(lang_mapping.size()) + " strings loaded");
}

28
utilities/uuid.hpp
View File

@@ -28,23 +28,17 @@ public:
}
std::string to_string() const {
std::ostringstream os;
os << std::hex << std::setfill('0');
auto put = [&](int i){
os << std::setw(2) << static_cast<int>(bytes[i]);
};
// format 8-4-4-4-12
for (int i = 0; i < 4; ++i) put(i);
for (int i = 4; i < 6; ++i) put(i);
os << '-';
for (int i = 6; i < 8; ++i) put(i);
os << '-';
for (int i = 8; i < 10; ++i) put(i);
os << '-';
for (int i = 10; i < 12; ++i) put(i);
os << '-';
for (int i = 12; i < 16; ++i) put(i);
return os.str();
// NOTE: done by hand as the text is made for each model instance of a scenery, and a string stream took a few microseconds each
static constexpr char digits[] = "0123456789abcdef";
std::string text;
text.reserve(36);
// format 12-4-4-4-8, the one node names in scenery files were written with
for (int i = 0; i < 16; ++i) {
if (i == 6 || i == 8 || i == 10 || i == 12) text += '-';
text += digits[bytes[i] >> 4];
text += digits[bytes[i] & 0x0F];
}
return text;
}
static UID from_string(const std::string& str) {