Merge branch 'master' into fix/sonar-reliability-security

2026-10-18 07:59:19 +02:00 · 2026-10-11 12:33:07 +02:00
parent b6f8cf669e c26041f2e7
commit 1e6b236c2f
179 changed files with 172242 additions and 16421 deletions
--- a/utilities/parser.cpp
+++ b/utilities/parser.cpp
@@ -12,6 +12,7 @@ http://mozilla.org/MPL/2.0/.
 #include "utilities/Logs.h"

 #include "scene/scenenodegroups.h"
+#include "scene/scenelayers.h"

 /*
    MaSzyna EU07 locomotive simulator parser
@@ -24,24 +25,14 @@ http://mozilla.org/MPL/2.0/.

 namespace
 {
-inline std::array<bool, 256> makeBreakTable(const char *brk)
-{
-	std::array<bool, 256> arr{};
-	for (unsigned char c : std::string_view(brk ? brk : ""))
-	{
-		arr[c] = true;
-	}
-	return arr;
-}
-
 inline char toLowerChar(char c)
 {
 	// Only fold ASCII letters. Bytes >= 0x80 belong to multibyte UTF-8
 	// sequences and must be passed through untouched, otherwise a non-"C"
 	// global locale could remap them and corrupt UTF-8 encoded text.
-	const unsigned char uc = static_cast<unsigned char>(c);
-	if (uc < 0x80)
-		return static_cast<char>(std::tolower(uc));
+	// The remaining ASCII characters have no lower case form, and don't need the call.
+	if (c >= 'A' && c <= 'Z')
+		return static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
 	return c;
 }

@@ -69,7 +60,28 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P
 	case buffer_FILE:
 	{
 		Path.append(Stream);
-		mStream = std::make_shared<std::ifstream>(Path, std::ios_base::binary);
+		// the file is read in one go, tokens are then extracted straight from the memory
+		std::ifstream file(Path, std::ios_base::binary);
+		if (file.fail())
+		{
+			mFailBit = true;
+		}
+		else
+		{
+			auto const size{file.rdbuf()->pubseekoff(0, std::ios_base::end)};
+			file.rdbuf()->pubseekoff(0, std::ios_base::beg);
+			if (size > 0 && static_cast<std::uintmax_t>(size) < mBuffer.max_size())
+			{
+				mBuffer.resize(static_cast<std::size_t>(size));
+				file.read(mBuffer.data(), static_cast<std::streamsize>(mBuffer.size()));
+				mBuffer.resize(static_cast<std::size_t>(std::max<std::streamsize>(0, file.gcount())));
+			}
+			else if (size != 0)
+			{
+				// not a file which can be read, e.g. a directory, which some systems allow to open
+				mFailBit = true;
+			}
+		}
 		// content of *.inc files is potentially grouped together
 		if (Stream.size() >= 4 && ToLower(Stream.substr(Stream.size() - 4)) == ".inc")
 		{
@@ -80,7 +92,7 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P
 	}
 	case buffer_TEXT:
 	{
-		mStream = std::make_shared<std::istringstream>(Stream);
+		mBuffer = Stream;
 		break;
 	}
 	default:
@@ -89,18 +101,14 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P
 	}
 	}
 	// calculate stream size
-	if (mStream)
+	if (true == mFailBit)
 	{
-		if (true == mStream->fail())
-		{
-			ErrorLog("Failed to open file \"" + Path + "\"");
-		}
-		else
-		{
-			mSize = mStream->rdbuf()->pubseekoff(0, std::ios_base::end);
-			mStream->rdbuf()->pubseekoff(0, std::ios_base::beg);
-			mLine = 1;
-		}
+		ErrorLog("Failed to open file \"" + Path + "\"");
+	}
+	else
+	{
+		mSize = static_cast<std::streamoff>(mBuffer.size());
+		mLine = 1;
 	}
 	// set parameter set if one was provided
 	if (false == Parameters.empty())
@@ -118,6 +126,15 @@ cParser::~cParser()
 		// wrap up the node group holding content of processed file
 		scene::Groups.close();
 	}
+	if (true == mLayerFile)
+	{
+		// content of the processed file ended, following nodes belong to the parent layer again
+		scene::Layers.close();
+	}
+	if (true == mInstanceFile)
+	{
+		scene::Layers.instance_end();
+	}
 }

 template <> glm::vec3 cParser::getToken(bool const ToLower, char const *Break)
@@ -137,7 +154,7 @@ template <> cParser &cParser::operator>>(std::string &Right)
 		return *this;
 	}

-	Right = this->tokens.front();
+	Right = std::move(this->tokens.front());
 	this->tokens.pop_front();

 	return *this;
@@ -157,6 +174,25 @@ template <> cParser &cParser::operator>>(bool &Right)
 	return *this;
 }

+template <> std::string cParser::getToken<std::string>(bool const ToLower, const char *Break)
+{
+	// does what the generic version would, minus the trip of the token through the queue
+	if (true == m_autoclear)
+	{
+		tokens.clear();
+	}
+	std::string token;
+	if (false == tokens.empty())
+	{
+		*this >> token;
+	}
+	else
+	{
+		readToken(token, ToLower, Break);
+	}
+	return token;
+}
+
 template <> bool cParser::getToken<bool>(bool const ToLower, const char *Break)
 {

@@ -224,30 +260,147 @@ bool cParser::getTokens(unsigned int Count, bool ToLower, const char *Break)
 		return true;
 }

-std::string cParser::readTokenFromStream(bool ToLower, const char *Break)
+// mimics std::istream::peek() != EOF of the stream the parser used to read from
+bool cParser::hasChar()
 {
-	std::string token;
-	token.reserve(64);
+	if (mEofBit || mFailBit)
+	{
+		// an attempt to read from a stream which isn't in good state marks it as failed
+		mFailBit = true;
+		return false;
+	}
+	if (static_cast<std::size_t>(mPosition) >= mBuffer.size())
+	{
+		mEofBit = true;
+		return false;
+	}
+	return true;
+}

-	const auto breakTable = makeBreakTable(Break);
+// mimics std::istream::get(char&). NOTE: doesn't advance the read position, the caller does
+bool cParser::getChar(char &Char)
+{
+	if (mEofBit || mFailBit)
+	{
+		mFailBit = true;
+		return false;
+	}
+	if (static_cast<std::size_t>(mPosition) >= mBuffer.size())
+	{
+		// failed extraction at the end of data sets both flags
+		mEofBit = true;
+		mFailBit = true;
+		return false;
+	}
+	Char = mBuffer[static_cast<std::size_t>(mPosition)];
+	return true;
+}
+
+// prepares character lookup for specified set of token separators
+void cParser::updateCharClasses(char const *Break)
+{
+	if (Break == nullptr)
+	{
+		Break = "";
+	}
+	if (mCharClassesValid && std::strcmp(mCharClassesBreak.c_str(), Break) == 0)
+	{
+		// the set of separators rarely changes from one token to another
+		return;
+	}
+	mCharClassesBreak = Break;
+	mCharClasses.fill(0);
+	for (unsigned char const c : mCharClassesBreak)
+	{
+		mCharClasses[c] |= char_break;
+	}
+	// characters which need more than being added to the token:
+	// line ends are counted,
+	mCharClasses[static_cast<unsigned char>('\n')] |= char_special;
+	// quotes glue the text they enclose into a single token,
+	mCharClasses[static_cast<unsigned char>('\"')] |= char_special;
+	// and a comment can start only once the last character of its opening mark arrives
+	for (auto const &comment : mComments)
+	{
+		for (auto character{0}; character < 256; ++character)
+		{
+			// NOTE: the mark is compared with the token, which can hold the character converted to lower case
+			if (comment.first.empty() || comment.first.back() == static_cast<char>(character) || comment.first.back() == toLowerChar(static_cast<char>(character)))
+			{
+				mCharClasses[character] |= char_special;
+			}
+		}
+	}
+	mCharClassesValid = true;
+}
+
+void cParser::readTokenFromStream(std::string &token, bool ToLower, const char *Break)
+{
+	token.clear();
+
+	updateCharClasses(Break);
 	char c = 0;
+	bool separated = false; // the token was ended by a separator, which isn't a part of it


-	while (token.empty() && mStream->peek() != EOF) {
-		while (mStream->peek() != EOF) { // idk why but with mStream->get(c) not all cars are loaded
-			c = static_cast<char>(mStream->get());
+	while (token.empty() && hasChar()) {
+		while (hasChar()) {
+			auto const *const data{mBuffer.data()};
+			auto const size{mBuffer.size()};
+			auto const begin{static_cast<std::size_t>(mPosition)};
+			if (mCharClasses[static_cast<unsigned char>(data[begin])] == 0)
+			{
+				// run of regular characters, which only need to be added to the token
+				auto end{begin + 1};
+				while (end < size && mCharClasses[static_cast<unsigned char>(data[end])] == 0)
+				{
+					++end;
+				}
+				if (token.empty())
+				{
+					mTokenBegin = mPosition;
+				}
+				auto const converted{token.size()};
+				token.append(data + begin, end - begin);
+				if (ToLower)
+				{
+					for (auto index{converted}; index < token.size(); ++index)
+					{
+						token[index] = toLowerChar(token[index]);
+					}
+				}
+				mPosition = static_cast<std::streamoff>(end);
+				continue;
+			}
+
+			c = data[begin];
+			++mPosition;
 			if (c == '\n') {
 				++mLine;
 			}

 			const unsigned char uc = static_cast<unsigned char>(c);
-			if (breakTable[uc]) {
+			if (mCharClasses[uc] & char_break) {
 				// separator ends token (or continues skipping if token empty)
-				if (!token.empty())
+				if (!token.empty()) {
+					separated = true;
 					break;
+				}
+				// the rest of a run of separators is skipped right away
+				auto next{begin + 1};
+				while (next < size && (mCharClasses[static_cast<unsigned char>(data[next])] & char_break)) {
+					if (data[next] == '\n') {
+						++mLine;
+					}
+					++next;
+				}
+				mPosition = static_cast<std::streamoff>(next);
 				continue;
 			}

+			if (token.empty()) {
+				mTokenBegin = mPosition - 1;
+			}
 			if (ToLower) c = toLowerChar(c);
 			token.push_back(c);

@@ -259,8 +412,8 @@ std::string cParser::readTokenFromStream(bool ToLower, const char *Break)
 			}
 		}
 	}
-
-	return token;
+	// NOTE: comment glued to the end of a token is counted as a part of it
+	mTokenEnd = separated ? mPosition - 1 : mPosition;
 }

 void cParser::stripFirstTokenBOM(std::string& token, bool ToLower, const char* Break) {
@@ -269,6 +422,7 @@ void cParser::stripFirstTokenBOM(std::string& token, bool ToLower, const char* B

 	if (startsWithBOM(token)) {
 		token.erase(0, 3);
+		mTokenBegin += 3;
 	}

 	// if first "token" was standalone BOM, read the next real token (avoid recursion)
@@ -331,10 +485,22 @@ void cParser::startIncludeFromParser(cParser& srcParser, bool ToLower, std::stri
 		return;
 	}

+	// layers are named by their files relative to the scenery folder; the terrain files (.txtf) come from includes made up by
+	// the loader, relative to the folder of the simulator
+	auto layername{mPath + includefile};
+	layername = (layername.starts_with(Global.asCurrentSceneryPath) ? layername.substr(Global.asCurrentSceneryPath.size()) : includefile);
+
 	const bool isTerrain = contains(includefile, "_ter.scm");
 	if (isTerrain && true == Global.file_binary_terrain_state) {
 		WriteLog("SBT found, ignoring: " + includefile);
 		readParameters(srcParser); // preserve original side-effect: still consume parameters
+		++Global.file_binary_terrain_skipped;
+		if (sceneryLayers)
+		{
+			// the file is still a part of the scenery, even though its content comes from the binary terrain file
+			scene::Layers.layer(scene::Layers.open(layername, include_site())).binary = true;
+			scene::Layers.close();
+		}
 		return;
 	}

@@ -354,11 +520,51 @@ void cParser::startIncludeFromParser(cParser& srcParser, bool ToLower, std::stri
 	if (mIncludeParser->mSize <= 0) {
 		ErrorLog("Bad include: can't open file \"" + includefile + "\"");
 	}
+
+	if (sceneryLayers)
+	{
+		mIncludeParser->sceneryLayers = true;
+		// content of an included scenery file forms a layer of its own. *.inc files are node templates
+		// reused all over the scenery, so their content stays in the layer of the file which includes them
+		if (mIncludeParser->mIncFile)
+		{
+			scene::Layers.template_used(includefile);
+			// include of a template made directly by a layer file is handled by the editor as a whole. templates
+			// included by that template are a part of it
+			auto const site{include_site()};
+			if (false == site.fixed && scene::Layers.instance() == 0 && mIncludeParser->mSize > 0)
+			{
+				scene::Layers.instance_begin(includefile, site.span);
+				mIncludeParser->mInstanceFile = true;
+			}
+		}
+		else if (mIncludeParser->mSize > 0)
+		{
+			auto site{include_site()};
+			site.parameters = false == mIncludeParser->parameters.empty();
+			scene::Layers.open(layername, site);
+			mIncludeParser->mLayerFile = true;
+		}
+	}
+}
+
+// describes the include directive being processed, for the scenery layer bookkeeping
+scene::include_site cParser::include_site() const
+{
+	scene::include_site site;
+	// the directive can be rewritten on scenery save only if we know where it is, and it sits in a layer file
+	site.fixed = mIncludeBegin < 0 || mFile.empty() || mIncFile;
+	if (false == site.fixed)
+	{
+		site.span = {mIncludeBegin, mTokenEnd};
+	}
+	return site;
 }

 bool cParser::handleIncludeIfPresent(std::string& token, bool ToLower, const char* Break) {
 	// token-mode include: token == "include"
 	if (expandIncludes && token == "include") {
+		mIncludeBegin = mTokenBegin;
 		std::string includefile;
 		if (allowRandomIncludes)
 			includefile = deserialize_random_set(*this);
@@ -374,6 +580,7 @@ bool cParser::handleIncludeIfPresent(std::string& token, bool ToLower, const cha

 	// line-mode HACK: Break == "\n\r" and line begins with "include"
 	if (std::strcmp(Break, "\n\r") == 0 && token.compare(0, 7, "include") == 0) {
+		mIncludeBegin = -1; // the directive is parsed out of a line of text, its exact location isn't known
 		cParser includeparser(token.substr(7));
 		std::string includefile;
 		if (allowRandomIncludes)
@@ -398,19 +605,30 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break)
 		if (out.empty())
 		{
 			mIncludeParser = nullptr;
-			out = readTokenFromStream(ToLower, Break);
+			readTokenFromStream(out, ToLower, Break);
 		}
 	}
 	else
 	{
-		out = readTokenFromStream(ToLower, Break);
+		readTokenFromStream(out, ToLower, Break);
 	}

-	stripFirstTokenBOM(out, ToLower, Break);
+	// NOTE: the checks repeat what the called methods test first, to skip the calls for vast majority of the tokens
+	if (mFirstToken)
+	{
+		stripFirstTokenBOM(out, ToLower, Break);
+	}

-	substituteParameters(out, ToLower);
+	if (false == parameters.empty())
+	{
+		substituteParameters(out, ToLower);
+	}

-	handleIncludeIfPresent(out, ToLower, Break);
+	if (out.size() >= 7 && out[0] == 'i')
+	{
+		// nothing else can be an include directive
+		handleIncludeIfPresent(out, ToLower, Break);
+	}
 }

 std::vector<std::string> cParser::readParameters(cParser &Input)
@@ -432,8 +650,9 @@ std::string cParser::readQuotes(char const Quote)
 	std::string token;
 	char c{0};
 	bool escaped = false;
-	while (mStream->get(c))
+	while (getChar(c))
 	{ // get all chars until the quote mark
+		++mPosition;
 		if (escaped)
 		{
 			escaped = false;
@@ -459,24 +678,23 @@ std::string cParser::readQuotes(char const Quote)

 void cParser::skipComment(std::string const &Endmark)
 { // pobieranie znaków aż do znalezienia znacznika końca
-	std::string input;
 	char c{0};
-	auto const endmarksize = Endmark.size();
-	while (mStream->get(c))
+	if (false == getChar(c))
 	{
-		if (c == '\n')
-		{
-			// update line counter
-			++mLine;
-		}
-		input += c;
-		if (input == Endmark) // szukanie znacznika końca
-			break;
-		if (input.size() >= endmarksize)
-		{
-			// keep the read text short, to avoid pointless string re-allocations on longer comments
-			input = input.substr(1);
-		}
+		return;
+	}
+	auto const begin{static_cast<std::size_t>(mPosition)};
+	// szukanie znacznika końca
+	// NOTE: with nothing to look for the comment runs until the end of data
+	auto const found{Endmark.empty() ? std::string::npos : mBuffer.find(Endmark, begin)};
+	auto const end{found != std::string::npos ? found + Endmark.size() : mBuffer.size()};
+	// update line counter
+	mLine += static_cast<std::size_t>(std::count(mBuffer.data() + begin, mBuffer.data() + end, '\n'));
+	mPosition = static_cast<std::streamoff>(end);
+	if (found == std::string::npos)
+	{
+		// the data ended before the comment did; record the attempt to read past the end
+		getChar(c);
 	}
 	return;
 }
@@ -523,12 +741,13 @@ void cParser::injectString(const std::string &str)
 	{
 		mIncludeParser = std::make_shared<cParser>(str, buffer_TEXT, "", LoadTraction, std::vector<std::string>(), allowRandomIncludes);
 		mIncludeParser->autoclear(m_autoclear);
+		mIncludeParser->sceneryLayers = sceneryLayers;
 	}
 }

 int cParser::getProgress() const
 {
-	return static_cast<int>(mStream->rdbuf()->pubseekoff(0, std::ios_base::cur) * 100 / mSize);
+	return (mSize > 0 ? static_cast<int>(mPosition * 100 / mSize) : 100);
 }

 int cParser::getFullProgress() const
@@ -566,6 +785,8 @@ void cParser::addCommentStyle(std::string const &Commentstart, std::string const
 {

 	mComments.insert(commentmap::value_type(Commentstart, Commentend));
+	// the character lookup depends on the comment marks
+	mCharClassesValid = false;
 }

 // returns name of currently open file, or empty string for text type stream
@@ -600,3 +821,59 @@ int cParser::LineMain() const
 {
 	return mIncludeParser ? -1 : mLine;
 }
+
+std::streamoff cParser::TokenBegin() const
+{
+	return mIncludeParser ? mIncludeParser->TokenBegin() : mTokenBegin;
+}
+
+std::streamoff cParser::TokenEnd() const
+{
+	return mIncludeParser ? mIncludeParser->TokenEnd() : mTokenEnd;
+}
+
+bool cParser::InLayerFile() const
+{
+	return mIncludeParser ? mIncludeParser->InLayerFile() : (false == mFile.empty() && false == mIncFile);
+}
+
+std::string_view cParser::remainingText() const
+{
+	if (false == tokens.empty())
+	{
+		return {};
+	}
+	// each parser on the way gets to change the tokens of the file being read
+	auto const *source{this};
+	while (true)
+	{
+		if (false == source->parameters.empty() || false == source->skipComments || source->mComments.size() != 2)
+		{
+			return {};
+		}
+		if (source->mIncludeParser == nullptr)
+		{
+			break;
+		}
+		source = source->mIncludeParser.get();
+	}
+	// NOTE: the first token of a file is left to the regular code, which deals with the byte order mark
+	if (source->mFirstToken || source->mEofBit || source->mFailBit || static_cast<std::size_t>(source->mPosition) >= source->mBuffer.size())
+	{
+		return {};
+	}
+	return std::string_view{source->mBuffer}.substr(static_cast<std::size_t>(source->mPosition));
+}
+
+void cParser::skipText(std::size_t const Count, std::size_t const Lines, std::size_t const Tokenlength)
+{
+	if (mIncludeParser)
+	{
+		mIncludeParser->skipText(Count, Lines, Tokenlength);
+		return;
+	}
+	mPosition = static_cast<std::streamoff>(std::min(static_cast<std::size_t>(mPosition) + Count, mBuffer.size()));
+	mLine += Lines;
+	mTokenEnd = mPosition;
+	mTokenBegin = mPosition - static_cast<std::streamoff>(std::min(Tokenlength, static_cast<std::size_t>(mPosition)));
+}