eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-22 19:49:19 +02:00
Code Issues 123 Wiki Activity

split parser with time-cutting enhancements

Browse Source
This commit is contained in:
jerrrrycho
2026-01-29 07:58:34 +01:00
parent ba6d3c09f2
commit fb1ed5c7b5
2 changed files with 215 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

330
parser.cpp
View File

@@ -23,9 +23,35 @@ http://mozilla.org/MPL/2.0/.
///////////////////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////////////////////
// cParser -- generic class for parsing text data. // cParser -- generic class for parsing text data.
namespace
{
inline std::array<bool, 256> makeBreakTable(const char *brk)
{
std::array<bool, 256> arr{};
for (unsigned char c : std::string_view(brk ? brk : ""))
{
arr[c] = true;
}
return arr;
}
inline char toLowerChar(char c)
{
return static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
}
inline bool startsWithBOM(const std::string &s)
{
return s.size() >= 3
&& static_cast<unsigned char>(s[0]) == 0xEF
&& static_cast<unsigned char>(s[1]) == 0xBB
&& static_cast<unsigned char>(s[2]) == 0xBF;
}
} // namespace
// constructors // constructors
cParser::cParser(std::string const &Stream, buffertype const Type, std::string Path, bool const Loadtraction, std::vector<std::string> Parameters, bool allowRandom) cParser::cParser(std::string const &Stream, buffertype const Type, std::string Path, bool const Loadtraction, std::vector<std::string> Parameters, bool allowRandom)
: mPath(Path), LoadTraction(Loadtraction), allowRandomIncludes(allowRandom) : allowRandomIncludes(allowRandom), LoadTraction(Loadtraction), mPath(Path)
{ {
// store to calculate sub-sequent includes from relative path // store to calculate sub-sequent includes from relative path
if (Type == buffertype::buffer_FILE) if (Type == buffertype::buffer_FILE)
@@ -192,181 +218,199 @@ bool cParser::getTokens(unsigned int Count, bool ToLower, const char *Break)
return true; return true;
} }
std::string cParser::readToken(bool ToLower, const char *Break) std::string cParser::readTokenFromDelegate(bool ToLower, const char *Break)
{ {
if (!mIncludeParser)
std::string token; return {};
if (mIncludeParser) std::string token = mIncludeParser->readToken(ToLower, Break);
{ if (token.empty())
// see if there's include parsing going on. clean up when it's done.
token = mIncludeParser->readToken(ToLower, Break);
if (true == token.empty())
{ {
mIncludeParser = nullptr; mIncludeParser = nullptr;
} }
return token;
} }
if (true == token.empty())
std::string cParser::readTokenFromStream(bool ToLower, const char *Break)
{ {
std::string token;
// get the token yourself if the delegation attempt failed // get the token yourself if the delegation attempt failed
char c{0};
do const auto breakTable = makeBreakTable(Break);
{ char c = 0;
while (mStream->peek() != EOF && strchr(Break, c = mStream->get()) == NULL)
{
if (ToLower) while (token.empty() && mStream->peek() != EOF) {
c = tolower(c); while (mStream->peek() != EOF) {
token += c; c = static_cast<char>(mStream->get());
if (findQuotes(token)) // do glue together words enclosed in quotes
continue; if (c == '\n') {
if (skipComments && trimComments(token)) // don't glue together words separated with comment
break;
}
if (c == '\n')
{
// update line counter
++mLine; ++mLine;
} }
} while (token == "" && mStream->peek() != EOF); // double check in case of consecutive separators
const unsigned char uc = static_cast<unsigned char>(c);
if (breakTable[uc]) {
// separator ends token (or continues skipping if token empty)
if (!token.empty())
break;
continue;
} }
// check the first token for potential presence of utf bom
if (mFirstToken) if (ToLower) c = toLowerChar(c);
{ token.push_back(c);
if (findQuotes(token)) {
continue; // glue quoted content
}
if (skipComments && trimComments(token)) {
break; // don't glue tokens separated by comment
}
}
}
return token;
}
void cParser::stripFirstTokenBOM(std::string& token, bool ToLower, const char* Break) {
if (!mFirstToken) return;
mFirstToken = false; mFirstToken = false;
if (token.rfind("\xef\xbb\xbf", 0) == 0)
{ if (startsWithBOM(token)) {
token.erase(0, 3); token.erase(0, 3);
} }
if (true == token.empty())
{ // if first "token" was standalone BOM, read the next real token (avoid recursion)
// potentially possible if our first token was standalone utf bom while (token.empty() && mStream->peek() != EOF) {
token = readToken(ToLower, Break); token = readToken(ToLower, Break);
// readToken will not re-enter BOM stripping because mFirstToken is now false
break;
} }
} }
if (false == parameters.empty()) void cParser::substituteParameters(std::string& token, bool ToLower) {
{ if (parameters.empty()) return;
// if there's parameter list, check the token for potential parameters to replace
size_t pos; // początek podmienianego ciągu // Replace occurrences of "(pN)" anywhere in token.
while ((pos = token.find("(p")) != std::string::npos) // Keep behavior: if missing parameter -> "none".
{ size_t pos = 0;
// check if the token is a parameter which should be replaced with stored true value while ((pos = token.find("(p", pos)) != std::string::npos) {
auto const parameter{token.substr(pos + 2, token.find(")", pos) - (pos + 2))}; // numer parametru const size_t close = token.find(')', pos);
token.erase(pos, token.find(")", pos) - pos + 1); // najpierw usunięcie "(pN)" if (close == std::string::npos) break; // malformed -> stop like old behavior (it would substr weirdly)
size_t nr = atoi(parameter.c_str()) - 1;
if (nr < parameters.size()) const std::string idxStr = token.substr(pos + 2, close - (pos + 2));
{ token.erase(pos, (close - pos) + 1);
token.insert(pos, parameters.at(nr)); // wklejenie wartości parametru
if (ToLower) const size_t nr = static_cast<size_t>(std::atoi(idxStr.c_str()));
for (; pos < parameters.at(nr).size(); ++pos) const std::string repl = (nr >= 1 && (nr - 1) < parameters.size())
token[pos] = tolower(token[pos]); ? parameters[nr - 1]
} : std::string("none");
else
token.insert(pos, "none"); // zabezpieczenie przed brakiem parametru const size_t insertPos = pos;
token.insert(insertPos, repl);
if (ToLower) {
// Lowercase only what we inserted (same intent as original)
for (size_t i = insertPos; i < insertPos + repl.size(); ++i) {
token[i] = toLowerChar(token[i]);
} }
} }
// launch child parser if include directive found. pos = insertPos + repl.size(); // continue after inserted text
// NOTE: parameter collecting uses default set of token separators. }
if (expandIncludes && token == "include") }
{
std::string includefile = allowRandomIncludes ? deserialize_random_set(*this) : readToken(ToLower); // nazwa pliku void cParser::skipIncludeBlock() {
// mimic original: while token != "end" readToken(true)
std::string t;
do {
t = readToken(true);
} while (t != "end" && !t.empty());
}
void cParser::startIncludeFromParser(cParser& srcParser, bool ToLower, std::string includefile) {
replace_slashes(includefile); replace_slashes(includefile);
if ((true == LoadTraction) || ((false == contains(includefile, "tr/")) && (false == contains(includefile, "tra/"))))
{ const bool allowTraction =
if (false == contains(includefile, "_ter.scm")) (true == LoadTraction) ||
{ ((false == contains(includefile, "tr/")) && (false == contains(includefile, "tra/")));
if (Global.ParserLogIncludes)
{ if (!allowTraction) {
// skip include block until "end" (original behavior in token-mode include)
skipIncludeBlock();
return;
}
const bool isTerrain = contains(includefile, "_ter.scm");
if (isTerrain && true == Global.file_binary_terrain_state) {
WriteLog("SBT found, ignoring: " + includefile);
readParameters(srcParser); // preserve original side-effect: still consume parameters
return;
}
if (Global.ParserLogIncludes) {
if (isTerrain) WriteLog("including terrain: " + includefile);
else {
// WriteLog("including: " + includefile); // WriteLog("including: " + includefile);
} }
mIncludeParser = std::make_shared<cParser>(includefile, buffer_FILE, mPath, LoadTraction, readParameters(*this)); }
mIncludeParser = std::make_shared<cParser>(
includefile, /*buffer_FILE*/ static_cast<buffertype>(/*buffer_FILE*/ 0), mPath, LoadTraction, readParameters(srcParser)
);
mIncludeParser->allowRandomIncludes = allowRandomIncludes; mIncludeParser->allowRandomIncludes = allowRandomIncludes;
mIncludeParser->autoclear(m_autoclear); mIncludeParser->autoclear(m_autoclear);
if (mIncludeParser->mSize <= 0)
{ if (mIncludeParser->mSize <= 0) {
ErrorLog("Bad include: can't open file \"" + includefile + "\""); ErrorLog("Bad include: can't open file \"" + includefile + "\"");
} }
} }
else
{ bool cParser::handleIncludeIfPresent(std::string& token, bool ToLower, const char* Break) {
if (true == Global.file_binary_terrain_state) // token-mode include: token == "include"
{ if (expandIncludes && token == "include") {
WriteLog("SBT found, ignoring: " + includefile); std::string includefile =
readParameters(*this); allowRandomIncludes ? deserialize_random_set(*this) : readToken(ToLower);
}
else startIncludeFromParser(*this, ToLower, std::move(includefile));
{
if (Global.ParserLogIncludes) // after processing include, return next token from current parser
{
WriteLog("including terrain: " + includefile);
}
mIncludeParser = std::make_shared<cParser>(includefile, buffer_FILE, mPath, LoadTraction, readParameters(*this));
mIncludeParser->allowRandomIncludes = allowRandomIncludes;
mIncludeParser->autoclear(m_autoclear);
if (mIncludeParser->mSize <= 0)
{
ErrorLog("Bad include: can't open file \"" + includefile + "\"");
}
}
}
}
else
{
while (token != "end")
{
token = readToken(true); // minimize risk of case mismatch on comparison
}
}
token = readToken(ToLower, Break); token = readToken(ToLower, Break);
return true;
} }
else if ((std::strcmp(Break, "\n\r") == 0) && (token.compare(0, 7, "include") == 0))
{ // line-mode HACK: Break == "\n\r" and line begins with "include"
// HACK: if the parser reads full lines we expect this line to contain entire include directive, to make parsing easier if ((std::strcmp(Break, "\n\r") == 0) && token.compare(0, 7, "include") == 0) {
cParser includeparser(token.substr(7)); cParser includeparser(token.substr(7));
std::string includefile = allowRandomIncludes ? deserialize_random_set(includeparser) : includeparser.readToken(ToLower); // nazwa pliku std::string includefile =
replace_slashes(includefile); allowRandomIncludes ? deserialize_random_set(includeparser) : includeparser.readToken(ToLower);
if ((true == LoadTraction) || ((false == contains(includefile, "tr/")) && (false == contains(includefile, "tra/"))))
{ startIncludeFromParser(includeparser, ToLower, std::move(includefile));
if (false == contains(includefile, "_ter.scm"))
{
if (Global.ParserLogIncludes)
{
// WriteLog("including: " + includefile);
}
mIncludeParser = std::make_shared<cParser>(includefile, buffer_FILE, mPath, LoadTraction, readParameters(includeparser));
mIncludeParser->allowRandomIncludes = allowRandomIncludes;
mIncludeParser->autoclear(m_autoclear);
if (mIncludeParser->mSize <= 0)
{
ErrorLog("Bad include: can't open file \"" + includefile + "\"");
}
}
else
{
if (true == Global.file_binary_terrain_state)
{
WriteLog("SBT found, ignoring: " + includefile);
readParameters(includeparser);
}
else
{
if (Global.ParserLogIncludes)
{
WriteLog("including terrain: " + includefile);
}
mIncludeParser = std::make_shared<cParser>(includefile, buffer_FILE, mPath, LoadTraction, readParameters(includeparser));
mIncludeParser->allowRandomIncludes = allowRandomIncludes;
mIncludeParser->autoclear(m_autoclear);
if (mIncludeParser->mSize <= 0)
{
ErrorLog("Bad include: can't open file \"" + includefile + "\"");
}
}
}
}
token = readToken(ToLower, Break); token = readToken(ToLower, Break);
return true;
} }
// all done
return false;
}
std::string cParser::readToken(bool ToLower, const char *Break)
{
std::string token;
token = readTokenFromDelegate(ToLower, Break);
if (token.empty())
{
token = readTokenFromStream(ToLower, Break);
}
stripFirstTokenBOM(token, ToLower, Break);
// 4) parameter substitution
substituteParameters(token, ToLower);
// 5) include directive handling (may mutate token to next token)
handleIncludeIfPresent(token, ToLower, Break);
return token; return token;
} }

9
parser.h
View File

@@ -66,6 +66,11 @@ class cParser //: public std::stringstream
return m_autoclear; } return m_autoclear; }
bool bool
getTokens( unsigned int Count = 1, bool ToLower = true, char const *Break = "\n\r\t ;" ); getTokens( unsigned int Count = 1, bool ToLower = true, char const *Break = "\n\r\t ;" );
std::string readTokenFromDelegate(bool ToLower, const char *Break);
std::string readTokenFromStream(bool ToLower, const char *Break);
void stripFirstTokenBOM(std::string &token, bool ToLower, const char *Break);
void substituteParameters(std::string &token, bool ToLower);
void skipIncludeBlock();
// returns next incoming token, if any, without removing it from the set // returns next incoming token, if any, without removing it from the set
inline inline
std::string std::string
@@ -95,9 +100,11 @@ class cParser //: public std::stringstream
bool skipComments = true; bool skipComments = true;
private: private:
void startIncludeFromParser(cParser &srcParser, bool ToLower, std::string includefile);
bool handleIncludeIfPresent(std::string &token, bool ToLower, const char *Break);
// methods: // methods:
std::string readToken(bool ToLower = true, const char *Break = "\n\r\t ;"); std::string readToken(bool ToLower = true, const char *Break = "\n\r\t ;");
std::vector<std::string> readParameters( cParser &Input ); static std::vector<std::string> readParameters( cParser &Input );
std::string readQuotes( char const Quote = '\"' ); std::string readQuotes( char const Quote = '\"' );
void skipComment( std::string const &Endmark ); void skipComment( std::string const &Endmark );
bool findQuotes( std::string &String ); bool findQuotes( std::string &String );