/* This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "stdafx.h" #include "parser.h" #include "utilities.h" #include "Logs.h" #include "scenenodegroups.h" /* MaSzyna EU07 locomotive simulator parser Copyright (C) 2003 TOLARIS */ ///////////////////////////////////////////////////////////////////////////////////////////////////// // cParser -- generic class for parsing text data. namespace { inline std::array makeBreakTable(const char *brk) { std::array arr{}; for (unsigned char c : std::string_view(brk ? brk : "")) { arr[c] = true; } return arr; } inline char toLowerChar(char c) { return static_cast(std::tolower(static_cast(c))); } inline bool startsWithBOM(const std::string &s) { return s.size() >= 3 && static_cast(s[0]) == 0xEF && static_cast(s[1]) == 0xBB && static_cast(s[2]) == 0xBF; } } // namespace // constructors cParser::cParser(std::string const &Stream, buffertype const Type, std::string Path, bool const Loadtraction, std::vector Parameters, bool allowRandom) : allowRandomIncludes(allowRandom), LoadTraction(Loadtraction), mPath(Path) { // store to calculate sub-sequent includes from relative path if (Type == buffertype::buffer_FILE) { mFile = Stream; } // reset pointers and attach proper type of buffer switch (Type) { case buffer_FILE: { Path.append(Stream); mStream = std::make_shared(Path, std::ios_base::binary); // content of *.inc files is potentially grouped together if ((Stream.size() >= 4) && (ToLower(Stream.substr(Stream.size() - 4)) == ".inc")) { mIncFile = true; scene::Groups.create(); } break; } case buffer_TEXT: { mStream = std::make_shared(Stream); break; } default: { break; } } // calculate stream size if (mStream) { if (true == mStream->fail()) { ErrorLog("Failed to open file \"" + Path + "\""); } else { mSize = mStream->rdbuf()->pubseekoff(0, std::ios_base::end); mStream->rdbuf()->pubseekoff(0, std::ios_base::beg); mLine = 1; } } // set parameter set if one was provided if (false == Parameters.empty()) { parameters.swap(Parameters); } } // destructor cParser::~cParser() { if (true == mIncFile) { // wrap up the node group holding content of processed file scene::Groups.close(); } } template <> glm::vec3 cParser::getToken(bool const ToLower, char const *Break) { // NOTE: this specialization ignores default arguments getTokens(3, false, "\n\r\t ,;[]"); glm::vec3 output; *this >> output.x >> output.y >> output.z; return output; }; template <> cParser &cParser::operator>>(std::string &Right) { if (true == this->tokens.empty()) { return *this; } Right = this->tokens.front(); this->tokens.pop_front(); return *this; } template <> cParser &cParser::operator>>(bool &Right) { if (true == this->tokens.empty()) { return *this; } Right = ((this->tokens.front() == "true") || (this->tokens.front() == "yes") || (this->tokens.front() == "1")); this->tokens.pop_front(); return *this; } template <> bool cParser::getToken(bool const ToLower, const char *Break) { auto const token = getToken(true, Break); return ((token == "true") || (token == "yes") || (token == "1")); } // methods cParser &cParser::autoclear(bool const Autoclear) { m_autoclear = Autoclear; if (mIncludeParser) { mIncludeParser->autoclear(Autoclear); } return *this; } bool cParser::getTokens(unsigned int Count, bool ToLower, const char *Break) { if (true == m_autoclear) { // legacy parser behaviour tokens.clear(); } /* if (LoadTraction==true) trtest="niemaproblema"; //wczytywać else trtest="x"; //nie wczytywać */ /* int i; this->str(""); this->clear(); */ for (unsigned int i = tokens.size(); i < Count; ++i) { std::string token = readToken(ToLower, Break); if (true == token.empty()) { // no more tokens break; } // collect parameters tokens.emplace_back(token); /* if (i == 0) this->str(token); else { std::string temp = this->str(); temp.append("\n"); temp.append(token); this->str(temp); } */ } if (tokens.size() < Count) return false; else return true; } std::string cParser::readTokenFromDelegate(bool ToLower, const char *Break) { if (!mIncludeParser) return {}; std::string token = mIncludeParser->readToken(ToLower, Break); if (token.empty()) { mIncludeParser = nullptr; } return token; } std::string cParser::readTokenFromStream(bool ToLower, const char *Break) { std::string token; // get the token yourself if the delegation attempt failed const auto breakTable = makeBreakTable(Break); char c = 0; while (token.empty() && mStream->peek() != EOF) { while (mStream->peek() != EOF) { c = static_cast(mStream->get()); if (c == '\n') { ++mLine; } const unsigned char uc = static_cast(c); if (breakTable[uc]) { // separator ends token (or continues skipping if token empty) if (!token.empty()) break; continue; } if (ToLower) c = toLowerChar(c); token.push_back(c); if (findQuotes(token)) { continue; // glue quoted content } if (skipComments && trimComments(token)) { break; // don't glue tokens separated by comment } } } return token; } void cParser::stripFirstTokenBOM(std::string& token, bool ToLower, const char* Break) { if (!mFirstToken) return; mFirstToken = false; if (startsWithBOM(token)) { token.erase(0, 3); } // if first "token" was standalone BOM, read the next real token (avoid recursion) while (token.empty() && mStream->peek() != EOF) { token = readToken(ToLower, Break); // readToken will not re-enter BOM stripping because mFirstToken is now false break; } } void cParser::substituteParameters(std::string& token, bool ToLower) { if (parameters.empty()) return; // Replace occurrences of "(pN)" anywhere in token. // Keep behavior: if missing parameter -> "none". size_t pos = 0; while ((pos = token.find("(p", pos)) != std::string::npos) { const size_t close = token.find(')', pos); if (close == std::string::npos) break; // malformed -> stop like old behavior (it would substr weirdly) const std::string idxStr = token.substr(pos + 2, close - (pos + 2)); token.erase(pos, (close - pos) + 1); const size_t nr = static_cast(std::atoi(idxStr.c_str())); const std::string repl = (nr >= 1 && (nr - 1) < parameters.size()) ? parameters[nr - 1] : std::string("none"); const size_t insertPos = pos; token.insert(insertPos, repl); if (ToLower) { // Lowercase only what we inserted (same intent as original) for (size_t i = insertPos; i < insertPos + repl.size(); ++i) { token[i] = toLowerChar(token[i]); } } pos = insertPos + repl.size(); // continue after inserted text } } void cParser::skipIncludeBlock() { // mimic original: while token != "end" readToken(true) std::string t; do { t = readToken(true); } while (t != "end" && !t.empty()); } void cParser::startIncludeFromParser(cParser& srcParser, bool ToLower, std::string includefile) { replace_slashes(includefile); const bool allowTraction = (true == LoadTraction) || ((false == contains(includefile, "tr/")) && (false == contains(includefile, "tra/"))); if (!allowTraction) { // skip include block until "end" (original behavior in token-mode include) skipIncludeBlock(); return; } const bool isTerrain = contains(includefile, "_ter.scm"); if (isTerrain && true == Global.file_binary_terrain_state) { WriteLog("SBT found, ignoring: " + includefile); readParameters(srcParser); // preserve original side-effect: still consume parameters return; } if (Global.ParserLogIncludes) { if (isTerrain) WriteLog("including terrain: " + includefile); else { // WriteLog("including: " + includefile); } } mIncludeParser = std::make_shared( includefile, /*buffer_FILE*/ static_cast(/*buffer_FILE*/ 0), mPath, LoadTraction, readParameters(srcParser) ); mIncludeParser->allowRandomIncludes = allowRandomIncludes; mIncludeParser->autoclear(m_autoclear); if (mIncludeParser->mSize <= 0) { ErrorLog("Bad include: can't open file \"" + includefile + "\""); } } bool cParser::handleIncludeIfPresent(std::string& token, bool ToLower, const char* Break) { // token-mode include: token == "include" if (expandIncludes && token == "include") { std::string includefile = allowRandomIncludes ? deserialize_random_set(*this) : readToken(ToLower); startIncludeFromParser(*this, ToLower, std::move(includefile)); // after processing include, return next token from current parser token = readToken(ToLower, Break); return true; } // line-mode HACK: Break == "\n\r" and line begins with "include" if ((std::strcmp(Break, "\n\r") == 0) && token.compare(0, 7, "include") == 0) { cParser includeparser(token.substr(7)); std::string includefile = allowRandomIncludes ? deserialize_random_set(includeparser) : includeparser.readToken(ToLower); startIncludeFromParser(includeparser, ToLower, std::move(includefile)); token = readToken(ToLower, Break); return true; } return false; } std::string cParser::readToken(bool ToLower, const char *Break) { std::string token; token = readTokenFromDelegate(ToLower, Break); if (token.empty()) { token = readTokenFromStream(ToLower, Break); } stripFirstTokenBOM(token, ToLower, Break); substituteParameters(token, ToLower); handleIncludeIfPresent(token, ToLower, Break); return token; } std::vector cParser::readParameters(cParser &Input) { std::vector includeparameters; std::string parameter = Input.readToken(false); // w parametrach nie zmniejszamy while ((parameter.empty() == false) && (parameter != "end")) { includeparameters.emplace_back(parameter); parameter = Input.readToken(false); } return includeparameters; } std::string cParser::readQuotes(char const Quote) { // read the stream until specified char or stream end std::string token = ""; char c{0}; bool escaped = false; while (mStream->peek() != EOF) { // get all chars until the quote mark c = mStream->get(); if (escaped) { escaped = false; } else { if (c == '\\') { escaped = true; continue; } else if (c == Quote) break; } if (c == '\n') ++mLine; // update line counter token += c; } return token; } void cParser::skipComment(std::string const &Endmark) { // pobieranie znaków aż do znalezienia znacznika końca std::string input = ""; char c{0}; auto const endmarksize = Endmark.size(); while (mStream->peek() != EOF) { // o ile nie koniec pliku c = mStream->get(); // pobranie znaku if (c == '\n') { // update line counter ++mLine; } input += c; if (input == Endmark) // szukanie znacznika końca break; if (input.size() >= endmarksize) { // keep the read text short, to avoid pointless string re-allocations on longer comments input = input.substr(1); } } return; } bool cParser::findQuotes(std::string &String) { if (String.back() == '\"') { String.pop_back(); String += readQuotes(); return true; } return false; } bool cParser::trimComments(std::string &String) { for (auto const &comment : mComments) { if (String.size() < comment.first.size()) { continue; } if (String.compare(String.size() - comment.first.size(), comment.first.size(), comment.first) == 0) { skipComment(comment.second); String.resize(String.rfind(comment.first)); return true; } } return false; } void cParser::injectString(const std::string &str) { if (mIncludeParser) { mIncludeParser->injectString(str); } else { mIncludeParser = std::make_shared(str, buffer_TEXT, "", LoadTraction, std::vector(), allowRandomIncludes); mIncludeParser->autoclear(m_autoclear); } } int cParser::getProgress() const { return static_cast(mStream->rdbuf()->pubseekoff(0, std::ios_base::cur) * 100 / mSize); } int cParser::getFullProgress() const { int progress = getProgress(); if (mIncludeParser) return progress + ((100 - progress) * (mIncludeParser->getProgress()) / 100); else return progress; } std::size_t cParser::countTokens(std::string const &Stream, std::string Path) { return cParser(Stream, buffer_FILE, Path).count(); } std::size_t cParser::count() { std::string token; size_t count{0}; do { token = ""; token = readToken(false); ++count; } while (false == token.empty()); return count - 1; } void cParser::addCommentStyle(std::string const &Commentstart, std::string const &Commentend) { mComments.insert(commentmap::value_type(Commentstart, Commentend)); } // returns name of currently open file, or empty string for text type stream std::string cParser::Name() const { if (mIncludeParser) { return mIncludeParser->Name(); } else { return mPath + mFile; } } // returns number of currently processed line std::size_t cParser::Line() const { if (mIncludeParser) { return mIncludeParser->Line(); } else { return mLine; } } int cParser::LineMain() const { return mIncludeParser ? -1 : mLine; }