diff --git a/utilities/parser.cpp b/utilities/parser.cpp index cb88a989..ff10c0b4 100644 --- a/utilities/parser.cpp +++ b/utilities/parser.cpp @@ -25,24 +25,14 @@ http://mozilla.org/MPL/2.0/. namespace { -inline std::array makeBreakTable(const char *brk) -{ - std::array arr{}; - for (unsigned char c : std::string_view(brk ? brk : "")) - { - arr[c] = true; - } - return arr; -} - inline char toLowerChar(char c) { // Only fold ASCII letters. Bytes >= 0x80 belong to multibyte UTF-8 // sequences and must be passed through untouched, otherwise a non-"C" // global locale could remap them and corrupt UTF-8 encoded text. - const unsigned char uc = static_cast(c); - if (uc < 0x80) - return static_cast(std::tolower(uc)); + // The remaining ASCII characters have no lower case form, and don't need the call. + if (c >= 'A' && c <= 'Z') + return static_cast(std::tolower(static_cast(c))); return c; } @@ -70,7 +60,28 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P case buffer_FILE: { Path.append(Stream); - mStream = std::make_shared(Path, std::ios_base::binary); + // the file is read in one go, tokens are then extracted straight from the memory + std::ifstream file(Path, std::ios_base::binary); + if (file.fail()) + { + mFailBit = true; + } + else + { + auto const size{file.rdbuf()->pubseekoff(0, std::ios_base::end)}; + file.rdbuf()->pubseekoff(0, std::ios_base::beg); + if (size > 0 && static_cast(size) < mBuffer.max_size()) + { + mBuffer.resize(static_cast(size)); + file.read(mBuffer.data(), static_cast(mBuffer.size())); + mBuffer.resize(static_cast(std::max(0, file.gcount()))); + } + else if (size != 0) + { + // not a file which can be read, e.g. a directory, which some systems allow to open + mFailBit = true; + } + } // content of *.inc files is potentially grouped together if (Stream.size() >= 4 && ToLower(Stream.substr(Stream.size() - 4)) == ".inc") { @@ -81,7 +92,7 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P } case buffer_TEXT: { - mStream = std::make_shared(Stream); + mBuffer = Stream; break; } default: @@ -90,18 +101,14 @@ cParser::cParser(std::string const &Stream, buffertype const Type, std::string P } } // calculate stream size - if (mStream) + if (true == mFailBit) { - if (true == mStream->fail()) - { - ErrorLog("Failed to open file \"" + Path + "\""); - } - else - { - mSize = mStream->rdbuf()->pubseekoff(0, std::ios_base::end); - mStream->rdbuf()->pubseekoff(0, std::ios_base::beg); - mLine = 1; - } + ErrorLog("Failed to open file \"" + Path + "\""); + } + else + { + mSize = static_cast(mBuffer.size()); + mLine = 1; } // set parameter set if one was provided if (false == Parameters.empty()) @@ -147,7 +154,7 @@ template <> cParser &cParser::operator>>(std::string &Right) return *this; } - Right = this->tokens.front(); + Right = std::move(this->tokens.front()); this->tokens.pop_front(); return *this; @@ -167,6 +174,25 @@ template <> cParser &cParser::operator>>(bool &Right) return *this; } +template <> std::string cParser::getToken(bool const ToLower, const char *Break) +{ + // does what the generic version would, minus the trip of the token through the queue + if (true == m_autoclear) + { + tokens.clear(); + } + std::string token; + if (false == tokens.empty()) + { + *this >> token; + } + else + { + readToken(token, ToLower, Break); + } + return token; +} + template <> bool cParser::getToken(bool const ToLower, const char *Break) { @@ -234,31 +260,141 @@ bool cParser::getTokens(unsigned int Count, bool ToLower, const char *Break) return true; } -std::string cParser::readTokenFromStream(bool ToLower, const char *Break) +// mimics std::istream::peek() != EOF of the stream the parser used to read from +bool cParser::hasChar() { - std::string token; - token.reserve(64); + if (mEofBit || mFailBit) + { + // an attempt to read from a stream which isn't in good state marks it as failed + mFailBit = true; + return false; + } + if (static_cast(mPosition) >= mBuffer.size()) + { + mEofBit = true; + return false; + } + return true; +} - const auto breakTable = makeBreakTable(Break); +// mimics std::istream::get(char&). NOTE: doesn't advance the read position, the caller does +bool cParser::getChar(char &Char) +{ + if (mEofBit || mFailBit) + { + mFailBit = true; + return false; + } + if (static_cast(mPosition) >= mBuffer.size()) + { + // failed extraction at the end of data sets both flags + mEofBit = true; + mFailBit = true; + return false; + } + Char = mBuffer[static_cast(mPosition)]; + return true; +} + +// prepares character lookup for specified set of token separators +void cParser::updateCharClasses(char const *Break) +{ + if (Break == nullptr) + { + Break = ""; + } + if (mCharClassesValid && std::strcmp(mCharClassesBreak.c_str(), Break) == 0) + { + // the set of separators rarely changes from one token to another + return; + } + mCharClassesBreak = Break; + mCharClasses.fill(0); + for (unsigned char const c : mCharClassesBreak) + { + mCharClasses[c] |= char_break; + } + // characters which need more than being added to the token: + // line ends are counted, + mCharClasses[static_cast('\n')] |= char_special; + // quotes glue the text they enclose into a single token, + mCharClasses[static_cast('\"')] |= char_special; + // and a comment can start only once the last character of its opening mark arrives + for (auto const &comment : mComments) + { + for (auto character{0}; character < 256; ++character) + { + // NOTE: the mark is compared with the token, which can hold the character converted to lower case + if (comment.first.empty() || comment.first.back() == static_cast(character) || comment.first.back() == toLowerChar(static_cast(character))) + { + mCharClasses[character] |= char_special; + } + } + } + mCharClassesValid = true; +} + +void cParser::readTokenFromStream(std::string &token, bool ToLower, const char *Break) +{ + token.clear(); + + updateCharClasses(Break); char c = 0; bool separated = false; // the token was ended by a separator, which isn't a part of it - while (token.empty() && mStream->peek() != EOF) { - while (mStream->peek() != EOF) { // idk why but with mStream->get(c) not all cars are loaded - c = static_cast(mStream->get()); + while (token.empty() && hasChar()) { + while (hasChar()) { + auto const *const data{mBuffer.data()}; + auto const size{mBuffer.size()}; + auto const begin{static_cast(mPosition)}; + if (mCharClasses[static_cast(data[begin])] == 0) + { + // run of regular characters, which only need to be added to the token + auto end{begin + 1}; + while (end < size && mCharClasses[static_cast(data[end])] == 0) + { + ++end; + } + if (token.empty()) + { + mTokenBegin = mPosition; + } + auto const converted{token.size()}; + token.append(data + begin, end - begin); + if (ToLower) + { + for (auto index{converted}; index < token.size(); ++index) + { + token[index] = toLowerChar(token[index]); + } + } + mPosition = static_cast(end); + continue; + } + + c = data[begin]; ++mPosition; if (c == '\n') { ++mLine; } const unsigned char uc = static_cast(c); - if (breakTable[uc]) { + if (mCharClasses[uc] & char_break) { // separator ends token (or continues skipping if token empty) if (!token.empty()) { separated = true; break; } + // the rest of a run of separators is skipped right away + auto next{begin + 1}; + while (next < size && (mCharClasses[static_cast(data[next])] & char_break)) { + if (data[next] == '\n') { + ++mLine; + } + ++next; + } + mPosition = static_cast(next); continue; } @@ -278,8 +414,6 @@ std::string cParser::readTokenFromStream(bool ToLower, const char *Break) } // NOTE: comment glued to the end of a token is counted as a part of it mTokenEnd = separated ? mPosition - 1 : mPosition; - - return token; } void cParser::stripFirstTokenBOM(std::string& token, bool ToLower, const char* Break) { @@ -292,7 +426,7 @@ void cParser::stripFirstTokenBOM(std::string& token, bool ToLower, const char* B } // if first "token" was standalone BOM, read the next real token (avoid recursion) - while (token.empty() && mStream->peek() != EOF) { + while (token.empty() && hasChar()) { readToken(token, ToLower, Break); // readToken will not re-enter BOM stripping because mFirstToken is now false break; @@ -467,19 +601,30 @@ void cParser::readToken(std::string &out, bool ToLower, const char *Break) if (out.empty()) { mIncludeParser = nullptr; - out = readTokenFromStream(ToLower, Break); + readTokenFromStream(out, ToLower, Break); } } else { - out = readTokenFromStream(ToLower, Break); + readTokenFromStream(out, ToLower, Break); } - stripFirstTokenBOM(out, ToLower, Break); + // NOTE: the checks repeat what the called methods test first, to skip the calls for vast majority of the tokens + if (mFirstToken) + { + stripFirstTokenBOM(out, ToLower, Break); + } - substituteParameters(out, ToLower); + if (false == parameters.empty()) + { + substituteParameters(out, ToLower); + } - handleIncludeIfPresent(out, ToLower, Break); + if (out.size() >= 7 && out[0] == 'i') + { + // nothing else can be an include directive + handleIncludeIfPresent(out, ToLower, Break); + } } std::vector cParser::readParameters(cParser &Input) @@ -501,7 +646,7 @@ std::string cParser::readQuotes(char const Quote) std::string token; char c{0}; bool escaped = false; - while (mStream->get(c)) + while (getChar(c)) { // get all chars until the quote mark ++mPosition; if (escaped) @@ -529,25 +674,23 @@ std::string cParser::readQuotes(char const Quote) void cParser::skipComment(std::string const &Endmark) { // pobieranie znaków aż do znalezienia znacznika końca - std::string input; char c{0}; - auto const endmarksize = Endmark.size(); - while (mStream->get(c)) + if (false == getChar(c)) { - ++mPosition; - if (c == '\n') - { - // update line counter - ++mLine; - } - input += c; - if (input == Endmark) // szukanie znacznika końca - break; - if (input.size() >= endmarksize) - { - // keep the read text short, to avoid pointless string re-allocations on longer comments - input = input.substr(1); - } + return; + } + auto const begin{static_cast(mPosition)}; + // szukanie znacznika końca + // NOTE: with nothing to look for the comment runs until the end of data + auto const found{Endmark.empty() ? std::string::npos : mBuffer.find(Endmark, begin)}; + auto const end{found != std::string::npos ? found + Endmark.size() : mBuffer.size()}; + // update line counter + mLine += static_cast(std::count(mBuffer.data() + begin, mBuffer.data() + end, '\n')); + mPosition = static_cast(end); + if (found == std::string::npos) + { + // the data ended before the comment did; record the attempt to read past the end + getChar(c); } return; } @@ -600,7 +743,7 @@ void cParser::injectString(const std::string &str) int cParser::getProgress() const { - return static_cast(mStream->rdbuf()->pubseekoff(0, std::ios_base::cur) * 100 / mSize); + return (mSize > 0 ? static_cast(mPosition * 100 / mSize) : 100); } int cParser::getFullProgress() const @@ -638,6 +781,8 @@ void cParser::addCommentStyle(std::string const &Commentstart, std::string const { mComments.insert(commentmap::value_type(Commentstart, Commentend)); + // the character lookup depends on the comment marks + mCharClassesValid = false; } // returns name of currently open file, or empty string for text type stream diff --git a/utilities/parser.h b/utilities/parser.h index bafcdff9..23ae0ef3 100644 --- a/utilities/parser.h +++ b/utilities/parser.h @@ -14,6 +14,9 @@ http://mozilla.org/MPL/2.0/. #include #include #include +#include +#include +#include namespace scene { @@ -61,11 +64,11 @@ class cParser //: public std::stringstream inline bool eof() { - return mStream->eof(); }; + return mEofBit; }; inline bool ok() { - return !mStream->fail(); }; + return !mFailBit; }; cParser & autoclear( bool const Autoclear ); inline @@ -74,7 +77,7 @@ class cParser //: public std::stringstream return m_autoclear; } bool getTokens( unsigned int Count = 1, bool ToLower = true, char const *Break = "\n\r\t ;" ); - std::string readTokenFromStream(bool ToLower, const char *Break); + void readTokenFromStream(std::string &token, bool ToLower, const char *Break); void stripFirstTokenBOM(std::string &token, bool ToLower, const char *Break); void substituteParameters(std::string &token, bool ToLower); void skipIncludeBlock(); @@ -110,6 +113,35 @@ class cParser //: public std::stringstream bool sceneryLayers = false; // included scenery files are registered as scenery layers (scenery opened for editing) private: + // types: + // retrieved tokens waiting for pick up. a vector with moving head instead of a deque, + // as the latter can allocate memory for each stored item + struct token_queue { + std::vector items; + std::size_t head { 0 }; + + bool empty() const { + return head == items.size(); } + std::size_t size() const { + return items.size() - head; } + std::string & front() { + return items[ head ]; } + std::string const & front() const { + return items[ head ]; } + void emplace_back( std::string &&Token ) { + items.emplace_back( std::move( Token ) ); } + void pop_front() { + ++head; + if( head == items.size() ) { + clear(); } + else if( head >= 64 && head * 2 >= items.size() ) { + // keep the queue from growing without end if it never gets fully emptied + items.erase( items.begin(), items.begin() + static_cast( head ) ); + head = 0; } } + void clear() { + items.clear(); + head = 0; } + }; void startIncludeFromParser(cParser &srcParser, bool ToLower, std::string includefile); scene::include_site include_site() const; bool handleIncludeIfPresent(std::string &token, bool ToLower, const char *Break); @@ -121,10 +153,26 @@ class cParser //: public std::stringstream bool findQuotes( std::string &String ); bool trimComments( std::string &String ); std::size_t count(); + // input access. these mimic peek() and get() of the stream the parser used to read from, + // including their effect on the state reported by eof() and ok() + bool hasChar(); + bool getChar( char &Char ); + void updateCharClasses( char const *Break ); + // converts text holding a plain number without involving a stream. returns: true on success, false if the text needs regular treatment + template + static bool parseNumber( std::string const &Token, Type_ &Output ); // members: bool m_autoclear { true }; // unretrieved tokens are discarded when another read command is issued (legacy behaviour) bool LoadTraction { true }; // load traction? - std::shared_ptr mStream; // relevant kind of buffer is attached on creation. + std::string mBuffer; // content of the open file or the provided text, loaded on creation. + bool mEofBit { false }; // input state, maintained the way a stream would do it + bool mFailBit { false }; + enum charclass : unsigned char { + char_break = 1, // separates tokens + char_special = 2 }; // can't be simply added to the token: line end, quote, or potential end of a comment mark + std::array mCharClasses {}; // lookup built for the most recently used set of token separators + std::string mCharClassesBreak; // set of separators the lookup was built for + bool mCharClassesValid { false }; std::string mFile; // name of the open file, if any std::string mPath; // path to open stream, for relative path lookups. std::streamoff mSize { 0 }; // size of open stream, for progress report. @@ -143,7 +191,7 @@ class cParser //: public std::stringstream commentmap::value_type( "//", "\n" ) }; std::shared_ptr mIncludeParser; // child class to handle include directives. std::vector parameters; // parameter list for included file. - std::deque tokens; + token_queue tokens; }; @@ -152,14 +200,53 @@ glm::vec3 cParser::getToken( bool const ToLower, const char *Break ); +template +bool +cParser::parseNumber( std::string const &Token, Type_ &Output ) { + + using type = std::remove_cv_t; + if constexpr( +#ifdef __cpp_lib_to_chars + std::is_floating_point_v || +#endif + ( std::is_integral_v + && false == std::is_same_v + && false == std::is_same_v + && false == std::is_same_v + && false == std::is_same_v + && false == std::is_same_v + && false == std::is_same_v + && false == std::is_same_v + && false == std::is_same_v ) ) { + + auto const *first { Token.data() }; + auto const *last { first + Token.size() }; + // only text which is a number from start to end is converted here. anything else, including forms + // the stream extraction doesn't recognize (inf, nan), is left for the stream to make sense of + auto const *digits { ( first != last && *first == '-' ) ? first + 1 : first }; + if( digits == last ) { return false; } + if( ( *digits < '0' || *digits > '9' ) && *digits != '.' ) { return false; } + type value {}; + auto const result { std::from_chars( first, last, value ) }; + if( result.ec != std::errc() || result.ptr != last ) { return false; } + Output = value; + return true; + } + else { + return false; + } +} + template cParser& cParser::operator>>( Type_ &Right ) { if( true == this->tokens.empty() ) { return *this; } - std::stringstream converter( this->tokens.front() ); - converter >> Right; + if( false == parseNumber( this->tokens.front(), Right ) ) { + std::stringstream converter( this->tokens.front() ); + converter >> Right; + } this->tokens.pop_front(); return *this; @@ -176,3 +263,7 @@ cParser::operator>>( bool &Right ); template<> bool cParser::getToken( bool const ToLower, const char *Break ); + +template<> +std::string +cParser::getToken( bool const ToLower, const char *Break );