perf(parser): tokens read from a memory buffer instead of a stream, plain numbers converted with from_chars, token queue without per-item allocations; stream state reported by eof() and ok() kept as it was

utilities/parser.h

@@ -14,6 +14,9 @@ http://mozilla.org/MPL/2.0/.
 #include <fstream>
 #include <vector>
 #include <map>
+#include <array>
+#include <charconv>
+#include <type_traits>
 
 namespace scene
 {
@@ -61,11 +64,11 @@ class cParser //: public std::stringstream
     inline
     bool
         eof() {
-            return mStream->eof(); };
+            return mEofBit; };
     inline
     bool
         ok() {
-            return !mStream->fail(); };
+            return !mFailBit; };
     cParser &
         autoclear( bool const Autoclear );
     inline
@@ -74,7 +77,7 @@ class cParser //: public std::stringstream
             return m_autoclear; }
     bool
         getTokens( unsigned int Count = 1, bool ToLower = true, char const *Break = "\n\r\t ;" );
-	std::string readTokenFromStream(bool ToLower, const char *Break);
+	void readTokenFromStream(std::string &token, bool ToLower, const char *Break);
 	void stripFirstTokenBOM(std::string &token, bool ToLower, const char *Break);
 	void substituteParameters(std::string &token, bool ToLower);
 	void skipIncludeBlock();
@@ -110,6 +113,35 @@ class cParser //: public std::stringstream
 	bool sceneryLayers = false; // included scenery files are registered as scenery layers (scenery opened for editing)
 
   private:
+    // types:
+    // retrieved tokens waiting for pick up. a vector with moving head instead of a deque,
+    // as the latter can allocate memory for each stored item
+    struct token_queue {
+        std::vector<std::string> items;
+        std::size_t head { 0 };
+
+        bool empty() const {
+            return head == items.size(); }
+        std::size_t size() const {
+            return items.size() - head; }
+        std::string & front() {
+            return items[ head ]; }
+        std::string const & front() const {
+            return items[ head ]; }
+        void emplace_back( std::string &&Token ) {
+            items.emplace_back( std::move( Token ) ); }
+        void pop_front() {
+            ++head;
+            if( head == items.size() ) {
+                clear(); }
+            else if( head >= 64 && head * 2 >= items.size() ) {
+                // keep the queue from growing without end if it never gets fully emptied
+                items.erase( items.begin(), items.begin() + static_cast<std::ptrdiff_t>( head ) );
+                head = 0; } }
+        void clear() {
+            items.clear();
+            head = 0; }
+    };
 	void startIncludeFromParser(cParser &srcParser, bool ToLower, std::string includefile);
 	scene::include_site include_site() const;
 	bool handleIncludeIfPresent(std::string &token, bool ToLower, const char *Break);
@@ -121,10 +153,26 @@ class cParser //: public std::stringstream
     bool findQuotes( std::string &String );
     bool trimComments( std::string &String );
     std::size_t count();
+    // input access. these mimic peek() and get() of the stream the parser used to read from,
+    // including their effect on the state reported by eof() and ok()
+    bool hasChar();
+    bool getChar( char &Char );
+    void updateCharClasses( char const *Break );
+    // converts text holding a plain number without involving a stream. returns: true on success, false if the text needs regular treatment
+    template <typename Type_>
+    static bool parseNumber( std::string const &Token, Type_ &Output );
     // members:
     bool m_autoclear { true }; // unretrieved tokens are discarded when another read command is issued (legacy behaviour)
     bool LoadTraction { true }; // load traction?
-    std::shared_ptr<std::istream> mStream; // relevant kind of buffer is attached on creation.
+    std::string mBuffer; // content of the open file or the provided text, loaded on creation.
+    bool mEofBit { false }; // input state, maintained the way a stream would do it
+    bool mFailBit { false };
+    enum charclass : unsigned char {
+        char_break = 1, // separates tokens
+        char_special = 2 }; // can't be simply added to the token: line end, quote, or potential end of a comment mark
+    std::array<unsigned char, 256> mCharClasses {}; // lookup built for the most recently used set of token separators
+    std::string mCharClassesBreak; // set of separators the lookup was built for
+    bool mCharClassesValid { false };
     std::string mFile; // name of the open file, if any
     std::string mPath; // path to open stream, for relative path lookups.
     std::streamoff mSize { 0 }; // size of open stream, for progress report.
@@ -143,7 +191,7 @@ class cParser //: public std::stringstream
         commentmap::value_type( "//", "\n" ) };
     std::shared_ptr<cParser> mIncludeParser; // child class to handle include directives.
     std::vector<std::string> parameters; // parameter list for included file.
-    std::deque<std::string> tokens;
+    token_queue tokens;
 };
 
 
@@ -152,14 +200,53 @@ glm::vec3
 cParser::getToken( bool const ToLower, const char *Break );
 
 
+template<typename Type_>
+bool
+cParser::parseNumber( std::string const &Token, Type_ &Output ) {
+
+    using type = std::remove_cv_t<Type_>;
+    if constexpr(
+#ifdef __cpp_lib_to_chars
+        std::is_floating_point_v<type> ||
+#endif
+        ( std::is_integral_v<type>
+       && false == std::is_same_v<type, bool>
+       && false == std::is_same_v<type, char>
+       && false == std::is_same_v<type, signed char>
+       && false == std::is_same_v<type, unsigned char>
+       && false == std::is_same_v<type, wchar_t>
+       && false == std::is_same_v<type, char8_t>
+       && false == std::is_same_v<type, char16_t>
+       && false == std::is_same_v<type, char32_t> ) ) {
+
+        auto const *first { Token.data() };
+        auto const *last { first + Token.size() };
+        // only text which is a number from start to end is converted here. anything else, including forms
+        // the stream extraction doesn't recognize (inf, nan), is left for the stream to make sense of
+        auto const *digits { ( first != last && *first == '-' ) ? first + 1 : first };
+        if( digits == last ) { return false; }
+        if( ( *digits < '0' || *digits > '9' ) && *digits != '.' ) { return false; }
+        type value {};
+        auto const result { std::from_chars( first, last, value ) };
+        if( result.ec != std::errc() || result.ptr != last ) { return false; }
+        Output = value;
+        return true;
+    }
+    else {
+        return false;
+    }
+}
+
 template<typename Type_>
 cParser&
 cParser::operator>>( Type_ &Right ) {
 
     if( true == this->tokens.empty() ) { return *this; }
 
-    std::stringstream converter( this->tokens.front() );
-    converter >> Right;
+    if( false == parseNumber( this->tokens.front(), Right ) ) {
+        std::stringstream converter( this->tokens.front() );
+        converter >> Right;
+    }
     this->tokens.pop_front();
 
     return *this;
@@ -176,3 +263,7 @@ cParser::operator>>( bool &Right );
 template<>
 bool
 cParser::getToken<bool>( bool const ToLower, const char *Break );
+
+template<>
+std::string
+cParser::getToken<std::string>( bool const ToLower, const char *Break );