eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-22 23:19:19 +02:00
Code Issues 123 Wiki Activity

Improve parser

Browse Source
This commit is contained in:
Hirek
2026-01-05 00:44:30 +01:00
parent 24c2cf2771
commit 77f817c930
2 changed files with 536 additions and 420 deletions
Download Patch File Download Diff File Expand all files Collapse all files

469
parser.cpp
View File

@@ -11,6 +11,7 @@ http://mozilla.org/MPL/2.0/.
#include "parser.h" #include "parser.h"
#include "utilities.h" #include "utilities.h"
#include "Logs.h" #include "Logs.h"
#include <limits>
#include "scenenodegroups.h" #include "scenenodegroups.h"
@@ -24,80 +25,87 @@ http://mozilla.org/MPL/2.0/.
// cParser -- generic class for parsing text data. // cParser -- generic class for parsing text data.
// constructors // constructors
cParser::cParser( std::string const &Stream, buffertype const Type, std::string Path, bool const Loadtraction, std::vector<std::string> Parameters, bool allowRandom ) : cParser::cParser(std::string const &Stream, buffertype const Type, std::string Path, bool const Loadtraction, std::vector<std::string> Parameters, bool allowRandom)
mPath(Path), : mPath(Path), LoadTraction(Loadtraction), allowRandomIncludes(allowRandom)
LoadTraction( Loadtraction ), {
allowRandomIncludes(allowRandom) {
// store to calculate sub-sequent includes from relative path // store to calculate sub-sequent includes from relative path
if( Type == buffertype::buffer_FILE ) { if (Type == buffertype::buffer_FILE)
{
mFile = Stream; mFile = Stream;
} }
// reset pointers and attach proper type of buffer // reset pointers and attach proper type of buffer
switch (Type) { switch (Type)
case buffer_FILE: { {
Path.append( Stream ); case buffer_FILE:
mStream = std::make_shared<std::ifstream>( Path, std::ios_base::binary ); {
Path.append(Stream);
mStream = std::make_shared<std::ifstream>(Path, std::ios_base::binary);
// content of *.inc files is potentially grouped together // content of *.inc files is potentially grouped together
if( ( Stream.size() >= 4 ) if ((Stream.size() >= 4) && (ToLower(Stream.substr(Stream.size() - 4)) == ".inc"))
&& ( ToLower( Stream.substr( Stream.size() - 4 ) ) == ".inc" ) ) { {
mIncFile = true; mIncFile = true;
scene::Groups.create(); scene::Groups.create();
} }
break; break;
} }
case buffer_TEXT: { case buffer_TEXT:
mStream = std::make_shared<std::istringstream>( Stream ); {
mStream = std::make_shared<std::istringstream>(Stream);
break; break;
} }
default: { default:
{
break; break;
} }
} }
// calculate stream size // calculate stream size
if (mStream) if (mStream)
{ {
if( true == mStream->fail() ) { if (true == mStream->fail())
ErrorLog( "Failed to open file \"" + Path + "\"" ); {
ErrorLog("Failed to open file \"" + Path + "\"");
} }
else { else
mSize = mStream->rdbuf()->pubseekoff( 0, std::ios_base::end ); {
mStream->rdbuf()->pubseekoff( 0, std::ios_base::beg ); mSize = mStream->rdbuf()->pubseekoff(0, std::ios_base::end);
mStream->rdbuf()->pubseekoff(0, std::ios_base::beg);
mLine = 1; mLine = 1;
} }
} }
// set parameter set if one was provided // set parameter set if one was provided
if( false == Parameters.empty() ) { if (false == Parameters.empty())
parameters.swap( Parameters ); {
parameters.swap(Parameters);
} }
} }
// destructor // destructor
cParser::~cParser() { cParser::~cParser()
{
if( true == mIncFile ) { if (true == mIncFile)
{
// wrap up the node group holding content of processed file // wrap up the node group holding content of processed file
scene::Groups.close(); scene::Groups.close();
} }
} }
template <> template <> glm::vec3 cParser::getToken(bool const ToLower, char const *Break)
glm::vec3 {
cParser::getToken( bool const ToLower, char const *Break ) {
// NOTE: this specialization ignores default arguments // NOTE: this specialization ignores default arguments
getTokens( 3, false, "\n\r\t ,;[]" ); getTokens(3, false, "\n\r\t ,;[]");
glm::vec3 output; glm::vec3 output;
*this *this >> output.x >> output.y >> output.z;
>> output.x
>> output.y
>> output.z;
return output; return output;
}; };
template<> template <> cParser &cParser::operator>>(std::string &Right)
cParser& {
cParser::operator>>( std::string &Right ) {
if( true == this->tokens.empty() ) { return *this; } if (true == this->tokens.empty())
{
return *this;
}
Right = this->tokens.front(); Right = this->tokens.front();
this->tokens.pop_front(); this->tokens.pop_front();
@@ -105,43 +113,44 @@ cParser::operator>>( std::string &Right ) {
return *this; return *this;
} }
template<> template <> cParser &cParser::operator>>(bool &Right)
cParser& {
cParser::operator>>( bool &Right ) {
if( true == this->tokens.empty() ) { return *this; } if (true == this->tokens.empty())
{
return *this;
}
Right = ( ( this->tokens.front() == "true" ) Right = ((this->tokens.front() == "true") || (this->tokens.front() == "yes") || (this->tokens.front() == "1"));
|| ( this->tokens.front() == "yes" )
|| ( this->tokens.front() == "1" ) );
this->tokens.pop_front(); this->tokens.pop_front();
return *this; return *this;
} }
template <> template <> bool cParser::getToken<bool>(bool const ToLower, const char *Break)
bool {
cParser::getToken<bool>( bool const ToLower, const char *Break ) {
auto const token = getToken<std::string>( true, Break ); auto const token = getToken<std::string>(true, Break);
return ( ( token == "true" ) return ((token == "true") || (token == "yes") || (token == "1"));
|| ( token == "yes" )
|| ( token == "1" ) );
} }
// methods // methods
cParser & cParser &cParser::autoclear(bool const Autoclear)
cParser::autoclear( bool const Autoclear ) { {
m_autoclear = Autoclear; m_autoclear = Autoclear;
if( mIncludeParser ) { mIncludeParser->autoclear( Autoclear ); } if (mIncludeParser)
{
mIncludeParser->autoclear(Autoclear);
}
return *this; return *this;
} }
bool cParser::getTokens(unsigned int Count, bool ToLower, const char *Break) bool cParser::getTokens(unsigned int Count, bool ToLower, const char *Break)
{ {
if( true == m_autoclear ) { if (true == m_autoclear)
{
// legacy parser behaviour // legacy parser behaviour
tokens.clear(); tokens.clear();
} }
@@ -151,21 +160,22 @@ bool cParser::getTokens(unsigned int Count, bool ToLower, const char *Break)
else else
trtest="x"; //nie wczytywać trtest="x"; //nie wczytywać
*/ */
/* /*
int i; int i;
this->str(""); this->str("");
this->clear(); this->clear();
*/ */
for (unsigned int i = tokens.size(); i < Count; ++i) for (unsigned int i = tokens.size(); i < Count; ++i)
{ {
std::string token = readToken(ToLower, Break); std::string token = readToken(ToLower, Break);
if( true == token.empty() ) { if (true == token.empty())
{
// no more tokens // no more tokens
break; break;
} }
// collect parameters // collect parameters
tokens.emplace_back( token ); tokens.emplace_back(token);
/* /*
if (i == 0) if (i == 0)
this->str(token); this->str(token);
else else
@@ -175,7 +185,7 @@ bool cParser::getTokens(unsigned int Count, bool ToLower, const char *Break)
temp.append(token); temp.append(token);
this->str(temp); this->str(temp);
} }
*/ */
} }
if (tokens.size() < Count) if (tokens.size() < Count)
return false; return false;
@@ -183,100 +193,148 @@ bool cParser::getTokens(unsigned int Count, bool ToLower, const char *Break)
return true; return true;
} }
std::string cParser::readToken( bool ToLower, const char *Break ) { std::string cParser::readToken(bool ToLower, const char *Break)
{
std::string token; std::string token;
if( mIncludeParser ) { token.clear();
token.reserve(64);
if (mIncludeParser)
{
// see if there's include parsing going on. clean up when it's done. // see if there's include parsing going on. clean up when it's done.
token = mIncludeParser->readToken( ToLower, Break ); token = mIncludeParser->readToken(ToLower, Break);
if( true == token.empty() ) { if (true == token.empty())
{
mIncludeParser = nullptr; mIncludeParser = nullptr;
} }
} }
if( true == token.empty() ) { if (true == token.empty())
{
// get the token yourself if the delegation attempt failed // get the token yourself if the delegation attempt failed
char c { 0 }; char c{0};
do { do
while( mStream->peek() != EOF && strchr( Break, c = mStream->get() ) == NULL ) { {
if( ToLower ) prepareBreakTable(Break);
c = tolower( c );
token += c; int ic = 0;
if( findQuotes( token ) ) // do glue together words enclosed in quotes while ((ic = mStream->get()) != EOF)
continue; {
if( skipComments && trimComments( token ) ) // don't glue together words separated with comment unsigned char uc = static_cast<unsigned char>(ic);
char c = static_cast<char>(uc);
if (isBreak(uc))
{
if (c == '\n')
++mLine;
break; break;
} }
if( c == '\n' ) {
// update line counter if (ToLower)
++mLine; c = static_cast<char>(std::tolower(uc));
token.push_back(c);
// Quotes: nie odpalaj findQuotes() na każdym znaku – tylko jak faktycznie trafiłeś "
if (c == '"')
{
token.pop_back(); // usuwamy "
token += readQuotes(); // doklejamy środek
continue;
} }
} while( token == "" && mStream->peek() != EOF ); // double check in case of consecutive separators
} // Comments: trimComments() jest drogie -> odpalaj tylko jak może się zaczynać komentarz
// check the first token for potential presence of utf bom if (skipComments && (c == '/' || c == '*') && trimComments(token))
if( mFirstToken ) { {
mFirstToken = false; break;
if( token.rfind( "\xef\xbb\xbf", 0 ) == 0 ) {
token.erase( 0, 3 );
}
if( true == token.empty() ) {
// potentially possible if our first token was standalone utf bom
token = readToken( ToLower, Break );
} }
} }
if( false == parameters.empty() ) { if (c == '\n')
{
// update line counter
++mLine;
}
} while (token == "" && mStream->peek() != EOF); // double check in case of consecutive separators
}
// check the first token for potential presence of utf bom
if (mFirstToken)
{
mFirstToken = false;
if (token.rfind("\xef\xbb\xbf", 0) == 0)
{
token.erase(0, 3);
}
if (true == token.empty())
{
// potentially possible if our first token was standalone utf bom
token = readToken(ToLower, Break);
}
}
if (false == parameters.empty())
{
// if there's parameter list, check the token for potential parameters to replace // if there's parameter list, check the token for potential parameters to replace
size_t pos; // początek podmienianego ciągu size_t pos; // początek podmienianego ciągu
while( ( pos = token.find( "(p" ) ) != std::string::npos ) { while ((pos = token.find("(p")) != std::string::npos)
{
// check if the token is a parameter which should be replaced with stored true value // check if the token is a parameter which should be replaced with stored true value
auto const parameter{ token.substr( pos + 2, token.find( ")", pos ) - ( pos + 2 ) ) }; // numer parametru auto const parameter{token.substr(pos + 2, token.find(")", pos) - (pos + 2))}; // numer parametru
token.erase( pos, token.find( ")", pos ) - pos + 1 ); // najpierw usunięcie "(pN)" token.erase(pos, token.find(")", pos) - pos + 1); // najpierw usunięcie "(pN)"
size_t nr = atoi( parameter.c_str() ) - 1; size_t nr = atoi(parameter.c_str()) - 1;
if( nr < parameters.size() ) { if (nr < parameters.size())
token.insert( pos, parameters.at( nr ) ); // wklejenie wartości parametru {
if( ToLower ) token.insert(pos, parameters.at(nr)); // wklejenie wartości parametru
for( ; pos < parameters.at( nr ).size(); ++pos ) if (ToLower)
token[ pos ] = tolower( token[ pos ] ); {
const auto &rep = parameters.at(nr);
for (size_t j = 0; j < rep.size(); ++j)
{
unsigned char uc = static_cast<unsigned char>(token[pos + j]);
token[pos + j] = static_cast<char>(std::tolower(uc));
}
}
} }
else else
token.insert( pos, "none" ); // zabezpieczenie przed brakiem parametru token.insert(pos, "none"); // zabezpieczenie przed brakiem parametru
} }
} }
// launch child parser if include directive found. // launch child parser if include directive found.
// NOTE: parameter collecting uses default set of token separators. // NOTE: parameter collecting uses default set of token separators.
if( expandIncludes && token == "include" ) { if (expandIncludes && token == "include")
{
std::string includefile = allowRandomIncludes ? deserialize_random_set(*this) : readToken(ToLower); // nazwa pliku std::string includefile = allowRandomIncludes ? deserialize_random_set(*this) : readToken(ToLower); // nazwa pliku
replace_slashes(includefile); replace_slashes(includefile);
if ((true == LoadTraction) || if ((true == LoadTraction) || ((false == contains(includefile, "tr/")) && (false == contains(includefile, "tra/"))))
((false == contains(includefile, "tr/")) && (false == contains(includefile, "tra/"))))
{ {
if (false == contains(includefile, "_ter.scm")) if (false == contains(includefile, "_ter.scm"))
{ {
if (Global.ParserLogIncludes) { if (Global.ParserLogIncludes)
{
// WriteLog("including: " + includefile); // WriteLog("including: " + includefile);
} }
mIncludeParser = std::make_shared<cParser>(includefile, buffer_FILE, mPath, LoadTraction, readParameters(*this)); mIncludeParser = std::make_shared<cParser>(includefile, buffer_FILE, mPath, LoadTraction, readParameters(*this));
mIncludeParser->allowRandomIncludes = allowRandomIncludes; mIncludeParser->allowRandomIncludes = allowRandomIncludes;
mIncludeParser->autoclear(m_autoclear); mIncludeParser->autoclear(m_autoclear);
if (mIncludeParser->mSize <= 0) { if (mIncludeParser->mSize <= 0)
{
ErrorLog("Bad include: can't open file \"" + includefile + "\""); ErrorLog("Bad include: can't open file \"" + includefile + "\"");
} }
} }
else else
{ {
if(true == Global.file_binary_terrain_state) if (true == Global.file_binary_terrain_state)
{ {
WriteLog("SBT found, ignoring: " + includefile); WriteLog("SBT found, ignoring: " + includefile);
readParameters(*this); readParameters(*this);
} }
else else
{ {
if (Global.ParserLogIncludes) { if (Global.ParserLogIncludes)
{
WriteLog("including terrain: " + includefile); WriteLog("including terrain: " + includefile);
} }
mIncludeParser = std::make_shared<cParser>(includefile, buffer_FILE, mPath, mIncludeParser = std::make_shared<cParser>(includefile, buffer_FILE, mPath, LoadTraction, readParameters(*this));
LoadTraction, readParameters(*this));
mIncludeParser->allowRandomIncludes = allowRandomIncludes; mIncludeParser->allowRandomIncludes = allowRandomIncludes;
mIncludeParser->autoclear(m_autoclear); mIncludeParser->autoclear(m_autoclear);
if (mIncludeParser->mSize <= 0) if (mIncludeParser->mSize <= 0)
@@ -286,28 +344,30 @@ std::string cParser::readToken( bool ToLower, const char *Break ) {
} }
} }
} }
else { else
while( token != "end" ) { {
token = readToken( true ); // minimize risk of case mismatch on comparison while (token != "end")
{
token = readToken(true); // minimize risk of case mismatch on comparison
} }
} }
token = readToken(ToLower, Break); token = readToken(ToLower, Break);
} }
else if( ( std::strcmp( Break, "\n\r" ) == 0 ) && ( token.compare( 0, 7, "include" ) == 0 ) ) { else if ((std::strcmp(Break, "\n\r") == 0) && (token.compare(0, 7, "include") == 0))
{
// HACK: if the parser reads full lines we expect this line to contain entire include directive, to make parsing easier // HACK: if the parser reads full lines we expect this line to contain entire include directive, to make parsing easier
cParser includeparser( token.substr( 7 ) ); cParser includeparser(token.substr(7));
std::string includefile = allowRandomIncludes ? deserialize_random_set( includeparser ) : includeparser.readToken( ToLower ); // nazwa pliku std::string includefile = allowRandomIncludes ? deserialize_random_set(includeparser) : includeparser.readToken(ToLower); // nazwa pliku
replace_slashes(includefile); replace_slashes(includefile);
if ((true == LoadTraction) || if ((true == LoadTraction) || ((false == contains(includefile, "tr/")) && (false == contains(includefile, "tra/"))))
((false == contains(includefile, "tr/")) && (false == contains(includefile, "tra/"))))
{ {
if (false == contains(includefile, "_ter.scm")) if (false == contains(includefile, "_ter.scm"))
{ {
if (Global.ParserLogIncludes) { if (Global.ParserLogIncludes)
{
// WriteLog("including: " + includefile); // WriteLog("including: " + includefile);
} }
mIncludeParser = std::make_shared<cParser>( mIncludeParser = std::make_shared<cParser>(includefile, buffer_FILE, mPath, LoadTraction, readParameters(includeparser));
includefile, buffer_FILE, mPath, LoadTraction, readParameters(includeparser));
mIncludeParser->allowRandomIncludes = allowRandomIncludes; mIncludeParser->allowRandomIncludes = allowRandomIncludes;
mIncludeParser->autoclear(m_autoclear); mIncludeParser->autoclear(m_autoclear);
if (mIncludeParser->mSize <= 0) if (mIncludeParser->mSize <= 0)
@@ -324,52 +384,57 @@ std::string cParser::readToken( bool ToLower, const char *Break ) {
} }
else else
{ {
if (Global.ParserLogIncludes) { if (Global.ParserLogIncludes)
{
WriteLog("including terrain: " + includefile); WriteLog("including terrain: " + includefile);
} }
mIncludeParser = mIncludeParser = std::make_shared<cParser>(includefile, buffer_FILE, mPath, LoadTraction, readParameters(includeparser));
std::make_shared<cParser>(includefile, buffer_FILE, mPath, LoadTraction,
readParameters(includeparser));
mIncludeParser->allowRandomIncludes = allowRandomIncludes; mIncludeParser->allowRandomIncludes = allowRandomIncludes;
mIncludeParser->autoclear(m_autoclear); mIncludeParser->autoclear(m_autoclear);
if (mIncludeParser->mSize <= 0) if (mIncludeParser->mSize <= 0)
{ {
ErrorLog("Bad include: can't open file \"" + includefile + "\""); ErrorLog("Bad include: can't open file \"" + includefile + "\"");
} }
} }
} }
} }
token = readToken( ToLower, Break ); token = readToken(ToLower, Break);
} }
// all done // all done
return token; return token;
} }
std::vector<std::string> cParser::readParameters( cParser &Input ) { std::vector<std::string> cParser::readParameters(cParser &Input)
{
std::vector<std::string> includeparameters; std::vector<std::string> includeparameters;
std::string parameter = Input.readToken( false ); // w parametrach nie zmniejszamy std::string parameter = Input.readToken(false); // w parametrach nie zmniejszamy
while( ( parameter.empty() == false ) while ((parameter.empty() == false) && (parameter != "end"))
&& ( parameter != "end" ) ) { {
includeparameters.emplace_back( parameter ); includeparameters.emplace_back(parameter);
parameter = Input.readToken( false ); parameter = Input.readToken(false);
} }
return includeparameters; return includeparameters;
} }
std::string cParser::readQuotes(char const Quote) { // read the stream until specified char or stream end std::string cParser::readQuotes(char const Quote)
std::string token = ""; { // read the stream until specified char or stream end
char c { 0 }; std::string token;
token.reserve(64);
char c{0};
bool escaped = false; bool escaped = false;
while( mStream->peek() != EOF ) { // get all chars until the quote mark while (mStream->peek() != EOF)
{ // get all chars until the quote mark
c = mStream->get(); c = mStream->get();
if (escaped) { if (escaped)
{
escaped = false; escaped = false;
} }
else { else
if (c == '\\') { {
if (c == '\\')
{
escaped = true; escaped = true;
continue; continue;
} }
@@ -385,31 +450,40 @@ std::string cParser::readQuotes(char const Quote) { // read the stream until spe
return token; return token;
} }
void cParser::skipComment( std::string const &Endmark ) { // pobieranie znaków aż do znalezienia znacznika końca void cParser::skipComment(std::string const &Endmark)
std::string input = ""; {
char c { 0 }; if (Endmark == "\n")
auto const endmarksize = Endmark.size(); {
while( mStream->peek() != EOF ) { mStream->ignore(std::numeric_limits<std::streamsize>::max(), '\n');
// o ile nie koniec pliku
c = mStream->get(); // pobranie znaku
if( c == '\n' ) {
// update line counter
++mLine; ++mLine;
}
input += c;
if( input == Endmark ) // szukanie znacznika końca
break;
if( input.size() >= endmarksize ) {
// keep the read text short, to avoid pointless string re-allocations on longer comments
input = input.substr( 1 );
}
}
return; return;
}
// dla "*/" itp. – rolling window, bez substr() co znak
std::string window;
window.reserve(Endmark.size());
int ic = 0;
while ((ic = mStream->get()) != EOF)
{
char c = static_cast<char>(ic);
if (c == '\n')
++mLine;
window.push_back(c);
if (window.size() > Endmark.size())
window.erase(0, window.size() - Endmark.size());
if (window.size() == Endmark.size() && window == Endmark)
break;
}
} }
bool cParser::findQuotes( std::string &String ) { bool cParser::findQuotes(std::string &String)
{
if( String.back() == '\"' ) { if (String.back() == '\"')
{
String.pop_back(); String.pop_back();
String += readQuotes(); String += readQuotes();
@@ -422,9 +496,12 @@ bool cParser::trimComments(std::string &String)
{ {
for (auto const &comment : mComments) for (auto const &comment : mComments)
{ {
if( String.size() < comment.first.size() ) { continue; } if (String.size() < comment.first.size())
{
continue;
}
if (String.compare( String.size() - comment.first.size(), comment.first.size(), comment.first ) == 0) if (String.compare(String.size() - comment.first.size(), comment.first.size(), comment.first) == 0)
{ {
skipComment(comment.second); skipComment(comment.second);
String.resize(String.rfind(comment.first)); String.resize(String.rfind(comment.first));
@@ -436,66 +513,88 @@ bool cParser::trimComments(std::string &String)
void cParser::injectString(const std::string &str) void cParser::injectString(const std::string &str)
{ {
if (mIncludeParser) { if (mIncludeParser)
{
mIncludeParser->injectString(str); mIncludeParser->injectString(str);
} }
else { else
mIncludeParser = std::make_shared<cParser>( str, buffer_TEXT, "", LoadTraction, std::vector<std::string>(), allowRandomIncludes ); {
mIncludeParser->autoclear( m_autoclear ); mIncludeParser = std::make_shared<cParser>(str, buffer_TEXT, "", LoadTraction, std::vector<std::string>(), allowRandomIncludes);
mIncludeParser->autoclear(m_autoclear);
} }
} }
int cParser::getProgress() const int cParser::getProgress() const
{ {
return static_cast<int>( mStream->rdbuf()->pubseekoff(0, std::ios_base::cur) * 100 / mSize ); return static_cast<int>(mStream->rdbuf()->pubseekoff(0, std::ios_base::cur) * 100 / mSize);
} }
int cParser::getFullProgress() const { int cParser::getFullProgress() const
{
int progress = getProgress(); int progress = getProgress();
if( mIncludeParser ) return progress + ( ( 100 - progress )*( mIncludeParser->getProgress() ) / 100 ); if (mIncludeParser)
else return progress; return progress + ((100 - progress) * (mIncludeParser->getProgress()) / 100);
else
return progress;
} }
std::size_t cParser::countTokens( std::string const &Stream, std::string Path ) { std::size_t cParser::countTokens(std::string const &Stream, std::string Path)
{
return cParser( Stream, buffer_FILE, Path ).count(); return cParser(Stream, buffer_FILE, Path).count();
} }
std::size_t cParser::count() { std::size_t cParser::count()
{
std::string token; std::string token;
size_t count { 0 }; size_t count{0};
do { do
{
token = ""; token = "";
token = readToken( false ); token = readToken(false);
++count; ++count;
} while( false == token.empty() ); } while (false == token.empty());
return count - 1; return count - 1;
} }
void cParser::addCommentStyle( std::string const &Commentstart, std::string const &Commentend ) { void cParser::addCommentStyle(std::string const &Commentstart, std::string const &Commentend)
{
mComments.insert( commentmap::value_type(Commentstart, Commentend) ); mComments.insert(commentmap::value_type(Commentstart, Commentend));
} }
// returns name of currently open file, or empty string for text type stream // returns name of currently open file, or empty string for text type stream
std::string std::string cParser::Name() const
cParser::Name() const { {
if( mIncludeParser ) { return mIncludeParser->Name(); } if (mIncludeParser)
else { return mPath + mFile; } {
return mIncludeParser->Name();
}
else
{
return mPath + mFile;
}
} }
// returns number of currently processed line // returns number of currently processed line
std::size_t std::size_t cParser::Line() const
cParser::Line() const { {
if( mIncludeParser ) { return mIncludeParser->Line(); } if (mIncludeParser)
else { return mLine; } {
return mIncludeParser->Line();
}
else
{
return mLine;
}
} }
int cParser::LineMain() const { int cParser::LineMain() const
{
return mIncludeParser ? -1 : mLine; return mIncludeParser ? -1 : mLine;
} }

163
parser.h
View File

@@ -14,6 +14,9 @@ http://mozilla.org/MPL/2.0/.
#include <fstream> #include <fstream>
#include <vector> #include <vector>
#include <map> #include <map>
#include <charconv>
#include <type_traits>
#include <cstdlib>
///////////////////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////////////////////
// cParser -- generic class for parsing text data, either from file or provided string // cParser -- generic class for parsing text data, either from file or provided string
@@ -28,52 +31,46 @@ class cParser //: public std::stringstream
buffer_TEXT buffer_TEXT
}; };
// constructors: // constructors:
cParser(std::string const &Stream, buffertype const Type = buffer_TEXT, std::string Path = "", bool const Loadtraction = true, std::vector<std::string> Parameters = std::vector<std::string>(), bool allowRandom = false ); cParser(std::string const &Stream, buffertype const Type = buffer_TEXT, std::string Path = "", bool const Loadtraction = true, std::vector<std::string> Parameters = std::vector<std::string>(),
bool allowRandom = false);
// destructor: // destructor:
virtual ~cParser(); virtual ~cParser();
// methods: // methods:
template <typename Type_> template <typename Type_> cParser &operator>>(Type_ &Right);
cParser & template <typename Output_> Output_ getToken(bool const ToLower = true, char const *Break = "\n\r\t ;")
operator>>( Type_ &Right ); {
template <typename Output_> getTokens(1, ToLower, Break);
Output_
getToken( bool const ToLower = true, char const *Break = "\n\r\t ;" ) {
getTokens( 1, ToLower, Break );
Output_ output; Output_ output;
*this >> output; *this >> output;
return output; }; return output;
inline };
void inline void ignoreToken()
ignoreToken() { {
readToken(); }; readToken();
inline };
bool inline bool expectToken(std::string const &Value)
expectToken( std::string const &Value ) { {
return readToken() == Value; }; return readToken() == Value;
inline };
bool inline bool eof()
eof() { {
return mStream->eof(); }; return mStream->eof();
inline };
bool inline bool ok()
ok() { {
return ( !mStream->fail() ); }; return (!mStream->fail());
cParser & };
autoclear( bool const Autoclear ); cParser &autoclear(bool const Autoclear);
inline inline bool autoclear() const
bool {
autoclear() const { return m_autoclear;
return m_autoclear; } }
bool bool getTokens(unsigned int Count = 1, bool ToLower = true, char const *Break = "\n\r\t ;");
getTokens( unsigned int Count = 1, bool ToLower = true, char const *Break = "\n\r\t ;" );
// returns next incoming token, if any, without removing it from the set // returns next incoming token, if any, without removing it from the set
inline inline std::string peek() const
std::string {
peek() const { return (false == tokens.empty() ? tokens.front() : "");
return ( }
false == tokens.empty() ?
tokens.front() :
"" ); }
// inject string as internal include // inject string as internal include
void injectString(const std::string &str); void injectString(const std::string &str);
@@ -81,9 +78,9 @@ class cParser //: public std::stringstream
int getProgress() const; int getProgress() const;
int getFullProgress() const; int getFullProgress() const;
// //
static std::size_t countTokens( std::string const &Stream, std::string Path = "" ); static std::size_t countTokens(std::string const &Stream, std::string Path = "");
// add custom definition of text which should be ignored when retrieving tokens // add custom definition of text which should be ignored when retrieving tokens
void addCommentStyle( std::string const &Commentstart, std::string const &Commentend ); void addCommentStyle(std::string const &Commentstart, std::string const &Commentend);
// returns name of currently open file, or empty string for text type stream // returns name of currently open file, or empty string for text type stream
std::string Name() const; std::string Name() const;
// returns number of currently processed line // returns number of currently processed line
@@ -97,58 +94,78 @@ class cParser //: public std::stringstream
private: private:
// methods: // methods:
std::string readToken(bool ToLower = true, const char *Break = "\n\r\t ;"); std::string readToken(bool ToLower = true, const char *Break = "\n\r\t ;");
std::vector<std::string> readParameters( cParser &Input ); std::vector<std::string> readParameters(cParser &Input);
std::string readQuotes( char const Quote = '\"' ); std::string readQuotes(char const Quote = '\"');
void skipComment( std::string const &Endmark ); void skipComment(std::string const &Endmark);
bool findQuotes( std::string &String ); bool findQuotes(std::string &String);
bool trimComments( std::string &String ); bool trimComments(std::string &String);
std::size_t count(); std::size_t count();
// members: // members:
bool m_autoclear { true }; // unretrieved tokens are discarded when another read command is issued (legacy behaviour) std::array<uint8_t, 256> mBreakTable{};
bool LoadTraction { true }; // load traction? const char* mBreakPtr{ nullptr };
inline void prepareBreakTable(const char* Break) {
if (Break == mBreakPtr) return;
mBreakPtr = Break;
mBreakTable.fill(0);
for (const unsigned char* p = reinterpret_cast<const unsigned char*>(Break); *p; ++p) {
mBreakTable[*p] = 1;
}
}
inline bool isBreak(unsigned char c) const { return mBreakTable[c] != 0; }
bool m_autoclear{true}; // unretrieved tokens are discarded when another read command is issued (legacy behaviour)
bool LoadTraction{true}; // load traction?
std::shared_ptr<std::istream> mStream; // relevant kind of buffer is attached on creation. std::shared_ptr<std::istream> mStream; // relevant kind of buffer is attached on creation.
std::string mFile; // name of the open file, if any std::string mFile; // name of the open file, if any
std::string mPath; // path to open stream, for relative path lookups. std::string mPath; // path to open stream, for relative path lookups.
std::streamoff mSize { 0 }; // size of open stream, for progress report. std::streamoff mSize{0}; // size of open stream, for progress report.
std::size_t mLine { 0 }; // currently processed line std::size_t mLine{0}; // currently processed line
bool mIncFile { false }; // the parser is processing an *.inc file bool mIncFile{false}; // the parser is processing an *.inc file
bool mFirstToken { true }; // processing first token in the current file; helper used when checking for utf bom bool mFirstToken{true}; // processing first token in the current file; helper used when checking for utf bom
typedef std::map<std::string, std::string> commentmap; typedef std::map<std::string, std::string> commentmap;
commentmap mComments { commentmap mComments{commentmap::value_type("/*", "*/"), commentmap::value_type("//", "\n")};
commentmap::value_type( "/*", "*/" ),
commentmap::value_type( "//", "\n" ) };
std::shared_ptr<cParser> mIncludeParser; // child class to handle include directives. std::shared_ptr<cParser> mIncludeParser; // child class to handle include directives.
std::vector<std::string> parameters; // parameter list for included file. std::vector<std::string> parameters; // parameter list for included file.
std::deque<std::string> tokens; std::deque<std::string> tokens;
}; };
template <> glm::vec3 cParser::getToken(bool const ToLower, const char *Break);
template <>
glm::vec3
cParser::getToken( bool const ToLower, const char *Break );
template<typename Type_> template<typename Type_>
cParser& cParser&
cParser::operator>>( Type_ &Right ) { cParser::operator>>( Type_ &Right ) {
if( true == this->tokens.empty() ) { return *this; } if (this->tokens.empty()) return *this;
std::stringstream converter( this->tokens.front() ); const std::string &s = this->tokens.front();
if constexpr (std::is_integral_v<Type_> && !std::is_same_v<Type_, bool>) {
// szybkie i bez-locale
Type_ v{};
auto res = std::from_chars(s.data(), s.data() + s.size(), v);
if (res.ec == std::errc{}) Right = v;
else Right = Type_{}; // albo zostaw bez zmian – jak wolisz
}
else if constexpr (std::is_floating_point_v<Type_>) {
// std::from_chars dla float bywa różnie wspierane; strtod jest OK w praktyce MaSzyny
char *end = nullptr;
double v = std::strtod(s.c_str(), &end);
Right = static_cast<Type_>(v);
}
else {
// fallback dla typów nie-liczbowych (np. jakieś własne)
std::stringstream converter(s);
converter >> Right; converter >> Right;
this->tokens.pop_front(); }
this->tokens.pop_front();
return *this; return *this;
} }
template<>
cParser&
cParser::operator>>( std::string &Right );
template<> template <> cParser &cParser::operator>>(std::string &Right);
cParser&
cParser::operator>>( bool &Right );
template<> template <> cParser &cParser::operator>>(bool &Right);
bool
cParser::getToken<bool>( bool const ToLower, const char *Break ); template <> bool cParser::getToken<bool>(bool const ToLower, const char *Break);