loading time fix

utilities/Globals.cpp

@@ -625,7 +625,31 @@ bool global_settings::ConfigParseSimulation(cParser& Parser, const std::string&
 
     if (token == "file.binary.terrain")
     {
-        ParseOne(Parser, file_binary_terrain, 1, false);
+       // ParseOne(Parser, file_binary_terrain, 1, false);
+        return true;
+    }
+
+    if (token == "scenario.fileexists.cache")
+    {
+        ParseOne(Parser, ScenarioFileExistsCache, 1, false);
+        return true;
+    }
+
+    if (token == "scenario.parser.filecache")
+    {
+        ParseOne(Parser, ScenarioParserFileCache, 1, false);
+        return true;
+    }
+
+    if (token == "scenario.parser.fastgeometry")
+    {
+        ParseOne(Parser, ScenarioParserFastGeometry, 1, false);
+        return true;
+    }
+
+    if (token == "scenario.parser.fastskip")
+    {
+        ParseOne(Parser, ScenarioParserFastSkip, 1, false);
         return true;
     }
 
@@ -1578,6 +1602,10 @@ global_settings::export_as_text( std::ostream &Output ) const {
     export_as_text( Output, "latitude", fLatitudeDeg );
     export_as_text( Output, "convertmodels", iConvertModels );
     export_as_text( Output, "file.binary.terrain", file_binary_terrain );
+    export_as_text( Output, "scenario.fileexists.cache", ScenarioFileExistsCache );
+    export_as_text( Output, "scenario.parser.filecache", ScenarioParserFileCache );
+    export_as_text( Output, "scenario.parser.fastgeometry", ScenarioParserFastGeometry );
+    export_as_text( Output, "scenario.parser.fastskip", ScenarioParserFastSkip );
     export_as_text( Output, "inactivepause", bInactivePause );
     export_as_text( Output, "slowmotion", iSlowMotionMask );
     export_as_text( Output, "hideconsole", bHideConsole );

utilities/Globals.h

@@ -80,8 +80,12 @@ struct global_settings {
     std::string local_start_vehicle{ "EU07-424" };
     int iConvertModels{ 0 }; // tworzenie plików binarnych
     int iConvertIndexRange{ 1000 }; // range of duplicate vertex scan
-    bool file_binary_terrain{ true }; // enable binary terrain (de)serialization
+    bool file_binary_terrain{ false }; // enable binary terrain (de)serialization
 	bool file_binary_terrain_state{true};
+	bool ScenarioFileExistsCache{ true }; // cache FileExists calls during scenario loading
+	bool ScenarioParserFileCache{ false }; // cache parser input file bytes during scenario loading
+	bool ScenarioParserFastGeometry{ false }; // fast float reads for triangles/lines import
+	bool ScenarioParserFastSkip{ false }; // fast skip_until for endtri/endline blocks
     // logs
 	bool priorityLoadText3D{false}; // ladowanie T3D priorytetowo
     int iWriteLogEnabled{ 3 }; // maska bitowa: 1-zapis do pliku, 2-okienko, 4-nazwy torów

utilities/Logs.cpp

@@ -25,50 +25,116 @@ char endstring[10] = "\n";
 
 std::deque<std::string> log_scrollback;
 
-std::string filename_date() {
-    ::SYSTEMTIME st;
+namespace
+{
+std::string sanitize_filename_component(std::string value)
+{
+	for (auto &character : value)
+	{
+		switch (character)
+		{
+		case '\\':
+		case '/':
+		case ':':
+		case '*':
+		case '?':
+		case '\"':
+		case '<':
+		case '>':
+		case '|':
+			character = '_';
+			break;
+		default:
+			break;
+		}
+	}
+	return value;
+}
+
+std::string filename_session_timestamp()
+{
+	::SYSTEMTIME st {};
 
 #ifdef __unix__
-    timespec ts;
-    clock_gettime(CLOCK_REALTIME, &ts);
-    tm *tms = localtime(&ts.tv_sec);
-    st.wYear = tms->tm_year;
-    st.wMonth = tms->tm_mon;
-    st.wDayOfWeek = tms->tm_wday;
-    st.wDay = tms->tm_mday;
-    st.wHour = tms->tm_hour;
-    st.wMinute = tms->tm_min;
-    st.wSecond = tms->tm_sec;
-    st.wMilliseconds = ts.tv_nsec / 1000000;
+	timespec ts {};
+	clock_gettime(CLOCK_REALTIME, &ts);
+	tm *tms = localtime(&ts.tv_sec);
+	if (tms != nullptr)
+	{
+		st.wYear = static_cast<unsigned short>(tms->tm_year + 1900);
+		st.wMonth = static_cast<unsigned short>(tms->tm_mon + 1);
+		st.wDay = static_cast<unsigned short>(tms->tm_mday);
+		st.wHour = static_cast<unsigned short>(tms->tm_hour);
+		st.wMinute = static_cast<unsigned short>(tms->tm_min);
+		st.wSecond = static_cast<unsigned short>(tms->tm_sec);
+		st.wMilliseconds = static_cast<unsigned short>(ts.tv_nsec / 1000000);
+	}
 #elif _WIN32
-    ::GetLocalTime( &st );
+	::GetLocalTime(&st);
 #endif
 
-    std::snprintf(
-        logbuffer,
-        sizeof(logbuffer),
-	    "%d%02d%02d_%02d%02d%03d",
-        st.wYear,
-        st.wMonth,
-        st.wDay,
-        st.wHour,
-	    st.wMinute,
-	    st.wMilliseconds);
+	std::snprintf(
+	    logbuffer,
+	    sizeof(logbuffer),
+	    "%04u%02u%02u_%02u%02u%02u_%03u",
+	    static_cast<unsigned>(st.wYear),
+	    static_cast<unsigned>(st.wMonth),
+	    static_cast<unsigned>(st.wDay),
+	    static_cast<unsigned>(st.wHour),
+	    static_cast<unsigned>(st.wMinute),
+	    static_cast<unsigned>(st.wSecond),
+	    static_cast<unsigned>(st.wMilliseconds));
 
-    return std::string( logbuffer );
+	return std::string(logbuffer);
 }
 
-std::string filename_scenery() {
-
-    auto extension = Global.SceneryFile.rfind( '.' );
-    if( extension != std::string::npos ) {
-        return Global.SceneryFile.substr( 0, extension );
-    }
-    else {
-        return Global.SceneryFile;
-    }
+std::string const &log_session_id()
+{
+	static std::string const session_id = []()
+	{
+		std::string session = filename_session_timestamp();
+#ifdef _WIN32
+		char pid_suffix[32];
+		std::snprintf(pid_suffix, sizeof(pid_suffix), "_%lu", static_cast<unsigned long>(::GetCurrentProcessId()));
+		session += pid_suffix;
+#endif
+		return session;
+	}();
+	return session_id;
 }
 
+std::string filename_scenery()
+{
+	auto extension = Global.SceneryFile.rfind('.');
+	if (extension != std::string::npos)
+	{
+		return sanitize_filename_component(Global.SceneryFile.substr(0, extension));
+	}
+
+	return sanitize_filename_component(Global.SceneryFile);
+}
+
+std::string log_output_filename()
+{
+	if (Global.MultipleLogs && false == Global.SceneryFile.empty())
+	{
+		return "logs/log (" + filename_scenery() + ") " + log_session_id() + ".txt";
+	}
+
+	return "logs/log_" + log_session_id() + ".txt";
+}
+
+std::string log_errors_filename()
+{
+	if (Global.MultipleLogs && false == Global.SceneryFile.empty())
+	{
+		return "logs/errors (" + filename_scenery() + ") " + log_session_id() + ".txt";
+	}
+
+	return "logs/errors_" + log_session_id() + ".txt";
+}
+} // namespace
+
 // log service stacks
 std::deque < std::pair<std::string, bool>> InfoStack;
 std::deque<std::string> ErrorStack;
@@ -108,8 +174,7 @@ void LogService()
 				{
 					if (!output.is_open())
 					{
-						std::string filename = (Global.MultipleLogs ? "logs/log (" + filename_scenery() + ") " + filename_date() + ".txt" : "log.txt");
-						output.open(filename, std::ios::trunc);
+						output.open(log_output_filename(), std::ios::trunc);
 					}
 					output << msg << "\n";
 					output.flush();
@@ -143,8 +208,7 @@ void LogService()
 
 				if (!errors.is_open())
 				{
-					std::string filename = (Global.MultipleLogs ? "logs/errors (" + filename_scenery() + ") " + filename_date() + ".txt" : "errors.txt");
-					errors.open(filename, std::ios::trunc);
+					errors.open(log_errors_filename(), std::ios::trunc);
 					errors << "EU07.EXE " + Global.asVersion << "\n";
 				}
 

utilities/parser.h

@@ -9,6 +9,7 @@ http://mozilla.org/MPL/2.0/.
 
 #pragma once
 
+#include <chrono>
 #include <string>
 #include <sstream>
 #include <fstream>
@@ -69,6 +70,11 @@ class cParser //: public std::stringstream
             return m_autoclear; }
     bool
         getTokens( unsigned int Count = 1, bool ToLower = true, char const *Break = "\n\r\t ;" );
+	// read one token and parse as number without using the tokens deque / stringstream
+	bool readTokenFloat( float &Value, bool ToLower = true, char const *Break = "\n\r\t ;" );
+	bool readTokenDouble( double &Value, bool ToLower = true, char const *Break = "\n\r\t ;" );
+	void readNextToken( std::string &Token, bool ToLower = true, char const *Break = "\n\r\t ;" );
+	void skipUntilKeyword( std::string const &Keyword, bool ToLower = true, char const *Break = "\n\r\t ;" );
 	std::string readTokenFromStream(bool ToLower, const char *Break);
 	void stripFirstTokenBOM(std::string &token, bool ToLower, const char *Break);
 	void substituteParameters(std::string &token, bool ToLower);
@@ -106,6 +112,8 @@ class cParser //: public std::stringstream
 	bool handleIncludeIfPresent(std::string &token, bool ToLower, const char *Break);
 	// methods:
     void readToken(std::string& out, bool ToLower = true, const char *Break = "\n\r\t ;");
+	void readTokenForSkip(std::string &out, bool ToLower = true, const char *Break = "\n\r\t ;");
+	std::string readTokenFromStreamFast(bool ToLower, const char *Break);
 	static std::vector<std::string> readParameters( cParser &Input );
     std::string readQuotes( char const Quote = '\"' );
     void skipComment( std::string const &Endmark );
@@ -131,6 +139,50 @@ class cParser //: public std::stringstream
     std::deque<std::string> tokens;
 };
 
+class ParserFileCacheScope
+{
+public:
+	ParserFileCacheScope();
+	~ParserFileCacheScope();
+
+	ParserFileCacheScope(ParserFileCacheScope const &) = delete;
+	ParserFileCacheScope &operator=(ParserFileCacheScope const &) = delete;
+
+	void end();
+
+private:
+	bool m_active { true };
+};
+
+class ParserMetricsScope
+{
+public:
+	ParserMetricsScope();
+	~ParserMetricsScope();
+
+	ParserMetricsScope(ParserMetricsScope const &) = delete;
+	ParserMetricsScope &operator=(ParserMetricsScope const &) = delete;
+
+	void end();
+
+private:
+	bool m_active { true };
+};
+
+namespace parser_metrics
+{
+bool active();
+
+struct convert_timer
+{
+	std::chrono::steady_clock::time_point start {};
+	bool enabled { false };
+
+	convert_timer();
+	~convert_timer();
+};
+} // namespace parser_metrics
+
 
 template <>
 glm::vec3
@@ -143,6 +195,7 @@ cParser::operator>>( Type_ &Right ) {
 
     if( true == this->tokens.empty() ) { return *this; }
 
+    parser_metrics::convert_timer timer;
     std::stringstream converter( this->tokens.front() );
     converter >> Right;
     this->tokens.pop_front();

utilities/utilities.cpp

@@ -15,7 +15,12 @@ Copyright (C) 2007-2014 Maciej Cierniak
 //
 //#include <sys/types.h>
 //#include <sys/stat.h>
+#include <atomic>
+#include <algorithm>
+#include <cctype>
 #include <ranges>
+#include <mutex>
+#include <unordered_map>
 //#ifndef WIN32
 //#include <unistd.h>
 //#endif
@@ -26,10 +31,9 @@ Copyright (C) 2007-2014 Maciej Cierniak
 
 #include "utilities/utilities.h"
 #include "utilities/Globals.h"
+#include "utilities/Logs.h"
 #include "utilities/parser.h"
 
-//#include "utilities/Logs.h"
-
 // TODO: This shouldn't be in Globals?
 bool DebugModeFlag = false;
 bool FreeFlyModeFlag = false;
@@ -37,6 +41,136 @@ bool EditorModeFlag = false;
 bool DebugCameraFlag = false;
 bool DebugTractionFlag = false;
 
+namespace
+{
+std::atomic<unsigned int> g_fileexists_cache_depth { 0 };
+bool g_fileexists_cache_enabled { false };
+std::mutex g_fileexists_cache_mutex;
+std::unordered_map<std::string, bool> g_fileexists_cache;
+
+struct fileexists_cache_metrics
+{
+	std::uint64_t total_calls { 0 };
+	std::uint64_t cache_hits { 0 };
+	std::uint64_t cache_misses { 0 };
+	std::uint64_t filesystem_exists_calls { 0 };
+	std::uint64_t positive_results { 0 };
+	std::uint64_t negative_results { 0 };
+	std::uint64_t untracked_path_calls { 0 };
+	std::chrono::steady_clock::duration filesystem_exists_time {};
+	std::unordered_map<std::string, std::uint64_t> path_counts;
+
+	void reset()
+	{
+		*this = {};
+	}
+};
+
+fileexists_cache_metrics g_fileexists_cache_metrics;
+
+std::string FileExistsCacheKey(std::string const &Filename)
+{
+	if (Filename.empty())
+	{
+		return Filename;
+	}
+
+	try
+	{
+		auto path = std::filesystem::path(Filename);
+		if (path.is_relative())
+		{
+			path = std::filesystem::absolute(path);
+		}
+
+		auto key = path.lexically_normal().generic_string();
+#ifdef _WIN32
+		for (auto &character : key)
+		{
+			auto const value = static_cast<unsigned char>(character);
+			if (value < 128)
+			{
+				character = static_cast<char>(std::tolower(value));
+			}
+		}
+#endif
+		return key;
+	}
+	catch (std::filesystem::filesystem_error const &)
+	{
+		auto key = Filename;
+		std::replace(key.begin(), key.end(), '\\', '/');
+		return key;
+	}
+}
+
+double FileExistsDurationMilliseconds(std::chrono::steady_clock::duration const Duration)
+{
+	return std::chrono::duration<double, std::milli>(Duration).count();
+}
+
+std::string FileExistsFormatPathCount(std::pair<std::string, std::uint64_t> const &Entry)
+{
+	return "  " + std::to_string(Entry.second) + "x " + Entry.first;
+}
+
+std::vector<std::string> FileExistsCacheReport()
+{
+	std::vector<std::string> report;
+	report.emplace_back(
+	    std::string("FileExists scope: mode=") + (g_fileexists_cache_enabled ? "cached" : "direct")
+	    + ", calls=" + std::to_string(g_fileexists_cache_metrics.total_calls)
+	    + ", hits=" + std::to_string(g_fileexists_cache_metrics.cache_hits)
+	    + ", misses=" + std::to_string(g_fileexists_cache_metrics.cache_misses)
+	    + ", fs_exists=" + std::to_string(g_fileexists_cache_metrics.filesystem_exists_calls)
+	    + ", found=" + std::to_string(g_fileexists_cache_metrics.positive_results)
+	    + ", missing=" + std::to_string(g_fileexists_cache_metrics.negative_results)
+	    + ", fs_time_ms=" + to_string(FileExistsDurationMilliseconds(g_fileexists_cache_metrics.filesystem_exists_time), 3)
+	    + ", untracked_paths=" + std::to_string(g_fileexists_cache_metrics.untracked_path_calls)
+	    + ", cached_paths=" + std::to_string(g_fileexists_cache.size()));
+
+	std::vector<std::pair<std::string, std::uint64_t>> top_paths;
+	top_paths.reserve(g_fileexists_cache_metrics.path_counts.size());
+	for (auto const &entry : g_fileexists_cache_metrics.path_counts)
+	{
+		if (entry.second > 1)
+		{
+			top_paths.emplace_back(entry);
+		}
+	}
+
+	constexpr std::size_t top_path_limit = 5;
+	auto const top_count = std::min(top_path_limit, top_paths.size());
+	std::partial_sort(
+	    top_paths.begin(),
+	    top_paths.begin() + top_count,
+	    top_paths.end(),
+	    [](auto const &Left, auto const &Right)
+	    {
+		    if (Left.second != Right.second)
+		    {
+			    return Left.second > Right.second;
+		    }
+		    return Left.first < Right.first;
+	    });
+
+	for (std::size_t index = 0; index < top_count; ++index)
+	{
+		report.emplace_back(FileExistsFormatPathCount(top_paths[index]));
+	}
+
+	return report;
+}
+
+void FileExistsCacheLogReport(std::vector<std::string> const &Report)
+{
+	for (auto const &line : Report)
+	{
+		WriteLog(line);
+	}
+}
+} // namespace
+
 std::string Now()
 {
 	auto now = std::chrono::system_clock::now();
@@ -328,7 +462,62 @@ template <> bool extract_value(bool &Variable, std::string const &Key, std::stri
 
 bool FileExists(std::string const &Filename)
 {
-	return std::filesystem::exists(Filename);
+	if (g_fileexists_cache_depth.load(std::memory_order_acquire) == 0)
+	{
+		return std::filesystem::exists(Filename);
+	}
+
+	std::lock_guard<std::mutex> lock(g_fileexists_cache_mutex);
+	if (g_fileexists_cache_depth.load(std::memory_order_relaxed) == 0)
+	{
+		return std::filesystem::exists(Filename);
+	}
+
+	auto const cache_enabled = g_fileexists_cache_enabled;
+	auto const key = cache_enabled ? FileExistsCacheKey(Filename) : Filename;
+	++g_fileexists_cache_metrics.total_calls;
+	if (auto lookup = g_fileexists_cache_metrics.path_counts.find(key); lookup != g_fileexists_cache_metrics.path_counts.end())
+	{
+		++lookup->second;
+	}
+	else if (g_fileexists_cache_metrics.path_counts.size() < 4096)
+	{
+		g_fileexists_cache_metrics.path_counts.emplace(key, 1);
+	}
+	else
+	{
+		++g_fileexists_cache_metrics.untracked_path_calls;
+	}
+
+	if (cache_enabled)
+	{
+		auto const lookup = g_fileexists_cache.find(key);
+		if (lookup != g_fileexists_cache.end())
+		{
+			++g_fileexists_cache_metrics.cache_hits;
+			return lookup->second;
+		}
+
+		++g_fileexists_cache_metrics.cache_misses;
+	}
+
+	++g_fileexists_cache_metrics.filesystem_exists_calls;
+	auto const timestart = std::chrono::steady_clock::now();
+	auto const exists = std::filesystem::exists(Filename);
+	g_fileexists_cache_metrics.filesystem_exists_time += std::chrono::steady_clock::now() - timestart;
+	if (exists)
+	{
+		++g_fileexists_cache_metrics.positive_results;
+	}
+	else
+	{
+		++g_fileexists_cache_metrics.negative_results;
+	}
+	if (cache_enabled)
+	{
+		g_fileexists_cache.emplace(key, exists);
+	}
+	return exists;
 }
 
 std::pair<std::string, std::string> FileExists(std::vector<std::string> const &Names, std::vector<std::string> const &Extensions)
@@ -348,6 +537,64 @@ std::pair<std::string, std::string> FileExists(std::vector<std::string> const &N
 	return {{}, {}};
 }
 
+FileExistsCacheScope::FileExistsCacheScope()
+{
+	std::lock_guard<std::mutex> lock(g_fileexists_cache_mutex);
+	if (g_fileexists_cache_depth.load(std::memory_order_relaxed) == 0)
+	{
+		g_fileexists_cache.clear();
+		g_fileexists_cache_enabled = Global.ScenarioFileExistsCache;
+		g_fileexists_cache_metrics.reset();
+	}
+	g_fileexists_cache_depth.fetch_add(1, std::memory_order_release);
+}
+
+FileExistsCacheScope::~FileExistsCacheScope()
+{
+	end();
+}
+
+void FileExistsCacheScope::clear()
+{
+	std::vector<std::string> report;
+	{
+		std::lock_guard<std::mutex> lock(g_fileexists_cache_mutex);
+		report = FileExistsCacheReport();
+		g_fileexists_cache.clear();
+		g_fileexists_cache_metrics.reset();
+	}
+	FileExistsCacheLogReport(report);
+}
+
+void FileExistsCacheScope::end()
+{
+	if (false == m_active)
+	{
+		return;
+	}
+
+	std::vector<std::string> report;
+	{
+		std::lock_guard<std::mutex> lock(g_fileexists_cache_mutex);
+		auto const depth = g_fileexists_cache_depth.load(std::memory_order_relaxed);
+		if (depth <= 1)
+		{
+			report = FileExistsCacheReport();
+			g_fileexists_cache_depth.store(0, std::memory_order_release);
+			g_fileexists_cache.clear();
+			g_fileexists_cache_enabled = false;
+			g_fileexists_cache_metrics.reset();
+		}
+		else
+		{
+			g_fileexists_cache_depth.store(depth - 1, std::memory_order_release);
+		}
+		m_active = false;
+	}
+
+	FileExistsCacheLogReport(report);
+}
+
 // returns time of last modification for specified file
 std::time_t last_modified(std::string const &Filename)
 {

utilities/utilities.h

@@ -201,6 +201,22 @@ bool FileExists(std::string const &Filename);
 
 std::pair<std::string, std::string> FileExists(std::vector<std::string> const &Names, std::vector<std::string> const &Extensions);
 
+class FileExistsCacheScope
+{
+public:
+	FileExistsCacheScope();
+	~FileExistsCacheScope();
+
+	FileExistsCacheScope(FileExistsCacheScope const &) = delete;
+	FileExistsCacheScope &operator=(FileExistsCacheScope const &) = delete;
+
+	void clear();
+	void end();
+
+private:
+	bool m_active { true };
+};
+
 // returns time of last modification for specified file
 std::time_t last_modified(std::string const &Filename);