16
0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-22 10:29:19 +02:00

Add headless parallel eu7v2 scenario bake with streaming and PLCE placements

Enable --eu7v2-bake from the main binary: parallel module pool, bounded-RAM
spool flush, streaming terrain triangles, flat include/model parsing, and
eu7v2 emit/load with optional verify. Large placement .scm files emit lean
PLCE records and bake referenced .inc modules separately for reuse.

- CLI: --eu7v2-bake, --eu7v2-verify, --eu7v2-mem-limit-gb, --eu7v2-threads,
  --eu7v2-max-parse; wire max_threads through to the bake parser
- eu7v2 v2 records: PLCE placements, runtime emitter/loader, batch verify
- Parallel bake pool with session cache; drop heavy-serial parse gate in spool
  mode; parse concurrency matches thread count
- Streaming terrain: batched parallel parse+bake, scan/bake pipeline, shape
  spool with persistent buffered I/O and flush-before-read
- Parallel flat-file streaming for models/includes; pack/model spool for
  low-memory incremental flush
- Optional 50 GB private-bytes guard during headless bake

Braniewo_szeroki: 160 modules, verify PASS, ~34s bake (nmt100 ~17s vs ~190s
serial baseline).

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
maj00r
2026-06-17 21:15:36 +02:00
parent 9574d2051b
commit beacc00932
40 changed files with 9327 additions and 193 deletions

View File

@@ -662,6 +662,18 @@ bool global_settings::ConfigParseSimulation(cParser& Parser, const std::string&
return true;
}
if (token == "eu7.bake.cpu.percent")
{
ParseOneClamped(Parser, eu7_bake_cpu_percent, 1, 100);
return true;
}
if (token == "eu7.v2.runtime")
{
ParseOne(Parser, eu7v2_runtime, 1, false);
return true;
}
if (token == "inactivepause")
{
ParseOne(Parser, bInactivePause);
@@ -1616,6 +1628,8 @@ global_settings::export_as_text( std::ostream &Output ) const {
export_as_text( Output, "eu7.pack.mesh.loader.workers", eu7_pack_mesh_loader_workers );
export_as_text( Output, "eu7.auto.bake", eu7_auto_bake );
export_as_text( Output, "eu7.bake.threads", eu7_bake_threads );
export_as_text( Output, "eu7.bake.cpu.percent", eu7_bake_cpu_percent );
export_as_text( Output, "eu7.v2.runtime", eu7v2_runtime );
export_as_text( Output, "inactivepause", bInactivePause );
export_as_text( Output, "slowmotion", iSlowMotionMask );
export_as_text( Output, "hideconsole", bHideConsole );

View File

@@ -86,6 +86,8 @@ struct global_settings {
int eu7_pack_mesh_loader_workers{ 0 }; // 0 = main-thread only (background mesh threads unsafe with Global)
bool eu7_auto_bake{ true }; // przy ladowaniu mapy: bake SCM→EU7 gdy brak lub nieaktualny .eu7
int eu7_bake_threads{ 0 }; // watki bake (0 = auto = hardware_concurrency)
int eu7_bake_cpu_percent{ 80 }; // bake: docelowy % rdzeni logicznych, gdy eu7_bake_threads=0 (auto)
bool eu7v2_runtime{ false }; // eksperyment: transkoduj/laduj moduly przez chudy format eu7v2 (.eu7.v2)
// logs
bool priorityLoadText3D{false}; // ladowanie T3D priorytetowo
int iWriteLogEnabled{ 3 }; // maska bitowa: 1-zapis do pliku, 2-okienko, 4-nazwy torów

View File

@@ -235,34 +235,40 @@ std::string cParser::readTokenFromStream(bool ToLower, const char *Break)
std::string token;
token.reserve(64);
const auto breakTable = makeBreakTable(Break);
char c = 0;
// rebuild the separator lookup table only when the break set actually changes;
// callers overwhelmingly pass the same string literal, so this stays cached.
if (Break != m_breakTableKey) {
m_breakTable = makeBreakTable(Break);
m_breakTableKey = Break;
}
const auto &breakTable = m_breakTable;
int ci = 0;
while ((ci = mStream->get()) != EOF) {
char c = static_cast<char>(ci);
if (c == '\n') {
++mLine;
}
while (token.empty() && mStream->peek() != EOF) {
while (mStream->peek() != EOF) { // idk why but with mStream->get(c) not all cars are loaded
c = static_cast<char>(mStream->get());
if (c == '\n') {
++mLine;
}
const unsigned char uc = static_cast<unsigned char>(c);
if (breakTable[uc]) {
// separator ends token (or continues skipping if token empty)
if (!token.empty())
break;
continue;
}
const unsigned char uc = static_cast<unsigned char>(c);
if (breakTable[uc]) {
// separator ends token (or continues skipping if token empty)
if (!token.empty())
break;
continue;
}
if (ToLower) c = toLowerChar(c);
token.push_back(c);
if (ToLower) c = toLowerChar(c);
token.push_back(c);
if (findQuotes(token)) {
continue; // glue quoted content
}
if (skipComments && trimComments(token)) {
break; // don't glue tokens separated by comment
}
if (findQuotes(token)) {
continue; // glue quoted content
}
if (skipComments && trimComments(token)) {
// comment stripped: return the token if we already have one,
// otherwise keep scanning for the next real token
if (!token.empty())
break;
}
}

View File

@@ -14,10 +14,30 @@ http://mozilla.org/MPL/2.0/.
#include <fstream>
#include <vector>
#include <map>
#include <array>
#include <charconv>
#include <type_traits>
/////////////////////////////////////////////////////////////////////////////////////////////////////
// cParser -- generic class for parsing text data, either from file or provided string
namespace cparser_detail {
// Types for which std::from_chars matches the legacy stringstream extraction semantics.
// Character types are excluded on purpose: narrow-stream extraction of char/wchar_t reads a
// single character, whereas std::from_chars would parse an integer.
template<typename T>
inline constexpr bool use_from_chars_v =
( std::is_integral_v<T>
&& !std::is_same_v<T, bool>
&& !std::is_same_v<T, char>
&& !std::is_same_v<T, signed char>
&& !std::is_same_v<T, unsigned char>
&& !std::is_same_v<T, wchar_t>
&& !std::is_same_v<T, char16_t>
&& !std::is_same_v<T, char32_t> )
|| std::is_floating_point_v<T>;
} // namespace cparser_detail
class cParser //: public std::stringstream
{
public:
@@ -126,6 +146,9 @@ class cParser //: public std::stringstream
commentmap mComments {
commentmap::value_type( "/*", "*/" ),
commentmap::value_type( "//", "\n" ) };
// cached separator lookup table to avoid rebuilding it on every token read
char const *m_breakTableKey { nullptr };
std::array<bool, 256> m_breakTable {};
std::shared_ptr<cParser> mIncludeParser; // child class to handle include directives.
std::vector<std::string> parameters; // parameter list for included file.
std::deque<std::string> tokens;
@@ -143,11 +166,34 @@ cParser::operator>>( Type_ &Right ) {
if( true == this->tokens.empty() ) { return *this; }
std::stringstream converter( this->tokens.front() );
converter >> Right;
this->tokens.pop_front();
if constexpr( cparser_detail::use_from_chars_v<Type_> ) {
std::string const &token = this->tokens.front();
char const *first { token.data() };
char const *const last { first + token.size() };
// legacy stream extraction accepts a leading '+', std::from_chars does not
if( first != last && *first == '+' ) {
++first;
}
Type_ value {};
auto const result { std::from_chars( first, last, value ) };
if( result.ec == std::errc() ) {
Right = value;
}
else {
// fall back to the legacy path for inputs from_chars rejects (inf/nan/hex/...)
std::stringstream converter( token );
converter >> Right;
}
this->tokens.pop_front();
return *this;
}
else {
std::stringstream converter( this->tokens.front() );
converter >> Right;
this->tokens.pop_front();
return *this;
return *this;
}
}
template<>