mirror of
https://github.com/MaSzyna-EU07/maszyna.git
synced 2026-07-22 17:29:18 +02:00
Add headless parallel eu7v2 scenario bake with streaming and PLCE placements
Enable --eu7v2-bake from the main binary: parallel module pool, bounded-RAM spool flush, streaming terrain triangles, flat include/model parsing, and eu7v2 emit/load with optional verify. Large placement .scm files emit lean PLCE records and bake referenced .inc modules separately for reuse. - CLI: --eu7v2-bake, --eu7v2-verify, --eu7v2-mem-limit-gb, --eu7v2-threads, --eu7v2-max-parse; wire max_threads through to the bake parser - eu7v2 v2 records: PLCE placements, runtime emitter/loader, batch verify - Parallel bake pool with session cache; drop heavy-serial parse gate in spool mode; parse concurrency matches thread count - Streaming terrain: batched parallel parse+bake, scan/bake pipeline, shape spool with persistent buffered I/O and flush-before-read - Parallel flat-file streaming for models/includes; pack/model spool for low-memory incremental flush - Optional 50 GB private-bytes guard during headless bake Braniewo_szeroki: 160 modules, verify PASS, ~34s bake (nmt100 ~17s vs ~190s serial baseline). Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
122
scene/eu7/eu7_bake_mem_guard.h
Normal file
122
scene/eu7/eu7_bake_mem_guard.h
Normal file
@@ -0,0 +1,122 @@
|
||||
/*
|
||||
This Source Code Form is subject to the
|
||||
terms of the Mozilla Public License, v.
|
||||
2.0. If a copy of the MPL was not
|
||||
distributed with this file, You can
|
||||
obtain one at
|
||||
http://mozilla.org/MPL/2.0/.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <thread>
|
||||
|
||||
#if defined( _WIN32 )
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#include <psapi.h>
|
||||
#else
|
||||
#include <cstdio>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
namespace scene::eu7::bake_parser {
|
||||
|
||||
inline constexpr unsigned kDefaultBakeMemLimitGb { 50u };
|
||||
|
||||
[[nodiscard]] inline std::uint64_t
|
||||
process_private_bytes() noexcept {
|
||||
#if defined( _WIN32 )
|
||||
PROCESS_MEMORY_COUNTERS_EX counters {};
|
||||
counters.cb = sizeof( counters );
|
||||
if( GetProcessMemoryInfo(
|
||||
GetCurrentProcess(),
|
||||
reinterpret_cast<PROCESS_MEMORY_COUNTERS *>( &counters ),
|
||||
sizeof( counters ) ) ) {
|
||||
return counters.PrivateUsage;
|
||||
}
|
||||
return 0;
|
||||
#else
|
||||
std::FILE *status { std::fopen( "/proc/self/statm", "r" ) };
|
||||
if( status == nullptr ) {
|
||||
return 0;
|
||||
}
|
||||
long total_pages { 0 };
|
||||
long resident_pages { 0 };
|
||||
if( std::fscanf( status, "%ld %ld", &total_pages, &resident_pages ) != 2 ) {
|
||||
std::fclose( status );
|
||||
return 0;
|
||||
}
|
||||
std::fclose( status );
|
||||
long const page_size { sysconf( _SC_PAGESIZE ) };
|
||||
if( page_size <= 0 ) {
|
||||
return 0;
|
||||
}
|
||||
return static_cast<std::uint64_t>( resident_pages ) * static_cast<std::uint64_t>( page_size );
|
||||
#endif
|
||||
}
|
||||
|
||||
// Background poll; abort() when private commit exceeds limit_bytes (0 = disabled).
|
||||
class bake_mem_guard {
|
||||
public:
|
||||
explicit bake_mem_guard( std::uint64_t const limit_bytes ) : m_limit_bytes( limit_bytes ) {}
|
||||
|
||||
bake_mem_guard( bake_mem_guard const & ) = delete;
|
||||
bake_mem_guard &operator=( bake_mem_guard const & ) = delete;
|
||||
|
||||
~bake_mem_guard() { stop(); }
|
||||
|
||||
void start() {
|
||||
if( m_limit_bytes == 0 || m_thread.joinable() ) {
|
||||
return;
|
||||
}
|
||||
m_stop.store( false, std::memory_order_release );
|
||||
m_thread = std::thread( [this]() { run(); } );
|
||||
}
|
||||
|
||||
void stop() {
|
||||
m_stop.store( true, std::memory_order_release );
|
||||
if( m_thread.joinable() ) {
|
||||
m_thread.join();
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
void run() const {
|
||||
while( !m_stop.load( std::memory_order_acquire ) ) {
|
||||
std::uint64_t const used { process_private_bytes() };
|
||||
if( used > m_limit_bytes ) {
|
||||
double const used_gb { static_cast<double>( used ) / ( 1024.0 * 1024.0 * 1024.0 ) };
|
||||
double const limit_gb {
|
||||
static_cast<double>( m_limit_bytes ) / ( 1024.0 * 1024.0 * 1024.0 ) };
|
||||
std::fprintf(
|
||||
stdout,
|
||||
"[EU7v2] FATAL: limit pamieci %.1f GB przekroczony (uzyte ~%.1f GB) — abort\n",
|
||||
limit_gb,
|
||||
used_gb );
|
||||
std::fflush( stdout );
|
||||
std::fprintf(
|
||||
stderr,
|
||||
"[EU7v2] FATAL: limit pamieci %.1f GB przekroczony (uzyte ~%.1f GB) — abort\n",
|
||||
limit_gb,
|
||||
used_gb );
|
||||
std::fflush( stderr );
|
||||
std::abort();
|
||||
}
|
||||
std::this_thread::sleep_for( std::chrono::milliseconds { 500 } );
|
||||
}
|
||||
}
|
||||
|
||||
std::uint64_t const m_limit_bytes { 0 };
|
||||
std::atomic<bool> m_stop { false };
|
||||
std::thread m_thread;
|
||||
};
|
||||
|
||||
} // namespace scene::eu7::bake_parser
|
||||
Reference in New Issue
Block a user