eu07/maszyna
16
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-21 13:39:18 +02:00
Code Issues 123 Wiki Activity

Add headless parallel eu7v2 scenario bake with streaming and PLCE placements

Browse Source 
Enable --eu7v2-bake from the main binary: parallel module pool, bounded-RAM
spool flush, streaming terrain triangles, flat include/model parsing, and
eu7v2 emit/load with optional verify. Large placement .scm files emit lean
PLCE records and bake referenced .inc modules separately for reuse.

- CLI: --eu7v2-bake, --eu7v2-verify, --eu7v2-mem-limit-gb, --eu7v2-threads,
  --eu7v2-max-parse; wire max_threads through to the bake parser
- eu7v2 v2 records: PLCE placements, runtime emitter/loader, batch verify
- Parallel bake pool with session cache; drop heavy-serial parse gate in spool
  mode; parse concurrency matches thread count
- Streaming terrain: batched parallel parse+bake, scan/bake pipeline, shape
  spool with persistent buffered I/O and flush-before-read
- Parallel flat-file streaming for models/includes; pack/model spool for
  low-memory incremental flush
- Optional 50 GB private-bytes guard during headless bake

Braniewo_szeroki: 160 modules, verify PASS, ~34s bake (nmt100 ~17s vs ~190s
serial baseline).

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
maj00r
2026-06-17 21:15:36 +02:00
parent 9574d2051b
commit beacc00932
40 changed files with 9327 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
eu07-parser/include/eu07/parser.hpp
View File

@@ -87,13 +87,20 @@ inline void tokenizeInto(
skipFieldSeparators(text);
continue;
}
// Unterminated '[' (no matching ']' in the remainder): it is an
// ordinary character, not a bracket token. The engine's default
// break set is "\n\r\t ;", so '[' belongs inside identifiers such
// as event names ("Kon_It2[Sch2"). Fall through to the general scan
// below, which does NOT stop at '[' and therefore consumes it,
// guaranteeing the outer loop makes forward progress (previously it
// spun forever: the '[' was neither a bracket nor a separator nor a
// token character, so text never shrank).
}
const char* const start = text.data();
while (!text.empty() &&
!isFieldSeparator(static_cast<unsigned char>(text.front())) &&
text.front() != '"' &&
text.front() != '[') {
text.front() != '"') {
text.remove_prefix(1);
}
const std::size_t len = static_cast<std::size_t>(text.data() - start);
@@ -459,10 +466,7 @@ struct ParseResult {
return tokens;
}
[[nodiscard]] inline ParseResult parseFile(
const std::filesystem::path& path,
const ParseOptions& options = {}) {
const RawFile raw = readRawFile(path);
[[nodiscard]] inline ParseResult parseRawFile(RawFile raw, const ParseOptions& options = {}) {
LogicalPass logical = toLogicalLines(raw.lines);
ParseResult result;
@@ -474,6 +478,12 @@ struct ParseResult {
return result;
}
[[nodiscard]] inline ParseResult parseFile(
const std::filesystem::path& path,
const ParseOptions& options = {}) {
return parseRawFile(readRawFile(path), options);
}
[[nodiscard]] inline ParseResult parseText(const std::string& text) {
RawFile raw;
raw.data = text;

350
eu07-parser/include/eu07/scene/bake/bake_tree.hpp
View File

@@ -12,13 +12,13 @@
#include <eu07/scene/bake/compose_pack_models.hpp>
#include <eu07/scene/bake/module.hpp>
#include <eu07/scene/bake/pack_model_spool.hpp>
#include <eu07/scene/bake/streaming_terrain.hpp>
#include <eu07/scene/binary/runtime_module.hpp>
#include <eu07/scene/include_resolve.hpp>
#include <eu07/scene/include_scan.hpp>
#include <eu07/scene/processor.hpp>
@@ -39,6 +39,8 @@
#include <iostream>
#include <memory>
#include <mutex>
#include <optional>
@@ -122,8 +124,44 @@ struct BakeTreeOptions {
BakeProgressCallback onProgress;
// Hook wywolywany po zbakowaniu kazdego modulu (przed/zamiast zapisu .eu7).
// Dla roota packBatches != nullptr (sploszczone modele PACK). isRoot == true
// tylko dla pliku-korzenia drzewa bake. Moze byc wolany z wielu watkow.
std::function<void(
const RuntimeModule& module,
const std::filesystem::path& textPath,
bool isRoot,
const std::vector<binary::codec::ModelSectionBatch>* packBatches,
ShapeSpoolFile* shapeSpool)>
onModuleBaked;
// Gdy true: pomijamy zapis legacy .eu7 (writeRuntimeModule).
bool skipLegacyWrite = false;
// eu7v2 PACK: modele dzieci sa juz w root PACK — nie duplikuj INST w .eu7v2 dziecka.
bool omitChildModuleModels = false;
// Max rownoleglych pelnych parseFile/processScene w session cache (0 = auto: 1).
// Ogranicza peak RAM gdy wiele ciezkich .scm startuje naraz (Braniewo / linie).
unsigned maxConcurrentParses = 0;
// Mniej RAM, wolniej: 1 watek bake/compose, agresywne zwalnianie cache PACK.
bool lowMemoryMode = false;
// Pliki wieksze niz X MB: parse/bake tylko pojedynczo (0 = wylaczone).
unsigned heavyParseThresholdMb = 0;
// PACK compose: spłucz sekcje modeli poza RAM gdy przekroczony prog (lowMemoryMode).
std::size_t packFlushThreshold = 0;
bool packFlushPerFile = false;
std::function<void(std::vector<runtime::RuntimeModelInstance>&&)> onPackModelsFlush;
};
[[nodiscard]] inline bool useIncrementalRootPack(const BakeTreeOptions& options) {
return options.lowMemoryMode;
}
struct BakeTreeContext {
@@ -256,6 +294,17 @@ struct BakePoolState {
}
[[nodiscard]] inline unsigned resolveParseConcurrencyLimit(const unsigned maxConcurrentParses) {
if (maxConcurrentParses != 0) {
return std::max(1u, maxConcurrentParses);
}
return 1u;
}
[[nodiscard]] inline std::size_t heavyParseThresholdBytes(const unsigned threshold_mb) {
return threshold_mb == 0 ? 0u : static_cast<std::size_t>(threshold_mb) * 1024u * 1024u;
}
inline void recordError(BakePoolState& pool) {
@@ -324,7 +373,9 @@ inline void bakeWorkItem(
const BakeTreeOptions& options,
const unsigned threadCount) {
const unsigned threadCount,
BakeParseSessionCache& sessionCache) {
if (pool.stop.load(std::memory_order_acquire)) {
@@ -358,34 +409,141 @@ inline void bakeWorkItem(
pool.inFlight.fetch_add(1, std::memory_order_relaxed);
if (options.onProgress) {
std::lock_guard lock(pool.progressMutex);
options.onProgress(
BakeProgress{
BakeProgressPhase::Start,
pool.completed.load(std::memory_order_relaxed),
0,
item.path,
threadCount,
});
}
bool models_pending_pack = false;
try {
const ParseResult parsed = parseFile(item.path);
SceneProcessOptions shallowOpts;
shallowOpts.expandIncludes = false;
const SceneDocument document =
processScene(parsed, context.sceneryRoot, shallowOpts).document;
RuntimeModule module;
const bool emitPackModels = canonicalPathKey(item.path) == rootKey;
RuntimeModule module = bakeModule(document, {.skipLocalModels = emitPackModels});
std::vector<ParsedInclude> deferred_child_includes;
eu07::scene::detail::FlatFileKind flat_kind =
eu07::scene::detail::FlatFileKind::None;
std::unique_ptr<ShapeSpoolFile> triangle_shape_spool;
{
if (options.onProgress) {
std::lock_guard lock(pool.progressMutex);
options.onProgress(
BakeProgress{
BakeProgressPhase::Start,
pool.completed.load(std::memory_order_relaxed),
0,
item.path,
threadCount,
});
}
const bool prefer_lightweight_document =
useIncrementalRootPack(options) ||
eu07::scene::detail::shouldStreamFlatSourceFile(item.path);
if (prefer_lightweight_document) {
flat_kind = eu07::scene::detail::scanFlatFileKindStreaming(item.path);
}
const bool try_triangle_terrain_stream =
prefer_lightweight_document &&
flat_kind == eu07::scene::detail::FlatFileKind::None &&
eu07::scene::detail::shouldStreamFlatSourceFile(item.path);
bool used_triangle_terrain_stream = false;
SceneDocument empty_document;
const SceneDocument* document_ptr = nullptr;
if (try_triangle_terrain_stream) {
if (options.lowMemoryMode) {
const std::size_t path_hash = std::hash<std::string> {}(
item.path.lexically_normal().generic_string());
triangle_shape_spool = std::make_unique<ShapeSpoolFile>(
std::filesystem::temp_directory_path() /
("eu7v2_shape_" + std::to_string(path_hash) + ".bin"));
}
if (detail::streamBakeTriangleTerrain(
item.path, module, triangle_shape_spool.get(), threadCount)) {
used_triangle_terrain_stream = true;
document_ptr = &empty_document;
}
}
if (!used_triangle_terrain_stream) {
const SceneDocument& document =
(prefer_lightweight_document &&
(flat_kind == eu07::scene::detail::FlatFileKind::Models ||
flat_kind == eu07::scene::detail::FlatFileKind::Includes))
? sessionCache.documentForPackCompose(
item.path, context.sceneryRoot, nullptr, true)
: sessionCache.documentFor(item.path, context.sceneryRoot, nullptr);
document_ptr = &document;
}
const SceneDocument& document = *document_ptr;
const auto enqueue_unique_inc_modules = [&](const SceneDocument& doc) {
if (!options.onModuleBaked) {
return;
}
BakeWorkItem child;
child.includeStack = item.includeStack;
child.includeStack.push_back(item.path);
for (const ParsedInclude& include : doc.include) {
if (!eu07::scene::detail::isIncFile(include.file)) {
continue;
}
child.path = eu07::scene::detail::resolveIncludeSourcePath(
context.sceneryRoot, item.relativeFile, include.file);
child.relativeFile = eu07::scene::detail::relativeSceneryFile(
context.sceneryRoot, child.path);
enqueueWork(pool, child);
}
};
const auto enqueue_discovered_includes = [&](const SceneDocument& doc) {
BakeWorkItem child;
child.includeStack = item.includeStack;
child.includeStack.push_back(item.path);
for (const ParsedInclude& include : doc.include) {
// Placementy .inc (flora, szablony) — tylko do PACK compose,
// nie osobne moduly bake (3_rosl1-leo: 17k+ linii).
if (eu07::scene::detail::isIncFile(include.file)) {
continue;
}
child.path = eu07::scene::detail::resolveIncludeSourcePath(
context.sceneryRoot, item.relativeFile, include.file);
child.relativeFile = eu07::scene::detail::relativeSceneryFile(
context.sceneryRoot, child.path);
enqueueWork(pool, child);
}
};
// Root PACK compose jest bardzo pamieciochlonny — nie startuj rownolegle
// pelnych parse linii (l204/l220/…) az compose roota sie nie skonczy.
// lowMemoryMode + spool: jeden modul = jeden plik .scm, PACK skladany per modul.
if (emitPackModels) {
if (useIncrementalRootPack(options)) {
enqueue_discovered_includes(document);
} else {
deferred_child_includes = document.include;
}
} else {
enqueue_discovered_includes(document);
}
enqueue_unique_inc_modules(document);
const bool skipLocalModels =
emitPackModels ||
(options.omitChildModuleModels && isModelOnlyDocument(document));
if (!used_triangle_terrain_stream) {
module = bakeModule(document, {.skipLocalModels = skipLocalModels});
}
models_pending_pack =
!used_triangle_terrain_stream &&
skipLocalModels &&
(isModelOnlyDocument(document) ||
flat_kind == eu07::scene::detail::FlatFileKind::Models);
if (!used_triangle_terrain_stream) {
sessionCache.releaseHeavyParseStorageAfterModuleBake(
item.path, models_pending_pack);
}
}
const std::filesystem::path eu7Path =
@@ -396,7 +554,7 @@ inline void bakeWorkItem(
binary::WriteRuntimeModuleOptions writeOpts;
writeOpts.emitPackModels = emitPackModels;
std::vector<binary::codec::ModelSectionBatch> packBatches;
if (writeOpts.emitPackModels) {
if (writeOpts.emitPackModels && !useIncrementalRootPack(options)) {
if (options.onProgress) {
std::lock_guard lock(pool.progressMutex);
options.onProgress(
@@ -431,11 +589,12 @@ inline void bakeWorkItem(
pack_options.progress = &pack_progress;
pack_options.stats = &pack_stats;
pack_options.max_threads = threadCount;
pack_options.low_memory = options.lowMemoryMode;
pack_options.pack_flush_threshold = options.packFlushThreshold;
pack_options.pack_flush_per_file = options.packFlushPerFile;
pack_options.on_pack_models_flush = options.onPackModelsFlush;
pack_options.pack_batches = &packBatches;
PackComposeSeed root_seed;
root_seed.path = item.path.lexically_normal();
root_seed.entry = makePackFileCacheEntry(document);
pack_options.root_seed = &root_seed;
pack_options.session_cache = &sessionCache;
if (options.onProgress) {
std::cerr << "[EU7] PACK compose: " << threadCount << " watkow\n" << std::flush;
}
@@ -457,21 +616,91 @@ inline void bakeWorkItem(
<< std::flush;
}
printPackComposeStats(pack_stats, std::cerr);
printBakeParseSessionStats(sessionCache, std::cerr);
if (stats != nullptr) {
stats->has_pack_diagnostics = true;
stats->pack_compose = snapshotPackComposeStats(pack_stats);
}
}
binary::PackWriteStats pack_write_stats;
if (writeOpts.emitPackModels) {
writeOpts.pack_write_stats = &pack_write_stats;
std::cerr << "[EU7] zapis .eu7 (PACK)...\n" << std::flush;
// lowMemoryMode: jeden plik = jeden modul — PACK tylko z tego .scm, potem spool/RAM zwolnione.
if (useIncrementalRootPack(options) && models_pending_pack && !emitPackModels) {
if (flat_kind == eu07::scene::detail::FlatFileKind::Models &&
options.onPackModelsFlush) {
detail::streamFlatModelsToPackFlush(item.path, options.onPackModelsFlush);
} else {
std::vector<binary::codec::ModelSectionBatch> module_pack_batches;
PackComposeOptions pack_options;
pack_options.low_memory = true;
pack_options.pack_flush_per_file = options.packFlushPerFile;
pack_options.on_pack_models_flush = options.onPackModelsFlush;
pack_options.pack_batches = &module_pack_batches;
pack_options.session_cache = &sessionCache;
pack_options.max_threads = 1;
(void)composePackModels(
item.path, context.sceneryRoot, item.relativeFile, pack_options);
if (options.onPackModelsFlush) {
std::vector<runtime::RuntimeModelInstance> tail;
for (binary::codec::ModelSectionBatch& batch : module_pack_batches) {
tail.insert(
tail.end(),
std::make_move_iterator(batch.models.begin()),
std::make_move_iterator(batch.models.end()));
}
if (!tail.empty()) {
options.onPackModelsFlush(std::move(tail));
}
}
}
sessionCache.dropPackCacheEntry(item.path);
}
binary::writeRuntimeModule(eu7Path, module, writeOpts);
if (writeOpts.emitPackModels) {
printPackWriteStats(pack_write_stats, std::cerr);
if (stats != nullptr) {
stats->pack_write = pack_write_stats;
// Hook: pozwala silnikowi wyemitowac wlasny format (eu7v2) bezposrednio
// z RuntimeModule, z dostepem do sploszczonych modeli PACK dla roota.
if (options.onModuleBaked) {
const bool root_with_pack =
emitPackModels && !useIncrementalRootPack(options);
const std::vector<binary::codec::ModelSectionBatch>* batchesPtr =
root_with_pack ? &packBatches : nullptr;
options.onModuleBaked(
module, item.path, emitPackModels, batchesPtr, triangle_shape_spool.get());
}
if (emitPackModels && !deferred_child_includes.empty() &&
!useIncrementalRootPack(options)) {
SceneDocument stub;
stub.include = std::move(deferred_child_includes);
BakeWorkItem child;
child.includeStack = item.includeStack;
child.includeStack.push_back(item.path);
for (const ParsedInclude& include : stub.include) {
if (eu07::scene::detail::isIncFile(include.file)) {
continue;
}
child.path = eu07::scene::detail::resolveIncludeSourcePath(
context.sceneryRoot, item.relativeFile, include.file);
child.relativeFile =
eu07::scene::detail::relativeSceneryFile(context.sceneryRoot, child.path);
enqueueWork(pool, child);
}
}
if (!models_pending_pack) {
sessionCache.evictEntryIfNotNeeded(item.path);
}
if (!options.skipLegacyWrite) {
binary::PackWriteStats pack_write_stats;
if (writeOpts.emitPackModels) {
writeOpts.pack_write_stats = &pack_write_stats;
std::cerr << "[EU7] zapis .eu7 (PACK)...\n" << std::flush;
}
binary::writeRuntimeModule(eu7Path, module, writeOpts);
if (writeOpts.emitPackModels) {
printPackWriteStats(pack_write_stats, std::cerr);
if (stats != nullptr) {
stats->pack_write = pack_write_stats;
}
}
}
@@ -525,30 +754,6 @@ inline void bakeWorkItem(
}
BakeWorkItem child;
child.includeStack = item.includeStack;
child.includeStack.push_back(item.path);
for (const std::string& includeFile : scanIncludeFilePaths(parsed.tokens)) {
child.path = eu07::scene::detail::resolveIncludeSourcePath(
context.sceneryRoot, item.relativeFile, includeFile);
child.relativeFile = eu07::scene::detail::relativeSceneryFile(
context.sceneryRoot, child.path);
enqueueWork(pool, child);
}
} catch (...) {
recordError(pool);
@@ -585,7 +790,9 @@ inline void bakePoolWorker(
const BakeTreeOptions& options,
const unsigned threadCount) {
const unsigned threadCount,
BakeParseSessionCache& sessionCache) {
while (true) {
@@ -639,7 +846,7 @@ inline void bakePoolWorker(
bakeWorkItem(
item, context, rootKey, pool, stats, rootModuleOut, options, threadCount);
item, context, rootKey, pool, stats, rootModuleOut, options, threadCount, sessionCache);
} catch (...) {
@@ -690,6 +897,11 @@ inline void runBakePool(
BakePoolState pool;
BakeParseSessionCache sessionCache {
detail::resolveParseConcurrencyLimit(options.maxConcurrentParses),
options.lowMemoryMode,
detail::heavyParseThresholdBytes(options.heavyParseThresholdMb) };
enqueueWork(
pool,
@@ -728,7 +940,9 @@ inline void runBakePool(
std::cref(options),
threadCount);
threadCount,
std::ref(sessionCache));
}

1481
eu07-parser/include/eu07/scene/bake/compose_pack_models.hpp
View File

File diff suppressed because it is too large Load Diff

138
eu07-parser/include/eu07/scene/bake/pack_model_spool.hpp Normal file
View File

@@ -0,0 +1,138 @@
#pragma once
// Append-only spool: modele spłukane z PACK compose (mniejszy peak RAM).
#include <eu07/scene/binary/runtime_module.hpp>
#include <eu07/scene/runtime/nodes.hpp>
#include <array>
#include <cstddef>
#include <filesystem>
#include <fstream>
#include <mutex>
#include <stdexcept>
#include <utility>
#include <vector>
namespace eu07::scene::bake {
inline constexpr std::size_t kSpoolStreamBufferBytes = 1024u * 1024u;
class PackModelSpoolFile {
public:
explicit PackModelSpoolFile(std::filesystem::path path)
: path_(std::move(path)),
out_(path_, std::ios::binary | std::ios::trunc) {
if (!out_) {
throw std::runtime_error("EU7 PACK spool: nie mozna utworzyc " + path_.string());
}
out_.rdbuf()->pubsetbuf(write_buffer_.data(), write_buffer_.size());
}
void append(std::vector<runtime::RuntimeModelInstance>&& models) {
if (models.empty()) {
return;
}
std::lock_guard lock(mutex_);
for (runtime::RuntimeModelInstance& model : models) {
binary::detail::collectModelStrings(table_, model);
binary::detail::writeRuntimeModel(out_, table_, model);
++model_count_;
}
}
template <typename Fn>
void for_each_model(Fn&& fn) const {
{
std::lock_guard lock(mutex_);
out_.flush();
}
std::ifstream in(path_, std::ios::binary);
if (!in) {
return;
}
while (in.peek() != EOF) {
fn(binary::detail::readRuntimeModel(in, table_));
}
}
[[nodiscard]] std::size_t model_count() const noexcept {
return model_count_;
}
[[nodiscard]] const std::filesystem::path& path() const noexcept {
return path_;
}
private:
std::filesystem::path path_;
binary::StringTable table_;
std::size_t model_count_ = 0;
std::array<char, kSpoolStreamBufferBytes> write_buffer_ {};
mutable std::ofstream out_;
mutable std::mutex mutex_;
};
class ShapeSpoolFile {
public:
explicit ShapeSpoolFile(std::filesystem::path path)
: path_(std::move(path)),
out_(path_, std::ios::binary | std::ios::trunc) {
if (!out_) {
throw std::runtime_error("EU7 shape spool: nie mozna utworzyc " + path_.string());
}
out_.rdbuf()->pubsetbuf(write_buffer_.data(), write_buffer_.size());
}
void append(runtime::RuntimeShapeNode&& shape) {
std::lock_guard lock(mutex_);
binary::detail::collectShapeStrings(table_, shape);
binary::detail::writeRuntimeShape(out_, table_, shape);
++shape_count_;
}
void append_batch(std::vector<runtime::RuntimeShapeNode>&& shapes) {
if (shapes.empty()) {
return;
}
std::lock_guard lock(mutex_);
for (runtime::RuntimeShapeNode& shape : shapes) {
binary::detail::collectShapeStrings(table_, shape);
binary::detail::writeRuntimeShape(out_, table_, shape);
++shape_count_;
}
}
template <typename Fn>
void for_each_shape(Fn&& fn) const {
{
std::lock_guard lock(mutex_);
out_.flush();
}
std::ifstream in(path_, std::ios::binary);
if (!in) {
return;
}
while (in.peek() != EOF) {
fn(binary::detail::readRuntimeShape(in, table_));
}
}
[[nodiscard]] std::size_t shape_count() const noexcept {
return shape_count_;
}
[[nodiscard]] const std::filesystem::path& path() const noexcept {
return path_;
}
private:
std::filesystem::path path_;
binary::StringTable table_;
std::size_t shape_count_ = 0;
std::array<char, kSpoolStreamBufferBytes> write_buffer_ {};
mutable std::ofstream out_;
mutable std::mutex mutex_;
};
} // namespace eu07::scene::bake

432
eu07-parser/include/eu07/scene/bake/streaming_terrain.hpp Normal file
View File

@@ -0,0 +1,432 @@
#pragma once
// Duze pliki terenu (same node triangles, np. nmt100_warmaz_ter.scm):
// linia po linii, parse per-wezel, bez readRawFile / tokenizacji calego pliku.
#include <eu07/parser.hpp>
#include <eu07/scene/bake/mesh.hpp>
#include <eu07/scene/bake/module.hpp>
#include <eu07/scene/bake/pack_model_spool.hpp>
#include <eu07/scene/context.hpp>
#include <eu07/scene/cursor.hpp>
#include <eu07/scene/match.hpp>
#include <eu07/scene/node/common.hpp>
#include <eu07/scene/node/triangles.hpp>
#include <eu07/scene/parallel_models.hpp>
#include <atomic>
#include <condition_variable>
#include <cstddef>
#include <deque>
#include <filesystem>
#include <functional>
#include <mutex>
#include <string>
#include <string_view>
#include <thread>
#include <vector>
namespace eu07::scene::bake::detail {
enum class TriangleTerrainLineKind {
Skip,
Header,
Vertex,
EndTri,
Invalid,
};
[[nodiscard]] inline TriangleTerrainLineKind classifyTriangleTerrainLine(std::string_view text) {
skipFieldSeparators(text);
if (text.empty()) {
return TriangleTerrainLineKind::Skip;
}
if (eu07::detail::isLineComment(text)) {
return TriangleTerrainLineKind::Skip;
}
if (eu07::scene::detail::segmentStartsWithKeyword(text, "node")) {
if (text.find("triangles") == std::string_view::npos) {
return TriangleTerrainLineKind::Invalid;
}
return TriangleTerrainLineKind::Header;
}
if (isKeyword(text, "endtri") || isKeyword(text, "endtriangles")) {
return TriangleTerrainLineKind::EndTri;
}
if (!text.empty() &&
(text.front() == '-' || text.front() == '+' ||
(text.front() >= '0' && text.front() <= '9'))) {
return TriangleTerrainLineKind::Vertex;
}
return TriangleTerrainLineKind::Invalid;
}
[[nodiscard]] inline bool parseTriangleTerrainBlock(
const std::vector<std::string>& lines,
const std::size_t header_line,
ParsedNodeTriangles& out) {
if (lines.size() < 2) {
return false;
}
std::vector<SourceToken> header_tokens;
tokenizeInto(lines.front(), header_tokens, header_line);
TokenStream stream(header_tokens);
NodeHeader header;
std::string subtype;
std::vector<SourceToken> raw;
if (!node::io::consumeHeader(stream, header, subtype, raw) ||
!isKeyword(subtype, node_triangles::kSubtype) ||
!node::io::takeString(stream, raw, out.texture)) {
return false;
}
out.header = header;
out.raw.clear();
out.vertices.clear();
out.vertices.reserve(lines.size() > 2 ? lines.size() - 2 : 0);
for (std::size_t i = 1; i + 1 < lines.size(); ++i) {
std::vector<SourceToken> vertex_tokens;
tokenizeInto(lines[i], vertex_tokens, header_line + i);
TokenStream vertex_stream(vertex_tokens);
MeshVertex vertex;
if (!node_triangles::takeVertex(vertex_stream, raw, vertex, !vertex_stream.empty())) {
return false;
}
out.vertices.push_back(vertex);
}
return !out.vertices.empty();
}
struct TriangleTerrainBlock {
std::size_t header_line = 0;
std::vector<std::string> lines;
};
constexpr std::size_t kTerrainParallelBatchSize = 8192;
constexpr std::size_t kTerrainParallelMinBlocks = 8;
constexpr std::size_t kTerrainPipelineMaxQueuedBatches = 2;
[[nodiscard]] inline unsigned resolveTerrainThreadCount(const unsigned max_threads) {
const unsigned hw =
std::thread::hardware_concurrency() == 0 ? 4u : std::thread::hardware_concurrency();
if (max_threads == 0) {
return std::max(1u, hw);
}
return std::max(1u, max_threads);
}
[[nodiscard]] inline bool bakeTriangleTerrainBatch(
const std::vector<TriangleTerrainBlock>& batch,
const unsigned max_threads,
std::vector<runtime::RuntimeShapeNode>& baked_out) {
baked_out.resize(batch.size());
if (batch.empty()) {
return true;
}
if (batch.size() < kTerrainParallelMinBlocks ||
resolveTerrainThreadCount(max_threads) <= 1) {
for (std::size_t i = 0; i < batch.size(); ++i) {
ParsedNodeTriangles parsed;
if (!parseTriangleTerrainBlock(batch[i].lines, batch[i].header_line, parsed)) {
return false;
}
baked_out[i] = bakeTriangles(parsed);
}
return true;
}
const unsigned worker_count = resolveTerrainThreadCount(max_threads);
std::atomic<std::size_t> next_index { 0 };
std::atomic<bool> ok { true };
const auto worker = [&]() {
while (ok.load(std::memory_order_relaxed)) {
const std::size_t index = next_index.fetch_add(1, std::memory_order_relaxed);
if (index >= batch.size()) {
return;
}
try {
ParsedNodeTriangles parsed;
if (!parseTriangleTerrainBlock(
batch[index].lines, batch[index].header_line, parsed)) {
ok.store(false, std::memory_order_relaxed);
return;
}
baked_out[index] = bakeTriangles(parsed);
} catch (...) {
ok.store(false, std::memory_order_relaxed);
return;
}
}
};
std::vector<std::thread> threads;
threads.reserve(worker_count);
for (unsigned i = 0; i < worker_count; ++i) {
threads.emplace_back(worker);
}
for (std::thread& thread : threads) {
thread.join();
}
return ok.load(std::memory_order_relaxed);
}
[[nodiscard]] inline bool scanTriangleTerrainOnlyStreaming(const std::filesystem::path& path) {
bool saw_header = false;
bool in_block = false;
bool ok = true;
eu07::scene::detail::forEachStreamingLogicalSegment(
path, [&](const LogicalSegment& segment) {
const TriangleTerrainLineKind kind = classifyTriangleTerrainLine(segment.text);
switch (kind) {
case TriangleTerrainLineKind::Skip:
return true;
case TriangleTerrainLineKind::Header:
saw_header = true;
in_block = true;
return true;
case TriangleTerrainLineKind::Vertex:
if (!in_block) {
ok = false;
return false;
}
return true;
case TriangleTerrainLineKind::EndTri:
if (!in_block) {
ok = false;
return false;
}
in_block = false;
return true;
case TriangleTerrainLineKind::Invalid:
default:
ok = false;
return false;
}
});
return ok && saw_header && !in_block;
}
[[nodiscard]] inline bool scanTriangleTerrainBlocksStreaming(
const std::filesystem::path& path,
const std::function<bool(std::vector<TriangleTerrainBlock>&&)>& on_batch) {
std::vector<TriangleTerrainBlock> batch;
batch.reserve(kTerrainParallelBatchSize);
TriangleTerrainBlock current_block;
bool in_block = false;
bool saw_header = false;
const auto finish_block = [&]() -> bool {
if (current_block.lines.empty()) {
return true;
}
batch.push_back(std::move(current_block));
current_block = {};
in_block = false;
if (batch.size() >= kTerrainParallelBatchSize) {
if (!on_batch(std::move(batch))) {
return false;
}
batch = {};
batch.reserve(kTerrainParallelBatchSize);
}
return true;
};
bool ok = true;
eu07::scene::detail::forEachStreamingLogicalSegment(
path, [&](const LogicalSegment& segment) {
const TriangleTerrainLineKind kind = classifyTriangleTerrainLine(segment.text);
switch (kind) {
case TriangleTerrainLineKind::Skip:
return true;
case TriangleTerrainLineKind::Header:
saw_header = true;
if (!finish_block()) {
ok = false;
return false;
}
current_block.header_line = segment.sourceLine;
current_block.lines.emplace_back(segment.text);
in_block = true;
return true;
case TriangleTerrainLineKind::Vertex:
if (!in_block) {
ok = false;
return false;
}
current_block.lines.emplace_back(segment.text);
return true;
case TriangleTerrainLineKind::EndTri:
if (!in_block) {
ok = false;
return false;
}
current_block.lines.emplace_back(segment.text);
if (!finish_block()) {
ok = false;
return false;
}
return true;
case TriangleTerrainLineKind::Invalid:
default:
ok = false;
return false;
}
});
if (ok && in_block) {
ok = finish_block();
}
if (ok && !batch.empty()) {
ok = on_batch(std::move(batch));
}
return ok && saw_header;
}
// Zwraca false gdy plik nie jest czystym terenem triangles (wtedy uzyj documentFor).
// Gdy shape_spool != nullptr, ksztalty trafiaja na dysk (module.scene.shapes puste).
[[nodiscard]] inline bool streamBakeTriangleTerrain(
const std::filesystem::path& path,
RuntimeModule& module,
ShapeSpoolFile* shape_spool = nullptr,
const unsigned max_threads = 0) {
module.scene.shapes.clear();
const unsigned bake_threads = resolveTerrainThreadCount(max_threads);
bool parsed_any = false;
const auto emit_baked = [&](std::vector<runtime::RuntimeShapeNode>& baked) -> bool {
if (baked.empty()) {
return true;
}
parsed_any = true;
if (shape_spool != nullptr) {
shape_spool->append_batch(std::move(baked));
} else {
module.scene.shapes.insert(
module.scene.shapes.end(),
std::make_move_iterator(baked.begin()),
std::make_move_iterator(baked.end()));
baked.clear();
}
return true;
};
const auto bake_and_emit = [&](std::vector<TriangleTerrainBlock>&& blocks) -> bool {
if (blocks.empty()) {
return true;
}
std::vector<runtime::RuntimeShapeNode> baked;
if (!bakeTriangleTerrainBatch(blocks, max_threads, baked)) {
return false;
}
return emit_baked(baked);
};
if (bake_threads <= 1) {
const bool ok = scanTriangleTerrainBlocksStreaming(path, bake_and_emit);
if (!ok || !parsed_any) {
module.scene.shapes.clear();
return false;
}
return true;
}
std::mutex queue_mutex;
std::condition_variable queue_cv;
std::deque<std::vector<TriangleTerrainBlock>> ready_batches;
std::atomic<bool> scan_ok { true };
std::atomic<bool> scan_done { false };
std::atomic<bool> bake_failed { false };
std::thread scanner([&]() {
try {
const bool ok = scanTriangleTerrainBlocksStreaming(
path, [&](std::vector<TriangleTerrainBlock>&& blocks) -> bool {
if (bake_failed.load(std::memory_order_relaxed)) {
return false;
}
std::unique_lock lock(queue_mutex);
queue_cv.wait(lock, [&]() {
return bake_failed.load(std::memory_order_relaxed) ||
ready_batches.size() < kTerrainPipelineMaxQueuedBatches;
});
if (bake_failed.load(std::memory_order_relaxed)) {
return false;
}
ready_batches.push_back(std::move(blocks));
lock.unlock();
queue_cv.notify_one();
return true;
});
scan_ok.store(ok, std::memory_order_relaxed);
} catch (...) {
scan_ok.store(false, std::memory_order_relaxed);
bake_failed.store(true, std::memory_order_relaxed);
queue_cv.notify_all();
}
scan_done.store(true, std::memory_order_release);
queue_cv.notify_all();
});
while (true) {
std::vector<TriangleTerrainBlock> batch;
{
std::unique_lock lock(queue_mutex);
queue_cv.wait(lock, [&]() {
return bake_failed.load(std::memory_order_relaxed) ||
!ready_batches.empty() ||
scan_done.load(std::memory_order_acquire);
});
if (bake_failed.load(std::memory_order_relaxed)) {
break;
}
if (ready_batches.empty() && scan_done.load(std::memory_order_acquire)) {
break;
}
if (ready_batches.empty()) {
continue;
}
batch = std::move(ready_batches.front());
ready_batches.pop_front();
}
queue_cv.notify_one();
std::vector<runtime::RuntimeShapeNode> baked;
if (!bakeTriangleTerrainBatch(batch, max_threads, baked)) {
bake_failed.store(true, std::memory_order_relaxed);
queue_cv.notify_all();
break;
}
if (!emit_baked(baked)) {
bake_failed.store(true, std::memory_order_relaxed);
queue_cv.notify_all();
break;
}
}
scanner.join();
if (bake_failed.load(std::memory_order_relaxed) ||
!scan_ok.load(std::memory_order_relaxed) || !parsed_any) {
module.scene.shapes.clear();
return false;
}
return true;
}
} // namespace eu07::scene::bake::detail

2
eu07-parser/include/eu07/scene/context.hpp
View File

@@ -16,6 +16,8 @@ struct ParseContext {
std::vector<std::filesystem::path> includeStack;
std::vector<std::vector<std::string>> activeIncludeParameters;
bool expandIncludes = true;
// PACK compose needs includes + models only; skip heavy geometry nodes (CityGML teren).
bool packComposeLightweight = false;
};
using DirectiveParser = bool (*)(TokenStream& stream, ParseContext& context);

8
eu07-parser/include/eu07/scene/cursor.hpp
View File

@@ -23,8 +23,12 @@ public:
return (*tokens_)[index_];
}
[[nodiscard]] SourceToken consume() {
SourceToken token = peek();
// Returns a reference into the underlying token vector (which outlives the
// stream), so callers that only read a field (e.g. .sourceLine) or bind to
// const& pay no per-token std::string copy. Callers that need an owning
// copy still get one via copy-initialisation, exactly as before.
[[nodiscard]] const SourceToken& consume() {
const SourceToken& token = peek();
++index_;
return token;
}

60
eu07-parser/include/eu07/scene/document.hpp
View File

@@ -106,4 +106,64 @@ struct SceneProcessResult {
doc.test.size();
}
// Plaski plik flory: same wpisy node model, bez torow/siatek/include.
[[nodiscard]] inline bool isModelOnlyDocument(const SceneDocument& document) noexcept {
if (document.nodeModel.empty()) {
return false;
}
return document.nodeDynamic.empty() && document.nodeTrackNormal.empty() &&
document.nodeTrackSwitch.empty() && document.nodeTrackRoad.empty() &&
document.nodeTrackCross.empty() && document.nodeTrackOther.empty() &&
document.nodeTraction.empty() && document.nodeTractionPower.empty() &&
document.nodeTriangles.empty() && document.nodeTriangleStrip.empty() &&
document.nodeTriangleFan.empty() && document.nodeLines.empty() &&
document.nodeLineStrip.empty() && document.nodeLineLoop.empty() &&
document.nodeMemcell.empty() && document.nodeEventlauncher.empty() &&
document.nodeSound.empty() && document.include.empty() && document.unknown.empty();
}
// Drops parsed node payloads already converted to RuntimeModule / PackFileCacheEntry.
// Keeps include lists and lightweight directives. When release_models is false,
// nodeModel is retained for a pending PACK build (model-only flora child).
inline void releaseHeavySceneParseStorage(SceneDocument& document, bool const release_models) {
if (release_models) {
document.nodeModel.clear();
document.nodeModel.shrink_to_fit();
}
document.nodeTriangles.clear();
document.nodeTriangles.shrink_to_fit();
document.nodeTriangleStrip.clear();
document.nodeTriangleStrip.shrink_to_fit();
document.nodeTriangleFan.clear();
document.nodeTriangleFan.shrink_to_fit();
document.nodeLines.clear();
document.nodeLines.shrink_to_fit();
document.nodeLineStrip.clear();
document.nodeLineStrip.shrink_to_fit();
document.nodeLineLoop.clear();
document.nodeLineLoop.shrink_to_fit();
document.nodeTrackNormal.clear();
document.nodeTrackNormal.shrink_to_fit();
document.nodeTrackSwitch.clear();
document.nodeTrackSwitch.shrink_to_fit();
document.nodeTrackRoad.clear();
document.nodeTrackRoad.shrink_to_fit();
document.nodeTrackCross.clear();
document.nodeTrackCross.shrink_to_fit();
document.nodeTrackOther.clear();
document.nodeTrackOther.shrink_to_fit();
document.nodeTraction.clear();
document.nodeTraction.shrink_to_fit();
document.nodeTractionPower.clear();
document.nodeTractionPower.shrink_to_fit();
document.nodeMemcell.clear();
document.nodeMemcell.shrink_to_fit();
document.nodeEventlauncher.clear();
document.nodeEventlauncher.shrink_to_fit();
document.nodeSound.clear();
document.nodeSound.shrink_to_fit();
document.nodeDynamic.clear();
document.nodeDynamic.shrink_to_fit();
}
} // namespace eu07::scene

12
eu07-parser/include/eu07/scene/node.hpp
View File

@@ -51,6 +51,18 @@ inline void applyScratchContext(NodeHeader& header, const SceneScratchpad& scrat
return false;
}
if (context.packComposeLightweight) {
std::vector<SourceToken> raw;
while (!stream.empty() && !io::atEnd(stream, kind->subtype, kind->endMarker)) {
stream.consume();
}
if (!io::consumeEnd(stream, raw, kind->subtype, kind->endMarker)) {
stream.rewind(anchor);
return false;
}
return true;
}
if (!kind->parseAndStore(stream, context.document, header, nullptr)) {
stream.rewind(anchor);
return false;

9
eu07-parser/include/eu07/scene/node/mesh.hpp
View File

@@ -30,6 +30,15 @@ template <typename Parsed>
return false;
}
// Rezerwacja z gornego oszacowania (>=8 tokenow/wierzcholek), z capem.
{
std::size_t estimate = stream.remaining() / 8;
if (estimate > 8192) {
estimate = 8192;
}
out.vertices.reserve(estimate);
}
while (!stream.empty() && !atEnd(stream, subtype, endMarker)) {
MeshVertex vertex;
const bool hasMore = stream.remaining() > 1;

43
eu07-parser/include/eu07/scene/node/parse_util.hpp
View File

@@ -6,9 +6,11 @@
#include <eu07/scene/match.hpp>
#include <eu07/scene/node/types.hpp>
#include <charconv>
#include <optional>
#include <string>
#include <string_view>
#include <system_error>
#include <vector>
namespace eu07::scene::node::io {
@@ -63,16 +65,23 @@ inline void appendRaw(std::vector<SourceToken>& raw, const SourceToken& token) {
return 0.0;
}
try {
std::size_t used = 0;
const double value = std::stod(std::string(text), &used);
if (used != text.size()) {
return std::nullopt;
}
return value;
} catch (...) {
// std::from_chars: bez alokacji std::string, bez wyjatkow, niezalezne od locale
// (separator dziesietny zawsze '.', co odpowiada formatowi EU07).
std::string_view body = text;
// std::stod akceptowal wiodacy '+'; from_chars nie — zachowujemy zgodnosc.
if (!body.empty() && body.front() == '+') {
body.remove_prefix(1);
}
double value = 0.0;
const char* const first = body.data();
const char* const last = body.data() + body.size();
const std::from_chars_result result =
std::from_chars(first, last, value, std::chars_format::general);
if (result.ec != std::errc{} || result.ptr != last) {
return std::nullopt;
}
return value;
}
[[nodiscard]] inline bool takeToken(
@@ -115,6 +124,24 @@ inline void appendRaw(std::vector<SourceToken>& raw, const SourceToken& token) {
return true;
}
// Szybki odczyt liczby z goracych petli wierzcholkow: bez kopii SourceToken
// (uzywa peek zamiast consume) i bez dopisywania do bufora raw. Bezpieczne tam,
// gdzie raw wierzcholkow nie jest pozniej czytany (siatki: triangles/strip/fan).
// Przy niepowodzeniu NIE konsumuje tokenu — wolajacy i tak cofa strumien do
// kotwicy wezla, wiec pozycja w miejscu bledu nie ma znaczenia.
[[nodiscard]] inline bool takeDoubleFast(TokenStream& stream, double& out) {
if (stream.empty()) {
return false;
}
const std::optional<double> value = parseDouble(stream.peek().value);
if (!value) {
return false;
}
stream.skip();
out = *value;
return true;
}
[[nodiscard]] inline bool takeDoubleOrPlaceholder(
TokenStream& stream,
std::vector<SourceToken>& raw,

20
eu07-parser/include/eu07/scene/node/triangles.hpp
View File

@@ -73,10 +73,12 @@ inline constexpr std::string_view kEndMarker = "endtriangles";
std::vector<SourceToken>& raw,
MeshVertex& vertex,
const bool allowEndSuffix) {
if (!node::io::takeDouble(stream, raw, vertex.x) || !node::io::takeDouble(stream, raw, vertex.y) ||
!node::io::takeDouble(stream, raw, vertex.z) || !node::io::takeDouble(stream, raw, vertex.nx) ||
!node::io::takeDouble(stream, raw, vertex.ny) || !node::io::takeDouble(stream, raw, vertex.nz) ||
!node::io::takeDouble(stream, raw, vertex.u) || !node::io::takeDouble(stream, raw, vertex.v)) {
// Goraca petla: 8 liczb na wierzcholek czytanych bez kopii tokenu i bez raw
// (raw wierzcholkow siatek nie jest pozniej czytany przy bake).
if (!node::io::takeDoubleFast(stream, vertex.x) || !node::io::takeDoubleFast(stream, vertex.y) ||
!node::io::takeDoubleFast(stream, vertex.z) || !node::io::takeDoubleFast(stream, vertex.nx) ||
!node::io::takeDoubleFast(stream, vertex.ny) || !node::io::takeDoubleFast(stream, vertex.nz) ||
!node::io::takeDoubleFast(stream, vertex.u) || !node::io::takeDoubleFast(stream, vertex.v)) {
return false;
}
if (allowEndSuffix && !stream.empty() && isKeyword(stream.peek().value, "end")) {
@@ -105,6 +107,16 @@ inline constexpr std::string_view kEndMarker = "endtriangles";
return false;
}
// Rezerwacja z gornego oszacowania (>=8 tokenow/wierzcholek), z capem aby
// nie przealokowac przy wczesnym wezle nad strumieniem calego pliku.
{
std::size_t estimate = stream.remaining() / 8;
if (estimate > 8192) {
estimate = 8192;
}
out.vertices.reserve(estimate);
}
while (!stream.empty() && !node::io::atEnd(stream, kSubtype, kEndMarker)) {
MeshVertex vertex;
const bool hasMore = stream.remaining() > 1;

766
eu07-parser/include/eu07/scene/parallel_models.hpp Normal file
View File

@@ -0,0 +1,766 @@
#pragma once
// Fast path: plaskie pliki .scm — kazdy wpis w jednej linii logicznej.
// (1) same "node model" (flora) albo (2) same "include;...;end" (placementy).
// Pomija tokenizacje calego pliku; parse linii rownolegle (per-line tokenize).
#include <eu07/nmt/parallel.hpp>
#include <eu07/parser.hpp>
#include <eu07/scene/context.hpp>
#include <eu07/scene/cursor.hpp>
#include <eu07/scene/dispatch_table.hpp>
#include <eu07/scene/document.hpp>
#include <eu07/scene/match.hpp>
#include <eu07/scene/node.hpp>
#include <eu07/scene/node/model.hpp>
#include <atomic>
#include <array>
#include <cstddef>
#include <filesystem>
#include <fstream>
#include <functional>
#include <optional>
#include <stdexcept>
#include <string>
#include <string_view>
#include <thread>
#include <utility>
#include <vector>
namespace eu07::scene::detail {
enum class FlatFileKind {
None,
Models,
Includes,
};
inline constexpr std::size_t kParallelFlatLineThreshold = 2048;
[[nodiscard]] inline bool segmentStartsWithKeyword(
std::string_view text,
const std::string_view keyword) {
skipFieldSeparators(text);
if (text.size() < keyword.size()) {
return false;
}
if (text.compare(0, keyword.size(), keyword) != 0) {
return false;
}
if (text.size() == keyword.size()) {
return true;
}
return isFieldSeparator(static_cast<unsigned char>(text[keyword.size()]));
}
[[nodiscard]] inline FlatFileKind classifyFlatFile(const std::vector<LogicalSegment>& segments) {
FlatFileKind kind = FlatFileKind::None;
for (const LogicalSegment& segment : segments) {
std::string_view text = segment.text;
skipFieldSeparators(text);
if (text.empty()) {
continue;
}
if (segmentStartsWithKeyword(text, "node")) {
if (text.find("endmodel") == std::string_view::npos) {
return FlatFileKind::None;
}
if (kind == FlatFileKind::Includes) {
return FlatFileKind::None;
}
kind = FlatFileKind::Models;
continue;
}
if (segmentStartsWithKeyword(text, "include")) {
if (text.find("end") == std::string_view::npos) {
return FlatFileKind::None;
}
if (kind == FlatFileKind::Models) {
return FlatFileKind::None;
}
kind = FlatFileKind::Includes;
continue;
}
return FlatFileKind::None;
}
return kind;
}
struct ModelParseJob {
std::size_t segment_index = 0;
NodeHeader header;
};
[[nodiscard]] inline bool skipModelBody(TokenStream& stream) {
std::vector<SourceToken> raw;
while (!stream.empty() && !node::io::atEnd(stream, node_model::kSubtype, node_model::kEndMarker)) {
stream.consume();
}
return node::io::consumeEnd(stream, raw, node_model::kSubtype, node_model::kEndMarker);
}
[[nodiscard]] inline bool parseModelSegment(
const LogicalSegment& segment,
ParsedNodeModel& out) {
const std::vector<SourceToken> tokens = tokenizeSegments({segment});
TokenStream stream(tokens);
NodeHeader header;
std::string subtype;
std::vector<SourceToken> raw;
if (!node::io::consumeHeader(stream, header, subtype, raw) ||
!isKeyword(subtype, node_model::kSubtype)) {
return false;
}
return node_model::parseBody(stream, raw, header, out);
}
[[nodiscard]] inline std::string_view trimFieldView(std::string_view text) {
skipFieldSeparators(text);
while (!text.empty() && isFieldSeparator(static_cast<unsigned char>(text.back()))) {
text.remove_suffix(1);
}
return text;
}
[[nodiscard]] inline std::vector<std::string_view> splitSemicolonFields(std::string_view text) {
std::vector<std::string_view> fields;
while (!text.empty()) {
skipFieldSeparators(text);
if (text.empty()) {
break;
}
const std::size_t separator = text.find(';');
if (separator == std::string_view::npos) {
fields.push_back(trimFieldView(text));
break;
}
fields.push_back(trimFieldView(text.substr(0, separator)));
text.remove_prefix(separator + 1);
}
return fields;
}
[[nodiscard]] inline bool parseIncludeSegmentSemicolon(
const LogicalSegment& segment,
ParsedInclude& out) {
std::string_view text = segment.text;
if (const std::size_t comment = text.find("//"); comment != std::string_view::npos) {
text = text.substr(0, comment);
}
skipFieldSeparators(text);
if (!segmentStartsWithKeyword(text, "include")) {
return false;
}
const std::vector<std::string_view> fields = splitSemicolonFields(text);
if (fields.empty() || !isKeyword(fields.front(), "include")) {
return false;
}
out = ParsedInclude {};
out.line = segment.sourceLine;
if (fields.size() < 2) {
out.error = "brak sciezki pliku";
return true;
}
out.file = std::string(fields[1]);
for (std::size_t index = 2; index < fields.size(); ++index) {
if (isKeyword(fields[index], "end")) {
break;
}
out.parameters.emplace_back(fields[index]);
}
return true;
}
[[nodiscard]] inline bool parseIncludeSegment(
const LogicalSegment& segment,
ParsedInclude& out) {
if (parseIncludeSegmentSemicolon(segment, out)) {
return true;
}
const std::vector<SourceToken> tokens = tokenizeSegments({segment});
TokenStream stream(tokens);
if (stream.empty() || !isKeyword(stream.peek().value, "include")) {
return false;
}
out = ParsedInclude {};
out.line = segment.sourceLine;
stream.consume();
if (stream.empty()) {
out.error = "brak sciezki pliku";
return true;
}
SourceToken file_token;
if (!node::io::takeToken(stream, out.raw, file_token)) {
out.error = "brak sciezki pliku";
return true;
}
out.file = file_token.value;
while (!stream.empty()) {
if (isKeyword(stream.peek().value, "end")) {
stream.consume();
break;
}
SourceToken param;
if (!node::io::takeToken(stream, out.raw, param)) {
break;
}
out.parameters.push_back(param.value);
}
out.raw.clear();
return true;
}
[[nodiscard]] inline std::optional<SceneDocument> parseFlatModels(
const std::vector<LogicalSegment>& segments) {
SceneDocument document;
if (segments.empty()) {
return document;
}
if (segments.size() < kParallelFlatLineThreshold) {
document.nodeModel.reserve(segments.size());
for (const LogicalSegment& segment : segments) {
ParsedNodeModel model;
if (!parseModelSegment(segment, model)) {
return std::nullopt;
}
document.nodeModel.push_back(std::move(model));
}
return document;
}
document.nodeModel.resize(segments.size());
const unsigned worker_count =
std::min(static_cast<unsigned>(segments.size()), eu07::nmt::workerThreadCount());
std::atomic<std::size_t> next_job { 0 };
std::atomic<bool> failed { false };
auto worker = [&]() {
while (true) {
if (failed.load(std::memory_order_relaxed)) {
return;
}
const std::size_t index = next_job.fetch_add(1, std::memory_order_relaxed);
if (index >= segments.size()) {
return;
}
ParsedNodeModel model;
if (!parseModelSegment(segments[index], model)) {
failed.store(true, std::memory_order_relaxed);
return;
}
document.nodeModel[index] = std::move(model);
}
};
std::vector<std::thread> threads;
const unsigned launch = std::max(1u, worker_count);
threads.reserve(launch);
for (unsigned i = 0; i < launch; ++i) {
threads.emplace_back(worker);
}
for (std::thread& thread : threads) {
thread.join();
}
if (failed.load(std::memory_order_relaxed)) {
return std::nullopt;
}
return document;
}
[[nodiscard]] inline std::optional<SceneDocument> parseFlatIncludes(
const std::vector<LogicalSegment>& segments) {
SceneDocument document;
document.include.resize(segments.size());
if (segments.size() < kParallelFlatLineThreshold) {
for (std::size_t index = 0; index < segments.size(); ++index) {
if (!parseIncludeSegment(segments[index], document.include[index])) {
return std::nullopt;
}
}
return document;
}
std::atomic<std::size_t> next_job { 0 };
std::atomic<bool> failed { false };
auto worker = [&]() {
while (true) {
if (failed.load(std::memory_order_relaxed)) {
return;
}
const std::size_t index = next_job.fetch_add(1, std::memory_order_relaxed);
if (index >= segments.size()) {
return;
}
ParsedInclude entry;
if (!parseIncludeSegment(segments[index], entry)) {
failed.store(true, std::memory_order_relaxed);
return;
}
document.include[index] = std::move(entry);
}
};
const unsigned worker_count =
std::min(static_cast<unsigned>(segments.size()), eu07::nmt::workerThreadCount());
std::vector<std::thread> threads;
const unsigned launch = std::max(1u, worker_count);
threads.reserve(launch);
for (unsigned i = 0; i < launch; ++i) {
threads.emplace_back(worker);
}
for (std::thread& thread : threads) {
thread.join();
}
if (failed.load(std::memory_order_relaxed)) {
return std::nullopt;
}
return document;
}
[[nodiscard]] inline std::optional<SceneDocument> tryProcessFlatLogical(
const LogicalPass& logical,
const std::filesystem::path& /*baseDirectory*/) {
const FlatFileKind kind = classifyFlatFile(logical.segments);
if (kind == FlatFileKind::None) {
return std::nullopt;
}
if (kind == FlatFileKind::Models) {
return parseFlatModels(logical.segments);
}
return parseFlatIncludes(logical.segments);
}
inline constexpr std::size_t kParallelModelParseThreshold = 4096;
[[nodiscard]] inline std::optional<SceneDocument> tryProcessFlatModels(
const ParseResult& parsed,
const std::filesystem::path& baseDirectory) {
if (parsed.tokens.empty()) {
return SceneDocument {};
}
SceneDocument document;
std::filesystem::path includeRoot = baseDirectory;
if (includeRoot.empty()) {
includeRoot = std::filesystem::current_path();
}
ParseContext context { document, {}, includeRoot, {} };
context.expandIncludes = false;
TokenStream stream(parsed.tokens);
std::vector<ModelParseJob> jobs;
jobs.reserve(parsed.tokens.size() / 16);
while (!stream.empty()) {
if (detail::dispatchDirective(stream, context)) {
continue;
}
if (!stream.empty() && isKeyword(stream.peek().value, "node")) {
const std::size_t anchor = stream.checkpoint();
NodeHeader header;
std::string subtype;
std::vector<SourceToken> raw;
if (!node::io::consumeHeader(stream, header, subtype, raw)) {
return std::nullopt;
}
if (!isKeyword(subtype, node_model::kSubtype)) {
return std::nullopt;
}
node::applyScratchContext(header, context.scratch);
jobs.push_back(ModelParseJob { anchor, header });
if (!skipModelBody(stream)) {
return std::nullopt;
}
continue;
}
UnknownEntry unknown;
unknown.line = stream.peek().sourceLine;
unknown.token = stream.consume().value;
document.unknown.push_back(std::move(unknown));
}
if (jobs.size() < kParallelModelParseThreshold) {
document.nodeModel.reserve(jobs.size());
for (const ModelParseJob& job : jobs) {
ParsedNodeModel model;
TokenStream job_stream(parsed.tokens);
job_stream.rewind(job.segment_index);
NodeHeader parsed_header;
std::string subtype;
std::vector<SourceToken> raw;
if (!node::io::consumeHeader(job_stream, parsed_header, subtype, raw) ||
!isKeyword(subtype, node_model::kSubtype)) {
return std::nullopt;
}
model.header = job.header;
if (!node_model::parseBody(job_stream, raw, job.header, model)) {
return std::nullopt;
}
document.nodeModel.push_back(std::move(model));
}
return document;
}
document.nodeModel.resize(jobs.size());
const unsigned worker_count =
std::min(static_cast<unsigned>(jobs.size()), eu07::nmt::workerThreadCount());
std::atomic<std::size_t> next_job { 0 };
std::atomic<bool> failed { false };
auto worker = [&]() {
while (true) {
if (failed.load(std::memory_order_relaxed)) {
return;
}
const std::size_t index = next_job.fetch_add(1, std::memory_order_relaxed);
if (index >= jobs.size()) {
return;
}
ParsedNodeModel model;
TokenStream job_stream(parsed.tokens);
job_stream.rewind(jobs[index].segment_index);
NodeHeader parsed_header;
std::string subtype;
std::vector<SourceToken> raw;
if (!node::io::consumeHeader(job_stream, parsed_header, subtype, raw) ||
!isKeyword(subtype, node_model::kSubtype)) {
failed.store(true, std::memory_order_relaxed);
return;
}
model.header = jobs[index].header;
if (!node_model::parseBody(job_stream, raw, jobs[index].header, model)) {
failed.store(true, std::memory_order_relaxed);
return;
}
document.nodeModel[index] = std::move(model);
}
};
std::vector<std::thread> threads;
const unsigned launch = std::max(1u, worker_count);
threads.reserve(launch);
for (unsigned i = 0; i < launch; ++i) {
threads.emplace_back(worker);
}
for (std::thread& thread : threads) {
thread.join();
}
if (failed.load(std::memory_order_relaxed)) {
return std::nullopt;
}
return document;
}
[[nodiscard]] inline std::optional<SceneDocument> tryProcessFlatSceneFile(
eu07::RawFile raw,
const std::filesystem::path& baseDirectory) {
const LogicalPass logical = toLogicalLines(raw.lines);
return tryProcessFlatLogical(logical, baseDirectory);
}
inline constexpr std::size_t kStreamFlatFileThresholdBytes = 4u * 1024u * 1024u;
[[nodiscard]] inline std::size_t streamSourceFileSizeBytes(const std::filesystem::path& path) {
std::error_code ec;
const auto size = std::filesystem::file_size(path, ec);
return ec ? 0u : static_cast<std::size_t>(size);
}
[[nodiscard]] inline bool shouldStreamFlatSourceFile(
const std::filesystem::path& path,
const std::size_t threshold_bytes = kStreamFlatFileThresholdBytes) {
if (threshold_bytes == 0) {
return false;
}
return streamSourceFileSizeBytes(path) > threshold_bytes;
}
// Linia po linii, z ta sama logika komentarzy co toLogicalLines — bez readRawFile.
template <typename Fn>
void forEachStreamingLogicalSegment(const std::filesystem::path& path, Fn&& on_segment) {
std::ifstream in(path, std::ios::binary);
if (!in) {
throw std::runtime_error("Nie mozna otworzyc: " + path.string());
}
std::vector<char> read_buffer(1024u * 1024u);
in.rdbuf()->pubsetbuf(read_buffer.data(), read_buffer.size());
bool in_block = false;
std::string line;
line.reserve(512);
std::size_t source_line = 0;
std::string spill;
const auto emit = [&](std::string_view piece, const std::size_t line_no) -> bool {
if (piece.empty()) {
return true;
}
skipFieldSeparators(piece);
if (eu07::detail::isStarter(piece)) {
return true;
}
if (eu07::detail::isLineComment(piece)) {
return true;
}
const std::string_view code = eu07::detail::stripInlineComment(piece);
if (code.empty()) {
return true;
}
LogicalSegment segment;
segment.sourceLine = line_no;
if (code.data() >= piece.data() && code.data() + code.size() <= piece.data() + piece.size()) {
segment.text = code;
} else {
spill.assign(code.begin(), code.end());
segment.text = spill;
}
return on_segment(segment);
};
while (std::getline(in, line)) {
++source_line;
if (!line.empty() && line.back() == '\r') {
line.pop_back();
}
{
std::string_view whole = line;
skipFieldSeparators(whole);
if (parseStarter(whole, source_line - 1)) {
continue;
}
}
if (!in_block) {
std::size_t at = 0;
std::string_view line_view = line;
while (at < line_view.size()) {
const std::size_t slash = line_view.find('/', at);
if (slash == std::string_view::npos) {
if (!emit(line_view.substr(at), source_line)) {
return;
}
break;
}
if (slash + 1 < line_view.size() && line_view[slash + 1] == '/') {
if (slash > at) {
if (!emit(line_view.substr(at, slash - at), source_line)) {
return;
}
}
break;
}
if (slash + 1 < line_view.size() && line_view[slash + 1] == '*') {
if (slash > at) {
if (!emit(line_view.substr(at, slash - at), source_line)) {
return;
}
}
in_block = true;
line_view = line_view.substr(slash + 2);
at = 0;
continue;
}
if (!emit(line_view.substr(at), source_line)) {
return;
}
break;
}
continue;
}
while (in_block && !line.empty()) {
const std::size_t close = line.find("*/");
if (close == std::string_view::npos) {
line.clear();
break;
}
line.erase(0, close + 2);
in_block = false;
}
if (!in_block && !line.empty()) {
if (!emit(line, source_line)) {
return;
}
}
}
}
[[nodiscard]] inline FlatFileKind scanFlatFileKindStreaming(const std::filesystem::path& path) {
FlatFileKind kind = FlatFileKind::None;
bool ok = true;
forEachStreamingLogicalSegment(path, [&](const LogicalSegment& segment) {
std::string_view text = segment.text;
skipFieldSeparators(text);
if (text.empty()) {
return true;
}
if (segmentStartsWithKeyword(text, "node")) {
if (text.find("endmodel") == std::string_view::npos) {
ok = false;
return false;
}
if (kind == FlatFileKind::Includes) {
ok = false;
return false;
}
kind = FlatFileKind::Models;
return true;
}
if (segmentStartsWithKeyword(text, "include")) {
if (text.find("end") == std::string_view::npos) {
ok = false;
return false;
}
if (kind == FlatFileKind::Models) {
ok = false;
return false;
}
kind = FlatFileKind::Includes;
return true;
}
ok = false;
return false;
});
return ok ? kind : FlatFileKind::None;
}
[[nodiscard]] inline std::optional<SceneDocument> buildFlatIncludesDocumentStreaming(
const std::filesystem::path& path) {
SceneDocument document;
struct OwnedIncludeLine {
std::size_t source_line = 0;
std::string text;
};
std::vector<OwnedIncludeLine> batch;
batch.reserve(8192);
bool ok = true;
const auto flush_batch = [&]() -> bool {
if (batch.empty()) {
return true;
}
const std::size_t base = document.include.size();
document.include.resize(base + batch.size());
const auto parse_one = [&](const std::size_t index) -> bool {
LogicalSegment segment;
segment.sourceLine = batch[index].source_line;
segment.text = batch[index].text;
ParsedInclude entry;
if (!parseIncludeSegment(segment, entry)) {
return false;
}
document.include[base + index] = std::move(entry);
return true;
};
if (batch.size() >= kParallelFlatLineThreshold) {
std::atomic<std::size_t> next_job { 0 };
std::atomic<bool> failed { false };
const unsigned worker_count = std::min(
static_cast<unsigned>(batch.size()), eu07::nmt::workerThreadCount());
const auto worker = [&]() {
while (!failed.load(std::memory_order_relaxed)) {
const std::size_t index =
next_job.fetch_add(1, std::memory_order_relaxed);
if (index >= batch.size()) {
return;
}
if (!parse_one(index)) {
failed.store(true, std::memory_order_relaxed);
return;
}
}
};
std::vector<std::thread> threads;
threads.reserve(worker_count);
for (unsigned i = 0; i < worker_count; ++i) {
threads.emplace_back(worker);
}
for (std::thread& thread : threads) {
thread.join();
}
if (failed.load(std::memory_order_relaxed)) {
return false;
}
} else {
for (std::size_t index = 0; index < batch.size(); ++index) {
if (!parse_one(index)) {
return false;
}
}
}
batch.clear();
return true;
};
forEachStreamingLogicalSegment(path, [&](const LogicalSegment& segment) {
std::string_view text = segment.text;
skipFieldSeparators(text);
if (text.empty() || !segmentStartsWithKeyword(text, "include")) {
return true;
}
batch.push_back(OwnedIncludeLine { segment.sourceLine, std::string(segment.text) });
if (batch.size() >= 8192) {
if (!flush_batch()) {
ok = false;
return false;
}
}
return true;
});
if (ok) {
ok = flush_batch();
}
return ok ? std::optional<SceneDocument>(std::move(document)) : std::nullopt;
}
template <typename Fn>
void streamFlatFileModels(const std::filesystem::path& path, Fn&& on_model) {
forEachStreamingLogicalSegment(path, [&](const LogicalSegment& segment) {
std::string_view text = segment.text;
skipFieldSeparators(text);
if (text.empty() || !segmentStartsWithKeyword(text, "node")) {
return true;
}
if (text.find("endmodel") == std::string_view::npos) {
throw std::runtime_error(
"Plaski plik modeli: oczekiwano endmodel w linii " +
std::to_string(segment.sourceLine));
}
ParsedNodeModel parsed;
if (!parseModelSegment(segment, parsed)) {
throw std::runtime_error(
"Plaski plik modeli: blad parse linii " + std::to_string(segment.sourceLine));
}
on_model(parsed);
return true;
});
}
} // namespace eu07::scene::detail

13
eu07-parser/include/eu07/scene/processor.hpp
View File

@@ -18,7 +18,7 @@
#include <eu07/scene/include_resolve.hpp>
#include <eu07/scene/parallel_models.hpp>
#include <filesystem>
@@ -194,6 +194,7 @@ inline void expandInclude(ParseContext& context, const IncludeExpansionRequest&
struct SceneProcessOptions {
bool expandIncludes = true;
bool packComposeLightweight = false;
};
[[nodiscard]] inline SceneProcessResult processScene(
@@ -204,6 +205,15 @@ struct SceneProcessOptions {
const SceneProcessOptions& options = {}) {
if (!options.expandIncludes) {
if (std::optional<SceneDocument> fast =
detail::tryProcessFlatModels(parsed, baseDirectory)) {
SceneProcessResult result;
result.document = std::move(*fast);
return result;
}
}
TokenStream stream(parsed.tokens);
SceneDocument document;
@@ -223,6 +233,7 @@ struct SceneProcessOptions {
ParseContext context{document, {}, includeRoot, {}};
context.expandIncludes = options.expandIncludes;
context.packComposeLightweight = options.packComposeLightweight;
detail::processStream(stream, context);