diff --git a/CMakeLists.txt b/CMakeLists.txt index 865a4f84..e1140bc5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -161,6 +161,8 @@ set(SOURCES "scene/eu7/eu7_emit.cpp" "scene/eu7/eu7_apply.cpp" "scene/eu7/eu7_bake.cpp" +"scene/eu7/v2/eu7v2_bake.cpp" +"scene/eu7/v2/eu7v2_load.cpp" "scene/eu7/eu7_parameters.cpp" "scene/scenenode.cpp" "simulation/simulation.cpp" @@ -415,11 +417,14 @@ if(WITH_EU7_PARSER) message(FATAL_ERROR "EU07 parser headers missing at ${EU07_PARSER_INCLUDE}. Sync eu07-parser/include from the parser repo.") endif() - add_library(eu07_bake STATIC "scene/eu7/eu7_bake_parser.cpp") + add_library(eu07_bake STATIC "scene/eu7/eu7_bake_parser.cpp" "scene/eu7/v2/eu7v2_emit_runtime.cpp") target_compile_features(eu07_bake PUBLIC cxx_std_23) set_target_properties(eu07_bake PROPERTIES CXX_STANDARD 23 CXX_STANDARD_REQUIRED ON) - target_include_directories(eu07_bake PRIVATE "${EU07_PARSER_INCLUDE}") + target_include_directories(eu07_bake PRIVATE "${EU07_PARSER_INCLUDE}" "${CMAKE_CURRENT_SOURCE_DIR}") target_link_libraries(eu07_bake PUBLIC Threads::Threads) + if(WIN32) + target_link_libraries(eu07_bake PUBLIC psapi) + endif() if(MSVC) target_compile_options(eu07_bake PRIVATE /utf-8 /Zc:__cplusplus /EHsc /wd4834) endif() diff --git a/EU07.cpp b/EU07.cpp index e1be8584..37509b18 100644 --- a/EU07.cpp +++ b/EU07.cpp @@ -21,6 +21,7 @@ Stele, firleju, szociu, hunter, ZiomalCl, OLI_EU and others #include "application/application.h" #include "utilities/Logs.h" #include +#include #ifdef WITHDUMPGEN #ifdef _WIN32 #include @@ -89,6 +90,14 @@ int main(int argc, char *argv[]) // init start timestamp Global.startTimestamp = std::chrono::steady_clock::now(); + // headless short-circuit: text -> eu7v2 bake (+optional verify), no window + if (eu07_application::wants_eu7v2_headless(argc, argv)) + { + int const code{eu07_application::run_eu7v2_headless(argc, argv)}; + std::cout.flush(); + std::_Exit(code); + } + // quick short-circuit for standalone e3d export if (argc == 6 && std::string(argv[1]) == "-e3d") { diff --git a/application/application.cpp b/application/application.cpp index ba09bde9..093b7f8e 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -30,7 +30,10 @@ http://mozilla.org/MPL/2.0/. #include "utilities/dictionary.h" #include "version_info.h" #include "ref/discord-rpc/include/discord_rpc.h" +#include "scene/eu7/eu7_bake_parser.h" +#include "scene/eu7/eu7_bake_mem_guard.h" #include +#include #include "utilities/translation.h" #if WITH_DISCORD_RPC @@ -1095,6 +1098,105 @@ void eu07_application::init_files() } namespace fs = std::filesystem; +bool eu07_application::wants_eu7v2_headless(int Argc, char *Argv[]) +{ + for (int i = 1; i < Argc; ++i) + { + if (std::string(Argv[i]) == "--eu7v2-bake") + return true; + } + return false; +} + +int eu07_application::run_eu7v2_headless(int Argc, char *Argv[]) +{ + std::string bake_path; + bool verify{false}; + unsigned mem_limit_gb{scene::eu7::bake_parser::kDefaultBakeMemLimitGb}; + unsigned max_threads{0u}; + unsigned max_parse{0u}; + unsigned heavy_parse_mb{0u}; + + for (int i = 1; i < Argc; ++i) + { + std::string const token{Argv[i]}; + if (token == "--eu7v2-bake") + { + if (i + 1 < Argc) + bake_path = Argv[++i]; + } + else if (token == "--eu7v2-verify") + { + verify = true; + } + else if (token == "--eu7v2-mem-limit-gb") + { + if (i + 1 < Argc) + mem_limit_gb = static_cast(std::stoul(Argv[++i])); + } + else if (token == "--eu7v2-max-parse") + { + if (i + 1 < Argc) + max_parse = static_cast(std::stoul(Argv[++i])); + } + else if (token == "--eu7v2-threads") + { + if (i + 1 < Argc) + max_threads = static_cast(std::stoul(Argv[++i])); + } + else if (token == "--eu7v2-heavy-parse-mb") + { + if (i + 1 < Argc) + heavy_parse_mb = static_cast(std::stoul(Argv[++i])); + } + } + + auto report_line = [](std::string const &line) { + std::cout << line << std::endl; + std::ofstream log{"log.txt", std::ios::app}; + if (log) + log << line << '\n'; + }; + + if (bake_path.empty()) + { + report_line("[eu7v2] BLAD: --eu7v2-bake wymaga sciezki do pliku .scn/.inc"); + return 2; + } + + report_line("[eu7v2] headless bake: " + bake_path + (verify ? " (+verify)" : "")); + + scene::eu7::bake_parser::Eu7v2BakeReport const report{ + scene::eu7::bake_parser::bake_scenario_tree_eu7v2( + bake_path, max_threads, verify, mem_limit_gb, max_parse, heavy_parse_mb)}; + + if (false == report.baked) + { + report_line("[eu7v2] BAKE FAIL: " + report.error); + return 1; + } + + report_line( + "[eu7v2] modulow=" + std::to_string(report.module_count) + + " modeli=" + std::to_string(report.model_count) + + " root=" + report.root_binary_path); + + if (false == report.error.empty()) + { + report_line("[eu7v2] OSTRZEZENIE: " + report.error); + } + + if (verify) + { + report_line(report.verify_ok ? "[eu7v2] VERIFY: PASS" : "[eu7v2] VERIFY: FAIL"); + if (false == report.verify_ok) + return 1; + } + + report_line("[eu7v2] OK"); + return (report.error.empty()) ? 0 : 1; +} + int eu07_application::init_settings(int Argc, char *Argv[]) { Global.asVersion = VERSION_INFO; diff --git a/application/application.h b/application/application.h index 11415603..99100826 100644 --- a/application/application.h +++ b/application/application.h @@ -36,6 +36,12 @@ public: init( int Argc, char *Argv[] ); int run(); + // Headless text -> eu7v2 bake (+optional roundtrip verify). Runs without a + // window / renderer / audio. Returns a process exit code (0 == OK). + static bool + wants_eu7v2_headless( int Argc, char *Argv[] ); + static int + run_eu7v2_headless( int Argc, char *Argv[] ); // issues request for a worker thread to perform specified task. returns: true if task was scheduled #ifdef WITH_DISCORD_RPC diff --git a/docs/eu7v2-format.md b/docs/eu7v2-format.md new file mode 100644 index 00000000..497d6fdd --- /dev/null +++ b/docs/eu7v2-format.md @@ -0,0 +1,381 @@ +# Format pliku EU7v2 (`.eu7v2`) + +> Dokumentacja oparta na kodzie źródłowym z: +> - `scene/eu7/v2/` — format, emitter, loader, testy roundtrip +> - `scene/eu7/eu7_bake_parser.cpp` — headless bake scenariusza +> - `eu07-parser/include/eu07/scene/bake/` — pool bake, streaming terenu +> +> Stan: **czerwiec 2026**, wersja kontenera **v1** (`kVersion = 1`). + +--- + +## Spis treści + +1. [Przegląd ogólny](#1-przegląd-ogólny) +2. [Nagłówek pliku](#2-nagłówek-pliku) +3. [Struktura chunku](#3-struktura-chunku) +4. [Rodzaje pliku (`file_kind`)](#4-rodzaje-pliku-file_kind) +5. [Katalog chunków](#5-katalog-chunków) +6. [Typy podstawowe](#6-typy-podstawowe) +7. [STRS — tablica łańcuchów](#7-strs--tablica-łańcuchów) +8. [META — metadane modułu](#8-meta--metadane-modułu) +9. [INCL — include rekurencyjne](#9-incl--include-rekurencyjne) +10. [PLCE — placementy modułów wielokrotnego użytku](#10-plce--placementy-modułów-wielokrotnego-użytku) +11. [PROT + INST — prototypy i instancje modeli](#11-prot--inst--prototypy-i-instancje-modeli) +12. [SHPE — kształty (triangles)](#12-shpe--kształty-triangles) +13. [MESH — teren (tile)](#13-mesh--teren-tile) +14. [LINE — linie](#14-line--linie) +15. [Rekordy symulacji](#15-rekordy-symulacji) +16. [Bake headless i drzewo modułów](#16-bake-headless-i-drzewo-modułów) +17. [Ścieżki plików](#17-ścieżki-plików) + +--- + +## 1. Przegląd ogólny + +Plik `.eu7v2` to binarny format skompilowanego modułu scenerii: + +- **Moduły wielokrotnego użytku** — osobny plik `.eu7v2` per `.inc` / `.scm`, placementy jako lean **PLCE** +- **Prototypy + instancje** — chunk **PROT** (deduplikacja meshów) + **INST** (transformacja w world-space) +- **Chunki pomijalne** — nieznany FourCC można pominąć po polu `size` +- **Little-endian** — deterministyczny zapis niezależny od endianness hosta +- **Centralna tablica STRS** — wszystkie stringi przez `string_id` (`uint32`) + +Magic pliku: **`EU7C`** (FourCC `'E','U','7','C'`). + +--- + +## 2. Nagłówek pliku + +``` +Offset Rozmiar Typ Opis +------ ------- -------- ------------------------- +0 4 uint32 Magic = 'EU7C' +4 2 uint16 Version (= 1) +6 2 uint16 file_kind (patrz sekcja 4) +8 4 uint32 Reserved (= 0) +12 4 uint32 Reserved (= 0) +16 ... chunks Strumień chunkowy (do EOF) +``` + +Nagłówek ma **16 bajtów**. Brak pola „rozmiar całego pliku”. + +--- + +## 3. Struktura chunku + +``` +Offset Rozmiar Typ Opis +------ ------- ------- ----------------------------------------- +0 4 uint32 Chunk ID — FourCC little-endian +4 8 uint64 Payload size (tylko payload, BEZ nagłówka chunku) +12 ... bytes Payload +``` + +Pole `size` to rozmiar **samego payloadu** (`uint64`), bez nagłówka chunku. + +Chunki są sekwencyjne do EOF. Nieznany chunk: `seek(payload_size)`. + +FourCC — bajty w pliku w kolejności ASCII (np. `STRS` → `53 54 52 53`). + +--- + +## 4. Rodzaje pliku (`file_kind`) + +| Wartość | Nazwa | Typowy źródłowy plik | Opis | +|---------|------------|-----------------------------|-------------------------------------------| +| `1` | `sim` | `.scn`, `.scm`, `.sbt`, `.ctr` | Moduł scenariusza / root | +| `2` | `module` | `.inc` | Moduł wielokrotnego użytku (trawa, drzewo…) | +| `3` | `tile` | (planowane) | Kafel 1 km — teren + instancje | +| `4` | `manifest` | (planowane) | Indeks kafli / modułów mapy | + +Emitter runtime (`eu7v2_emit_runtime.cpp`) ustawia: +- `.inc` → `file_kind::module` +- pozostałe rozszerzenia modułów → `file_kind::sim` + +Loader akceptuje `sim` i `module` (`eu7v2_load.cpp`). + +--- + +## 5. Katalog chunków + +| FourCC | Stała C++ | Typ pliku | Opis | +|---------|---------------|-------------|---------------------------------------------------| +| `STRS` | `chunk::strs` | wszystkie | Tablica stringów (zawsze pierwszy logicznie) | +| `META` | `chunk::meta` | sim/module | Metadane modułu (first-init, placement params) | +| `INCL` | `chunk::incl` | sim/module | Include rekurencyjne (submoduły `.eu7v2`) | +| `PLCE` | `chunk::plce` | sim/module | Lean placementy → osobne moduły `.eu7v2` | +| `PROT` | `chunk::prot` | sim/module/tile | Prototypy modeli (mesh bez transformacji) | +| `INST` | `chunk::inst` | sim/module/tile | Instancje modeli (proto + transform) | +| `MESH` | `chunk::mesh` | tile/sim | Teren — mesh z origin f64 + wierzchołki f32 | +| `SHPE` | `chunk::shpe` | sim/module | Kształty triangles/strip/fan (nie-teren) | +| `LINE` | `chunk::line` | sim/module | Geometria linii | +| `TRAK` | `chunk::trak` | sim | Tory | +| `TRAC` | `chunk::trac` | sim | Przewody trakcyjne | +| `PWRS` | `chunk::pwrs` | sim | Źródła zasilania | +| `MEMC` | `chunk::memc` | sim | Komórki pamięci | +| `LAUN` | `chunk::laun` | sim | Wyzwalacze zdarzeń | +| `EVNT` | `chunk::evnt` | sim | Zdarzenia | +| `SOND` | `chunk::sond` | sim | Dźwięki | +| `DYNM` | `chunk::dynm` | sim | Pojazdy dynamiczne | +| `TRST` | `chunk::trst` | sim/module | Zestawy wagonowe | +| `TRGR` | `chunk::trgr` | sim | Precomputed track graph (planowane) | +| `SIDX` | `chunk::sidx` | manifest | Indeks sekcji przestrzennej (planowane) | + +Chunki o zerowej liczności (pusta tablica) **nie są zapisywane** przez emitter. + +--- + +## 6. Typy podstawowe + +| Typ w pliku | Rozmiar | Opis | +|-------------|---------|---------------------------------------------------| +| `uint8` | 1 B | Bajt bez znaku | +| `uint16` | 2 B | LE | +| `uint32` | 4 B | LE | +| `uint64` | 8 B | LE | +| `int32` | 4 B | LE, ze znakiem | +| `float32` | 4 B | IEEE 754 | +| `float64` | 8 B | IEEE 754 — pozycje world-space, originy mesh | +| `string_id` | 4 B | Indeks STRS; `0xFFFFFFFF` = brak (`kNoString`) | + +Współrzędne instancji modeli i placementów: **float64** (x/y/z). +Wierzchołki mesh/shape: **float32** względem origin (f64). + +--- + +## 7. STRS — tablica łańcuchów + +``` +uint32 count +Powtórzone count razy: + uint32 length + char[] data (length bajtów, bez null-terminatora) +``` + +Indeks `0xFFFFFFFF` (`kNoString`) oznacza brak stringu. + +--- + +## 8. META — metadane modułu + +Layout wersjonowany (pole `layout_version`): + +``` +uint32 layout_version (= 1) +uint32 first_init_count +uint8 has_terrain_chunk (informacyjne) +uint8 has_pack_chunk (informacyjne) +uint8 placement_origin_x (numer parametru include, 0 = brak) +uint8 placement_origin_y +uint8 placement_origin_z +uint8 placement_rotation_y +``` + +--- + +## 9. INCL — include rekurencyjne + +Submoduły wczytywane rekurencyjnie (np. zagnieżdżone `.scm`, tory, CTR): + +``` +uint32 count +Powtórzone count razy: + uint32 source_line + string_id source_path (.scm/.inc tekstowy) + string_id binary_path (.eu7v2) + uint32 param_count + string_id[] parameters + TransformRecord site_transform: + uint32 origin_stack_count + dvec3[] origin_stack + uint32 scale_stack_count + dvec3[] scale_stack + dvec3 rotation (stopnie, XYZ) + uint32 group_depth +``` + +--- + +## 10. PLCE — placementy modułów wielokrotnego użytku + +Pliki placement (flora, dekoracje) emitują lean rekordy wskazujące na **osobny** moduł `.eu7v2`: + +``` +uint32 count +Powtórzone count razy: + string_id module_path (np. scenery/grass_l61/20.eu7v2) + string_id texture_override (kNoString jeśli brak) + float64 x, y, z (world-space) + float32 rotation_y (stopnie) + uint8 cell_id (0xFF = brak) +``` + +**Bake:** unikalne `.inc` z placement file są bake'owane jako osobne moduły `file_kind::module`. +**Runtime:** loader syntetyzuje include z PLCE (ładuje wskazany moduł z transformacją). + +Typowy rozmiar: **~37 B / placement**. + +Przykład: `204_trawky_ter.scm` (110k linii include) → `.eu7v2` ~4 MB (PLCE) + osobne `grass_l61/20.eu7v2` per unikalny `.inc`. + +--- + +## 11. PROT + INST — prototypy i instancje modeli + +### PROT — prototyp + +``` +uint32 count +Powtórzone count razy: + string_id model_file + string_id texture_file + uint8 flags (bit 0=transition, 1=is_terrain, 2=instanceable) + float32 range_min + float32 range_max + uint32 light_state_count + float32[] light_states + uint32 light_color_count + uint32[] light_colors +``` + +### INST — instancja + +``` +uint32 count +Powtórzone count razy: + uint8 flags (bit 0=has_scale, 1=texture_override, 2=has_node) + uint32 proto (indeks PROT) + float64 x, y, z + float32 ax, ay, az (kąty Eulera, stopnie) + uint8 cell_id + [flags & 1]: float32 sx, sy, sz + [flags & 2]: string_id texture_override + [flags & 4]: node_record +``` + +Root scenariusza: modele z PACK bake są deduplikowane do PROT+INST (flattened world-space). + +--- + +## 12. SHPE — kształty (triangles) + +``` +uint32 count +Powtórzone count razy: + node_record node + uint8 translucent + string_id material + lighting_block (12 × float32: diffuse/ambient/specular RGBA) + float64 ox, oy, oz (origin world-space) + uint32 vertex_count + Powtórzone vertex_count razy: + float32 px, py, pz (względem origin) + float32 nx, ny, nz + float32 u, v +``` + +Typ węzła (`node.type`) niesie subtype: `triangles`, `triangle_strip`, `triangle_fan`. + +Duże pliki terenu (same `node … triangles`) mogą być streamowane przy bake; shapes trafiają do **shape spool** na dysku, potem do chunku SHPE przy emit. + +--- + +## 13. MESH — teren (tile) + +Format terenu dla kafli 1 km (origin f64 + relative f32): + +``` +uint32 count +Powtórzone count razy: + string_id material + uint8 translucent + float64 ox, oy, oz + uint32 vertex_count + Powtórzone vertex_count razy: + float32 px, py, pz, nx, ny, nz, u, v +``` + +--- + +## 14. LINE — linie + +``` +uint32 count +Powtórzone count razy: + node_record + lighting_block + float32 line_width + float64 ox, oy, oz + uint32 vertex_count + dvec3[] vertices (float64 × 3) +``` + +--- + +## 15. Rekordy symulacji + +Chunki symulacji: `TRAK`, `TRAC`, `PWRS`, `MEMC`, `LAUN`, `EVNT`, `SOND`, `DYNM`, `TRST`. +Wspólny nagłówek węzła: **`node_record`**. Stringi przez STRS. Pozycje torów/krzywych Béziera: **float64**. + +Szczegóły pól: `scene/eu7/v2/eu7v2_records.h` (funkcje `write_*` / `read_*`). + +--- + +## 16. Bake headless i drzewo modułów + +CLI (binarny `eu07`, bez GUI): + +``` +eu07.exe --eu7v2-bake [opcje] +``` + +| Opcja | Opis | +|--------------------------|---------------------------------------------------| +| `--eu7v2-verify` | Roundtrip: przeładuj `.eu7v2`, porównaj liczności rekordów | +| `--eu7v2-mem-limit-gb N` | Limit pamięci procesu (domyślnie 50 GB); włącza tryb spool | +| `--eu7v2-threads N` | Wątki pool bake (0 = auto = liczba rdzeni) | +| `--eu7v2-max-parse N` | Równoległe parse (0 = auto) | +| `--eu7v2-heavy-parse-mb N` | Próg serial parse dużych plików (0 = wyłączony w trybie spool) | + +**Przepływ bake scenariusza:** + +1. Root `.scn` → enqueue drzewa modułów (dedup po ścieżce kanonicznej) +2. Pool workerów bake'uje moduły równolegle +3. Każdy `.scm`/`.inc` → osobny `.eu7v2` obok źródła tekstowego +4. Placement `.scm` → **PLCE** + enqueue unikalnych `.inc` jako `module` +5. Root emit na końcu (PACK → PROT+INST, includes z podmodułów) +6. Opcjonalnie verify batch wszystkich modułów + +Implementacja: `scene/eu7/eu7_bake_parser.cpp`, `eu07-parser/.../bake_tree.hpp`. + +--- + +## 17. Ścieżki plików + +```cpp +binary_path_from_text("scenery/foo/bar.scm") → scenery/foo/bar.eu7v2 +binary_path_from_text("scenery/grass_l61/20.inc") → scenery/grass_l61/20.eu7v2 +binary_path_from_text("scenery/foo/base.ctr") → scenery/foo/base.ctr.eu7v2 +``` + +Rozszerzenia `.scm`, `.scn`, `.sbt`, `.inc` → `.eu7v2`. +Inne (np. `.ctr`) → `.eu7v2` (bez utraty suffixu). + +--- + +## Pliki implementacji + +| Temat | Plik | +|--------------------|---------------------------------------------------| +| Format core | `scene/eu7/v2/eu7v2_format.h` | +| Scene payloads | `scene/eu7/v2/eu7v2_scene.h` | +| Sim records | `scene/eu7/v2/eu7v2_records.h` | +| Emit RuntimeModule | `scene/eu7/v2/eu7v2_emit_runtime.cpp` | +| Load runtime | `scene/eu7/v2/eu7v2_load.cpp` | +| Test roundtrip | `scene/eu7/v2/eu7v2_test.cpp` | +| Headless bake | `scene/eu7/eu7_bake_parser.cpp` | +| Bake pool | `eu07-parser/include/eu07/scene/bake/bake_tree.hpp` | + +--- + +*Dokumentacja na podstawie kodu; stan na 17 czerwca 2026.* diff --git a/eu07-parser/include/eu07/parser.hpp b/eu07-parser/include/eu07/parser.hpp index ae4b38b8..f528d309 100644 --- a/eu07-parser/include/eu07/parser.hpp +++ b/eu07-parser/include/eu07/parser.hpp @@ -87,13 +87,20 @@ inline void tokenizeInto( skipFieldSeparators(text); continue; } + // Unterminated '[' (no matching ']' in the remainder): it is an + // ordinary character, not a bracket token. The engine's default + // break set is "\n\r\t ;", so '[' belongs inside identifiers such + // as event names ("Kon_It2[Sch2"). Fall through to the general scan + // below, which does NOT stop at '[' and therefore consumes it, + // guaranteeing the outer loop makes forward progress (previously it + // spun forever: the '[' was neither a bracket nor a separator nor a + // token character, so text never shrank). } const char* const start = text.data(); while (!text.empty() && !isFieldSeparator(static_cast(text.front())) && - text.front() != '"' && - text.front() != '[') { + text.front() != '"') { text.remove_prefix(1); } const std::size_t len = static_cast(text.data() - start); @@ -459,10 +466,7 @@ struct ParseResult { return tokens; } -[[nodiscard]] inline ParseResult parseFile( - const std::filesystem::path& path, - const ParseOptions& options = {}) { - const RawFile raw = readRawFile(path); +[[nodiscard]] inline ParseResult parseRawFile(RawFile raw, const ParseOptions& options = {}) { LogicalPass logical = toLogicalLines(raw.lines); ParseResult result; @@ -474,6 +478,12 @@ struct ParseResult { return result; } +[[nodiscard]] inline ParseResult parseFile( + const std::filesystem::path& path, + const ParseOptions& options = {}) { + return parseRawFile(readRawFile(path), options); +} + [[nodiscard]] inline ParseResult parseText(const std::string& text) { RawFile raw; raw.data = text; diff --git a/eu07-parser/include/eu07/scene/bake/bake_tree.hpp b/eu07-parser/include/eu07/scene/bake/bake_tree.hpp index 701f3c73..0b38c34b 100644 --- a/eu07-parser/include/eu07/scene/bake/bake_tree.hpp +++ b/eu07-parser/include/eu07/scene/bake/bake_tree.hpp @@ -12,13 +12,13 @@ #include #include +#include +#include #include #include -#include - #include @@ -39,6 +39,8 @@ #include +#include + #include #include @@ -122,8 +124,44 @@ struct BakeTreeOptions { BakeProgressCallback onProgress; + // Hook wywolywany po zbakowaniu kazdego modulu (przed/zamiast zapisu .eu7). + // Dla roota packBatches != nullptr (sploszczone modele PACK). isRoot == true + // tylko dla pliku-korzenia drzewa bake. Moze byc wolany z wielu watkow. + std::function* packBatches, + ShapeSpoolFile* shapeSpool)> + onModuleBaked; + + // Gdy true: pomijamy zapis legacy .eu7 (writeRuntimeModule). + bool skipLegacyWrite = false; + + // eu7v2 PACK: modele dzieci sa juz w root PACK — nie duplikuj INST w .eu7v2 dziecka. + bool omitChildModuleModels = false; + + // Max rownoleglych pelnych parseFile/processScene w session cache (0 = auto: 1). + // Ogranicza peak RAM gdy wiele ciezkich .scm startuje naraz (Braniewo / linie). + unsigned maxConcurrentParses = 0; + + // Mniej RAM, wolniej: 1 watek bake/compose, agresywne zwalnianie cache PACK. + bool lowMemoryMode = false; + + // Pliki wieksze niz X MB: parse/bake tylko pojedynczo (0 = wylaczone). + unsigned heavyParseThresholdMb = 0; + + // PACK compose: spłucz sekcje modeli poza RAM gdy przekroczony prog (lowMemoryMode). + std::size_t packFlushThreshold = 0; + bool packFlushPerFile = false; + std::function&&)> onPackModelsFlush; + }; +[[nodiscard]] inline bool useIncrementalRootPack(const BakeTreeOptions& options) { + return options.lowMemoryMode; +} + struct BakeTreeContext { @@ -256,6 +294,17 @@ struct BakePoolState { } +[[nodiscard]] inline unsigned resolveParseConcurrencyLimit(const unsigned maxConcurrentParses) { + if (maxConcurrentParses != 0) { + return std::max(1u, maxConcurrentParses); + } + return 1u; +} + +[[nodiscard]] inline std::size_t heavyParseThresholdBytes(const unsigned threshold_mb) { + return threshold_mb == 0 ? 0u : static_cast(threshold_mb) * 1024u * 1024u; +} + inline void recordError(BakePoolState& pool) { @@ -324,7 +373,9 @@ inline void bakeWorkItem( const BakeTreeOptions& options, - const unsigned threadCount) { + const unsigned threadCount, + + BakeParseSessionCache& sessionCache) { if (pool.stop.load(std::memory_order_acquire)) { @@ -358,34 +409,141 @@ inline void bakeWorkItem( pool.inFlight.fetch_add(1, std::memory_order_relaxed); - if (options.onProgress) { - std::lock_guard lock(pool.progressMutex); - options.onProgress( - BakeProgress{ - BakeProgressPhase::Start, - pool.completed.load(std::memory_order_relaxed), - 0, - item.path, - threadCount, - }); - } + bool models_pending_pack = false; try { - const ParseResult parsed = parseFile(item.path); - - SceneProcessOptions shallowOpts; - - shallowOpts.expandIncludes = false; - - const SceneDocument document = - - processScene(parsed, context.sceneryRoot, shallowOpts).document; - - - + RuntimeModule module; const bool emitPackModels = canonicalPathKey(item.path) == rootKey; - RuntimeModule module = bakeModule(document, {.skipLocalModels = emitPackModels}); + std::vector deferred_child_includes; + eu07::scene::detail::FlatFileKind flat_kind = + eu07::scene::detail::FlatFileKind::None; + std::unique_ptr triangle_shape_spool; + { + if (options.onProgress) { + std::lock_guard lock(pool.progressMutex); + options.onProgress( + BakeProgress{ + BakeProgressPhase::Start, + pool.completed.load(std::memory_order_relaxed), + 0, + item.path, + threadCount, + }); + } + + const bool prefer_lightweight_document = + useIncrementalRootPack(options) || + eu07::scene::detail::shouldStreamFlatSourceFile(item.path); + if (prefer_lightweight_document) { + flat_kind = eu07::scene::detail::scanFlatFileKindStreaming(item.path); + } + + const bool try_triangle_terrain_stream = + prefer_lightweight_document && + flat_kind == eu07::scene::detail::FlatFileKind::None && + eu07::scene::detail::shouldStreamFlatSourceFile(item.path); + + bool used_triangle_terrain_stream = false; + SceneDocument empty_document; + const SceneDocument* document_ptr = nullptr; + + if (try_triangle_terrain_stream) { + if (options.lowMemoryMode) { + const std::size_t path_hash = std::hash {}( + item.path.lexically_normal().generic_string()); + triangle_shape_spool = std::make_unique( + std::filesystem::temp_directory_path() / + ("eu7v2_shape_" + std::to_string(path_hash) + ".bin")); + } + if (detail::streamBakeTriangleTerrain( + item.path, module, triangle_shape_spool.get(), threadCount)) { + used_triangle_terrain_stream = true; + document_ptr = &empty_document; + } + } + + if (!used_triangle_terrain_stream) { + const SceneDocument& document = + (prefer_lightweight_document && + (flat_kind == eu07::scene::detail::FlatFileKind::Models || + flat_kind == eu07::scene::detail::FlatFileKind::Includes)) + ? sessionCache.documentForPackCompose( + item.path, context.sceneryRoot, nullptr, true) + : sessionCache.documentFor(item.path, context.sceneryRoot, nullptr); + document_ptr = &document; + } + + const SceneDocument& document = *document_ptr; + + const auto enqueue_unique_inc_modules = [&](const SceneDocument& doc) { + if (!options.onModuleBaked) { + return; + } + BakeWorkItem child; + child.includeStack = item.includeStack; + child.includeStack.push_back(item.path); + for (const ParsedInclude& include : doc.include) { + if (!eu07::scene::detail::isIncFile(include.file)) { + continue; + } + child.path = eu07::scene::detail::resolveIncludeSourcePath( + context.sceneryRoot, item.relativeFile, include.file); + child.relativeFile = eu07::scene::detail::relativeSceneryFile( + context.sceneryRoot, child.path); + enqueueWork(pool, child); + } + }; + + const auto enqueue_discovered_includes = [&](const SceneDocument& doc) { + BakeWorkItem child; + child.includeStack = item.includeStack; + child.includeStack.push_back(item.path); + for (const ParsedInclude& include : doc.include) { + // Placementy .inc (flora, szablony) — tylko do PACK compose, + // nie osobne moduly bake (3_rosl1-leo: 17k+ linii). + if (eu07::scene::detail::isIncFile(include.file)) { + continue; + } + child.path = eu07::scene::detail::resolveIncludeSourcePath( + context.sceneryRoot, item.relativeFile, include.file); + child.relativeFile = eu07::scene::detail::relativeSceneryFile( + context.sceneryRoot, child.path); + enqueueWork(pool, child); + } + }; + + // Root PACK compose jest bardzo pamieciochlonny — nie startuj rownolegle + // pelnych parse linii (l204/l220/…) az compose roota sie nie skonczy. + // lowMemoryMode + spool: jeden modul = jeden plik .scm, PACK skladany per modul. + if (emitPackModels) { + if (useIncrementalRootPack(options)) { + enqueue_discovered_includes(document); + } else { + deferred_child_includes = document.include; + } + } else { + enqueue_discovered_includes(document); + } + enqueue_unique_inc_modules(document); + + const bool skipLocalModels = + emitPackModels || + (options.omitChildModuleModels && isModelOnlyDocument(document)); + if (!used_triangle_terrain_stream) { + module = bakeModule(document, {.skipLocalModels = skipLocalModels}); + } + + models_pending_pack = + !used_triangle_terrain_stream && + skipLocalModels && + (isModelOnlyDocument(document) || + flat_kind == eu07::scene::detail::FlatFileKind::Models); + if (!used_triangle_terrain_stream) { + sessionCache.releaseHeavyParseStorageAfterModuleBake( + item.path, models_pending_pack); + } + } const std::filesystem::path eu7Path = @@ -396,7 +554,7 @@ inline void bakeWorkItem( binary::WriteRuntimeModuleOptions writeOpts; writeOpts.emitPackModels = emitPackModels; std::vector packBatches; - if (writeOpts.emitPackModels) { + if (writeOpts.emitPackModels && !useIncrementalRootPack(options)) { if (options.onProgress) { std::lock_guard lock(pool.progressMutex); options.onProgress( @@ -431,11 +589,12 @@ inline void bakeWorkItem( pack_options.progress = &pack_progress; pack_options.stats = &pack_stats; pack_options.max_threads = threadCount; + pack_options.low_memory = options.lowMemoryMode; + pack_options.pack_flush_threshold = options.packFlushThreshold; + pack_options.pack_flush_per_file = options.packFlushPerFile; + pack_options.on_pack_models_flush = options.onPackModelsFlush; pack_options.pack_batches = &packBatches; - PackComposeSeed root_seed; - root_seed.path = item.path.lexically_normal(); - root_seed.entry = makePackFileCacheEntry(document); - pack_options.root_seed = &root_seed; + pack_options.session_cache = &sessionCache; if (options.onProgress) { std::cerr << "[EU7] PACK compose: " << threadCount << " watkow\n" << std::flush; } @@ -457,21 +616,91 @@ inline void bakeWorkItem( << std::flush; } printPackComposeStats(pack_stats, std::cerr); + printBakeParseSessionStats(sessionCache, std::cerr); if (stats != nullptr) { stats->has_pack_diagnostics = true; stats->pack_compose = snapshotPackComposeStats(pack_stats); } } - binary::PackWriteStats pack_write_stats; - if (writeOpts.emitPackModels) { - writeOpts.pack_write_stats = &pack_write_stats; - std::cerr << "[EU7] zapis .eu7 (PACK)...\n" << std::flush; + + // lowMemoryMode: jeden plik = jeden modul — PACK tylko z tego .scm, potem spool/RAM zwolnione. + if (useIncrementalRootPack(options) && models_pending_pack && !emitPackModels) { + if (flat_kind == eu07::scene::detail::FlatFileKind::Models && + options.onPackModelsFlush) { + detail::streamFlatModelsToPackFlush(item.path, options.onPackModelsFlush); + } else { + std::vector module_pack_batches; + PackComposeOptions pack_options; + pack_options.low_memory = true; + pack_options.pack_flush_per_file = options.packFlushPerFile; + pack_options.on_pack_models_flush = options.onPackModelsFlush; + pack_options.pack_batches = &module_pack_batches; + pack_options.session_cache = &sessionCache; + pack_options.max_threads = 1; + (void)composePackModels( + item.path, context.sceneryRoot, item.relativeFile, pack_options); + if (options.onPackModelsFlush) { + std::vector tail; + for (binary::codec::ModelSectionBatch& batch : module_pack_batches) { + tail.insert( + tail.end(), + std::make_move_iterator(batch.models.begin()), + std::make_move_iterator(batch.models.end())); + } + if (!tail.empty()) { + options.onPackModelsFlush(std::move(tail)); + } + } + } + sessionCache.dropPackCacheEntry(item.path); } - binary::writeRuntimeModule(eu7Path, module, writeOpts); - if (writeOpts.emitPackModels) { - printPackWriteStats(pack_write_stats, std::cerr); - if (stats != nullptr) { - stats->pack_write = pack_write_stats; + + // Hook: pozwala silnikowi wyemitowac wlasny format (eu7v2) bezposrednio + // z RuntimeModule, z dostepem do sploszczonych modeli PACK dla roota. + if (options.onModuleBaked) { + const bool root_with_pack = + emitPackModels && !useIncrementalRootPack(options); + const std::vector* batchesPtr = + root_with_pack ? &packBatches : nullptr; + options.onModuleBaked( + module, item.path, emitPackModels, batchesPtr, triangle_shape_spool.get()); + } + + if (emitPackModels && !deferred_child_includes.empty() && + !useIncrementalRootPack(options)) { + SceneDocument stub; + stub.include = std::move(deferred_child_includes); + BakeWorkItem child; + child.includeStack = item.includeStack; + child.includeStack.push_back(item.path); + for (const ParsedInclude& include : stub.include) { + if (eu07::scene::detail::isIncFile(include.file)) { + continue; + } + child.path = eu07::scene::detail::resolveIncludeSourcePath( + context.sceneryRoot, item.relativeFile, include.file); + child.relativeFile = + eu07::scene::detail::relativeSceneryFile(context.sceneryRoot, child.path); + enqueueWork(pool, child); + } + } + + if (!models_pending_pack) { + sessionCache.evictEntryIfNotNeeded(item.path); + } + + if (!options.skipLegacyWrite) { + binary::PackWriteStats pack_write_stats; + if (writeOpts.emitPackModels) { + writeOpts.pack_write_stats = &pack_write_stats; + std::cerr << "[EU7] zapis .eu7 (PACK)...\n" << std::flush; + } + binary::writeRuntimeModule(eu7Path, module, writeOpts); + if (writeOpts.emitPackModels) { + printPackWriteStats(pack_write_stats, std::cerr); + if (stats != nullptr) { + stats->pack_write = pack_write_stats; + } } } @@ -525,30 +754,6 @@ inline void bakeWorkItem( } - - - BakeWorkItem child; - - child.includeStack = item.includeStack; - - child.includeStack.push_back(item.path); - - - - for (const std::string& includeFile : scanIncludeFilePaths(parsed.tokens)) { - - child.path = eu07::scene::detail::resolveIncludeSourcePath( - - context.sceneryRoot, item.relativeFile, includeFile); - - child.relativeFile = eu07::scene::detail::relativeSceneryFile( - - context.sceneryRoot, child.path); - - enqueueWork(pool, child); - - } - } catch (...) { recordError(pool); @@ -585,7 +790,9 @@ inline void bakePoolWorker( const BakeTreeOptions& options, - const unsigned threadCount) { + const unsigned threadCount, + + BakeParseSessionCache& sessionCache) { while (true) { @@ -639,7 +846,7 @@ inline void bakePoolWorker( bakeWorkItem( - item, context, rootKey, pool, stats, rootModuleOut, options, threadCount); + item, context, rootKey, pool, stats, rootModuleOut, options, threadCount, sessionCache); } catch (...) { @@ -690,6 +897,11 @@ inline void runBakePool( BakePoolState pool; + BakeParseSessionCache sessionCache { + detail::resolveParseConcurrencyLimit(options.maxConcurrentParses), + options.lowMemoryMode, + detail::heavyParseThresholdBytes(options.heavyParseThresholdMb) }; + enqueueWork( pool, @@ -728,7 +940,9 @@ inline void runBakePool( std::cref(options), - threadCount); + threadCount, + + std::ref(sessionCache)); } diff --git a/eu07-parser/include/eu07/scene/bake/compose_pack_models.hpp b/eu07-parser/include/eu07/scene/bake/compose_pack_models.hpp index 79f99dc0..c769b6a0 100644 --- a/eu07-parser/include/eu07/scene/bake/compose_pack_models.hpp +++ b/eu07-parser/include/eu07/scene/bake/compose_pack_models.hpp @@ -8,7 +8,10 @@ #include #include #include +#include +#include #include +#include #include #include @@ -74,7 +77,11 @@ struct PackComposeStats { std::atomic queue_peak { 0 }; std::atomic unique_files_cached { 0 }; std::atomic parse_us { 0 }; + std::atomic tokenize_us { 0 }; // parseFile (read + tokenize) + std::atomic process_us { 0 }; // processScene (build SceneDocument) + std::atomic makeentry_us { 0 }; // makePackFileCacheEntry (bakeModel loop) std::atomic instantiate_us { 0 }; + std::atomic sink_us { 0 }; // PackSectionAccumulator::add (sharding) std::atomic finalize_us { 0 }; std::atomic compose_us { 0 }; std::atomic threads { 0 }; @@ -82,6 +89,16 @@ struct PackComposeStats { std::atomic chain_fold_steps { 0 }; std::atomic chain_fold_hops { 0 }; std::atomic inc_includes_inlined { 0 }; + // --- live deadlock diagnostics (read by external watchdog) --- + std::atomic diag_pending { 0 }; // outstanding queued/running tasks + std::atomic diag_gate_active { 0 }; // gate slots in use + std::atomic diag_gate_max { 0 }; // gate slot limit + std::atomic diag_gate_waiting { 0 }; // workers blocked in gate acquire + std::atomic diag_workers_alive { 0 }; // worker threads currently running + std::atomic diag_workers_idle { 0 }; // workers blocked on queue_cv + std::atomic diag_workers_parse { 0 }; // workers inside buildCacheEntry/parse + std::atomic diag_workers_inst { 0 }; // workers inside instantiate/process + std::atomic diag_main_wait { 0 }; // 1 while main blocks on done_cv }; struct PackComposeStatsSnapshot { @@ -99,7 +116,11 @@ struct PackComposeStatsSnapshot { std::size_t queue_peak = 0; std::size_t unique_files_cached = 0; std::uint64_t parse_us = 0; + std::uint64_t tokenize_us = 0; + std::uint64_t process_us = 0; + std::uint64_t makeentry_us = 0; std::uint64_t instantiate_us = 0; + std::uint64_t sink_us = 0; std::uint64_t finalize_us = 0; std::uint64_t compose_us = 0; unsigned threads = 0; @@ -126,7 +147,11 @@ struct PackComposeStatsSnapshot { out.queue_peak = stats.queue_peak.load(std::memory_order_relaxed); out.unique_files_cached = stats.unique_files_cached.load(std::memory_order_relaxed); out.parse_us = stats.parse_us.load(std::memory_order_relaxed); + out.tokenize_us = stats.tokenize_us.load(std::memory_order_relaxed); + out.process_us = stats.process_us.load(std::memory_order_relaxed); + out.makeentry_us = stats.makeentry_us.load(std::memory_order_relaxed); out.instantiate_us = stats.instantiate_us.load(std::memory_order_relaxed); + out.sink_us = stats.sink_us.load(std::memory_order_relaxed); out.finalize_us = stats.finalize_us.load(std::memory_order_relaxed); out.compose_us = stats.compose_us.load(std::memory_order_relaxed); out.threads = stats.threads.load(std::memory_order_relaxed); @@ -197,16 +222,31 @@ inline void printPackComposeStats(const PackComposeStats& stats, std::ostream& o printPackComposeStats(snapshotPackComposeStats(stats), out); } +namespace detail { +class BakeParseSessionCache; +} + struct PackComposeOptions { PackComposeProgress* progress = nullptr; PackComposeStats* stats = nullptr; unsigned max_threads = 0; // Root juz sparsowany w bake_tree — bez drugiego parseFile. const PackComposeSeed* root_seed = nullptr; + // Wspolny cache z bake_tree (eliminuje drugi parse tego samego pliku). + detail::BakeParseSessionCache* session_cache = nullptr; // Wynik: sekcje 1 km (serializacja PACK w writeRuntimeModule). std::vector* pack_batches = nullptr; + // Strumieniuj modele do pack_batches (bez duzego akumulatora na koncu). + bool low_memory = false; + // low_memory: gdy w RAM jest wiecej niz threshold, przenies sekcje do on_flush. + std::size_t pack_flush_threshold = 0; + // low_memory: po kazdym pliku w compose — spłucz caly PACK z RAM (ignoruje threshold). + bool pack_flush_per_file = false; + std::function&&)> on_pack_models_flush; }; +inline constexpr std::size_t kDefaultPackFlushThreshold = 65536u; + namespace detail { // Po tylu hopach skladania wysylamy kontynuacje do kolejki (rownolegle segmenty lancucha). @@ -228,14 +268,18 @@ public: void acquire() { std::unique_lock lock(mutex_); + waiting_.fetch_add(1, std::memory_order_relaxed); cv_.wait(lock, [this]() { return active_slots_ < max_slots_; }); + waiting_.fetch_sub(1, std::memory_order_relaxed); ++active_slots_; + active_snapshot_.store(active_slots_, std::memory_order_relaxed); } void release() { { std::lock_guard lock(mutex_); --active_slots_; + active_snapshot_.store(active_slots_, std::memory_order_relaxed); } cv_.notify_one(); } @@ -252,9 +296,21 @@ public: return Scope { *this }; } + [[nodiscard]] unsigned maxSlots() const noexcept { + return max_slots_; + } + [[nodiscard]] unsigned activeSlots() const noexcept { + return active_snapshot_.load(std::memory_order_relaxed); + } + [[nodiscard]] unsigned waiting() const noexcept { + return waiting_.load(std::memory_order_relaxed); + } + private: unsigned max_slots_; unsigned active_slots_ = 0; + std::atomic active_snapshot_ { 0 }; + std::atomic waiting_ { 0 }; std::mutex mutex_; std::condition_variable cv_; }; @@ -421,19 +477,791 @@ inline void applyIncludePlacementToModels( using PackModelBatchSink = std::function&&)>; -[[nodiscard]] inline PackFileCacheEntry makePackFileCacheEntry(const SceneDocument& document) { +inline constexpr std::size_t kParallelBakeModelThreshold = 8192; + +[[nodiscard]] inline PackFileCacheEntry makePackFileCacheEntry( + const SceneDocument& document, + const unsigned max_threads = 0) { PackFileCacheEntry entry; entry.includes = document.include; if (const std::optional placement = extractIncludePlacement(document)) { entry.placement = *placement; } - entry.base_models.reserve(document.nodeModel.size()); - for (const ParsedNodeModel& parsedModel : document.nodeModel) { - entry.base_models.push_back(bakeModel(parsedModel)); + const std::size_t model_count = document.nodeModel.size(); + entry.base_models.resize(model_count); + + if (model_count < kParallelBakeModelThreshold) { + for (std::size_t i = 0; i < model_count; ++i) { + entry.base_models[i] = bakeModel(document.nodeModel[i]); + } + return entry; + } + + const unsigned worker_count = resolveComposeThreadCount(max_threads); + std::atomic next_index { 0 }; + auto worker = [&]() { + while (true) { + const std::size_t index = next_index.fetch_add(1, std::memory_order_relaxed); + if (index >= model_count) { + return; + } + entry.base_models[index] = bakeModel(document.nodeModel[index]); + } + }; + + std::vector threads; + threads.reserve(worker_count); + for (unsigned i = 0; i < worker_count; ++i) { + threads.emplace_back(worker); + } + for (std::thread& thread : threads) { + thread.join(); } return entry; } +// Buduje PACK zwalniajac nodeModel w trakcie (mniejszy peak niz pelna kopia + bake). +[[nodiscard]] inline PackFileCacheEntry makePackFileCacheEntryConsuming(SceneDocument& document) { + PackFileCacheEntry entry; + entry.includes = std::move(document.include); + if (const std::optional placement = extractIncludePlacement(document)) { + entry.placement = *placement; + } + const std::size_t model_count = document.nodeModel.size(); + entry.base_models.reserve(model_count); + while (!document.nodeModel.empty()) { + const std::size_t index = document.nodeModel.size() - 1; + entry.base_models.push_back(bakeModel(document.nodeModel[index])); + document.nodeModel.pop_back(); + } + std::reverse(entry.base_models.begin(), entry.base_models.end()); + document.nodeModel.shrink_to_fit(); + return entry; +} + +// low_memory .inc: parse jednego modelu → bake → base_models (bez wektora ParsedNodeModel). +[[nodiscard]] inline bool streamFlatSegmentsToPackModels( + const std::vector& segments, + std::vector& base_models_out) { + for (const LogicalSegment& segment : segments) { + std::string_view text = segment.text; + eu07::skipFieldSeparators(text); + if (text.empty()) { + continue; + } + ParsedNodeModel parsed; + if (!eu07::scene::detail::parseModelSegment(segment, parsed)) { + return false; + } + base_models_out.push_back(bakeModel(parsed)); + } + return true; +} + +[[nodiscard]] inline bool tryStreamTokenModelOnlyToPack( + const ParseResult& parsed, + const std::filesystem::path& base_directory, + PackFileCacheEntry& entry) { + SceneDocument scratch; + std::filesystem::path include_root = base_directory; + if (include_root.empty()) { + include_root = std::filesystem::current_path(); + } + + ParseContext context { scratch, {}, include_root, {} }; + context.expandIncludes = false; + + TokenStream stream(parsed.tokens); + std::vector jobs; + jobs.reserve(parsed.tokens.size() / 16); + + while (!stream.empty()) { + if (eu07::scene::detail::dispatchDirective(stream, context)) { + continue; + } + + if (!stream.empty() && isKeyword(stream.peek().value, "node")) { + const std::size_t anchor = stream.checkpoint(); + NodeHeader header; + std::string subtype; + std::vector raw; + if (!node::io::consumeHeader(stream, header, subtype, raw)) { + return false; + } + if (!isKeyword(subtype, node_model::kSubtype)) { + return false; + } + node::applyScratchContext(header, context.scratch); + jobs.push_back(eu07::scene::detail::ModelParseJob { anchor, header }); + if (!eu07::scene::detail::skipModelBody(stream)) { + return false; + } + continue; + } + + UnknownEntry unknown; + unknown.line = stream.peek().sourceLine; + unknown.token = stream.consume().value; + scratch.unknown.push_back(std::move(unknown)); + } + + if (jobs.empty()) { + return false; + } + + entry.includes = std::move(scratch.include); + if (const std::optional placement = extractIncludePlacement(scratch)) { + entry.placement = *placement; + } + + entry.base_models.clear(); + entry.base_models.reserve(jobs.size()); + for (const eu07::scene::detail::ModelParseJob& job : jobs) { + ParsedNodeModel model; + TokenStream job_stream(parsed.tokens); + job_stream.rewind(job.segment_index); + NodeHeader parsed_header; + std::string subtype; + std::vector raw; + if (!node::io::consumeHeader(job_stream, parsed_header, subtype, raw) || + !isKeyword(subtype, node_model::kSubtype)) { + return false; + } + model.header = job.header; + if (!node_model::parseBody(job_stream, raw, job.header, model)) { + return false; + } + entry.base_models.push_back(bakeModel(model)); + } + return true; +} + +// PACK compose (lightweight): tylko include z plaskich linii — bez modeli i tokenizacji. +[[nodiscard]] inline std::optional +extractFlatIncludesOnly(const std::vector& segments) { + SceneDocument document; + for (const LogicalSegment& segment : segments) { + std::string_view text = segment.text; + eu07::skipFieldSeparators(text); + if (text.empty()) { + continue; + } + if (!eu07::scene::detail::segmentStartsWithKeyword(text, "include")) { + continue; + } + ParsedInclude entry; + if (!eu07::scene::detail::parseIncludeSegment(segment, entry)) { + return std::nullopt; + } + document.include.push_back(std::move(entry)); + } + return document; +} + +// Duze plaskie .scm: linia po linii → batch → spool (bez readRawFile / base_models w RAM). +inline void streamFlatModelsToPackFlush( + const std::filesystem::path& path, + const std::function&&)>& on_flush, + const std::size_t batch_size = 4096) { + if (!on_flush) { + return; + } + std::vector batch; + batch.reserve(batch_size); + eu07::scene::detail::streamFlatFileModels(path, [&](const ParsedNodeModel& parsed) { + batch.push_back(bakeModel(parsed)); + if (batch.size() >= batch_size) { + on_flush(std::move(batch)); + batch.clear(); + batch.reserve(batch_size); + } + }); + if (!batch.empty()) { + on_flush(std::move(batch)); + } +} + +// Wspolny cache parse na cala sesje bakeModuleTree — bake_tree i PACK compose +// czytaja ten sam SceneDocument + PackFileCacheEntry (bez drugiego parseFile). +class ParseConcurrencyGate { + public: + explicit ParseConcurrencyGate(const unsigned limit) + : limit_(std::max(1u, limit)) {} + + void acquire() { + std::unique_lock lock(mutex_); + cv_.wait(lock, [&]() { return in_flight_ < limit_; }); + ++in_flight_; + } + + void release() { + { + std::lock_guard lock(mutex_); + if (in_flight_ > 0) { + --in_flight_; + } + } + cv_.notify_one(); + } + + struct Guard { + ParseConcurrencyGate& gate; + explicit Guard(ParseConcurrencyGate& g) : gate(g) { gate.acquire(); } + Guard(const Guard&) = delete; + Guard& operator=(const Guard&) = delete; + ~Guard() { gate.release(); } + }; + + private: + unsigned const limit_; + unsigned in_flight_ = 0; + std::mutex mutex_; + std::condition_variable cv_; +}; + +// Pliki > heavy_threshold_bytes: parse/bake wyłącznie pojedynczo (zero innych parse). +// Mniejsze pliki: do normal_limit rownolegle. +class PathAwareParseGate { + public: + PathAwareParseGate( + const unsigned normal_limit, + const std::size_t heavy_threshold_bytes, + const bool allow_parallel_heavy = false) + : normal_limit_(std::max(1u, normal_limit)), + heavy_threshold_bytes_(heavy_threshold_bytes), + allow_parallel_heavy_(allow_parallel_heavy) {} + + [[nodiscard]] static std::size_t sourceFileSizeBytes(const std::filesystem::path& path) { + std::error_code ec; + const auto size = std::filesystem::file_size(path, ec); + return ec ? 0u : static_cast(size); + } + + struct Guard { + PathAwareParseGate& gate; + explicit Guard(PathAwareParseGate& g, const std::filesystem::path& path) : gate(g) { + gate.acquire(path); + } + Guard(const Guard&) = delete; + Guard& operator=(const Guard&) = delete; + ~Guard() { gate.release(); } + }; + + private: + friend struct Guard; + + void acquire(const std::filesystem::path& path) { + const bool heavy = !allow_parallel_heavy_ && heavy_threshold_bytes_ > 0 && + sourceFileSizeBytes(path) > heavy_threshold_bytes_; + std::unique_lock lock(mutex_); + if (heavy) { + cv_.wait(lock, [&]() { return in_flight_ == 0; }); + heavy_active_ = true; + in_flight_ = 1; + return; + } + cv_.wait(lock, [&]() { return !heavy_active_ && in_flight_ < normal_limit_; }); + ++in_flight_; + } + + void release() { + { + std::lock_guard lock(mutex_); + if (heavy_active_) { + heavy_active_ = false; + in_flight_ = 0; + } else if (in_flight_ > 0) { + --in_flight_; + } + } + cv_.notify_all(); + } + + unsigned const normal_limit_; + std::size_t const heavy_threshold_bytes_; + bool const allow_parallel_heavy_; + unsigned in_flight_ = 0; + bool heavy_active_ = false; + std::mutex mutex_; + std::condition_variable cv_; +}; + +class BakeParseSessionCache { +public: + explicit BakeParseSessionCache( + const unsigned max_concurrent_parses = 1, + const bool low_memory = false, + const std::size_t heavy_parse_threshold_bytes = 0) + : parse_gate_(max_concurrent_parses, heavy_parse_threshold_bytes, low_memory), + low_memory_mode_(low_memory) {} + + struct Entry { + std::filesystem::file_time_type mtime {}; + bool has_document = false; + SceneDocument document; + bool pack_lightweight_only = false; + bool has_pack = false; + PackFileCacheEntry pack; + }; + + std::atomic document_hits { 0 }; + std::atomic document_misses { 0 }; + std::atomic pack_hits { 0 }; + std::atomic pack_builds { 0 }; + std::atomic full_parses { 0 }; + + [[nodiscard]] std::size_t fileCount() const { + std::shared_lock lock(mutex_); + return entries_.size(); + } + + [[nodiscard]] const SceneDocument& documentFor( + const std::filesystem::path& path, + const std::filesystem::path& scenery_root, + PackComposeStats* stats = nullptr) { + const std::string key = path.lexically_normal().generic_string(); + const std::filesystem::file_time_type mtime = readMtime(path); + + { + std::shared_lock read_lock(mutex_); + if (const auto found = entries_.find(key); found != entries_.end() && + found->second.has_document && + !found->second.pack_lightweight_only && + found->second.mtime == mtime) { + document_hits.fetch_add(1, std::memory_order_relaxed); + return found->second.document; + } + } + + std::unique_lock write_lock(mutex_); + Entry& slot = entries_[key]; + if (slot.has_document && !slot.pack_lightweight_only && slot.mtime == mtime) { + document_hits.fetch_add(1, std::memory_order_relaxed); + return slot.document; + } + + document_misses.fetch_add(1, std::memory_order_relaxed); + full_parses.fetch_add(1, std::memory_order_relaxed); + + // Nie trzymaj globalnego mutexa cache podczas parse (wolne I/O + tokenize). + write_lock.unlock(); + PathAwareParseGate::Guard const parse_slot(parse_gate_, path); + write_lock.lock(); + Entry& fresh = entries_[key]; + if (fresh.has_document && !fresh.pack_lightweight_only && fresh.mtime == mtime) { + document_hits.fetch_add(1, std::memory_order_relaxed); + return fresh.document; + } + + parseIntoEntry(fresh, path, scenery_root, mtime, stats, false); + return fresh.document; + } + + [[nodiscard]] const SceneDocument& documentForPackCompose( + const std::filesystem::path& path, + const std::filesystem::path& scenery_root, + PackComposeStats* stats = nullptr, + const bool pack_lightweight = true) { + const std::string key = path.lexically_normal().generic_string(); + const std::filesystem::file_time_type mtime = readMtime(path); + + { + std::shared_lock read_lock(mutex_); + if (const auto found = entries_.find(key); found != entries_.end() && + found->second.has_document && + found->second.mtime == mtime) { + document_hits.fetch_add(1, std::memory_order_relaxed); + return found->second.document; + } + } + + std::unique_lock write_lock(mutex_); + Entry& slot = entries_[key]; + if (slot.has_document && slot.mtime == mtime) { + document_hits.fetch_add(1, std::memory_order_relaxed); + return slot.document; + } + + document_misses.fetch_add(1, std::memory_order_relaxed); + full_parses.fetch_add(1, std::memory_order_relaxed); + + write_lock.unlock(); + PathAwareParseGate::Guard const parse_slot(parse_gate_, path); + write_lock.lock(); + Entry& fresh = entries_[key]; + if (fresh.has_document && fresh.mtime == mtime) { + document_hits.fetch_add(1, std::memory_order_relaxed); + return fresh.document; + } + + parseIntoEntry(fresh, path, scenery_root, mtime, stats, pack_lightweight); + return fresh.document; + } + + [[nodiscard]] const PackFileCacheEntry* tryPackEntry(const std::filesystem::path& path) const { + const std::string key = path.lexically_normal().generic_string(); + std::shared_lock read_lock(mutex_); + if (const auto found = entries_.find(key); found != entries_.end() && found->second.has_pack) { + return &found->second.pack; + } + return nullptr; + } + + [[nodiscard]] const PackFileCacheEntry& packEntryFor( + const std::filesystem::path& path, + const std::filesystem::path& scenery_root, + PackComposeStats* stats = nullptr) { + if (const PackFileCacheEntry* cached = tryPackEntry(path)) { + pack_hits.fetch_add(1, std::memory_order_relaxed); + return *cached; + } + + const std::string key = path.lexically_normal().generic_string(); + const std::filesystem::file_time_type mtime = readMtime(path); + + if (low_memory_mode_ && + (eu07::scene::detail::isIncFile(path.filename().string()) || + eu07::scene::detail::shouldStreamFlatSourceFile(path))) { + std::unique_lock write_lock(mutex_); + Entry& slot = entries_[key]; + if (slot.has_pack && slot.mtime == mtime) { + pack_hits.fetch_add(1, std::memory_order_relaxed); + return slot.pack; + } + + write_lock.unlock(); + PathAwareParseGate::Guard const parse_slot(parse_gate_, path); + write_lock.lock(); + Entry& fresh = entries_[key]; + if (fresh.has_pack && fresh.mtime == mtime) { + pack_hits.fetch_add(1, std::memory_order_relaxed); + return fresh.pack; + } + + if (tryStreamIncPackEntry(path, scenery_root, fresh, mtime, stats)) { + pack_builds.fetch_add(1, std::memory_order_relaxed); + full_parses.fetch_add(1, std::memory_order_relaxed); + return fresh.pack; + } + write_lock.unlock(); + } + + const bool pack_lightweight = + !eu07::scene::detail::isIncFile(path.filename().string()); + documentForPackCompose(path, scenery_root, stats, pack_lightweight); + + std::unique_lock write_lock(mutex_); + Entry& slot = entries_[key]; + if (!slot.has_pack) { + buildPackFromDocument(slot, stats); + } else { + pack_hits.fetch_add(1, std::memory_order_relaxed); + } + return slot.pack; + } + + // After pack exists, parsed nodeModel is redundant (see buildPackFromDocument). + void releaseHeavyParseStorageAfterModuleBake( + const std::filesystem::path& path, + const bool models_pending_pack) { + const std::string key = path.lexically_normal().generic_string(); + std::unique_lock write_lock(mutex_); + if (const auto found = entries_.find(key); found != entries_.end()) { + if (!found->second.has_document) { + return; + } + const bool release_models = !models_pending_pack || found->second.has_pack; + releaseHeavySceneParseStorage(found->second.document, release_models); + } + } + + // Drop parsed document after module bake+emit; retain PACK cache for root compose. + void evictEntryIfNotNeeded(const std::filesystem::path& path) { + const std::string key = path.lexically_normal().generic_string(); + std::unique_lock write_lock(mutex_); + const auto found = entries_.find(key); + if (found == entries_.end()) { + return; + } + if (found->second.has_pack) { + found->second.document = {}; + found->second.has_document = false; + found->second.pack_lightweight_only = false; + return; + } + (void)entries_.erase(key); + } + + // PACK compose skopiowal modele — zwolnij cache (w low_memory: cale wpisy). + void releasePackCacheAfterComposeUse(const std::filesystem::path& path) { + if (low_memory_mode_) { + dropPackCacheEntry(path); + return; + } + releasePackBaseModelsUnlessInc(path); + } + + void dropPackCacheEntry(const std::filesystem::path& path) { + const std::string key = path.lexically_normal().generic_string(); + std::unique_lock write_lock(mutex_); + (void)entries_.erase(key); + } + +private: + void releasePackBaseModelsUnlessInc(const std::filesystem::path& path) { + if (eu07::scene::detail::isIncFile(path.filename().string())) { + return; + } + const std::string key = path.lexically_normal().generic_string(); + std::unique_lock write_lock(mutex_); + if (const auto found = entries_.find(key); found != entries_.end() && found->second.has_pack) { + found->second.pack.base_models.clear(); + found->second.pack.base_models.shrink_to_fit(); + } + } + + [[nodiscard]] static std::filesystem::file_time_type readMtime( + const std::filesystem::path& path) { + std::error_code ec; + const auto mtime = std::filesystem::last_write_time(path, ec); + return ec ? std::filesystem::file_time_type {} : mtime; + } + + static void addTimingUs( + PackComposeStats* stats, + std::atomic& dst, + const std::chrono::steady_clock::time_point begin, + const std::chrono::steady_clock::time_point end) { + if (stats == nullptr) { + return; + } + dst.fetch_add( + static_cast( + std::chrono::duration_cast(end - begin).count()), + std::memory_order_relaxed); + } + + void parseIntoEntry( + Entry& slot, + const std::filesystem::path& path, + const std::filesystem::path& scenery_root, + const std::filesystem::file_time_type mtime, + PackComposeStats* stats, + const bool pack_lightweight) { + const auto parse_begin = std::chrono::steady_clock::now(); + + const bool stream_source = pack_lightweight; + const eu07::scene::detail::FlatFileKind flat_kind = + (stream_source || + eu07::scene::detail::shouldStreamFlatSourceFile(path)) + ? eu07::scene::detail::scanFlatFileKindStreaming(path) + : eu07::scene::detail::FlatFileKind::None; + if (stream_source || + flat_kind != eu07::scene::detail::FlatFileKind::None) { + if (flat_kind == eu07::scene::detail::FlatFileKind::Includes || + flat_kind == eu07::scene::detail::FlatFileKind::Models) { + const std::optional flat = + eu07::scene::detail::buildFlatIncludesDocumentStreaming(path); + if (flat) { + slot.document = *flat; + const auto process_end = std::chrono::steady_clock::now(); + slot.mtime = mtime; + slot.has_document = true; + slot.pack_lightweight_only = true; + if (stats != nullptr) { + addTimingUs(stats, stats->process_us, parse_begin, process_end); + stats->parse_us.fetch_add( + static_cast( + std::chrono::duration_cast( + process_end - parse_begin) + .count()), + std::memory_order_relaxed); + } + return; + } + } + if (stream_source && + eu07::scene::detail::shouldStreamFlatSourceFile(path) && + flat_kind == eu07::scene::detail::FlatFileKind::None) { + throw std::runtime_error( + "EU7: duzy plik nie-plaski — nie mozna zaladowac calego do RAM: " + + path.string()); + } + } + + eu07::RawFile raw = readRawFile(path); + SceneProcessOptions options; + options.expandIncludes = false; + options.packComposeLightweight = pack_lightweight; + + if (pack_lightweight) { + // PACK compose: nie ladowac plaskiej flory (miliony modeli) — tylko include'y. + const LogicalPass logical = toLogicalLines(raw.lines); + const eu07::scene::detail::FlatFileKind kind = + eu07::scene::detail::classifyFlatFile(logical.segments); + if (kind == eu07::scene::detail::FlatFileKind::Includes) { + if (const std::optional flat = + eu07::scene::detail::parseFlatIncludes(logical.segments)) { + slot.document = std::move(*flat); + const auto process_end = std::chrono::steady_clock::now(); + slot.mtime = mtime; + slot.has_document = true; + slot.pack_lightweight_only = true; + if (stats != nullptr) { + addTimingUs(stats, stats->process_us, parse_begin, process_end); + stats->parse_us.fetch_add( + static_cast( + std::chrono::duration_cast( + process_end - parse_begin) + .count()), + std::memory_order_relaxed); + } + return; + } + } + if (kind == eu07::scene::detail::FlatFileKind::Models || + kind == eu07::scene::detail::FlatFileKind::None) { + if (const std::optional flat = + extractFlatIncludesOnly(logical.segments)) { + slot.document = std::move(*flat); + const auto process_end = std::chrono::steady_clock::now(); + slot.mtime = mtime; + slot.has_document = true; + slot.pack_lightweight_only = true; + if (stats != nullptr) { + addTimingUs(stats, stats->process_us, parse_begin, process_end); + stats->parse_us.fetch_add( + static_cast( + std::chrono::duration_cast( + process_end - parse_begin) + .count()), + std::memory_order_relaxed); + } + return; + } + } + } else if (std::optional flat = + eu07::scene::detail::tryProcessFlatSceneFile(raw, scenery_root)) { + slot.document = std::move(*flat); + const auto process_end = std::chrono::steady_clock::now(); + slot.mtime = mtime; + slot.has_document = true; + slot.pack_lightweight_only = pack_lightweight; + if (stats != nullptr) { + addTimingUs(stats, stats->process_us, parse_begin, process_end); + stats->parse_us.fetch_add( + static_cast( + std::chrono::duration_cast( + process_end - parse_begin) + .count()), + std::memory_order_relaxed); + } + return; + } + + const ParseResult parsed = parseRawFile(std::move(raw)); + const auto tokenize_end = std::chrono::steady_clock::now(); + slot.document = processScene(parsed, scenery_root, options).document; + const auto process_end = std::chrono::steady_clock::now(); + + slot.mtime = mtime; + slot.has_document = true; + slot.pack_lightweight_only = pack_lightweight; + + if (stats != nullptr) { + addTimingUs(stats, stats->tokenize_us, parse_begin, tokenize_end); + addTimingUs(stats, stats->process_us, tokenize_end, process_end); + stats->parse_us.fetch_add( + static_cast( + std::chrono::duration_cast(process_end - parse_begin) + .count()), + std::memory_order_relaxed); + } + } + + void buildPackFromDocument(Entry& slot, PackComposeStats* stats) { + const auto begin = std::chrono::steady_clock::now(); + slot.pack = low_memory_mode_ ? makePackFileCacheEntryConsuming(slot.document) + : makePackFileCacheEntry(slot.document); + const auto end = std::chrono::steady_clock::now(); + slot.has_pack = true; + slot.document = {}; + slot.has_document = false; + slot.pack_lightweight_only = false; + pack_builds.fetch_add(1, std::memory_order_relaxed); + addTimingUs(stats, stats->makeentry_us, begin, end); + } + + [[nodiscard]] bool tryStreamIncPackEntry( + const std::filesystem::path& path, + const std::filesystem::path& scenery_root, + Entry& slot, + const std::filesystem::file_time_type mtime, + PackComposeStats* stats) { + (void)scenery_root; + const auto parse_begin = std::chrono::steady_clock::now(); + + const eu07::scene::detail::FlatFileKind kind = + eu07::scene::detail::scanFlatFileKindStreaming(path); + if (kind == eu07::scene::detail::FlatFileKind::None) { + return false; + } + + PackFileCacheEntry built; + + if (kind == eu07::scene::detail::FlatFileKind::Models) { + eu07::scene::detail::streamFlatFileModels( + path, [&](const ParsedNodeModel& parsed) { + built.base_models.push_back(bakeModel(parsed)); + }); + } else if (kind == eu07::scene::detail::FlatFileKind::Includes) { + const std::optional doc = + eu07::scene::detail::buildFlatIncludesDocumentStreaming(path); + if (!doc) { + return false; + } + built.includes = std::move(doc->include); + if (const std::optional placement = extractIncludePlacement(*doc)) { + built.placement = *placement; + } + } else { + return false; + } + + const auto end = std::chrono::steady_clock::now(); + if (stats != nullptr) { + addTimingUs(stats, stats->process_us, parse_begin, end); + addTimingUs(stats, stats->makeentry_us, parse_begin, end); + stats->parse_us.fetch_add( + static_cast( + std::chrono::duration_cast(end - parse_begin) + .count()), + std::memory_order_relaxed); + } + slot.pack = std::move(built); + slot.mtime = mtime; + slot.has_pack = true; + slot.has_document = false; + slot.pack_lightweight_only = true; + return true; + } + + mutable std::shared_mutex mutex_; + std::unordered_map entries_; + PathAwareParseGate parse_gate_; + bool const low_memory_mode_; +}; + +inline void printBakeParseSessionStats(const BakeParseSessionCache& cache, std::ostream& out) { + out << "[EU7] session cache: pliki=" << cache.fileCount() + << " doc_hit=" << cache.document_hits.load(std::memory_order_relaxed) + << " doc_miss=" << cache.document_misses.load(std::memory_order_relaxed) + << " pack_hit=" << cache.pack_hits.load(std::memory_order_relaxed) + << " pack_build=" << cache.pack_builds.load(std::memory_order_relaxed) + << " parse=" << cache.full_parses.load(std::memory_order_relaxed) << '\n' + << std::flush; +} + class PackSectionAccumulator { public: static constexpr unsigned kShardCount = 64; @@ -549,6 +1377,199 @@ private: std::size_t models_total_ = 0; }; +// low_memory: od razu do pack_batches (bez osobnego duzego akumulatora + finalize merge). +class PackBatchDirectBuilder { +public: + PackBatchDirectBuilder( + std::vector* batches, + const bool thread_safe) + : batches_(batches), thread_safe_(thread_safe) { + batches_->clear(); + } + + void set_flush( + std::size_t threshold, + std::function&&)> on_flush) { + flush_threshold_ = threshold; + on_flush_ = std::move(on_flush); + } + + void add(std::vector&& batch, PackComposeStats* stats = nullptr) { + if (batch.empty()) { + return; + } + if (stats != nullptr) { + stats->sink_batches.fetch_add(1, std::memory_order_relaxed); + } + models_total_ += batch.size(); + in_ram_models_ += batch.size(); + + std::array< + std::unordered_map< + binary::codec::TerrSectionKey, + std::vector, + binary::codec::TerrSectionKeyHash>, + PackSectionAccumulator::kShardCount> + staged {}; + for (runtime::RuntimeModelInstance& model : batch) { + const binary::codec::TerrSectionKey section_key = + binary::codec::clampTerrSectionKey(binary::codec::terrSectionKey(model.location)); + const std::size_t shard = + binary::codec::TerrSectionKeyHash {}(section_key) % + PackSectionAccumulator::kShardCount; + staged[shard][section_key].push_back(std::move(model)); + } + batch.clear(); + + if (thread_safe_) { + std::lock_guard lock(mutex_); + mergeStaged(staged); + maybe_flush(); + } else { + mergeStaged(staged); + maybe_flush(); + } + } + + void finalizeSort() { + if (batches_ == nullptr || batches_->empty()) { + return; + } + if (thread_safe_) { + std::lock_guard lock(mutex_); + sortBatches(); + } else { + sortBatches(); + } + } + + [[nodiscard]] std::size_t modelsTotal() const noexcept { + return models_total_; + } + + [[nodiscard]] std::size_t flushedTotal() const noexcept { + return flushed_total_; + } + + // Spłucz wszystkie sekcje PACK z RAM (np. po kazdym pliku w compose). + void flushAll() { + if (!on_flush_ || batches_ == nullptr || batches_->empty()) { + return; + } + if (thread_safe_) { + std::lock_guard lock(mutex_); + flushAllUnlocked(); + } else { + flushAllUnlocked(); + } + } + + private: + void flushAllUnlocked() { + if (!on_flush_ || batches_->empty()) { + return; + } + std::vector chunk; + chunk.reserve(in_ram_models_); + for (binary::codec::ModelSectionBatch& batch : *batches_) { + chunk.insert( + chunk.end(), + std::make_move_iterator(batch.models.begin()), + std::make_move_iterator(batch.models.end())); + batch.models.clear(); + } + batches_->clear(); + section_index_.clear(); + flushed_total_ += in_ram_models_; + in_ram_models_ = 0; + on_flush_(std::move(chunk)); + } + + void rebuildSectionIndex() { + section_index_.clear(); + for (std::size_t index = 0; index < batches_->size(); ++index) { + section_index_.emplace(batches_->at(index).section, index); + } + } + + void maybe_flush() { + if (flush_threshold_ == 0 || !on_flush_ || in_ram_models_ <= flush_threshold_) { + return; + } + while (in_ram_models_ > flush_threshold_ && !batches_->empty()) { + std::size_t best = 0; + for (std::size_t index = 1; index < batches_->size(); ++index) { + if (batches_->at(index).models.size() > batches_->at(best).models.size()) { + best = index; + } + } + binary::codec::ModelSectionBatch& batch = batches_->at(best); + const std::size_t flushed_count = batch.models.size(); + in_ram_models_ -= flushed_count; + flushed_total_ += flushed_count; + std::vector chunk = std::move(batch.models); + batches_->erase(batches_->begin() + static_cast(best)); + rebuildSectionIndex(); + on_flush_(std::move(chunk)); + } + } + + void sortBatches() { + std::sort( + batches_->begin(), + batches_->end(), + [](const binary::codec::ModelSectionBatch& a, + const binary::codec::ModelSectionBatch& b) noexcept { + if (a.section.z != b.section.z) { + return a.section.z < b.section.z; + } + return a.section.x < b.section.x; + }); + } + + void mergeStaged( + std::array< + std::unordered_map< + binary::codec::TerrSectionKey, + std::vector, + binary::codec::TerrSectionKeyHash>, + PackSectionAccumulator::kShardCount>& staged) { + for (unsigned shard = 0; shard < PackSectionAccumulator::kShardCount; ++shard) { + for (auto& [section_key, models] : staged[shard]) { + const auto found = section_index_.find(section_key); + if (found == section_index_.end()) { + binary::codec::ModelSectionBatch batch; + batch.section = section_key; + batch.models = std::move(models); + batches_->push_back(std::move(batch)); + section_index_.emplace(section_key, batches_->size() - 1); + } else { + std::vector& target = + batches_->at(found->second).models; + target.insert( + target.end(), + std::make_move_iterator(models.begin()), + std::make_move_iterator(models.end())); + } + } + } + } + + std::vector* batches_ = nullptr; + std::unordered_map< + binary::codec::TerrSectionKey, + std::size_t, + binary::codec::TerrSectionKeyHash> + section_index_; + std::mutex mutex_; + bool const thread_safe_ = false; + std::size_t models_total_ = 0; + std::size_t in_ram_models_ = 0; + std::size_t flushed_total_ = 0; + std::size_t flush_threshold_ = 0; + std::function&&)> on_flush_; +}; + struct PackComposeWorkItem { std::filesystem::path path; std::string relativeFile; @@ -577,6 +1598,11 @@ struct PackIncludeFoldPlan { if (stats != nullptr) { stats->instantiate_fast.fetch_add(1, std::memory_order_relaxed); } + // Copy is unavoidable when callers mutate/sink owns the batch, but avoid + // doubling peak RAM on empty templates (common for .inc placement chains). + if (entry.base_models.empty()) { + return {}; + } return entry.base_models; } if (stats != nullptr) { @@ -602,12 +1628,14 @@ public: const unsigned max_threads, PackComposeProgress* progress, const std::filesystem::path& scenery_root, - PackComposeStats* stats = nullptr) + PackComposeStats* stats = nullptr, + BakeParseSessionCache* session_cache = nullptr) : max_threads_(resolveComposeThreadCount(max_threads)) , gate_(max_threads_) , progress_(progress) , scenery_root_(scenery_root) - , stats_(stats) {} + , stats_(stats) + , session_cache_(session_cache) {} void seedCache(const std::filesystem::path& resolved, PackFileCacheEntry entry) { const std::string key = resolved.lexically_normal().generic_string(); @@ -625,18 +1653,24 @@ public: const std::vector& includeStack, const runtime::TransformContext& includePrefix, const std::span includeParameters, - PackModelBatchSink& sink) { + PackModelBatchSink& sink, + std::function* after_file_done = nullptr) { std::mutex queue_mutex; std::deque queue; std::condition_variable queue_cv; std::condition_variable done_cv; std::atomic pending_tasks { 0 }; std::atomic stop_workers { false }; + std::atomic failed { false }; std::exception_ptr first_error; std::mutex error_mutex; auto submit = [&](PackComposeWorkItem item) { - pending_tasks.fetch_add(1, std::memory_order_acq_rel); + std::size_t const now_pending { + pending_tasks.fetch_add(1, std::memory_order_acq_rel) + 1 }; + if (stats_ != nullptr) { + stats_->diag_pending.store(now_pending, std::memory_order_relaxed); + } std::size_t queue_size = 0; { std::lock_guard lock(queue_mutex); @@ -654,7 +1688,17 @@ public: }; auto finish_task = [&]() { - if (pending_tasks.fetch_sub(1, std::memory_order_acq_rel) == 1) { + // The completion notification must be issued under queue_mutex: the + // main thread waits on done_cv with that mutex and re-checks the + // atomic predicate, so notifying without the lock can drop the final + // wakeup (deadlock with all threads idle). Callers must NOT already + // hold queue_mutex. + std::size_t const prev { pending_tasks.fetch_sub(1, std::memory_order_acq_rel) }; + if (stats_ != nullptr) { + stats_->diag_pending.store(prev - 1, std::memory_order_relaxed); + } + if (prev == 1) { + std::lock_guard lock(queue_mutex); done_cv.notify_all(); } }; @@ -662,8 +1706,26 @@ public: auto process_one = [&](PackComposeWorkItem work) { bool folded_hop = false; std::size_t task_fold_steps = 0; + auto release_visited_pack = [&](const std::filesystem::path& file) { + if (session_cache_ != nullptr) { + session_cache_->releasePackCacheAfterComposeUse(file); + return; + } + if (eu07::scene::detail::isIncFile(file.filename().string())) { + return; + } + const std::string key = file.lexically_normal().generic_string(); + std::unique_lock lock(cache_mutex_); + if (const auto found = cache_.find(key); found != cache_.end()) { + found->second.base_models.clear(); + found->second.base_models.shrink_to_fit(); + } + }; while (true) { const std::filesystem::path resolved = work.path.lexically_normal(); + // Surface the file we are about to process so a long (or stuck) + // compose shows which scenery file it is currently working on. + reportActiveFile(resolved); if (eu07::scene::detail::isIncludeCycle(resolved, work.includeStack)) { if (stats_ != nullptr) { stats_->cycles_skipped.fetch_add(1, std::memory_order_relaxed); @@ -671,7 +1733,9 @@ public: return; } - if (!std::filesystem::exists(resolved)) { + if (const PackFileCacheEntry* known = tryGetCached(resolved)) { + (void)known; + } else if (!std::filesystem::exists(resolved)) { if (work.includeStack.empty()) { throw std::runtime_error("EU7 PACK: brak pliku: " + resolved.string()); } @@ -777,24 +1841,27 @@ public: std::vector placement_params = start_inc.parameters; while (true) { - const std::filesystem::path inc_path = - eu07::scene::detail::resolveIncludeSourcePath( - scenery_root_, relative_file, step->file) - .lexically_normal(); + // Single memoized lookup yields the resolved path AND the + // parsed entry (or a missing marker) behind one shared lock, + // avoiding the per-placement resolve/stat/probe storm. + const ResolvedInclude& resolved_inc = + resolveIncludeCached(relative_file, step->file); + const std::filesystem::path& inc_path = resolved_inc.path; if (is_inc_walk_cycle(inc_path, suffix)) { if (stats_ != nullptr) { stats_->cycles_skipped.fetch_add(1, std::memory_order_relaxed); } return; } - if (!std::filesystem::exists(inc_path)) { + if (resolved_inc.missing || resolved_inc.entry == nullptr) { if (stats_ != nullptr) { stats_->missing_skipped.fetch_add(1, std::memory_order_relaxed); } return; } + noteCacheHit(); - const PackFileCacheEntry& inc_entry = loadCached(inc_path); + const PackFileCacheEntry& inc_entry = *resolved_inc.entry; if (!inc_entry.placement.empty()) { work.pending_placement = inc_entry.placement; work.pending_placement_parameters = placement_params; @@ -809,8 +1876,7 @@ public: } note_inc_inline(inc_path); - const std::string inc_relative = - eu07::scene::detail::relativeSceneryFile(scenery_root_, inc_path); + const std::string& inc_relative = resolved_inc.relative; std::vector non_inc; non_inc.reserve(inc_entry.includes.size()); const ParsedInclude* next_inc = nullptr; @@ -907,8 +1973,10 @@ public: if (!models.empty()) { sink(std::move(models)); } + release_visited_pack(resolved); return; } + release_visited_pack(resolved); work.includeStack.push_back(resolved); work.path = next_path; work.relativeFile = @@ -931,6 +1999,7 @@ public: if (!models.empty()) { sink(std::move(models)); } + release_visited_pack(resolved); return; } continue; @@ -945,19 +2014,32 @@ public: if (!models.empty()) { sink(std::move(models)); } + release_visited_pack(resolved); return; } }; auto worker_loop = [&]() { + if (stats_ != nullptr) { + stats_->diag_workers_alive.fetch_add(1, std::memory_order_relaxed); + } while (true) { PackComposeWorkItem work; { std::unique_lock lock(queue_mutex); + if (stats_ != nullptr) { + stats_->diag_workers_idle.fetch_add(1, std::memory_order_relaxed); + } queue_cv.wait(lock, [&]() { return stop_workers.load(std::memory_order_acquire) || !queue.empty(); }); + if (stats_ != nullptr) { + stats_->diag_workers_idle.fetch_sub(1, std::memory_order_relaxed); + } if (stop_workers.load(std::memory_order_acquire) && queue.empty()) { + if (stats_ != nullptr) { + stats_->diag_workers_alive.fetch_sub(1, std::memory_order_relaxed); + } return; } if (queue.empty()) { @@ -967,22 +2049,42 @@ public: queue.pop_front(); } + if (stats_ != nullptr) { + stats_->diag_workers_inst.fetch_add(1, std::memory_order_relaxed); + } try { process_one(std::move(work)); + if (after_file_done != nullptr && *after_file_done) { + (*after_file_done)(); + } + if (stats_ != nullptr) { + stats_->diag_workers_inst.fetch_sub(1, std::memory_order_relaxed); + } } catch (...) { + if (stats_ != nullptr) { + stats_->diag_workers_inst.fetch_sub(1, std::memory_order_relaxed); + } { std::lock_guard lock(error_mutex); if (!first_error) { first_error = std::current_exception(); } } + failed.store(true, std::memory_order_release); stop_workers.store(true, std::memory_order_release); queue_cv.notify_all(); + { + std::lock_guard lock(queue_mutex); + done_cv.notify_all(); + } } finish_task(); } }; + if (stats_ != nullptr) { + stats_->diag_gate_max.store(gate_.maxSlots(), std::memory_order_relaxed); + } std::vector workers; workers.reserve(max_threads_); for (unsigned worker = 0; worker < max_threads_; ++worker) { @@ -999,17 +2101,26 @@ public: { std::unique_lock lock(queue_mutex); + if (stats_ != nullptr) { + stats_->diag_main_wait.store(1, std::memory_order_relaxed); + } done_cv.wait(lock, [&]() { - return pending_tasks.load(std::memory_order_acquire) == 0 || first_error != nullptr; + return pending_tasks.load(std::memory_order_acquire) == 0 || + failed.load(std::memory_order_acquire); }); + if (stats_ != nullptr) { + stats_->diag_main_wait.store(0, std::memory_order_relaxed); + } } stop_workers.store(true, std::memory_order_release); { + // Already holding queue_mutex here, so decrement directly instead of + // finish_task() (which would re-lock queue_mutex and deadlock). std::lock_guard lock(queue_mutex); while (!queue.empty()) { queue.pop_front(); - finish_task(); + pending_tasks.fetch_sub(1, std::memory_order_acq_rel); } } queue_cv.notify_all(); @@ -1034,15 +2145,89 @@ public: private: [[nodiscard]] PackFileCacheEntry buildCacheEntry(const std::filesystem::path& resolved) { + const auto add_us = [&](std::atomic& dst, + const std::chrono::steady_clock::time_point a, + const std::chrono::steady_clock::time_point b) { + if (stats_ != nullptr) { + dst.fetch_add( + static_cast( + std::chrono::duration_cast(b - a).count()), + std::memory_order_relaxed); + } + }; const auto parse_begin = std::chrono::steady_clock::now(); - const ParseResult parsed = parseFile(resolved); + + if (eu07::scene::detail::shouldStreamFlatSourceFile(resolved)) { + const eu07::scene::detail::FlatFileKind kind = + eu07::scene::detail::scanFlatFileKindStreaming(resolved); + if (kind == eu07::scene::detail::FlatFileKind::Models) { + PackFileCacheEntry entry; + eu07::scene::detail::streamFlatFileModels( + resolved, [&](const ParsedNodeModel& parsed) { + entry.base_models.push_back(bakeModel(parsed)); + }); + const auto parse_end = std::chrono::steady_clock::now(); + if (stats_ != nullptr) { + add_us(stats_->process_us, parse_begin, parse_end); + stats_->parse_us.fetch_add( + static_cast( + std::chrono::duration_cast( + parse_end - parse_begin) + .count()), + std::memory_order_relaxed); + } + return entry; + } + if (kind == eu07::scene::detail::FlatFileKind::Includes) { + if (const std::optional doc = + eu07::scene::detail::buildFlatIncludesDocumentStreaming(resolved)) { + const auto t_proc = std::chrono::steady_clock::now(); + PackFileCacheEntry entry = makePackFileCacheEntry(*doc, max_threads_); + const auto parse_end = std::chrono::steady_clock::now(); + if (stats_ != nullptr) { + add_us(stats_->process_us, parse_begin, t_proc); + add_us(stats_->makeentry_us, t_proc, parse_end); + stats_->parse_us.fetch_add( + static_cast( + std::chrono::duration_cast( + parse_end - parse_begin) + .count()), + std::memory_order_relaxed); + } + return entry; + } + } + throw std::runtime_error( + "EU7: duzy plik nie-plaski — nie mozna zaladowac calego do RAM: " + + resolved.string()); + } + + eu07::RawFile raw = readRawFile(resolved); SceneProcessOptions options; options.expandIncludes = false; - const SceneDocument document = - processScene(parsed, scenery_root_, options).document; - PackFileCacheEntry entry = makePackFileCacheEntry(document); + SceneDocument document; + if (std::optional flat = + eu07::scene::detail::tryProcessFlatSceneFile(raw, scenery_root_)) { + document = std::move(*flat); + const auto t_proc = std::chrono::steady_clock::now(); + if (stats_ != nullptr) { + add_us(stats_->process_us, parse_begin, t_proc); + } + } else { + const ParseResult parsed = parseRawFile(std::move(raw)); + const auto t_tok = std::chrono::steady_clock::now(); + document = processScene(parsed, scenery_root_, options).document; + const auto t_proc = std::chrono::steady_clock::now(); + if (stats_ != nullptr) { + add_us(stats_->tokenize_us, parse_begin, t_tok); + add_us(stats_->process_us, t_tok, t_proc); + } + } + const auto t_proc = std::chrono::steady_clock::now(); + PackFileCacheEntry entry = makePackFileCacheEntry(document, max_threads_); + const auto parse_end = std::chrono::steady_clock::now(); if (stats_ != nullptr) { - const auto parse_end = std::chrono::steady_clock::now(); + add_us(stats_->makeentry_us, t_proc, parse_end); stats_->parse_us.fetch_add( static_cast( std::chrono::duration_cast(parse_end - parse_begin) @@ -1052,7 +2237,98 @@ private: return entry; } + // Lock-light cache probe used by the placement hot path. A hit proves the + // file exists and is parsed, so the caller can skip the per-placement + // filesystem::exists() syscall and the parse gate entirely. References into + // cache_ stay valid because entries are never erased (node-stable map). + // Fully resolved include reference: the normalized path plus the parsed cache + // entry (or a "missing" marker). Caching this per (currentRelativeFile, + // fileToken) collapses the flora placement hot path (~243k repeats of the same + // few .inc templates) into a single map lookup behind one shared lock — instead + // of re-running resolveIncludeSourcePath (which stats the disk), a separate + // exists() syscall, and the parse-cache probe on every placement. + struct ResolvedInclude { + std::filesystem::path path; + std::string relative; // relativeSceneryFile(scenery_root_, path), memoized + const PackFileCacheEntry* entry = nullptr; + bool missing = false; + }; + + // Returns a stable reference into resolve_cache_ (node-stable map, never erased), + // so callers may hold it freely. Resolution and parsing are deterministic for a + // fixed scenery on disk, so memoizing is safe. + [[nodiscard]] const ResolvedInclude& resolveIncludeCached( + const std::string& currentRelativeFile, const std::string& fileToken) { + std::string key; + key.reserve(currentRelativeFile.size() + fileToken.size() + 1); + key.append(currentRelativeFile); + key.push_back('\x1f'); + key.append(fileToken); + { + std::shared_lock read_lock(resolve_cache_mutex_); + if (const auto found = resolve_cache_.find(key); found != resolve_cache_.end()) { + return found->second; + } + } + ResolvedInclude resolved; + resolved.path = eu07::scene::detail::resolveIncludeSourcePath( + scenery_root_, currentRelativeFile, fileToken) + .lexically_normal(); + if (const PackFileCacheEntry* cached = tryGetCached(resolved.path)) { + resolved.entry = cached; + } else if (!std::filesystem::exists(resolved.path)) { + resolved.missing = true; + } else { + resolved.entry = &loadCached(resolved.path); + } + if (!resolved.missing) { + resolved.relative = + eu07::scene::detail::relativeSceneryFile(scenery_root_, resolved.path); + } + std::unique_lock write_lock(resolve_cache_mutex_); + const auto [it, inserted] = resolve_cache_.emplace(std::move(key), std::move(resolved)); + return it->second; + } + + [[nodiscard]] const PackFileCacheEntry* tryGetCached(const std::filesystem::path& resolved) { + if (session_cache_ != nullptr) { + return session_cache_->tryPackEntry(resolved); + } + const std::string key = resolved.generic_string(); + std::shared_lock read_lock(cache_mutex_); + if (const auto found = cache_.find(key); found != cache_.end()) { + return &found->second; + } + return nullptr; + } + [[nodiscard]] const PackFileCacheEntry& loadCached(const std::filesystem::path& resolved) { + if (session_cache_ != nullptr) { + if (const PackFileCacheEntry* cached = session_cache_->tryPackEntry(resolved)) { + noteCacheHit(); + return *cached; + } + const ComposeConcurrencyGate::Scope gate_scope = gate_.scoped(); + if (stats_ != nullptr) { + stats_->diag_gate_active.store(gate_.activeSlots(), std::memory_order_relaxed); + stats_->diag_gate_waiting.store(gate_.waiting(), std::memory_order_relaxed); + } + if (const PackFileCacheEntry* cached = session_cache_->tryPackEntry(resolved)) { + noteCacheHit(); + return *cached; + } + if (stats_ != nullptr) { + stats_->diag_workers_parse.fetch_add(1, std::memory_order_relaxed); + } + const PackFileCacheEntry& built = + session_cache_->packEntryFor(resolved, scenery_root_, stats_); + if (stats_ != nullptr) { + stats_->diag_workers_parse.fetch_sub(1, std::memory_order_relaxed); + } + noteCacheHit(); + return built; + } + const std::string key = resolved.generic_string(); { std::shared_lock read_lock(cache_mutex_); @@ -1063,6 +2339,10 @@ private: } const ComposeConcurrencyGate::Scope gate_scope = gate_.scoped(); + if (stats_ != nullptr) { + stats_->diag_gate_active.store(gate_.activeSlots(), std::memory_order_relaxed); + stats_->diag_gate_waiting.store(gate_.waiting(), std::memory_order_relaxed); + } { std::shared_lock read_lock(cache_mutex_); if (const auto found = cache_.find(key); found != cache_.end()) { @@ -1071,7 +2351,13 @@ private: } } + if (stats_ != nullptr) { + stats_->diag_workers_parse.fetch_add(1, std::memory_order_relaxed); + } PackFileCacheEntry entry = buildCacheEntry(resolved); + if (stats_ != nullptr) { + stats_->diag_workers_parse.fetch_sub(1, std::memory_order_relaxed); + } std::unique_lock write_lock(cache_mutex_); const auto [it, inserted] = cache_.emplace(key, std::move(entry)); @@ -1100,6 +2386,32 @@ private: } } + // Reports the file currently entering compose (no counter mutation), so the + // active scenery file is visible even before its models are instantiated. + // Lightly throttled to avoid flooding on dense include trees. + void reportActiveFile(const std::filesystem::path& file) { + if (progress_ == nullptr || progress_->on_progress == nullptr) { + return; + } + const auto now = std::chrono::steady_clock::now(); + std::lock_guard lock(progress_mutex_); + const bool timed = + !progress_->reported_once || + std::chrono::duration_cast(now - progress_->last_report) + .count() >= 250; + if (!timed) { + return; + } + progress_->reported_once = true; + progress_->last_report = now; + progress_->on_progress( + file, + stats_ != nullptr ? stats_->files_visited.load(std::memory_order_relaxed) + : files_visited_.load(std::memory_order_relaxed), + stats_ != nullptr ? stats_->models_emitted.load(std::memory_order_relaxed) + : models_total_.load(std::memory_order_relaxed)); + } + void reportProgress(const std::filesystem::path& file, const std::size_t local_models) { const std::size_t files_visited = files_visited_.fetch_add(1, std::memory_order_relaxed) + 1; const std::size_t models_total = @@ -1137,8 +2449,16 @@ private: PackComposeProgress* progress_; std::filesystem::path scenery_root_; PackComposeStats* stats_ = nullptr; + BakeParseSessionCache* session_cache_ = nullptr; std::shared_mutex cache_mutex_; std::unordered_map cache_; + std::shared_mutex resolve_cache_mutex_; + // Memoized include-path resolution keyed on (currentRelativeFile, fileToken). + // The flora placement hot path re-resolves the same .inc template hundreds of + // thousands of times; resolveIncludeSourcePath stats the filesystem and builds + // paths, so caching avoids ~243k redundant syscalls + allocations. Resolution + // is deterministic for a fixed scenery on disk, so memoizing is safe. + std::unordered_map resolve_cache_; std::mutex progress_mutex_; std::atomic files_visited_ { 0 }; std::atomic models_total_ { 0 }; @@ -1148,7 +2468,9 @@ private: } // namespace detail +using detail::BakeParseSessionCache; using detail::makePackFileCacheEntry; +using detail::printBakeParseSessionStats; [[nodiscard]] inline std::size_t composePackModels( const std::filesystem::path& path, @@ -1160,8 +2482,12 @@ using detail::makePackFileCacheEntry; const std::span includeParameters = {}) { const auto compose_begin = std::chrono::steady_clock::now(); detail::PackComposeSession session( - options.max_threads, options.progress, sceneryRoot, options.stats); - if (options.root_seed != nullptr) { + options.max_threads, + options.progress, + sceneryRoot, + options.stats, + options.session_cache); + if (options.session_cache == nullptr && options.root_seed != nullptr) { session.seedCache(options.root_seed->path, options.root_seed->entry); } @@ -1173,31 +2499,84 @@ using detail::makePackFileCacheEntry; if (options.stats != nullptr) { options.stats->threads.store(thread_count, std::memory_order_relaxed); } - std::vector thread_accumulators(thread_count); - std::atomic next_worker_id { 0 }; - - detail::PackModelBatchSink sink { - [&thread_accumulators, &next_worker_id, stats = options.stats]( - std::vector&& batch) { - thread_local unsigned worker_index = static_cast(-1); - if (worker_index == static_cast(-1)) { - worker_index = next_worker_id.fetch_add(1, std::memory_order_relaxed); - } - if (worker_index < thread_accumulators.size()) { - thread_accumulators[worker_index].add(std::move(batch), stats); - } else { - thread_accumulators[worker_index % thread_accumulators.size()].add( - std::move(batch), stats); - } - } }; - session.runCompose(path, relativeFile, includeStack, includePrefix, includeParameters, sink); const auto finalize_begin = std::chrono::steady_clock::now(); - detail::PackSectionAccumulator merged; - for (detail::PackSectionAccumulator& accumulator : thread_accumulators) { - merged.mergeFrom(std::move(accumulator)); + std::size_t models_total = 0; + + if (options.low_memory) { + detail::PackBatchDirectBuilder direct_builder( + options.pack_batches, thread_count > 1); + std::function flush_after_file; + if (options.on_pack_models_flush) { + if (options.pack_flush_per_file) { + direct_builder.set_flush(0, options.on_pack_models_flush); + flush_after_file = [&direct_builder]() { direct_builder.flushAll(); }; + } else if (options.pack_flush_threshold != 0) { + direct_builder.set_flush( + options.pack_flush_threshold, options.on_pack_models_flush); + } + } + detail::PackModelBatchSink sink { + [&direct_builder, stats = options.stats]( + std::vector&& batch) { + const auto sink_begin = std::chrono::steady_clock::now(); + direct_builder.add(std::move(batch), stats); + if (stats != nullptr) { + stats->sink_us.fetch_add( + static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - sink_begin) + .count()), + std::memory_order_relaxed); + } + } }; + session.runCompose( + path, + relativeFile, + includeStack, + includePrefix, + includeParameters, + sink, + options.pack_flush_per_file && flush_after_file ? &flush_after_file : nullptr); + direct_builder.finalizeSort(); + models_total = direct_builder.modelsTotal(); + } else { + std::vector thread_accumulators(thread_count); + std::atomic next_worker_id { 0 }; + detail::PackModelBatchSink sink { + [&thread_accumulators, &next_worker_id, stats = options.stats]( + std::vector&& batch) { + thread_local unsigned worker_index = static_cast(-1); + if (worker_index == static_cast(-1)) { + worker_index = next_worker_id.fetch_add(1, std::memory_order_relaxed); + } + const auto sink_begin = std::chrono::steady_clock::now(); + if (worker_index < thread_accumulators.size()) { + thread_accumulators[worker_index].add(std::move(batch), stats); + } else { + thread_accumulators[worker_index % thread_accumulators.size()].add( + std::move(batch), stats); + } + if (stats != nullptr) { + stats->sink_us.fetch_add( + static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - sink_begin) + .count()), + std::memory_order_relaxed); + } + } }; + session.runCompose( + path, relativeFile, includeStack, includePrefix, includeParameters, sink); + + detail::PackSectionAccumulator merged; + for (detail::PackSectionAccumulator& accumulator : thread_accumulators) { + merged.mergeFrom(std::move(accumulator)); + } + *options.pack_batches = merged.finalize(); + models_total = merged.modelsTotal(); } - *options.pack_batches = merged.finalize(); + const auto compose_end = std::chrono::steady_clock::now(); if (options.stats != nullptr) { @@ -1213,10 +2592,10 @@ using detail::makePackFileCacheEntry; .count()), std::memory_order_relaxed); if (options.stats->models_emitted.load(std::memory_order_relaxed) == 0) { - options.stats->models_emitted.store(merged.modelsTotal(), std::memory_order_relaxed); + options.stats->models_emitted.store(models_total, std::memory_order_relaxed); } } - return merged.modelsTotal(); + return models_total; } } // namespace eu07::scene::bake diff --git a/eu07-parser/include/eu07/scene/bake/pack_model_spool.hpp b/eu07-parser/include/eu07/scene/bake/pack_model_spool.hpp new file mode 100644 index 00000000..d3ef8cc5 --- /dev/null +++ b/eu07-parser/include/eu07/scene/bake/pack_model_spool.hpp @@ -0,0 +1,138 @@ +#pragma once + +// Append-only spool: modele spłukane z PACK compose (mniejszy peak RAM). + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace eu07::scene::bake { + +inline constexpr std::size_t kSpoolStreamBufferBytes = 1024u * 1024u; + +class PackModelSpoolFile { +public: + explicit PackModelSpoolFile(std::filesystem::path path) + : path_(std::move(path)), + out_(path_, std::ios::binary | std::ios::trunc) { + if (!out_) { + throw std::runtime_error("EU7 PACK spool: nie mozna utworzyc " + path_.string()); + } + out_.rdbuf()->pubsetbuf(write_buffer_.data(), write_buffer_.size()); + } + + void append(std::vector&& models) { + if (models.empty()) { + return; + } + std::lock_guard lock(mutex_); + for (runtime::RuntimeModelInstance& model : models) { + binary::detail::collectModelStrings(table_, model); + binary::detail::writeRuntimeModel(out_, table_, model); + ++model_count_; + } + } + + template + void for_each_model(Fn&& fn) const { + { + std::lock_guard lock(mutex_); + out_.flush(); + } + std::ifstream in(path_, std::ios::binary); + if (!in) { + return; + } + while (in.peek() != EOF) { + fn(binary::detail::readRuntimeModel(in, table_)); + } + } + + [[nodiscard]] std::size_t model_count() const noexcept { + return model_count_; + } + + [[nodiscard]] const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; + binary::StringTable table_; + std::size_t model_count_ = 0; + std::array write_buffer_ {}; + mutable std::ofstream out_; + mutable std::mutex mutex_; +}; + +class ShapeSpoolFile { +public: + explicit ShapeSpoolFile(std::filesystem::path path) + : path_(std::move(path)), + out_(path_, std::ios::binary | std::ios::trunc) { + if (!out_) { + throw std::runtime_error("EU7 shape spool: nie mozna utworzyc " + path_.string()); + } + out_.rdbuf()->pubsetbuf(write_buffer_.data(), write_buffer_.size()); + } + + void append(runtime::RuntimeShapeNode&& shape) { + std::lock_guard lock(mutex_); + binary::detail::collectShapeStrings(table_, shape); + binary::detail::writeRuntimeShape(out_, table_, shape); + ++shape_count_; + } + + void append_batch(std::vector&& shapes) { + if (shapes.empty()) { + return; + } + std::lock_guard lock(mutex_); + for (runtime::RuntimeShapeNode& shape : shapes) { + binary::detail::collectShapeStrings(table_, shape); + binary::detail::writeRuntimeShape(out_, table_, shape); + ++shape_count_; + } + } + + template + void for_each_shape(Fn&& fn) const { + { + std::lock_guard lock(mutex_); + out_.flush(); + } + std::ifstream in(path_, std::ios::binary); + if (!in) { + return; + } + while (in.peek() != EOF) { + fn(binary::detail::readRuntimeShape(in, table_)); + } + } + + [[nodiscard]] std::size_t shape_count() const noexcept { + return shape_count_; + } + + [[nodiscard]] const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; + binary::StringTable table_; + std::size_t shape_count_ = 0; + std::array write_buffer_ {}; + mutable std::ofstream out_; + mutable std::mutex mutex_; +}; + +} // namespace eu07::scene::bake diff --git a/eu07-parser/include/eu07/scene/bake/streaming_terrain.hpp b/eu07-parser/include/eu07/scene/bake/streaming_terrain.hpp new file mode 100644 index 00000000..65254ffa --- /dev/null +++ b/eu07-parser/include/eu07/scene/bake/streaming_terrain.hpp @@ -0,0 +1,432 @@ +#pragma once + +// Duze pliki terenu (same node triangles, np. nmt100_warmaz_ter.scm): +// linia po linii, parse per-wezel, bez readRawFile / tokenizacji calego pliku. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace eu07::scene::bake::detail { + +enum class TriangleTerrainLineKind { + Skip, + Header, + Vertex, + EndTri, + Invalid, +}; + +[[nodiscard]] inline TriangleTerrainLineKind classifyTriangleTerrainLine(std::string_view text) { + skipFieldSeparators(text); + if (text.empty()) { + return TriangleTerrainLineKind::Skip; + } + if (eu07::detail::isLineComment(text)) { + return TriangleTerrainLineKind::Skip; + } + + if (eu07::scene::detail::segmentStartsWithKeyword(text, "node")) { + if (text.find("triangles") == std::string_view::npos) { + return TriangleTerrainLineKind::Invalid; + } + return TriangleTerrainLineKind::Header; + } + + if (isKeyword(text, "endtri") || isKeyword(text, "endtriangles")) { + return TriangleTerrainLineKind::EndTri; + } + + if (!text.empty() && + (text.front() == '-' || text.front() == '+' || + (text.front() >= '0' && text.front() <= '9'))) { + return TriangleTerrainLineKind::Vertex; + } + + return TriangleTerrainLineKind::Invalid; +} + +[[nodiscard]] inline bool parseTriangleTerrainBlock( + const std::vector& lines, + const std::size_t header_line, + ParsedNodeTriangles& out) { + if (lines.size() < 2) { + return false; + } + + std::vector header_tokens; + tokenizeInto(lines.front(), header_tokens, header_line); + + TokenStream stream(header_tokens); + NodeHeader header; + std::string subtype; + std::vector raw; + if (!node::io::consumeHeader(stream, header, subtype, raw) || + !isKeyword(subtype, node_triangles::kSubtype) || + !node::io::takeString(stream, raw, out.texture)) { + return false; + } + + out.header = header; + out.raw.clear(); + out.vertices.clear(); + out.vertices.reserve(lines.size() > 2 ? lines.size() - 2 : 0); + + for (std::size_t i = 1; i + 1 < lines.size(); ++i) { + std::vector vertex_tokens; + tokenizeInto(lines[i], vertex_tokens, header_line + i); + + TokenStream vertex_stream(vertex_tokens); + MeshVertex vertex; + if (!node_triangles::takeVertex(vertex_stream, raw, vertex, !vertex_stream.empty())) { + return false; + } + out.vertices.push_back(vertex); + } + + return !out.vertices.empty(); +} + +struct TriangleTerrainBlock { + std::size_t header_line = 0; + std::vector lines; +}; + +constexpr std::size_t kTerrainParallelBatchSize = 8192; +constexpr std::size_t kTerrainParallelMinBlocks = 8; +constexpr std::size_t kTerrainPipelineMaxQueuedBatches = 2; + +[[nodiscard]] inline unsigned resolveTerrainThreadCount(const unsigned max_threads) { + const unsigned hw = + std::thread::hardware_concurrency() == 0 ? 4u : std::thread::hardware_concurrency(); + if (max_threads == 0) { + return std::max(1u, hw); + } + return std::max(1u, max_threads); +} + +[[nodiscard]] inline bool bakeTriangleTerrainBatch( + const std::vector& batch, + const unsigned max_threads, + std::vector& baked_out) { + baked_out.resize(batch.size()); + if (batch.empty()) { + return true; + } + if (batch.size() < kTerrainParallelMinBlocks || + resolveTerrainThreadCount(max_threads) <= 1) { + for (std::size_t i = 0; i < batch.size(); ++i) { + ParsedNodeTriangles parsed; + if (!parseTriangleTerrainBlock(batch[i].lines, batch[i].header_line, parsed)) { + return false; + } + baked_out[i] = bakeTriangles(parsed); + } + return true; + } + + const unsigned worker_count = resolveTerrainThreadCount(max_threads); + std::atomic next_index { 0 }; + std::atomic ok { true }; + + const auto worker = [&]() { + while (ok.load(std::memory_order_relaxed)) { + const std::size_t index = next_index.fetch_add(1, std::memory_order_relaxed); + if (index >= batch.size()) { + return; + } + try { + ParsedNodeTriangles parsed; + if (!parseTriangleTerrainBlock( + batch[index].lines, batch[index].header_line, parsed)) { + ok.store(false, std::memory_order_relaxed); + return; + } + baked_out[index] = bakeTriangles(parsed); + } catch (...) { + ok.store(false, std::memory_order_relaxed); + return; + } + } + }; + + std::vector threads; + threads.reserve(worker_count); + for (unsigned i = 0; i < worker_count; ++i) { + threads.emplace_back(worker); + } + for (std::thread& thread : threads) { + thread.join(); + } + return ok.load(std::memory_order_relaxed); +} + +[[nodiscard]] inline bool scanTriangleTerrainOnlyStreaming(const std::filesystem::path& path) { + bool saw_header = false; + bool in_block = false; + bool ok = true; + + eu07::scene::detail::forEachStreamingLogicalSegment( + path, [&](const LogicalSegment& segment) { + const TriangleTerrainLineKind kind = classifyTriangleTerrainLine(segment.text); + switch (kind) { + case TriangleTerrainLineKind::Skip: + return true; + case TriangleTerrainLineKind::Header: + saw_header = true; + in_block = true; + return true; + case TriangleTerrainLineKind::Vertex: + if (!in_block) { + ok = false; + return false; + } + return true; + case TriangleTerrainLineKind::EndTri: + if (!in_block) { + ok = false; + return false; + } + in_block = false; + return true; + case TriangleTerrainLineKind::Invalid: + default: + ok = false; + return false; + } + }); + + return ok && saw_header && !in_block; +} + +[[nodiscard]] inline bool scanTriangleTerrainBlocksStreaming( + const std::filesystem::path& path, + const std::function&&)>& on_batch) { + std::vector batch; + batch.reserve(kTerrainParallelBatchSize); + + TriangleTerrainBlock current_block; + bool in_block = false; + bool saw_header = false; + + const auto finish_block = [&]() -> bool { + if (current_block.lines.empty()) { + return true; + } + batch.push_back(std::move(current_block)); + current_block = {}; + in_block = false; + if (batch.size() >= kTerrainParallelBatchSize) { + if (!on_batch(std::move(batch))) { + return false; + } + batch = {}; + batch.reserve(kTerrainParallelBatchSize); + } + return true; + }; + + bool ok = true; + eu07::scene::detail::forEachStreamingLogicalSegment( + path, [&](const LogicalSegment& segment) { + const TriangleTerrainLineKind kind = classifyTriangleTerrainLine(segment.text); + switch (kind) { + case TriangleTerrainLineKind::Skip: + return true; + case TriangleTerrainLineKind::Header: + saw_header = true; + if (!finish_block()) { + ok = false; + return false; + } + current_block.header_line = segment.sourceLine; + current_block.lines.emplace_back(segment.text); + in_block = true; + return true; + case TriangleTerrainLineKind::Vertex: + if (!in_block) { + ok = false; + return false; + } + current_block.lines.emplace_back(segment.text); + return true; + case TriangleTerrainLineKind::EndTri: + if (!in_block) { + ok = false; + return false; + } + current_block.lines.emplace_back(segment.text); + if (!finish_block()) { + ok = false; + return false; + } + return true; + case TriangleTerrainLineKind::Invalid: + default: + ok = false; + return false; + } + }); + + if (ok && in_block) { + ok = finish_block(); + } + if (ok && !batch.empty()) { + ok = on_batch(std::move(batch)); + } + + return ok && saw_header; +} + +// Zwraca false gdy plik nie jest czystym terenem triangles (wtedy uzyj documentFor). +// Gdy shape_spool != nullptr, ksztalty trafiaja na dysk (module.scene.shapes puste). +[[nodiscard]] inline bool streamBakeTriangleTerrain( + const std::filesystem::path& path, + RuntimeModule& module, + ShapeSpoolFile* shape_spool = nullptr, + const unsigned max_threads = 0) { + module.scene.shapes.clear(); + + const unsigned bake_threads = resolveTerrainThreadCount(max_threads); + bool parsed_any = false; + + const auto emit_baked = [&](std::vector& baked) -> bool { + if (baked.empty()) { + return true; + } + parsed_any = true; + if (shape_spool != nullptr) { + shape_spool->append_batch(std::move(baked)); + } else { + module.scene.shapes.insert( + module.scene.shapes.end(), + std::make_move_iterator(baked.begin()), + std::make_move_iterator(baked.end())); + baked.clear(); + } + return true; + }; + + const auto bake_and_emit = [&](std::vector&& blocks) -> bool { + if (blocks.empty()) { + return true; + } + std::vector baked; + if (!bakeTriangleTerrainBatch(blocks, max_threads, baked)) { + return false; + } + return emit_baked(baked); + }; + + if (bake_threads <= 1) { + const bool ok = scanTriangleTerrainBlocksStreaming(path, bake_and_emit); + if (!ok || !parsed_any) { + module.scene.shapes.clear(); + return false; + } + return true; + } + + std::mutex queue_mutex; + std::condition_variable queue_cv; + std::deque> ready_batches; + std::atomic scan_ok { true }; + std::atomic scan_done { false }; + std::atomic bake_failed { false }; + + std::thread scanner([&]() { + try { + const bool ok = scanTriangleTerrainBlocksStreaming( + path, [&](std::vector&& blocks) -> bool { + if (bake_failed.load(std::memory_order_relaxed)) { + return false; + } + std::unique_lock lock(queue_mutex); + queue_cv.wait(lock, [&]() { + return bake_failed.load(std::memory_order_relaxed) || + ready_batches.size() < kTerrainPipelineMaxQueuedBatches; + }); + if (bake_failed.load(std::memory_order_relaxed)) { + return false; + } + ready_batches.push_back(std::move(blocks)); + lock.unlock(); + queue_cv.notify_one(); + return true; + }); + scan_ok.store(ok, std::memory_order_relaxed); + } catch (...) { + scan_ok.store(false, std::memory_order_relaxed); + bake_failed.store(true, std::memory_order_relaxed); + queue_cv.notify_all(); + } + scan_done.store(true, std::memory_order_release); + queue_cv.notify_all(); + }); + + while (true) { + std::vector batch; + { + std::unique_lock lock(queue_mutex); + queue_cv.wait(lock, [&]() { + return bake_failed.load(std::memory_order_relaxed) || + !ready_batches.empty() || + scan_done.load(std::memory_order_acquire); + }); + if (bake_failed.load(std::memory_order_relaxed)) { + break; + } + if (ready_batches.empty() && scan_done.load(std::memory_order_acquire)) { + break; + } + if (ready_batches.empty()) { + continue; + } + batch = std::move(ready_batches.front()); + ready_batches.pop_front(); + } + queue_cv.notify_one(); + + std::vector baked; + if (!bakeTriangleTerrainBatch(batch, max_threads, baked)) { + bake_failed.store(true, std::memory_order_relaxed); + queue_cv.notify_all(); + break; + } + if (!emit_baked(baked)) { + bake_failed.store(true, std::memory_order_relaxed); + queue_cv.notify_all(); + break; + } + } + + scanner.join(); + + if (bake_failed.load(std::memory_order_relaxed) || + !scan_ok.load(std::memory_order_relaxed) || !parsed_any) { + module.scene.shapes.clear(); + return false; + } + return true; +} + +} // namespace eu07::scene::bake::detail diff --git a/eu07-parser/include/eu07/scene/context.hpp b/eu07-parser/include/eu07/scene/context.hpp index 88ff1417..2ab80445 100644 --- a/eu07-parser/include/eu07/scene/context.hpp +++ b/eu07-parser/include/eu07/scene/context.hpp @@ -16,6 +16,8 @@ struct ParseContext { std::vector includeStack; std::vector> activeIncludeParameters; bool expandIncludes = true; + // PACK compose needs includes + models only; skip heavy geometry nodes (CityGML teren). + bool packComposeLightweight = false; }; using DirectiveParser = bool (*)(TokenStream& stream, ParseContext& context); diff --git a/eu07-parser/include/eu07/scene/cursor.hpp b/eu07-parser/include/eu07/scene/cursor.hpp index bca28442..bbaef5ea 100644 --- a/eu07-parser/include/eu07/scene/cursor.hpp +++ b/eu07-parser/include/eu07/scene/cursor.hpp @@ -23,8 +23,12 @@ public: return (*tokens_)[index_]; } - [[nodiscard]] SourceToken consume() { - SourceToken token = peek(); + // Returns a reference into the underlying token vector (which outlives the + // stream), so callers that only read a field (e.g. .sourceLine) or bind to + // const& pay no per-token std::string copy. Callers that need an owning + // copy still get one via copy-initialisation, exactly as before. + [[nodiscard]] const SourceToken& consume() { + const SourceToken& token = peek(); ++index_; return token; } diff --git a/eu07-parser/include/eu07/scene/document.hpp b/eu07-parser/include/eu07/scene/document.hpp index 6c5f352e..af12f0c9 100644 --- a/eu07-parser/include/eu07/scene/document.hpp +++ b/eu07-parser/include/eu07/scene/document.hpp @@ -106,4 +106,64 @@ struct SceneProcessResult { doc.test.size(); } +// Plaski plik flory: same wpisy node model, bez torow/siatek/include. +[[nodiscard]] inline bool isModelOnlyDocument(const SceneDocument& document) noexcept { + if (document.nodeModel.empty()) { + return false; + } + return document.nodeDynamic.empty() && document.nodeTrackNormal.empty() && + document.nodeTrackSwitch.empty() && document.nodeTrackRoad.empty() && + document.nodeTrackCross.empty() && document.nodeTrackOther.empty() && + document.nodeTraction.empty() && document.nodeTractionPower.empty() && + document.nodeTriangles.empty() && document.nodeTriangleStrip.empty() && + document.nodeTriangleFan.empty() && document.nodeLines.empty() && + document.nodeLineStrip.empty() && document.nodeLineLoop.empty() && + document.nodeMemcell.empty() && document.nodeEventlauncher.empty() && + document.nodeSound.empty() && document.include.empty() && document.unknown.empty(); +} + +// Drops parsed node payloads already converted to RuntimeModule / PackFileCacheEntry. +// Keeps include lists and lightweight directives. When release_models is false, +// nodeModel is retained for a pending PACK build (model-only flora child). +inline void releaseHeavySceneParseStorage(SceneDocument& document, bool const release_models) { + if (release_models) { + document.nodeModel.clear(); + document.nodeModel.shrink_to_fit(); + } + document.nodeTriangles.clear(); + document.nodeTriangles.shrink_to_fit(); + document.nodeTriangleStrip.clear(); + document.nodeTriangleStrip.shrink_to_fit(); + document.nodeTriangleFan.clear(); + document.nodeTriangleFan.shrink_to_fit(); + document.nodeLines.clear(); + document.nodeLines.shrink_to_fit(); + document.nodeLineStrip.clear(); + document.nodeLineStrip.shrink_to_fit(); + document.nodeLineLoop.clear(); + document.nodeLineLoop.shrink_to_fit(); + document.nodeTrackNormal.clear(); + document.nodeTrackNormal.shrink_to_fit(); + document.nodeTrackSwitch.clear(); + document.nodeTrackSwitch.shrink_to_fit(); + document.nodeTrackRoad.clear(); + document.nodeTrackRoad.shrink_to_fit(); + document.nodeTrackCross.clear(); + document.nodeTrackCross.shrink_to_fit(); + document.nodeTrackOther.clear(); + document.nodeTrackOther.shrink_to_fit(); + document.nodeTraction.clear(); + document.nodeTraction.shrink_to_fit(); + document.nodeTractionPower.clear(); + document.nodeTractionPower.shrink_to_fit(); + document.nodeMemcell.clear(); + document.nodeMemcell.shrink_to_fit(); + document.nodeEventlauncher.clear(); + document.nodeEventlauncher.shrink_to_fit(); + document.nodeSound.clear(); + document.nodeSound.shrink_to_fit(); + document.nodeDynamic.clear(); + document.nodeDynamic.shrink_to_fit(); +} + } // namespace eu07::scene diff --git a/eu07-parser/include/eu07/scene/node.hpp b/eu07-parser/include/eu07/scene/node.hpp index 7b10e63b..fc1ab3d8 100644 --- a/eu07-parser/include/eu07/scene/node.hpp +++ b/eu07-parser/include/eu07/scene/node.hpp @@ -51,6 +51,18 @@ inline void applyScratchContext(NodeHeader& header, const SceneScratchpad& scrat return false; } + if (context.packComposeLightweight) { + std::vector raw; + while (!stream.empty() && !io::atEnd(stream, kind->subtype, kind->endMarker)) { + stream.consume(); + } + if (!io::consumeEnd(stream, raw, kind->subtype, kind->endMarker)) { + stream.rewind(anchor); + return false; + } + return true; + } + if (!kind->parseAndStore(stream, context.document, header, nullptr)) { stream.rewind(anchor); return false; diff --git a/eu07-parser/include/eu07/scene/node/mesh.hpp b/eu07-parser/include/eu07/scene/node/mesh.hpp index 848d8b30..37ce40bc 100644 --- a/eu07-parser/include/eu07/scene/node/mesh.hpp +++ b/eu07-parser/include/eu07/scene/node/mesh.hpp @@ -30,6 +30,15 @@ template return false; } + // Rezerwacja z gornego oszacowania (>=8 tokenow/wierzcholek), z capem. + { + std::size_t estimate = stream.remaining() / 8; + if (estimate > 8192) { + estimate = 8192; + } + out.vertices.reserve(estimate); + } + while (!stream.empty() && !atEnd(stream, subtype, endMarker)) { MeshVertex vertex; const bool hasMore = stream.remaining() > 1; diff --git a/eu07-parser/include/eu07/scene/node/parse_util.hpp b/eu07-parser/include/eu07/scene/node/parse_util.hpp index e9d93184..cdb33bea 100644 --- a/eu07-parser/include/eu07/scene/node/parse_util.hpp +++ b/eu07-parser/include/eu07/scene/node/parse_util.hpp @@ -6,9 +6,11 @@ #include #include +#include #include #include #include +#include #include namespace eu07::scene::node::io { @@ -63,16 +65,23 @@ inline void appendRaw(std::vector& raw, const SourceToken& token) { return 0.0; } - try { - std::size_t used = 0; - const double value = std::stod(std::string(text), &used); - if (used != text.size()) { - return std::nullopt; - } - return value; - } catch (...) { + // std::from_chars: bez alokacji std::string, bez wyjatkow, niezalezne od locale + // (separator dziesietny zawsze '.', co odpowiada formatowi EU07). + std::string_view body = text; + // std::stod akceptowal wiodacy '+'; from_chars nie — zachowujemy zgodnosc. + if (!body.empty() && body.front() == '+') { + body.remove_prefix(1); + } + + double value = 0.0; + const char* const first = body.data(); + const char* const last = body.data() + body.size(); + const std::from_chars_result result = + std::from_chars(first, last, value, std::chars_format::general); + if (result.ec != std::errc{} || result.ptr != last) { return std::nullopt; } + return value; } [[nodiscard]] inline bool takeToken( @@ -115,6 +124,24 @@ inline void appendRaw(std::vector& raw, const SourceToken& token) { return true; } +// Szybki odczyt liczby z goracych petli wierzcholkow: bez kopii SourceToken +// (uzywa peek zamiast consume) i bez dopisywania do bufora raw. Bezpieczne tam, +// gdzie raw wierzcholkow nie jest pozniej czytany (siatki: triangles/strip/fan). +// Przy niepowodzeniu NIE konsumuje tokenu — wolajacy i tak cofa strumien do +// kotwicy wezla, wiec pozycja w miejscu bledu nie ma znaczenia. +[[nodiscard]] inline bool takeDoubleFast(TokenStream& stream, double& out) { + if (stream.empty()) { + return false; + } + const std::optional value = parseDouble(stream.peek().value); + if (!value) { + return false; + } + stream.skip(); + out = *value; + return true; +} + [[nodiscard]] inline bool takeDoubleOrPlaceholder( TokenStream& stream, std::vector& raw, diff --git a/eu07-parser/include/eu07/scene/node/triangles.hpp b/eu07-parser/include/eu07/scene/node/triangles.hpp index 37da46ff..0f7e782b 100644 --- a/eu07-parser/include/eu07/scene/node/triangles.hpp +++ b/eu07-parser/include/eu07/scene/node/triangles.hpp @@ -73,10 +73,12 @@ inline constexpr std::string_view kEndMarker = "endtriangles"; std::vector& raw, MeshVertex& vertex, const bool allowEndSuffix) { - if (!node::io::takeDouble(stream, raw, vertex.x) || !node::io::takeDouble(stream, raw, vertex.y) || - !node::io::takeDouble(stream, raw, vertex.z) || !node::io::takeDouble(stream, raw, vertex.nx) || - !node::io::takeDouble(stream, raw, vertex.ny) || !node::io::takeDouble(stream, raw, vertex.nz) || - !node::io::takeDouble(stream, raw, vertex.u) || !node::io::takeDouble(stream, raw, vertex.v)) { + // Goraca petla: 8 liczb na wierzcholek czytanych bez kopii tokenu i bez raw + // (raw wierzcholkow siatek nie jest pozniej czytany przy bake). + if (!node::io::takeDoubleFast(stream, vertex.x) || !node::io::takeDoubleFast(stream, vertex.y) || + !node::io::takeDoubleFast(stream, vertex.z) || !node::io::takeDoubleFast(stream, vertex.nx) || + !node::io::takeDoubleFast(stream, vertex.ny) || !node::io::takeDoubleFast(stream, vertex.nz) || + !node::io::takeDoubleFast(stream, vertex.u) || !node::io::takeDoubleFast(stream, vertex.v)) { return false; } if (allowEndSuffix && !stream.empty() && isKeyword(stream.peek().value, "end")) { @@ -105,6 +107,16 @@ inline constexpr std::string_view kEndMarker = "endtriangles"; return false; } + // Rezerwacja z gornego oszacowania (>=8 tokenow/wierzcholek), z capem aby + // nie przealokowac przy wczesnym wezle nad strumieniem calego pliku. + { + std::size_t estimate = stream.remaining() / 8; + if (estimate > 8192) { + estimate = 8192; + } + out.vertices.reserve(estimate); + } + while (!stream.empty() && !node::io::atEnd(stream, kSubtype, kEndMarker)) { MeshVertex vertex; const bool hasMore = stream.remaining() > 1; diff --git a/eu07-parser/include/eu07/scene/parallel_models.hpp b/eu07-parser/include/eu07/scene/parallel_models.hpp new file mode 100644 index 00000000..12b7058b --- /dev/null +++ b/eu07-parser/include/eu07/scene/parallel_models.hpp @@ -0,0 +1,766 @@ +#pragma once + +// Fast path: plaskie pliki .scm — kazdy wpis w jednej linii logicznej. +// (1) same "node model" (flora) albo (2) same "include;...;end" (placementy). +// Pomija tokenizacje calego pliku; parse linii rownolegle (per-line tokenize). + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace eu07::scene::detail { + +enum class FlatFileKind { + None, + Models, + Includes, +}; + +inline constexpr std::size_t kParallelFlatLineThreshold = 2048; + +[[nodiscard]] inline bool segmentStartsWithKeyword( + std::string_view text, + const std::string_view keyword) { + skipFieldSeparators(text); + if (text.size() < keyword.size()) { + return false; + } + if (text.compare(0, keyword.size(), keyword) != 0) { + return false; + } + if (text.size() == keyword.size()) { + return true; + } + return isFieldSeparator(static_cast(text[keyword.size()])); +} + +[[nodiscard]] inline FlatFileKind classifyFlatFile(const std::vector& segments) { + FlatFileKind kind = FlatFileKind::None; + for (const LogicalSegment& segment : segments) { + std::string_view text = segment.text; + skipFieldSeparators(text); + if (text.empty()) { + continue; + } + + if (segmentStartsWithKeyword(text, "node")) { + if (text.find("endmodel") == std::string_view::npos) { + return FlatFileKind::None; + } + if (kind == FlatFileKind::Includes) { + return FlatFileKind::None; + } + kind = FlatFileKind::Models; + continue; + } + + if (segmentStartsWithKeyword(text, "include")) { + if (text.find("end") == std::string_view::npos) { + return FlatFileKind::None; + } + if (kind == FlatFileKind::Models) { + return FlatFileKind::None; + } + kind = FlatFileKind::Includes; + continue; + } + + return FlatFileKind::None; + } + return kind; +} + +struct ModelParseJob { + std::size_t segment_index = 0; + NodeHeader header; +}; + +[[nodiscard]] inline bool skipModelBody(TokenStream& stream) { + std::vector raw; + while (!stream.empty() && !node::io::atEnd(stream, node_model::kSubtype, node_model::kEndMarker)) { + stream.consume(); + } + return node::io::consumeEnd(stream, raw, node_model::kSubtype, node_model::kEndMarker); +} + +[[nodiscard]] inline bool parseModelSegment( + const LogicalSegment& segment, + ParsedNodeModel& out) { + const std::vector tokens = tokenizeSegments({segment}); + TokenStream stream(tokens); + NodeHeader header; + std::string subtype; + std::vector raw; + if (!node::io::consumeHeader(stream, header, subtype, raw) || + !isKeyword(subtype, node_model::kSubtype)) { + return false; + } + return node_model::parseBody(stream, raw, header, out); +} + +[[nodiscard]] inline std::string_view trimFieldView(std::string_view text) { + skipFieldSeparators(text); + while (!text.empty() && isFieldSeparator(static_cast(text.back()))) { + text.remove_suffix(1); + } + return text; +} + +[[nodiscard]] inline std::vector splitSemicolonFields(std::string_view text) { + std::vector fields; + while (!text.empty()) { + skipFieldSeparators(text); + if (text.empty()) { + break; + } + const std::size_t separator = text.find(';'); + if (separator == std::string_view::npos) { + fields.push_back(trimFieldView(text)); + break; + } + fields.push_back(trimFieldView(text.substr(0, separator))); + text.remove_prefix(separator + 1); + } + return fields; +} + +[[nodiscard]] inline bool parseIncludeSegmentSemicolon( + const LogicalSegment& segment, + ParsedInclude& out) { + std::string_view text = segment.text; + if (const std::size_t comment = text.find("//"); comment != std::string_view::npos) { + text = text.substr(0, comment); + } + skipFieldSeparators(text); + if (!segmentStartsWithKeyword(text, "include")) { + return false; + } + + const std::vector fields = splitSemicolonFields(text); + if (fields.empty() || !isKeyword(fields.front(), "include")) { + return false; + } + + out = ParsedInclude {}; + out.line = segment.sourceLine; + if (fields.size() < 2) { + out.error = "brak sciezki pliku"; + return true; + } + + out.file = std::string(fields[1]); + for (std::size_t index = 2; index < fields.size(); ++index) { + if (isKeyword(fields[index], "end")) { + break; + } + out.parameters.emplace_back(fields[index]); + } + return true; +} + +[[nodiscard]] inline bool parseIncludeSegment( + const LogicalSegment& segment, + ParsedInclude& out) { + if (parseIncludeSegmentSemicolon(segment, out)) { + return true; + } + const std::vector tokens = tokenizeSegments({segment}); + TokenStream stream(tokens); + if (stream.empty() || !isKeyword(stream.peek().value, "include")) { + return false; + } + + out = ParsedInclude {}; + out.line = segment.sourceLine; + stream.consume(); + + if (stream.empty()) { + out.error = "brak sciezki pliku"; + return true; + } + + SourceToken file_token; + if (!node::io::takeToken(stream, out.raw, file_token)) { + out.error = "brak sciezki pliku"; + return true; + } + out.file = file_token.value; + + while (!stream.empty()) { + if (isKeyword(stream.peek().value, "end")) { + stream.consume(); + break; + } + SourceToken param; + if (!node::io::takeToken(stream, out.raw, param)) { + break; + } + out.parameters.push_back(param.value); + } + + out.raw.clear(); + return true; +} + +[[nodiscard]] inline std::optional parseFlatModels( + const std::vector& segments) { + SceneDocument document; + if (segments.empty()) { + return document; + } + + if (segments.size() < kParallelFlatLineThreshold) { + document.nodeModel.reserve(segments.size()); + for (const LogicalSegment& segment : segments) { + ParsedNodeModel model; + if (!parseModelSegment(segment, model)) { + return std::nullopt; + } + document.nodeModel.push_back(std::move(model)); + } + return document; + } + + document.nodeModel.resize(segments.size()); + const unsigned worker_count = + std::min(static_cast(segments.size()), eu07::nmt::workerThreadCount()); + std::atomic next_job { 0 }; + std::atomic failed { false }; + + auto worker = [&]() { + while (true) { + if (failed.load(std::memory_order_relaxed)) { + return; + } + const std::size_t index = next_job.fetch_add(1, std::memory_order_relaxed); + if (index >= segments.size()) { + return; + } + ParsedNodeModel model; + if (!parseModelSegment(segments[index], model)) { + failed.store(true, std::memory_order_relaxed); + return; + } + document.nodeModel[index] = std::move(model); + } + }; + + std::vector threads; + const unsigned launch = std::max(1u, worker_count); + threads.reserve(launch); + for (unsigned i = 0; i < launch; ++i) { + threads.emplace_back(worker); + } + for (std::thread& thread : threads) { + thread.join(); + } + + if (failed.load(std::memory_order_relaxed)) { + return std::nullopt; + } + return document; +} + +[[nodiscard]] inline std::optional parseFlatIncludes( + const std::vector& segments) { + SceneDocument document; + document.include.resize(segments.size()); + + if (segments.size() < kParallelFlatLineThreshold) { + for (std::size_t index = 0; index < segments.size(); ++index) { + if (!parseIncludeSegment(segments[index], document.include[index])) { + return std::nullopt; + } + } + return document; + } + + std::atomic next_job { 0 }; + std::atomic failed { false }; + + auto worker = [&]() { + while (true) { + if (failed.load(std::memory_order_relaxed)) { + return; + } + const std::size_t index = next_job.fetch_add(1, std::memory_order_relaxed); + if (index >= segments.size()) { + return; + } + ParsedInclude entry; + if (!parseIncludeSegment(segments[index], entry)) { + failed.store(true, std::memory_order_relaxed); + return; + } + document.include[index] = std::move(entry); + } + }; + + const unsigned worker_count = + std::min(static_cast(segments.size()), eu07::nmt::workerThreadCount()); + std::vector threads; + const unsigned launch = std::max(1u, worker_count); + threads.reserve(launch); + for (unsigned i = 0; i < launch; ++i) { + threads.emplace_back(worker); + } + for (std::thread& thread : threads) { + thread.join(); + } + + if (failed.load(std::memory_order_relaxed)) { + return std::nullopt; + } + return document; +} + +[[nodiscard]] inline std::optional tryProcessFlatLogical( + const LogicalPass& logical, + const std::filesystem::path& /*baseDirectory*/) { + const FlatFileKind kind = classifyFlatFile(logical.segments); + if (kind == FlatFileKind::None) { + return std::nullopt; + } + if (kind == FlatFileKind::Models) { + return parseFlatModels(logical.segments); + } + return parseFlatIncludes(logical.segments); +} + +inline constexpr std::size_t kParallelModelParseThreshold = 4096; + +[[nodiscard]] inline std::optional tryProcessFlatModels( + const ParseResult& parsed, + const std::filesystem::path& baseDirectory) { + if (parsed.tokens.empty()) { + return SceneDocument {}; + } + + SceneDocument document; + std::filesystem::path includeRoot = baseDirectory; + if (includeRoot.empty()) { + includeRoot = std::filesystem::current_path(); + } + + ParseContext context { document, {}, includeRoot, {} }; + context.expandIncludes = false; + + TokenStream stream(parsed.tokens); + std::vector jobs; + jobs.reserve(parsed.tokens.size() / 16); + + while (!stream.empty()) { + if (detail::dispatchDirective(stream, context)) { + continue; + } + + if (!stream.empty() && isKeyword(stream.peek().value, "node")) { + const std::size_t anchor = stream.checkpoint(); + NodeHeader header; + std::string subtype; + std::vector raw; + if (!node::io::consumeHeader(stream, header, subtype, raw)) { + return std::nullopt; + } + if (!isKeyword(subtype, node_model::kSubtype)) { + return std::nullopt; + } + node::applyScratchContext(header, context.scratch); + jobs.push_back(ModelParseJob { anchor, header }); + if (!skipModelBody(stream)) { + return std::nullopt; + } + continue; + } + + UnknownEntry unknown; + unknown.line = stream.peek().sourceLine; + unknown.token = stream.consume().value; + document.unknown.push_back(std::move(unknown)); + } + + if (jobs.size() < kParallelModelParseThreshold) { + document.nodeModel.reserve(jobs.size()); + for (const ModelParseJob& job : jobs) { + ParsedNodeModel model; + TokenStream job_stream(parsed.tokens); + job_stream.rewind(job.segment_index); + NodeHeader parsed_header; + std::string subtype; + std::vector raw; + if (!node::io::consumeHeader(job_stream, parsed_header, subtype, raw) || + !isKeyword(subtype, node_model::kSubtype)) { + return std::nullopt; + } + model.header = job.header; + if (!node_model::parseBody(job_stream, raw, job.header, model)) { + return std::nullopt; + } + document.nodeModel.push_back(std::move(model)); + } + return document; + } + + document.nodeModel.resize(jobs.size()); + const unsigned worker_count = + std::min(static_cast(jobs.size()), eu07::nmt::workerThreadCount()); + std::atomic next_job { 0 }; + std::atomic failed { false }; + + auto worker = [&]() { + while (true) { + if (failed.load(std::memory_order_relaxed)) { + return; + } + const std::size_t index = next_job.fetch_add(1, std::memory_order_relaxed); + if (index >= jobs.size()) { + return; + } + ParsedNodeModel model; + TokenStream job_stream(parsed.tokens); + job_stream.rewind(jobs[index].segment_index); + NodeHeader parsed_header; + std::string subtype; + std::vector raw; + if (!node::io::consumeHeader(job_stream, parsed_header, subtype, raw) || + !isKeyword(subtype, node_model::kSubtype)) { + failed.store(true, std::memory_order_relaxed); + return; + } + model.header = jobs[index].header; + if (!node_model::parseBody(job_stream, raw, jobs[index].header, model)) { + failed.store(true, std::memory_order_relaxed); + return; + } + document.nodeModel[index] = std::move(model); + } + }; + + std::vector threads; + const unsigned launch = std::max(1u, worker_count); + threads.reserve(launch); + for (unsigned i = 0; i < launch; ++i) { + threads.emplace_back(worker); + } + for (std::thread& thread : threads) { + thread.join(); + } + + if (failed.load(std::memory_order_relaxed)) { + return std::nullopt; + } + return document; +} + +[[nodiscard]] inline std::optional tryProcessFlatSceneFile( + eu07::RawFile raw, + const std::filesystem::path& baseDirectory) { + const LogicalPass logical = toLogicalLines(raw.lines); + return tryProcessFlatLogical(logical, baseDirectory); +} + +inline constexpr std::size_t kStreamFlatFileThresholdBytes = 4u * 1024u * 1024u; + +[[nodiscard]] inline std::size_t streamSourceFileSizeBytes(const std::filesystem::path& path) { + std::error_code ec; + const auto size = std::filesystem::file_size(path, ec); + return ec ? 0u : static_cast(size); +} + +[[nodiscard]] inline bool shouldStreamFlatSourceFile( + const std::filesystem::path& path, + const std::size_t threshold_bytes = kStreamFlatFileThresholdBytes) { + if (threshold_bytes == 0) { + return false; + } + return streamSourceFileSizeBytes(path) > threshold_bytes; +} + +// Linia po linii, z ta sama logika komentarzy co toLogicalLines — bez readRawFile. +template +void forEachStreamingLogicalSegment(const std::filesystem::path& path, Fn&& on_segment) { + std::ifstream in(path, std::ios::binary); + if (!in) { + throw std::runtime_error("Nie mozna otworzyc: " + path.string()); + } + + std::vector read_buffer(1024u * 1024u); + in.rdbuf()->pubsetbuf(read_buffer.data(), read_buffer.size()); + + bool in_block = false; + std::string line; + line.reserve(512); + std::size_t source_line = 0; + std::string spill; + + const auto emit = [&](std::string_view piece, const std::size_t line_no) -> bool { + if (piece.empty()) { + return true; + } + skipFieldSeparators(piece); + if (eu07::detail::isStarter(piece)) { + return true; + } + if (eu07::detail::isLineComment(piece)) { + return true; + } + const std::string_view code = eu07::detail::stripInlineComment(piece); + if (code.empty()) { + return true; + } + LogicalSegment segment; + segment.sourceLine = line_no; + if (code.data() >= piece.data() && code.data() + code.size() <= piece.data() + piece.size()) { + segment.text = code; + } else { + spill.assign(code.begin(), code.end()); + segment.text = spill; + } + return on_segment(segment); + }; + + while (std::getline(in, line)) { + ++source_line; + if (!line.empty() && line.back() == '\r') { + line.pop_back(); + } + + { + std::string_view whole = line; + skipFieldSeparators(whole); + if (parseStarter(whole, source_line - 1)) { + continue; + } + } + + if (!in_block) { + std::size_t at = 0; + std::string_view line_view = line; + while (at < line_view.size()) { + const std::size_t slash = line_view.find('/', at); + if (slash == std::string_view::npos) { + if (!emit(line_view.substr(at), source_line)) { + return; + } + break; + } + if (slash + 1 < line_view.size() && line_view[slash + 1] == '/') { + if (slash > at) { + if (!emit(line_view.substr(at, slash - at), source_line)) { + return; + } + } + break; + } + if (slash + 1 < line_view.size() && line_view[slash + 1] == '*') { + if (slash > at) { + if (!emit(line_view.substr(at, slash - at), source_line)) { + return; + } + } + in_block = true; + line_view = line_view.substr(slash + 2); + at = 0; + continue; + } + if (!emit(line_view.substr(at), source_line)) { + return; + } + break; + } + continue; + } + + while (in_block && !line.empty()) { + const std::size_t close = line.find("*/"); + if (close == std::string_view::npos) { + line.clear(); + break; + } + line.erase(0, close + 2); + in_block = false; + } + if (!in_block && !line.empty()) { + if (!emit(line, source_line)) { + return; + } + } + } +} + +[[nodiscard]] inline FlatFileKind scanFlatFileKindStreaming(const std::filesystem::path& path) { + FlatFileKind kind = FlatFileKind::None; + bool ok = true; + forEachStreamingLogicalSegment(path, [&](const LogicalSegment& segment) { + std::string_view text = segment.text; + skipFieldSeparators(text); + if (text.empty()) { + return true; + } + if (segmentStartsWithKeyword(text, "node")) { + if (text.find("endmodel") == std::string_view::npos) { + ok = false; + return false; + } + if (kind == FlatFileKind::Includes) { + ok = false; + return false; + } + kind = FlatFileKind::Models; + return true; + } + if (segmentStartsWithKeyword(text, "include")) { + if (text.find("end") == std::string_view::npos) { + ok = false; + return false; + } + if (kind == FlatFileKind::Models) { + ok = false; + return false; + } + kind = FlatFileKind::Includes; + return true; + } + ok = false; + return false; + }); + return ok ? kind : FlatFileKind::None; +} + +[[nodiscard]] inline std::optional buildFlatIncludesDocumentStreaming( + const std::filesystem::path& path) { + SceneDocument document; + struct OwnedIncludeLine { + std::size_t source_line = 0; + std::string text; + }; + std::vector batch; + batch.reserve(8192); + bool ok = true; + + const auto flush_batch = [&]() -> bool { + if (batch.empty()) { + return true; + } + const std::size_t base = document.include.size(); + document.include.resize(base + batch.size()); + + const auto parse_one = [&](const std::size_t index) -> bool { + LogicalSegment segment; + segment.sourceLine = batch[index].source_line; + segment.text = batch[index].text; + ParsedInclude entry; + if (!parseIncludeSegment(segment, entry)) { + return false; + } + document.include[base + index] = std::move(entry); + return true; + }; + + if (batch.size() >= kParallelFlatLineThreshold) { + std::atomic next_job { 0 }; + std::atomic failed { false }; + const unsigned worker_count = std::min( + static_cast(batch.size()), eu07::nmt::workerThreadCount()); + const auto worker = [&]() { + while (!failed.load(std::memory_order_relaxed)) { + const std::size_t index = + next_job.fetch_add(1, std::memory_order_relaxed); + if (index >= batch.size()) { + return; + } + if (!parse_one(index)) { + failed.store(true, std::memory_order_relaxed); + return; + } + } + }; + std::vector threads; + threads.reserve(worker_count); + for (unsigned i = 0; i < worker_count; ++i) { + threads.emplace_back(worker); + } + for (std::thread& thread : threads) { + thread.join(); + } + if (failed.load(std::memory_order_relaxed)) { + return false; + } + } else { + for (std::size_t index = 0; index < batch.size(); ++index) { + if (!parse_one(index)) { + return false; + } + } + } + + batch.clear(); + return true; + }; + + forEachStreamingLogicalSegment(path, [&](const LogicalSegment& segment) { + std::string_view text = segment.text; + skipFieldSeparators(text); + if (text.empty() || !segmentStartsWithKeyword(text, "include")) { + return true; + } + batch.push_back(OwnedIncludeLine { segment.sourceLine, std::string(segment.text) }); + if (batch.size() >= 8192) { + if (!flush_batch()) { + ok = false; + return false; + } + } + return true; + }); + + if (ok) { + ok = flush_batch(); + } + return ok ? std::optional(std::move(document)) : std::nullopt; +} + +template +void streamFlatFileModels(const std::filesystem::path& path, Fn&& on_model) { + forEachStreamingLogicalSegment(path, [&](const LogicalSegment& segment) { + std::string_view text = segment.text; + skipFieldSeparators(text); + if (text.empty() || !segmentStartsWithKeyword(text, "node")) { + return true; + } + if (text.find("endmodel") == std::string_view::npos) { + throw std::runtime_error( + "Plaski plik modeli: oczekiwano endmodel w linii " + + std::to_string(segment.sourceLine)); + } + ParsedNodeModel parsed; + if (!parseModelSegment(segment, parsed)) { + throw std::runtime_error( + "Plaski plik modeli: blad parse linii " + std::to_string(segment.sourceLine)); + } + on_model(parsed); + return true; + }); +} + +} // namespace eu07::scene::detail diff --git a/eu07-parser/include/eu07/scene/processor.hpp b/eu07-parser/include/eu07/scene/processor.hpp index d57367b9..901a8762 100644 --- a/eu07-parser/include/eu07/scene/processor.hpp +++ b/eu07-parser/include/eu07/scene/processor.hpp @@ -18,7 +18,7 @@ #include - +#include #include @@ -194,6 +194,7 @@ inline void expandInclude(ParseContext& context, const IncludeExpansionRequest& struct SceneProcessOptions { bool expandIncludes = true; + bool packComposeLightweight = false; }; [[nodiscard]] inline SceneProcessResult processScene( @@ -204,6 +205,15 @@ struct SceneProcessOptions { const SceneProcessOptions& options = {}) { + if (!options.expandIncludes) { + if (std::optional fast = + detail::tryProcessFlatModels(parsed, baseDirectory)) { + SceneProcessResult result; + result.document = std::move(*fast); + return result; + } + } + TokenStream stream(parsed.tokens); SceneDocument document; @@ -223,6 +233,7 @@ struct SceneProcessOptions { ParseContext context{document, {}, includeRoot, {}}; context.expandIncludes = options.expandIncludes; + context.packComposeLightweight = options.packComposeLightweight; detail::processStream(stream, context); diff --git a/scene/eu7/eu7_bake.cpp b/scene/eu7/eu7_bake.cpp index 18f086a7..f671061b 100644 --- a/scene/eu7/eu7_bake.cpp +++ b/scene/eu7/eu7_bake.cpp @@ -32,6 +32,8 @@ http://mozilla.org/MPL/2.0/. +#include + #include @@ -62,7 +64,31 @@ bake_thread_count() { auto const hardware { std::thread::hardware_concurrency() }; - return hardware > 0 ? hardware : 1u; + if( hardware == 0 ) { + + return 1u; + + } + + int percent { Global.eu7_bake_cpu_percent }; + + if( percent < 1 ) { percent = 1; } + + if( percent > 100 ) { percent = 100; } + + unsigned const threads { + + std::max( 1u, + + static_cast( ( static_cast( hardware ) * percent + 50 ) / 100 ) ) }; + + WriteLog( + + "EU7 bake: auto watki=" + std::to_string( threads ) + + + " (" + std::to_string( percent ) + "% z " + std::to_string( hardware ) + " rdzeni logicznych)" ); + + return threads; } @@ -90,7 +116,27 @@ run_scenario_tree_bake( std::string error; - if( false == bake_parser::bake_scenario_tree( text_root, bake_thread_count(), error ) ) { + bool bake_ok { false }; + + if( Global.eu7v2_runtime ) { + + auto const report { + + bake_parser::bake_scenario_tree_eu7v2( text_root, bake_thread_count(), false ) }; + + bake_ok = report.baked; + + error = report.error; + + } + + else { + + bake_ok = bake_parser::bake_scenario_tree( text_root, bake_thread_count(), error ); + + } + + if( false == bake_ok ) { if( probe_file( eu7_path ) ) { diff --git a/scene/eu7/eu7_bake_mem_guard.h b/scene/eu7/eu7_bake_mem_guard.h new file mode 100644 index 00000000..fa020a72 --- /dev/null +++ b/scene/eu7/eu7_bake_mem_guard.h @@ -0,0 +1,122 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#if defined( _WIN32 ) +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#else +#include +#include +#endif + +namespace scene::eu7::bake_parser { + +inline constexpr unsigned kDefaultBakeMemLimitGb { 50u }; + +[[nodiscard]] inline std::uint64_t +process_private_bytes() noexcept { +#if defined( _WIN32 ) + PROCESS_MEMORY_COUNTERS_EX counters {}; + counters.cb = sizeof( counters ); + if( GetProcessMemoryInfo( + GetCurrentProcess(), + reinterpret_cast( &counters ), + sizeof( counters ) ) ) { + return counters.PrivateUsage; + } + return 0; +#else + std::FILE *status { std::fopen( "/proc/self/statm", "r" ) }; + if( status == nullptr ) { + return 0; + } + long total_pages { 0 }; + long resident_pages { 0 }; + if( std::fscanf( status, "%ld %ld", &total_pages, &resident_pages ) != 2 ) { + std::fclose( status ); + return 0; + } + std::fclose( status ); + long const page_size { sysconf( _SC_PAGESIZE ) }; + if( page_size <= 0 ) { + return 0; + } + return static_cast( resident_pages ) * static_cast( page_size ); +#endif +} + +// Background poll; abort() when private commit exceeds limit_bytes (0 = disabled). +class bake_mem_guard { + public: + explicit bake_mem_guard( std::uint64_t const limit_bytes ) : m_limit_bytes( limit_bytes ) {} + + bake_mem_guard( bake_mem_guard const & ) = delete; + bake_mem_guard &operator=( bake_mem_guard const & ) = delete; + + ~bake_mem_guard() { stop(); } + + void start() { + if( m_limit_bytes == 0 || m_thread.joinable() ) { + return; + } + m_stop.store( false, std::memory_order_release ); + m_thread = std::thread( [this]() { run(); } ); + } + + void stop() { + m_stop.store( true, std::memory_order_release ); + if( m_thread.joinable() ) { + m_thread.join(); + } + } + + private: + void run() const { + while( !m_stop.load( std::memory_order_acquire ) ) { + std::uint64_t const used { process_private_bytes() }; + if( used > m_limit_bytes ) { + double const used_gb { static_cast( used ) / ( 1024.0 * 1024.0 * 1024.0 ) }; + double const limit_gb { + static_cast( m_limit_bytes ) / ( 1024.0 * 1024.0 * 1024.0 ) }; + std::fprintf( + stdout, + "[EU7v2] FATAL: limit pamieci %.1f GB przekroczony (uzyte ~%.1f GB) — abort\n", + limit_gb, + used_gb ); + std::fflush( stdout ); + std::fprintf( + stderr, + "[EU7v2] FATAL: limit pamieci %.1f GB przekroczony (uzyte ~%.1f GB) — abort\n", + limit_gb, + used_gb ); + std::fflush( stderr ); + std::abort(); + } + std::this_thread::sleep_for( std::chrono::milliseconds { 500 } ); + } + } + + std::uint64_t const m_limit_bytes { 0 }; + std::atomic m_stop { false }; + std::thread m_thread; +}; + +} // namespace scene::eu7::bake_parser diff --git a/scene/eu7/eu7_bake_parser.cpp b/scene/eu7/eu7_bake_parser.cpp index f85683de..e2c96e10 100644 --- a/scene/eu7/eu7_bake_parser.cpp +++ b/scene/eu7/eu7_bake_parser.cpp @@ -17,15 +17,43 @@ http://mozilla.org/MPL/2.0/. #include +#include +#include + +#include "scene/eu7/eu7_bake_parser.h" + +#include "scene/eu7/eu7_bake_mem_guard.h" +#include "scene/eu7/v2/eu7v2_emit_runtime.h" + +#include "utilities/Logs.h" +#include + +#include + +#include + #include +#include + +#include + +#include + #include +#include +#include + #include +#include + +#include + namespace scene::eu7::bake_parser { @@ -36,43 +64,218 @@ namespace { namespace fs = std::filesystem; +using clock_type = std::chrono::steady_clock; -std::mutex g_progress_log_mutex; + +// Bake logs go to stdout (headless console) and WriteLog (-> log.txt when the +// game's LogService is running). One mutex keeps interleaved lines readable +// across the bake worker threads. +std::mutex g_log_mutex; + +void + +bake_log( std::string const &line ) { + + std::lock_guard lock { g_log_mutex }; + + std::cout << line << '\n' << std::flush; + + WriteLog( line ); + + // Also tee to a dedicated file so headless runs (where stdout lands in a + // detached console and the game LogService is not flushing) still leave a + // readable record of timings/summaries. + static std::ofstream bake_file { "eu7v2_bake.log", std::ios::app }; + if( bake_file ) { + bake_file << line << '\n'; + bake_file.flush(); + } + +} + + + +[[nodiscard]] std::string + +fmt_ms( double const ms ) { + + std::ostringstream out; + + out << std::fixed << std::setprecision( 1 ) << ms; + + return out.str(); + +} + + + +[[nodiscard]] std::string + +fmt_s( double const s ) { + + std::ostringstream out; + + out << std::fixed << std::setprecision( 2 ) << s; + + return out.str(); + +} + + + +// Shared progress/timing state for one bake run. The bake_tree onProgress +// callback and the onModuleBaked hook both fire from worker threads, so all +// mutable access is guarded. +struct bake_progress_state { + + clock_type::time_point start { clock_type::now() }; + + std::mutex mutex; + + std::unordered_map module_start; + + clock_type::time_point compose_last_log {}; + + bool compose_started { false }; + + std::atomic threads { 0 }; + + std::atomic started { 0 }; + + std::atomic module_index { 0 }; + + std::atomic emit_us { 0 }; + + // Live PACK-compose diagnostics, read by the watchdog thread so progress is + + // visible (and a stall is pinpointed) even if the worker callbacks stop. + + std::atomic compose_stats { nullptr }; + + std::atomic compose_active { false }; + + std::string compose_file; + +}; void -log_bake_progress( eu07::scene::bake::BakeProgress const &progress ) { +on_bake_progress( bake_progress_state &state, eu07::scene::bake::BakeProgress const &progress ) { - std::lock_guard lock { g_progress_log_mutex }; + using Phase = eu07::scene::bake::BakeProgressPhase; - auto const path { progress.path.u8string() }; + if( progress.threads > 0 ) { + + state.threads.store( progress.threads, std::memory_order_relaxed ); + + } + + auto const name { progress.path.filename().generic_string() }; switch( progress.phase ) { - case eu07::scene::bake::BakeProgressPhase::Starting: + case Phase::Start: { - case eu07::scene::bake::BakeProgressPhase::Start: + std::size_t const i { state.started.fetch_add( 1, std::memory_order_relaxed ) + 1 }; - break; + { - case eu07::scene::bake::BakeProgressPhase::PackModels: + std::lock_guard lock { state.mutex }; - case eu07::scene::bake::BakeProgressPhase::PackCompose: + state.module_start[ progress.path.generic_string() ] = clock_type::now(); - case eu07::scene::bake::BakeProgressPhase::PackComposeDone: + } - case eu07::scene::bake::BakeProgressPhase::Bake: - - case eu07::scene::bake::BakeProgressPhase::Done: + bake_log( "[EU7v2] (" + std::to_string( i ) + ") BAKING: " + progress.path.generic_string() ); break; } - (void)path; + case Phase::PackModels: + + state.compose_active.store( true, std::memory_order_relaxed ); + + bake_log( + + "[EU7v2] ROOT " + name + ": PACK compose start (watki=" + + + std::to_string( progress.threads ) + ")" ); + + break; + + case Phase::PackCompose: { + + if( progress.pack_stats != nullptr ) { + + state.compose_stats.store( progress.pack_stats, std::memory_order_relaxed ); + + } + + // Throttle to ~2 s so a long compose visibly advances (and the last + + // file pinpoints a stall) without flooding the log. + + auto const now { clock_type::now() }; + + std::lock_guard lock { state.mutex }; + + state.compose_file = name; + + bool const due { + + ( false == state.compose_started ) || + + std::chrono::duration_cast( + + now - state.compose_last_log ) + + .count() >= 2000 }; + + if( due ) { + + state.compose_started = true; + + state.compose_last_log = now; + + bake_log( + + "[EU7v2] PACK compose: pliki=" + std::to_string( progress.current ) + + + " modele=" + std::to_string( progress.total ) + " (plik: " + name + ")" ); + + } + + break; + + } + + case Phase::PackComposeDone: + + state.compose_active.store( false, std::memory_order_relaxed ); + + state.compose_stats.store( nullptr, std::memory_order_relaxed ); + + bake_log( + + "[EU7v2] PACK compose gotowe: pliki=" + std::to_string( progress.current ) + + + " modele=" + std::to_string( progress.total ) ); + + break; + + case Phase::Starting: + + case Phase::Bake: + + case Phase::Done: + + break; + + } } @@ -134,8 +337,6 @@ bake_scenario_tree( options.maxThreads = max_threads; - options.onProgress = log_bake_progress; - eu07::scene::bake::BakeTreeStats stats; @@ -166,5 +367,466 @@ bake_scenario_tree( +Eu7v2BakeReport +bake_scenario_tree_eu7v2( + std::string const &text_scenario_path, + unsigned const max_threads, + bool const verify, + unsigned const mem_limit_gb, + unsigned const max_concurrent_parses, + unsigned const heavy_parse_threshold_mb ) { + + Eu7v2BakeReport report; + report.verify_requested = verify; + + std::uint64_t const mem_limit_bytes { + mem_limit_gb == 0 ? 0u + : static_cast( mem_limit_gb ) * 1024u * 1024u * 1024u }; + bake_mem_guard mem_guard { mem_limit_bytes }; + if( mem_limit_gb != 0 ) { + bake_log( + "[EU7v2] limit pamieci: " + std::to_string( mem_limit_gb ) + + " GB (wylacz: --eu7v2-mem-limit-gb 0)" ); + } + mem_guard.start(); + + fs::path const input { text_scenario_path }; + if( false == fs::exists( input ) ) { + report.error = "brak pliku: " + text_scenario_path; + return report; + } + if( false == is_text_scenery_path( input ) ) { + report.error = "nie jest plikiem SCM/SCN/INC: " + text_scenario_path; + return report; + } + + std::mutex emit_mutex; + std::atomic module_count { 0 }; + std::atomic model_count { 0 }; + std::atomic verify_ok { true }; + std::atomic emit_failed { false }; + std::string root_binary_path; + + struct pending_verify_job { + fs::path path; + bool is_root { false }; + eu7v2::module_verify_spec spec {}; + std::size_t pack_models { 0 }; + }; + std::vector verify_jobs; + std::mutex verify_jobs_mutex; + + bake_progress_state pstate; + + bake_log( + "[EU7v2] BAKE START: " + input.generic_string() + + ( verify ? " (+verify)" : "" ) ); + + try { + eu07::scene::bake::BakeTreeOptions options; + bool const spool_flush { mem_limit_gb != 0 }; + if( spool_flush ) { + // PACK/shape spool + per-module flush bound RAM; pool can bake modules in parallel. + options.lowMemoryMode = true; + options.maxThreads = max_threads; // 0 => hardware default in runBakePool + if( max_concurrent_parses != 0 ) { + options.maxConcurrentParses = max_concurrent_parses; + } else { + options.maxConcurrentParses = + eu07::scene::bake::detail::defaultPoolThreadCount(); + } + } else { + options.maxThreads = max_threads != 0 ? max_threads : 1u; + options.maxConcurrentParses = + max_concurrent_parses != 0 ? max_concurrent_parses : 1u; + } + if( heavy_parse_threshold_mb != 0 ) { + options.heavyParseThresholdMb = heavy_parse_threshold_mb; + } else if( spool_flush ) { + // Streaming/spool bounds RAM — duze pliki na dysku nie wymagaja serial parse. + options.heavyParseThresholdMb = 0u; + } + std::unique_ptr root_pack_spool; + if( spool_flush ) { + fs::path const spool_path { + fs::temp_directory_path() / "eu7v2_pack_flush.bin" }; + root_pack_spool = + std::make_unique( spool_path ); + options.packFlushPerFile = true; + options.onPackModelsFlush = + [&root_pack_spool]( + std::vector &&models ) { + root_pack_spool->append( std::move( models ) ); + }; + bake_log( + "[EU7v2] tryb: jeden plik = jeden modul, PACK per .scm -> " + + spool_path.generic_string() + + " (rownolegly bake modulow)" ); + } + { + unsigned const pool_threads { + eu07::scene::bake::detail::resolvePoolThreadCount( options.maxThreads ) }; + unsigned const parse_limit { + eu07::scene::bake::detail::resolveParseConcurrencyLimit( + options.maxConcurrentParses ) }; + bake_log( + "[EU7v2] parse/bake throttle: watki=" + + std::to_string( pool_threads ) + + ( options.maxThreads == 0 ? " (auto)" : "" ) + + ", rownolegle parse=" + std::to_string( parse_limit ) + + ( options.heavyParseThresholdMb != 0 + ? ", heavy serial >" + std::to_string( options.heavyParseThresholdMb ) + " MB" + : "" ) ); + } + options.onProgress = + [&pstate]( eu07::scene::bake::BakeProgress const &progress ) { + on_bake_progress( pstate, progress ); + }; + options.skipLegacyWrite = true; + options.omitChildModuleModels = true; + + struct deferred_root_emit { + bool pending { false }; + eu07::scene::bake::RuntimeModule module; + fs::path path; + } deferred_root; + bool const incremental_pack { options.lowMemoryMode }; + + auto finish_module_emit = + [&]( eu07::scene::bake::RuntimeModule const &module, + fs::path const &text_path, + bool const is_root, + std::vector const *pack_batches, + eu07::scene::bake::ShapeSpoolFile const *shape_spool ) { + eu7v2::emit_outcome const outcome { eu7v2::emit_runtime_module( + module, + text_path, + is_root, + pack_batches, + false, + is_root && root_pack_spool != nullptr ? root_pack_spool.get() : nullptr, + shape_spool ) }; + + double parse_bake_ms { 0.0 }; + { + std::lock_guard lock { pstate.mutex }; + auto const it { pstate.module_start.find( text_path.generic_string() ) }; + if( it != pstate.module_start.end() ) { + parse_bake_ms = std::chrono::duration( + clock_type::now() - it->second ) + .count(); + pstate.module_start.erase( it ); + } + } + + std::size_t const idx { + module_count.fetch_add( 1, std::memory_order_relaxed ) + 1 }; + model_count.fetch_add( outcome.model_total, std::memory_order_relaxed ); + pstate.emit_us.fetch_add( + static_cast( + ( outcome.build_ms + outcome.write_ms ) * 1000.0 ), + std::memory_order_relaxed ); + if( false == outcome.ok ) { + emit_failed.store( true, std::memory_order_relaxed ); + } + + if( verify && outcome.ok ) { + pending_verify_job job; + job.path = fs::path { outcome.written_path }; + job.is_root = is_root; + for( auto const &inc : module.includes ) { + if( eu07::scene::detail::isIncFile( inc.sourcePath ) && + inc.parameters.size() >= 5 ) { + try { + (void)std::stod( inc.parameters[ 1 ] ); + (void)std::stod( inc.parameters[ 2 ] ); + (void)std::stod( inc.parameters[ 3 ] ); + (void)std::stod( inc.parameters[ 4 ] ); + ++job.spec.placements; + continue; + } catch( ... ) { + } + } + ++job.spec.includes; + } + job.spec.models = module.scene.models.size(); + job.spec.shapes = module.scene.shapes.size(); + if( shape_spool != nullptr ) { + job.spec.shapes += shape_spool->shape_count(); + } + job.spec.lines = module.scene.lines.size(); + job.spec.tracks = module.scene.tracks.size(); + job.spec.traction = module.scene.traction.size(); + job.spec.power = module.scene.powerSources.size(); + job.spec.memcells = module.scene.memcells.size(); + job.spec.launchers = module.scene.eventLaunchers.size(); + job.spec.events = module.scene.events.size(); + job.spec.sounds = module.scene.sounds.size(); + job.spec.dynamics = module.scene.dynamics.size(); + job.spec.trainsets = module.scene.trainsets.size(); + if( is_root && pack_batches != nullptr ) { + for( auto const &batch : *pack_batches ) { + job.pack_models += batch.models.size(); + } + } + if( is_root && root_pack_spool != nullptr ) { + job.pack_models += root_pack_spool->model_count(); + } + std::lock_guard lock { verify_jobs_mutex }; + verify_jobs.push_back( std::move( job ) ); + } + + bool const verbose_module { + is_root || false == outcome.ok || parse_bake_ms >= 300.0 || + outcome.build_ms >= 100.0 }; + if( !verbose_module ) { + { + std::lock_guard lock { emit_mutex }; + if( is_root ) { + root_binary_path = outcome.written_path; + } + } + return; + } + + std::string line { + "[EU7v2] (" + std::to_string( idx ) + ") " + + ( is_root ? "ROOT " : "" ) + text_path.filename().generic_string() + + " - parse+bake " + fmt_ms( parse_bake_ms ) + "ms, build " + + fmt_ms( outcome.build_ms ) + "ms, zapis " + fmt_ms( outcome.write_ms ) + "ms" }; + line += " (models=" + std::to_string( outcome.model_total ) + ", " + + std::to_string( ( outcome.byte_size + 1023 ) / 1024 ) + " KB)"; + if( false == outcome.ok ) { + line += " ZAPIS-BLAD: " + outcome.message; + } + + { + std::lock_guard lock { emit_mutex }; + if( is_root ) { + root_binary_path = outcome.written_path; + } + } + bake_log( line ); + + if( is_root && ( pack_batches != nullptr || root_pack_spool != nullptr ) ) { + std::size_t pack_models { 0 }; + if( root_pack_spool != nullptr ) { + pack_models += root_pack_spool->model_count(); + } + if( pack_batches != nullptr ) { + for( auto const &batch : *pack_batches ) { + pack_models += batch.models.size(); + } + } + bake_log( + "[EU7v2] ROOT packModels=" + std::to_string( pack_models ) + + ( root_pack_spool != nullptr + ? " (spool=" + std::to_string( root_pack_spool->model_count() ) + + ")" + : "" ) ); + } + }; + + options.onModuleBaked = + [&]( eu07::scene::bake::RuntimeModule const &module, + fs::path const &text_path, + bool const is_root, + std::vector const *pack_batches, + eu07::scene::bake::ShapeSpoolFile *shape_spool ) { + if( is_root && incremental_pack ) { + deferred_root.pending = true; + deferred_root.module = module; + deferred_root.path = text_path; + return; + } + finish_module_emit( module, text_path, is_root, pack_batches, shape_spool ); + }; + + // Watchdog: prints a heartbeat every ~2 s so a long (or stuck) bake is + // visibly alive. During PACK compose it reads the live counters + // directly, so a stall shows which counter is frozen and on which file. + std::atomic watchdog_done { false }; + std::thread watchdog( [&]() { + while( false == watchdog_done.load( std::memory_order_relaxed ) ) { + for( int slice { 0 }; + slice < 20 && + false == watchdog_done.load( std::memory_order_relaxed ); + ++slice ) { + std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); + } + if( watchdog_done.load( std::memory_order_relaxed ) ) { + break; + } + double const elapsed { + std::chrono::duration( clock_type::now() - pstate.start ).count() }; + std::string msg { + "[EU7v2] ... pracuje: elapsed=" + fmt_s( elapsed ) + + "s, moduly start=" + std::to_string( pstate.started.load( std::memory_order_relaxed ) ) + + " done=" + std::to_string( module_count.load( std::memory_order_relaxed ) ) }; + auto const *cs { pstate.compose_stats.load( std::memory_order_relaxed ) }; + if( pstate.compose_active.load( std::memory_order_relaxed ) && cs != nullptr ) { + std::string file; + { + std::lock_guard lock { pstate.mutex }; + file = pstate.compose_file; + } + msg += + " | PACK compose: pliki=" + + std::to_string( cs->files_visited.load( std::memory_order_relaxed ) ) + + " submit=" + + std::to_string( cs->includes_submitted.load( std::memory_order_relaxed ) ) + + " inst=" + + std::to_string( + cs->instantiate_fast.load( std::memory_order_relaxed ) + + cs->instantiate_full.load( std::memory_order_relaxed ) ) + + " miss=" + + std::to_string( cs->cache_misses.load( std::memory_order_relaxed ) ) + + " kolejka_max=" + + std::to_string( cs->queue_peak.load( std::memory_order_relaxed ) ) + + " (plik: " + file + ")"; + // Live deadlock diagnostics: what every worker/gate is doing. + msg += + "\n[EU7v2] DIAG: pending=" + + std::to_string( cs->diag_pending.load( std::memory_order_relaxed ) ) + + " workers_alive=" + + std::to_string( cs->diag_workers_alive.load( std::memory_order_relaxed ) ) + + " idle=" + + std::to_string( cs->diag_workers_idle.load( std::memory_order_relaxed ) ) + + " parse=" + + std::to_string( cs->diag_workers_parse.load( std::memory_order_relaxed ) ) + + " proc=" + + std::to_string( cs->diag_workers_inst.load( std::memory_order_relaxed ) ) + + " gate=" + + std::to_string( cs->diag_gate_active.load( std::memory_order_relaxed ) ) + + "/" + + std::to_string( cs->diag_gate_max.load( std::memory_order_relaxed ) ) + + " gate_wait=" + + std::to_string( cs->diag_gate_waiting.load( std::memory_order_relaxed ) ) + + " main_wait=" + + std::to_string( cs->diag_main_wait.load( std::memory_order_relaxed ) ); + } + bake_log( msg ); + } + } ); + + eu07::scene::bake::BakeTreeStats stats; + try { + (void)eu07::scene::bake::bakeModuleTree( input, &stats, options ); + } + catch( ... ) { + watchdog_done.store( true, std::memory_order_relaxed ); + watchdog.join(); + throw; + } + watchdog_done.store( true, std::memory_order_relaxed ); + watchdog.join(); + + if( deferred_root.pending ) { + bake_log( "[EU7v2] ROOT emit (po wszystkich modulach)..." ); + finish_module_emit( deferred_root.module, deferred_root.path, true, nullptr, nullptr ); + } + + if( verify && false == verify_jobs.empty() ) { + auto const verify_begin { clock_type::now() }; + std::atomic next_job { 0 }; + std::mutex verify_error_mutex; + std::string verify_error; + const unsigned worker_count { std::max( + 1u, + std::min( + static_cast( verify_jobs.size() ), + std::thread::hardware_concurrency() == 0 ? 4u + : std::thread::hardware_concurrency() ) ) }; + std::vector verify_workers; + verify_workers.reserve( worker_count ); + for( unsigned worker_index { 0 }; worker_index < worker_count; ++worker_index ) { + verify_workers.emplace_back( [&]() { + while( true ) { + std::size_t const job_index { + next_job.fetch_add( 1, std::memory_order_relaxed ) }; + if( job_index >= verify_jobs.size() ) { + return; + } + pending_verify_job const &job { verify_jobs[ job_index ] }; + std::string message; + if( false == + eu7v2::verify_written_module( + job.path, job.spec, job.is_root, job.pack_models, &message ) ) { + verify_ok.store( false, std::memory_order_relaxed ); + std::lock_guard lock { verify_error_mutex }; + if( verify_error.empty() ) { + verify_error = job.path.filename().generic_string() + ":\n" + message; + } + } + } + } ); + } + for( std::thread &worker : verify_workers ) { + worker.join(); + } + double const verify_s { + std::chrono::duration( clock_type::now() - verify_begin ).count() }; + pstate.emit_us.fetch_add( + static_cast( verify_s * 1.0e6 ), + std::memory_order_relaxed ); + bake_log( + "[EU7v2] VERIFY batch: " + std::to_string( verify_jobs.size() ) + " plikow, " + + fmt_s( verify_s ) + "s" ); + if( false == verify_ok.load() && false == verify_error.empty() ) { + bake_log( verify_error ); + } + } + + report.baked = true; + report.module_count = module_count.load(); + report.model_count = model_count.load(); + report.root_binary_path = root_binary_path; + report.verify_ok = verify_ok.load() && ( false == emit_failed.load() ); + if( emit_failed.load() ) { + report.error = "blad zapisu eu7v2 (patrz log)"; + } + + double const total_s { + std::chrono::duration( clock_type::now() - pstate.start ).count() }; + double const emit_s { + static_cast( pstate.emit_us.load( std::memory_order_relaxed ) ) / 1.0e6 }; + std::string summary { + "[EU7v2] BAKE DONE: modules=" + std::to_string( report.module_count ) + + ", total=" + fmt_s( total_s ) + "s, watki=" + + std::to_string( pstate.threads.load( std::memory_order_relaxed ) ) + + ", emit=" + fmt_s( emit_s ) + "s" }; + if( stats.has_pack_diagnostics ) { + auto const &pc { stats.pack_compose }; + summary += ", compose=" + fmt_s( static_cast( pc.compose_us ) / 1.0e6 ) + + "s, parse=" + fmt_s( static_cast( pc.parse_us ) / 1.0e6 ) + + "s [tok=" + fmt_s( static_cast( pc.tokenize_us ) / 1.0e6 ) + + " proc=" + fmt_s( static_cast( pc.process_us ) / 1.0e6 ) + + " make=" + fmt_s( static_cast( pc.makeentry_us ) / 1.0e6 ) + + "], inst=" + fmt_s( static_cast( pc.instantiate_us ) / 1.0e6 ) + + "s, sink=" + fmt_s( static_cast( pc.sink_us ) / 1.0e6 ) + + "s, finalize=" + fmt_s( static_cast( pc.finalize_us ) / 1.0e6 ) + + "s (pliki=" + std::to_string( pc.files_visited ) + + ", modele=" + std::to_string( pc.models_emitted ) + + ", sekcje=" + std::to_string( pc.sections ) + + ", inc_inline=" + std::to_string( pc.inc_includes_inlined ) + + ", miss=" + std::to_string( pc.cache_misses ) + + ", hit=" + std::to_string( pc.cache_hits ) + ")"; + } + bake_log( summary ); + if( verify ) { + bake_log( report.verify_ok ? "[EU7v2] VERIFY: PASS" : "[EU7v2] VERIFY: FAIL" ); + } + return report; + } + catch( std::exception const &ex ) { + report.error = ex.what(); + return report; + } + catch( ... ) { + report.error = "nieznany wyjatek bake eu7v2"; + return report; + } +} + } // namespace scene::eu7::bake_parser diff --git a/scene/eu7/eu7_bake_parser.h b/scene/eu7/eu7_bake_parser.h index 0e5f557d..3c2f5a50 100644 --- a/scene/eu7/eu7_bake_parser.h +++ b/scene/eu7/eu7_bake_parser.h @@ -9,6 +9,7 @@ http://mozilla.org/MPL/2.0/. #pragma once +#include #include namespace scene::eu7::bake_parser { @@ -19,4 +20,32 @@ bake_scenario_tree( unsigned MaxThreads, std::string &ErrorOut ); +// Result of a headless text -> eu7v2 bake (and optional roundtrip verify). +struct Eu7v2BakeReport { + bool baked { false }; // bake tree completed without throwing + bool verify_requested { false }; + bool verify_ok { true }; // all per-module record counts matched + std::size_t module_count { 0 }; + std::size_t model_count { 0 }; + std::string root_binary_path; // emitted root .eu7v2 + std::string error; +}; + +// Bakes the whole text scenery tree directly into .eu7v2 modules (no legacy +// .eu7). When Verify is set each emitted module is reloaded and its record +// counts are compared against the source RuntimeModule, with a PASS/FAIL +// report printed to stdout. This is the headless consistency guard. +// MemLimitGb: abort when process private memory exceeds this (0 = no limit). +// Default 50 when called from headless CLI. +// MaxConcurrentParses: cap parallel parseFile/processScene (0 = auto: 1 with MemLimitGb spool). +// MaxThreads 0 with spool = 1 (serial module bake); use --eu7v2-threads N to override. +[[nodiscard]] Eu7v2BakeReport +bake_scenario_tree_eu7v2( + std::string const &TextScenarioPath, + unsigned MaxThreads, + bool Verify, + unsigned MemLimitGb = 50u, + unsigned MaxConcurrentParses = 0u, + unsigned HeavyParseThresholdMb = 0u ); + } // namespace scene::eu7::bake_parser diff --git a/scene/eu7/eu7_load_stats.cpp b/scene/eu7/eu7_load_stats.cpp index 2717cb41..4ae3af30 100644 --- a/scene/eu7/eu7_load_stats.cpp +++ b/scene/eu7/eu7_load_stats.cpp @@ -80,7 +80,9 @@ log_load_stats() { WriteLog( " read/cache: " + format_seconds( s.read_ms ) + " reads=" + std::to_string( s.module_read ) + - " cache_hit=" + std::to_string( s.module_cache_hit ) ); + " cache_hit=" + std::to_string( s.module_cache_hit ) + + " v2_loaded=" + std::to_string( s.module_v2_loaded ) + + " v2_baked=" + std::to_string( s.module_v2_baked ) ); WriteLog( " scm fallback: " + format_seconds( s.scm_fallback_ms ) + " count=" + std::to_string( s.scm_fallback ) ); diff --git a/scene/eu7/eu7_load_stats.h b/scene/eu7/eu7_load_stats.h index d9ce7504..5a5f5159 100644 --- a/scene/eu7/eu7_load_stats.h +++ b/scene/eu7/eu7_load_stats.h @@ -52,6 +52,8 @@ struct Eu7LoadStats { std::uint64_t sounds { 0 }; std::uint64_t events { 0 }; std::uint64_t trainsets { 0 }; + std::uint64_t module_v2_loaded { 0 }; + std::uint64_t module_v2_baked { 0 }; std::uint64_t pack_skipped_includes { 0 }; std::uint64_t pack_sections_loaded { 0 }; std::uint64_t pack_models { 0 }; diff --git a/scene/eu7/eu7_loader.cpp b/scene/eu7/eu7_loader.cpp index caecdf98..0176f0ef 100644 --- a/scene/eu7/eu7_loader.cpp +++ b/scene/eu7/eu7_loader.cpp @@ -18,6 +18,9 @@ http://mozilla.org/MPL/2.0/. #include "scene/eu7/eu7_section_stream.h" #include "scene/eu7/eu7_parameters.h" #include "scene/eu7/eu7_transform.h" +#include "scene/eu7/v2/eu7v2_bake.h" +#include "scene/eu7/v2/eu7v2_format.h" +#include "scene/eu7/v2/eu7v2_load.h" #include "scene/scene.h" #include "scene/scenenode.h" #include "rendering/renderer.h" @@ -30,11 +33,14 @@ http://mozilla.org/MPL/2.0/. #include #include +#include #include +#include #include #include #include #include +#include namespace scene::eu7 { @@ -43,6 +49,135 @@ namespace { std::unordered_map g_module_file_cache; bool g_packed_root_active { false }; +// --- eu7v2 experiment: lossy lean-format transcode/load --------------------- +// Plik towarzyszacy obok legacy .eu7: .v2 +[[nodiscard]] std::string +eu7v2_sidecar_path( std::string const &eu7_path ) { + return eu7_path + ".v2"; +} + +// .v2 jest swiezy gdy istnieje i nie jest starszy niz zrodlowy .eu7. +[[nodiscard]] bool +eu7v2_sidecar_fresh( std::string const &eu7_path, std::string const &v2_path ) { + std::error_code ec; + if( false == std::filesystem::exists( v2_path, ec ) ) { + return false; + } + auto const v2_time { std::filesystem::last_write_time( v2_path, ec ) }; + if( ec ) { + return false; + } + auto const src_time { std::filesystem::last_write_time( eu7_path, ec ) }; + if( ec ) { + return false; + } + return v2_time >= src_time; +} + +// Odczyt modulu z chudego .v2. Stratny: odtwarza wylacznie scene (PROT/INST/MESH +// + rekordy), bez includes/pack/prototypow/flag. Zwraca false przy bledzie. +[[nodiscard]] bool +read_eu7v2_module( std::string const &v2_path, Eu7Module &out ) { + std::ifstream input { v2_path, std::ios::binary }; + if( !input ) { + return false; + } + std::vector bytes( + ( std::istreambuf_iterator( input ) ), + std::istreambuf_iterator() ); + if( bytes.empty() ) { + return false; + } + Eu7Scene scene; + if( false == eu7v2::load_scene( bytes.data(), bytes.size(), scene ) ) { + return false; + } + out = Eu7Module{}; + out.scene = std::move( scene ); + return true; +} + +// True gdy plik zaczyna sie magiem eu7v2 ('E','U','7','C'). +[[nodiscard]] bool +file_is_eu7v2( std::string const &path ) { + std::ifstream input { path, std::ios::binary }; + if( !input ) { + return false; + } + char magic[ 4 ] { 0, 0, 0, 0 }; + input.read( magic, 4 ); + if( input.gcount() != 4 ) { + return false; + } + return magic[ 0 ] == 'E' && magic[ 1 ] == 'U' && magic[ 2 ] == '7' && magic[ 3 ] == 'C'; +} + +// Pelny modul z formatu .eu7v2 (lossless): scena + includes + placement + flagi. +[[nodiscard]] bool +read_eu7v2_module_full( std::string const &path, Eu7Module &out ) { + std::ifstream input { path, std::ios::binary }; + if( !input ) { + return false; + } + std::vector bytes( + ( std::istreambuf_iterator( input ) ), + std::istreambuf_iterator() ); + if( bytes.empty() ) { + return false; + } + out = Eu7Module{}; + return eu7v2::load_module( bytes.data(), bytes.size(), out ); +} + +// Zapis chudego .v2 obok legacy .eu7 (best-effort, ciche niepowodzenie). +void +write_eu7v2_module( std::string const &v2_path, Eu7Module const &module ) { + auto const bytes { eu7v2::bake_scene( module.scene ) }; + std::ofstream output { v2_path, std::ios::binary | std::ios::trunc }; + if( !output ) { + return; + } + output.write( + reinterpret_cast( bytes.data() ), + static_cast( bytes.size() ) ); +} + +// Odczyt modulu z transkodem eu7v2 gdy Global.eu7v2_runtime wlaczone: +// - jest swiezy .v2 -> czytaj z niego (szybka, stratna sciezka), +// - brak/nieswiezy -> czytaj legacy i (dla nie-PACK) zapisz .v2 na nastepny raz. +[[nodiscard]] Eu7Module +read_module_maybe_v2( std::string const &resolved ) { + if( false == Global.eu7v2_runtime ) { + return read_module( resolved ); + } + // Primary path: the module file itself is a native .eu7v2 container + // (text -> eu7v2 bake, no legacy .eu7). Load it losslessly. + if( file_is_eu7v2( resolved ) ) { + Eu7Module v2_module; + if( read_eu7v2_module_full( resolved, v2_module ) ) { + ++load_stats().module_v2_loaded; + return v2_module; + } + throw std::runtime_error( "EU7v2: uszkodzony modul \"" + resolved + "\"" ); + } + // Legacy .eu7 with optional ".v2" sidecar transcode (older experiment). + auto const v2_path { eu7v2_sidecar_path( resolved ) }; + if( eu7v2_sidecar_fresh( resolved, v2_path ) ) { + Eu7Module v2_module; + if( read_eu7v2_module( v2_path, v2_module ) ) { + ++load_stats().module_v2_loaded; + return v2_module; + } + WriteLog( "EU7v2: nieudany odczyt \"" + v2_path + "\", fallback legacy .eu7" ); + } + auto legacy { read_module( resolved ) }; + if( false == legacy.has_pack_chunk ) { + write_eu7v2_module( v2_path, legacy ); + ++load_stats().module_v2_baked; + } + return legacy; +} + void apply_material( shape_node::shapenode_data &data, std::string material_name ) { replace_slashes( material_name ); @@ -212,7 +347,7 @@ load_module_recursive( } else { ScopedTimer const read_timer { load_stats().read_ms }; - owned_module = read_module( resolved ); + owned_module = read_module_maybe_v2( resolved ); ++load_stats().module_read; auto const emplaced { g_module_file_cache.emplace( resolved, std::move( owned_module ) ) }; template_module = &emplaced.first->second; @@ -370,13 +505,17 @@ resolve_scenery_path( std::string const &reference ) { std::string binary_path( std::string const &reference ) { auto path { resolve_scenery_path( reference ) }; + if( Global.eu7v2_runtime ) { + return eu7v2::binary_path_from_text( path ).generic_string(); + } + char const *const ext { ".eu7" }; if( path.ends_with( ".scm" ) || path.ends_with( ".sbt" ) || path.ends_with( ".inc" ) || path.ends_with( ".scn" ) ) { - path.replace( path.size() - 4, 4, ".eu7" ); + path.replace( path.size() - 4, 4, ext ); } else if( false == path.ends_with( ".eu7" ) ) { erase_extension( path ); - path += ".eu7"; + path += ext; } return path; } @@ -421,7 +560,13 @@ terrain_binary_path( std::string const &terrain_reference ) { bool probe_file( std::string const &path ) { - return FileExists( path ) && is_valid_eu7b_file( path ); + if( false == FileExists( path ) ) { + return false; + } + if( Global.eu7v2_runtime && file_is_eu7v2( path ) ) { + return true; + } + return is_valid_eu7b_file( path ); } bool diff --git a/scene/eu7/v2/eu7v2_bake.cpp b/scene/eu7/v2/eu7v2_bake.cpp new file mode 100644 index 00000000..73bfeb65 --- /dev/null +++ b/scene/eu7/v2/eu7v2_bake.cpp @@ -0,0 +1,561 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#include "scene/eu7/v2/eu7v2_bake.h" + +#include "scene/eu7/v2/eu7v2_format.h" +#include "scene/eu7/v2/eu7v2_scene.h" +#include "scene/eu7/v2/eu7v2_records.h" +#include "scene/eu7/eu7_types.h" + +#include +#include + +namespace eu7v2 { + +namespace { + +// Bakes the parser output into the dependency-free eu7v2 structs, owning the +// string table all chunks share. +class scene_baker { + public: + explicit scene_baker( scene::eu7::Eu7Scene const &scene ) : m_scene( scene ) {} + scene_baker( scene::eu7::Eu7Scene const &scene, scene::eu7::Eu7Module const &module ) + : m_scene( scene ), m_module( &module ) {} + + std::vector run() { + convert_models(); + convert_terrain(); + convert_shapes(); + convert_lines(); + convert_tracks(); + convert_traction(); + convert_power_sources(); + convert_memcells(); + convert_launchers(); + convert_events(); + convert_sounds(); + convert_dynamics(); + convert_trainsets(); + if( m_module != nullptr ) { + convert_includes(); + convert_meta(); + } + return serialize(); + } + + private: + [[nodiscard]] std::uint32_t str( std::string const &s ) { return m_strings.intern( s ); } + + [[nodiscard]] node_record node( scene::eu7::Eu7BasicNode const &n ) { + node_record out; + out.name = str( n.name ); + out.type = str( n.node_type ); + out.area_center = { n.area.center.x, n.area.center.y, n.area.center.z }; + out.area_radius = n.area.radius; + out.range_sq_min = n.range_squared_min; + out.range_sq_max = n.range_squared_max; + out.visible = n.visible; + return out; + } + + // -- models: deduplicate identical definitions into prototypes ---------- + void convert_models() { + for( auto const &model : m_scene.models ) { + auto const proto { intern_prototype( model ) }; + + model_instance inst; + inst.proto = proto; + inst.x = model.location.x; + inst.y = model.location.y; + inst.z = model.location.z; + inst.ax = static_cast( model.angles.x ); + inst.ay = static_cast( model.angles.y ); + inst.az = static_cast( model.angles.z ); + inst.sx = static_cast( model.scale.x ); + inst.sy = static_cast( model.scale.y ); + inst.sz = static_cast( model.scale.z ); + inst.cell_id = model.pack_cell_id; + inst.texture_override = kNoString; // within a single bake the proto already carries the skin + inst.has_node = true; + inst.node = node( model.node ); + m_instances.push_back( inst ); + } + } + + [[nodiscard]] std::uint32_t intern_prototype( scene::eu7::Eu7Model const &model ) { + std::string key; + key.reserve( model.model_file.size() + model.texture_file.size() + 32 ); + key.append( model.model_file ); + key.push_back( '\x1f' ); + key.append( model.texture_file ); + key.push_back( '\x1f' ); + key.push_back( model.transition ? '1' : '0' ); + key.push_back( model.is_terrain ? '1' : '0' ); + key.append( std::to_string( model.baked_range_min ) ); + key.push_back( '\x1f' ); + key.append( std::to_string( model.baked_range_max ) ); + key.push_back( '\x1f' ); + for( auto const s : model.light_states ) { + key.append( std::to_string( s ) ); + key.push_back( ',' ); + } + key.push_back( '\x1f' ); + for( auto const c : model.light_colors ) { + key.append( std::to_string( c ) ); + key.push_back( ',' ); + } + + auto const it { m_prototype_lookup.find( key ) }; + if( it != m_prototype_lookup.end() ) { + return it->second; + } + + model_prototype proto; + proto.model_file = str( model.model_file ); + proto.texture_file = model.texture_file.empty() ? kNoString : str( model.texture_file ); + proto.flags = 0; + if( model.transition ) { + proto.flags |= proto_flag::transition; + } + if( model.is_terrain ) { + proto.flags |= proto_flag::is_terrain; + } + if( model.pack_flags & scene::eu7::kEu7PackFlagInstanceableHint ) { + proto.flags |= proto_flag::instanceable; + } + proto.range_min = model.baked_range_min; + proto.range_max = model.baked_range_max; + proto.light_states = model.light_states; + proto.light_colors = model.light_colors; + + auto const id { static_cast( m_prototypes.size() ) }; + m_prototypes.push_back( std::move( proto ) ); + m_prototype_lookup.emplace( std::move( key ), id ); + return id; + } + + // -- terrain: store origin in f64, vertices f32 relative to origin ------ + void convert_terrain() { + for( auto const &shape : m_scene.terrain_shapes ) { + terrain_mesh mesh; + mesh.material = shape.material_path.empty() ? kNoString : str( shape.material_path ); + mesh.translucent = shape.translucent; + mesh.ox = shape.origin.x; + mesh.oy = shape.origin.y; + mesh.oz = shape.origin.z; + mesh.vertices.reserve( shape.vertices.size() ); + for( auto const &v : shape.vertices ) { + mesh_vertex mv; + mv.px = static_cast( v.position.x - shape.origin.x ); + mv.py = static_cast( v.position.y - shape.origin.y ); + mv.pz = static_cast( v.position.z - shape.origin.z ); + mv.nx = v.normal.x; + mv.ny = v.normal.y; + mv.nz = v.normal.z; + mv.u = static_cast( v.u ); + mv.v = static_cast( v.v ); + mesh.vertices.push_back( mv ); + } + m_meshes.push_back( std::move( mesh ) ); + } + } + + [[nodiscard]] lighting_block lighting( scene::eu7::Eu7LightingData const &l ) { + lighting_block out; + out.diffuse[ 0 ] = l.diffuse.x; out.diffuse[ 1 ] = l.diffuse.y; + out.diffuse[ 2 ] = l.diffuse.z; out.diffuse[ 3 ] = l.diffuse.w; + out.ambient[ 0 ] = l.ambient.x; out.ambient[ 1 ] = l.ambient.y; + out.ambient[ 2 ] = l.ambient.z; out.ambient[ 3 ] = l.ambient.w; + out.specular[ 0 ] = l.specular.x; out.specular[ 1 ] = l.specular.y; + out.specular[ 2 ] = l.specular.z; out.specular[ 3 ] = l.specular.w; + return out; + } + + // -- non-terrain shapes: same encoding as terrain but lossless on node --- + void convert_shapes() { + for( auto const &shape : m_scene.shapes ) { + shape_record r; + r.node = node( shape.node ); + r.translucent = shape.translucent; + r.material = shape.material_path.empty() ? kNoString : str( shape.material_path ); + r.lighting = lighting( shape.lighting ); + r.ox = shape.origin.x; + r.oy = shape.origin.y; + r.oz = shape.origin.z; + r.vertices.reserve( shape.vertices.size() ); + for( auto const &v : shape.vertices ) { + mesh_vertex mv; + mv.px = static_cast( v.position.x - shape.origin.x ); + mv.py = static_cast( v.position.y - shape.origin.y ); + mv.pz = static_cast( v.position.z - shape.origin.z ); + mv.nx = v.normal.x; + mv.ny = v.normal.y; + mv.nz = v.normal.z; + mv.u = static_cast( v.u ); + mv.v = static_cast( v.v ); + r.vertices.push_back( mv ); + } + m_shapes.push_back( std::move( r ) ); + } + } + + void convert_lines() { + for( auto const &line : m_scene.lines ) { + lines_record r; + r.node = node( line.node ); + r.lighting = lighting( line.lighting ); + r.line_width = line.line_width; + r.ox = line.origin.x; + r.oy = line.origin.y; + r.oz = line.origin.z; + r.vertices.reserve( line.vertices.size() ); + for( auto const &v : line.vertices ) { + r.vertices.push_back( { v.position.x, v.position.y, v.position.z } ); + } + m_lines.push_back( std::move( r ) ); + } + } + + void convert_tracks() { + for( auto const &t : m_scene.tracks ) { + track_record r; + r.node = node( t.node ); + r.track_type = static_cast( t.track_type ); + r.category = static_cast( t.category ); + r.length = t.length; + r.track_width = t.track_width; + r.friction = t.friction; + r.sound_distance = t.sound_distance; + r.quality_flag = t.quality_flag; + r.damage_flag = t.damage_flag; + r.environment = static_cast( t.environment ); + if( t.visibility.has_value() ) { + r.has_visibility = true; + r.visibility.material1 = str( t.visibility->material1 ); + r.visibility.tex_length = t.visibility->tex_length; + r.visibility.material2 = str( t.visibility->material2 ); + r.visibility.tex_height1 = t.visibility->tex_height1; + r.visibility.tex_width = t.visibility->tex_width; + r.visibility.tex_slope = t.visibility->tex_slope; + } + r.paths.reserve( t.paths.size() ); + for( auto const &p : t.paths ) { + track_path tp; + tp.p_start = { p.p_start.x, p.p_start.y, p.p_start.z }; + tp.roll_start = p.roll_start; + tp.cp_out = { p.cp_out.x, p.cp_out.y, p.cp_out.z }; + tp.cp_in = { p.cp_in.x, p.cp_in.y, p.cp_in.z }; + tp.p_end = { p.p_end.x, p.p_end.y, p.p_end.z }; + tp.roll_end = p.roll_end; + tp.radius = p.radius; + r.paths.push_back( tp ); + } + r.tail_keywords.reserve( t.tail_keywords.size() ); + for( auto const &kv : t.tail_keywords ) { + r.tail_keywords.emplace_back( str( kv.first ), str( kv.second ) ); + } + m_tracks.push_back( std::move( r ) ); + } + } + + void convert_traction() { + for( auto const &t : m_scene.traction ) { + traction_record r; + r.node = node( t.node ); + r.power_supply_name = str( t.power_supply_name ); + r.nominal_voltage = t.nominal_voltage; + r.max_current = t.max_current; + r.resistivity = t.resistivity_ohm_per_m; + r.material = static_cast( t.material ); + r.wire_thickness = t.wire_thickness; + r.damage_flag = t.damage_flag; + r.wire_p1 = { t.wire_p1.x, t.wire_p1.y, t.wire_p1.z }; + r.wire_p2 = { t.wire_p2.x, t.wire_p2.y, t.wire_p2.z }; + r.wire_p3 = { t.wire_p3.x, t.wire_p3.y, t.wire_p3.z }; + r.wire_p4 = { t.wire_p4.x, t.wire_p4.y, t.wire_p4.z }; + r.min_height = t.min_height; + r.segment_length = t.segment_length; + r.wire_count = t.wire_count; + r.wire_offset = t.wire_offset; + if( t.parallel_name.has_value() ) { + r.has_parallel = true; + r.parallel_name = str( *t.parallel_name ); + } + m_traction.push_back( std::move( r ) ); + } + } + + void convert_power_sources() { + for( auto const &p : m_scene.power_sources ) { + power_source_record r; + r.node = node( p.node ); + r.position = { p.position.x, p.position.y, p.position.z }; + r.nominal_voltage = p.nominal_voltage; + r.voltage_frequency = p.voltage_frequency; + r.internal_resistance = p.internal_resistance; + r.max_output_current = p.max_output_current; + r.fast_fuse_timeout = p.fast_fuse_timeout; + r.fast_fuse_repetition = p.fast_fuse_repetition; + r.slow_fuse_timeout = p.slow_fuse_timeout; + r.modifier = static_cast( p.modifier ); + m_power.push_back( std::move( r ) ); + } + } + + void convert_memcells() { + for( auto const &m : m_scene.memcells ) { + memcell_record r; + r.node = node( m.node ); + r.text = str( m.text ); + r.value1 = m.value1; + r.value2 = m.value2; + if( m.track_name.has_value() ) { + r.has_track = true; + r.track_name = str( *m.track_name ); + } + m_memcells.push_back( std::move( r ) ); + } + } + + void convert_launchers() { + for( auto const &l : m_scene.event_launchers ) { + launcher_record r; + r.node = node( l.node ); + r.location = { l.location.x, l.location.y, l.location.z }; + r.radius_squared = l.radius_squared; + r.activation_key = l.activation_key; + r.delta_time = l.delta_time; + r.event1_name = str( l.event1_name ); + r.event2_name = str( l.event2_name ); + if( l.condition.has_value() ) { + r.has_condition = true; + r.condition.memcell_name = str( l.condition->memcell_name ); + r.condition.compare_text = str( l.condition->compare_text ); + r.condition.compare_value1 = l.condition->compare_value1; + r.condition.compare_value2 = l.condition->compare_value2; + r.condition.check_mask = l.condition->check_mask; + } + r.train_triggered = l.train_triggered; + r.launch_hour = l.launch_hour; + r.launch_minute = l.launch_minute; + m_launchers.push_back( std::move( r ) ); + } + } + + void convert_events() { + for( auto const &e : m_scene.events ) { + event_record r; + r.name = str( e.name ); + r.type = static_cast( e.type ); + r.delay = e.delay; + r.delay_random = e.delay_random; + r.delay_departure = e.delay_departure; + r.ignored = e.ignored; + r.passive = e.passive; + r.targets.reserve( e.targets.size() ); + for( auto const &t : e.targets ) { + r.targets.push_back( str( t ) ); + } + r.payload.reserve( e.payload.size() ); + for( auto const &kv : e.payload ) { + r.payload.emplace_back( str( kv.first ), str( kv.second ) ); + } + m_events.push_back( std::move( r ) ); + } + } + + void convert_sounds() { + for( auto const &s : m_scene.sounds ) { + sound_record r; + r.node = node( s.node ); + r.location = { s.location.x, s.location.y, s.location.z }; + r.wav_file = str( s.wav_file ); + m_sounds.push_back( std::move( r ) ); + } + } + + void convert_dynamics() { + for( auto const &d : m_scene.dynamics ) { + dynamic_record r; + r.node = node( d.node ); + r.data_folder = str( d.data_folder ); + r.skin_file = str( d.skin_file ); + r.mmd_file = str( d.mmd_file ); + r.track_name = str( d.track_name ); + r.offset = d.offset; + r.driver_type = str( d.driver_type ); + r.coupling = d.coupling; + r.coupling_raw = str( d.coupling_raw ); + r.coupling_params = str( d.coupling_params ); + r.velocity = d.velocity; + r.load_count = d.load_count; + r.load_type = str( d.load_type ); + if( d.destination.has_value() ) { + r.has_destination = true; + r.destination = str( *d.destination ); + } + if( d.trainset_index.has_value() ) { + r.has_trainset = true; + r.trainset_index = static_cast( *d.trainset_index ); + } + m_dynamics.push_back( std::move( r ) ); + } + } + + void convert_trainsets() { + for( auto const &t : m_scene.trainsets ) { + trainset_record r; + r.name = str( t.name ); + r.track = str( t.track ); + r.offset = t.offset; + r.velocity = t.velocity; + r.assignment.reserve( t.assignment.size() ); + for( auto const &kv : t.assignment ) { + r.assignment.emplace_back( str( kv.first ), str( kv.second ) ); + } + r.vehicle_indices.reserve( t.vehicle_indices.size() ); + for( auto const idx : t.vehicle_indices ) { + r.vehicle_indices.push_back( static_cast( idx ) ); + } + r.couplings.reserve( t.couplings.size() ); + for( auto const c : t.couplings ) { + r.couplings.push_back( c ); + } + r.driver_index = + t.driver_index == static_cast( -1 ) + ? 0xffffffffu + : static_cast( t.driver_index ); + m_trainsets.push_back( std::move( r ) ); + } + } + + void convert_includes() { + for( auto const &inc : m_module->includes ) { + include_record r; + r.source_line = inc.source_line; + r.source_path = inc.source_path.empty() ? kNoString : str( inc.source_path ); + r.binary_path = inc.binary_path.empty() ? kNoString : str( inc.binary_path ); + r.parameters.reserve( inc.parameters.size() ); + for( auto const &p : inc.parameters ) { + r.parameters.push_back( str( p ) ); + } + r.site_transform = transform( inc.site_transform ); + m_includes.push_back( std::move( r ) ); + } + } + + [[nodiscard]] transform_record transform( scene::eu7::Eu7TransformContext const &t ) { + transform_record out; + out.origin_stack.reserve( t.origin_stack.size() ); + for( auto const &v : t.origin_stack ) { + out.origin_stack.push_back( { v.x, v.y, v.z } ); + } + out.scale_stack.reserve( t.scale_stack.size() ); + for( auto const &v : t.scale_stack ) { + out.scale_stack.push_back( { v.x, v.y, v.z } ); + } + out.rotation = { t.rotation.x, t.rotation.y, t.rotation.z }; + out.group_depth = static_cast( t.group_depth ); + return out; + } + + void convert_meta() { + m_meta.first_init_count = m_scene.first_init_count; + m_meta.has_terrain_chunk = m_module->has_terrain_chunk; + m_meta.has_pack_chunk = m_module->has_pack_chunk; + m_meta.placement_origin_x = m_module->include_placement.origin_x_param; + m_meta.placement_origin_y = m_module->include_placement.origin_y_param; + m_meta.placement_origin_z = m_module->include_placement.origin_z_param; + m_meta.placement_rotation_y = m_module->include_placement.rotation_y_param; + m_has_meta = true; + } + + [[nodiscard]] std::vector serialize() const { + container_writer writer( file_kind::sim ); + + byte_writer strs; + m_strings.serialize( strs ); + writer.add_chunk( chunk::strs, strs ); + + // META must be emitted before the chunks it describes are consumed; + // a single pass over chunks resolves it (loader reads it whenever seen). + if( m_has_meta ) { + byte_writer meta; + write_meta( meta, m_meta ); + writer.add_chunk( chunk::meta, meta ); + } + + auto emit { [&]( std::uint32_t id, auto const &write_fn, auto const &items ) { + if( items.empty() ) { + return; + } + byte_writer payload; + write_fn( payload, items ); + writer.add_chunk( id, payload ); + } }; + + emit( chunk::incl, &write_includes, m_includes ); + emit( chunk::prot, &write_prototypes, m_prototypes ); + emit( chunk::inst, &write_instances, m_instances ); + emit( chunk::mesh, &write_terrain_meshes, m_meshes ); + emit( chunk::shpe, &write_shapes, m_shapes ); + emit( chunk::line, &write_lines, m_lines ); + emit( chunk::trak, &write_tracks, m_tracks ); + emit( chunk::trac, &write_traction, m_traction ); + emit( chunk::pwrs, &write_power_sources, m_power ); + emit( chunk::memc, &write_memcells, m_memcells ); + emit( chunk::laun, &write_launchers, m_launchers ); + emit( chunk::evnt, &write_events, m_events ); + emit( chunk::sond, &write_sounds, m_sounds ); + emit( chunk::dynm, &write_dynamics, m_dynamics ); + emit( chunk::trst, &write_trainsets, m_trainsets ); + + return writer.data(); + } + + scene::eu7::Eu7Scene const &m_scene; + scene::eu7::Eu7Module const *m_module { nullptr }; + string_table m_strings; + + std::vector m_prototypes; + std::unordered_map m_prototype_lookup; + std::vector m_instances; + std::vector m_meshes; + std::vector m_shapes; + std::vector m_lines; + std::vector m_tracks; + std::vector m_traction; + std::vector m_power; + std::vector m_memcells; + std::vector m_launchers; + std::vector m_events; + std::vector m_sounds; + std::vector m_dynamics; + std::vector m_trainsets; + std::vector m_includes; + module_meta m_meta; + bool m_has_meta { false }; +}; + +} // namespace + +std::vector +bake_scene( scene::eu7::Eu7Scene const &scene ) { + scene_baker baker( scene ); + return baker.run(); +} + +std::vector +bake_module( scene::eu7::Eu7Module const &module ) { + scene_baker baker( module.scene, module ); + return baker.run(); +} + +} // namespace eu7v2 diff --git a/scene/eu7/v2/eu7v2_bake.h b/scene/eu7/v2/eu7v2_bake.h new file mode 100644 index 00000000..1094e733 --- /dev/null +++ b/scene/eu7/v2/eu7v2_bake.h @@ -0,0 +1,36 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +#include +#include + +namespace scene::eu7 { +struct Eu7Scene; +struct Eu7Module; +} // namespace scene::eu7 + +namespace eu7v2 { + +// Bakes a parsed scene (output of the text parser) into a self-contained +// eu7v2 "sim" container: STRS + PROT/INST (deduplicated models) + MESH +// (terrain) + SHPE/LINE + simulation record chunks. This is the clean-slate +// replacement for the legacy EU7B emitter; it reuses the existing parser +// front-end and only changes the on-disk representation. +[[nodiscard]] std::vector +bake_scene( scene::eu7::Eu7Scene const &scene ); + +// Lossless module bake: everything bake_scene() emits, plus INCL (includes +// for recursion) and META (flags, placement, first_init_count). This is the +// format the runtime loads as a module. +[[nodiscard]] std::vector +bake_module( scene::eu7::Eu7Module const &module ); + +} // namespace eu7v2 diff --git a/scene/eu7/v2/eu7v2_emit_runtime.cpp b/scene/eu7/v2/eu7v2_emit_runtime.cpp new file mode 100644 index 00000000..8d86005f --- /dev/null +++ b/scene/eu7/v2/eu7v2_emit_runtime.cpp @@ -0,0 +1,888 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#include "scene/eu7/v2/eu7v2_emit_runtime.h" + +#include "scene/eu7/v2/eu7v2_format.h" +#include "scene/eu7/v2/eu7v2_scene.h" +#include "scene/eu7/v2/eu7v2_records.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace eu7v2 { + +namespace { + +namespace rt = eu07::scene::runtime; +namespace bake = eu07::scene::bake; +namespace codec = eu07::scene::binary::codec; + +[[nodiscard]] bool is_model_inc_placement( bake::ModuleInclude const &inc ) { + if( !eu07::scene::detail::isIncFile( inc.sourcePath ) ) { + return false; + } + if( inc.parameters.size() < 5 ) { + return false; + } + try { + (void)std::stod( inc.parameters[ 1 ] ); + (void)std::stod( inc.parameters[ 2 ] ); + (void)std::stod( inc.parameters[ 3 ] ); + (void)std::stod( inc.parameters[ 4 ] ); + return true; + } catch( ... ) { + return false; + } +} + +[[nodiscard]] bool try_parse_inc_placement( + std::span const params, + double &x, + double &y, + double &z, + float &rot_y ) { + if( params.size() < 5 ) { + return false; + } + try { + x = std::stod( params[ 1 ] ); + y = std::stod( params[ 2 ] ); + z = std::stod( params[ 3 ] ); + rot_y = static_cast( std::stod( params[ 4 ] ) ); + return true; + } catch( ... ) { + return false; + } +} + +[[nodiscard]] file_kind file_kind_for_text_path( std::filesystem::path const &text_path ) { + auto const ext { text_path.extension().string() }; + if( ext == ".inc" ) { + return file_kind::module; + } + return file_kind::sim; +} + +// Mirror of scene_baker but sourced from the parser's Runtime* records. +class runtime_baker { + public: + runtime_baker( + bake::RuntimeModule const &module, + bool const is_root, + std::vector const *pack_batches, + bake::PackModelSpoolFile const *pack_spool, + bake::ShapeSpoolFile const *shape_spool, + std::filesystem::path const &text_path ) + : m_module( module ) + , m_is_root( is_root ) + , m_pack_batches( pack_batches ) + , m_pack_spool( pack_spool ) + , m_shape_spool( shape_spool ) + , m_file_kind( file_kind_for_text_path( text_path ) ) {} + + [[nodiscard]] std::vector run() { + convert_models(); + convert_shapes(); + convert_lines(); + convert_tracks(); + convert_traction(); + convert_power_sources(); + convert_memcells(); + convert_launchers(); + convert_events(); + convert_sounds(); + convert_dynamics(); + convert_trainsets(); + convert_includes(); + convert_meta(); + return serialize(); + } + + [[nodiscard]] std::size_t model_total() const { return m_instances.size(); } + [[nodiscard]] std::size_t placement_total() const { return m_placements.size(); } + [[nodiscard]] std::size_t structural_include_total() const { return m_includes.size(); } + + private: + [[nodiscard]] std::uint32_t str( std::string const &s ) { return m_strings.intern( s ); } + [[nodiscard]] std::uint32_t opt_str( std::string const &s ) { + return s.empty() ? kNoString : m_strings.intern( s ); + } + + [[nodiscard]] node_record node( rt::BasicNode const &n ) { + node_record out; + out.name = opt_str( n.name ); + out.type = opt_str( n.nodeType ); + out.area_center = { n.area.center.x, n.area.center.y, n.area.center.z }; + out.area_radius = n.area.radius; + out.range_sq_min = n.rangeSquaredMin; + out.range_sq_max = n.rangeSquaredMax; + out.visible = n.visible; + return out; + } + + [[nodiscard]] lighting_block lighting( rt::LightingData const &l ) { + lighting_block out; + out.diffuse[ 0 ] = l.diffuse.x; out.diffuse[ 1 ] = l.diffuse.y; + out.diffuse[ 2 ] = l.diffuse.z; out.diffuse[ 3 ] = l.diffuse.w; + out.ambient[ 0 ] = l.ambient.x; out.ambient[ 1 ] = l.ambient.y; + out.ambient[ 2 ] = l.ambient.z; out.ambient[ 3 ] = l.ambient.w; + out.specular[ 0 ] = l.specular.x; out.specular[ 1 ] = l.specular.y; + out.specular[ 2 ] = l.specular.z; out.specular[ 3 ] = l.specular.w; + return out; + } + + [[nodiscard]] std::uint32_t intern_prototype( rt::RuntimeModelInstance const &model ) { + std::string key; + key.reserve( model.modelFile.size() + model.textureFile.size() + 32 ); + key.append( model.modelFile ); + key.push_back( '\x1f' ); + key.append( model.textureFile ); + key.push_back( '\x1f' ); + key.push_back( model.transition ? '1' : '0' ); + key.push_back( model.isTerrain ? '1' : '0' ); + for( auto const s : model.lightStates ) { + key.append( std::to_string( s ) ); + key.push_back( ',' ); + } + key.push_back( '\x1f' ); + for( auto const c : model.lightColors ) { + key.append( std::to_string( c ) ); + key.push_back( ',' ); + } + + auto const it { m_prototype_lookup.find( key ) }; + if( it != m_prototype_lookup.end() ) { + return it->second; + } + + model_prototype proto; + proto.model_file = str( model.modelFile ); + proto.texture_file = opt_str( model.textureFile ); + proto.flags = 0; + if( model.transition ) { + proto.flags |= proto_flag::transition; + } + if( model.isTerrain ) { + proto.flags |= proto_flag::is_terrain; + } + proto.range_min = -1.f; + proto.range_max = -1.f; + proto.light_states = model.lightStates; + proto.light_colors = model.lightColors; + + auto const id { static_cast( m_prototypes.size() ) }; + m_prototypes.push_back( std::move( proto ) ); + m_prototype_lookup.emplace( std::move( key ), id ); + return id; + } + + void emit_model( rt::RuntimeModelInstance const &model ) { + auto const proto { intern_prototype( model ) }; + model_instance inst; + inst.proto = proto; + inst.x = model.location.x; + inst.y = model.location.y; + inst.z = model.location.z; + inst.ax = static_cast( model.angles.x ); + inst.ay = static_cast( model.angles.y ); + inst.az = static_cast( model.angles.z ); + inst.sx = static_cast( model.scale.x ); + inst.sy = static_cast( model.scale.y ); + inst.sz = static_cast( model.scale.z ); + inst.cell_id = 0xffu; + inst.texture_override = kNoString; + inst.has_node = true; + inst.node = node( model.node ); + m_instances.push_back( std::move( inst ) ); + } + + void convert_models() { + if( m_is_root && m_pack_spool != nullptr ) { + m_pack_spool->for_each_model( [&]( rt::RuntimeModelInstance const &model ) { + emit_model( model ); + } ); + } + if( m_is_root && m_pack_batches != nullptr ) { + // Root: models live in the flattened PACK batches, not scene.models. + for( auto const &batch : *m_pack_batches ) { + for( auto const &model : batch.models ) { + emit_model( model ); + } + } + } + for( auto const &model : m_module.scene.models ) { + emit_model( model ); + } + } + + [[nodiscard]] shape_record make_shape_record( rt::RuntimeShapeNode const &shape ) { + shape_record r; + r.node = node( shape.node ); + r.translucent = shape.translucent; + r.material = opt_str( shape.materialPath ); + r.lighting = lighting( shape.lighting ); + r.ox = shape.origin.x; + r.oy = shape.origin.y; + r.oz = shape.origin.z; + r.vertices.reserve( shape.vertices.size() ); + for( auto const &v : shape.vertices ) { + mesh_vertex mv; + mv.px = static_cast( v.position.x - shape.origin.x ); + mv.py = static_cast( v.position.y - shape.origin.y ); + mv.pz = static_cast( v.position.z - shape.origin.z ); + mv.nx = static_cast( v.normal.x ); + mv.ny = static_cast( v.normal.y ); + mv.nz = static_cast( v.normal.z ); + mv.u = static_cast( v.u ); + mv.v = static_cast( v.v ); + r.vertices.push_back( mv ); + } + return r; + } + + void convert_shapes() { + if( m_shape_spool != nullptr ) { + m_shape_spool->for_each_shape( [&]( rt::RuntimeShapeNode const &shape ) { + (void)opt_str( shape.materialPath ); + (void)node( shape.node ); + } ); + return; + } + for( auto const &shape : m_module.scene.shapes ) { + m_shapes.push_back( make_shape_record( shape ) ); + } + } + + void convert_lines() { + for( auto const &line : m_module.scene.lines ) { + lines_record r; + r.node = node( line.node ); + r.lighting = lighting( line.lighting ); + r.line_width = line.lineWidth; + r.ox = line.origin.x; + r.oy = line.origin.y; + r.oz = line.origin.z; + r.vertices.reserve( line.vertices.size() ); + for( auto const &v : line.vertices ) { + r.vertices.push_back( { v.position.x, v.position.y, v.position.z } ); + } + m_lines.push_back( std::move( r ) ); + } + } + + void convert_tracks() { + for( auto const &t : m_module.scene.tracks ) { + track_record r; + r.node = node( t.node ); + r.track_type = static_cast( t.trackType ); + r.category = static_cast( t.category ); + r.length = t.length; + r.track_width = t.trackWidth; + r.friction = t.friction; + r.sound_distance = t.soundDistance; + r.quality_flag = t.qualityFlag; + r.damage_flag = t.damageFlag; + r.environment = static_cast( t.environment ); + if( t.visibility.has_value() ) { + r.has_visibility = true; + r.visibility.material1 = str( t.visibility->material1 ); + r.visibility.tex_length = t.visibility->texLength; + r.visibility.material2 = str( t.visibility->material2 ); + r.visibility.tex_height1 = t.visibility->texHeight1; + r.visibility.tex_width = t.visibility->texWidth; + r.visibility.tex_slope = t.visibility->texSlope; + } + r.paths.reserve( t.paths.size() ); + for( auto const &p : t.paths ) { + track_path tp; + tp.p_start = { p.pStart.x, p.pStart.y, p.pStart.z }; + tp.roll_start = p.rollStart; + tp.cp_out = { p.cpOut.x, p.cpOut.y, p.cpOut.z }; + tp.cp_in = { p.cpIn.x, p.cpIn.y, p.cpIn.z }; + tp.p_end = { p.pEnd.x, p.pEnd.y, p.pEnd.z }; + tp.roll_end = p.rollEnd; + tp.radius = p.radius; + r.paths.push_back( tp ); + } + r.tail_keywords.reserve( t.tailKeywords.size() ); + for( auto const &kv : t.tailKeywords ) { + r.tail_keywords.emplace_back( str( kv.first ), str( kv.second ) ); + } + m_tracks.push_back( std::move( r ) ); + } + } + + void convert_traction() { + for( auto const &t : m_module.scene.traction ) { + traction_record r; + r.node = node( t.node ); + r.power_supply_name = opt_str( t.powerSupplyName ); + r.nominal_voltage = t.nominalVoltage; + r.max_current = t.maxCurrent; + r.resistivity = t.resistivityOhmPerM; + r.material = static_cast( t.material ); + r.wire_thickness = t.wireThickness; + r.damage_flag = t.damageFlag; + r.wire_p1 = { t.wireP1.x, t.wireP1.y, t.wireP1.z }; + r.wire_p2 = { t.wireP2.x, t.wireP2.y, t.wireP2.z }; + r.wire_p3 = { t.wireP3.x, t.wireP3.y, t.wireP3.z }; + r.wire_p4 = { t.wireP4.x, t.wireP4.y, t.wireP4.z }; + r.min_height = t.minHeight; + r.segment_length = t.segmentLength; + r.wire_count = t.wireCount; + r.wire_offset = t.wireOffset; + if( t.parallelName.has_value() ) { + r.has_parallel = true; + r.parallel_name = str( *t.parallelName ); + } + m_traction.push_back( std::move( r ) ); + } + } + + void convert_power_sources() { + for( auto const &p : m_module.scene.powerSources ) { + power_source_record r; + r.node = node( p.node ); + r.position = { p.position.x, p.position.y, p.position.z }; + r.nominal_voltage = p.nominalVoltage; + r.voltage_frequency = p.voltageFrequency; + r.internal_resistance = p.internalResistance; + r.max_output_current = p.maxOutputCurrent; + r.fast_fuse_timeout = p.fastFuseTimeout; + r.fast_fuse_repetition = p.fastFuseRepetition; + r.slow_fuse_timeout = p.slowFuseTimeout; + r.modifier = static_cast( p.modifier ); + m_power.push_back( std::move( r ) ); + } + } + + void convert_memcells() { + for( auto const &m : m_module.scene.memcells ) { + memcell_record r; + r.node = node( m.node ); + r.text = opt_str( m.text ); + r.value1 = m.value1; + r.value2 = m.value2; + if( m.trackName.has_value() ) { + r.has_track = true; + r.track_name = str( *m.trackName ); + } + m_memcells.push_back( std::move( r ) ); + } + } + + void convert_launchers() { + for( auto const &l : m_module.scene.eventLaunchers ) { + launcher_record r; + r.node = node( l.node ); + r.location = { l.location.x, l.location.y, l.location.z }; + r.radius_squared = l.radiusSquared; + r.activation_key = l.activationKey; + r.delta_time = l.deltaTime; + r.event1_name = opt_str( l.event1Name ); + r.event2_name = opt_str( l.event2Name ); + if( l.condition.has_value() ) { + r.has_condition = true; + r.condition.memcell_name = str( l.condition->memcellName ); + r.condition.compare_text = str( l.condition->compareText ); + r.condition.compare_value1 = l.condition->compareValue1; + r.condition.compare_value2 = l.condition->compareValue2; + r.condition.check_mask = l.condition->checkMask; + } + r.train_triggered = l.trainTriggered; + r.launch_hour = l.launchHour; + r.launch_minute = l.launchMinute; + m_launchers.push_back( std::move( r ) ); + } + } + + void convert_events() { + for( auto const &e : m_module.scene.events ) { + event_record r; + r.name = opt_str( e.name ); + r.type = static_cast( e.type ); + r.delay = e.delay; + r.delay_random = e.delayRandom; + r.delay_departure = e.delayDeparture; + r.ignored = e.ignored; + r.passive = e.passive; + r.targets.reserve( e.targets.size() ); + for( auto const &t : e.targets ) { + r.targets.push_back( str( t ) ); + } + r.payload.reserve( e.payload.size() ); + for( auto const &kv : e.payload ) { + r.payload.emplace_back( str( kv.first ), str( kv.second ) ); + } + m_events.push_back( std::move( r ) ); + } + } + + void convert_sounds() { + for( auto const &s : m_module.scene.sounds ) { + sound_record r; + r.node = node( s.node ); + r.location = { s.location.x, s.location.y, s.location.z }; + r.wav_file = opt_str( s.wavFile ); + m_sounds.push_back( std::move( r ) ); + } + } + + void convert_dynamics() { + for( auto const &d : m_module.scene.dynamics ) { + dynamic_record r; + r.node = node( d.node ); + r.data_folder = opt_str( d.dataFolder ); + r.skin_file = opt_str( d.skinFile ); + r.mmd_file = opt_str( d.mmdFile ); + r.track_name = opt_str( d.trackName ); + r.offset = d.offset; + r.driver_type = opt_str( d.driverType ); + r.coupling = d.coupling; + r.coupling_raw = opt_str( d.couplingRaw ); + r.coupling_params = opt_str( d.couplingParams ); + r.velocity = d.velocity; + r.load_count = d.loadCount; + r.load_type = opt_str( d.loadType ); + if( d.destination.has_value() ) { + r.has_destination = true; + r.destination = str( *d.destination ); + } + if( d.trainsetIndex.has_value() ) { + r.has_trainset = true; + r.trainset_index = static_cast( *d.trainsetIndex ); + } + m_dynamics.push_back( std::move( r ) ); + } + } + + void convert_trainsets() { + for( auto const &t : m_module.scene.trainsets ) { + trainset_record r; + r.name = opt_str( t.name ); + r.track = opt_str( t.track ); + r.offset = t.offset; + r.velocity = t.velocity; + r.assignment.reserve( t.assignment.size() ); + for( auto const &kv : t.assignment ) { + r.assignment.emplace_back( str( kv.first ), str( kv.second ) ); + } + r.vehicle_indices.reserve( t.vehicleIndices.size() ); + for( auto const idx : t.vehicleIndices ) { + r.vehicle_indices.push_back( static_cast( idx ) ); + } + r.couplings.reserve( t.couplings.size() ); + for( auto const c : t.couplings ) { + r.couplings.push_back( c ); + } + r.driver_index = + t.driverIndex == static_cast( -1 ) + ? 0xffffffffu + : static_cast( t.driverIndex ); + m_trainsets.push_back( std::move( r ) ); + } + } + + [[nodiscard]] transform_record transform( rt::TransformContext const &t ) { + transform_record out; + out.origin_stack.reserve( t.originStack.size() ); + for( auto const &v : t.originStack ) { + out.origin_stack.push_back( { v.x, v.y, v.z } ); + } + out.scale_stack.reserve( t.scaleStack.size() ); + for( auto const &v : t.scaleStack ) { + out.scale_stack.push_back( { v.x, v.y, v.z } ); + } + out.rotation = { t.rotation.x, t.rotation.y, t.rotation.z }; + out.group_depth = static_cast( t.groupStackDepth ); + return out; + } + + void convert_includes() { + for( auto const &inc : m_module.includes ) { + if( is_model_inc_placement( inc ) ) { + double x { 0.0 }, y { 0.0 }, z { 0.0 }; + float rot_y { 0.f }; + if( !try_parse_inc_placement( inc.parameters, x, y, z, rot_y ) ) { + continue; + } + module_placement_record p; + p.module_path = str( + binary_path_from_text( std::filesystem::path { inc.sourcePath } ) + .generic_string() ); + p.texture_override = + ( inc.parameters.empty() || inc.parameters[ 0 ] == "none" ) + ? kNoString + : str( inc.parameters[ 0 ] ); + p.x = x; + p.y = y; + p.z = z; + p.rotation_y = rot_y; + m_placements.push_back( std::move( p ) ); + continue; + } + include_record r; + r.source_line = inc.sourceLine; + r.source_path = opt_str( inc.sourcePath ); + r.binary_path = inc.sourcePath.empty() + ? kNoString + : str( binary_path_from_text( + std::filesystem::path { inc.sourcePath } ) + .generic_string() ); + r.parameters.reserve( inc.parameters.size() ); + for( auto const ¶m : inc.parameters ) { + r.parameters.push_back( str( param ) ); + } + r.site_transform = transform( inc.siteTransform ); + m_includes.push_back( std::move( r ) ); + } + } + + void convert_meta() { + m_meta.first_init_count = m_module.scene.firstInitCount; + m_meta.has_terrain_chunk = false; + // Root PACK batches are flattened into plain INST records, so the file + // carries no PACK streaming chunk; never advertise one. + m_meta.has_pack_chunk = false; + m_meta.placement_origin_x = m_module.includePlacement.origin_x_param; + m_meta.placement_origin_y = m_module.includePlacement.origin_y_param; + m_meta.placement_origin_z = m_module.includePlacement.origin_z_param; + m_meta.placement_rotation_y = m_module.includePlacement.rotation_y_param; + } + + [[nodiscard]] std::vector serialize() { + container_writer writer( m_file_kind ); + + byte_writer strs; + m_strings.serialize( strs ); + writer.add_chunk( chunk::strs, strs ); + + byte_writer meta; + write_meta( meta, m_meta ); + writer.add_chunk( chunk::meta, meta ); + + auto emit { [&]( std::uint32_t id, auto const &write_fn, auto const &items ) { + if( items.empty() ) { + return; + } + byte_writer payload; + write_fn( payload, items ); + writer.add_chunk( id, payload ); + } }; + + emit( chunk::incl, &write_includes, m_includes ); + emit( chunk::plce, &write_module_placements, m_placements ); + emit( chunk::prot, &write_prototypes, m_prototypes ); + emit( chunk::inst, &write_instances, m_instances ); + if( m_shape_spool != nullptr && m_shape_spool->shape_count() != 0 ) { + byte_writer payload; + payload.put_u32( static_cast( m_shape_spool->shape_count() ) ); + m_shape_spool->for_each_shape( [&]( rt::RuntimeShapeNode const &shape ) { + write_shape_record( payload, make_shape_record( shape ) ); + } ); + writer.add_chunk( chunk::shpe, payload ); + } else { + emit( chunk::shpe, &write_shapes, m_shapes ); + } + emit( chunk::line, &write_lines, m_lines ); + emit( chunk::trak, &write_tracks, m_tracks ); + emit( chunk::trac, &write_traction, m_traction ); + emit( chunk::pwrs, &write_power_sources, m_power ); + emit( chunk::memc, &write_memcells, m_memcells ); + emit( chunk::laun, &write_launchers, m_launchers ); + emit( chunk::evnt, &write_events, m_events ); + emit( chunk::sond, &write_sounds, m_sounds ); + emit( chunk::dynm, &write_dynamics, m_dynamics ); + emit( chunk::trst, &write_trainsets, m_trainsets ); + + return writer.data(); + } + + bake::RuntimeModule const &m_module; + bool m_is_root; + std::vector const *m_pack_batches; + bake::PackModelSpoolFile const *m_pack_spool; + bake::ShapeSpoolFile const *m_shape_spool; + file_kind m_file_kind { file_kind::sim }; + std::vector m_placements; + string_table m_strings; + + std::vector m_prototypes; + std::unordered_map m_prototype_lookup; + std::vector m_instances; + std::vector m_shapes; + std::vector m_lines; + std::vector m_tracks; + std::vector m_traction; + std::vector m_power; + std::vector m_memcells; + std::vector m_launchers; + std::vector m_events; + std::vector m_sounds; + std::vector m_dynamics; + std::vector m_trainsets; + std::vector m_includes; + module_meta m_meta; +}; + +// Decoded record counts of an emitted eu7v2 image (chunk-by-chunk). +struct decoded_counts { + std::size_t includes { 0 }; + std::size_t placements { 0 }; + std::size_t instances { 0 }; + std::size_t shapes { 0 }; + std::size_t lines { 0 }; + std::size_t tracks { 0 }; + std::size_t traction { 0 }; + std::size_t power { 0 }; + std::size_t memcells { 0 }; + std::size_t launchers { 0 }; + std::size_t events { 0 }; + std::size_t sounds { 0 }; + std::size_t dynamics { 0 }; + std::size_t trainsets { 0 }; + bool ok { true }; +}; + +[[nodiscard]] decoded_counts decode_counts( std::vector const &bytes ) { + decoded_counts c; + try { + container_reader reader( bytes.data(), bytes.size() ); + chunk_view chunk; + while( reader.next( chunk ) ) { + auto r { chunk.reader() }; + switch( chunk.id ) { + case chunk::incl: c.includes = read_includes( r ).size(); break; + case chunk::plce: c.placements = read_module_placements( r ).size(); break; + case chunk::inst: c.instances = read_instances( r ).size(); break; + case chunk::shpe: c.shapes = read_shapes( r ).size(); break; + case chunk::line: c.lines = read_lines( r ).size(); break; + case chunk::trak: c.tracks = read_tracks( r ).size(); break; + case chunk::trac: c.traction = read_traction( r ).size(); break; + case chunk::pwrs: c.power = read_power_sources( r ).size(); break; + case chunk::memc: c.memcells = read_memcells( r ).size(); break; + case chunk::laun: c.launchers = read_launchers( r ).size(); break; + case chunk::evnt: c.events = read_events( r ).size(); break; + case chunk::sond: c.sounds = read_sounds( r ).size(); break; + case chunk::dynm: c.dynamics = read_dynamics( r ).size(); break; + case chunk::trst: c.trainsets = read_trainsets( r ).size(); break; + default: break; + } + } + } + catch( parse_error const & ) { + c.ok = false; + } + return c; +} + +[[nodiscard]] std::filesystem::path eu7v2_path_for( std::filesystem::path const &text_path ) { + return binary_path_from_text( text_path ); +} + +} // namespace + +std::vector +emit_runtime_module_bytes( + bake::RuntimeModule const &module, + bool const is_root, + std::vector const *pack_batches, + bake::PackModelSpoolFile const *pack_spool, + bake::ShapeSpoolFile const *shape_spool, + std::filesystem::path const &text_path ) { + runtime_baker baker( module, is_root, pack_batches, pack_spool, shape_spool, text_path ); + return baker.run(); +} + +emit_outcome +emit_runtime_module( + bake::RuntimeModule const &module, + std::filesystem::path const &text_path, + bool const is_root, + std::vector const *pack_batches, + bool const verify, + bake::PackModelSpoolFile const *pack_spool, + bake::ShapeSpoolFile const *shape_spool ) { + emit_outcome outcome; + + auto const ms_since { []( std::chrono::steady_clock::time_point const t0 ) { + return std::chrono::duration( + std::chrono::steady_clock::now() - t0 ) + .count(); + } }; + + auto const build_begin { std::chrono::steady_clock::now() }; + runtime_baker baker( module, is_root, pack_batches, pack_spool, shape_spool, text_path ); + std::vector const bytes { baker.run() }; + outcome.model_total = baker.model_total(); + outcome.byte_size = bytes.size(); + outcome.build_ms = ms_since( build_begin ); + + auto const out_path { eu7v2_path_for( text_path ) }; + outcome.written_path = out_path.generic_string(); + + { + auto const write_begin { std::chrono::steady_clock::now() }; + std::ofstream output { out_path, std::ios::binary | std::ios::trunc }; + if( !output ) { + outcome.ok = false; + outcome.message = "nie mozna zapisac " + outcome.written_path; + return outcome; + } + output.write( + reinterpret_cast( bytes.data() ), + static_cast( bytes.size() ) ); + output.flush(); + outcome.write_ms = ms_since( write_begin ); + } + + if( !verify ) { + return outcome; + } + + auto const verify_begin { std::chrono::steady_clock::now() }; + outcome.verified = true; + decoded_counts const dec { decode_counts( bytes ) }; + + std::size_t src_models { module.scene.models.size() }; + if( is_root && pack_spool != nullptr ) { + src_models += pack_spool->model_count(); + } + if( is_root && pack_batches != nullptr ) { + for( auto const &batch : *pack_batches ) { + src_models += batch.models.size(); + } + } + + std::size_t src_placements { 0 }; + std::size_t src_structural { 0 }; + for( auto const &inc : module.includes ) { + if( is_model_inc_placement( inc ) ) { + ++src_placements; + } else { + ++src_structural; + } + } + + std::ostringstream report; + bool pass { dec.ok }; + auto cmp { [&]( char const *label, std::size_t const src, std::size_t const got ) { + bool const ok { src == got }; + pass = pass && ok; + report << " " << ( ok ? "ok " : "FAIL" ) << ' ' << label << " src=" << src + << " eu7v2=" << got << '\n'; + } }; + + cmp( "includes", src_structural, dec.includes ); + cmp( "placements", src_placements, dec.placements ); + cmp( "models", src_models, dec.instances ); + cmp( "shapes", module.scene.shapes.size() + ( shape_spool != nullptr ? shape_spool->shape_count() : 0 ), + dec.shapes ); + cmp( "lines", module.scene.lines.size(), dec.lines ); + cmp( "tracks", module.scene.tracks.size(), dec.tracks ); + cmp( "traction", module.scene.traction.size(), dec.traction ); + cmp( "power", module.scene.powerSources.size(), dec.power ); + cmp( "memcells", module.scene.memcells.size(), dec.memcells ); + cmp( "launchers", module.scene.eventLaunchers.size(), dec.launchers ); + cmp( "events", module.scene.events.size(), dec.events ); + cmp( "sounds", module.scene.sounds.size(), dec.sounds ); + cmp( "dynamics", module.scene.dynamics.size(), dec.dynamics ); + cmp( "trainsets", module.scene.trainsets.size(), dec.trainsets ); + + outcome.verify_ok = pass; + outcome.message = report.str(); + outcome.verify_ms = ms_since( verify_begin ); + return outcome; +} + +bool +verify_written_module( + std::filesystem::path const &path, + module_verify_spec const &spec, + bool const is_root, + std::size_t const pack_models, + std::string *message_out ) { + std::ifstream input { path, std::ios::binary }; + if( !input ) { + if( message_out != nullptr ) { + *message_out = "nie mozna odczytac " + path.generic_string(); + } + return false; + } + + input.seekg( 0, std::ios::end ); + const std::streamoff file_size { input.tellg() }; + input.seekg( 0, std::ios::beg ); + if( file_size <= 0 ) { + if( message_out != nullptr ) { + *message_out = "pusty plik " + path.generic_string(); + } + return false; + } + + std::vector bytes( static_cast( file_size ) ); + input.read( + reinterpret_cast( bytes.data() ), + static_cast( bytes.size() ) ); + + decoded_counts const dec { decode_counts( bytes ) }; + if( !dec.ok ) { + if( message_out != nullptr ) { + *message_out = "decode blad " + path.generic_string(); + } + return false; + } + + std::size_t src_models { spec.models }; + if( is_root ) { + src_models += pack_models; + } + + std::ostringstream report; + bool pass { dec.ok }; + auto cmp { [&]( char const *label, std::size_t const src, std::size_t const got ) { + bool const ok { src == got }; + pass = pass && ok; + report << " " << ( ok ? "ok " : "FAIL" ) << ' ' << label << " src=" << src + << " eu7v2=" << got << '\n'; + } }; + + cmp( "includes", spec.includes, dec.includes ); + cmp( "placements", spec.placements, dec.placements ); + cmp( "models", src_models, dec.instances ); + cmp( "shapes", spec.shapes, dec.shapes ); + cmp( "lines", spec.lines, dec.lines ); + cmp( "tracks", spec.tracks, dec.tracks ); + cmp( "traction", spec.traction, dec.traction ); + cmp( "power", spec.power, dec.power ); + cmp( "memcells", spec.memcells, dec.memcells ); + cmp( "launchers", spec.launchers, dec.launchers ); + cmp( "events", spec.events, dec.events ); + cmp( "sounds", spec.sounds, dec.sounds ); + cmp( "dynamics", spec.dynamics, dec.dynamics ); + cmp( "trainsets", spec.trainsets, dec.trainsets ); + + if( message_out != nullptr ) { + *message_out = report.str(); + } + return pass; +} + +} // namespace eu7v2 diff --git a/scene/eu7/v2/eu7v2_emit_runtime.h b/scene/eu7/v2/eu7v2_emit_runtime.h new file mode 100644 index 00000000..38cd30b0 --- /dev/null +++ b/scene/eu7/v2/eu7v2_emit_runtime.h @@ -0,0 +1,99 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +// --------------------------------------------------------------------------- +// RuntimeScene -> eu7v2 emitter. Lives in the eu07_bake target because it +// touches the parser's RuntimeModule types directly. It maps Runtime* records +// into the same dependency-free eu7v2 structs that scene_baker uses (one +// writer/reader per record), so the produced .eu7v2 matches what the engine +// loads via eu7v2::load_module. For the scenario root the flattened PACK model +// batches are deduplicated into prototypes + instances. +// --------------------------------------------------------------------------- + +#include +#include + +#include +#include +#include +#include +#include + +namespace eu07::scene::bake { +class PackModelSpoolFile; +class ShapeSpoolFile; +} + +namespace eu7v2 { + +struct emit_outcome { + bool ok { true }; + bool verified { false }; + bool verify_ok { true }; + std::string written_path; + std::string message; + std::size_t model_total { 0 }; + std::size_t byte_size { 0 }; + double build_ms { 0.0 }; + double write_ms { 0.0 }; + double verify_ms { 0.0 }; +}; + +struct module_verify_spec { + std::size_t includes { 0 }; + std::size_t placements { 0 }; + std::size_t models { 0 }; + std::size_t shapes { 0 }; + std::size_t lines { 0 }; + std::size_t tracks { 0 }; + std::size_t traction { 0 }; + std::size_t power { 0 }; + std::size_t memcells { 0 }; + std::size_t launchers { 0 }; + std::size_t events { 0 }; + std::size_t sounds { 0 }; + std::size_t dynamics { 0 }; + std::size_t trainsets { 0 }; +}; + +// Builds the eu7v2 byte image for one baked module. +[[nodiscard]] std::vector +emit_runtime_module_bytes( + eu07::scene::bake::RuntimeModule const &module, + bool is_root, + std::vector const *pack_batches, + eu07::scene::bake::PackModelSpoolFile const *pack_spool = nullptr, + eu07::scene::bake::ShapeSpoolFile const *shape_spool = nullptr, + std::filesystem::path const &text_path = {} ); + +// Emits the module, writes ".eu7v2" next to the text source, and (when +// verify is true) reloads the bytes and compares record counts against the +// source RuntimeModule, filling outcome.message with a PASS/FAIL report. +[[nodiscard]] emit_outcome +emit_runtime_module( + eu07::scene::bake::RuntimeModule const &module, + std::filesystem::path const &text_path, + bool is_root, + std::vector const *pack_batches, + bool verify, + eu07::scene::bake::PackModelSpoolFile const *pack_spool = nullptr, + eu07::scene::bake::ShapeSpoolFile const *shape_spool = nullptr ); + +// Porownanie zapisanej eu7v2 z oczekiwanymi licznikami (do odroczonego verify). +[[nodiscard]] bool +verify_written_module( + std::filesystem::path const &path, + module_verify_spec const &spec, + bool const is_root, + std::size_t const pack_models, + std::string *message_out = nullptr ); + +} // namespace eu7v2 diff --git a/scene/eu7/v2/eu7v2_format.h b/scene/eu7/v2/eu7v2_format.h new file mode 100644 index 00000000..466a07c1 --- /dev/null +++ b/scene/eu7/v2/eu7v2_format.h @@ -0,0 +1,383 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +// --------------------------------------------------------------------------- +// eu7v2 - clean-slate compiled scenery format core (no backward compatibility +// with the legacy EU7B v4..v13 chunks). +// +// Design goals: +// * self-describing, skippable chunks: [u32 id][u64 size][payload] +// * explicit little-endian on disk so files are deterministic & portable +// * a central string table so payloads reference strings by u32 id +// * a separation of file kinds: sim core / reusable module / streamable tile +// * header-only and dependency-free so it can be unit tested in isolation +// +// This is the foundation layer only (iteration 1). Higher layers (baker, +// runtime loader, streaming) build on top of these primitives. +// --------------------------------------------------------------------------- + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace eu7v2 { + +// FourCC packed little-endian: 'A' is the least significant byte so the bytes +// appear in source order when the file is viewed in a hex editor. +[[nodiscard]] constexpr std::uint32_t +fourcc( char const a, char const b, char const c, char const d ) noexcept { + return ( static_cast( static_cast( a ) ) ) + | ( static_cast( static_cast( b ) ) << 8 ) + | ( static_cast( static_cast( c ) ) << 16 ) + | ( static_cast( static_cast( d ) ) << 24 ); +} + +// File magic + single current version. No legacy versions are recognised. +constexpr std::uint32_t kMagic { fourcc( 'E', 'U', '7', 'C' ) }; +constexpr std::uint16_t kVersion { 1 }; + +enum class file_kind : std::uint16_t { + sim = 1, // global simulation core, loaded once + module = 2, // reusable module / prototype library (baked .inc) + tile = 3, // streamable 1km visual tile (terrain + instances) + manifest = 4, // index of tiles & modules for a map +}; + +// Chunk identifiers (FourCC). Kept small & explicit; grows as layers land. +namespace chunk { +constexpr std::uint32_t strs { fourcc( 'S', 'T', 'R', 'S' ) }; // string table +constexpr std::uint32_t meta { fourcc( 'M', 'E', 'T', 'A' ) }; // key/value metadata +constexpr std::uint32_t prot { fourcc( 'P', 'R', 'O', 'T' ) }; // model prototypes (module) +constexpr std::uint32_t inst { fourcc( 'I', 'N', 'S', 'T' ) }; // lean instances (tile) +constexpr std::uint32_t mesh { fourcc( 'M', 'E', 'S', 'H' ) }; // baked terrain mesh (tile) +constexpr std::uint32_t shpe { fourcc( 'S', 'H', 'P', 'E' ) }; // non-terrain shapes (triangles) +constexpr std::uint32_t line { fourcc( 'L', 'I', 'N', 'E' ) }; // line geometry nodes +constexpr std::uint32_t incl { fourcc( 'I', 'N', 'C', 'L' ) }; // module includes (recursion refs) +constexpr std::uint32_t plce { fourcc( 'P', 'L', 'C', 'E' ) }; // lean .inc placements (binary coords) +constexpr std::uint32_t trst { fourcc( 'T', 'R', 'S', 'T' ) }; // trainsets +constexpr std::uint32_t trgr { fourcc( 'T', 'R', 'G', 'R' ) }; // precomputed track graph (sim) +constexpr std::uint32_t sidx { fourcc( 'S', 'I', 'D', 'X' ) }; // spatial section index (sim/manifest) +// simulation record chunks (sim) +constexpr std::uint32_t trak { fourcc( 'T', 'R', 'A', 'K' ) }; // tracks +constexpr std::uint32_t trac { fourcc( 'T', 'R', 'A', 'C' ) }; // traction wires +constexpr std::uint32_t pwrs { fourcc( 'P', 'W', 'R', 'S' ) }; // traction power sources +constexpr std::uint32_t memc { fourcc( 'M', 'E', 'M', 'C' ) }; // memory cells +constexpr std::uint32_t laun { fourcc( 'L', 'A', 'U', 'N' ) }; // event launchers +constexpr std::uint32_t evnt { fourcc( 'E', 'V', 'N', 'T' ) }; // events +constexpr std::uint32_t sond { fourcc( 'S', 'O', 'N', 'D' ) }; // sounds +constexpr std::uint32_t dynm { fourcc( 'D', 'Y', 'N', 'M' ) }; // dynamic vehicles +} // namespace chunk + +// --------------------------------------------------------------------------- +// Writer side +// --------------------------------------------------------------------------- + +// Little-endian byte sink. All multi-byte values are written explicitly so the +// output does not depend on host endianness. +class byte_writer { + public: + void put_u8( std::uint8_t const v ) { m_data.push_back( v ); } + + void put_u16( std::uint16_t const v ) { + put_u8( static_cast( v ) ); + put_u8( static_cast( v >> 8 ) ); + } + + void put_u32( std::uint32_t const v ) { + put_u16( static_cast( v ) ); + put_u16( static_cast( v >> 16 ) ); + } + + void put_u64( std::uint64_t const v ) { + put_u32( static_cast( v ) ); + put_u32( static_cast( v >> 32 ) ); + } + + void put_i32( std::int32_t const v ) { + put_u32( static_cast( v ) ); + } + + void put_f32( float const v ) { + std::uint32_t bits; + std::memcpy( &bits, &v, sizeof( bits ) ); + put_u32( bits ); + } + + // doubles are stored as f32 on disk (positions excepted via put_f64 where + // the extra precision actually matters) + void put_f64( double const v ) { + std::uint64_t bits; + std::memcpy( &bits, &v, sizeof( bits ) ); + put_u64( bits ); + } + + void put_vec3f( double const x, double const y, double const z ) { + put_f32( static_cast( x ) ); + put_f32( static_cast( y ) ); + put_f32( static_cast( z ) ); + } + + void put_bytes( void const *data, std::size_t const size ) { + auto const *bytes { static_cast( data ) }; + m_data.insert( m_data.end(), bytes, bytes + size ); + } + + [[nodiscard]] std::vector const &data() const noexcept { return m_data; } + [[nodiscard]] std::vector &data() noexcept { return m_data; } + [[nodiscard]] std::size_t size() const noexcept { return m_data.size(); } + + private: + std::vector m_data; +}; + +// Deduplicating string table. Returns a stable u32 id per unique string. +class string_table { + public: + [[nodiscard]] std::uint32_t intern( std::string_view const s ) { + auto const key { std::string( s ) }; + auto const it { m_lookup.find( key ) }; + if( it != m_lookup.end() ) { + return it->second; + } + auto const id { static_cast( m_strings.size() ) }; + m_strings.emplace_back( key ); + m_lookup.emplace( key, id ); + return id; + } + + [[nodiscard]] std::size_t size() const noexcept { return m_strings.size(); } + + // Serialise as: [u32 count]( [u32 len][bytes] )* + void serialize( byte_writer &out ) const { + out.put_u32( static_cast( m_strings.size() ) ); + for( auto const &s : m_strings ) { + out.put_u32( static_cast( s.size() ) ); + out.put_bytes( s.data(), s.size() ); + } + } + + private: + std::vector m_strings; + std::unordered_map m_lookup; +}; + +// Top-level container writer: file header followed by [id][size][payload] chunks. +class container_writer { + public: + explicit container_writer( file_kind const kind ) : m_kind( kind ) { + m_out.put_u32( kMagic ); + m_out.put_u16( kVersion ); + m_out.put_u16( static_cast( kind ) ); + // reserved for flags / future use, keeps header 16-byte aligned + m_out.put_u32( 0 ); + m_out.put_u32( 0 ); + } + + // Append a fully-built chunk payload under the given id. + void add_chunk( std::uint32_t const id, std::vector const &payload ) { + m_out.put_u32( id ); + m_out.put_u64( static_cast( payload.size() ) ); + m_out.put_bytes( payload.data(), payload.size() ); + } + + void add_chunk( std::uint32_t const id, byte_writer const &payload ) { + add_chunk( id, payload.data() ); + } + + [[nodiscard]] file_kind kind() const noexcept { return m_kind; } + [[nodiscard]] std::vector const &data() const noexcept { return m_out.data(); } + + private: + file_kind m_kind; + byte_writer m_out; +}; + +// --------------------------------------------------------------------------- +// Reader side (bounds-checked) +// --------------------------------------------------------------------------- + +class parse_error : public std::runtime_error { + public: + explicit parse_error( std::string const &what ) : std::runtime_error( what ) {} +}; + +// Little-endian cursor over a byte span. Every read is bounds-checked and +// throws parse_error on truncation so a corrupt file can never read OOB. +class byte_reader { + public: + byte_reader( std::uint8_t const *data, std::size_t const size ) noexcept + : m_data( data ), m_size( size ) {} + + [[nodiscard]] std::size_t remaining() const noexcept { return m_size - m_pos; } + [[nodiscard]] bool empty() const noexcept { return m_pos >= m_size; } + [[nodiscard]] std::size_t position() const noexcept { return m_pos; } + + std::uint8_t get_u8() { + require( 1 ); + return m_data[ m_pos++ ]; + } + + std::uint16_t get_u16() { + std::uint16_t const lo { get_u8() }; + std::uint16_t const hi { get_u8() }; + return static_cast( lo | ( hi << 8 ) ); + } + + std::uint32_t get_u32() { + std::uint32_t const lo { get_u16() }; + std::uint32_t const hi { get_u16() }; + return lo | ( hi << 16 ); + } + + std::uint64_t get_u64() { + std::uint64_t const lo { get_u32() }; + std::uint64_t const hi { get_u32() }; + return lo | ( hi << 32 ); + } + + std::int32_t get_i32() { return static_cast( get_u32() ); } + + float get_f32() { + std::uint32_t const bits { get_u32() }; + float v; + std::memcpy( &v, &bits, sizeof( v ) ); + return v; + } + + double get_f64() { + std::uint64_t const bits { get_u64() }; + double v; + std::memcpy( &v, &bits, sizeof( v ) ); + return v; + } + + void get_bytes( void *dst, std::size_t const size ) { + require( size ); + std::memcpy( dst, m_data + m_pos, size ); + m_pos += size; + } + + [[nodiscard]] std::uint8_t const *take( std::size_t const size ) { + require( size ); + auto const *ptr { m_data + m_pos }; + m_pos += size; + return ptr; + } + + private: + void require( std::size_t const n ) const { + if( m_pos + n > m_size ) { + throw parse_error( "eu7v2: unexpected end of data" ); + } + } + + std::uint8_t const *m_data; + std::size_t m_size; + std::size_t m_pos { 0 }; +}; + +// Decoded string table: id -> string view into the owned backing store. +class string_pool { + public: + void deserialize( byte_reader &in ) { + auto const count { in.get_u32() }; + m_strings.clear(); + m_strings.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + auto const len { in.get_u32() }; + std::string s; + s.resize( len ); + if( len != 0 ) { + in.get_bytes( s.data(), len ); + } + m_strings.emplace_back( std::move( s ) ); + } + } + + [[nodiscard]] std::string const &get( std::uint32_t const id ) const { + if( id >= m_strings.size() ) { + throw parse_error( "eu7v2: string id out of range" ); + } + return m_strings[ id ]; + } + + [[nodiscard]] std::size_t size() const noexcept { return m_strings.size(); } + + private: + std::vector m_strings; +}; + +// One chunk located inside a container: its id and a view of its payload. +struct chunk_view { + std::uint32_t id { 0 }; + std::uint8_t const *data { nullptr }; + std::size_t size { 0 }; + + [[nodiscard]] byte_reader reader() const noexcept { return byte_reader( data, size ); } +}; + +// Reads the file header and iterates chunks without copying payloads. +class container_reader { + public: + container_reader( std::uint8_t const *data, std::size_t const size ) : m_cursor( data, size ) { + auto const magic { m_cursor.get_u32() }; + if( magic != kMagic ) { + throw parse_error( "eu7v2: bad magic" ); + } + m_version = m_cursor.get_u16(); + if( m_version != kVersion ) { + throw parse_error( "eu7v2: unsupported version" ); + } + m_kind = static_cast( m_cursor.get_u16() ); + (void)m_cursor.get_u32(); // reserved + (void)m_cursor.get_u32(); // reserved + } + + [[nodiscard]] file_kind kind() const noexcept { return m_kind; } + [[nodiscard]] std::uint16_t version() const noexcept { return m_version; } + + // Returns false when there are no more chunks. + [[nodiscard]] bool next( chunk_view &out ) { + if( m_cursor.remaining() == 0 ) { + return false; + } + out.id = m_cursor.get_u32(); + auto const size { m_cursor.get_u64() }; + out.size = static_cast( size ); + out.data = m_cursor.take( out.size ); + return true; + } + + private: + byte_reader m_cursor; + file_kind m_kind { file_kind::sim }; + std::uint16_t m_version { 0 }; +}; + +// Maps a text scenery source to its compiled .eu7v2 path. Standard module +// extensions (.scm/.scn/.sbt/.inc) become ".eu7v2"; other extensions +// (.ctr, …) keep the source suffix so e.g. foo.scm and foo.ctr do not collide. +[[nodiscard]] inline std::filesystem::path +binary_path_from_text( std::filesystem::path const &text_path ) { + auto const ext { text_path.extension().string() }; + if( ext == ".scm" || ext == ".scn" || ext == ".sbt" || ext == ".inc" ) { + return text_path.parent_path() / ( text_path.stem().string() + ".eu7v2" ); + } + return text_path.parent_path() / ( text_path.filename().string() + ".eu7v2" ); +} + +} // namespace eu7v2 diff --git a/scene/eu7/v2/eu7v2_load.cpp b/scene/eu7/v2/eu7v2_load.cpp new file mode 100644 index 00000000..4d62c45a --- /dev/null +++ b/scene/eu7/v2/eu7v2_load.cpp @@ -0,0 +1,515 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#include "scene/eu7/v2/eu7v2_load.h" + +#include "scene/eu7/v2/eu7v2_format.h" +#include "scene/eu7/v2/eu7v2_scene.h" +#include "scene/eu7/v2/eu7v2_records.h" +#include "scene/eu7/eu7_types.h" + +#include + +namespace eu7v2 { + +namespace { + +// Reverse of scene_baker: turns decoded eu7v2 structs back into Eu7Scene. +class scene_loader { + public: + scene_loader( string_pool const &pool, scene::eu7::Eu7Scene &out ) + : m_pool( pool ), m_out( out ) {} + + [[nodiscard]] std::string str( std::uint32_t const id ) const { + return id == kNoString ? std::string() : m_pool.get( id ); + } + + void apply_node( scene::eu7::Eu7BasicNode &n, node_record const &r ) const { + n.name = str( r.name ); + n.node_type = str( r.type ); + n.area.center = { r.area_center.x, r.area_center.y, r.area_center.z }; + n.area.radius = r.area_radius; + n.range_squared_min = r.range_sq_min; + n.range_squared_max = r.range_sq_max; + n.visible = r.visible; + } + + void build_models( + std::vector const &protos, + std::vector const &instances ) const { + m_out.models.reserve( instances.size() ); + for( auto const &inst : instances ) { + if( inst.proto >= protos.size() ) { + continue; + } + auto const &proto { protos[ inst.proto ] }; + scene::eu7::Eu7Model m; + m.location = { inst.x, inst.y, inst.z }; + m.angles = { inst.ax, inst.ay, inst.az }; + m.scale = { inst.sx, inst.sy, inst.sz }; + m.model_file = str( proto.model_file ); + m.texture_file = + inst.texture_override != kNoString ? str( inst.texture_override ) + : str( proto.texture_file ); + m.light_states = proto.light_states; + m.light_colors = proto.light_colors; + m.transition = ( proto.flags & proto_flag::transition ) != 0; + m.is_terrain = ( proto.flags & proto_flag::is_terrain ) != 0; + m.pack_flags = + ( proto.flags & proto_flag::instanceable ) ? scene::eu7::kEu7PackFlagInstanceableHint : 0u; + m.baked_range_min = proto.range_min; + m.baked_range_max = proto.range_max; + m.pack_cell_id = inst.cell_id; + if( inst.has_node ) { + apply_node( m.node, inst.node ); + } + else { + m.node.name = m.model_file; + m.node.node_type = "model"; + m.node.area.center = m.location; + } + m_out.models.push_back( std::move( m ) ); + } + } + + static void apply_lighting( + scene::eu7::Eu7LightingData &dst, lighting_block const &src ) { + dst.diffuse = { src.diffuse[ 0 ], src.diffuse[ 1 ], src.diffuse[ 2 ], src.diffuse[ 3 ] }; + dst.ambient = { src.ambient[ 0 ], src.ambient[ 1 ], src.ambient[ 2 ], src.ambient[ 3 ] }; + dst.specular = { src.specular[ 0 ], src.specular[ 1 ], src.specular[ 2 ], src.specular[ 3 ] }; + } + + void build_shapes( std::vector const &shapes ) const { + m_out.shapes.reserve( shapes.size() ); + for( auto const &r : shapes ) { + scene::eu7::Eu7Shape s; + apply_node( s.node, r.node ); + s.translucent = r.translucent; + s.material_path = str( r.material ); + apply_lighting( s.lighting, r.lighting ); + s.origin = { r.ox, r.oy, r.oz }; + s.vertices.reserve( r.vertices.size() ); + for( auto const &v : r.vertices ) { + scene::eu7::Eu7WorldVertex wv; + wv.position = { + r.ox + static_cast( v.px ), + r.oy + static_cast( v.py ), + r.oz + static_cast( v.pz ) }; + wv.normal = { v.nx, v.ny, v.nz }; + wv.u = v.u; + wv.v = v.v; + s.vertices.push_back( wv ); + } + m_out.shapes.push_back( std::move( s ) ); + } + } + + void build_lines( std::vector const &items ) const { + m_out.lines.reserve( items.size() ); + for( auto const &r : items ) { + scene::eu7::Eu7Lines l; + apply_node( l.node, r.node ); + apply_lighting( l.lighting, r.lighting ); + l.line_width = r.line_width; + l.origin = { r.ox, r.oy, r.oz }; + l.vertices.reserve( r.vertices.size() ); + for( auto const &v : r.vertices ) { + scene::eu7::Eu7WorldVertex wv; + wv.position = { v.x, v.y, v.z }; + l.vertices.push_back( wv ); + } + m_out.lines.push_back( std::move( l ) ); + } + } + + void build_trainsets( std::vector const &items ) const { + m_out.trainsets.reserve( items.size() ); + for( auto const &r : items ) { + scene::eu7::Eu7Trainset t; + t.name = str( r.name ); + t.track = str( r.track ); + t.offset = r.offset; + t.velocity = r.velocity; + for( auto const &kv : r.assignment ) { + t.assignment.emplace( str( kv.first ), str( kv.second ) ); + } + t.vehicle_indices.reserve( r.vehicle_indices.size() ); + for( auto const idx : r.vehicle_indices ) { + t.vehicle_indices.push_back( static_cast( idx ) ); + } + t.couplings.reserve( r.couplings.size() ); + for( auto const c : r.couplings ) { + t.couplings.push_back( c ); + } + t.driver_index = + r.driver_index == 0xffffffffu + ? static_cast( -1 ) + : static_cast( r.driver_index ); + m_out.trainsets.push_back( std::move( t ) ); + } + } + + void build_terrain( std::vector const &meshes ) const { + m_out.terrain_shapes.reserve( meshes.size() ); + for( auto const &mesh : meshes ) { + scene::eu7::Eu7Shape s; + s.material_path = str( mesh.material ); + s.translucent = mesh.translucent; + s.origin = { mesh.ox, mesh.oy, mesh.oz }; + s.vertices.reserve( mesh.vertices.size() ); + for( auto const &v : mesh.vertices ) { + scene::eu7::Eu7WorldVertex wv; + wv.position = { + mesh.ox + static_cast( v.px ), + mesh.oy + static_cast( v.py ), + mesh.oz + static_cast( v.pz ) }; + wv.normal = { v.nx, v.ny, v.nz }; + wv.u = v.u; + wv.v = v.v; + s.vertices.push_back( wv ); + } + m_out.terrain_shapes.push_back( std::move( s ) ); + } + } + + void build_tracks( std::vector const &tracks ) const { + m_out.tracks.reserve( tracks.size() ); + for( auto const &r : tracks ) { + scene::eu7::Eu7Track t; + apply_node( t.node, r.node ); + t.track_type = static_cast( r.track_type ); + t.category = static_cast( r.category ); + t.length = r.length; + t.track_width = r.track_width; + t.friction = r.friction; + t.sound_distance = r.sound_distance; + t.quality_flag = r.quality_flag; + t.damage_flag = r.damage_flag; + t.environment = static_cast( r.environment ); + if( r.has_visibility ) { + scene::eu7::Eu7TrackVisibility vis; + vis.material1 = str( r.visibility.material1 ); + vis.tex_length = r.visibility.tex_length; + vis.material2 = str( r.visibility.material2 ); + vis.tex_height1 = r.visibility.tex_height1; + vis.tex_width = r.visibility.tex_width; + vis.tex_slope = r.visibility.tex_slope; + t.visibility = vis; + } + t.paths.reserve( r.paths.size() ); + for( auto const &p : r.paths ) { + scene::eu7::Eu7SegmentPath sp; + sp.p_start = { p.p_start.x, p.p_start.y, p.p_start.z }; + sp.roll_start = p.roll_start; + sp.cp_out = { p.cp_out.x, p.cp_out.y, p.cp_out.z }; + sp.cp_in = { p.cp_in.x, p.cp_in.y, p.cp_in.z }; + sp.p_end = { p.p_end.x, p.p_end.y, p.p_end.z }; + sp.roll_end = p.roll_end; + sp.radius = p.radius; + t.paths.push_back( sp ); + } + t.tail_keywords.reserve( r.tail_keywords.size() ); + for( auto const &kv : r.tail_keywords ) { + t.tail_keywords.emplace_back( str( kv.first ), str( kv.second ) ); + } + m_out.tracks.push_back( std::move( t ) ); + } + } + + void build_traction( std::vector const &items ) const { + m_out.traction.reserve( items.size() ); + for( auto const &r : items ) { + scene::eu7::Eu7Traction t; + apply_node( t.node, r.node ); + t.power_supply_name = str( r.power_supply_name ); + t.nominal_voltage = r.nominal_voltage; + t.max_current = r.max_current; + t.resistivity_ohm_per_m = r.resistivity; + t.material = static_cast( r.material ); + t.wire_thickness = r.wire_thickness; + t.damage_flag = r.damage_flag; + t.wire_p1 = { r.wire_p1.x, r.wire_p1.y, r.wire_p1.z }; + t.wire_p2 = { r.wire_p2.x, r.wire_p2.y, r.wire_p2.z }; + t.wire_p3 = { r.wire_p3.x, r.wire_p3.y, r.wire_p3.z }; + t.wire_p4 = { r.wire_p4.x, r.wire_p4.y, r.wire_p4.z }; + t.min_height = r.min_height; + t.segment_length = r.segment_length; + t.wire_count = r.wire_count; + t.wire_offset = r.wire_offset; + if( r.has_parallel ) { + t.parallel_name = str( r.parallel_name ); + } + m_out.traction.push_back( std::move( t ) ); + } + } + + void build_power_sources( std::vector const &items ) const { + m_out.power_sources.reserve( items.size() ); + for( auto const &r : items ) { + scene::eu7::Eu7TractionPowerSource p; + apply_node( p.node, r.node ); + p.position = { r.position.x, r.position.y, r.position.z }; + p.nominal_voltage = r.nominal_voltage; + p.voltage_frequency = r.voltage_frequency; + p.internal_resistance = r.internal_resistance; + p.max_output_current = r.max_output_current; + p.fast_fuse_timeout = r.fast_fuse_timeout; + p.fast_fuse_repetition = r.fast_fuse_repetition; + p.slow_fuse_timeout = r.slow_fuse_timeout; + p.modifier = static_cast( r.modifier ); + m_out.power_sources.push_back( std::move( p ) ); + } + } + + void build_memcells( std::vector const &items ) const { + m_out.memcells.reserve( items.size() ); + for( auto const &r : items ) { + scene::eu7::Eu7MemCell m; + apply_node( m.node, r.node ); + m.text = str( r.text ); + m.value1 = r.value1; + m.value2 = r.value2; + if( r.has_track ) { + m.track_name = str( r.track_name ); + } + m_out.memcells.push_back( std::move( m ) ); + } + } + + void build_launchers( std::vector const &items ) const { + m_out.event_launchers.reserve( items.size() ); + for( auto const &r : items ) { + scene::eu7::Eu7EventLauncher l; + apply_node( l.node, r.node ); + l.location = { r.location.x, r.location.y, r.location.z }; + l.radius_squared = r.radius_squared; + l.activation_key = r.activation_key; + l.delta_time = r.delta_time; + l.event1_name = str( r.event1_name ); + l.event2_name = str( r.event2_name ); + if( r.has_condition ) { + scene::eu7::Eu7EventLauncherCondition c; + c.memcell_name = str( r.condition.memcell_name ); + c.compare_text = str( r.condition.compare_text ); + c.compare_value1 = r.condition.compare_value1; + c.compare_value2 = r.condition.compare_value2; + c.check_mask = r.condition.check_mask; + l.condition = c; + } + l.train_triggered = r.train_triggered; + l.launch_hour = r.launch_hour; + l.launch_minute = r.launch_minute; + m_out.event_launchers.push_back( std::move( l ) ); + } + } + + void build_events( std::vector const &items ) const { + m_out.events.reserve( items.size() ); + for( auto const &r : items ) { + scene::eu7::Eu7Event e; + e.name = str( r.name ); + e.type = static_cast( r.type ); + e.delay = r.delay; + e.delay_random = r.delay_random; + e.delay_departure = r.delay_departure; + e.ignored = r.ignored; + e.passive = r.passive; + e.targets.reserve( r.targets.size() ); + for( auto const t : r.targets ) { + e.targets.push_back( str( t ) ); + } + e.payload.reserve( r.payload.size() ); + for( auto const &kv : r.payload ) { + e.payload.emplace_back( str( kv.first ), str( kv.second ) ); + } + m_out.events.push_back( std::move( e ) ); + } + } + + void build_sounds( std::vector const &items ) const { + m_out.sounds.reserve( items.size() ); + for( auto const &r : items ) { + scene::eu7::Eu7Sound s; + apply_node( s.node, r.node ); + s.location = { r.location.x, r.location.y, r.location.z }; + s.wav_file = str( r.wav_file ); + m_out.sounds.push_back( std::move( s ) ); + } + } + + void build_dynamics( std::vector const &items ) const { + m_out.dynamics.reserve( items.size() ); + for( auto const &r : items ) { + scene::eu7::Eu7Dynamic d; + apply_node( d.node, r.node ); + d.data_folder = str( r.data_folder ); + d.skin_file = str( r.skin_file ); + d.mmd_file = str( r.mmd_file ); + d.track_name = str( r.track_name ); + d.offset = r.offset; + d.driver_type = str( r.driver_type ); + d.coupling = r.coupling; + d.coupling_raw = str( r.coupling_raw ); + d.coupling_params = str( r.coupling_params ); + d.velocity = r.velocity; + d.load_count = r.load_count; + d.load_type = str( r.load_type ); + if( r.has_destination ) { + d.destination = str( r.destination ); + } + if( r.has_trainset ) { + d.trainset_index = static_cast( r.trainset_index ); + } + m_out.dynamics.push_back( std::move( d ) ); + } + } + + private: + string_pool const &m_pool; + scene::eu7::Eu7Scene &m_out; +}; + +} // namespace + +namespace { + +// Shared scan: reconstructs the Eu7Scene; when module != nullptr it also +// reconstructs includes / placement / flags / first_init_count. +[[nodiscard]] bool +load_into( + std::uint8_t const *data, + std::size_t const size, + scene::eu7::Eu7Scene &out, + scene::eu7::Eu7Module *module ) { + try { + container_reader reader( data, size ); + if( reader.kind() != file_kind::sim && reader.kind() != file_kind::module ) { + return false; + } + + string_pool pool; + std::vector protos; + std::vector instances; + std::vector includes; + std::vector placements; + bool saw_meta { false }; + module_meta meta; + + // STRS must be resolved before the records that reference it; the baker + // always writes it first, so a single pass over chunks is enough. + chunk_view chunk; + scene_loader loader( pool, out ); + while( reader.next( chunk ) ) { + auto r { chunk.reader() }; + switch( chunk.id ) { + case chunk::strs: pool.deserialize( r ); break; + case chunk::meta: meta = read_meta( r ); saw_meta = true; break; + case chunk::incl: includes = read_includes( r ); break; + case chunk::plce: placements = read_module_placements( r ); break; + case chunk::prot: protos = read_prototypes( r ); break; + case chunk::inst: instances = read_instances( r ); break; + case chunk::mesh: loader.build_terrain( read_terrain_meshes( r ) ); break; + case chunk::shpe: loader.build_shapes( read_shapes( r ) ); break; + case chunk::line: loader.build_lines( read_lines( r ) ); break; + case chunk::trak: loader.build_tracks( read_tracks( r ) ); break; + case chunk::trac: loader.build_traction( read_traction( r ) ); break; + case chunk::pwrs: loader.build_power_sources( read_power_sources( r ) ); break; + case chunk::memc: loader.build_memcells( read_memcells( r ) ); break; + case chunk::laun: loader.build_launchers( read_launchers( r ) ); break; + case chunk::evnt: loader.build_events( read_events( r ) ); break; + case chunk::sond: loader.build_sounds( read_sounds( r ) ); break; + case chunk::dynm: loader.build_dynamics( read_dynamics( r ) ); break; + case chunk::trst: loader.build_trainsets( read_trainsets( r ) ); break; + default: break; // unknown chunk: skip + } + } + + // models depend on both PROT and INST, build them after the scan + loader.build_models( protos, instances ); + + if( saw_meta ) { + out.first_init_count = meta.first_init_count; + } + + if( module != nullptr ) { + auto resolve { [&]( std::uint32_t const id ) { + return id == kNoString ? std::string() : pool.get( id ); + } }; + module->includes.reserve( includes.size() + placements.size() ); + for( auto const &inc : includes ) { + scene::eu7::Eu7Include e; + e.source_line = inc.source_line; + e.source_path = resolve( inc.source_path ); + e.binary_path = resolve( inc.binary_path ); + e.parameters.reserve( inc.parameters.size() ); + for( auto const p : inc.parameters ) { + e.parameters.push_back( resolve( p ) ); + } + for( auto const &v : inc.site_transform.origin_stack ) { + e.site_transform.origin_stack.push_back( { v.x, v.y, v.z } ); + } + for( auto const &v : inc.site_transform.scale_stack ) { + e.site_transform.scale_stack.push_back( { v.x, v.y, v.z } ); + } + e.site_transform.rotation = { + inc.site_transform.rotation.x, + inc.site_transform.rotation.y, + inc.site_transform.rotation.z }; + e.site_transform.group_depth = + static_cast( inc.site_transform.group_depth ); + module->includes.push_back( std::move( e ) ); + } + for( auto const &p : placements ) { + scene::eu7::Eu7Include e; + e.binary_path = resolve( p.module_path ); + if( !e.binary_path.empty() ) { + std::filesystem::path source { e.binary_path }; + source.replace_extension( ".inc" ); + e.source_path = source.generic_string(); + } + if( p.texture_override != kNoString ) { + e.parameters.push_back( resolve( p.texture_override ) ); + } else { + e.parameters.push_back( "none" ); + } + e.parameters.push_back( std::to_string( p.x ) ); + e.parameters.push_back( std::to_string( p.y ) ); + e.parameters.push_back( std::to_string( p.z ) ); + e.parameters.push_back( std::to_string( p.rotation_y ) ); + module->includes.push_back( std::move( e ) ); + } + if( saw_meta ) { + module->has_terrain_chunk = meta.has_terrain_chunk; + module->has_pack_chunk = meta.has_pack_chunk; + module->include_placement.origin_x_param = meta.placement_origin_x; + module->include_placement.origin_y_param = meta.placement_origin_y; + module->include_placement.origin_z_param = meta.placement_origin_z; + module->include_placement.rotation_y_param = meta.placement_rotation_y; + } + } + return true; + } + catch( parse_error const & ) { + return false; + } +} + +} // namespace + +bool +load_scene( std::uint8_t const *data, std::size_t const size, scene::eu7::Eu7Scene &out ) { + return load_into( data, size, out, nullptr ); +} + +bool +load_module( std::uint8_t const *data, std::size_t const size, scene::eu7::Eu7Module &out ) { + return load_into( data, size, out.scene, &out ); +} + +} // namespace eu7v2 diff --git a/scene/eu7/v2/eu7v2_load.h b/scene/eu7/v2/eu7v2_load.h new file mode 100644 index 00000000..376cd854 --- /dev/null +++ b/scene/eu7/v2/eu7v2_load.h @@ -0,0 +1,36 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +#include +#include + +namespace scene::eu7 { +struct Eu7Scene; +struct Eu7Module; +} // namespace scene::eu7 + +namespace eu7v2 { + +// Reconstructs a parsed scene from an eu7v2 container produced by bake_scene() +// / bake_module(). The reconstructed Eu7Scene can be fed to the existing apply +// path, so switching to the new format only swaps serialization and reuses the +// proven world-construction code. Returns false on a malformed container (the +// scene is left in a partially-populated but safe state). +[[nodiscard]] bool +load_scene( std::uint8_t const *data, std::size_t size, scene::eu7::Eu7Scene &out ); + +// Reconstructs a full module (scene + includes + placement + flags) from an +// eu7v2 container. The runtime needs the includes to recurse into child +// modules. Returns false on a malformed container. +[[nodiscard]] bool +load_module( std::uint8_t const *data, std::size_t size, scene::eu7::Eu7Module &out ); + +} // namespace eu7v2 diff --git a/scene/eu7/v2/eu7v2_records.h b/scene/eu7/v2/eu7v2_records.h new file mode 100644 index 00000000..4a747967 --- /dev/null +++ b/scene/eu7/v2/eu7v2_records.h @@ -0,0 +1,816 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +// --------------------------------------------------------------------------- +// eu7v2 simulation records (iteration 2b): the non-visual scene data that is +// loaded once into the sim core - tracks, traction, power sources, memory +// cells, event launchers, events, sounds and dynamic vehicles. +// +// All strings are referenced by string-table id; optional fields use a leading +// presence flag so absent data costs a single byte. Dependency-free so the +// encode/decode path stays unit testable with a standalone compiler. +// --------------------------------------------------------------------------- + +#include "eu7v2_format.h" +#include "eu7v2_scene.h" + +#include +#include +#include +#include +#include + +namespace eu7v2 { + +// Shared helpers (put_strid/dvec3/node_record/...) live in eu7v2_scene.h so +// scene payloads and these sim records share a single definition. + +// --- TRAK : tracks --------------------------------------------------------- + +struct track_path { + dvec3 p_start; + double roll_start { 0.0 }; + dvec3 cp_out; + dvec3 cp_in; + dvec3 p_end; + double roll_end { 0.0 }; + double radius { 0.0 }; +}; + +struct track_visibility { + std::uint32_t material1 { kNoString }; + float tex_length { 4.f }; + std::uint32_t material2 { kNoString }; + float tex_height1 { 0.f }; + float tex_width { 0.f }; + float tex_slope { 0.f }; +}; + +struct track_record { + node_record node; + std::uint8_t track_type { 0 }; + std::uint8_t category { 1 }; + float length { 0.f }; + float track_width { 0.f }; + float friction { 0.f }; + float sound_distance { 0.f }; + std::int32_t quality_flag { 0 }; + std::int32_t damage_flag { 0 }; + std::int8_t environment { -1 }; + bool has_visibility { false }; + track_visibility visibility; + std::vector paths; + std::vector> tail_keywords; // (key strid, value strid) +}; + +inline void write_tracks( byte_writer &out, std::vector const &tracks ) { + out.put_u32( static_cast( tracks.size() ) ); + for( auto const &t : tracks ) { + write_node( out, t.node ); + out.put_u8( t.track_type ); + out.put_u8( t.category ); + out.put_f32( t.length ); + out.put_f32( t.track_width ); + out.put_f32( t.friction ); + out.put_f32( t.sound_distance ); + out.put_i32( t.quality_flag ); + out.put_i32( t.damage_flag ); + out.put_u8( static_cast( t.environment ) ); + out.put_u8( t.has_visibility ? 1u : 0u ); + if( t.has_visibility ) { + out.put_u32( t.visibility.material1 ); + out.put_f32( t.visibility.tex_length ); + out.put_u32( t.visibility.material2 ); + out.put_f32( t.visibility.tex_height1 ); + out.put_f32( t.visibility.tex_width ); + out.put_f32( t.visibility.tex_slope ); + } + out.put_u32( static_cast( t.paths.size() ) ); + for( auto const &p : t.paths ) { + put_dvec3( out, p.p_start.x, p.p_start.y, p.p_start.z ); + out.put_f64( p.roll_start ); + put_dvec3( out, p.cp_out.x, p.cp_out.y, p.cp_out.z ); + put_dvec3( out, p.cp_in.x, p.cp_in.y, p.cp_in.z ); + put_dvec3( out, p.p_end.x, p.p_end.y, p.p_end.z ); + out.put_f64( p.roll_end ); + out.put_f64( p.radius ); + } + out.put_u32( static_cast( t.tail_keywords.size() ) ); + for( auto const &kv : t.tail_keywords ) { + out.put_u32( kv.first ); + out.put_u32( kv.second ); + } + } +} + +inline std::vector read_tracks( byte_reader &in ) { + std::vector tracks; + auto const count { in.get_u32() }; + tracks.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + track_record t; + t.node = read_node( in ); + t.track_type = in.get_u8(); + t.category = in.get_u8(); + t.length = in.get_f32(); + t.track_width = in.get_f32(); + t.friction = in.get_f32(); + t.sound_distance = in.get_f32(); + t.quality_flag = in.get_i32(); + t.damage_flag = in.get_i32(); + t.environment = static_cast( in.get_u8() ); + t.has_visibility = in.get_u8() != 0; + if( t.has_visibility ) { + t.visibility.material1 = in.get_u32(); + t.visibility.tex_length = in.get_f32(); + t.visibility.material2 = in.get_u32(); + t.visibility.tex_height1 = in.get_f32(); + t.visibility.tex_width = in.get_f32(); + t.visibility.tex_slope = in.get_f32(); + } + auto const paths { in.get_u32() }; + t.paths.reserve( paths ); + for( std::uint32_t p { 0 }; p < paths; ++p ) { + track_path tp; + tp.p_start = get_dvec3( in ); + tp.roll_start = in.get_f64(); + tp.cp_out = get_dvec3( in ); + tp.cp_in = get_dvec3( in ); + tp.p_end = get_dvec3( in ); + tp.roll_end = in.get_f64(); + tp.radius = in.get_f64(); + t.paths.push_back( tp ); + } + auto const kws { in.get_u32() }; + t.tail_keywords.reserve( kws ); + for( std::uint32_t k { 0 }; k < kws; ++k ) { + auto const key { in.get_u32() }; + auto const value { in.get_u32() }; + t.tail_keywords.emplace_back( key, value ); + } + tracks.push_back( std::move( t ) ); + } + return tracks; +} + +// --- TRAC : traction ------------------------------------------------------- + +struct traction_record { + node_record node; + std::uint32_t power_supply_name { kNoString }; + float nominal_voltage { 0.f }; + float max_current { 0.f }; + float resistivity { 0.f }; + std::uint8_t material { 1 }; + float wire_thickness { 0.f }; + std::int32_t damage_flag { 0 }; + dvec3 wire_p1, wire_p2, wire_p3, wire_p4; + double min_height { 0.0 }; + double segment_length { 0.0 }; + std::int32_t wire_count { 0 }; + float wire_offset { 0.f }; + bool has_parallel { false }; + std::uint32_t parallel_name { kNoString }; +}; + +inline void write_traction( byte_writer &out, std::vector const &items ) { + out.put_u32( static_cast( items.size() ) ); + for( auto const &t : items ) { + write_node( out, t.node ); + out.put_u32( t.power_supply_name ); + out.put_f32( t.nominal_voltage ); + out.put_f32( t.max_current ); + out.put_f32( t.resistivity ); + out.put_u8( t.material ); + out.put_f32( t.wire_thickness ); + out.put_i32( t.damage_flag ); + put_dvec3( out, t.wire_p1.x, t.wire_p1.y, t.wire_p1.z ); + put_dvec3( out, t.wire_p2.x, t.wire_p2.y, t.wire_p2.z ); + put_dvec3( out, t.wire_p3.x, t.wire_p3.y, t.wire_p3.z ); + put_dvec3( out, t.wire_p4.x, t.wire_p4.y, t.wire_p4.z ); + out.put_f64( t.min_height ); + out.put_f64( t.segment_length ); + out.put_i32( t.wire_count ); + out.put_f32( t.wire_offset ); + put_opt_strid( out, t.has_parallel, t.parallel_name ); + } +} + +inline std::vector read_traction( byte_reader &in ) { + std::vector items; + auto const count { in.get_u32() }; + items.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + traction_record t; + t.node = read_node( in ); + t.power_supply_name = in.get_u32(); + t.nominal_voltage = in.get_f32(); + t.max_current = in.get_f32(); + t.resistivity = in.get_f32(); + t.material = in.get_u8(); + t.wire_thickness = in.get_f32(); + t.damage_flag = in.get_i32(); + t.wire_p1 = get_dvec3( in ); + t.wire_p2 = get_dvec3( in ); + t.wire_p3 = get_dvec3( in ); + t.wire_p4 = get_dvec3( in ); + t.min_height = in.get_f64(); + t.segment_length = in.get_f64(); + t.wire_count = in.get_i32(); + t.wire_offset = in.get_f32(); + t.has_parallel = in.get_u8() != 0; + if( t.has_parallel ) { + t.parallel_name = in.get_u32(); + } + items.push_back( t ); + } + return items; +} + +// --- PWRS : traction power sources ----------------------------------------- + +struct power_source_record { + node_record node; + dvec3 position; + float nominal_voltage { 0.f }; + float voltage_frequency { 0.f }; + float internal_resistance { 0.2f }; + float max_output_current { 0.f }; + float fast_fuse_timeout { 0.f }; + float fast_fuse_repetition { 0.f }; + float slow_fuse_timeout { 0.f }; + std::uint8_t modifier { 0 }; +}; + +inline void write_power_sources( byte_writer &out, std::vector const &items ) { + out.put_u32( static_cast( items.size() ) ); + for( auto const &p : items ) { + write_node( out, p.node ); + put_dvec3( out, p.position.x, p.position.y, p.position.z ); + out.put_f32( p.nominal_voltage ); + out.put_f32( p.voltage_frequency ); + out.put_f32( p.internal_resistance ); + out.put_f32( p.max_output_current ); + out.put_f32( p.fast_fuse_timeout ); + out.put_f32( p.fast_fuse_repetition ); + out.put_f32( p.slow_fuse_timeout ); + out.put_u8( p.modifier ); + } +} + +inline std::vector read_power_sources( byte_reader &in ) { + std::vector items; + auto const count { in.get_u32() }; + items.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + power_source_record p; + p.node = read_node( in ); + p.position = get_dvec3( in ); + p.nominal_voltage = in.get_f32(); + p.voltage_frequency = in.get_f32(); + p.internal_resistance = in.get_f32(); + p.max_output_current = in.get_f32(); + p.fast_fuse_timeout = in.get_f32(); + p.fast_fuse_repetition = in.get_f32(); + p.slow_fuse_timeout = in.get_f32(); + p.modifier = in.get_u8(); + items.push_back( p ); + } + return items; +} + +// --- MEMC : memory cells --------------------------------------------------- + +struct memcell_record { + node_record node; + std::uint32_t text { kNoString }; + double value1 { 0.0 }; + double value2 { 0.0 }; + bool has_track { false }; + std::uint32_t track_name { kNoString }; +}; + +inline void write_memcells( byte_writer &out, std::vector const &items ) { + out.put_u32( static_cast( items.size() ) ); + for( auto const &m : items ) { + write_node( out, m.node ); + out.put_u32( m.text ); + out.put_f64( m.value1 ); + out.put_f64( m.value2 ); + put_opt_strid( out, m.has_track, m.track_name ); + } +} + +inline std::vector read_memcells( byte_reader &in ) { + std::vector items; + auto const count { in.get_u32() }; + items.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + memcell_record m; + m.node = read_node( in ); + m.text = in.get_u32(); + m.value1 = in.get_f64(); + m.value2 = in.get_f64(); + m.has_track = in.get_u8() != 0; + if( m.has_track ) { + m.track_name = in.get_u32(); + } + items.push_back( m ); + } + return items; +} + +// --- LAUN : event launchers ------------------------------------------------ + +struct launcher_condition { + std::uint32_t memcell_name { kNoString }; + std::uint32_t compare_text { kNoString }; + double compare_value1 { 0.0 }; + double compare_value2 { 0.0 }; + std::int32_t check_mask { 0 }; +}; + +struct launcher_record { + node_record node; + dvec3 location; + double radius_squared { 0.0 }; + std::int32_t activation_key { 0 }; + double delta_time { -1.0 }; + std::uint32_t event1_name { kNoString }; + std::uint32_t event2_name { kNoString }; + bool has_condition { false }; + launcher_condition condition; + bool train_triggered { false }; + std::int32_t launch_hour { -1 }; + std::int32_t launch_minute { -1 }; +}; + +inline void write_launchers( byte_writer &out, std::vector const &items ) { + out.put_u32( static_cast( items.size() ) ); + for( auto const &l : items ) { + write_node( out, l.node ); + put_dvec3( out, l.location.x, l.location.y, l.location.z ); + out.put_f64( l.radius_squared ); + out.put_i32( l.activation_key ); + out.put_f64( l.delta_time ); + out.put_u32( l.event1_name ); + out.put_u32( l.event2_name ); + out.put_u8( l.has_condition ? 1u : 0u ); + if( l.has_condition ) { + out.put_u32( l.condition.memcell_name ); + out.put_u32( l.condition.compare_text ); + out.put_f64( l.condition.compare_value1 ); + out.put_f64( l.condition.compare_value2 ); + out.put_i32( l.condition.check_mask ); + } + out.put_u8( l.train_triggered ? 1u : 0u ); + out.put_i32( l.launch_hour ); + out.put_i32( l.launch_minute ); + } +} + +inline std::vector read_launchers( byte_reader &in ) { + std::vector items; + auto const count { in.get_u32() }; + items.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + launcher_record l; + l.node = read_node( in ); + l.location = get_dvec3( in ); + l.radius_squared = in.get_f64(); + l.activation_key = in.get_i32(); + l.delta_time = in.get_f64(); + l.event1_name = in.get_u32(); + l.event2_name = in.get_u32(); + l.has_condition = in.get_u8() != 0; + if( l.has_condition ) { + l.condition.memcell_name = in.get_u32(); + l.condition.compare_text = in.get_u32(); + l.condition.compare_value1 = in.get_f64(); + l.condition.compare_value2 = in.get_f64(); + l.condition.check_mask = in.get_i32(); + } + l.train_triggered = in.get_u8() != 0; + l.launch_hour = in.get_i32(); + l.launch_minute = in.get_i32(); + items.push_back( l ); + } + return items; +} + +// --- EVNT : events --------------------------------------------------------- + +struct event_record { + std::uint32_t name { kNoString }; + std::uint8_t type { 0 }; + double delay { 0.0 }; + double delay_random { 0.0 }; + double delay_departure { 0.0 }; + bool ignored { false }; + bool passive { false }; + std::vector targets; // string ids + std::vector> payload; // (key strid, value strid) +}; + +inline void write_events( byte_writer &out, std::vector const &items ) { + out.put_u32( static_cast( items.size() ) ); + for( auto const &e : items ) { + out.put_u32( e.name ); + out.put_u8( e.type ); + out.put_f64( e.delay ); + out.put_f64( e.delay_random ); + out.put_f64( e.delay_departure ); + out.put_u8( e.ignored ? 1u : 0u ); + out.put_u8( e.passive ? 1u : 0u ); + out.put_u32( static_cast( e.targets.size() ) ); + for( auto const t : e.targets ) { + out.put_u32( t ); + } + out.put_u32( static_cast( e.payload.size() ) ); + for( auto const &kv : e.payload ) { + out.put_u32( kv.first ); + out.put_u32( kv.second ); + } + } +} + +inline std::vector read_events( byte_reader &in ) { + std::vector items; + auto const count { in.get_u32() }; + items.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + event_record e; + e.name = in.get_u32(); + e.type = in.get_u8(); + e.delay = in.get_f64(); + e.delay_random = in.get_f64(); + e.delay_departure = in.get_f64(); + e.ignored = in.get_u8() != 0; + e.passive = in.get_u8() != 0; + auto const targets { in.get_u32() }; + e.targets.reserve( targets ); + for( std::uint32_t t { 0 }; t < targets; ++t ) { + e.targets.push_back( in.get_u32() ); + } + auto const payload { in.get_u32() }; + e.payload.reserve( payload ); + for( std::uint32_t p { 0 }; p < payload; ++p ) { + auto const key { in.get_u32() }; + auto const value { in.get_u32() }; + e.payload.emplace_back( key, value ); + } + items.push_back( std::move( e ) ); + } + return items; +} + +// --- SOND : sounds --------------------------------------------------------- + +struct sound_record { + node_record node; + dvec3 location; + std::uint32_t wav_file { kNoString }; +}; + +inline void write_sounds( byte_writer &out, std::vector const &items ) { + out.put_u32( static_cast( items.size() ) ); + for( auto const &s : items ) { + write_node( out, s.node ); + put_dvec3( out, s.location.x, s.location.y, s.location.z ); + out.put_u32( s.wav_file ); + } +} + +inline std::vector read_sounds( byte_reader &in ) { + std::vector items; + auto const count { in.get_u32() }; + items.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + sound_record s; + s.node = read_node( in ); + s.location = get_dvec3( in ); + s.wav_file = in.get_u32(); + items.push_back( s ); + } + return items; +} + +// --- DYNM : dynamic vehicles ----------------------------------------------- + +struct dynamic_record { + node_record node; + std::uint32_t data_folder { kNoString }; + std::uint32_t skin_file { kNoString }; + std::uint32_t mmd_file { kNoString }; + std::uint32_t track_name { kNoString }; + double offset { -1.0 }; + std::uint32_t driver_type { kNoString }; + std::int32_t coupling { 3 }; + std::uint32_t coupling_raw { kNoString }; + std::uint32_t coupling_params { kNoString }; + float velocity { 0.f }; + std::int32_t load_count { 0 }; + std::uint32_t load_type { kNoString }; + bool has_destination { false }; + std::uint32_t destination { kNoString }; + bool has_trainset { false }; + std::uint32_t trainset_index { 0 }; +}; + +inline void write_dynamics( byte_writer &out, std::vector const &items ) { + out.put_u32( static_cast( items.size() ) ); + for( auto const &d : items ) { + write_node( out, d.node ); + out.put_u32( d.data_folder ); + out.put_u32( d.skin_file ); + out.put_u32( d.mmd_file ); + out.put_u32( d.track_name ); + out.put_f64( d.offset ); + out.put_u32( d.driver_type ); + out.put_i32( d.coupling ); + out.put_u32( d.coupling_raw ); + out.put_u32( d.coupling_params ); + out.put_f32( d.velocity ); + out.put_i32( d.load_count ); + out.put_u32( d.load_type ); + put_opt_strid( out, d.has_destination, d.destination ); + out.put_u8( d.has_trainset ? 1u : 0u ); + if( d.has_trainset ) { + out.put_u32( d.trainset_index ); + } + } +} + +inline std::vector read_dynamics( byte_reader &in ) { + std::vector items; + auto const count { in.get_u32() }; + items.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + dynamic_record d; + d.node = read_node( in ); + d.data_folder = in.get_u32(); + d.skin_file = in.get_u32(); + d.mmd_file = in.get_u32(); + d.track_name = in.get_u32(); + d.offset = in.get_f64(); + d.driver_type = in.get_u32(); + d.coupling = in.get_i32(); + d.coupling_raw = in.get_u32(); + d.coupling_params = in.get_u32(); + d.velocity = in.get_f32(); + d.load_count = in.get_i32(); + d.load_type = in.get_u32(); + d.has_destination = in.get_u8() != 0; + if( d.has_destination ) { + d.destination = in.get_u32(); + } + d.has_trainset = in.get_u8() != 0; + if( d.has_trainset ) { + d.trainset_index = in.get_u32(); + } + items.push_back( d ); + } + return items; +} + +// --- TRST : trainsets ------------------------------------------------------ + +struct trainset_record { + std::uint32_t name { kNoString }; + std::uint32_t track { kNoString }; + float offset { 0.f }; + float velocity { 0.f }; + std::vector> assignment; // (key strid, value strid) + std::vector vehicle_indices; + std::vector couplings; + std::uint32_t driver_index { 0xffffffffu }; // (size_t)-1 sentinel +}; + +inline void write_trainsets( byte_writer &out, std::vector const &items ) { + out.put_u32( static_cast( items.size() ) ); + for( auto const &t : items ) { + out.put_u32( t.name ); + out.put_u32( t.track ); + out.put_f32( t.offset ); + out.put_f32( t.velocity ); + out.put_u32( static_cast( t.assignment.size() ) ); + for( auto const &kv : t.assignment ) { + out.put_u32( kv.first ); + out.put_u32( kv.second ); + } + out.put_u32( static_cast( t.vehicle_indices.size() ) ); + for( auto const idx : t.vehicle_indices ) { + out.put_u32( idx ); + } + out.put_u32( static_cast( t.couplings.size() ) ); + for( auto const c : t.couplings ) { + out.put_i32( c ); + } + out.put_u32( t.driver_index ); + } +} + +inline std::vector read_trainsets( byte_reader &in ) { + std::vector items; + auto const count { in.get_u32() }; + items.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + trainset_record t; + t.name = in.get_u32(); + t.track = in.get_u32(); + t.offset = in.get_f32(); + t.velocity = in.get_f32(); + auto const assign { in.get_u32() }; + t.assignment.reserve( assign ); + for( std::uint32_t a { 0 }; a < assign; ++a ) { + auto const key { in.get_u32() }; + auto const value { in.get_u32() }; + t.assignment.emplace_back( key, value ); + } + auto const vehicles { in.get_u32() }; + t.vehicle_indices.reserve( vehicles ); + for( std::uint32_t v { 0 }; v < vehicles; ++v ) { + t.vehicle_indices.push_back( in.get_u32() ); + } + auto const couplings { in.get_u32() }; + t.couplings.reserve( couplings ); + for( std::uint32_t c { 0 }; c < couplings; ++c ) { + t.couplings.push_back( in.get_i32() ); + } + t.driver_index = in.get_u32(); + items.push_back( std::move( t ) ); + } + return items; +} + +// --- PLCE : lean reusable-module placements (.inc with x/y/z/rot as f64/f32) - + +struct module_placement_record { + std::uint32_t module_path { kNoString }; // e.g. scenery/grass_l61/20.eu7v2 + std::uint32_t texture_override { kNoString }; + double x { 0.0 }, y { 0.0 }, z { 0.0 }; + float rotation_y { 0.f }; + std::uint8_t cell_id { 0xffu }; +}; + +inline void write_module_placements( + byte_writer &out, + std::vector const &items ) { + out.put_u32( static_cast( items.size() ) ); + for( auto const &p : items ) { + out.put_u32( p.module_path ); + out.put_u32( p.texture_override ); + out.put_f64( p.x ); + out.put_f64( p.y ); + out.put_f64( p.z ); + out.put_f32( p.rotation_y ); + out.put_u8( p.cell_id ); + } +} + +inline std::vector read_module_placements( byte_reader &in ) { + std::vector items; + auto const count { in.get_u32() }; + items.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + module_placement_record p; + p.module_path = in.get_u32(); + p.texture_override = in.get_u32(); + p.x = in.get_f64(); + p.y = in.get_f64(); + p.z = in.get_f64(); + p.rotation_y = in.get_f32(); + p.cell_id = in.get_u8(); + items.push_back( std::move( p ) ); + } + return items; +} + +// --- INCL : module includes (recursion references) ------------------------- + +// Snapshot of the origin/scale/rotation stacks at include site (detokenizer). +struct transform_record { + std::vector origin_stack; + std::vector scale_stack; + dvec3 rotation; + std::uint32_t group_depth { 0 }; +}; + +inline void write_transform( byte_writer &out, transform_record const &t ) { + out.put_u32( static_cast( t.origin_stack.size() ) ); + for( auto const &v : t.origin_stack ) { + put_dvec3( out, v.x, v.y, v.z ); + } + out.put_u32( static_cast( t.scale_stack.size() ) ); + for( auto const &v : t.scale_stack ) { + put_dvec3( out, v.x, v.y, v.z ); + } + put_dvec3( out, t.rotation.x, t.rotation.y, t.rotation.z ); + out.put_u32( t.group_depth ); +} + +inline transform_record read_transform( byte_reader &in ) { + transform_record t; + auto const origins { in.get_u32() }; + t.origin_stack.reserve( origins ); + for( std::uint32_t i { 0 }; i < origins; ++i ) { + t.origin_stack.push_back( get_dvec3( in ) ); + } + auto const scales { in.get_u32() }; + t.scale_stack.reserve( scales ); + for( std::uint32_t i { 0 }; i < scales; ++i ) { + t.scale_stack.push_back( get_dvec3( in ) ); + } + t.rotation = get_dvec3( in ); + t.group_depth = in.get_u32(); + return t; +} + +struct include_record { + std::uint32_t source_line { 0 }; + std::uint32_t source_path { kNoString }; + std::uint32_t binary_path { kNoString }; // points at the .eu7v2 module + std::vector parameters; // string ids + transform_record site_transform; +}; + +inline void write_includes( byte_writer &out, std::vector const &items ) { + out.put_u32( static_cast( items.size() ) ); + for( auto const &inc : items ) { + out.put_u32( inc.source_line ); + out.put_u32( inc.source_path ); + out.put_u32( inc.binary_path ); + out.put_u32( static_cast( inc.parameters.size() ) ); + for( auto const p : inc.parameters ) { + out.put_u32( p ); + } + write_transform( out, inc.site_transform ); + } +} + +inline std::vector read_includes( byte_reader &in ) { + std::vector items; + auto const count { in.get_u32() }; + items.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + include_record inc; + inc.source_line = in.get_u32(); + inc.source_path = in.get_u32(); + inc.binary_path = in.get_u32(); + auto const params { in.get_u32() }; + inc.parameters.reserve( params ); + for( std::uint32_t p { 0 }; p < params; ++p ) { + inc.parameters.push_back( in.get_u32() ); + } + inc.site_transform = read_transform( in ); + items.push_back( std::move( inc ) ); + } + return items; +} + +// --- META : module-level metadata (flags, placement, counts) --------------- + +struct module_meta { + std::uint32_t first_init_count { 0 }; + bool has_terrain_chunk { false }; + bool has_pack_chunk { false }; + // include placement (parameter indices, 0 = unused) + std::uint8_t placement_origin_x { 0 }; + std::uint8_t placement_origin_y { 0 }; + std::uint8_t placement_origin_z { 0 }; + std::uint8_t placement_rotation_y { 0 }; +}; + +inline void write_meta( byte_writer &out, module_meta const &m ) { + out.put_u32( 1u ); // meta layout version (forward-tolerant) + out.put_u32( m.first_init_count ); + out.put_u8( m.has_terrain_chunk ? 1u : 0u ); + out.put_u8( m.has_pack_chunk ? 1u : 0u ); + out.put_u8( m.placement_origin_x ); + out.put_u8( m.placement_origin_y ); + out.put_u8( m.placement_origin_z ); + out.put_u8( m.placement_rotation_y ); +} + +inline module_meta read_meta( byte_reader &in ) { + module_meta m; + (void)in.get_u32(); // layout version + m.first_init_count = in.get_u32(); + m.has_terrain_chunk = in.get_u8() != 0; + m.has_pack_chunk = in.get_u8() != 0; + m.placement_origin_x = in.get_u8(); + m.placement_origin_y = in.get_u8(); + m.placement_origin_z = in.get_u8(); + m.placement_rotation_y = in.get_u8(); + return m; +} + +} // namespace eu7v2 diff --git a/scene/eu7/v2/eu7v2_scene.h b/scene/eu7/v2/eu7v2_scene.h new file mode 100644 index 00000000..f0b89825 --- /dev/null +++ b/scene/eu7/v2/eu7v2_scene.h @@ -0,0 +1,472 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +// --------------------------------------------------------------------------- +// eu7v2 scene payloads (iteration 2 slice): the lean model + terrain records +// that fix the "heavy per-object" problem of the legacy streaming path. +// +// PROT - deduplicated model prototypes (mesh + material + LOD + lights), +// stored once per unique model, referenced by index. +// INST - lean instance: prototype index + world transform + optional +// per-instance texture override. No engine object is created here; +// the runtime resolves prototype -> shared mesh and renders via +// instancing. +// MESH - baked terrain mesh: origin (f64) + vertices relative to origin +// (f32) so a 1km tile keeps precision without paying f64 per vertex. +// +// Dependency-free on purpose so the whole encode/decode path is unit testable +// with a standalone compiler (see eu7v2_test.cpp), independent of the engine. +// --------------------------------------------------------------------------- + +#include "eu7v2_format.h" + +#include +#include +#include + +namespace eu7v2 { + +constexpr std::uint32_t kNoString { 0xffffffffu }; + +// --- shared serialization helpers (used by scene payloads and sim records) - + +inline void put_strid( byte_writer &out, std::uint32_t const id ) { out.put_u32( id ); } +inline std::uint32_t get_strid( byte_reader &in ) { return in.get_u32(); } + +inline void put_dvec3( byte_writer &out, double const x, double const y, double const z ) { + out.put_f64( x ); + out.put_f64( y ); + out.put_f64( z ); +} + +struct dvec3 { + double x { 0.0 }, y { 0.0 }, z { 0.0 }; +}; + +inline dvec3 get_dvec3( byte_reader &in ) { + dvec3 v; + v.x = in.get_f64(); + v.y = in.get_f64(); + v.z = in.get_f64(); + return v; +} + +inline void put_opt_strid( byte_writer &out, bool const present, std::uint32_t const id ) { + out.put_u8( present ? 1u : 0u ); + if( present ) { + out.put_u32( id ); + } +} + +// Common node metadata kept after baking (world-space, transform already applied). +struct node_record { + std::uint32_t name { kNoString }; + std::uint32_t type { kNoString }; + dvec3 area_center; + float area_radius { -1.f }; + double range_sq_min { 0.0 }; + double range_sq_max { 0.0 }; + bool visible { true }; +}; + +inline void write_node( byte_writer &out, node_record const &n ) { + out.put_u32( n.name ); + out.put_u32( n.type ); + put_dvec3( out, n.area_center.x, n.area_center.y, n.area_center.z ); + out.put_f32( n.area_radius ); + out.put_f64( n.range_sq_min ); + out.put_f64( n.range_sq_max ); + out.put_u8( n.visible ? 1u : 0u ); +} + +inline node_record read_node( byte_reader &in ) { + node_record n; + n.name = in.get_u32(); + n.type = in.get_u32(); + n.area_center = get_dvec3( in ); + n.area_radius = in.get_f32(); + n.range_sq_min = in.get_f64(); + n.range_sq_max = in.get_f64(); + n.visible = in.get_u8() != 0; + return n; +} + +// RGBA-ish lighting block stored as 12 floats (diffuse/ambient/specular vec4). +struct lighting_block { + float diffuse[ 4 ] { 0.8f, 0.8f, 0.8f, 1.f }; + float ambient[ 4 ] { 0.2f, 0.2f, 0.2f, 1.f }; + float specular[ 4 ] { 0.f, 0.f, 0.f, 1.f }; +}; + +inline void write_lighting( byte_writer &out, lighting_block const &l ) { + for( auto const v : l.diffuse ) { out.put_f32( v ); } + for( auto const v : l.ambient ) { out.put_f32( v ); } + for( auto const v : l.specular ) { out.put_f32( v ); } +} + +inline lighting_block read_lighting( byte_reader &in ) { + lighting_block l; + for( auto &v : l.diffuse ) { v = in.get_f32(); } + for( auto &v : l.ambient ) { v = in.get_f32(); } + for( auto &v : l.specular ) { v = in.get_f32(); } + return l; +} + +// Prototype flags packed into a single byte. +namespace proto_flag { +constexpr std::uint8_t transition { 1u << 0 }; // model has LOD transition +constexpr std::uint8_t is_terrain { 1u << 1 }; // terrain-style submodel split +constexpr std::uint8_t instanceable { 1u << 2 }; // safe to GPU-instance +} // namespace proto_flag + +struct model_prototype { + std::uint32_t model_file { kNoString }; // string id + std::uint32_t texture_file { kNoString }; // string id (default skin) + std::uint8_t flags { 0 }; + float range_min { -1.f }; + float range_max { -1.f }; + std::vector light_states; + std::vector light_colors; +}; + +// Lean instance: which prototype, where, and an optional skin override. +struct model_instance { + std::uint32_t proto { 0 }; + double x { 0.0 }, y { 0.0 }, z { 0.0 }; + float ax { 0.f }, ay { 0.f }, az { 0.f }; // euler angles (deg), engine convention + float sx { 1.f }, sy { 1.f }, sz { 1.f }; // scale + std::uint32_t texture_override { kNoString }; + std::uint8_t cell_id { 0xffu }; + bool has_node { false }; // full node metadata present (lossless path) + node_record node; +}; + +struct mesh_vertex { + float px { 0.f }, py { 0.f }, pz { 0.f }; // position relative to mesh origin + float nx { 0.f }, ny { 0.f }, nz { 0.f }; // normal + float u { 0.f }, v { 0.f }; // texcoord +}; + +struct terrain_mesh { + std::uint32_t material { kNoString }; // string id + bool translucent { false }; + double ox { 0.0 }, oy { 0.0 }, oz { 0.0 }; // world origin (f64) + std::vector vertices; +}; + +// --- PROT ------------------------------------------------------------------ + +inline void +write_prototypes( byte_writer &out, std::vector const &protos ) { + out.put_u32( static_cast( protos.size() ) ); + for( auto const &p : protos ) { + out.put_u32( p.model_file ); + out.put_u32( p.texture_file ); + out.put_u8( p.flags ); + out.put_f32( p.range_min ); + out.put_f32( p.range_max ); + out.put_u32( static_cast( p.light_states.size() ) ); + for( auto const s : p.light_states ) { + out.put_f32( s ); + } + out.put_u32( static_cast( p.light_colors.size() ) ); + for( auto const c : p.light_colors ) { + out.put_u32( c ); + } + } +} + +inline std::vector +read_prototypes( byte_reader &in ) { + std::vector protos; + auto const count { in.get_u32() }; + protos.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + model_prototype p; + p.model_file = in.get_u32(); + p.texture_file = in.get_u32(); + p.flags = in.get_u8(); + p.range_min = in.get_f32(); + p.range_max = in.get_f32(); + auto const states { in.get_u32() }; + p.light_states.reserve( states ); + for( std::uint32_t s { 0 }; s < states; ++s ) { + p.light_states.push_back( in.get_f32() ); + } + auto const colors { in.get_u32() }; + p.light_colors.reserve( colors ); + for( std::uint32_t c { 0 }; c < colors; ++c ) { + p.light_colors.push_back( in.get_u32() ); + } + protos.push_back( std::move( p ) ); + } + return protos; +} + +// --- INST ------------------------------------------------------------------ + +// Per-instance presence flags so common cases (unit scale, no override) stay +// compact instead of always paying for scale + override. +namespace inst_flag { +constexpr std::uint8_t has_scale { 1u << 0 }; +constexpr std::uint8_t has_texture_override { 1u << 1 }; +constexpr std::uint8_t has_node { 1u << 2 }; +} // namespace inst_flag + +inline void +write_instances( byte_writer &out, std::vector const &instances ) { + out.put_u32( static_cast( instances.size() ) ); + for( auto const &i : instances ) { + std::uint8_t flags { 0 }; + bool const unit_scale { i.sx == 1.f && i.sy == 1.f && i.sz == 1.f }; + if( !unit_scale ) { + flags |= inst_flag::has_scale; + } + if( i.texture_override != kNoString ) { + flags |= inst_flag::has_texture_override; + } + if( i.has_node ) { + flags |= inst_flag::has_node; + } + out.put_u8( flags ); + out.put_u32( i.proto ); + out.put_f64( i.x ); + out.put_f64( i.y ); + out.put_f64( i.z ); + out.put_f32( i.ax ); + out.put_f32( i.ay ); + out.put_f32( i.az ); + out.put_u8( i.cell_id ); + if( flags & inst_flag::has_scale ) { + out.put_f32( i.sx ); + out.put_f32( i.sy ); + out.put_f32( i.sz ); + } + if( flags & inst_flag::has_texture_override ) { + out.put_u32( i.texture_override ); + } + if( flags & inst_flag::has_node ) { + write_node( out, i.node ); + } + } +} + +inline std::vector +read_instances( byte_reader &in ) { + std::vector instances; + auto const count { in.get_u32() }; + instances.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + model_instance m; + auto const flags { in.get_u8() }; + m.proto = in.get_u32(); + m.x = in.get_f64(); + m.y = in.get_f64(); + m.z = in.get_f64(); + m.ax = in.get_f32(); + m.ay = in.get_f32(); + m.az = in.get_f32(); + m.cell_id = in.get_u8(); + if( flags & inst_flag::has_scale ) { + m.sx = in.get_f32(); + m.sy = in.get_f32(); + m.sz = in.get_f32(); + } + if( flags & inst_flag::has_texture_override ) { + m.texture_override = in.get_u32(); + } + if( flags & inst_flag::has_node ) { + m.has_node = true; + m.node = read_node( in ); + } + instances.push_back( m ); + } + return instances; +} + +// --- MESH ------------------------------------------------------------------ + +inline void +write_terrain_meshes( byte_writer &out, std::vector const &meshes ) { + out.put_u32( static_cast( meshes.size() ) ); + for( auto const &m : meshes ) { + out.put_u32( m.material ); + out.put_u8( m.translucent ? 1u : 0u ); + out.put_f64( m.ox ); + out.put_f64( m.oy ); + out.put_f64( m.oz ); + out.put_u32( static_cast( m.vertices.size() ) ); + for( auto const &v : m.vertices ) { + out.put_f32( v.px ); + out.put_f32( v.py ); + out.put_f32( v.pz ); + out.put_f32( v.nx ); + out.put_f32( v.ny ); + out.put_f32( v.nz ); + out.put_f32( v.u ); + out.put_f32( v.v ); + } + } +} + +inline std::vector +read_terrain_meshes( byte_reader &in ) { + std::vector meshes; + auto const count { in.get_u32() }; + meshes.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + terrain_mesh m; + m.material = in.get_u32(); + m.translucent = in.get_u8() != 0; + m.ox = in.get_f64(); + m.oy = in.get_f64(); + m.oz = in.get_f64(); + auto const verts { in.get_u32() }; + m.vertices.reserve( verts ); + for( std::uint32_t v { 0 }; v < verts; ++v ) { + mesh_vertex mv; + mv.px = in.get_f32(); + mv.py = in.get_f32(); + mv.pz = in.get_f32(); + mv.nx = in.get_f32(); + mv.ny = in.get_f32(); + mv.nz = in.get_f32(); + mv.u = in.get_f32(); + mv.v = in.get_f32(); + m.vertices.push_back( mv ); + } + meshes.push_back( std::move( m ) ); + } + return meshes; +} + +// --- SHPE : non-terrain shape nodes (triangles/strip/fan, baked world-space) - +// Lossless counterpart of MESH: keeps node metadata, lighting and translucency. +// Vertex positions are stored relative to origin (f64) as f32, matching MESH. + +struct shape_record { + node_record node; + bool translucent { false }; + std::uint32_t material { kNoString }; + lighting_block lighting; + double ox { 0.0 }, oy { 0.0 }, oz { 0.0 }; + std::vector vertices; +}; + +inline void write_shape_record( byte_writer &out, shape_record const &s ) { + write_node( out, s.node ); + out.put_u8( s.translucent ? 1u : 0u ); + out.put_u32( s.material ); + write_lighting( out, s.lighting ); + out.put_f64( s.ox ); + out.put_f64( s.oy ); + out.put_f64( s.oz ); + out.put_u32( static_cast( s.vertices.size() ) ); + for( auto const &v : s.vertices ) { + out.put_f32( v.px ); + out.put_f32( v.py ); + out.put_f32( v.pz ); + out.put_f32( v.nx ); + out.put_f32( v.ny ); + out.put_f32( v.nz ); + out.put_f32( v.u ); + out.put_f32( v.v ); + } +} + +inline void write_shapes( byte_writer &out, std::vector const &shapes ) { + out.put_u32( static_cast( shapes.size() ) ); + for( auto const &s : shapes ) { + write_shape_record( out, s ); + } +} + +inline std::vector read_shapes( byte_reader &in ) { + std::vector shapes; + auto const count { in.get_u32() }; + shapes.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + shape_record s; + s.node = read_node( in ); + s.translucent = in.get_u8() != 0; + s.material = in.get_u32(); + s.lighting = read_lighting( in ); + s.ox = in.get_f64(); + s.oy = in.get_f64(); + s.oz = in.get_f64(); + auto const verts { in.get_u32() }; + s.vertices.reserve( verts ); + for( std::uint32_t v { 0 }; v < verts; ++v ) { + mesh_vertex mv; + mv.px = in.get_f32(); + mv.py = in.get_f32(); + mv.pz = in.get_f32(); + mv.nx = in.get_f32(); + mv.ny = in.get_f32(); + mv.nz = in.get_f32(); + mv.u = in.get_f32(); + mv.v = in.get_f32(); + s.vertices.push_back( mv ); + } + shapes.push_back( std::move( s ) ); + } + return shapes; +} + +// --- LINE : line geometry nodes (only vertex positions are meaningful) ------- + +struct lines_record { + node_record node; + lighting_block lighting; + float line_width { 1.f }; + double ox { 0.0 }, oy { 0.0 }, oz { 0.0 }; + std::vector vertices; // world-space positions +}; + +inline void write_lines( byte_writer &out, std::vector const &items ) { + out.put_u32( static_cast( items.size() ) ); + for( auto const &l : items ) { + write_node( out, l.node ); + write_lighting( out, l.lighting ); + out.put_f32( l.line_width ); + out.put_f64( l.ox ); + out.put_f64( l.oy ); + out.put_f64( l.oz ); + out.put_u32( static_cast( l.vertices.size() ) ); + for( auto const &v : l.vertices ) { + put_dvec3( out, v.x, v.y, v.z ); + } + } +} + +inline std::vector read_lines( byte_reader &in ) { + std::vector items; + auto const count { in.get_u32() }; + items.reserve( count ); + for( std::uint32_t i { 0 }; i < count; ++i ) { + lines_record l; + l.node = read_node( in ); + l.lighting = read_lighting( in ); + l.line_width = in.get_f32(); + l.ox = in.get_f64(); + l.oy = in.get_f64(); + l.oz = in.get_f64(); + auto const verts { in.get_u32() }; + l.vertices.reserve( verts ); + for( std::uint32_t v { 0 }; v < verts; ++v ) { + l.vertices.push_back( get_dvec3( in ) ); + } + items.push_back( std::move( l ) ); + } + return items; +} + +} // namespace eu7v2 diff --git a/scene/eu7/v2/eu7v2_test.cpp b/scene/eu7/v2/eu7v2_test.cpp new file mode 100644 index 00000000..a5b65ecd --- /dev/null +++ b/scene/eu7/v2/eu7v2_test.cpp @@ -0,0 +1,673 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +// Standalone round-trip test for the eu7v2 format core. Builds in isolation +// (no engine dependencies) so the foundation can be validated without running +// the simulator: +// g++ -std=c++20 scene/eu7/v2/eu7v2_test.cpp -o eu7v2_test && ./eu7v2_test + +#include "eu7v2_format.h" +#include "eu7v2_scene.h" +#include "eu7v2_records.h" + +#include +#include +#include +#include + +namespace { + +using namespace eu7v2; + +int g_checks { 0 }; +int g_failures { 0 }; + +void +check( bool const ok, char const *what ) { + ++g_checks; + if( !ok ) { + ++g_failures; + std::printf( " FAIL: %s\n", what ); + } +} + +// A lean instance record: prototype id + world transform + optional texture override. +struct sample_instance { + std::uint32_t proto_id; + double x, y, z; + float yaw; + std::uint32_t texture_override; // string id, 0xffffffff = none +}; + +// A precomputed track-graph edge: track index -> neighbour at an endpoint. +struct sample_edge { + std::uint32_t track; + std::uint32_t neighbour; + std::uint8_t end; // 0 = prev, 1 = next + std::uint8_t neighbour_end; +}; + +std::vector +build_tile() { + string_table strings; + auto const tex_a { strings.intern( "textures/rail.dds" ) }; + auto const tex_b { strings.intern( "textures/grass.dds" ) }; + auto const proto_a { strings.intern( "models/sem.e3d" ) }; + (void)tex_b; + (void)proto_a; + + std::vector const instances { + { 0, 10.0, 0.0, -5.0, 1.5707963f, tex_a }, + { 1, 1234.5, 12.25, -6789.0, 0.0f, 0xffffffffu }, + { 2, -42.0, 3.0, 7.0, 3.14159f, tex_b }, + }; + + byte_writer strs_payload; + strings.serialize( strs_payload ); + + byte_writer inst_payload; + inst_payload.put_u32( static_cast( instances.size() ) ); + for( auto const &i : instances ) { + inst_payload.put_u32( i.proto_id ); + inst_payload.put_f64( i.x ); + inst_payload.put_f64( i.y ); + inst_payload.put_f64( i.z ); + inst_payload.put_f32( i.yaw ); + inst_payload.put_u32( i.texture_override ); + } + + container_writer writer( file_kind::tile ); + writer.add_chunk( chunk::strs, strs_payload ); + writer.add_chunk( chunk::inst, inst_payload ); + return writer.data(); +} + +void +verify_tile( std::vector const &bytes ) { + container_reader reader( bytes.data(), bytes.size() ); + check( reader.kind() == file_kind::tile, "tile kind" ); + check( reader.version() == kVersion, "tile version" ); + + string_pool pool; + bool saw_strs { false }; + bool saw_inst { false }; + + chunk_view chunk; + while( reader.next( chunk ) ) { + if( chunk.id == chunk::strs ) { + saw_strs = true; + auto r { chunk.reader() }; + pool.deserialize( r ); + check( pool.size() == 3, "string count" ); + check( pool.get( 0 ) == "textures/rail.dds", "string 0" ); + check( pool.get( 2 ) == "models/sem.e3d", "string 2" ); + } + else if( chunk.id == chunk::inst ) { + saw_inst = true; + auto r { chunk.reader() }; + auto const count { r.get_u32() }; + check( count == 3, "instance count" ); + + auto const p0 { r.get_u32() }; + auto const x0 { r.get_f64() }; + auto const y0 { r.get_f64() }; + auto const z0 { r.get_f64() }; + auto const yaw0 { r.get_f32() }; + auto const tex0 { r.get_u32() }; + check( p0 == 0, "inst0 proto" ); + check( x0 == 10.0 && y0 == 0.0 && z0 == -5.0, "inst0 pos (f64 exact)" ); + check( std::fabs( yaw0 - 1.5707963f ) < 1e-6f, "inst0 yaw" ); + check( tex0 == 0, "inst0 tex override id" ); + + // second record: verify large f64 coordinate keeps full precision + auto const p1 { r.get_u32() }; + auto const x1 { r.get_f64() }; + (void)r.get_f64(); + auto const z1 { r.get_f64() }; + (void)r.get_f32(); + auto const tex1 { r.get_u32() }; + check( p1 == 1, "inst1 proto" ); + check( x1 == 1234.5 && z1 == -6789.0, "inst1 large coord precision" ); + check( tex1 == 0xffffffffu, "inst1 no tex override" ); + } + } + + check( saw_strs, "tile has STRS" ); + check( saw_inst, "tile has INST" ); +} + +void +verify_trgr_roundtrip() { + std::vector const edges { + { 0, 1, 1, 0 }, + { 1, 2, 1, 0 }, + { 2, 0, 1, 0 }, + }; + + byte_writer payload; + payload.put_u32( static_cast( edges.size() ) ); + for( auto const &e : edges ) { + payload.put_u32( e.track ); + payload.put_u32( e.neighbour ); + payload.put_u8( e.end ); + payload.put_u8( e.neighbour_end ); + } + + container_writer writer( file_kind::sim ); + writer.add_chunk( chunk::trgr, payload ); + auto const bytes { writer.data() }; + + container_reader reader( bytes.data(), bytes.size() ); + check( reader.kind() == file_kind::sim, "sim kind" ); + + chunk_view chunk; + bool ok { false }; + while( reader.next( chunk ) ) { + if( chunk.id != chunk::trgr ) { + continue; + } + auto r { chunk.reader() }; + auto const count { r.get_u32() }; + check( count == edges.size(), "edge count" ); + for( auto const &expected : edges ) { + auto const track { r.get_u32() }; + auto const neighbour { r.get_u32() }; + auto const end { r.get_u8() }; + auto const nend { r.get_u8() }; + check( + track == expected.track && neighbour == expected.neighbour && + end == expected.end && nend == expected.neighbour_end, + "edge values" ); + } + ok = true; + } + check( ok, "sim has TRGR" ); +} + +void +verify_truncation_is_rejected() { + auto bytes { build_tile() }; + bytes.resize( bytes.size() - 4 ); // chop the tail of the last chunk + bool threw { false }; + try { + container_reader reader( bytes.data(), bytes.size() ); + chunk_view chunk; + while( reader.next( chunk ) ) { + auto r { chunk.reader() }; + // force payload reads + while( r.remaining() > 0 ) { + (void)r.get_u8(); + } + } + } + catch( parse_error const & ) { + threw = true; + } + check( threw, "truncated file rejected with parse_error" ); +} + +void +verify_scene_roundtrip() { + string_table strings; + auto const sem_model { strings.intern( "models/sem.e3d" ) }; + auto const sem_tex { strings.intern( "textures/sem.dds" ) }; + auto const grass { strings.intern( "textures/grass.dds" ) }; + auto const tex_override { strings.intern( "textures/sem_winter.dds" ) }; + + std::vector protos( 2 ); + protos[ 0 ].model_file = sem_model; + protos[ 0 ].texture_file = sem_tex; + protos[ 0 ].flags = proto_flag::instanceable | proto_flag::transition; + protos[ 0 ].range_min = 0.f; + protos[ 0 ].range_max = 4000.f; + protos[ 0 ].light_states = { 1.f, 0.5f, 0.f }; + protos[ 0 ].light_colors = { 0xff0000ffu, 0x00ff00ffu }; + protos[ 1 ].model_file = strings.intern( "models/tree.e3d" ); + protos[ 1 ].flags = proto_flag::instanceable; + + std::vector instances( 3 ); + instances[ 0 ] = { 0, 100.0, 0.0, -200.0, 0.f, 90.f, 0.f, 1.f, 1.f, 1.f, kNoString, 7 }; + instances[ 1 ] = { 0, 999999.5, 1.25, -888888.0, 0.f, 0.f, 0.f, 2.f, 2.f, 2.f, tex_override, 3 }; + instances[ 2 ] = { 1, -5.0, 0.0, 5.0, 0.f, 45.f, 0.f, 1.f, 1.f, 1.f, kNoString, 0xffu }; + + std::vector meshes( 1 ); + meshes[ 0 ].material = grass; + meshes[ 0 ].translucent = false; + meshes[ 0 ].ox = 500000.0; + meshes[ 0 ].oy = 0.0; + meshes[ 0 ].oz = -500000.0; + meshes[ 0 ].vertices = { + { 0.f, 0.f, 0.f, 0.f, 1.f, 0.f, 0.f, 0.f }, + { 1000.f, 0.f, 0.f, 0.f, 1.f, 0.f, 1.f, 0.f }, + { 0.f, 0.f, -1000.f, 0.f, 1.f, 0.f, 0.f, 1.f }, + }; + + byte_writer strs_payload; + strings.serialize( strs_payload ); + byte_writer prot_payload; + write_prototypes( prot_payload, protos ); + byte_writer inst_payload; + write_instances( inst_payload, instances ); + byte_writer mesh_payload; + write_terrain_meshes( mesh_payload, meshes ); + + container_writer writer( file_kind::tile ); + writer.add_chunk( chunk::strs, strs_payload ); + writer.add_chunk( chunk::prot, prot_payload ); + writer.add_chunk( chunk::inst, inst_payload ); + writer.add_chunk( chunk::mesh, mesh_payload ); + auto const bytes { writer.data() }; + + container_reader reader( bytes.data(), bytes.size() ); + string_pool pool; + std::vector rp; + std::vector ri; + std::vector rm; + chunk_view chunk; + while( reader.next( chunk ) ) { + auto r { chunk.reader() }; + if( chunk.id == chunk::strs ) { + pool.deserialize( r ); + } + else if( chunk.id == chunk::prot ) { + rp = read_prototypes( r ); + } + else if( chunk.id == chunk::inst ) { + ri = read_instances( r ); + } + else if( chunk.id == chunk::mesh ) { + rm = read_terrain_meshes( r ); + } + } + + check( rp.size() == 2, "proto count" ); + check( rp[ 0 ].flags == ( proto_flag::instanceable | proto_flag::transition ), "proto0 flags" ); + check( rp[ 0 ].range_max == 4000.f, "proto0 range" ); + check( rp[ 0 ].light_states.size() == 3 && rp[ 0 ].light_states[ 1 ] == 0.5f, "proto0 light states" ); + check( rp[ 0 ].light_colors.size() == 2 && rp[ 0 ].light_colors[ 0 ] == 0xff0000ffu, "proto0 light colors" ); + check( pool.get( rp[ 0 ].model_file ) == "models/sem.e3d", "proto0 model string" ); + + check( ri.size() == 3, "instance count" ); + check( ri[ 0 ].proto == 0 && ri[ 0 ].cell_id == 7, "inst0 proto/cell" ); + check( ri[ 0 ].sx == 1.f && ri[ 0 ].texture_override == kNoString, "inst0 defaults survive" ); + check( ri[ 1 ].x == 999999.5 && ri[ 1 ].z == -888888.0, "inst1 large coord precision" ); + check( ri[ 1 ].sx == 2.f && ri[ 1 ].sy == 2.f && ri[ 1 ].sz == 2.f, "inst1 scale" ); + check( pool.get( ri[ 1 ].texture_override ) == "textures/sem_winter.dds", "inst1 override" ); + check( ri[ 2 ].cell_id == 0xffu, "inst2 no cell" ); + + check( rm.size() == 1, "mesh count" ); + check( rm[ 0 ].ox == 500000.0 && rm[ 0 ].oz == -500000.0, "mesh origin f64 precision" ); + check( rm[ 0 ].vertices.size() == 3, "mesh vertex count" ); + check( rm[ 0 ].vertices[ 1 ].px == 1000.f && rm[ 0 ].vertices[ 1 ].u == 1.f, "mesh vertex data" ); + check( pool.get( rm[ 0 ].material ) == "textures/grass.dds", "mesh material string" ); +} + +void +verify_records_roundtrip() { + string_table s; + + // --- tracks --- + std::vector tracks( 1 ); + tracks[ 0 ].node.name = s.intern( "track_42" ); + tracks[ 0 ].node.type = s.intern( "track" ); + tracks[ 0 ].node.area_center = { 123456.5, 0.0, -654321.0 }; + tracks[ 0 ].node.range_sq_max = 1.0e12; + tracks[ 0 ].track_type = 2; // switch + tracks[ 0 ].category = 1; + tracks[ 0 ].length = 50.0f; + tracks[ 0 ].environment = 3; + tracks[ 0 ].has_visibility = true; + tracks[ 0 ].visibility.material1 = s.intern( "rail.mat" ); + tracks[ 0 ].visibility.tex_length = 8.f; + tracks[ 0 ].paths.push_back( { { 1.0, 2.0, 3.0 }, 0.1, { 4.0, 5.0, 6.0 }, { 7.0, 8.0, 9.0 }, { 10.0, 11.0, 12.0 }, 0.2, 300.0 } ); + tracks[ 0 ].tail_keywords.emplace_back( s.intern( "event0" ), s.intern( "ev_start" ) ); + + // --- traction --- + std::vector traction( 1 ); + traction[ 0 ].node.name = s.intern( "trakcja1" ); + traction[ 0 ].power_supply_name = s.intern( "psupply" ); + traction[ 0 ].nominal_voltage = 3000.f; + traction[ 0 ].material = 1; + traction[ 0 ].wire_p1 = { 0.0, 6.0, 0.0 }; + traction[ 0 ].wire_p2 = { 100.0, 6.0, 0.0 }; + traction[ 0 ].wire_count = 2; + traction[ 0 ].has_parallel = true; + traction[ 0 ].parallel_name = s.intern( "trakcja0" ); + + // --- power sources --- + std::vector power( 1 ); + power[ 0 ].node.name = s.intern( "psupply" ); + power[ 0 ].position = { 50.0, 0.0, -50.0 }; + power[ 0 ].nominal_voltage = 3000.f; + power[ 0 ].max_output_current = 4000.f; + power[ 0 ].modifier = 2; + + // --- memcells --- + std::vector memcells( 1 ); + memcells[ 0 ].node.name = s.intern( "mc_station" ); + memcells[ 0 ].text = s.intern( "PASS depot 1" ); + memcells[ 0 ].value1 = 12.5; + memcells[ 0 ].value2 = -7.0; + memcells[ 0 ].has_track = true; + memcells[ 0 ].track_name = s.intern( "track_42" ); + + // --- launchers --- + std::vector launchers( 1 ); + launchers[ 0 ].node.name = s.intern( "launch1" ); + launchers[ 0 ].location = { 200.0, 1.0, -200.0 }; + launchers[ 0 ].radius_squared = 400.0; + launchers[ 0 ].activation_key = 65; + launchers[ 0 ].event1_name = s.intern( "ev_start" ); + launchers[ 0 ].has_condition = true; + launchers[ 0 ].condition.memcell_name = s.intern( "mc_station" ); + launchers[ 0 ].condition.compare_value1 = 12.5; + launchers[ 0 ].condition.check_mask = 3; + launchers[ 0 ].launch_hour = 14; + launchers[ 0 ].launch_minute = 30; + + // --- events --- + std::vector events( 1 ); + events[ 0 ].name = s.intern( "ev_start" ); + events[ 0 ].type = 7; // multiple + events[ 0 ].delay = 2.5; + events[ 0 ].passive = true; + events[ 0 ].targets = { s.intern( "ev_a" ), s.intern( "ev_b" ) }; + events[ 0 ].payload.emplace_back( s.intern( "k1" ), s.intern( "v1" ) ); + + // --- sounds --- + std::vector sounds( 1 ); + sounds[ 0 ].node.name = s.intern( "snd1" ); + sounds[ 0 ].location = { -10.0, 2.0, 10.0 }; + sounds[ 0 ].wav_file = s.intern( "ambient.wav" ); + + // --- dynamics --- + std::vector dynamics( 1 ); + dynamics[ 0 ].node.name = s.intern( "veh1" ); + dynamics[ 0 ].data_folder = s.intern( "pkp/ep07" ); + dynamics[ 0 ].mmd_file = s.intern( "ep07.mmd" ); + dynamics[ 0 ].track_name = s.intern( "track_42" ); + dynamics[ 0 ].offset = 25.0; + dynamics[ 0 ].coupling = 3; + dynamics[ 0 ].velocity = 40.f; + dynamics[ 0 ].has_destination = true; + dynamics[ 0 ].destination = s.intern( "Warszawa" ); + dynamics[ 0 ].has_trainset = true; + dynamics[ 0 ].trainset_index = 5; + + byte_writer strs_payload, trak_p, trac_p, pwrs_p, memc_p, laun_p, evnt_p, sond_p, dynm_p; + s.serialize( strs_payload ); + write_tracks( trak_p, tracks ); + write_traction( trac_p, traction ); + write_power_sources( pwrs_p, power ); + write_memcells( memc_p, memcells ); + write_launchers( laun_p, launchers ); + write_events( evnt_p, events ); + write_sounds( sond_p, sounds ); + write_dynamics( dynm_p, dynamics ); + + container_writer w( file_kind::sim ); + w.add_chunk( chunk::strs, strs_payload ); + w.add_chunk( chunk::trak, trak_p ); + w.add_chunk( chunk::trac, trac_p ); + w.add_chunk( chunk::pwrs, pwrs_p ); + w.add_chunk( chunk::memc, memc_p ); + w.add_chunk( chunk::laun, laun_p ); + w.add_chunk( chunk::evnt, evnt_p ); + w.add_chunk( chunk::sond, sond_p ); + w.add_chunk( chunk::dynm, dynm_p ); + auto const bytes { w.data() }; + + container_reader r( bytes.data(), bytes.size() ); + check( r.kind() == file_kind::sim, "records sim kind" ); + + string_pool pool; + std::vector rt; + std::vector rtr; + std::vector rp; + std::vector rmc; + std::vector rl; + std::vector re; + std::vector rs; + std::vector rd; + chunk_view c; + while( r.next( c ) ) { + auto rr { c.reader() }; + switch( c.id ) { + case chunk::strs: pool.deserialize( rr ); break; + case chunk::trak: rt = read_tracks( rr ); break; + case chunk::trac: rtr = read_traction( rr ); break; + case chunk::pwrs: rp = read_power_sources( rr ); break; + case chunk::memc: rmc = read_memcells( rr ); break; + case chunk::laun: rl = read_launchers( rr ); break; + case chunk::evnt: re = read_events( rr ); break; + case chunk::sond: rs = read_sounds( rr ); break; + case chunk::dynm: rd = read_dynamics( rr ); break; + default: break; + } + } + + check( rt.size() == 1 && pool.get( rt[ 0 ].node.name ) == "track_42", "track name" ); + check( rt[ 0 ].node.area_center.x == 123456.5 && rt[ 0 ].node.area_center.z == -654321.0, "track center f64" ); + check( rt[ 0 ].track_type == 2 && rt[ 0 ].environment == 3, "track type/env" ); + check( rt[ 0 ].has_visibility && pool.get( rt[ 0 ].visibility.material1 ) == "rail.mat", "track visibility" ); + check( rt[ 0 ].paths.size() == 1 && rt[ 0 ].paths[ 0 ].radius == 300.0, "track path" ); + check( rt[ 0 ].paths[ 0 ].p_end.z == 12.0, "track path endpoint" ); + check( rt[ 0 ].tail_keywords.size() == 1 && pool.get( rt[ 0 ].tail_keywords[ 0 ].second ) == "ev_start", "track keyword" ); + + check( rtr.size() == 1 && rtr[ 0 ].nominal_voltage == 3000.f, "traction voltage" ); + check( rtr[ 0 ].wire_p2.x == 100.0 && rtr[ 0 ].wire_count == 2, "traction wire" ); + check( rtr[ 0 ].has_parallel && pool.get( rtr[ 0 ].parallel_name ) == "trakcja0", "traction parallel" ); + + check( rp.size() == 1 && rp[ 0 ].max_output_current == 4000.f && rp[ 0 ].modifier == 2, "power source" ); + + check( rmc.size() == 1 && pool.get( rmc[ 0 ].text ) == "PASS depot 1", "memcell text" ); + check( rmc[ 0 ].value1 == 12.5 && rmc[ 0 ].has_track, "memcell value/track" ); + + check( rl.size() == 1 && rl[ 0 ].activation_key == 65, "launcher key" ); + check( rl[ 0 ].has_condition && rl[ 0 ].condition.check_mask == 3, "launcher condition" ); + check( rl[ 0 ].launch_hour == 14 && rl[ 0 ].launch_minute == 30, "launcher time" ); + + check( re.size() == 1 && re[ 0 ].type == 7 && re[ 0 ].passive, "event type" ); + check( re[ 0 ].targets.size() == 2 && pool.get( re[ 0 ].targets[ 1 ] ) == "ev_b", "event targets" ); + check( re[ 0 ].payload.size() == 1 && pool.get( re[ 0 ].payload[ 0 ].first ) == "k1", "event payload" ); + + check( rs.size() == 1 && pool.get( rs[ 0 ].wav_file ) == "ambient.wav", "sound wav" ); + + check( rd.size() == 1 && pool.get( rd[ 0 ].data_folder ) == "pkp/ep07", "dynamic folder" ); + check( rd[ 0 ].offset == 25.0 && rd[ 0 ].velocity == 40.f, "dynamic offset/vel" ); + check( rd[ 0 ].has_destination && pool.get( rd[ 0 ].destination ) == "Warszawa", "dynamic destination" ); + check( rd[ 0 ].has_trainset && rd[ 0 ].trainset_index == 5, "dynamic trainset" ); +} + +void +verify_extensions_roundtrip() { + string_table s; + + // --- shapes (SHPE) --- + std::vector shapes( 1 ); + shapes[ 0 ].node.name = s.intern( "shape0" ); + shapes[ 0 ].node.type = s.intern( "triangles" ); + shapes[ 0 ].translucent = true; + shapes[ 0 ].material = s.intern( "bricks.mat" ); + shapes[ 0 ].lighting.diffuse[ 0 ] = 0.5f; + shapes[ 0 ].ox = 12345.5; + shapes[ 0 ].oy = 1.0; + shapes[ 0 ].oz = -54321.0; + shapes[ 0 ].vertices = { + { 0.f, 0.f, 0.f, 0.f, 1.f, 0.f, 0.f, 0.f }, + { 10.f, 0.f, 0.f, 0.f, 1.f, 0.f, 1.f, 0.f }, + { 0.f, 5.f, 0.f, 0.f, 1.f, 0.f, 0.f, 1.f }, + }; + + // --- lines (LINE) --- + std::vector lines( 1 ); + lines[ 0 ].node.name = s.intern( "line0" ); + lines[ 0 ].line_width = 2.f; + lines[ 0 ].ox = 100.0; + lines[ 0 ].vertices = { { 0.0, 0.0, 0.0 }, { 50.0, 0.0, -3.0 } }; + + // --- trainsets (TRST) --- + std::vector trainsets( 1 ); + trainsets[ 0 ].name = s.intern( "ts_warszawa" ); + trainsets[ 0 ].track = s.intern( "track_42" ); + trainsets[ 0 ].offset = 25.5f; + trainsets[ 0 ].velocity = 60.f; + trainsets[ 0 ].assignment.emplace_back( s.intern( "k" ), s.intern( "v" ) ); + trainsets[ 0 ].vehicle_indices = { 0u, 1u, 2u }; + trainsets[ 0 ].couplings = { 3, -1 }; + trainsets[ 0 ].driver_index = 1u; + + // --- includes (INCL) --- + std::vector includes( 1 ); + includes[ 0 ].source_line = 17; + includes[ 0 ].source_path = s.intern( "scenery/sub.inc" ); + includes[ 0 ].binary_path = s.intern( "scenery/sub.eu7v2" ); + includes[ 0 ].parameters = { s.intern( "p1" ), s.intern( "p2" ) }; + includes[ 0 ].site_transform.origin_stack.push_back( { 10.0, 0.0, -20.0 } ); + includes[ 0 ].site_transform.rotation = { 0.0, 90.0, 0.0 }; + includes[ 0 ].site_transform.group_depth = 1; + + // --- instance with full node --- + std::vector protos( 1 ); + protos[ 0 ].model_file = s.intern( "models/x.e3d" ); + std::vector instances( 1 ); + instances[ 0 ].proto = 0; + instances[ 0 ].x = 7.0; + instances[ 0 ].has_node = true; + instances[ 0 ].node.name = s.intern( "inst_named" ); + instances[ 0 ].node.type = s.intern( "model" ); + instances[ 0 ].node.range_sq_max = 1.0e9; + instances[ 0 ].node.visible = false; + + // --- meta (META) --- + module_meta meta; + meta.first_init_count = 4; + meta.has_terrain_chunk = true; + meta.placement_origin_x = 1; + meta.placement_rotation_y = 4; + + byte_writer strs_p, shpe_p, line_p, trst_p, incl_p, prot_p, inst_p, meta_p; + s.serialize( strs_p ); + write_shapes( shpe_p, shapes ); + write_lines( line_p, lines ); + write_trainsets( trst_p, trainsets ); + write_includes( incl_p, includes ); + write_prototypes( prot_p, protos ); + write_instances( inst_p, instances ); + write_meta( meta_p, meta ); + + container_writer w( file_kind::module ); + w.add_chunk( chunk::strs, strs_p ); + w.add_chunk( chunk::meta, meta_p ); + w.add_chunk( chunk::incl, incl_p ); + w.add_chunk( chunk::shpe, shpe_p ); + w.add_chunk( chunk::line, line_p ); + w.add_chunk( chunk::trst, trst_p ); + w.add_chunk( chunk::prot, prot_p ); + w.add_chunk( chunk::inst, inst_p ); + auto const bytes { w.data() }; + + container_reader r( bytes.data(), bytes.size() ); + check( r.kind() == file_kind::module, "ext module kind" ); + + string_pool pool; + std::vector rsh; + std::vector rln; + std::vector rts; + std::vector rin; + std::vector ri; + module_meta rmeta; + chunk_view c; + while( r.next( c ) ) { + auto rr { c.reader() }; + switch( c.id ) { + case chunk::strs: pool.deserialize( rr ); break; + case chunk::meta: rmeta = read_meta( rr ); break; + case chunk::incl: rin = read_includes( rr ); break; + case chunk::shpe: rsh = read_shapes( rr ); break; + case chunk::line: rln = read_lines( rr ); break; + case chunk::trst: rts = read_trainsets( rr ); break; + case chunk::inst: ri = read_instances( rr ); break; + default: break; + } + } + + check( rsh.size() == 1 && rsh[ 0 ].translucent, "shape translucent" ); + check( rsh[ 0 ].ox == 12345.5 && rsh[ 0 ].oz == -54321.0, "shape origin f64" ); + check( rsh[ 0 ].vertices.size() == 3 && rsh[ 0 ].vertices[ 1 ].px == 10.f, "shape verts" ); + check( pool.get( rsh[ 0 ].material ) == "bricks.mat", "shape material" ); + check( rsh[ 0 ].lighting.diffuse[ 0 ] == 0.5f, "shape lighting" ); + + check( rln.size() == 1 && rln[ 0 ].line_width == 2.f, "line width" ); + check( rln[ 0 ].vertices.size() == 2 && rln[ 0 ].vertices[ 1 ].x == 50.0, "line verts" ); + + check( rts.size() == 1 && pool.get( rts[ 0 ].name ) == "ts_warszawa", "trainset name" ); + check( rts[ 0 ].vehicle_indices.size() == 3 && rts[ 0 ].vehicle_indices[ 2 ] == 2u, "trainset vehicles" ); + check( rts[ 0 ].couplings.size() == 2 && rts[ 0 ].couplings[ 1 ] == -1, "trainset couplings" ); + check( rts[ 0 ].driver_index == 1u && rts[ 0 ].offset == 25.5f, "trainset driver/offset" ); + + check( rin.size() == 1 && rin[ 0 ].source_line == 17, "include line" ); + check( pool.get( rin[ 0 ].binary_path ) == "scenery/sub.eu7v2", "include binary path .eu7v2" ); + check( rin[ 0 ].parameters.size() == 2, "include params" ); + check( rin[ 0 ].site_transform.origin_stack.size() == 1 && + rin[ 0 ].site_transform.origin_stack[ 0 ].z == -20.0, "include transform origin" ); + check( rin[ 0 ].site_transform.rotation.y == 90.0 && + rin[ 0 ].site_transform.group_depth == 1, "include transform rot/group" ); + + check( ri.size() == 1 && ri[ 0 ].has_node, "inst has node" ); + check( pool.get( ri[ 0 ].node.name ) == "inst_named" && !ri[ 0 ].node.visible, "inst node fields" ); + check( ri[ 0 ].node.range_sq_max == 1.0e9, "inst node range" ); + + check( rmeta.first_init_count == 4 && rmeta.has_terrain_chunk, "meta counts/flags" ); + check( rmeta.placement_origin_x == 1 && rmeta.placement_rotation_y == 4, "meta placement" ); +} + +void +verify_file_io() { + auto const bytes { build_tile() }; + char const *path { "eu7v2_roundtrip.bin" }; + { + std::ofstream os( path, std::ios::binary ); + os.write( reinterpret_cast( bytes.data() ), + static_cast( bytes.size() ) ); + } + std::ifstream is( path, std::ios::binary | std::ios::ate ); + auto const size { static_cast( is.tellg() ) }; + is.seekg( 0 ); + std::vector loaded( size ); + is.read( reinterpret_cast( loaded.data() ), + static_cast( size ) ); + check( loaded == bytes, "file write/read byte-identical" ); + verify_tile( loaded ); + std::remove( path ); +} + +} // namespace + +int +main() { + std::printf( "eu7v2 format core round-trip test\n" ); + + auto const tile { build_tile() }; + verify_tile( tile ); + verify_trgr_roundtrip(); + verify_scene_roundtrip(); + verify_records_roundtrip(); + verify_extensions_roundtrip(); + verify_truncation_is_rejected(); + verify_file_io(); + + std::printf( "checks: %d, failures: %d\n", g_checks, g_failures ); + if( g_failures == 0 ) { + std::printf( "OK\n" ); + return 0; + } + std::printf( "FAILED\n" ); + return 1; +} diff --git a/utilities/Globals.cpp b/utilities/Globals.cpp index ac71e391..d533eb3a 100644 --- a/utilities/Globals.cpp +++ b/utilities/Globals.cpp @@ -662,6 +662,18 @@ bool global_settings::ConfigParseSimulation(cParser& Parser, const std::string& return true; } + if (token == "eu7.bake.cpu.percent") + { + ParseOneClamped(Parser, eu7_bake_cpu_percent, 1, 100); + return true; + } + + if (token == "eu7.v2.runtime") + { + ParseOne(Parser, eu7v2_runtime, 1, false); + return true; + } + if (token == "inactivepause") { ParseOne(Parser, bInactivePause); @@ -1616,6 +1628,8 @@ global_settings::export_as_text( std::ostream &Output ) const { export_as_text( Output, "eu7.pack.mesh.loader.workers", eu7_pack_mesh_loader_workers ); export_as_text( Output, "eu7.auto.bake", eu7_auto_bake ); export_as_text( Output, "eu7.bake.threads", eu7_bake_threads ); + export_as_text( Output, "eu7.bake.cpu.percent", eu7_bake_cpu_percent ); + export_as_text( Output, "eu7.v2.runtime", eu7v2_runtime ); export_as_text( Output, "inactivepause", bInactivePause ); export_as_text( Output, "slowmotion", iSlowMotionMask ); export_as_text( Output, "hideconsole", bHideConsole ); diff --git a/utilities/Globals.h b/utilities/Globals.h index 28696cb0..37d977dc 100644 --- a/utilities/Globals.h +++ b/utilities/Globals.h @@ -86,6 +86,8 @@ struct global_settings { int eu7_pack_mesh_loader_workers{ 0 }; // 0 = main-thread only (background mesh threads unsafe with Global) bool eu7_auto_bake{ true }; // przy ladowaniu mapy: bake SCM→EU7 gdy brak lub nieaktualny .eu7 int eu7_bake_threads{ 0 }; // watki bake (0 = auto = hardware_concurrency) + int eu7_bake_cpu_percent{ 80 }; // bake: docelowy % rdzeni logicznych, gdy eu7_bake_threads=0 (auto) + bool eu7v2_runtime{ false }; // eksperyment: transkoduj/laduj moduly przez chudy format eu7v2 (.eu7.v2) // logs bool priorityLoadText3D{false}; // ladowanie T3D priorytetowo int iWriteLogEnabled{ 3 }; // maska bitowa: 1-zapis do pliku, 2-okienko, 4-nazwy torów diff --git a/utilities/parser.cpp b/utilities/parser.cpp index be90ba6e..be863cf7 100644 --- a/utilities/parser.cpp +++ b/utilities/parser.cpp @@ -235,34 +235,40 @@ std::string cParser::readTokenFromStream(bool ToLower, const char *Break) std::string token; token.reserve(64); - const auto breakTable = makeBreakTable(Break); - char c = 0; + // rebuild the separator lookup table only when the break set actually changes; + // callers overwhelmingly pass the same string literal, so this stays cached. + if (Break != m_breakTableKey) { + m_breakTable = makeBreakTable(Break); + m_breakTableKey = Break; + } + const auto &breakTable = m_breakTable; + int ci = 0; + while ((ci = mStream->get()) != EOF) { + char c = static_cast(ci); + if (c == '\n') { + ++mLine; + } - while (token.empty() && mStream->peek() != EOF) { - while (mStream->peek() != EOF) { // idk why but with mStream->get(c) not all cars are loaded - c = static_cast(mStream->get()); - if (c == '\n') { - ++mLine; - } + const unsigned char uc = static_cast(c); + if (breakTable[uc]) { + // separator ends token (or continues skipping if token empty) + if (!token.empty()) + break; + continue; + } - const unsigned char uc = static_cast(c); - if (breakTable[uc]) { - // separator ends token (or continues skipping if token empty) - if (!token.empty()) - break; - continue; - } + if (ToLower) c = toLowerChar(c); + token.push_back(c); - if (ToLower) c = toLowerChar(c); - token.push_back(c); - - if (findQuotes(token)) { - continue; // glue quoted content - } - if (skipComments && trimComments(token)) { - break; // don't glue tokens separated by comment - } + if (findQuotes(token)) { + continue; // glue quoted content + } + if (skipComments && trimComments(token)) { + // comment stripped: return the token if we already have one, + // otherwise keep scanning for the next real token + if (!token.empty()) + break; } } diff --git a/utilities/parser.h b/utilities/parser.h index 6bac77e7..7b6606c5 100644 --- a/utilities/parser.h +++ b/utilities/parser.h @@ -14,10 +14,30 @@ http://mozilla.org/MPL/2.0/. #include #include #include +#include +#include +#include ///////////////////////////////////////////////////////////////////////////////////////////////////// // cParser -- generic class for parsing text data, either from file or provided string +namespace cparser_detail { +// Types for which std::from_chars matches the legacy stringstream extraction semantics. +// Character types are excluded on purpose: narrow-stream extraction of char/wchar_t reads a +// single character, whereas std::from_chars would parse an integer. +template +inline constexpr bool use_from_chars_v = + ( std::is_integral_v + && !std::is_same_v + && !std::is_same_v + && !std::is_same_v + && !std::is_same_v + && !std::is_same_v + && !std::is_same_v + && !std::is_same_v ) + || std::is_floating_point_v; +} // namespace cparser_detail + class cParser //: public std::stringstream { public: @@ -126,6 +146,9 @@ class cParser //: public std::stringstream commentmap mComments { commentmap::value_type( "/*", "*/" ), commentmap::value_type( "//", "\n" ) }; + // cached separator lookup table to avoid rebuilding it on every token read + char const *m_breakTableKey { nullptr }; + std::array m_breakTable {}; std::shared_ptr mIncludeParser; // child class to handle include directives. std::vector parameters; // parameter list for included file. std::deque tokens; @@ -143,11 +166,34 @@ cParser::operator>>( Type_ &Right ) { if( true == this->tokens.empty() ) { return *this; } - std::stringstream converter( this->tokens.front() ); - converter >> Right; - this->tokens.pop_front(); + if constexpr( cparser_detail::use_from_chars_v ) { + std::string const &token = this->tokens.front(); + char const *first { token.data() }; + char const *const last { first + token.size() }; + // legacy stream extraction accepts a leading '+', std::from_chars does not + if( first != last && *first == '+' ) { + ++first; + } + Type_ value {}; + auto const result { std::from_chars( first, last, value ) }; + if( result.ec == std::errc() ) { + Right = value; + } + else { + // fall back to the legacy path for inputs from_chars rejects (inf/nan/hex/...) + std::stringstream converter( token ); + converter >> Right; + } + this->tokens.pop_front(); + return *this; + } + else { + std::stringstream converter( this->tokens.front() ); + converter >> Right; + this->tokens.pop_front(); - return *this; + return *this; + } } template<>