16
0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-07-18 00:49:19 +02:00

Add headless parallel eu7v2 scenario bake with streaming and PLCE placements

Enable --eu7v2-bake from the main binary: parallel module pool, bounded-RAM
spool flush, streaming terrain triangles, flat include/model parsing, and
eu7v2 emit/load with optional verify. Large placement .scm files emit lean
PLCE records and bake referenced .inc modules separately for reuse.

- CLI: --eu7v2-bake, --eu7v2-verify, --eu7v2-mem-limit-gb, --eu7v2-threads,
  --eu7v2-max-parse; wire max_threads through to the bake parser
- eu7v2 v2 records: PLCE placements, runtime emitter/loader, batch verify
- Parallel bake pool with session cache; drop heavy-serial parse gate in spool
  mode; parse concurrency matches thread count
- Streaming terrain: batched parallel parse+bake, scan/bake pipeline, shape
  spool with persistent buffered I/O and flush-before-read
- Parallel flat-file streaming for models/includes; pack/model spool for
  low-memory incremental flush
- Optional 50 GB private-bytes guard during headless bake

Braniewo_szeroki: 160 modules, verify PASS, ~34s bake (nmt100 ~17s vs ~190s
serial baseline).

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
maj00r
2026-06-17 21:15:36 +02:00
parent 9574d2051b
commit beacc00932
40 changed files with 9327 additions and 193 deletions

View File

@@ -161,6 +161,8 @@ set(SOURCES
"scene/eu7/eu7_emit.cpp" "scene/eu7/eu7_emit.cpp"
"scene/eu7/eu7_apply.cpp" "scene/eu7/eu7_apply.cpp"
"scene/eu7/eu7_bake.cpp" "scene/eu7/eu7_bake.cpp"
"scene/eu7/v2/eu7v2_bake.cpp"
"scene/eu7/v2/eu7v2_load.cpp"
"scene/eu7/eu7_parameters.cpp" "scene/eu7/eu7_parameters.cpp"
"scene/scenenode.cpp" "scene/scenenode.cpp"
"simulation/simulation.cpp" "simulation/simulation.cpp"
@@ -415,11 +417,14 @@ if(WITH_EU7_PARSER)
message(FATAL_ERROR "EU07 parser headers missing at ${EU07_PARSER_INCLUDE}. Sync eu07-parser/include from the parser repo.") message(FATAL_ERROR "EU07 parser headers missing at ${EU07_PARSER_INCLUDE}. Sync eu07-parser/include from the parser repo.")
endif() endif()
add_library(eu07_bake STATIC "scene/eu7/eu7_bake_parser.cpp") add_library(eu07_bake STATIC "scene/eu7/eu7_bake_parser.cpp" "scene/eu7/v2/eu7v2_emit_runtime.cpp")
target_compile_features(eu07_bake PUBLIC cxx_std_23) target_compile_features(eu07_bake PUBLIC cxx_std_23)
set_target_properties(eu07_bake PROPERTIES CXX_STANDARD 23 CXX_STANDARD_REQUIRED ON) set_target_properties(eu07_bake PROPERTIES CXX_STANDARD 23 CXX_STANDARD_REQUIRED ON)
target_include_directories(eu07_bake PRIVATE "${EU07_PARSER_INCLUDE}") target_include_directories(eu07_bake PRIVATE "${EU07_PARSER_INCLUDE}" "${CMAKE_CURRENT_SOURCE_DIR}")
target_link_libraries(eu07_bake PUBLIC Threads::Threads) target_link_libraries(eu07_bake PUBLIC Threads::Threads)
if(WIN32)
target_link_libraries(eu07_bake PUBLIC psapi)
endif()
if(MSVC) if(MSVC)
target_compile_options(eu07_bake PRIVATE /utf-8 /Zc:__cplusplus /EHsc /wd4834) target_compile_options(eu07_bake PRIVATE /utf-8 /Zc:__cplusplus /EHsc /wd4834)
endif() endif()

View File

@@ -21,6 +21,7 @@ Stele, firleju, szociu, hunter, ZiomalCl, OLI_EU and others
#include "application/application.h" #include "application/application.h"
#include "utilities/Logs.h" #include "utilities/Logs.h"
#include <cstdlib> #include <cstdlib>
#include <iostream>
#ifdef WITHDUMPGEN #ifdef WITHDUMPGEN
#ifdef _WIN32 #ifdef _WIN32
#include <Windows.h> #include <Windows.h>
@@ -89,6 +90,14 @@ int main(int argc, char *argv[])
// init start timestamp // init start timestamp
Global.startTimestamp = std::chrono::steady_clock::now(); Global.startTimestamp = std::chrono::steady_clock::now();
// headless short-circuit: text -> eu7v2 bake (+optional verify), no window
if (eu07_application::wants_eu7v2_headless(argc, argv))
{
int const code{eu07_application::run_eu7v2_headless(argc, argv)};
std::cout.flush();
std::_Exit(code);
}
// quick short-circuit for standalone e3d export // quick short-circuit for standalone e3d export
if (argc == 6 && std::string(argv[1]) == "-e3d") if (argc == 6 && std::string(argv[1]) == "-e3d")
{ {

View File

@@ -30,7 +30,10 @@ http://mozilla.org/MPL/2.0/.
#include "utilities/dictionary.h" #include "utilities/dictionary.h"
#include "version_info.h" #include "version_info.h"
#include "ref/discord-rpc/include/discord_rpc.h" #include "ref/discord-rpc/include/discord_rpc.h"
#include "scene/eu7/eu7_bake_parser.h"
#include "scene/eu7/eu7_bake_mem_guard.h"
#include <chrono> #include <chrono>
#include <fstream>
#include "utilities/translation.h" #include "utilities/translation.h"
#if WITH_DISCORD_RPC #if WITH_DISCORD_RPC
@@ -1095,6 +1098,105 @@ void eu07_application::init_files()
} }
namespace fs = std::filesystem; namespace fs = std::filesystem;
bool eu07_application::wants_eu7v2_headless(int Argc, char *Argv[])
{
for (int i = 1; i < Argc; ++i)
{
if (std::string(Argv[i]) == "--eu7v2-bake")
return true;
}
return false;
}
int eu07_application::run_eu7v2_headless(int Argc, char *Argv[])
{
std::string bake_path;
bool verify{false};
unsigned mem_limit_gb{scene::eu7::bake_parser::kDefaultBakeMemLimitGb};
unsigned max_threads{0u};
unsigned max_parse{0u};
unsigned heavy_parse_mb{0u};
for (int i = 1; i < Argc; ++i)
{
std::string const token{Argv[i]};
if (token == "--eu7v2-bake")
{
if (i + 1 < Argc)
bake_path = Argv[++i];
}
else if (token == "--eu7v2-verify")
{
verify = true;
}
else if (token == "--eu7v2-mem-limit-gb")
{
if (i + 1 < Argc)
mem_limit_gb = static_cast<unsigned>(std::stoul(Argv[++i]));
}
else if (token == "--eu7v2-max-parse")
{
if (i + 1 < Argc)
max_parse = static_cast<unsigned>(std::stoul(Argv[++i]));
}
else if (token == "--eu7v2-threads")
{
if (i + 1 < Argc)
max_threads = static_cast<unsigned>(std::stoul(Argv[++i]));
}
else if (token == "--eu7v2-heavy-parse-mb")
{
if (i + 1 < Argc)
heavy_parse_mb = static_cast<unsigned>(std::stoul(Argv[++i]));
}
}
auto report_line = [](std::string const &line) {
std::cout << line << std::endl;
std::ofstream log{"log.txt", std::ios::app};
if (log)
log << line << '\n';
};
if (bake_path.empty())
{
report_line("[eu7v2] BLAD: --eu7v2-bake wymaga sciezki do pliku .scn/.inc");
return 2;
}
report_line("[eu7v2] headless bake: " + bake_path + (verify ? " (+verify)" : ""));
scene::eu7::bake_parser::Eu7v2BakeReport const report{
scene::eu7::bake_parser::bake_scenario_tree_eu7v2(
bake_path, max_threads, verify, mem_limit_gb, max_parse, heavy_parse_mb)};
if (false == report.baked)
{
report_line("[eu7v2] BAKE FAIL: " + report.error);
return 1;
}
report_line(
"[eu7v2] modulow=" + std::to_string(report.module_count) +
" modeli=" + std::to_string(report.model_count) +
" root=" + report.root_binary_path);
if (false == report.error.empty())
{
report_line("[eu7v2] OSTRZEZENIE: " + report.error);
}
if (verify)
{
report_line(report.verify_ok ? "[eu7v2] VERIFY: PASS" : "[eu7v2] VERIFY: FAIL");
if (false == report.verify_ok)
return 1;
}
report_line("[eu7v2] OK");
return (report.error.empty()) ? 0 : 1;
}
int eu07_application::init_settings(int Argc, char *Argv[]) int eu07_application::init_settings(int Argc, char *Argv[])
{ {
Global.asVersion = VERSION_INFO; Global.asVersion = VERSION_INFO;

View File

@@ -36,6 +36,12 @@ public:
init( int Argc, char *Argv[] ); init( int Argc, char *Argv[] );
int int
run(); run();
// Headless text -> eu7v2 bake (+optional roundtrip verify). Runs without a
// window / renderer / audio. Returns a process exit code (0 == OK).
static bool
wants_eu7v2_headless( int Argc, char *Argv[] );
static int
run_eu7v2_headless( int Argc, char *Argv[] );
// issues request for a worker thread to perform specified task. returns: true if task was scheduled // issues request for a worker thread to perform specified task. returns: true if task was scheduled
#ifdef WITH_DISCORD_RPC #ifdef WITH_DISCORD_RPC

381
docs/eu7v2-format.md Normal file
View File

@@ -0,0 +1,381 @@
# Format pliku EU7v2 (`.eu7v2`)
> Dokumentacja oparta na kodzie źródłowym z:
> - `scene/eu7/v2/` — format, emitter, loader, testy roundtrip
> - `scene/eu7/eu7_bake_parser.cpp` — headless bake scenariusza
> - `eu07-parser/include/eu07/scene/bake/` — pool bake, streaming terenu
>
> Stan: **czerwiec 2026**, wersja kontenera **v1** (`kVersion = 1`).
---
## Spis treści
1. [Przegląd ogólny](#1-przegląd-ogólny)
2. [Nagłówek pliku](#2-nagłówek-pliku)
3. [Struktura chunku](#3-struktura-chunku)
4. [Rodzaje pliku (`file_kind`)](#4-rodzaje-pliku-file_kind)
5. [Katalog chunków](#5-katalog-chunków)
6. [Typy podstawowe](#6-typy-podstawowe)
7. [STRS — tablica łańcuchów](#7-strs--tablica-łańcuchów)
8. [META — metadane modułu](#8-meta--metadane-modułu)
9. [INCL — include rekurencyjne](#9-incl--include-rekurencyjne)
10. [PLCE — placementy modułów wielokrotnego użytku](#10-plce--placementy-modułów-wielokrotnego-użytku)
11. [PROT + INST — prototypy i instancje modeli](#11-prot--inst--prototypy-i-instancje-modeli)
12. [SHPE — kształty (triangles)](#12-shpe--kształty-triangles)
13. [MESH — teren (tile)](#13-mesh--teren-tile)
14. [LINE — linie](#14-line--linie)
15. [Rekordy symulacji](#15-rekordy-symulacji)
16. [Bake headless i drzewo modułów](#16-bake-headless-i-drzewo-modułów)
17. [Ścieżki plików](#17-ścieżki-plików)
---
## 1. Przegląd ogólny
Plik `.eu7v2` to binarny format skompilowanego modułu scenerii:
- **Moduły wielokrotnego użytku** — osobny plik `.eu7v2` per `.inc` / `.scm`, placementy jako lean **PLCE**
- **Prototypy + instancje** — chunk **PROT** (deduplikacja meshów) + **INST** (transformacja w world-space)
- **Chunki pomijalne** — nieznany FourCC można pominąć po polu `size`
- **Little-endian** — deterministyczny zapis niezależny od endianness hosta
- **Centralna tablica STRS** — wszystkie stringi przez `string_id` (`uint32`)
Magic pliku: **`EU7C`** (FourCC `'E','U','7','C'`).
---
## 2. Nagłówek pliku
```
Offset Rozmiar Typ Opis
------ ------- -------- -------------------------
0 4 uint32 Magic = 'EU7C'
4 2 uint16 Version (= 1)
6 2 uint16 file_kind (patrz sekcja 4)
8 4 uint32 Reserved (= 0)
12 4 uint32 Reserved (= 0)
16 ... chunks Strumień chunkowy (do EOF)
```
Nagłówek ma **16 bajtów**. Brak pola „rozmiar całego pliku”.
---
## 3. Struktura chunku
```
Offset Rozmiar Typ Opis
------ ------- ------- -----------------------------------------
0 4 uint32 Chunk ID — FourCC little-endian
4 8 uint64 Payload size (tylko payload, BEZ nagłówka chunku)
12 ... bytes Payload
```
Pole `size` to rozmiar **samego payloadu** (`uint64`), bez nagłówka chunku.
Chunki są sekwencyjne do EOF. Nieznany chunk: `seek(payload_size)`.
FourCC — bajty w pliku w kolejności ASCII (np. `STRS``53 54 52 53`).
---
## 4. Rodzaje pliku (`file_kind`)
| Wartość | Nazwa | Typowy źródłowy plik | Opis |
|---------|------------|-----------------------------|-------------------------------------------|
| `1` | `sim` | `.scn`, `.scm`, `.sbt`, `.ctr` | Moduł scenariusza / root |
| `2` | `module` | `.inc` | Moduł wielokrotnego użytku (trawa, drzewo…) |
| `3` | `tile` | (planowane) | Kafel 1 km — teren + instancje |
| `4` | `manifest` | (planowane) | Indeks kafli / modułów mapy |
Emitter runtime (`eu7v2_emit_runtime.cpp`) ustawia:
- `.inc``file_kind::module`
- pozostałe rozszerzenia modułów → `file_kind::sim`
Loader akceptuje `sim` i `module` (`eu7v2_load.cpp`).
---
## 5. Katalog chunków
| FourCC | Stała C++ | Typ pliku | Opis |
|---------|---------------|-------------|---------------------------------------------------|
| `STRS` | `chunk::strs` | wszystkie | Tablica stringów (zawsze pierwszy logicznie) |
| `META` | `chunk::meta` | sim/module | Metadane modułu (first-init, placement params) |
| `INCL` | `chunk::incl` | sim/module | Include rekurencyjne (submoduły `.eu7v2`) |
| `PLCE` | `chunk::plce` | sim/module | Lean placementy → osobne moduły `.eu7v2` |
| `PROT` | `chunk::prot` | sim/module/tile | Prototypy modeli (mesh bez transformacji) |
| `INST` | `chunk::inst` | sim/module/tile | Instancje modeli (proto + transform) |
| `MESH` | `chunk::mesh` | tile/sim | Teren — mesh z origin f64 + wierzchołki f32 |
| `SHPE` | `chunk::shpe` | sim/module | Kształty triangles/strip/fan (nie-teren) |
| `LINE` | `chunk::line` | sim/module | Geometria linii |
| `TRAK` | `chunk::trak` | sim | Tory |
| `TRAC` | `chunk::trac` | sim | Przewody trakcyjne |
| `PWRS` | `chunk::pwrs` | sim | Źródła zasilania |
| `MEMC` | `chunk::memc` | sim | Komórki pamięci |
| `LAUN` | `chunk::laun` | sim | Wyzwalacze zdarzeń |
| `EVNT` | `chunk::evnt` | sim | Zdarzenia |
| `SOND` | `chunk::sond` | sim | Dźwięki |
| `DYNM` | `chunk::dynm` | sim | Pojazdy dynamiczne |
| `TRST` | `chunk::trst` | sim/module | Zestawy wagonowe |
| `TRGR` | `chunk::trgr` | sim | Precomputed track graph (planowane) |
| `SIDX` | `chunk::sidx` | manifest | Indeks sekcji przestrzennej (planowane) |
Chunki o zerowej liczności (pusta tablica) **nie są zapisywane** przez emitter.
---
## 6. Typy podstawowe
| Typ w pliku | Rozmiar | Opis |
|-------------|---------|---------------------------------------------------|
| `uint8` | 1 B | Bajt bez znaku |
| `uint16` | 2 B | LE |
| `uint32` | 4 B | LE |
| `uint64` | 8 B | LE |
| `int32` | 4 B | LE, ze znakiem |
| `float32` | 4 B | IEEE 754 |
| `float64` | 8 B | IEEE 754 — pozycje world-space, originy mesh |
| `string_id` | 4 B | Indeks STRS; `0xFFFFFFFF` = brak (`kNoString`) |
Współrzędne instancji modeli i placementów: **float64** (x/y/z).
Wierzchołki mesh/shape: **float32** względem origin (f64).
---
## 7. STRS — tablica łańcuchów
```
uint32 count
Powtórzone count razy:
uint32 length
char[] data (length bajtów, bez null-terminatora)
```
Indeks `0xFFFFFFFF` (`kNoString`) oznacza brak stringu.
---
## 8. META — metadane modułu
Layout wersjonowany (pole `layout_version`):
```
uint32 layout_version (= 1)
uint32 first_init_count
uint8 has_terrain_chunk (informacyjne)
uint8 has_pack_chunk (informacyjne)
uint8 placement_origin_x (numer parametru include, 0 = brak)
uint8 placement_origin_y
uint8 placement_origin_z
uint8 placement_rotation_y
```
---
## 9. INCL — include rekurencyjne
Submoduły wczytywane rekurencyjnie (np. zagnieżdżone `.scm`, tory, CTR):
```
uint32 count
Powtórzone count razy:
uint32 source_line
string_id source_path (.scm/.inc tekstowy)
string_id binary_path (.eu7v2)
uint32 param_count
string_id[] parameters
TransformRecord site_transform:
uint32 origin_stack_count
dvec3[] origin_stack
uint32 scale_stack_count
dvec3[] scale_stack
dvec3 rotation (stopnie, XYZ)
uint32 group_depth
```
---
## 10. PLCE — placementy modułów wielokrotnego użytku
Pliki placement (flora, dekoracje) emitują lean rekordy wskazujące na **osobny** moduł `.eu7v2`:
```
uint32 count
Powtórzone count razy:
string_id module_path (np. scenery/grass_l61/20.eu7v2)
string_id texture_override (kNoString jeśli brak)
float64 x, y, z (world-space)
float32 rotation_y (stopnie)
uint8 cell_id (0xFF = brak)
```
**Bake:** unikalne `.inc` z placement file są bake'owane jako osobne moduły `file_kind::module`.
**Runtime:** loader syntetyzuje include z PLCE (ładuje wskazany moduł z transformacją).
Typowy rozmiar: **~37 B / placement**.
Przykład: `204_trawky_ter.scm` (110k linii include) → `.eu7v2` ~4 MB (PLCE) + osobne `grass_l61/20.eu7v2` per unikalny `.inc`.
---
## 11. PROT + INST — prototypy i instancje modeli
### PROT — prototyp
```
uint32 count
Powtórzone count razy:
string_id model_file
string_id texture_file
uint8 flags (bit 0=transition, 1=is_terrain, 2=instanceable)
float32 range_min
float32 range_max
uint32 light_state_count
float32[] light_states
uint32 light_color_count
uint32[] light_colors
```
### INST — instancja
```
uint32 count
Powtórzone count razy:
uint8 flags (bit 0=has_scale, 1=texture_override, 2=has_node)
uint32 proto (indeks PROT)
float64 x, y, z
float32 ax, ay, az (kąty Eulera, stopnie)
uint8 cell_id
[flags & 1]: float32 sx, sy, sz
[flags & 2]: string_id texture_override
[flags & 4]: node_record
```
Root scenariusza: modele z PACK bake są deduplikowane do PROT+INST (flattened world-space).
---
## 12. SHPE — kształty (triangles)
```
uint32 count
Powtórzone count razy:
node_record node
uint8 translucent
string_id material
lighting_block (12 × float32: diffuse/ambient/specular RGBA)
float64 ox, oy, oz (origin world-space)
uint32 vertex_count
Powtórzone vertex_count razy:
float32 px, py, pz (względem origin)
float32 nx, ny, nz
float32 u, v
```
Typ węzła (`node.type`) niesie subtype: `triangles`, `triangle_strip`, `triangle_fan`.
Duże pliki terenu (same `node … triangles`) mogą być streamowane przy bake; shapes trafiają do **shape spool** na dysku, potem do chunku SHPE przy emit.
---
## 13. MESH — teren (tile)
Format terenu dla kafli 1 km (origin f64 + relative f32):
```
uint32 count
Powtórzone count razy:
string_id material
uint8 translucent
float64 ox, oy, oz
uint32 vertex_count
Powtórzone vertex_count razy:
float32 px, py, pz, nx, ny, nz, u, v
```
---
## 14. LINE — linie
```
uint32 count
Powtórzone count razy:
node_record
lighting_block
float32 line_width
float64 ox, oy, oz
uint32 vertex_count
dvec3[] vertices (float64 × 3)
```
---
## 15. Rekordy symulacji
Chunki symulacji: `TRAK`, `TRAC`, `PWRS`, `MEMC`, `LAUN`, `EVNT`, `SOND`, `DYNM`, `TRST`.
Wspólny nagłówek węzła: **`node_record`**. Stringi przez STRS. Pozycje torów/krzywych Béziera: **float64**.
Szczegóły pól: `scene/eu7/v2/eu7v2_records.h` (funkcje `write_*` / `read_*`).
---
## 16. Bake headless i drzewo modułów
CLI (binarny `eu07`, bez GUI):
```
eu07.exe --eu7v2-bake <sciezka.scn|.inc> [opcje]
```
| Opcja | Opis |
|--------------------------|---------------------------------------------------|
| `--eu7v2-verify` | Roundtrip: przeładuj `.eu7v2`, porównaj liczności rekordów |
| `--eu7v2-mem-limit-gb N` | Limit pamięci procesu (domyślnie 50 GB); włącza tryb spool |
| `--eu7v2-threads N` | Wątki pool bake (0 = auto = liczba rdzeni) |
| `--eu7v2-max-parse N` | Równoległe parse (0 = auto) |
| `--eu7v2-heavy-parse-mb N` | Próg serial parse dużych plików (0 = wyłączony w trybie spool) |
**Przepływ bake scenariusza:**
1. Root `.scn` → enqueue drzewa modułów (dedup po ścieżce kanonicznej)
2. Pool workerów bake'uje moduły równolegle
3. Każdy `.scm`/`.inc` → osobny `.eu7v2` obok źródła tekstowego
4. Placement `.scm`**PLCE** + enqueue unikalnych `.inc` jako `module`
5. Root emit na końcu (PACK → PROT+INST, includes z podmodułów)
6. Opcjonalnie verify batch wszystkich modułów
Implementacja: `scene/eu7/eu7_bake_parser.cpp`, `eu07-parser/.../bake_tree.hpp`.
---
## 17. Ścieżki plików
```cpp
binary_path_from_text("scenery/foo/bar.scm") scenery/foo/bar.eu7v2
binary_path_from_text("scenery/grass_l61/20.inc") scenery/grass_l61/20.eu7v2
binary_path_from_text("scenery/foo/base.ctr") scenery/foo/base.ctr.eu7v2
```
Rozszerzenia `.scm`, `.scn`, `.sbt`, `.inc``<stem>.eu7v2`.
Inne (np. `.ctr`) → `<nazwa_pliku>.eu7v2` (bez utraty suffixu).
---
## Pliki implementacji
| Temat | Plik |
|--------------------|---------------------------------------------------|
| Format core | `scene/eu7/v2/eu7v2_format.h` |
| Scene payloads | `scene/eu7/v2/eu7v2_scene.h` |
| Sim records | `scene/eu7/v2/eu7v2_records.h` |
| Emit RuntimeModule | `scene/eu7/v2/eu7v2_emit_runtime.cpp` |
| Load runtime | `scene/eu7/v2/eu7v2_load.cpp` |
| Test roundtrip | `scene/eu7/v2/eu7v2_test.cpp` |
| Headless bake | `scene/eu7/eu7_bake_parser.cpp` |
| Bake pool | `eu07-parser/include/eu07/scene/bake/bake_tree.hpp` |
---
*Dokumentacja na podstawie kodu; stan na 17 czerwca 2026.*

View File

@@ -87,13 +87,20 @@ inline void tokenizeInto(
skipFieldSeparators(text); skipFieldSeparators(text);
continue; continue;
} }
// Unterminated '[' (no matching ']' in the remainder): it is an
// ordinary character, not a bracket token. The engine's default
// break set is "\n\r\t ;", so '[' belongs inside identifiers such
// as event names ("Kon_It2[Sch2"). Fall through to the general scan
// below, which does NOT stop at '[' and therefore consumes it,
// guaranteeing the outer loop makes forward progress (previously it
// spun forever: the '[' was neither a bracket nor a separator nor a
// token character, so text never shrank).
} }
const char* const start = text.data(); const char* const start = text.data();
while (!text.empty() && while (!text.empty() &&
!isFieldSeparator(static_cast<unsigned char>(text.front())) && !isFieldSeparator(static_cast<unsigned char>(text.front())) &&
text.front() != '"' && text.front() != '"') {
text.front() != '[') {
text.remove_prefix(1); text.remove_prefix(1);
} }
const std::size_t len = static_cast<std::size_t>(text.data() - start); const std::size_t len = static_cast<std::size_t>(text.data() - start);
@@ -459,10 +466,7 @@ struct ParseResult {
return tokens; return tokens;
} }
[[nodiscard]] inline ParseResult parseFile( [[nodiscard]] inline ParseResult parseRawFile(RawFile raw, const ParseOptions& options = {}) {
const std::filesystem::path& path,
const ParseOptions& options = {}) {
const RawFile raw = readRawFile(path);
LogicalPass logical = toLogicalLines(raw.lines); LogicalPass logical = toLogicalLines(raw.lines);
ParseResult result; ParseResult result;
@@ -474,6 +478,12 @@ struct ParseResult {
return result; return result;
} }
[[nodiscard]] inline ParseResult parseFile(
const std::filesystem::path& path,
const ParseOptions& options = {}) {
return parseRawFile(readRawFile(path), options);
}
[[nodiscard]] inline ParseResult parseText(const std::string& text) { [[nodiscard]] inline ParseResult parseText(const std::string& text) {
RawFile raw; RawFile raw;
raw.data = text; raw.data = text;

View File

@@ -12,13 +12,13 @@
#include <eu07/scene/bake/compose_pack_models.hpp> #include <eu07/scene/bake/compose_pack_models.hpp>
#include <eu07/scene/bake/module.hpp> #include <eu07/scene/bake/module.hpp>
#include <eu07/scene/bake/pack_model_spool.hpp>
#include <eu07/scene/bake/streaming_terrain.hpp>
#include <eu07/scene/binary/runtime_module.hpp> #include <eu07/scene/binary/runtime_module.hpp>
#include <eu07/scene/include_resolve.hpp> #include <eu07/scene/include_resolve.hpp>
#include <eu07/scene/include_scan.hpp>
#include <eu07/scene/processor.hpp> #include <eu07/scene/processor.hpp>
@@ -39,6 +39,8 @@
#include <iostream> #include <iostream>
#include <memory>
#include <mutex> #include <mutex>
#include <optional> #include <optional>
@@ -122,8 +124,44 @@ struct BakeTreeOptions {
BakeProgressCallback onProgress; BakeProgressCallback onProgress;
// Hook wywolywany po zbakowaniu kazdego modulu (przed/zamiast zapisu .eu7).
// Dla roota packBatches != nullptr (sploszczone modele PACK). isRoot == true
// tylko dla pliku-korzenia drzewa bake. Moze byc wolany z wielu watkow.
std::function<void(
const RuntimeModule& module,
const std::filesystem::path& textPath,
bool isRoot,
const std::vector<binary::codec::ModelSectionBatch>* packBatches,
ShapeSpoolFile* shapeSpool)>
onModuleBaked;
// Gdy true: pomijamy zapis legacy .eu7 (writeRuntimeModule).
bool skipLegacyWrite = false;
// eu7v2 PACK: modele dzieci sa juz w root PACK — nie duplikuj INST w .eu7v2 dziecka.
bool omitChildModuleModels = false;
// Max rownoleglych pelnych parseFile/processScene w session cache (0 = auto: 1).
// Ogranicza peak RAM gdy wiele ciezkich .scm startuje naraz (Braniewo / linie).
unsigned maxConcurrentParses = 0;
// Mniej RAM, wolniej: 1 watek bake/compose, agresywne zwalnianie cache PACK.
bool lowMemoryMode = false;
// Pliki wieksze niz X MB: parse/bake tylko pojedynczo (0 = wylaczone).
unsigned heavyParseThresholdMb = 0;
// PACK compose: spłucz sekcje modeli poza RAM gdy przekroczony prog (lowMemoryMode).
std::size_t packFlushThreshold = 0;
bool packFlushPerFile = false;
std::function<void(std::vector<runtime::RuntimeModelInstance>&&)> onPackModelsFlush;
}; };
[[nodiscard]] inline bool useIncrementalRootPack(const BakeTreeOptions& options) {
return options.lowMemoryMode;
}
struct BakeTreeContext { struct BakeTreeContext {
@@ -256,6 +294,17 @@ struct BakePoolState {
} }
[[nodiscard]] inline unsigned resolveParseConcurrencyLimit(const unsigned maxConcurrentParses) {
if (maxConcurrentParses != 0) {
return std::max(1u, maxConcurrentParses);
}
return 1u;
}
[[nodiscard]] inline std::size_t heavyParseThresholdBytes(const unsigned threshold_mb) {
return threshold_mb == 0 ? 0u : static_cast<std::size_t>(threshold_mb) * 1024u * 1024u;
}
inline void recordError(BakePoolState& pool) { inline void recordError(BakePoolState& pool) {
@@ -324,7 +373,9 @@ inline void bakeWorkItem(
const BakeTreeOptions& options, const BakeTreeOptions& options,
const unsigned threadCount) { const unsigned threadCount,
BakeParseSessionCache& sessionCache) {
if (pool.stop.load(std::memory_order_acquire)) { if (pool.stop.load(std::memory_order_acquire)) {
@@ -358,34 +409,141 @@ inline void bakeWorkItem(
pool.inFlight.fetch_add(1, std::memory_order_relaxed); pool.inFlight.fetch_add(1, std::memory_order_relaxed);
if (options.onProgress) { bool models_pending_pack = false;
std::lock_guard lock(pool.progressMutex);
options.onProgress(
BakeProgress{
BakeProgressPhase::Start,
pool.completed.load(std::memory_order_relaxed),
0,
item.path,
threadCount,
});
}
try { try {
const ParseResult parsed = parseFile(item.path); RuntimeModule module;
SceneProcessOptions shallowOpts;
shallowOpts.expandIncludes = false;
const SceneDocument document =
processScene(parsed, context.sceneryRoot, shallowOpts).document;
const bool emitPackModels = canonicalPathKey(item.path) == rootKey; const bool emitPackModels = canonicalPathKey(item.path) == rootKey;
RuntimeModule module = bakeModule(document, {.skipLocalModels = emitPackModels}); std::vector<ParsedInclude> deferred_child_includes;
eu07::scene::detail::FlatFileKind flat_kind =
eu07::scene::detail::FlatFileKind::None;
std::unique_ptr<ShapeSpoolFile> triangle_shape_spool;
{
if (options.onProgress) {
std::lock_guard lock(pool.progressMutex);
options.onProgress(
BakeProgress{
BakeProgressPhase::Start,
pool.completed.load(std::memory_order_relaxed),
0,
item.path,
threadCount,
});
}
const bool prefer_lightweight_document =
useIncrementalRootPack(options) ||
eu07::scene::detail::shouldStreamFlatSourceFile(item.path);
if (prefer_lightweight_document) {
flat_kind = eu07::scene::detail::scanFlatFileKindStreaming(item.path);
}
const bool try_triangle_terrain_stream =
prefer_lightweight_document &&
flat_kind == eu07::scene::detail::FlatFileKind::None &&
eu07::scene::detail::shouldStreamFlatSourceFile(item.path);
bool used_triangle_terrain_stream = false;
SceneDocument empty_document;
const SceneDocument* document_ptr = nullptr;
if (try_triangle_terrain_stream) {
if (options.lowMemoryMode) {
const std::size_t path_hash = std::hash<std::string> {}(
item.path.lexically_normal().generic_string());
triangle_shape_spool = std::make_unique<ShapeSpoolFile>(
std::filesystem::temp_directory_path() /
("eu7v2_shape_" + std::to_string(path_hash) + ".bin"));
}
if (detail::streamBakeTriangleTerrain(
item.path, module, triangle_shape_spool.get(), threadCount)) {
used_triangle_terrain_stream = true;
document_ptr = &empty_document;
}
}
if (!used_triangle_terrain_stream) {
const SceneDocument& document =
(prefer_lightweight_document &&
(flat_kind == eu07::scene::detail::FlatFileKind::Models ||
flat_kind == eu07::scene::detail::FlatFileKind::Includes))
? sessionCache.documentForPackCompose(
item.path, context.sceneryRoot, nullptr, true)
: sessionCache.documentFor(item.path, context.sceneryRoot, nullptr);
document_ptr = &document;
}
const SceneDocument& document = *document_ptr;
const auto enqueue_unique_inc_modules = [&](const SceneDocument& doc) {
if (!options.onModuleBaked) {
return;
}
BakeWorkItem child;
child.includeStack = item.includeStack;
child.includeStack.push_back(item.path);
for (const ParsedInclude& include : doc.include) {
if (!eu07::scene::detail::isIncFile(include.file)) {
continue;
}
child.path = eu07::scene::detail::resolveIncludeSourcePath(
context.sceneryRoot, item.relativeFile, include.file);
child.relativeFile = eu07::scene::detail::relativeSceneryFile(
context.sceneryRoot, child.path);
enqueueWork(pool, child);
}
};
const auto enqueue_discovered_includes = [&](const SceneDocument& doc) {
BakeWorkItem child;
child.includeStack = item.includeStack;
child.includeStack.push_back(item.path);
for (const ParsedInclude& include : doc.include) {
// Placementy .inc (flora, szablony) — tylko do PACK compose,
// nie osobne moduly bake (3_rosl1-leo: 17k+ linii).
if (eu07::scene::detail::isIncFile(include.file)) {
continue;
}
child.path = eu07::scene::detail::resolveIncludeSourcePath(
context.sceneryRoot, item.relativeFile, include.file);
child.relativeFile = eu07::scene::detail::relativeSceneryFile(
context.sceneryRoot, child.path);
enqueueWork(pool, child);
}
};
// Root PACK compose jest bardzo pamieciochlonny — nie startuj rownolegle
// pelnych parse linii (l204/l220/…) az compose roota sie nie skonczy.
// lowMemoryMode + spool: jeden modul = jeden plik .scm, PACK skladany per modul.
if (emitPackModels) {
if (useIncrementalRootPack(options)) {
enqueue_discovered_includes(document);
} else {
deferred_child_includes = document.include;
}
} else {
enqueue_discovered_includes(document);
}
enqueue_unique_inc_modules(document);
const bool skipLocalModels =
emitPackModels ||
(options.omitChildModuleModels && isModelOnlyDocument(document));
if (!used_triangle_terrain_stream) {
module = bakeModule(document, {.skipLocalModels = skipLocalModels});
}
models_pending_pack =
!used_triangle_terrain_stream &&
skipLocalModels &&
(isModelOnlyDocument(document) ||
flat_kind == eu07::scene::detail::FlatFileKind::Models);
if (!used_triangle_terrain_stream) {
sessionCache.releaseHeavyParseStorageAfterModuleBake(
item.path, models_pending_pack);
}
}
const std::filesystem::path eu7Path = const std::filesystem::path eu7Path =
@@ -396,7 +554,7 @@ inline void bakeWorkItem(
binary::WriteRuntimeModuleOptions writeOpts; binary::WriteRuntimeModuleOptions writeOpts;
writeOpts.emitPackModels = emitPackModels; writeOpts.emitPackModels = emitPackModels;
std::vector<binary::codec::ModelSectionBatch> packBatches; std::vector<binary::codec::ModelSectionBatch> packBatches;
if (writeOpts.emitPackModels) { if (writeOpts.emitPackModels && !useIncrementalRootPack(options)) {
if (options.onProgress) { if (options.onProgress) {
std::lock_guard lock(pool.progressMutex); std::lock_guard lock(pool.progressMutex);
options.onProgress( options.onProgress(
@@ -431,11 +589,12 @@ inline void bakeWorkItem(
pack_options.progress = &pack_progress; pack_options.progress = &pack_progress;
pack_options.stats = &pack_stats; pack_options.stats = &pack_stats;
pack_options.max_threads = threadCount; pack_options.max_threads = threadCount;
pack_options.low_memory = options.lowMemoryMode;
pack_options.pack_flush_threshold = options.packFlushThreshold;
pack_options.pack_flush_per_file = options.packFlushPerFile;
pack_options.on_pack_models_flush = options.onPackModelsFlush;
pack_options.pack_batches = &packBatches; pack_options.pack_batches = &packBatches;
PackComposeSeed root_seed; pack_options.session_cache = &sessionCache;
root_seed.path = item.path.lexically_normal();
root_seed.entry = makePackFileCacheEntry(document);
pack_options.root_seed = &root_seed;
if (options.onProgress) { if (options.onProgress) {
std::cerr << "[EU7] PACK compose: " << threadCount << " watkow\n" << std::flush; std::cerr << "[EU7] PACK compose: " << threadCount << " watkow\n" << std::flush;
} }
@@ -457,21 +616,91 @@ inline void bakeWorkItem(
<< std::flush; << std::flush;
} }
printPackComposeStats(pack_stats, std::cerr); printPackComposeStats(pack_stats, std::cerr);
printBakeParseSessionStats(sessionCache, std::cerr);
if (stats != nullptr) { if (stats != nullptr) {
stats->has_pack_diagnostics = true; stats->has_pack_diagnostics = true;
stats->pack_compose = snapshotPackComposeStats(pack_stats); stats->pack_compose = snapshotPackComposeStats(pack_stats);
} }
} }
binary::PackWriteStats pack_write_stats;
if (writeOpts.emitPackModels) { // lowMemoryMode: jeden plik = jeden modul — PACK tylko z tego .scm, potem spool/RAM zwolnione.
writeOpts.pack_write_stats = &pack_write_stats; if (useIncrementalRootPack(options) && models_pending_pack && !emitPackModels) {
std::cerr << "[EU7] zapis .eu7 (PACK)...\n" << std::flush; if (flat_kind == eu07::scene::detail::FlatFileKind::Models &&
options.onPackModelsFlush) {
detail::streamFlatModelsToPackFlush(item.path, options.onPackModelsFlush);
} else {
std::vector<binary::codec::ModelSectionBatch> module_pack_batches;
PackComposeOptions pack_options;
pack_options.low_memory = true;
pack_options.pack_flush_per_file = options.packFlushPerFile;
pack_options.on_pack_models_flush = options.onPackModelsFlush;
pack_options.pack_batches = &module_pack_batches;
pack_options.session_cache = &sessionCache;
pack_options.max_threads = 1;
(void)composePackModels(
item.path, context.sceneryRoot, item.relativeFile, pack_options);
if (options.onPackModelsFlush) {
std::vector<runtime::RuntimeModelInstance> tail;
for (binary::codec::ModelSectionBatch& batch : module_pack_batches) {
tail.insert(
tail.end(),
std::make_move_iterator(batch.models.begin()),
std::make_move_iterator(batch.models.end()));
}
if (!tail.empty()) {
options.onPackModelsFlush(std::move(tail));
}
}
}
sessionCache.dropPackCacheEntry(item.path);
} }
binary::writeRuntimeModule(eu7Path, module, writeOpts);
if (writeOpts.emitPackModels) { // Hook: pozwala silnikowi wyemitowac wlasny format (eu7v2) bezposrednio
printPackWriteStats(pack_write_stats, std::cerr); // z RuntimeModule, z dostepem do sploszczonych modeli PACK dla roota.
if (stats != nullptr) { if (options.onModuleBaked) {
stats->pack_write = pack_write_stats; const bool root_with_pack =
emitPackModels && !useIncrementalRootPack(options);
const std::vector<binary::codec::ModelSectionBatch>* batchesPtr =
root_with_pack ? &packBatches : nullptr;
options.onModuleBaked(
module, item.path, emitPackModels, batchesPtr, triangle_shape_spool.get());
}
if (emitPackModels && !deferred_child_includes.empty() &&
!useIncrementalRootPack(options)) {
SceneDocument stub;
stub.include = std::move(deferred_child_includes);
BakeWorkItem child;
child.includeStack = item.includeStack;
child.includeStack.push_back(item.path);
for (const ParsedInclude& include : stub.include) {
if (eu07::scene::detail::isIncFile(include.file)) {
continue;
}
child.path = eu07::scene::detail::resolveIncludeSourcePath(
context.sceneryRoot, item.relativeFile, include.file);
child.relativeFile =
eu07::scene::detail::relativeSceneryFile(context.sceneryRoot, child.path);
enqueueWork(pool, child);
}
}
if (!models_pending_pack) {
sessionCache.evictEntryIfNotNeeded(item.path);
}
if (!options.skipLegacyWrite) {
binary::PackWriteStats pack_write_stats;
if (writeOpts.emitPackModels) {
writeOpts.pack_write_stats = &pack_write_stats;
std::cerr << "[EU7] zapis .eu7 (PACK)...\n" << std::flush;
}
binary::writeRuntimeModule(eu7Path, module, writeOpts);
if (writeOpts.emitPackModels) {
printPackWriteStats(pack_write_stats, std::cerr);
if (stats != nullptr) {
stats->pack_write = pack_write_stats;
}
} }
} }
@@ -525,30 +754,6 @@ inline void bakeWorkItem(
} }
BakeWorkItem child;
child.includeStack = item.includeStack;
child.includeStack.push_back(item.path);
for (const std::string& includeFile : scanIncludeFilePaths(parsed.tokens)) {
child.path = eu07::scene::detail::resolveIncludeSourcePath(
context.sceneryRoot, item.relativeFile, includeFile);
child.relativeFile = eu07::scene::detail::relativeSceneryFile(
context.sceneryRoot, child.path);
enqueueWork(pool, child);
}
} catch (...) { } catch (...) {
recordError(pool); recordError(pool);
@@ -585,7 +790,9 @@ inline void bakePoolWorker(
const BakeTreeOptions& options, const BakeTreeOptions& options,
const unsigned threadCount) { const unsigned threadCount,
BakeParseSessionCache& sessionCache) {
while (true) { while (true) {
@@ -639,7 +846,7 @@ inline void bakePoolWorker(
bakeWorkItem( bakeWorkItem(
item, context, rootKey, pool, stats, rootModuleOut, options, threadCount); item, context, rootKey, pool, stats, rootModuleOut, options, threadCount, sessionCache);
} catch (...) { } catch (...) {
@@ -690,6 +897,11 @@ inline void runBakePool(
BakePoolState pool; BakePoolState pool;
BakeParseSessionCache sessionCache {
detail::resolveParseConcurrencyLimit(options.maxConcurrentParses),
options.lowMemoryMode,
detail::heavyParseThresholdBytes(options.heavyParseThresholdMb) };
enqueueWork( enqueueWork(
pool, pool,
@@ -728,7 +940,9 @@ inline void runBakePool(
std::cref(options), std::cref(options),
threadCount); threadCount,
std::ref(sessionCache));
} }

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,138 @@
#pragma once
// Append-only spool: modele spłukane z PACK compose (mniejszy peak RAM).
#include <eu07/scene/binary/runtime_module.hpp>
#include <eu07/scene/runtime/nodes.hpp>
#include <array>
#include <cstddef>
#include <filesystem>
#include <fstream>
#include <mutex>
#include <stdexcept>
#include <utility>
#include <vector>
namespace eu07::scene::bake {
inline constexpr std::size_t kSpoolStreamBufferBytes = 1024u * 1024u;
class PackModelSpoolFile {
public:
explicit PackModelSpoolFile(std::filesystem::path path)
: path_(std::move(path)),
out_(path_, std::ios::binary | std::ios::trunc) {
if (!out_) {
throw std::runtime_error("EU7 PACK spool: nie mozna utworzyc " + path_.string());
}
out_.rdbuf()->pubsetbuf(write_buffer_.data(), write_buffer_.size());
}
void append(std::vector<runtime::RuntimeModelInstance>&& models) {
if (models.empty()) {
return;
}
std::lock_guard lock(mutex_);
for (runtime::RuntimeModelInstance& model : models) {
binary::detail::collectModelStrings(table_, model);
binary::detail::writeRuntimeModel(out_, table_, model);
++model_count_;
}
}
template <typename Fn>
void for_each_model(Fn&& fn) const {
{
std::lock_guard lock(mutex_);
out_.flush();
}
std::ifstream in(path_, std::ios::binary);
if (!in) {
return;
}
while (in.peek() != EOF) {
fn(binary::detail::readRuntimeModel(in, table_));
}
}
[[nodiscard]] std::size_t model_count() const noexcept {
return model_count_;
}
[[nodiscard]] const std::filesystem::path& path() const noexcept {
return path_;
}
private:
std::filesystem::path path_;
binary::StringTable table_;
std::size_t model_count_ = 0;
std::array<char, kSpoolStreamBufferBytes> write_buffer_ {};
mutable std::ofstream out_;
mutable std::mutex mutex_;
};
class ShapeSpoolFile {
public:
explicit ShapeSpoolFile(std::filesystem::path path)
: path_(std::move(path)),
out_(path_, std::ios::binary | std::ios::trunc) {
if (!out_) {
throw std::runtime_error("EU7 shape spool: nie mozna utworzyc " + path_.string());
}
out_.rdbuf()->pubsetbuf(write_buffer_.data(), write_buffer_.size());
}
void append(runtime::RuntimeShapeNode&& shape) {
std::lock_guard lock(mutex_);
binary::detail::collectShapeStrings(table_, shape);
binary::detail::writeRuntimeShape(out_, table_, shape);
++shape_count_;
}
void append_batch(std::vector<runtime::RuntimeShapeNode>&& shapes) {
if (shapes.empty()) {
return;
}
std::lock_guard lock(mutex_);
for (runtime::RuntimeShapeNode& shape : shapes) {
binary::detail::collectShapeStrings(table_, shape);
binary::detail::writeRuntimeShape(out_, table_, shape);
++shape_count_;
}
}
template <typename Fn>
void for_each_shape(Fn&& fn) const {
{
std::lock_guard lock(mutex_);
out_.flush();
}
std::ifstream in(path_, std::ios::binary);
if (!in) {
return;
}
while (in.peek() != EOF) {
fn(binary::detail::readRuntimeShape(in, table_));
}
}
[[nodiscard]] std::size_t shape_count() const noexcept {
return shape_count_;
}
[[nodiscard]] const std::filesystem::path& path() const noexcept {
return path_;
}
private:
std::filesystem::path path_;
binary::StringTable table_;
std::size_t shape_count_ = 0;
std::array<char, kSpoolStreamBufferBytes> write_buffer_ {};
mutable std::ofstream out_;
mutable std::mutex mutex_;
};
} // namespace eu07::scene::bake

View File

@@ -0,0 +1,432 @@
#pragma once
// Duze pliki terenu (same node triangles, np. nmt100_warmaz_ter.scm):
// linia po linii, parse per-wezel, bez readRawFile / tokenizacji calego pliku.
#include <eu07/parser.hpp>
#include <eu07/scene/bake/mesh.hpp>
#include <eu07/scene/bake/module.hpp>
#include <eu07/scene/bake/pack_model_spool.hpp>
#include <eu07/scene/context.hpp>
#include <eu07/scene/cursor.hpp>
#include <eu07/scene/match.hpp>
#include <eu07/scene/node/common.hpp>
#include <eu07/scene/node/triangles.hpp>
#include <eu07/scene/parallel_models.hpp>
#include <atomic>
#include <condition_variable>
#include <cstddef>
#include <deque>
#include <filesystem>
#include <functional>
#include <mutex>
#include <string>
#include <string_view>
#include <thread>
#include <vector>
namespace eu07::scene::bake::detail {
enum class TriangleTerrainLineKind {
Skip,
Header,
Vertex,
EndTri,
Invalid,
};
[[nodiscard]] inline TriangleTerrainLineKind classifyTriangleTerrainLine(std::string_view text) {
skipFieldSeparators(text);
if (text.empty()) {
return TriangleTerrainLineKind::Skip;
}
if (eu07::detail::isLineComment(text)) {
return TriangleTerrainLineKind::Skip;
}
if (eu07::scene::detail::segmentStartsWithKeyword(text, "node")) {
if (text.find("triangles") == std::string_view::npos) {
return TriangleTerrainLineKind::Invalid;
}
return TriangleTerrainLineKind::Header;
}
if (isKeyword(text, "endtri") || isKeyword(text, "endtriangles")) {
return TriangleTerrainLineKind::EndTri;
}
if (!text.empty() &&
(text.front() == '-' || text.front() == '+' ||
(text.front() >= '0' && text.front() <= '9'))) {
return TriangleTerrainLineKind::Vertex;
}
return TriangleTerrainLineKind::Invalid;
}
[[nodiscard]] inline bool parseTriangleTerrainBlock(
const std::vector<std::string>& lines,
const std::size_t header_line,
ParsedNodeTriangles& out) {
if (lines.size() < 2) {
return false;
}
std::vector<SourceToken> header_tokens;
tokenizeInto(lines.front(), header_tokens, header_line);
TokenStream stream(header_tokens);
NodeHeader header;
std::string subtype;
std::vector<SourceToken> raw;
if (!node::io::consumeHeader(stream, header, subtype, raw) ||
!isKeyword(subtype, node_triangles::kSubtype) ||
!node::io::takeString(stream, raw, out.texture)) {
return false;
}
out.header = header;
out.raw.clear();
out.vertices.clear();
out.vertices.reserve(lines.size() > 2 ? lines.size() - 2 : 0);
for (std::size_t i = 1; i + 1 < lines.size(); ++i) {
std::vector<SourceToken> vertex_tokens;
tokenizeInto(lines[i], vertex_tokens, header_line + i);
TokenStream vertex_stream(vertex_tokens);
MeshVertex vertex;
if (!node_triangles::takeVertex(vertex_stream, raw, vertex, !vertex_stream.empty())) {
return false;
}
out.vertices.push_back(vertex);
}
return !out.vertices.empty();
}
struct TriangleTerrainBlock {
std::size_t header_line = 0;
std::vector<std::string> lines;
};
constexpr std::size_t kTerrainParallelBatchSize = 8192;
constexpr std::size_t kTerrainParallelMinBlocks = 8;
constexpr std::size_t kTerrainPipelineMaxQueuedBatches = 2;
[[nodiscard]] inline unsigned resolveTerrainThreadCount(const unsigned max_threads) {
const unsigned hw =
std::thread::hardware_concurrency() == 0 ? 4u : std::thread::hardware_concurrency();
if (max_threads == 0) {
return std::max(1u, hw);
}
return std::max(1u, max_threads);
}
[[nodiscard]] inline bool bakeTriangleTerrainBatch(
const std::vector<TriangleTerrainBlock>& batch,
const unsigned max_threads,
std::vector<runtime::RuntimeShapeNode>& baked_out) {
baked_out.resize(batch.size());
if (batch.empty()) {
return true;
}
if (batch.size() < kTerrainParallelMinBlocks ||
resolveTerrainThreadCount(max_threads) <= 1) {
for (std::size_t i = 0; i < batch.size(); ++i) {
ParsedNodeTriangles parsed;
if (!parseTriangleTerrainBlock(batch[i].lines, batch[i].header_line, parsed)) {
return false;
}
baked_out[i] = bakeTriangles(parsed);
}
return true;
}
const unsigned worker_count = resolveTerrainThreadCount(max_threads);
std::atomic<std::size_t> next_index { 0 };
std::atomic<bool> ok { true };
const auto worker = [&]() {
while (ok.load(std::memory_order_relaxed)) {
const std::size_t index = next_index.fetch_add(1, std::memory_order_relaxed);
if (index >= batch.size()) {
return;
}
try {
ParsedNodeTriangles parsed;
if (!parseTriangleTerrainBlock(
batch[index].lines, batch[index].header_line, parsed)) {
ok.store(false, std::memory_order_relaxed);
return;
}
baked_out[index] = bakeTriangles(parsed);
} catch (...) {
ok.store(false, std::memory_order_relaxed);
return;
}
}
};
std::vector<std::thread> threads;
threads.reserve(worker_count);
for (unsigned i = 0; i < worker_count; ++i) {
threads.emplace_back(worker);
}
for (std::thread& thread : threads) {
thread.join();
}
return ok.load(std::memory_order_relaxed);
}
[[nodiscard]] inline bool scanTriangleTerrainOnlyStreaming(const std::filesystem::path& path) {
bool saw_header = false;
bool in_block = false;
bool ok = true;
eu07::scene::detail::forEachStreamingLogicalSegment(
path, [&](const LogicalSegment& segment) {
const TriangleTerrainLineKind kind = classifyTriangleTerrainLine(segment.text);
switch (kind) {
case TriangleTerrainLineKind::Skip:
return true;
case TriangleTerrainLineKind::Header:
saw_header = true;
in_block = true;
return true;
case TriangleTerrainLineKind::Vertex:
if (!in_block) {
ok = false;
return false;
}
return true;
case TriangleTerrainLineKind::EndTri:
if (!in_block) {
ok = false;
return false;
}
in_block = false;
return true;
case TriangleTerrainLineKind::Invalid:
default:
ok = false;
return false;
}
});
return ok && saw_header && !in_block;
}
[[nodiscard]] inline bool scanTriangleTerrainBlocksStreaming(
const std::filesystem::path& path,
const std::function<bool(std::vector<TriangleTerrainBlock>&&)>& on_batch) {
std::vector<TriangleTerrainBlock> batch;
batch.reserve(kTerrainParallelBatchSize);
TriangleTerrainBlock current_block;
bool in_block = false;
bool saw_header = false;
const auto finish_block = [&]() -> bool {
if (current_block.lines.empty()) {
return true;
}
batch.push_back(std::move(current_block));
current_block = {};
in_block = false;
if (batch.size() >= kTerrainParallelBatchSize) {
if (!on_batch(std::move(batch))) {
return false;
}
batch = {};
batch.reserve(kTerrainParallelBatchSize);
}
return true;
};
bool ok = true;
eu07::scene::detail::forEachStreamingLogicalSegment(
path, [&](const LogicalSegment& segment) {
const TriangleTerrainLineKind kind = classifyTriangleTerrainLine(segment.text);
switch (kind) {
case TriangleTerrainLineKind::Skip:
return true;
case TriangleTerrainLineKind::Header:
saw_header = true;
if (!finish_block()) {
ok = false;
return false;
}
current_block.header_line = segment.sourceLine;
current_block.lines.emplace_back(segment.text);
in_block = true;
return true;
case TriangleTerrainLineKind::Vertex:
if (!in_block) {
ok = false;
return false;
}
current_block.lines.emplace_back(segment.text);
return true;
case TriangleTerrainLineKind::EndTri:
if (!in_block) {
ok = false;
return false;
}
current_block.lines.emplace_back(segment.text);
if (!finish_block()) {
ok = false;
return false;
}
return true;
case TriangleTerrainLineKind::Invalid:
default:
ok = false;
return false;
}
});
if (ok && in_block) {
ok = finish_block();
}
if (ok && !batch.empty()) {
ok = on_batch(std::move(batch));
}
return ok && saw_header;
}
// Zwraca false gdy plik nie jest czystym terenem triangles (wtedy uzyj documentFor).
// Gdy shape_spool != nullptr, ksztalty trafiaja na dysk (module.scene.shapes puste).
[[nodiscard]] inline bool streamBakeTriangleTerrain(
const std::filesystem::path& path,
RuntimeModule& module,
ShapeSpoolFile* shape_spool = nullptr,
const unsigned max_threads = 0) {
module.scene.shapes.clear();
const unsigned bake_threads = resolveTerrainThreadCount(max_threads);
bool parsed_any = false;
const auto emit_baked = [&](std::vector<runtime::RuntimeShapeNode>& baked) -> bool {
if (baked.empty()) {
return true;
}
parsed_any = true;
if (shape_spool != nullptr) {
shape_spool->append_batch(std::move(baked));
} else {
module.scene.shapes.insert(
module.scene.shapes.end(),
std::make_move_iterator(baked.begin()),
std::make_move_iterator(baked.end()));
baked.clear();
}
return true;
};
const auto bake_and_emit = [&](std::vector<TriangleTerrainBlock>&& blocks) -> bool {
if (blocks.empty()) {
return true;
}
std::vector<runtime::RuntimeShapeNode> baked;
if (!bakeTriangleTerrainBatch(blocks, max_threads, baked)) {
return false;
}
return emit_baked(baked);
};
if (bake_threads <= 1) {
const bool ok = scanTriangleTerrainBlocksStreaming(path, bake_and_emit);
if (!ok || !parsed_any) {
module.scene.shapes.clear();
return false;
}
return true;
}
std::mutex queue_mutex;
std::condition_variable queue_cv;
std::deque<std::vector<TriangleTerrainBlock>> ready_batches;
std::atomic<bool> scan_ok { true };
std::atomic<bool> scan_done { false };
std::atomic<bool> bake_failed { false };
std::thread scanner([&]() {
try {
const bool ok = scanTriangleTerrainBlocksStreaming(
path, [&](std::vector<TriangleTerrainBlock>&& blocks) -> bool {
if (bake_failed.load(std::memory_order_relaxed)) {
return false;
}
std::unique_lock lock(queue_mutex);
queue_cv.wait(lock, [&]() {
return bake_failed.load(std::memory_order_relaxed) ||
ready_batches.size() < kTerrainPipelineMaxQueuedBatches;
});
if (bake_failed.load(std::memory_order_relaxed)) {
return false;
}
ready_batches.push_back(std::move(blocks));
lock.unlock();
queue_cv.notify_one();
return true;
});
scan_ok.store(ok, std::memory_order_relaxed);
} catch (...) {
scan_ok.store(false, std::memory_order_relaxed);
bake_failed.store(true, std::memory_order_relaxed);
queue_cv.notify_all();
}
scan_done.store(true, std::memory_order_release);
queue_cv.notify_all();
});
while (true) {
std::vector<TriangleTerrainBlock> batch;
{
std::unique_lock lock(queue_mutex);
queue_cv.wait(lock, [&]() {
return bake_failed.load(std::memory_order_relaxed) ||
!ready_batches.empty() ||
scan_done.load(std::memory_order_acquire);
});
if (bake_failed.load(std::memory_order_relaxed)) {
break;
}
if (ready_batches.empty() && scan_done.load(std::memory_order_acquire)) {
break;
}
if (ready_batches.empty()) {
continue;
}
batch = std::move(ready_batches.front());
ready_batches.pop_front();
}
queue_cv.notify_one();
std::vector<runtime::RuntimeShapeNode> baked;
if (!bakeTriangleTerrainBatch(batch, max_threads, baked)) {
bake_failed.store(true, std::memory_order_relaxed);
queue_cv.notify_all();
break;
}
if (!emit_baked(baked)) {
bake_failed.store(true, std::memory_order_relaxed);
queue_cv.notify_all();
break;
}
}
scanner.join();
if (bake_failed.load(std::memory_order_relaxed) ||
!scan_ok.load(std::memory_order_relaxed) || !parsed_any) {
module.scene.shapes.clear();
return false;
}
return true;
}
} // namespace eu07::scene::bake::detail

View File

@@ -16,6 +16,8 @@ struct ParseContext {
std::vector<std::filesystem::path> includeStack; std::vector<std::filesystem::path> includeStack;
std::vector<std::vector<std::string>> activeIncludeParameters; std::vector<std::vector<std::string>> activeIncludeParameters;
bool expandIncludes = true; bool expandIncludes = true;
// PACK compose needs includes + models only; skip heavy geometry nodes (CityGML teren).
bool packComposeLightweight = false;
}; };
using DirectiveParser = bool (*)(TokenStream& stream, ParseContext& context); using DirectiveParser = bool (*)(TokenStream& stream, ParseContext& context);

View File

@@ -23,8 +23,12 @@ public:
return (*tokens_)[index_]; return (*tokens_)[index_];
} }
[[nodiscard]] SourceToken consume() { // Returns a reference into the underlying token vector (which outlives the
SourceToken token = peek(); // stream), so callers that only read a field (e.g. .sourceLine) or bind to
// const& pay no per-token std::string copy. Callers that need an owning
// copy still get one via copy-initialisation, exactly as before.
[[nodiscard]] const SourceToken& consume() {
const SourceToken& token = peek();
++index_; ++index_;
return token; return token;
} }

View File

@@ -106,4 +106,64 @@ struct SceneProcessResult {
doc.test.size(); doc.test.size();
} }
// Plaski plik flory: same wpisy node model, bez torow/siatek/include.
[[nodiscard]] inline bool isModelOnlyDocument(const SceneDocument& document) noexcept {
if (document.nodeModel.empty()) {
return false;
}
return document.nodeDynamic.empty() && document.nodeTrackNormal.empty() &&
document.nodeTrackSwitch.empty() && document.nodeTrackRoad.empty() &&
document.nodeTrackCross.empty() && document.nodeTrackOther.empty() &&
document.nodeTraction.empty() && document.nodeTractionPower.empty() &&
document.nodeTriangles.empty() && document.nodeTriangleStrip.empty() &&
document.nodeTriangleFan.empty() && document.nodeLines.empty() &&
document.nodeLineStrip.empty() && document.nodeLineLoop.empty() &&
document.nodeMemcell.empty() && document.nodeEventlauncher.empty() &&
document.nodeSound.empty() && document.include.empty() && document.unknown.empty();
}
// Drops parsed node payloads already converted to RuntimeModule / PackFileCacheEntry.
// Keeps include lists and lightweight directives. When release_models is false,
// nodeModel is retained for a pending PACK build (model-only flora child).
inline void releaseHeavySceneParseStorage(SceneDocument& document, bool const release_models) {
if (release_models) {
document.nodeModel.clear();
document.nodeModel.shrink_to_fit();
}
document.nodeTriangles.clear();
document.nodeTriangles.shrink_to_fit();
document.nodeTriangleStrip.clear();
document.nodeTriangleStrip.shrink_to_fit();
document.nodeTriangleFan.clear();
document.nodeTriangleFan.shrink_to_fit();
document.nodeLines.clear();
document.nodeLines.shrink_to_fit();
document.nodeLineStrip.clear();
document.nodeLineStrip.shrink_to_fit();
document.nodeLineLoop.clear();
document.nodeLineLoop.shrink_to_fit();
document.nodeTrackNormal.clear();
document.nodeTrackNormal.shrink_to_fit();
document.nodeTrackSwitch.clear();
document.nodeTrackSwitch.shrink_to_fit();
document.nodeTrackRoad.clear();
document.nodeTrackRoad.shrink_to_fit();
document.nodeTrackCross.clear();
document.nodeTrackCross.shrink_to_fit();
document.nodeTrackOther.clear();
document.nodeTrackOther.shrink_to_fit();
document.nodeTraction.clear();
document.nodeTraction.shrink_to_fit();
document.nodeTractionPower.clear();
document.nodeTractionPower.shrink_to_fit();
document.nodeMemcell.clear();
document.nodeMemcell.shrink_to_fit();
document.nodeEventlauncher.clear();
document.nodeEventlauncher.shrink_to_fit();
document.nodeSound.clear();
document.nodeSound.shrink_to_fit();
document.nodeDynamic.clear();
document.nodeDynamic.shrink_to_fit();
}
} // namespace eu07::scene } // namespace eu07::scene

View File

@@ -51,6 +51,18 @@ inline void applyScratchContext(NodeHeader& header, const SceneScratchpad& scrat
return false; return false;
} }
if (context.packComposeLightweight) {
std::vector<SourceToken> raw;
while (!stream.empty() && !io::atEnd(stream, kind->subtype, kind->endMarker)) {
stream.consume();
}
if (!io::consumeEnd(stream, raw, kind->subtype, kind->endMarker)) {
stream.rewind(anchor);
return false;
}
return true;
}
if (!kind->parseAndStore(stream, context.document, header, nullptr)) { if (!kind->parseAndStore(stream, context.document, header, nullptr)) {
stream.rewind(anchor); stream.rewind(anchor);
return false; return false;

View File

@@ -30,6 +30,15 @@ template <typename Parsed>
return false; return false;
} }
// Rezerwacja z gornego oszacowania (>=8 tokenow/wierzcholek), z capem.
{
std::size_t estimate = stream.remaining() / 8;
if (estimate > 8192) {
estimate = 8192;
}
out.vertices.reserve(estimate);
}
while (!stream.empty() && !atEnd(stream, subtype, endMarker)) { while (!stream.empty() && !atEnd(stream, subtype, endMarker)) {
MeshVertex vertex; MeshVertex vertex;
const bool hasMore = stream.remaining() > 1; const bool hasMore = stream.remaining() > 1;

View File

@@ -6,9 +6,11 @@
#include <eu07/scene/match.hpp> #include <eu07/scene/match.hpp>
#include <eu07/scene/node/types.hpp> #include <eu07/scene/node/types.hpp>
#include <charconv>
#include <optional> #include <optional>
#include <string> #include <string>
#include <string_view> #include <string_view>
#include <system_error>
#include <vector> #include <vector>
namespace eu07::scene::node::io { namespace eu07::scene::node::io {
@@ -63,16 +65,23 @@ inline void appendRaw(std::vector<SourceToken>& raw, const SourceToken& token) {
return 0.0; return 0.0;
} }
try { // std::from_chars: bez alokacji std::string, bez wyjatkow, niezalezne od locale
std::size_t used = 0; // (separator dziesietny zawsze '.', co odpowiada formatowi EU07).
const double value = std::stod(std::string(text), &used); std::string_view body = text;
if (used != text.size()) { // std::stod akceptowal wiodacy '+'; from_chars nie — zachowujemy zgodnosc.
return std::nullopt; if (!body.empty() && body.front() == '+') {
} body.remove_prefix(1);
return value; }
} catch (...) {
double value = 0.0;
const char* const first = body.data();
const char* const last = body.data() + body.size();
const std::from_chars_result result =
std::from_chars(first, last, value, std::chars_format::general);
if (result.ec != std::errc{} || result.ptr != last) {
return std::nullopt; return std::nullopt;
} }
return value;
} }
[[nodiscard]] inline bool takeToken( [[nodiscard]] inline bool takeToken(
@@ -115,6 +124,24 @@ inline void appendRaw(std::vector<SourceToken>& raw, const SourceToken& token) {
return true; return true;
} }
// Szybki odczyt liczby z goracych petli wierzcholkow: bez kopii SourceToken
// (uzywa peek zamiast consume) i bez dopisywania do bufora raw. Bezpieczne tam,
// gdzie raw wierzcholkow nie jest pozniej czytany (siatki: triangles/strip/fan).
// Przy niepowodzeniu NIE konsumuje tokenu — wolajacy i tak cofa strumien do
// kotwicy wezla, wiec pozycja w miejscu bledu nie ma znaczenia.
[[nodiscard]] inline bool takeDoubleFast(TokenStream& stream, double& out) {
if (stream.empty()) {
return false;
}
const std::optional<double> value = parseDouble(stream.peek().value);
if (!value) {
return false;
}
stream.skip();
out = *value;
return true;
}
[[nodiscard]] inline bool takeDoubleOrPlaceholder( [[nodiscard]] inline bool takeDoubleOrPlaceholder(
TokenStream& stream, TokenStream& stream,
std::vector<SourceToken>& raw, std::vector<SourceToken>& raw,

View File

@@ -73,10 +73,12 @@ inline constexpr std::string_view kEndMarker = "endtriangles";
std::vector<SourceToken>& raw, std::vector<SourceToken>& raw,
MeshVertex& vertex, MeshVertex& vertex,
const bool allowEndSuffix) { const bool allowEndSuffix) {
if (!node::io::takeDouble(stream, raw, vertex.x) || !node::io::takeDouble(stream, raw, vertex.y) || // Goraca petla: 8 liczb na wierzcholek czytanych bez kopii tokenu i bez raw
!node::io::takeDouble(stream, raw, vertex.z) || !node::io::takeDouble(stream, raw, vertex.nx) || // (raw wierzcholkow siatek nie jest pozniej czytany przy bake).
!node::io::takeDouble(stream, raw, vertex.ny) || !node::io::takeDouble(stream, raw, vertex.nz) || if (!node::io::takeDoubleFast(stream, vertex.x) || !node::io::takeDoubleFast(stream, vertex.y) ||
!node::io::takeDouble(stream, raw, vertex.u) || !node::io::takeDouble(stream, raw, vertex.v)) { !node::io::takeDoubleFast(stream, vertex.z) || !node::io::takeDoubleFast(stream, vertex.nx) ||
!node::io::takeDoubleFast(stream, vertex.ny) || !node::io::takeDoubleFast(stream, vertex.nz) ||
!node::io::takeDoubleFast(stream, vertex.u) || !node::io::takeDoubleFast(stream, vertex.v)) {
return false; return false;
} }
if (allowEndSuffix && !stream.empty() && isKeyword(stream.peek().value, "end")) { if (allowEndSuffix && !stream.empty() && isKeyword(stream.peek().value, "end")) {
@@ -105,6 +107,16 @@ inline constexpr std::string_view kEndMarker = "endtriangles";
return false; return false;
} }
// Rezerwacja z gornego oszacowania (>=8 tokenow/wierzcholek), z capem aby
// nie przealokowac przy wczesnym wezle nad strumieniem calego pliku.
{
std::size_t estimate = stream.remaining() / 8;
if (estimate > 8192) {
estimate = 8192;
}
out.vertices.reserve(estimate);
}
while (!stream.empty() && !node::io::atEnd(stream, kSubtype, kEndMarker)) { while (!stream.empty() && !node::io::atEnd(stream, kSubtype, kEndMarker)) {
MeshVertex vertex; MeshVertex vertex;
const bool hasMore = stream.remaining() > 1; const bool hasMore = stream.remaining() > 1;

View File

@@ -0,0 +1,766 @@
#pragma once
// Fast path: plaskie pliki .scm — kazdy wpis w jednej linii logicznej.
// (1) same "node model" (flora) albo (2) same "include;...;end" (placementy).
// Pomija tokenizacje calego pliku; parse linii rownolegle (per-line tokenize).
#include <eu07/nmt/parallel.hpp>
#include <eu07/parser.hpp>
#include <eu07/scene/context.hpp>
#include <eu07/scene/cursor.hpp>
#include <eu07/scene/dispatch_table.hpp>
#include <eu07/scene/document.hpp>
#include <eu07/scene/match.hpp>
#include <eu07/scene/node.hpp>
#include <eu07/scene/node/model.hpp>
#include <atomic>
#include <array>
#include <cstddef>
#include <filesystem>
#include <fstream>
#include <functional>
#include <optional>
#include <stdexcept>
#include <string>
#include <string_view>
#include <thread>
#include <utility>
#include <vector>
namespace eu07::scene::detail {
enum class FlatFileKind {
None,
Models,
Includes,
};
inline constexpr std::size_t kParallelFlatLineThreshold = 2048;
[[nodiscard]] inline bool segmentStartsWithKeyword(
std::string_view text,
const std::string_view keyword) {
skipFieldSeparators(text);
if (text.size() < keyword.size()) {
return false;
}
if (text.compare(0, keyword.size(), keyword) != 0) {
return false;
}
if (text.size() == keyword.size()) {
return true;
}
return isFieldSeparator(static_cast<unsigned char>(text[keyword.size()]));
}
[[nodiscard]] inline FlatFileKind classifyFlatFile(const std::vector<LogicalSegment>& segments) {
FlatFileKind kind = FlatFileKind::None;
for (const LogicalSegment& segment : segments) {
std::string_view text = segment.text;
skipFieldSeparators(text);
if (text.empty()) {
continue;
}
if (segmentStartsWithKeyword(text, "node")) {
if (text.find("endmodel") == std::string_view::npos) {
return FlatFileKind::None;
}
if (kind == FlatFileKind::Includes) {
return FlatFileKind::None;
}
kind = FlatFileKind::Models;
continue;
}
if (segmentStartsWithKeyword(text, "include")) {
if (text.find("end") == std::string_view::npos) {
return FlatFileKind::None;
}
if (kind == FlatFileKind::Models) {
return FlatFileKind::None;
}
kind = FlatFileKind::Includes;
continue;
}
return FlatFileKind::None;
}
return kind;
}
struct ModelParseJob {
std::size_t segment_index = 0;
NodeHeader header;
};
[[nodiscard]] inline bool skipModelBody(TokenStream& stream) {
std::vector<SourceToken> raw;
while (!stream.empty() && !node::io::atEnd(stream, node_model::kSubtype, node_model::kEndMarker)) {
stream.consume();
}
return node::io::consumeEnd(stream, raw, node_model::kSubtype, node_model::kEndMarker);
}
[[nodiscard]] inline bool parseModelSegment(
const LogicalSegment& segment,
ParsedNodeModel& out) {
const std::vector<SourceToken> tokens = tokenizeSegments({segment});
TokenStream stream(tokens);
NodeHeader header;
std::string subtype;
std::vector<SourceToken> raw;
if (!node::io::consumeHeader(stream, header, subtype, raw) ||
!isKeyword(subtype, node_model::kSubtype)) {
return false;
}
return node_model::parseBody(stream, raw, header, out);
}
[[nodiscard]] inline std::string_view trimFieldView(std::string_view text) {
skipFieldSeparators(text);
while (!text.empty() && isFieldSeparator(static_cast<unsigned char>(text.back()))) {
text.remove_suffix(1);
}
return text;
}
[[nodiscard]] inline std::vector<std::string_view> splitSemicolonFields(std::string_view text) {
std::vector<std::string_view> fields;
while (!text.empty()) {
skipFieldSeparators(text);
if (text.empty()) {
break;
}
const std::size_t separator = text.find(';');
if (separator == std::string_view::npos) {
fields.push_back(trimFieldView(text));
break;
}
fields.push_back(trimFieldView(text.substr(0, separator)));
text.remove_prefix(separator + 1);
}
return fields;
}
[[nodiscard]] inline bool parseIncludeSegmentSemicolon(
const LogicalSegment& segment,
ParsedInclude& out) {
std::string_view text = segment.text;
if (const std::size_t comment = text.find("//"); comment != std::string_view::npos) {
text = text.substr(0, comment);
}
skipFieldSeparators(text);
if (!segmentStartsWithKeyword(text, "include")) {
return false;
}
const std::vector<std::string_view> fields = splitSemicolonFields(text);
if (fields.empty() || !isKeyword(fields.front(), "include")) {
return false;
}
out = ParsedInclude {};
out.line = segment.sourceLine;
if (fields.size() < 2) {
out.error = "brak sciezki pliku";
return true;
}
out.file = std::string(fields[1]);
for (std::size_t index = 2; index < fields.size(); ++index) {
if (isKeyword(fields[index], "end")) {
break;
}
out.parameters.emplace_back(fields[index]);
}
return true;
}
[[nodiscard]] inline bool parseIncludeSegment(
const LogicalSegment& segment,
ParsedInclude& out) {
if (parseIncludeSegmentSemicolon(segment, out)) {
return true;
}
const std::vector<SourceToken> tokens = tokenizeSegments({segment});
TokenStream stream(tokens);
if (stream.empty() || !isKeyword(stream.peek().value, "include")) {
return false;
}
out = ParsedInclude {};
out.line = segment.sourceLine;
stream.consume();
if (stream.empty()) {
out.error = "brak sciezki pliku";
return true;
}
SourceToken file_token;
if (!node::io::takeToken(stream, out.raw, file_token)) {
out.error = "brak sciezki pliku";
return true;
}
out.file = file_token.value;
while (!stream.empty()) {
if (isKeyword(stream.peek().value, "end")) {
stream.consume();
break;
}
SourceToken param;
if (!node::io::takeToken(stream, out.raw, param)) {
break;
}
out.parameters.push_back(param.value);
}
out.raw.clear();
return true;
}
[[nodiscard]] inline std::optional<SceneDocument> parseFlatModels(
const std::vector<LogicalSegment>& segments) {
SceneDocument document;
if (segments.empty()) {
return document;
}
if (segments.size() < kParallelFlatLineThreshold) {
document.nodeModel.reserve(segments.size());
for (const LogicalSegment& segment : segments) {
ParsedNodeModel model;
if (!parseModelSegment(segment, model)) {
return std::nullopt;
}
document.nodeModel.push_back(std::move(model));
}
return document;
}
document.nodeModel.resize(segments.size());
const unsigned worker_count =
std::min(static_cast<unsigned>(segments.size()), eu07::nmt::workerThreadCount());
std::atomic<std::size_t> next_job { 0 };
std::atomic<bool> failed { false };
auto worker = [&]() {
while (true) {
if (failed.load(std::memory_order_relaxed)) {
return;
}
const std::size_t index = next_job.fetch_add(1, std::memory_order_relaxed);
if (index >= segments.size()) {
return;
}
ParsedNodeModel model;
if (!parseModelSegment(segments[index], model)) {
failed.store(true, std::memory_order_relaxed);
return;
}
document.nodeModel[index] = std::move(model);
}
};
std::vector<std::thread> threads;
const unsigned launch = std::max(1u, worker_count);
threads.reserve(launch);
for (unsigned i = 0; i < launch; ++i) {
threads.emplace_back(worker);
}
for (std::thread& thread : threads) {
thread.join();
}
if (failed.load(std::memory_order_relaxed)) {
return std::nullopt;
}
return document;
}
[[nodiscard]] inline std::optional<SceneDocument> parseFlatIncludes(
const std::vector<LogicalSegment>& segments) {
SceneDocument document;
document.include.resize(segments.size());
if (segments.size() < kParallelFlatLineThreshold) {
for (std::size_t index = 0; index < segments.size(); ++index) {
if (!parseIncludeSegment(segments[index], document.include[index])) {
return std::nullopt;
}
}
return document;
}
std::atomic<std::size_t> next_job { 0 };
std::atomic<bool> failed { false };
auto worker = [&]() {
while (true) {
if (failed.load(std::memory_order_relaxed)) {
return;
}
const std::size_t index = next_job.fetch_add(1, std::memory_order_relaxed);
if (index >= segments.size()) {
return;
}
ParsedInclude entry;
if (!parseIncludeSegment(segments[index], entry)) {
failed.store(true, std::memory_order_relaxed);
return;
}
document.include[index] = std::move(entry);
}
};
const unsigned worker_count =
std::min(static_cast<unsigned>(segments.size()), eu07::nmt::workerThreadCount());
std::vector<std::thread> threads;
const unsigned launch = std::max(1u, worker_count);
threads.reserve(launch);
for (unsigned i = 0; i < launch; ++i) {
threads.emplace_back(worker);
}
for (std::thread& thread : threads) {
thread.join();
}
if (failed.load(std::memory_order_relaxed)) {
return std::nullopt;
}
return document;
}
[[nodiscard]] inline std::optional<SceneDocument> tryProcessFlatLogical(
const LogicalPass& logical,
const std::filesystem::path& /*baseDirectory*/) {
const FlatFileKind kind = classifyFlatFile(logical.segments);
if (kind == FlatFileKind::None) {
return std::nullopt;
}
if (kind == FlatFileKind::Models) {
return parseFlatModels(logical.segments);
}
return parseFlatIncludes(logical.segments);
}
inline constexpr std::size_t kParallelModelParseThreshold = 4096;
[[nodiscard]] inline std::optional<SceneDocument> tryProcessFlatModels(
const ParseResult& parsed,
const std::filesystem::path& baseDirectory) {
if (parsed.tokens.empty()) {
return SceneDocument {};
}
SceneDocument document;
std::filesystem::path includeRoot = baseDirectory;
if (includeRoot.empty()) {
includeRoot = std::filesystem::current_path();
}
ParseContext context { document, {}, includeRoot, {} };
context.expandIncludes = false;
TokenStream stream(parsed.tokens);
std::vector<ModelParseJob> jobs;
jobs.reserve(parsed.tokens.size() / 16);
while (!stream.empty()) {
if (detail::dispatchDirective(stream, context)) {
continue;
}
if (!stream.empty() && isKeyword(stream.peek().value, "node")) {
const std::size_t anchor = stream.checkpoint();
NodeHeader header;
std::string subtype;
std::vector<SourceToken> raw;
if (!node::io::consumeHeader(stream, header, subtype, raw)) {
return std::nullopt;
}
if (!isKeyword(subtype, node_model::kSubtype)) {
return std::nullopt;
}
node::applyScratchContext(header, context.scratch);
jobs.push_back(ModelParseJob { anchor, header });
if (!skipModelBody(stream)) {
return std::nullopt;
}
continue;
}
UnknownEntry unknown;
unknown.line = stream.peek().sourceLine;
unknown.token = stream.consume().value;
document.unknown.push_back(std::move(unknown));
}
if (jobs.size() < kParallelModelParseThreshold) {
document.nodeModel.reserve(jobs.size());
for (const ModelParseJob& job : jobs) {
ParsedNodeModel model;
TokenStream job_stream(parsed.tokens);
job_stream.rewind(job.segment_index);
NodeHeader parsed_header;
std::string subtype;
std::vector<SourceToken> raw;
if (!node::io::consumeHeader(job_stream, parsed_header, subtype, raw) ||
!isKeyword(subtype, node_model::kSubtype)) {
return std::nullopt;
}
model.header = job.header;
if (!node_model::parseBody(job_stream, raw, job.header, model)) {
return std::nullopt;
}
document.nodeModel.push_back(std::move(model));
}
return document;
}
document.nodeModel.resize(jobs.size());
const unsigned worker_count =
std::min(static_cast<unsigned>(jobs.size()), eu07::nmt::workerThreadCount());
std::atomic<std::size_t> next_job { 0 };
std::atomic<bool> failed { false };
auto worker = [&]() {
while (true) {
if (failed.load(std::memory_order_relaxed)) {
return;
}
const std::size_t index = next_job.fetch_add(1, std::memory_order_relaxed);
if (index >= jobs.size()) {
return;
}
ParsedNodeModel model;
TokenStream job_stream(parsed.tokens);
job_stream.rewind(jobs[index].segment_index);
NodeHeader parsed_header;
std::string subtype;
std::vector<SourceToken> raw;
if (!node::io::consumeHeader(job_stream, parsed_header, subtype, raw) ||
!isKeyword(subtype, node_model::kSubtype)) {
failed.store(true, std::memory_order_relaxed);
return;
}
model.header = jobs[index].header;
if (!node_model::parseBody(job_stream, raw, jobs[index].header, model)) {
failed.store(true, std::memory_order_relaxed);
return;
}
document.nodeModel[index] = std::move(model);
}
};
std::vector<std::thread> threads;
const unsigned launch = std::max(1u, worker_count);
threads.reserve(launch);
for (unsigned i = 0; i < launch; ++i) {
threads.emplace_back(worker);
}
for (std::thread& thread : threads) {
thread.join();
}
if (failed.load(std::memory_order_relaxed)) {
return std::nullopt;
}
return document;
}
[[nodiscard]] inline std::optional<SceneDocument> tryProcessFlatSceneFile(
eu07::RawFile raw,
const std::filesystem::path& baseDirectory) {
const LogicalPass logical = toLogicalLines(raw.lines);
return tryProcessFlatLogical(logical, baseDirectory);
}
inline constexpr std::size_t kStreamFlatFileThresholdBytes = 4u * 1024u * 1024u;
[[nodiscard]] inline std::size_t streamSourceFileSizeBytes(const std::filesystem::path& path) {
std::error_code ec;
const auto size = std::filesystem::file_size(path, ec);
return ec ? 0u : static_cast<std::size_t>(size);
}
[[nodiscard]] inline bool shouldStreamFlatSourceFile(
const std::filesystem::path& path,
const std::size_t threshold_bytes = kStreamFlatFileThresholdBytes) {
if (threshold_bytes == 0) {
return false;
}
return streamSourceFileSizeBytes(path) > threshold_bytes;
}
// Linia po linii, z ta sama logika komentarzy co toLogicalLines — bez readRawFile.
template <typename Fn>
void forEachStreamingLogicalSegment(const std::filesystem::path& path, Fn&& on_segment) {
std::ifstream in(path, std::ios::binary);
if (!in) {
throw std::runtime_error("Nie mozna otworzyc: " + path.string());
}
std::vector<char> read_buffer(1024u * 1024u);
in.rdbuf()->pubsetbuf(read_buffer.data(), read_buffer.size());
bool in_block = false;
std::string line;
line.reserve(512);
std::size_t source_line = 0;
std::string spill;
const auto emit = [&](std::string_view piece, const std::size_t line_no) -> bool {
if (piece.empty()) {
return true;
}
skipFieldSeparators(piece);
if (eu07::detail::isStarter(piece)) {
return true;
}
if (eu07::detail::isLineComment(piece)) {
return true;
}
const std::string_view code = eu07::detail::stripInlineComment(piece);
if (code.empty()) {
return true;
}
LogicalSegment segment;
segment.sourceLine = line_no;
if (code.data() >= piece.data() && code.data() + code.size() <= piece.data() + piece.size()) {
segment.text = code;
} else {
spill.assign(code.begin(), code.end());
segment.text = spill;
}
return on_segment(segment);
};
while (std::getline(in, line)) {
++source_line;
if (!line.empty() && line.back() == '\r') {
line.pop_back();
}
{
std::string_view whole = line;
skipFieldSeparators(whole);
if (parseStarter(whole, source_line - 1)) {
continue;
}
}
if (!in_block) {
std::size_t at = 0;
std::string_view line_view = line;
while (at < line_view.size()) {
const std::size_t slash = line_view.find('/', at);
if (slash == std::string_view::npos) {
if (!emit(line_view.substr(at), source_line)) {
return;
}
break;
}
if (slash + 1 < line_view.size() && line_view[slash + 1] == '/') {
if (slash > at) {
if (!emit(line_view.substr(at, slash - at), source_line)) {
return;
}
}
break;
}
if (slash + 1 < line_view.size() && line_view[slash + 1] == '*') {
if (slash > at) {
if (!emit(line_view.substr(at, slash - at), source_line)) {
return;
}
}
in_block = true;
line_view = line_view.substr(slash + 2);
at = 0;
continue;
}
if (!emit(line_view.substr(at), source_line)) {
return;
}
break;
}
continue;
}
while (in_block && !line.empty()) {
const std::size_t close = line.find("*/");
if (close == std::string_view::npos) {
line.clear();
break;
}
line.erase(0, close + 2);
in_block = false;
}
if (!in_block && !line.empty()) {
if (!emit(line, source_line)) {
return;
}
}
}
}
[[nodiscard]] inline FlatFileKind scanFlatFileKindStreaming(const std::filesystem::path& path) {
FlatFileKind kind = FlatFileKind::None;
bool ok = true;
forEachStreamingLogicalSegment(path, [&](const LogicalSegment& segment) {
std::string_view text = segment.text;
skipFieldSeparators(text);
if (text.empty()) {
return true;
}
if (segmentStartsWithKeyword(text, "node")) {
if (text.find("endmodel") == std::string_view::npos) {
ok = false;
return false;
}
if (kind == FlatFileKind::Includes) {
ok = false;
return false;
}
kind = FlatFileKind::Models;
return true;
}
if (segmentStartsWithKeyword(text, "include")) {
if (text.find("end") == std::string_view::npos) {
ok = false;
return false;
}
if (kind == FlatFileKind::Models) {
ok = false;
return false;
}
kind = FlatFileKind::Includes;
return true;
}
ok = false;
return false;
});
return ok ? kind : FlatFileKind::None;
}
[[nodiscard]] inline std::optional<SceneDocument> buildFlatIncludesDocumentStreaming(
const std::filesystem::path& path) {
SceneDocument document;
struct OwnedIncludeLine {
std::size_t source_line = 0;
std::string text;
};
std::vector<OwnedIncludeLine> batch;
batch.reserve(8192);
bool ok = true;
const auto flush_batch = [&]() -> bool {
if (batch.empty()) {
return true;
}
const std::size_t base = document.include.size();
document.include.resize(base + batch.size());
const auto parse_one = [&](const std::size_t index) -> bool {
LogicalSegment segment;
segment.sourceLine = batch[index].source_line;
segment.text = batch[index].text;
ParsedInclude entry;
if (!parseIncludeSegment(segment, entry)) {
return false;
}
document.include[base + index] = std::move(entry);
return true;
};
if (batch.size() >= kParallelFlatLineThreshold) {
std::atomic<std::size_t> next_job { 0 };
std::atomic<bool> failed { false };
const unsigned worker_count = std::min(
static_cast<unsigned>(batch.size()), eu07::nmt::workerThreadCount());
const auto worker = [&]() {
while (!failed.load(std::memory_order_relaxed)) {
const std::size_t index =
next_job.fetch_add(1, std::memory_order_relaxed);
if (index >= batch.size()) {
return;
}
if (!parse_one(index)) {
failed.store(true, std::memory_order_relaxed);
return;
}
}
};
std::vector<std::thread> threads;
threads.reserve(worker_count);
for (unsigned i = 0; i < worker_count; ++i) {
threads.emplace_back(worker);
}
for (std::thread& thread : threads) {
thread.join();
}
if (failed.load(std::memory_order_relaxed)) {
return false;
}
} else {
for (std::size_t index = 0; index < batch.size(); ++index) {
if (!parse_one(index)) {
return false;
}
}
}
batch.clear();
return true;
};
forEachStreamingLogicalSegment(path, [&](const LogicalSegment& segment) {
std::string_view text = segment.text;
skipFieldSeparators(text);
if (text.empty() || !segmentStartsWithKeyword(text, "include")) {
return true;
}
batch.push_back(OwnedIncludeLine { segment.sourceLine, std::string(segment.text) });
if (batch.size() >= 8192) {
if (!flush_batch()) {
ok = false;
return false;
}
}
return true;
});
if (ok) {
ok = flush_batch();
}
return ok ? std::optional<SceneDocument>(std::move(document)) : std::nullopt;
}
template <typename Fn>
void streamFlatFileModels(const std::filesystem::path& path, Fn&& on_model) {
forEachStreamingLogicalSegment(path, [&](const LogicalSegment& segment) {
std::string_view text = segment.text;
skipFieldSeparators(text);
if (text.empty() || !segmentStartsWithKeyword(text, "node")) {
return true;
}
if (text.find("endmodel") == std::string_view::npos) {
throw std::runtime_error(
"Plaski plik modeli: oczekiwano endmodel w linii " +
std::to_string(segment.sourceLine));
}
ParsedNodeModel parsed;
if (!parseModelSegment(segment, parsed)) {
throw std::runtime_error(
"Plaski plik modeli: blad parse linii " + std::to_string(segment.sourceLine));
}
on_model(parsed);
return true;
});
}
} // namespace eu07::scene::detail

View File

@@ -18,7 +18,7 @@
#include <eu07/scene/include_resolve.hpp> #include <eu07/scene/include_resolve.hpp>
#include <eu07/scene/parallel_models.hpp>
#include <filesystem> #include <filesystem>
@@ -194,6 +194,7 @@ inline void expandInclude(ParseContext& context, const IncludeExpansionRequest&
struct SceneProcessOptions { struct SceneProcessOptions {
bool expandIncludes = true; bool expandIncludes = true;
bool packComposeLightweight = false;
}; };
[[nodiscard]] inline SceneProcessResult processScene( [[nodiscard]] inline SceneProcessResult processScene(
@@ -204,6 +205,15 @@ struct SceneProcessOptions {
const SceneProcessOptions& options = {}) { const SceneProcessOptions& options = {}) {
if (!options.expandIncludes) {
if (std::optional<SceneDocument> fast =
detail::tryProcessFlatModels(parsed, baseDirectory)) {
SceneProcessResult result;
result.document = std::move(*fast);
return result;
}
}
TokenStream stream(parsed.tokens); TokenStream stream(parsed.tokens);
SceneDocument document; SceneDocument document;
@@ -223,6 +233,7 @@ struct SceneProcessOptions {
ParseContext context{document, {}, includeRoot, {}}; ParseContext context{document, {}, includeRoot, {}};
context.expandIncludes = options.expandIncludes; context.expandIncludes = options.expandIncludes;
context.packComposeLightweight = options.packComposeLightweight;
detail::processStream(stream, context); detail::processStream(stream, context);

View File

@@ -32,6 +32,8 @@ http://mozilla.org/MPL/2.0/.
#include <algorithm>
#include <thread> #include <thread>
@@ -62,7 +64,31 @@ bake_thread_count() {
auto const hardware { std::thread::hardware_concurrency() }; auto const hardware { std::thread::hardware_concurrency() };
return hardware > 0 ? hardware : 1u; if( hardware == 0 ) {
return 1u;
}
int percent { Global.eu7_bake_cpu_percent };
if( percent < 1 ) { percent = 1; }
if( percent > 100 ) { percent = 100; }
unsigned const threads {
std::max( 1u,
static_cast<unsigned>( ( static_cast<unsigned long long>( hardware ) * percent + 50 ) / 100 ) ) };
WriteLog(
"EU7 bake: auto watki=" + std::to_string( threads ) +
" (" + std::to_string( percent ) + "% z " + std::to_string( hardware ) + " rdzeni logicznych)" );
return threads;
} }
@@ -90,7 +116,27 @@ run_scenario_tree_bake(
std::string error; std::string error;
if( false == bake_parser::bake_scenario_tree( text_root, bake_thread_count(), error ) ) { bool bake_ok { false };
if( Global.eu7v2_runtime ) {
auto const report {
bake_parser::bake_scenario_tree_eu7v2( text_root, bake_thread_count(), false ) };
bake_ok = report.baked;
error = report.error;
}
else {
bake_ok = bake_parser::bake_scenario_tree( text_root, bake_thread_count(), error );
}
if( false == bake_ok ) {
if( probe_file( eu7_path ) ) { if( probe_file( eu7_path ) ) {

View File

@@ -0,0 +1,122 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#pragma once
#include <atomic>
#include <chrono>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <thread>
#if defined( _WIN32 )
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#include <psapi.h>
#else
#include <cstdio>
#include <unistd.h>
#endif
namespace scene::eu7::bake_parser {
inline constexpr unsigned kDefaultBakeMemLimitGb { 50u };
[[nodiscard]] inline std::uint64_t
process_private_bytes() noexcept {
#if defined( _WIN32 )
PROCESS_MEMORY_COUNTERS_EX counters {};
counters.cb = sizeof( counters );
if( GetProcessMemoryInfo(
GetCurrentProcess(),
reinterpret_cast<PROCESS_MEMORY_COUNTERS *>( &counters ),
sizeof( counters ) ) ) {
return counters.PrivateUsage;
}
return 0;
#else
std::FILE *status { std::fopen( "/proc/self/statm", "r" ) };
if( status == nullptr ) {
return 0;
}
long total_pages { 0 };
long resident_pages { 0 };
if( std::fscanf( status, "%ld %ld", &total_pages, &resident_pages ) != 2 ) {
std::fclose( status );
return 0;
}
std::fclose( status );
long const page_size { sysconf( _SC_PAGESIZE ) };
if( page_size <= 0 ) {
return 0;
}
return static_cast<std::uint64_t>( resident_pages ) * static_cast<std::uint64_t>( page_size );
#endif
}
// Background poll; abort() when private commit exceeds limit_bytes (0 = disabled).
class bake_mem_guard {
public:
explicit bake_mem_guard( std::uint64_t const limit_bytes ) : m_limit_bytes( limit_bytes ) {}
bake_mem_guard( bake_mem_guard const & ) = delete;
bake_mem_guard &operator=( bake_mem_guard const & ) = delete;
~bake_mem_guard() { stop(); }
void start() {
if( m_limit_bytes == 0 || m_thread.joinable() ) {
return;
}
m_stop.store( false, std::memory_order_release );
m_thread = std::thread( [this]() { run(); } );
}
void stop() {
m_stop.store( true, std::memory_order_release );
if( m_thread.joinable() ) {
m_thread.join();
}
}
private:
void run() const {
while( !m_stop.load( std::memory_order_acquire ) ) {
std::uint64_t const used { process_private_bytes() };
if( used > m_limit_bytes ) {
double const used_gb { static_cast<double>( used ) / ( 1024.0 * 1024.0 * 1024.0 ) };
double const limit_gb {
static_cast<double>( m_limit_bytes ) / ( 1024.0 * 1024.0 * 1024.0 ) };
std::fprintf(
stdout,
"[EU7v2] FATAL: limit pamieci %.1f GB przekroczony (uzyte ~%.1f GB) — abort\n",
limit_gb,
used_gb );
std::fflush( stdout );
std::fprintf(
stderr,
"[EU7v2] FATAL: limit pamieci %.1f GB przekroczony (uzyte ~%.1f GB) — abort\n",
limit_gb,
used_gb );
std::fflush( stderr );
std::abort();
}
std::this_thread::sleep_for( std::chrono::milliseconds { 500 } );
}
}
std::uint64_t const m_limit_bytes { 0 };
std::atomic<bool> m_stop { false };
std::thread m_thread;
};
} // namespace scene::eu7::bake_parser

View File

@@ -17,15 +17,43 @@ http://mozilla.org/MPL/2.0/.
#include <eu07/scene/bake/bake_tree.hpp> #include <eu07/scene/bake/bake_tree.hpp>
#include <eu07/scene/bake/pack_model_spool.hpp>
#include <eu07/scene/include_resolve.hpp>
#include "scene/eu7/eu7_bake_parser.h"
#include "scene/eu7/eu7_bake_mem_guard.h"
#include "scene/eu7/v2/eu7v2_emit_runtime.h"
#include "utilities/Logs.h"
#include <atomic>
#include <chrono>
#include <deque>
#include <filesystem> #include <filesystem>
#include <iomanip>
#include <iostream>
#include <memory>
#include <mutex> #include <mutex>
#include <fstream>
#include <sstream>
#include <string> #include <string>
#include <thread>
#include <unordered_map>
namespace scene::eu7::bake_parser { namespace scene::eu7::bake_parser {
@@ -36,43 +64,218 @@ namespace {
namespace fs = std::filesystem; namespace fs = std::filesystem;
using clock_type = std::chrono::steady_clock;
std::mutex g_progress_log_mutex;
// Bake logs go to stdout (headless console) and WriteLog (-> log.txt when the
// game's LogService is running). One mutex keeps interleaved lines readable
// across the bake worker threads.
std::mutex g_log_mutex;
void
bake_log( std::string const &line ) {
std::lock_guard<std::mutex> lock { g_log_mutex };
std::cout << line << '\n' << std::flush;
WriteLog( line );
// Also tee to a dedicated file so headless runs (where stdout lands in a
// detached console and the game LogService is not flushing) still leave a
// readable record of timings/summaries.
static std::ofstream bake_file { "eu7v2_bake.log", std::ios::app };
if( bake_file ) {
bake_file << line << '\n';
bake_file.flush();
}
}
[[nodiscard]] std::string
fmt_ms( double const ms ) {
std::ostringstream out;
out << std::fixed << std::setprecision( 1 ) << ms;
return out.str();
}
[[nodiscard]] std::string
fmt_s( double const s ) {
std::ostringstream out;
out << std::fixed << std::setprecision( 2 ) << s;
return out.str();
}
// Shared progress/timing state for one bake run. The bake_tree onProgress
// callback and the onModuleBaked hook both fire from worker threads, so all
// mutable access is guarded.
struct bake_progress_state {
clock_type::time_point start { clock_type::now() };
std::mutex mutex;
std::unordered_map<std::string, clock_type::time_point> module_start;
clock_type::time_point compose_last_log {};
bool compose_started { false };
std::atomic<unsigned> threads { 0 };
std::atomic<std::size_t> started { 0 };
std::atomic<std::size_t> module_index { 0 };
std::atomic<std::uint64_t> emit_us { 0 };
// Live PACK-compose diagnostics, read by the watchdog thread so progress is
// visible (and a stall is pinpointed) even if the worker callbacks stop.
std::atomic<eu07::scene::bake::PackComposeStats const *> compose_stats { nullptr };
std::atomic<bool> compose_active { false };
std::string compose_file;
};
void void
log_bake_progress( eu07::scene::bake::BakeProgress const &progress ) { on_bake_progress( bake_progress_state &state, eu07::scene::bake::BakeProgress const &progress ) {
std::lock_guard<std::mutex> lock { g_progress_log_mutex }; using Phase = eu07::scene::bake::BakeProgressPhase;
auto const path { progress.path.u8string() }; if( progress.threads > 0 ) {
state.threads.store( progress.threads, std::memory_order_relaxed );
}
auto const name { progress.path.filename().generic_string() };
switch( progress.phase ) { switch( progress.phase ) {
case eu07::scene::bake::BakeProgressPhase::Starting: case Phase::Start: {
case eu07::scene::bake::BakeProgressPhase::Start: std::size_t const i { state.started.fetch_add( 1, std::memory_order_relaxed ) + 1 };
break; {
case eu07::scene::bake::BakeProgressPhase::PackModels: std::lock_guard<std::mutex> lock { state.mutex };
case eu07::scene::bake::BakeProgressPhase::PackCompose: state.module_start[ progress.path.generic_string() ] = clock_type::now();
case eu07::scene::bake::BakeProgressPhase::PackComposeDone: }
case eu07::scene::bake::BakeProgressPhase::Bake: bake_log( "[EU7v2] (" + std::to_string( i ) + ") BAKING: " + progress.path.generic_string() );
case eu07::scene::bake::BakeProgressPhase::Done:
break; break;
} }
(void)path; case Phase::PackModels:
state.compose_active.store( true, std::memory_order_relaxed );
bake_log(
"[EU7v2] ROOT " + name + ": PACK compose start (watki=" +
std::to_string( progress.threads ) + ")" );
break;
case Phase::PackCompose: {
if( progress.pack_stats != nullptr ) {
state.compose_stats.store( progress.pack_stats, std::memory_order_relaxed );
}
// Throttle to ~2 s so a long compose visibly advances (and the last
// file pinpoints a stall) without flooding the log.
auto const now { clock_type::now() };
std::lock_guard<std::mutex> lock { state.mutex };
state.compose_file = name;
bool const due {
( false == state.compose_started ) ||
std::chrono::duration_cast<std::chrono::milliseconds>(
now - state.compose_last_log )
.count() >= 2000 };
if( due ) {
state.compose_started = true;
state.compose_last_log = now;
bake_log(
"[EU7v2] PACK compose: pliki=" + std::to_string( progress.current ) +
" modele=" + std::to_string( progress.total ) + " (plik: " + name + ")" );
}
break;
}
case Phase::PackComposeDone:
state.compose_active.store( false, std::memory_order_relaxed );
state.compose_stats.store( nullptr, std::memory_order_relaxed );
bake_log(
"[EU7v2] PACK compose gotowe: pliki=" + std::to_string( progress.current ) +
" modele=" + std::to_string( progress.total ) );
break;
case Phase::Starting:
case Phase::Bake:
case Phase::Done:
break;
}
} }
@@ -134,8 +337,6 @@ bake_scenario_tree(
options.maxThreads = max_threads; options.maxThreads = max_threads;
options.onProgress = log_bake_progress;
eu07::scene::bake::BakeTreeStats stats; eu07::scene::bake::BakeTreeStats stats;
@@ -166,5 +367,466 @@ bake_scenario_tree(
Eu7v2BakeReport
bake_scenario_tree_eu7v2(
std::string const &text_scenario_path,
unsigned const max_threads,
bool const verify,
unsigned const mem_limit_gb,
unsigned const max_concurrent_parses,
unsigned const heavy_parse_threshold_mb ) {
Eu7v2BakeReport report;
report.verify_requested = verify;
std::uint64_t const mem_limit_bytes {
mem_limit_gb == 0 ? 0u
: static_cast<std::uint64_t>( mem_limit_gb ) * 1024u * 1024u * 1024u };
bake_mem_guard mem_guard { mem_limit_bytes };
if( mem_limit_gb != 0 ) {
bake_log(
"[EU7v2] limit pamieci: " + std::to_string( mem_limit_gb ) +
" GB (wylacz: --eu7v2-mem-limit-gb 0)" );
}
mem_guard.start();
fs::path const input { text_scenario_path };
if( false == fs::exists( input ) ) {
report.error = "brak pliku: " + text_scenario_path;
return report;
}
if( false == is_text_scenery_path( input ) ) {
report.error = "nie jest plikiem SCM/SCN/INC: " + text_scenario_path;
return report;
}
std::mutex emit_mutex;
std::atomic<std::size_t> module_count { 0 };
std::atomic<std::size_t> model_count { 0 };
std::atomic<bool> verify_ok { true };
std::atomic<bool> emit_failed { false };
std::string root_binary_path;
struct pending_verify_job {
fs::path path;
bool is_root { false };
eu7v2::module_verify_spec spec {};
std::size_t pack_models { 0 };
};
std::vector<pending_verify_job> verify_jobs;
std::mutex verify_jobs_mutex;
bake_progress_state pstate;
bake_log(
"[EU7v2] BAKE START: " + input.generic_string() +
( verify ? " (+verify)" : "" ) );
try {
eu07::scene::bake::BakeTreeOptions options;
bool const spool_flush { mem_limit_gb != 0 };
if( spool_flush ) {
// PACK/shape spool + per-module flush bound RAM; pool can bake modules in parallel.
options.lowMemoryMode = true;
options.maxThreads = max_threads; // 0 => hardware default in runBakePool
if( max_concurrent_parses != 0 ) {
options.maxConcurrentParses = max_concurrent_parses;
} else {
options.maxConcurrentParses =
eu07::scene::bake::detail::defaultPoolThreadCount();
}
} else {
options.maxThreads = max_threads != 0 ? max_threads : 1u;
options.maxConcurrentParses =
max_concurrent_parses != 0 ? max_concurrent_parses : 1u;
}
if( heavy_parse_threshold_mb != 0 ) {
options.heavyParseThresholdMb = heavy_parse_threshold_mb;
} else if( spool_flush ) {
// Streaming/spool bounds RAM — duze pliki na dysku nie wymagaja serial parse.
options.heavyParseThresholdMb = 0u;
}
std::unique_ptr<eu07::scene::bake::PackModelSpoolFile> root_pack_spool;
if( spool_flush ) {
fs::path const spool_path {
fs::temp_directory_path() / "eu7v2_pack_flush.bin" };
root_pack_spool =
std::make_unique<eu07::scene::bake::PackModelSpoolFile>( spool_path );
options.packFlushPerFile = true;
options.onPackModelsFlush =
[&root_pack_spool](
std::vector<eu07::scene::runtime::RuntimeModelInstance> &&models ) {
root_pack_spool->append( std::move( models ) );
};
bake_log(
"[EU7v2] tryb: jeden plik = jeden modul, PACK per .scm -> " +
spool_path.generic_string() +
" (rownolegly bake modulow)" );
}
{
unsigned const pool_threads {
eu07::scene::bake::detail::resolvePoolThreadCount( options.maxThreads ) };
unsigned const parse_limit {
eu07::scene::bake::detail::resolveParseConcurrencyLimit(
options.maxConcurrentParses ) };
bake_log(
"[EU7v2] parse/bake throttle: watki=" +
std::to_string( pool_threads ) +
( options.maxThreads == 0 ? " (auto)" : "" ) +
", rownolegle parse=" + std::to_string( parse_limit ) +
( options.heavyParseThresholdMb != 0
? ", heavy serial >" + std::to_string( options.heavyParseThresholdMb ) + " MB"
: "" ) );
}
options.onProgress =
[&pstate]( eu07::scene::bake::BakeProgress const &progress ) {
on_bake_progress( pstate, progress );
};
options.skipLegacyWrite = true;
options.omitChildModuleModels = true;
struct deferred_root_emit {
bool pending { false };
eu07::scene::bake::RuntimeModule module;
fs::path path;
} deferred_root;
bool const incremental_pack { options.lowMemoryMode };
auto finish_module_emit =
[&]( eu07::scene::bake::RuntimeModule const &module,
fs::path const &text_path,
bool const is_root,
std::vector<eu07::scene::binary::codec::ModelSectionBatch> const *pack_batches,
eu07::scene::bake::ShapeSpoolFile const *shape_spool ) {
eu7v2::emit_outcome const outcome { eu7v2::emit_runtime_module(
module,
text_path,
is_root,
pack_batches,
false,
is_root && root_pack_spool != nullptr ? root_pack_spool.get() : nullptr,
shape_spool ) };
double parse_bake_ms { 0.0 };
{
std::lock_guard<std::mutex> lock { pstate.mutex };
auto const it { pstate.module_start.find( text_path.generic_string() ) };
if( it != pstate.module_start.end() ) {
parse_bake_ms = std::chrono::duration<double, std::milli>(
clock_type::now() - it->second )
.count();
pstate.module_start.erase( it );
}
}
std::size_t const idx {
module_count.fetch_add( 1, std::memory_order_relaxed ) + 1 };
model_count.fetch_add( outcome.model_total, std::memory_order_relaxed );
pstate.emit_us.fetch_add(
static_cast<std::uint64_t>(
( outcome.build_ms + outcome.write_ms ) * 1000.0 ),
std::memory_order_relaxed );
if( false == outcome.ok ) {
emit_failed.store( true, std::memory_order_relaxed );
}
if( verify && outcome.ok ) {
pending_verify_job job;
job.path = fs::path { outcome.written_path };
job.is_root = is_root;
for( auto const &inc : module.includes ) {
if( eu07::scene::detail::isIncFile( inc.sourcePath ) &&
inc.parameters.size() >= 5 ) {
try {
(void)std::stod( inc.parameters[ 1 ] );
(void)std::stod( inc.parameters[ 2 ] );
(void)std::stod( inc.parameters[ 3 ] );
(void)std::stod( inc.parameters[ 4 ] );
++job.spec.placements;
continue;
} catch( ... ) {
}
}
++job.spec.includes;
}
job.spec.models = module.scene.models.size();
job.spec.shapes = module.scene.shapes.size();
if( shape_spool != nullptr ) {
job.spec.shapes += shape_spool->shape_count();
}
job.spec.lines = module.scene.lines.size();
job.spec.tracks = module.scene.tracks.size();
job.spec.traction = module.scene.traction.size();
job.spec.power = module.scene.powerSources.size();
job.spec.memcells = module.scene.memcells.size();
job.spec.launchers = module.scene.eventLaunchers.size();
job.spec.events = module.scene.events.size();
job.spec.sounds = module.scene.sounds.size();
job.spec.dynamics = module.scene.dynamics.size();
job.spec.trainsets = module.scene.trainsets.size();
if( is_root && pack_batches != nullptr ) {
for( auto const &batch : *pack_batches ) {
job.pack_models += batch.models.size();
}
}
if( is_root && root_pack_spool != nullptr ) {
job.pack_models += root_pack_spool->model_count();
}
std::lock_guard<std::mutex> lock { verify_jobs_mutex };
verify_jobs.push_back( std::move( job ) );
}
bool const verbose_module {
is_root || false == outcome.ok || parse_bake_ms >= 300.0 ||
outcome.build_ms >= 100.0 };
if( !verbose_module ) {
{
std::lock_guard<std::mutex> lock { emit_mutex };
if( is_root ) {
root_binary_path = outcome.written_path;
}
}
return;
}
std::string line {
"[EU7v2] (" + std::to_string( idx ) + ") " +
( is_root ? "ROOT " : "" ) + text_path.filename().generic_string() +
" - parse+bake " + fmt_ms( parse_bake_ms ) + "ms, build " +
fmt_ms( outcome.build_ms ) + "ms, zapis " + fmt_ms( outcome.write_ms ) + "ms" };
line += " (models=" + std::to_string( outcome.model_total ) + ", " +
std::to_string( ( outcome.byte_size + 1023 ) / 1024 ) + " KB)";
if( false == outcome.ok ) {
line += " ZAPIS-BLAD: " + outcome.message;
}
{
std::lock_guard<std::mutex> lock { emit_mutex };
if( is_root ) {
root_binary_path = outcome.written_path;
}
}
bake_log( line );
if( is_root && ( pack_batches != nullptr || root_pack_spool != nullptr ) ) {
std::size_t pack_models { 0 };
if( root_pack_spool != nullptr ) {
pack_models += root_pack_spool->model_count();
}
if( pack_batches != nullptr ) {
for( auto const &batch : *pack_batches ) {
pack_models += batch.models.size();
}
}
bake_log(
"[EU7v2] ROOT packModels=" + std::to_string( pack_models ) +
( root_pack_spool != nullptr
? " (spool=" + std::to_string( root_pack_spool->model_count() ) +
")"
: "" ) );
}
};
options.onModuleBaked =
[&]( eu07::scene::bake::RuntimeModule const &module,
fs::path const &text_path,
bool const is_root,
std::vector<eu07::scene::binary::codec::ModelSectionBatch> const *pack_batches,
eu07::scene::bake::ShapeSpoolFile *shape_spool ) {
if( is_root && incremental_pack ) {
deferred_root.pending = true;
deferred_root.module = module;
deferred_root.path = text_path;
return;
}
finish_module_emit( module, text_path, is_root, pack_batches, shape_spool );
};
// Watchdog: prints a heartbeat every ~2 s so a long (or stuck) bake is
// visibly alive. During PACK compose it reads the live counters
// directly, so a stall shows which counter is frozen and on which file.
std::atomic<bool> watchdog_done { false };
std::thread watchdog( [&]() {
while( false == watchdog_done.load( std::memory_order_relaxed ) ) {
for( int slice { 0 };
slice < 20 &&
false == watchdog_done.load( std::memory_order_relaxed );
++slice ) {
std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) );
}
if( watchdog_done.load( std::memory_order_relaxed ) ) {
break;
}
double const elapsed {
std::chrono::duration<double>( clock_type::now() - pstate.start ).count() };
std::string msg {
"[EU7v2] ... pracuje: elapsed=" + fmt_s( elapsed ) +
"s, moduly start=" + std::to_string( pstate.started.load( std::memory_order_relaxed ) ) +
" done=" + std::to_string( module_count.load( std::memory_order_relaxed ) ) };
auto const *cs { pstate.compose_stats.load( std::memory_order_relaxed ) };
if( pstate.compose_active.load( std::memory_order_relaxed ) && cs != nullptr ) {
std::string file;
{
std::lock_guard<std::mutex> lock { pstate.mutex };
file = pstate.compose_file;
}
msg +=
" | PACK compose: pliki=" +
std::to_string( cs->files_visited.load( std::memory_order_relaxed ) ) +
" submit=" +
std::to_string( cs->includes_submitted.load( std::memory_order_relaxed ) ) +
" inst=" +
std::to_string(
cs->instantiate_fast.load( std::memory_order_relaxed ) +
cs->instantiate_full.load( std::memory_order_relaxed ) ) +
" miss=" +
std::to_string( cs->cache_misses.load( std::memory_order_relaxed ) ) +
" kolejka_max=" +
std::to_string( cs->queue_peak.load( std::memory_order_relaxed ) ) +
" (plik: " + file + ")";
// Live deadlock diagnostics: what every worker/gate is doing.
msg +=
"\n[EU7v2] DIAG: pending=" +
std::to_string( cs->diag_pending.load( std::memory_order_relaxed ) ) +
" workers_alive=" +
std::to_string( cs->diag_workers_alive.load( std::memory_order_relaxed ) ) +
" idle=" +
std::to_string( cs->diag_workers_idle.load( std::memory_order_relaxed ) ) +
" parse=" +
std::to_string( cs->diag_workers_parse.load( std::memory_order_relaxed ) ) +
" proc=" +
std::to_string( cs->diag_workers_inst.load( std::memory_order_relaxed ) ) +
" gate=" +
std::to_string( cs->diag_gate_active.load( std::memory_order_relaxed ) ) +
"/" +
std::to_string( cs->diag_gate_max.load( std::memory_order_relaxed ) ) +
" gate_wait=" +
std::to_string( cs->diag_gate_waiting.load( std::memory_order_relaxed ) ) +
" main_wait=" +
std::to_string( cs->diag_main_wait.load( std::memory_order_relaxed ) );
}
bake_log( msg );
}
} );
eu07::scene::bake::BakeTreeStats stats;
try {
(void)eu07::scene::bake::bakeModuleTree( input, &stats, options );
}
catch( ... ) {
watchdog_done.store( true, std::memory_order_relaxed );
watchdog.join();
throw;
}
watchdog_done.store( true, std::memory_order_relaxed );
watchdog.join();
if( deferred_root.pending ) {
bake_log( "[EU7v2] ROOT emit (po wszystkich modulach)..." );
finish_module_emit( deferred_root.module, deferred_root.path, true, nullptr, nullptr );
}
if( verify && false == verify_jobs.empty() ) {
auto const verify_begin { clock_type::now() };
std::atomic<std::size_t> next_job { 0 };
std::mutex verify_error_mutex;
std::string verify_error;
const unsigned worker_count { std::max(
1u,
std::min(
static_cast<unsigned>( verify_jobs.size() ),
std::thread::hardware_concurrency() == 0 ? 4u
: std::thread::hardware_concurrency() ) ) };
std::vector<std::thread> verify_workers;
verify_workers.reserve( worker_count );
for( unsigned worker_index { 0 }; worker_index < worker_count; ++worker_index ) {
verify_workers.emplace_back( [&]() {
while( true ) {
std::size_t const job_index {
next_job.fetch_add( 1, std::memory_order_relaxed ) };
if( job_index >= verify_jobs.size() ) {
return;
}
pending_verify_job const &job { verify_jobs[ job_index ] };
std::string message;
if( false ==
eu7v2::verify_written_module(
job.path, job.spec, job.is_root, job.pack_models, &message ) ) {
verify_ok.store( false, std::memory_order_relaxed );
std::lock_guard<std::mutex> lock { verify_error_mutex };
if( verify_error.empty() ) {
verify_error = job.path.filename().generic_string() + ":\n" + message;
}
}
}
} );
}
for( std::thread &worker : verify_workers ) {
worker.join();
}
double const verify_s {
std::chrono::duration<double>( clock_type::now() - verify_begin ).count() };
pstate.emit_us.fetch_add(
static_cast<std::uint64_t>( verify_s * 1.0e6 ),
std::memory_order_relaxed );
bake_log(
"[EU7v2] VERIFY batch: " + std::to_string( verify_jobs.size() ) + " plikow, " +
fmt_s( verify_s ) + "s" );
if( false == verify_ok.load() && false == verify_error.empty() ) {
bake_log( verify_error );
}
}
report.baked = true;
report.module_count = module_count.load();
report.model_count = model_count.load();
report.root_binary_path = root_binary_path;
report.verify_ok = verify_ok.load() && ( false == emit_failed.load() );
if( emit_failed.load() ) {
report.error = "blad zapisu eu7v2 (patrz log)";
}
double const total_s {
std::chrono::duration<double>( clock_type::now() - pstate.start ).count() };
double const emit_s {
static_cast<double>( pstate.emit_us.load( std::memory_order_relaxed ) ) / 1.0e6 };
std::string summary {
"[EU7v2] BAKE DONE: modules=" + std::to_string( report.module_count ) +
", total=" + fmt_s( total_s ) + "s, watki=" +
std::to_string( pstate.threads.load( std::memory_order_relaxed ) ) +
", emit=" + fmt_s( emit_s ) + "s" };
if( stats.has_pack_diagnostics ) {
auto const &pc { stats.pack_compose };
summary += ", compose=" + fmt_s( static_cast<double>( pc.compose_us ) / 1.0e6 ) +
"s, parse=" + fmt_s( static_cast<double>( pc.parse_us ) / 1.0e6 ) +
"s [tok=" + fmt_s( static_cast<double>( pc.tokenize_us ) / 1.0e6 ) +
" proc=" + fmt_s( static_cast<double>( pc.process_us ) / 1.0e6 ) +
" make=" + fmt_s( static_cast<double>( pc.makeentry_us ) / 1.0e6 ) +
"], inst=" + fmt_s( static_cast<double>( pc.instantiate_us ) / 1.0e6 ) +
"s, sink=" + fmt_s( static_cast<double>( pc.sink_us ) / 1.0e6 ) +
"s, finalize=" + fmt_s( static_cast<double>( pc.finalize_us ) / 1.0e6 ) +
"s (pliki=" + std::to_string( pc.files_visited ) +
", modele=" + std::to_string( pc.models_emitted ) +
", sekcje=" + std::to_string( pc.sections ) +
", inc_inline=" + std::to_string( pc.inc_includes_inlined ) +
", miss=" + std::to_string( pc.cache_misses ) +
", hit=" + std::to_string( pc.cache_hits ) + ")";
}
bake_log( summary );
if( verify ) {
bake_log( report.verify_ok ? "[EU7v2] VERIFY: PASS" : "[EU7v2] VERIFY: FAIL" );
}
return report;
}
catch( std::exception const &ex ) {
report.error = ex.what();
return report;
}
catch( ... ) {
report.error = "nieznany wyjatek bake eu7v2";
return report;
}
}
} // namespace scene::eu7::bake_parser } // namespace scene::eu7::bake_parser

View File

@@ -9,6 +9,7 @@ http://mozilla.org/MPL/2.0/.
#pragma once #pragma once
#include <cstddef>
#include <string> #include <string>
namespace scene::eu7::bake_parser { namespace scene::eu7::bake_parser {
@@ -19,4 +20,32 @@ bake_scenario_tree(
unsigned MaxThreads, unsigned MaxThreads,
std::string &ErrorOut ); std::string &ErrorOut );
// Result of a headless text -> eu7v2 bake (and optional roundtrip verify).
struct Eu7v2BakeReport {
bool baked { false }; // bake tree completed without throwing
bool verify_requested { false };
bool verify_ok { true }; // all per-module record counts matched
std::size_t module_count { 0 };
std::size_t model_count { 0 };
std::string root_binary_path; // emitted root .eu7v2
std::string error;
};
// Bakes the whole text scenery tree directly into .eu7v2 modules (no legacy
// .eu7). When Verify is set each emitted module is reloaded and its record
// counts are compared against the source RuntimeModule, with a PASS/FAIL
// report printed to stdout. This is the headless consistency guard.
// MemLimitGb: abort when process private memory exceeds this (0 = no limit).
// Default 50 when called from headless CLI.
// MaxConcurrentParses: cap parallel parseFile/processScene (0 = auto: 1 with MemLimitGb spool).
// MaxThreads 0 with spool = 1 (serial module bake); use --eu7v2-threads N to override.
[[nodiscard]] Eu7v2BakeReport
bake_scenario_tree_eu7v2(
std::string const &TextScenarioPath,
unsigned MaxThreads,
bool Verify,
unsigned MemLimitGb = 50u,
unsigned MaxConcurrentParses = 0u,
unsigned HeavyParseThresholdMb = 0u );
} // namespace scene::eu7::bake_parser } // namespace scene::eu7::bake_parser

View File

@@ -80,7 +80,9 @@ log_load_stats() {
WriteLog( WriteLog(
" read/cache: " + format_seconds( s.read_ms ) + " read/cache: " + format_seconds( s.read_ms ) +
" reads=" + std::to_string( s.module_read ) + " reads=" + std::to_string( s.module_read ) +
" cache_hit=" + std::to_string( s.module_cache_hit ) ); " cache_hit=" + std::to_string( s.module_cache_hit ) +
" v2_loaded=" + std::to_string( s.module_v2_loaded ) +
" v2_baked=" + std::to_string( s.module_v2_baked ) );
WriteLog( WriteLog(
" scm fallback: " + format_seconds( s.scm_fallback_ms ) + " scm fallback: " + format_seconds( s.scm_fallback_ms ) +
" count=" + std::to_string( s.scm_fallback ) ); " count=" + std::to_string( s.scm_fallback ) );

View File

@@ -52,6 +52,8 @@ struct Eu7LoadStats {
std::uint64_t sounds { 0 }; std::uint64_t sounds { 0 };
std::uint64_t events { 0 }; std::uint64_t events { 0 };
std::uint64_t trainsets { 0 }; std::uint64_t trainsets { 0 };
std::uint64_t module_v2_loaded { 0 };
std::uint64_t module_v2_baked { 0 };
std::uint64_t pack_skipped_includes { 0 }; std::uint64_t pack_skipped_includes { 0 };
std::uint64_t pack_sections_loaded { 0 }; std::uint64_t pack_sections_loaded { 0 };
std::uint64_t pack_models { 0 }; std::uint64_t pack_models { 0 };

View File

@@ -18,6 +18,9 @@ http://mozilla.org/MPL/2.0/.
#include "scene/eu7/eu7_section_stream.h" #include "scene/eu7/eu7_section_stream.h"
#include "scene/eu7/eu7_parameters.h" #include "scene/eu7/eu7_parameters.h"
#include "scene/eu7/eu7_transform.h" #include "scene/eu7/eu7_transform.h"
#include "scene/eu7/v2/eu7v2_bake.h"
#include "scene/eu7/v2/eu7v2_format.h"
#include "scene/eu7/v2/eu7v2_load.h"
#include "scene/scene.h" #include "scene/scene.h"
#include "scene/scenenode.h" #include "scene/scenenode.h"
#include "rendering/renderer.h" #include "rendering/renderer.h"
@@ -30,11 +33,14 @@ http://mozilla.org/MPL/2.0/.
#include <cmath> #include <cmath>
#include <cstring> #include <cstring>
#include <filesystem>
#include <fstream> #include <fstream>
#include <iterator>
#include <limits> #include <limits>
#include <stdexcept> #include <stdexcept>
#include <unordered_map> #include <unordered_map>
#include <unordered_set> #include <unordered_set>
#include <vector>
namespace scene::eu7 { namespace scene::eu7 {
@@ -43,6 +49,135 @@ namespace {
std::unordered_map<std::string, Eu7Module> g_module_file_cache; std::unordered_map<std::string, Eu7Module> g_module_file_cache;
bool g_packed_root_active { false }; bool g_packed_root_active { false };
// --- eu7v2 experiment: lossy lean-format transcode/load ---------------------
// Plik towarzyszacy obok legacy .eu7: <eu7>.v2
[[nodiscard]] std::string
eu7v2_sidecar_path( std::string const &eu7_path ) {
return eu7_path + ".v2";
}
// .v2 jest swiezy gdy istnieje i nie jest starszy niz zrodlowy .eu7.
[[nodiscard]] bool
eu7v2_sidecar_fresh( std::string const &eu7_path, std::string const &v2_path ) {
std::error_code ec;
if( false == std::filesystem::exists( v2_path, ec ) ) {
return false;
}
auto const v2_time { std::filesystem::last_write_time( v2_path, ec ) };
if( ec ) {
return false;
}
auto const src_time { std::filesystem::last_write_time( eu7_path, ec ) };
if( ec ) {
return false;
}
return v2_time >= src_time;
}
// Odczyt modulu z chudego .v2. Stratny: odtwarza wylacznie scene (PROT/INST/MESH
// + rekordy), bez includes/pack/prototypow/flag. Zwraca false przy bledzie.
[[nodiscard]] bool
read_eu7v2_module( std::string const &v2_path, Eu7Module &out ) {
std::ifstream input { v2_path, std::ios::binary };
if( !input ) {
return false;
}
std::vector<std::uint8_t> bytes(
( std::istreambuf_iterator<char>( input ) ),
std::istreambuf_iterator<char>() );
if( bytes.empty() ) {
return false;
}
Eu7Scene scene;
if( false == eu7v2::load_scene( bytes.data(), bytes.size(), scene ) ) {
return false;
}
out = Eu7Module{};
out.scene = std::move( scene );
return true;
}
// True gdy plik zaczyna sie magiem eu7v2 ('E','U','7','C').
[[nodiscard]] bool
file_is_eu7v2( std::string const &path ) {
std::ifstream input { path, std::ios::binary };
if( !input ) {
return false;
}
char magic[ 4 ] { 0, 0, 0, 0 };
input.read( magic, 4 );
if( input.gcount() != 4 ) {
return false;
}
return magic[ 0 ] == 'E' && magic[ 1 ] == 'U' && magic[ 2 ] == '7' && magic[ 3 ] == 'C';
}
// Pelny modul z formatu .eu7v2 (lossless): scena + includes + placement + flagi.
[[nodiscard]] bool
read_eu7v2_module_full( std::string const &path, Eu7Module &out ) {
std::ifstream input { path, std::ios::binary };
if( !input ) {
return false;
}
std::vector<std::uint8_t> bytes(
( std::istreambuf_iterator<char>( input ) ),
std::istreambuf_iterator<char>() );
if( bytes.empty() ) {
return false;
}
out = Eu7Module{};
return eu7v2::load_module( bytes.data(), bytes.size(), out );
}
// Zapis chudego .v2 obok legacy .eu7 (best-effort, ciche niepowodzenie).
void
write_eu7v2_module( std::string const &v2_path, Eu7Module const &module ) {
auto const bytes { eu7v2::bake_scene( module.scene ) };
std::ofstream output { v2_path, std::ios::binary | std::ios::trunc };
if( !output ) {
return;
}
output.write(
reinterpret_cast<char const *>( bytes.data() ),
static_cast<std::streamsize>( bytes.size() ) );
}
// Odczyt modulu z transkodem eu7v2 gdy Global.eu7v2_runtime wlaczone:
// - jest swiezy .v2 -> czytaj z niego (szybka, stratna sciezka),
// - brak/nieswiezy -> czytaj legacy i (dla nie-PACK) zapisz .v2 na nastepny raz.
[[nodiscard]] Eu7Module
read_module_maybe_v2( std::string const &resolved ) {
if( false == Global.eu7v2_runtime ) {
return read_module( resolved );
}
// Primary path: the module file itself is a native .eu7v2 container
// (text -> eu7v2 bake, no legacy .eu7). Load it losslessly.
if( file_is_eu7v2( resolved ) ) {
Eu7Module v2_module;
if( read_eu7v2_module_full( resolved, v2_module ) ) {
++load_stats().module_v2_loaded;
return v2_module;
}
throw std::runtime_error( "EU7v2: uszkodzony modul \"" + resolved + "\"" );
}
// Legacy .eu7 with optional ".v2" sidecar transcode (older experiment).
auto const v2_path { eu7v2_sidecar_path( resolved ) };
if( eu7v2_sidecar_fresh( resolved, v2_path ) ) {
Eu7Module v2_module;
if( read_eu7v2_module( v2_path, v2_module ) ) {
++load_stats().module_v2_loaded;
return v2_module;
}
WriteLog( "EU7v2: nieudany odczyt \"" + v2_path + "\", fallback legacy .eu7" );
}
auto legacy { read_module( resolved ) };
if( false == legacy.has_pack_chunk ) {
write_eu7v2_module( v2_path, legacy );
++load_stats().module_v2_baked;
}
return legacy;
}
void void
apply_material( shape_node::shapenode_data &data, std::string material_name ) { apply_material( shape_node::shapenode_data &data, std::string material_name ) {
replace_slashes( material_name ); replace_slashes( material_name );
@@ -212,7 +347,7 @@ load_module_recursive(
} }
else { else {
ScopedTimer const read_timer { load_stats().read_ms }; ScopedTimer const read_timer { load_stats().read_ms };
owned_module = read_module( resolved ); owned_module = read_module_maybe_v2( resolved );
++load_stats().module_read; ++load_stats().module_read;
auto const emplaced { g_module_file_cache.emplace( resolved, std::move( owned_module ) ) }; auto const emplaced { g_module_file_cache.emplace( resolved, std::move( owned_module ) ) };
template_module = &emplaced.first->second; template_module = &emplaced.first->second;
@@ -370,13 +505,17 @@ resolve_scenery_path( std::string const &reference ) {
std::string std::string
binary_path( std::string const &reference ) { binary_path( std::string const &reference ) {
auto path { resolve_scenery_path( reference ) }; auto path { resolve_scenery_path( reference ) };
if( Global.eu7v2_runtime ) {
return eu7v2::binary_path_from_text( path ).generic_string();
}
char const *const ext { ".eu7" };
if( path.ends_with( ".scm" ) || path.ends_with( ".sbt" ) || path.ends_with( ".inc" ) || if( path.ends_with( ".scm" ) || path.ends_with( ".sbt" ) || path.ends_with( ".inc" ) ||
path.ends_with( ".scn" ) ) { path.ends_with( ".scn" ) ) {
path.replace( path.size() - 4, 4, ".eu7" ); path.replace( path.size() - 4, 4, ext );
} }
else if( false == path.ends_with( ".eu7" ) ) { else if( false == path.ends_with( ".eu7" ) ) {
erase_extension( path ); erase_extension( path );
path += ".eu7"; path += ext;
} }
return path; return path;
} }
@@ -421,7 +560,13 @@ terrain_binary_path( std::string const &terrain_reference ) {
bool bool
probe_file( std::string const &path ) { probe_file( std::string const &path ) {
return FileExists( path ) && is_valid_eu7b_file( path ); if( false == FileExists( path ) ) {
return false;
}
if( Global.eu7v2_runtime && file_is_eu7v2( path ) ) {
return true;
}
return is_valid_eu7b_file( path );
} }
bool bool

561
scene/eu7/v2/eu7v2_bake.cpp Normal file
View File

@@ -0,0 +1,561 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#include "scene/eu7/v2/eu7v2_bake.h"
#include "scene/eu7/v2/eu7v2_format.h"
#include "scene/eu7/v2/eu7v2_scene.h"
#include "scene/eu7/v2/eu7v2_records.h"
#include "scene/eu7/eu7_types.h"
#include <string>
#include <unordered_map>
namespace eu7v2 {
namespace {
// Bakes the parser output into the dependency-free eu7v2 structs, owning the
// string table all chunks share.
class scene_baker {
public:
explicit scene_baker( scene::eu7::Eu7Scene const &scene ) : m_scene( scene ) {}
scene_baker( scene::eu7::Eu7Scene const &scene, scene::eu7::Eu7Module const &module )
: m_scene( scene ), m_module( &module ) {}
std::vector<std::uint8_t> run() {
convert_models();
convert_terrain();
convert_shapes();
convert_lines();
convert_tracks();
convert_traction();
convert_power_sources();
convert_memcells();
convert_launchers();
convert_events();
convert_sounds();
convert_dynamics();
convert_trainsets();
if( m_module != nullptr ) {
convert_includes();
convert_meta();
}
return serialize();
}
private:
[[nodiscard]] std::uint32_t str( std::string const &s ) { return m_strings.intern( s ); }
[[nodiscard]] node_record node( scene::eu7::Eu7BasicNode const &n ) {
node_record out;
out.name = str( n.name );
out.type = str( n.node_type );
out.area_center = { n.area.center.x, n.area.center.y, n.area.center.z };
out.area_radius = n.area.radius;
out.range_sq_min = n.range_squared_min;
out.range_sq_max = n.range_squared_max;
out.visible = n.visible;
return out;
}
// -- models: deduplicate identical definitions into prototypes ----------
void convert_models() {
for( auto const &model : m_scene.models ) {
auto const proto { intern_prototype( model ) };
model_instance inst;
inst.proto = proto;
inst.x = model.location.x;
inst.y = model.location.y;
inst.z = model.location.z;
inst.ax = static_cast<float>( model.angles.x );
inst.ay = static_cast<float>( model.angles.y );
inst.az = static_cast<float>( model.angles.z );
inst.sx = static_cast<float>( model.scale.x );
inst.sy = static_cast<float>( model.scale.y );
inst.sz = static_cast<float>( model.scale.z );
inst.cell_id = model.pack_cell_id;
inst.texture_override = kNoString; // within a single bake the proto already carries the skin
inst.has_node = true;
inst.node = node( model.node );
m_instances.push_back( inst );
}
}
[[nodiscard]] std::uint32_t intern_prototype( scene::eu7::Eu7Model const &model ) {
std::string key;
key.reserve( model.model_file.size() + model.texture_file.size() + 32 );
key.append( model.model_file );
key.push_back( '\x1f' );
key.append( model.texture_file );
key.push_back( '\x1f' );
key.push_back( model.transition ? '1' : '0' );
key.push_back( model.is_terrain ? '1' : '0' );
key.append( std::to_string( model.baked_range_min ) );
key.push_back( '\x1f' );
key.append( std::to_string( model.baked_range_max ) );
key.push_back( '\x1f' );
for( auto const s : model.light_states ) {
key.append( std::to_string( s ) );
key.push_back( ',' );
}
key.push_back( '\x1f' );
for( auto const c : model.light_colors ) {
key.append( std::to_string( c ) );
key.push_back( ',' );
}
auto const it { m_prototype_lookup.find( key ) };
if( it != m_prototype_lookup.end() ) {
return it->second;
}
model_prototype proto;
proto.model_file = str( model.model_file );
proto.texture_file = model.texture_file.empty() ? kNoString : str( model.texture_file );
proto.flags = 0;
if( model.transition ) {
proto.flags |= proto_flag::transition;
}
if( model.is_terrain ) {
proto.flags |= proto_flag::is_terrain;
}
if( model.pack_flags & scene::eu7::kEu7PackFlagInstanceableHint ) {
proto.flags |= proto_flag::instanceable;
}
proto.range_min = model.baked_range_min;
proto.range_max = model.baked_range_max;
proto.light_states = model.light_states;
proto.light_colors = model.light_colors;
auto const id { static_cast<std::uint32_t>( m_prototypes.size() ) };
m_prototypes.push_back( std::move( proto ) );
m_prototype_lookup.emplace( std::move( key ), id );
return id;
}
// -- terrain: store origin in f64, vertices f32 relative to origin ------
void convert_terrain() {
for( auto const &shape : m_scene.terrain_shapes ) {
terrain_mesh mesh;
mesh.material = shape.material_path.empty() ? kNoString : str( shape.material_path );
mesh.translucent = shape.translucent;
mesh.ox = shape.origin.x;
mesh.oy = shape.origin.y;
mesh.oz = shape.origin.z;
mesh.vertices.reserve( shape.vertices.size() );
for( auto const &v : shape.vertices ) {
mesh_vertex mv;
mv.px = static_cast<float>( v.position.x - shape.origin.x );
mv.py = static_cast<float>( v.position.y - shape.origin.y );
mv.pz = static_cast<float>( v.position.z - shape.origin.z );
mv.nx = v.normal.x;
mv.ny = v.normal.y;
mv.nz = v.normal.z;
mv.u = static_cast<float>( v.u );
mv.v = static_cast<float>( v.v );
mesh.vertices.push_back( mv );
}
m_meshes.push_back( std::move( mesh ) );
}
}
[[nodiscard]] lighting_block lighting( scene::eu7::Eu7LightingData const &l ) {
lighting_block out;
out.diffuse[ 0 ] = l.diffuse.x; out.diffuse[ 1 ] = l.diffuse.y;
out.diffuse[ 2 ] = l.diffuse.z; out.diffuse[ 3 ] = l.diffuse.w;
out.ambient[ 0 ] = l.ambient.x; out.ambient[ 1 ] = l.ambient.y;
out.ambient[ 2 ] = l.ambient.z; out.ambient[ 3 ] = l.ambient.w;
out.specular[ 0 ] = l.specular.x; out.specular[ 1 ] = l.specular.y;
out.specular[ 2 ] = l.specular.z; out.specular[ 3 ] = l.specular.w;
return out;
}
// -- non-terrain shapes: same encoding as terrain but lossless on node ---
void convert_shapes() {
for( auto const &shape : m_scene.shapes ) {
shape_record r;
r.node = node( shape.node );
r.translucent = shape.translucent;
r.material = shape.material_path.empty() ? kNoString : str( shape.material_path );
r.lighting = lighting( shape.lighting );
r.ox = shape.origin.x;
r.oy = shape.origin.y;
r.oz = shape.origin.z;
r.vertices.reserve( shape.vertices.size() );
for( auto const &v : shape.vertices ) {
mesh_vertex mv;
mv.px = static_cast<float>( v.position.x - shape.origin.x );
mv.py = static_cast<float>( v.position.y - shape.origin.y );
mv.pz = static_cast<float>( v.position.z - shape.origin.z );
mv.nx = v.normal.x;
mv.ny = v.normal.y;
mv.nz = v.normal.z;
mv.u = static_cast<float>( v.u );
mv.v = static_cast<float>( v.v );
r.vertices.push_back( mv );
}
m_shapes.push_back( std::move( r ) );
}
}
void convert_lines() {
for( auto const &line : m_scene.lines ) {
lines_record r;
r.node = node( line.node );
r.lighting = lighting( line.lighting );
r.line_width = line.line_width;
r.ox = line.origin.x;
r.oy = line.origin.y;
r.oz = line.origin.z;
r.vertices.reserve( line.vertices.size() );
for( auto const &v : line.vertices ) {
r.vertices.push_back( { v.position.x, v.position.y, v.position.z } );
}
m_lines.push_back( std::move( r ) );
}
}
void convert_tracks() {
for( auto const &t : m_scene.tracks ) {
track_record r;
r.node = node( t.node );
r.track_type = static_cast<std::uint8_t>( t.track_type );
r.category = static_cast<std::uint8_t>( t.category );
r.length = t.length;
r.track_width = t.track_width;
r.friction = t.friction;
r.sound_distance = t.sound_distance;
r.quality_flag = t.quality_flag;
r.damage_flag = t.damage_flag;
r.environment = static_cast<std::int8_t>( t.environment );
if( t.visibility.has_value() ) {
r.has_visibility = true;
r.visibility.material1 = str( t.visibility->material1 );
r.visibility.tex_length = t.visibility->tex_length;
r.visibility.material2 = str( t.visibility->material2 );
r.visibility.tex_height1 = t.visibility->tex_height1;
r.visibility.tex_width = t.visibility->tex_width;
r.visibility.tex_slope = t.visibility->tex_slope;
}
r.paths.reserve( t.paths.size() );
for( auto const &p : t.paths ) {
track_path tp;
tp.p_start = { p.p_start.x, p.p_start.y, p.p_start.z };
tp.roll_start = p.roll_start;
tp.cp_out = { p.cp_out.x, p.cp_out.y, p.cp_out.z };
tp.cp_in = { p.cp_in.x, p.cp_in.y, p.cp_in.z };
tp.p_end = { p.p_end.x, p.p_end.y, p.p_end.z };
tp.roll_end = p.roll_end;
tp.radius = p.radius;
r.paths.push_back( tp );
}
r.tail_keywords.reserve( t.tail_keywords.size() );
for( auto const &kv : t.tail_keywords ) {
r.tail_keywords.emplace_back( str( kv.first ), str( kv.second ) );
}
m_tracks.push_back( std::move( r ) );
}
}
void convert_traction() {
for( auto const &t : m_scene.traction ) {
traction_record r;
r.node = node( t.node );
r.power_supply_name = str( t.power_supply_name );
r.nominal_voltage = t.nominal_voltage;
r.max_current = t.max_current;
r.resistivity = t.resistivity_ohm_per_m;
r.material = static_cast<std::uint8_t>( t.material );
r.wire_thickness = t.wire_thickness;
r.damage_flag = t.damage_flag;
r.wire_p1 = { t.wire_p1.x, t.wire_p1.y, t.wire_p1.z };
r.wire_p2 = { t.wire_p2.x, t.wire_p2.y, t.wire_p2.z };
r.wire_p3 = { t.wire_p3.x, t.wire_p3.y, t.wire_p3.z };
r.wire_p4 = { t.wire_p4.x, t.wire_p4.y, t.wire_p4.z };
r.min_height = t.min_height;
r.segment_length = t.segment_length;
r.wire_count = t.wire_count;
r.wire_offset = t.wire_offset;
if( t.parallel_name.has_value() ) {
r.has_parallel = true;
r.parallel_name = str( *t.parallel_name );
}
m_traction.push_back( std::move( r ) );
}
}
void convert_power_sources() {
for( auto const &p : m_scene.power_sources ) {
power_source_record r;
r.node = node( p.node );
r.position = { p.position.x, p.position.y, p.position.z };
r.nominal_voltage = p.nominal_voltage;
r.voltage_frequency = p.voltage_frequency;
r.internal_resistance = p.internal_resistance;
r.max_output_current = p.max_output_current;
r.fast_fuse_timeout = p.fast_fuse_timeout;
r.fast_fuse_repetition = p.fast_fuse_repetition;
r.slow_fuse_timeout = p.slow_fuse_timeout;
r.modifier = static_cast<std::uint8_t>( p.modifier );
m_power.push_back( std::move( r ) );
}
}
void convert_memcells() {
for( auto const &m : m_scene.memcells ) {
memcell_record r;
r.node = node( m.node );
r.text = str( m.text );
r.value1 = m.value1;
r.value2 = m.value2;
if( m.track_name.has_value() ) {
r.has_track = true;
r.track_name = str( *m.track_name );
}
m_memcells.push_back( std::move( r ) );
}
}
void convert_launchers() {
for( auto const &l : m_scene.event_launchers ) {
launcher_record r;
r.node = node( l.node );
r.location = { l.location.x, l.location.y, l.location.z };
r.radius_squared = l.radius_squared;
r.activation_key = l.activation_key;
r.delta_time = l.delta_time;
r.event1_name = str( l.event1_name );
r.event2_name = str( l.event2_name );
if( l.condition.has_value() ) {
r.has_condition = true;
r.condition.memcell_name = str( l.condition->memcell_name );
r.condition.compare_text = str( l.condition->compare_text );
r.condition.compare_value1 = l.condition->compare_value1;
r.condition.compare_value2 = l.condition->compare_value2;
r.condition.check_mask = l.condition->check_mask;
}
r.train_triggered = l.train_triggered;
r.launch_hour = l.launch_hour;
r.launch_minute = l.launch_minute;
m_launchers.push_back( std::move( r ) );
}
}
void convert_events() {
for( auto const &e : m_scene.events ) {
event_record r;
r.name = str( e.name );
r.type = static_cast<std::uint8_t>( e.type );
r.delay = e.delay;
r.delay_random = e.delay_random;
r.delay_departure = e.delay_departure;
r.ignored = e.ignored;
r.passive = e.passive;
r.targets.reserve( e.targets.size() );
for( auto const &t : e.targets ) {
r.targets.push_back( str( t ) );
}
r.payload.reserve( e.payload.size() );
for( auto const &kv : e.payload ) {
r.payload.emplace_back( str( kv.first ), str( kv.second ) );
}
m_events.push_back( std::move( r ) );
}
}
void convert_sounds() {
for( auto const &s : m_scene.sounds ) {
sound_record r;
r.node = node( s.node );
r.location = { s.location.x, s.location.y, s.location.z };
r.wav_file = str( s.wav_file );
m_sounds.push_back( std::move( r ) );
}
}
void convert_dynamics() {
for( auto const &d : m_scene.dynamics ) {
dynamic_record r;
r.node = node( d.node );
r.data_folder = str( d.data_folder );
r.skin_file = str( d.skin_file );
r.mmd_file = str( d.mmd_file );
r.track_name = str( d.track_name );
r.offset = d.offset;
r.driver_type = str( d.driver_type );
r.coupling = d.coupling;
r.coupling_raw = str( d.coupling_raw );
r.coupling_params = str( d.coupling_params );
r.velocity = d.velocity;
r.load_count = d.load_count;
r.load_type = str( d.load_type );
if( d.destination.has_value() ) {
r.has_destination = true;
r.destination = str( *d.destination );
}
if( d.trainset_index.has_value() ) {
r.has_trainset = true;
r.trainset_index = static_cast<std::uint32_t>( *d.trainset_index );
}
m_dynamics.push_back( std::move( r ) );
}
}
void convert_trainsets() {
for( auto const &t : m_scene.trainsets ) {
trainset_record r;
r.name = str( t.name );
r.track = str( t.track );
r.offset = t.offset;
r.velocity = t.velocity;
r.assignment.reserve( t.assignment.size() );
for( auto const &kv : t.assignment ) {
r.assignment.emplace_back( str( kv.first ), str( kv.second ) );
}
r.vehicle_indices.reserve( t.vehicle_indices.size() );
for( auto const idx : t.vehicle_indices ) {
r.vehicle_indices.push_back( static_cast<std::uint32_t>( idx ) );
}
r.couplings.reserve( t.couplings.size() );
for( auto const c : t.couplings ) {
r.couplings.push_back( c );
}
r.driver_index =
t.driver_index == static_cast<std::size_t>( -1 )
? 0xffffffffu
: static_cast<std::uint32_t>( t.driver_index );
m_trainsets.push_back( std::move( r ) );
}
}
void convert_includes() {
for( auto const &inc : m_module->includes ) {
include_record r;
r.source_line = inc.source_line;
r.source_path = inc.source_path.empty() ? kNoString : str( inc.source_path );
r.binary_path = inc.binary_path.empty() ? kNoString : str( inc.binary_path );
r.parameters.reserve( inc.parameters.size() );
for( auto const &p : inc.parameters ) {
r.parameters.push_back( str( p ) );
}
r.site_transform = transform( inc.site_transform );
m_includes.push_back( std::move( r ) );
}
}
[[nodiscard]] transform_record transform( scene::eu7::Eu7TransformContext const &t ) {
transform_record out;
out.origin_stack.reserve( t.origin_stack.size() );
for( auto const &v : t.origin_stack ) {
out.origin_stack.push_back( { v.x, v.y, v.z } );
}
out.scale_stack.reserve( t.scale_stack.size() );
for( auto const &v : t.scale_stack ) {
out.scale_stack.push_back( { v.x, v.y, v.z } );
}
out.rotation = { t.rotation.x, t.rotation.y, t.rotation.z };
out.group_depth = static_cast<std::uint32_t>( t.group_depth );
return out;
}
void convert_meta() {
m_meta.first_init_count = m_scene.first_init_count;
m_meta.has_terrain_chunk = m_module->has_terrain_chunk;
m_meta.has_pack_chunk = m_module->has_pack_chunk;
m_meta.placement_origin_x = m_module->include_placement.origin_x_param;
m_meta.placement_origin_y = m_module->include_placement.origin_y_param;
m_meta.placement_origin_z = m_module->include_placement.origin_z_param;
m_meta.placement_rotation_y = m_module->include_placement.rotation_y_param;
m_has_meta = true;
}
[[nodiscard]] std::vector<std::uint8_t> serialize() const {
container_writer writer( file_kind::sim );
byte_writer strs;
m_strings.serialize( strs );
writer.add_chunk( chunk::strs, strs );
// META must be emitted before the chunks it describes are consumed;
// a single pass over chunks resolves it (loader reads it whenever seen).
if( m_has_meta ) {
byte_writer meta;
write_meta( meta, m_meta );
writer.add_chunk( chunk::meta, meta );
}
auto emit { [&]( std::uint32_t id, auto const &write_fn, auto const &items ) {
if( items.empty() ) {
return;
}
byte_writer payload;
write_fn( payload, items );
writer.add_chunk( id, payload );
} };
emit( chunk::incl, &write_includes, m_includes );
emit( chunk::prot, &write_prototypes, m_prototypes );
emit( chunk::inst, &write_instances, m_instances );
emit( chunk::mesh, &write_terrain_meshes, m_meshes );
emit( chunk::shpe, &write_shapes, m_shapes );
emit( chunk::line, &write_lines, m_lines );
emit( chunk::trak, &write_tracks, m_tracks );
emit( chunk::trac, &write_traction, m_traction );
emit( chunk::pwrs, &write_power_sources, m_power );
emit( chunk::memc, &write_memcells, m_memcells );
emit( chunk::laun, &write_launchers, m_launchers );
emit( chunk::evnt, &write_events, m_events );
emit( chunk::sond, &write_sounds, m_sounds );
emit( chunk::dynm, &write_dynamics, m_dynamics );
emit( chunk::trst, &write_trainsets, m_trainsets );
return writer.data();
}
scene::eu7::Eu7Scene const &m_scene;
scene::eu7::Eu7Module const *m_module { nullptr };
string_table m_strings;
std::vector<model_prototype> m_prototypes;
std::unordered_map<std::string, std::uint32_t> m_prototype_lookup;
std::vector<model_instance> m_instances;
std::vector<terrain_mesh> m_meshes;
std::vector<shape_record> m_shapes;
std::vector<lines_record> m_lines;
std::vector<track_record> m_tracks;
std::vector<traction_record> m_traction;
std::vector<power_source_record> m_power;
std::vector<memcell_record> m_memcells;
std::vector<launcher_record> m_launchers;
std::vector<event_record> m_events;
std::vector<sound_record> m_sounds;
std::vector<dynamic_record> m_dynamics;
std::vector<trainset_record> m_trainsets;
std::vector<include_record> m_includes;
module_meta m_meta;
bool m_has_meta { false };
};
} // namespace
std::vector<std::uint8_t>
bake_scene( scene::eu7::Eu7Scene const &scene ) {
scene_baker baker( scene );
return baker.run();
}
std::vector<std::uint8_t>
bake_module( scene::eu7::Eu7Module const &module ) {
scene_baker baker( module.scene, module );
return baker.run();
}
} // namespace eu7v2

36
scene/eu7/v2/eu7v2_bake.h Normal file
View File

@@ -0,0 +1,36 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#pragma once
#include <cstdint>
#include <vector>
namespace scene::eu7 {
struct Eu7Scene;
struct Eu7Module;
} // namespace scene::eu7
namespace eu7v2 {
// Bakes a parsed scene (output of the text parser) into a self-contained
// eu7v2 "sim" container: STRS + PROT/INST (deduplicated models) + MESH
// (terrain) + SHPE/LINE + simulation record chunks. This is the clean-slate
// replacement for the legacy EU7B emitter; it reuses the existing parser
// front-end and only changes the on-disk representation.
[[nodiscard]] std::vector<std::uint8_t>
bake_scene( scene::eu7::Eu7Scene const &scene );
// Lossless module bake: everything bake_scene() emits, plus INCL (includes
// for recursion) and META (flags, placement, first_init_count). This is the
// format the runtime loads as a module.
[[nodiscard]] std::vector<std::uint8_t>
bake_module( scene::eu7::Eu7Module const &module );
} // namespace eu7v2

View File

@@ -0,0 +1,888 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#include "scene/eu7/v2/eu7v2_emit_runtime.h"
#include "scene/eu7/v2/eu7v2_format.h"
#include "scene/eu7/v2/eu7v2_scene.h"
#include "scene/eu7/v2/eu7v2_records.h"
#include <eu07/scene/bake/pack_model_spool.hpp>
#include <eu07/scene/include_resolve.hpp>
#include <eu07/scene/runtime/scene.hpp>
#include <chrono>
#include <fstream>
#include <limits>
#include <span>
#include <sstream>
#include <stdexcept>
#include <string>
#include <unordered_map>
namespace eu7v2 {
namespace {
namespace rt = eu07::scene::runtime;
namespace bake = eu07::scene::bake;
namespace codec = eu07::scene::binary::codec;
[[nodiscard]] bool is_model_inc_placement( bake::ModuleInclude const &inc ) {
if( !eu07::scene::detail::isIncFile( inc.sourcePath ) ) {
return false;
}
if( inc.parameters.size() < 5 ) {
return false;
}
try {
(void)std::stod( inc.parameters[ 1 ] );
(void)std::stod( inc.parameters[ 2 ] );
(void)std::stod( inc.parameters[ 3 ] );
(void)std::stod( inc.parameters[ 4 ] );
return true;
} catch( ... ) {
return false;
}
}
[[nodiscard]] bool try_parse_inc_placement(
std::span<std::string const> const params,
double &x,
double &y,
double &z,
float &rot_y ) {
if( params.size() < 5 ) {
return false;
}
try {
x = std::stod( params[ 1 ] );
y = std::stod( params[ 2 ] );
z = std::stod( params[ 3 ] );
rot_y = static_cast<float>( std::stod( params[ 4 ] ) );
return true;
} catch( ... ) {
return false;
}
}
[[nodiscard]] file_kind file_kind_for_text_path( std::filesystem::path const &text_path ) {
auto const ext { text_path.extension().string() };
if( ext == ".inc" ) {
return file_kind::module;
}
return file_kind::sim;
}
// Mirror of scene_baker but sourced from the parser's Runtime* records.
class runtime_baker {
public:
runtime_baker(
bake::RuntimeModule const &module,
bool const is_root,
std::vector<codec::ModelSectionBatch> const *pack_batches,
bake::PackModelSpoolFile const *pack_spool,
bake::ShapeSpoolFile const *shape_spool,
std::filesystem::path const &text_path )
: m_module( module )
, m_is_root( is_root )
, m_pack_batches( pack_batches )
, m_pack_spool( pack_spool )
, m_shape_spool( shape_spool )
, m_file_kind( file_kind_for_text_path( text_path ) ) {}
[[nodiscard]] std::vector<std::uint8_t> run() {
convert_models();
convert_shapes();
convert_lines();
convert_tracks();
convert_traction();
convert_power_sources();
convert_memcells();
convert_launchers();
convert_events();
convert_sounds();
convert_dynamics();
convert_trainsets();
convert_includes();
convert_meta();
return serialize();
}
[[nodiscard]] std::size_t model_total() const { return m_instances.size(); }
[[nodiscard]] std::size_t placement_total() const { return m_placements.size(); }
[[nodiscard]] std::size_t structural_include_total() const { return m_includes.size(); }
private:
[[nodiscard]] std::uint32_t str( std::string const &s ) { return m_strings.intern( s ); }
[[nodiscard]] std::uint32_t opt_str( std::string const &s ) {
return s.empty() ? kNoString : m_strings.intern( s );
}
[[nodiscard]] node_record node( rt::BasicNode const &n ) {
node_record out;
out.name = opt_str( n.name );
out.type = opt_str( n.nodeType );
out.area_center = { n.area.center.x, n.area.center.y, n.area.center.z };
out.area_radius = n.area.radius;
out.range_sq_min = n.rangeSquaredMin;
out.range_sq_max = n.rangeSquaredMax;
out.visible = n.visible;
return out;
}
[[nodiscard]] lighting_block lighting( rt::LightingData const &l ) {
lighting_block out;
out.diffuse[ 0 ] = l.diffuse.x; out.diffuse[ 1 ] = l.diffuse.y;
out.diffuse[ 2 ] = l.diffuse.z; out.diffuse[ 3 ] = l.diffuse.w;
out.ambient[ 0 ] = l.ambient.x; out.ambient[ 1 ] = l.ambient.y;
out.ambient[ 2 ] = l.ambient.z; out.ambient[ 3 ] = l.ambient.w;
out.specular[ 0 ] = l.specular.x; out.specular[ 1 ] = l.specular.y;
out.specular[ 2 ] = l.specular.z; out.specular[ 3 ] = l.specular.w;
return out;
}
[[nodiscard]] std::uint32_t intern_prototype( rt::RuntimeModelInstance const &model ) {
std::string key;
key.reserve( model.modelFile.size() + model.textureFile.size() + 32 );
key.append( model.modelFile );
key.push_back( '\x1f' );
key.append( model.textureFile );
key.push_back( '\x1f' );
key.push_back( model.transition ? '1' : '0' );
key.push_back( model.isTerrain ? '1' : '0' );
for( auto const s : model.lightStates ) {
key.append( std::to_string( s ) );
key.push_back( ',' );
}
key.push_back( '\x1f' );
for( auto const c : model.lightColors ) {
key.append( std::to_string( c ) );
key.push_back( ',' );
}
auto const it { m_prototype_lookup.find( key ) };
if( it != m_prototype_lookup.end() ) {
return it->second;
}
model_prototype proto;
proto.model_file = str( model.modelFile );
proto.texture_file = opt_str( model.textureFile );
proto.flags = 0;
if( model.transition ) {
proto.flags |= proto_flag::transition;
}
if( model.isTerrain ) {
proto.flags |= proto_flag::is_terrain;
}
proto.range_min = -1.f;
proto.range_max = -1.f;
proto.light_states = model.lightStates;
proto.light_colors = model.lightColors;
auto const id { static_cast<std::uint32_t>( m_prototypes.size() ) };
m_prototypes.push_back( std::move( proto ) );
m_prototype_lookup.emplace( std::move( key ), id );
return id;
}
void emit_model( rt::RuntimeModelInstance const &model ) {
auto const proto { intern_prototype( model ) };
model_instance inst;
inst.proto = proto;
inst.x = model.location.x;
inst.y = model.location.y;
inst.z = model.location.z;
inst.ax = static_cast<float>( model.angles.x );
inst.ay = static_cast<float>( model.angles.y );
inst.az = static_cast<float>( model.angles.z );
inst.sx = static_cast<float>( model.scale.x );
inst.sy = static_cast<float>( model.scale.y );
inst.sz = static_cast<float>( model.scale.z );
inst.cell_id = 0xffu;
inst.texture_override = kNoString;
inst.has_node = true;
inst.node = node( model.node );
m_instances.push_back( std::move( inst ) );
}
void convert_models() {
if( m_is_root && m_pack_spool != nullptr ) {
m_pack_spool->for_each_model( [&]( rt::RuntimeModelInstance const &model ) {
emit_model( model );
} );
}
if( m_is_root && m_pack_batches != nullptr ) {
// Root: models live in the flattened PACK batches, not scene.models.
for( auto const &batch : *m_pack_batches ) {
for( auto const &model : batch.models ) {
emit_model( model );
}
}
}
for( auto const &model : m_module.scene.models ) {
emit_model( model );
}
}
[[nodiscard]] shape_record make_shape_record( rt::RuntimeShapeNode const &shape ) {
shape_record r;
r.node = node( shape.node );
r.translucent = shape.translucent;
r.material = opt_str( shape.materialPath );
r.lighting = lighting( shape.lighting );
r.ox = shape.origin.x;
r.oy = shape.origin.y;
r.oz = shape.origin.z;
r.vertices.reserve( shape.vertices.size() );
for( auto const &v : shape.vertices ) {
mesh_vertex mv;
mv.px = static_cast<float>( v.position.x - shape.origin.x );
mv.py = static_cast<float>( v.position.y - shape.origin.y );
mv.pz = static_cast<float>( v.position.z - shape.origin.z );
mv.nx = static_cast<float>( v.normal.x );
mv.ny = static_cast<float>( v.normal.y );
mv.nz = static_cast<float>( v.normal.z );
mv.u = static_cast<float>( v.u );
mv.v = static_cast<float>( v.v );
r.vertices.push_back( mv );
}
return r;
}
void convert_shapes() {
if( m_shape_spool != nullptr ) {
m_shape_spool->for_each_shape( [&]( rt::RuntimeShapeNode const &shape ) {
(void)opt_str( shape.materialPath );
(void)node( shape.node );
} );
return;
}
for( auto const &shape : m_module.scene.shapes ) {
m_shapes.push_back( make_shape_record( shape ) );
}
}
void convert_lines() {
for( auto const &line : m_module.scene.lines ) {
lines_record r;
r.node = node( line.node );
r.lighting = lighting( line.lighting );
r.line_width = line.lineWidth;
r.ox = line.origin.x;
r.oy = line.origin.y;
r.oz = line.origin.z;
r.vertices.reserve( line.vertices.size() );
for( auto const &v : line.vertices ) {
r.vertices.push_back( { v.position.x, v.position.y, v.position.z } );
}
m_lines.push_back( std::move( r ) );
}
}
void convert_tracks() {
for( auto const &t : m_module.scene.tracks ) {
track_record r;
r.node = node( t.node );
r.track_type = static_cast<std::uint8_t>( t.trackType );
r.category = static_cast<std::uint8_t>( t.category );
r.length = t.length;
r.track_width = t.trackWidth;
r.friction = t.friction;
r.sound_distance = t.soundDistance;
r.quality_flag = t.qualityFlag;
r.damage_flag = t.damageFlag;
r.environment = static_cast<std::int8_t>( t.environment );
if( t.visibility.has_value() ) {
r.has_visibility = true;
r.visibility.material1 = str( t.visibility->material1 );
r.visibility.tex_length = t.visibility->texLength;
r.visibility.material2 = str( t.visibility->material2 );
r.visibility.tex_height1 = t.visibility->texHeight1;
r.visibility.tex_width = t.visibility->texWidth;
r.visibility.tex_slope = t.visibility->texSlope;
}
r.paths.reserve( t.paths.size() );
for( auto const &p : t.paths ) {
track_path tp;
tp.p_start = { p.pStart.x, p.pStart.y, p.pStart.z };
tp.roll_start = p.rollStart;
tp.cp_out = { p.cpOut.x, p.cpOut.y, p.cpOut.z };
tp.cp_in = { p.cpIn.x, p.cpIn.y, p.cpIn.z };
tp.p_end = { p.pEnd.x, p.pEnd.y, p.pEnd.z };
tp.roll_end = p.rollEnd;
tp.radius = p.radius;
r.paths.push_back( tp );
}
r.tail_keywords.reserve( t.tailKeywords.size() );
for( auto const &kv : t.tailKeywords ) {
r.tail_keywords.emplace_back( str( kv.first ), str( kv.second ) );
}
m_tracks.push_back( std::move( r ) );
}
}
void convert_traction() {
for( auto const &t : m_module.scene.traction ) {
traction_record r;
r.node = node( t.node );
r.power_supply_name = opt_str( t.powerSupplyName );
r.nominal_voltage = t.nominalVoltage;
r.max_current = t.maxCurrent;
r.resistivity = t.resistivityOhmPerM;
r.material = static_cast<std::uint8_t>( t.material );
r.wire_thickness = t.wireThickness;
r.damage_flag = t.damageFlag;
r.wire_p1 = { t.wireP1.x, t.wireP1.y, t.wireP1.z };
r.wire_p2 = { t.wireP2.x, t.wireP2.y, t.wireP2.z };
r.wire_p3 = { t.wireP3.x, t.wireP3.y, t.wireP3.z };
r.wire_p4 = { t.wireP4.x, t.wireP4.y, t.wireP4.z };
r.min_height = t.minHeight;
r.segment_length = t.segmentLength;
r.wire_count = t.wireCount;
r.wire_offset = t.wireOffset;
if( t.parallelName.has_value() ) {
r.has_parallel = true;
r.parallel_name = str( *t.parallelName );
}
m_traction.push_back( std::move( r ) );
}
}
void convert_power_sources() {
for( auto const &p : m_module.scene.powerSources ) {
power_source_record r;
r.node = node( p.node );
r.position = { p.position.x, p.position.y, p.position.z };
r.nominal_voltage = p.nominalVoltage;
r.voltage_frequency = p.voltageFrequency;
r.internal_resistance = p.internalResistance;
r.max_output_current = p.maxOutputCurrent;
r.fast_fuse_timeout = p.fastFuseTimeout;
r.fast_fuse_repetition = p.fastFuseRepetition;
r.slow_fuse_timeout = p.slowFuseTimeout;
r.modifier = static_cast<std::uint8_t>( p.modifier );
m_power.push_back( std::move( r ) );
}
}
void convert_memcells() {
for( auto const &m : m_module.scene.memcells ) {
memcell_record r;
r.node = node( m.node );
r.text = opt_str( m.text );
r.value1 = m.value1;
r.value2 = m.value2;
if( m.trackName.has_value() ) {
r.has_track = true;
r.track_name = str( *m.trackName );
}
m_memcells.push_back( std::move( r ) );
}
}
void convert_launchers() {
for( auto const &l : m_module.scene.eventLaunchers ) {
launcher_record r;
r.node = node( l.node );
r.location = { l.location.x, l.location.y, l.location.z };
r.radius_squared = l.radiusSquared;
r.activation_key = l.activationKey;
r.delta_time = l.deltaTime;
r.event1_name = opt_str( l.event1Name );
r.event2_name = opt_str( l.event2Name );
if( l.condition.has_value() ) {
r.has_condition = true;
r.condition.memcell_name = str( l.condition->memcellName );
r.condition.compare_text = str( l.condition->compareText );
r.condition.compare_value1 = l.condition->compareValue1;
r.condition.compare_value2 = l.condition->compareValue2;
r.condition.check_mask = l.condition->checkMask;
}
r.train_triggered = l.trainTriggered;
r.launch_hour = l.launchHour;
r.launch_minute = l.launchMinute;
m_launchers.push_back( std::move( r ) );
}
}
void convert_events() {
for( auto const &e : m_module.scene.events ) {
event_record r;
r.name = opt_str( e.name );
r.type = static_cast<std::uint8_t>( e.type );
r.delay = e.delay;
r.delay_random = e.delayRandom;
r.delay_departure = e.delayDeparture;
r.ignored = e.ignored;
r.passive = e.passive;
r.targets.reserve( e.targets.size() );
for( auto const &t : e.targets ) {
r.targets.push_back( str( t ) );
}
r.payload.reserve( e.payload.size() );
for( auto const &kv : e.payload ) {
r.payload.emplace_back( str( kv.first ), str( kv.second ) );
}
m_events.push_back( std::move( r ) );
}
}
void convert_sounds() {
for( auto const &s : m_module.scene.sounds ) {
sound_record r;
r.node = node( s.node );
r.location = { s.location.x, s.location.y, s.location.z };
r.wav_file = opt_str( s.wavFile );
m_sounds.push_back( std::move( r ) );
}
}
void convert_dynamics() {
for( auto const &d : m_module.scene.dynamics ) {
dynamic_record r;
r.node = node( d.node );
r.data_folder = opt_str( d.dataFolder );
r.skin_file = opt_str( d.skinFile );
r.mmd_file = opt_str( d.mmdFile );
r.track_name = opt_str( d.trackName );
r.offset = d.offset;
r.driver_type = opt_str( d.driverType );
r.coupling = d.coupling;
r.coupling_raw = opt_str( d.couplingRaw );
r.coupling_params = opt_str( d.couplingParams );
r.velocity = d.velocity;
r.load_count = d.loadCount;
r.load_type = opt_str( d.loadType );
if( d.destination.has_value() ) {
r.has_destination = true;
r.destination = str( *d.destination );
}
if( d.trainsetIndex.has_value() ) {
r.has_trainset = true;
r.trainset_index = static_cast<std::uint32_t>( *d.trainsetIndex );
}
m_dynamics.push_back( std::move( r ) );
}
}
void convert_trainsets() {
for( auto const &t : m_module.scene.trainsets ) {
trainset_record r;
r.name = opt_str( t.name );
r.track = opt_str( t.track );
r.offset = t.offset;
r.velocity = t.velocity;
r.assignment.reserve( t.assignment.size() );
for( auto const &kv : t.assignment ) {
r.assignment.emplace_back( str( kv.first ), str( kv.second ) );
}
r.vehicle_indices.reserve( t.vehicleIndices.size() );
for( auto const idx : t.vehicleIndices ) {
r.vehicle_indices.push_back( static_cast<std::uint32_t>( idx ) );
}
r.couplings.reserve( t.couplings.size() );
for( auto const c : t.couplings ) {
r.couplings.push_back( c );
}
r.driver_index =
t.driverIndex == static_cast<std::size_t>( -1 )
? 0xffffffffu
: static_cast<std::uint32_t>( t.driverIndex );
m_trainsets.push_back( std::move( r ) );
}
}
[[nodiscard]] transform_record transform( rt::TransformContext const &t ) {
transform_record out;
out.origin_stack.reserve( t.originStack.size() );
for( auto const &v : t.originStack ) {
out.origin_stack.push_back( { v.x, v.y, v.z } );
}
out.scale_stack.reserve( t.scaleStack.size() );
for( auto const &v : t.scaleStack ) {
out.scale_stack.push_back( { v.x, v.y, v.z } );
}
out.rotation = { t.rotation.x, t.rotation.y, t.rotation.z };
out.group_depth = static_cast<std::uint32_t>( t.groupStackDepth );
return out;
}
void convert_includes() {
for( auto const &inc : m_module.includes ) {
if( is_model_inc_placement( inc ) ) {
double x { 0.0 }, y { 0.0 }, z { 0.0 };
float rot_y { 0.f };
if( !try_parse_inc_placement( inc.parameters, x, y, z, rot_y ) ) {
continue;
}
module_placement_record p;
p.module_path = str(
binary_path_from_text( std::filesystem::path { inc.sourcePath } )
.generic_string() );
p.texture_override =
( inc.parameters.empty() || inc.parameters[ 0 ] == "none" )
? kNoString
: str( inc.parameters[ 0 ] );
p.x = x;
p.y = y;
p.z = z;
p.rotation_y = rot_y;
m_placements.push_back( std::move( p ) );
continue;
}
include_record r;
r.source_line = inc.sourceLine;
r.source_path = opt_str( inc.sourcePath );
r.binary_path = inc.sourcePath.empty()
? kNoString
: str( binary_path_from_text(
std::filesystem::path { inc.sourcePath } )
.generic_string() );
r.parameters.reserve( inc.parameters.size() );
for( auto const &param : inc.parameters ) {
r.parameters.push_back( str( param ) );
}
r.site_transform = transform( inc.siteTransform );
m_includes.push_back( std::move( r ) );
}
}
void convert_meta() {
m_meta.first_init_count = m_module.scene.firstInitCount;
m_meta.has_terrain_chunk = false;
// Root PACK batches are flattened into plain INST records, so the file
// carries no PACK streaming chunk; never advertise one.
m_meta.has_pack_chunk = false;
m_meta.placement_origin_x = m_module.includePlacement.origin_x_param;
m_meta.placement_origin_y = m_module.includePlacement.origin_y_param;
m_meta.placement_origin_z = m_module.includePlacement.origin_z_param;
m_meta.placement_rotation_y = m_module.includePlacement.rotation_y_param;
}
[[nodiscard]] std::vector<std::uint8_t> serialize() {
container_writer writer( m_file_kind );
byte_writer strs;
m_strings.serialize( strs );
writer.add_chunk( chunk::strs, strs );
byte_writer meta;
write_meta( meta, m_meta );
writer.add_chunk( chunk::meta, meta );
auto emit { [&]( std::uint32_t id, auto const &write_fn, auto const &items ) {
if( items.empty() ) {
return;
}
byte_writer payload;
write_fn( payload, items );
writer.add_chunk( id, payload );
} };
emit( chunk::incl, &write_includes, m_includes );
emit( chunk::plce, &write_module_placements, m_placements );
emit( chunk::prot, &write_prototypes, m_prototypes );
emit( chunk::inst, &write_instances, m_instances );
if( m_shape_spool != nullptr && m_shape_spool->shape_count() != 0 ) {
byte_writer payload;
payload.put_u32( static_cast<std::uint32_t>( m_shape_spool->shape_count() ) );
m_shape_spool->for_each_shape( [&]( rt::RuntimeShapeNode const &shape ) {
write_shape_record( payload, make_shape_record( shape ) );
} );
writer.add_chunk( chunk::shpe, payload );
} else {
emit( chunk::shpe, &write_shapes, m_shapes );
}
emit( chunk::line, &write_lines, m_lines );
emit( chunk::trak, &write_tracks, m_tracks );
emit( chunk::trac, &write_traction, m_traction );
emit( chunk::pwrs, &write_power_sources, m_power );
emit( chunk::memc, &write_memcells, m_memcells );
emit( chunk::laun, &write_launchers, m_launchers );
emit( chunk::evnt, &write_events, m_events );
emit( chunk::sond, &write_sounds, m_sounds );
emit( chunk::dynm, &write_dynamics, m_dynamics );
emit( chunk::trst, &write_trainsets, m_trainsets );
return writer.data();
}
bake::RuntimeModule const &m_module;
bool m_is_root;
std::vector<codec::ModelSectionBatch> const *m_pack_batches;
bake::PackModelSpoolFile const *m_pack_spool;
bake::ShapeSpoolFile const *m_shape_spool;
file_kind m_file_kind { file_kind::sim };
std::vector<module_placement_record> m_placements;
string_table m_strings;
std::vector<model_prototype> m_prototypes;
std::unordered_map<std::string, std::uint32_t> m_prototype_lookup;
std::vector<model_instance> m_instances;
std::vector<shape_record> m_shapes;
std::vector<lines_record> m_lines;
std::vector<track_record> m_tracks;
std::vector<traction_record> m_traction;
std::vector<power_source_record> m_power;
std::vector<memcell_record> m_memcells;
std::vector<launcher_record> m_launchers;
std::vector<event_record> m_events;
std::vector<sound_record> m_sounds;
std::vector<dynamic_record> m_dynamics;
std::vector<trainset_record> m_trainsets;
std::vector<include_record> m_includes;
module_meta m_meta;
};
// Decoded record counts of an emitted eu7v2 image (chunk-by-chunk).
struct decoded_counts {
std::size_t includes { 0 };
std::size_t placements { 0 };
std::size_t instances { 0 };
std::size_t shapes { 0 };
std::size_t lines { 0 };
std::size_t tracks { 0 };
std::size_t traction { 0 };
std::size_t power { 0 };
std::size_t memcells { 0 };
std::size_t launchers { 0 };
std::size_t events { 0 };
std::size_t sounds { 0 };
std::size_t dynamics { 0 };
std::size_t trainsets { 0 };
bool ok { true };
};
[[nodiscard]] decoded_counts decode_counts( std::vector<std::uint8_t> const &bytes ) {
decoded_counts c;
try {
container_reader reader( bytes.data(), bytes.size() );
chunk_view chunk;
while( reader.next( chunk ) ) {
auto r { chunk.reader() };
switch( chunk.id ) {
case chunk::incl: c.includes = read_includes( r ).size(); break;
case chunk::plce: c.placements = read_module_placements( r ).size(); break;
case chunk::inst: c.instances = read_instances( r ).size(); break;
case chunk::shpe: c.shapes = read_shapes( r ).size(); break;
case chunk::line: c.lines = read_lines( r ).size(); break;
case chunk::trak: c.tracks = read_tracks( r ).size(); break;
case chunk::trac: c.traction = read_traction( r ).size(); break;
case chunk::pwrs: c.power = read_power_sources( r ).size(); break;
case chunk::memc: c.memcells = read_memcells( r ).size(); break;
case chunk::laun: c.launchers = read_launchers( r ).size(); break;
case chunk::evnt: c.events = read_events( r ).size(); break;
case chunk::sond: c.sounds = read_sounds( r ).size(); break;
case chunk::dynm: c.dynamics = read_dynamics( r ).size(); break;
case chunk::trst: c.trainsets = read_trainsets( r ).size(); break;
default: break;
}
}
}
catch( parse_error const & ) {
c.ok = false;
}
return c;
}
[[nodiscard]] std::filesystem::path eu7v2_path_for( std::filesystem::path const &text_path ) {
return binary_path_from_text( text_path );
}
} // namespace
std::vector<std::uint8_t>
emit_runtime_module_bytes(
bake::RuntimeModule const &module,
bool const is_root,
std::vector<codec::ModelSectionBatch> const *pack_batches,
bake::PackModelSpoolFile const *pack_spool,
bake::ShapeSpoolFile const *shape_spool,
std::filesystem::path const &text_path ) {
runtime_baker baker( module, is_root, pack_batches, pack_spool, shape_spool, text_path );
return baker.run();
}
emit_outcome
emit_runtime_module(
bake::RuntimeModule const &module,
std::filesystem::path const &text_path,
bool const is_root,
std::vector<codec::ModelSectionBatch> const *pack_batches,
bool const verify,
bake::PackModelSpoolFile const *pack_spool,
bake::ShapeSpoolFile const *shape_spool ) {
emit_outcome outcome;
auto const ms_since { []( std::chrono::steady_clock::time_point const t0 ) {
return std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - t0 )
.count();
} };
auto const build_begin { std::chrono::steady_clock::now() };
runtime_baker baker( module, is_root, pack_batches, pack_spool, shape_spool, text_path );
std::vector<std::uint8_t> const bytes { baker.run() };
outcome.model_total = baker.model_total();
outcome.byte_size = bytes.size();
outcome.build_ms = ms_since( build_begin );
auto const out_path { eu7v2_path_for( text_path ) };
outcome.written_path = out_path.generic_string();
{
auto const write_begin { std::chrono::steady_clock::now() };
std::ofstream output { out_path, std::ios::binary | std::ios::trunc };
if( !output ) {
outcome.ok = false;
outcome.message = "nie mozna zapisac " + outcome.written_path;
return outcome;
}
output.write(
reinterpret_cast<char const *>( bytes.data() ),
static_cast<std::streamsize>( bytes.size() ) );
output.flush();
outcome.write_ms = ms_since( write_begin );
}
if( !verify ) {
return outcome;
}
auto const verify_begin { std::chrono::steady_clock::now() };
outcome.verified = true;
decoded_counts const dec { decode_counts( bytes ) };
std::size_t src_models { module.scene.models.size() };
if( is_root && pack_spool != nullptr ) {
src_models += pack_spool->model_count();
}
if( is_root && pack_batches != nullptr ) {
for( auto const &batch : *pack_batches ) {
src_models += batch.models.size();
}
}
std::size_t src_placements { 0 };
std::size_t src_structural { 0 };
for( auto const &inc : module.includes ) {
if( is_model_inc_placement( inc ) ) {
++src_placements;
} else {
++src_structural;
}
}
std::ostringstream report;
bool pass { dec.ok };
auto cmp { [&]( char const *label, std::size_t const src, std::size_t const got ) {
bool const ok { src == got };
pass = pass && ok;
report << " " << ( ok ? "ok " : "FAIL" ) << ' ' << label << " src=" << src
<< " eu7v2=" << got << '\n';
} };
cmp( "includes", src_structural, dec.includes );
cmp( "placements", src_placements, dec.placements );
cmp( "models", src_models, dec.instances );
cmp( "shapes", module.scene.shapes.size() + ( shape_spool != nullptr ? shape_spool->shape_count() : 0 ),
dec.shapes );
cmp( "lines", module.scene.lines.size(), dec.lines );
cmp( "tracks", module.scene.tracks.size(), dec.tracks );
cmp( "traction", module.scene.traction.size(), dec.traction );
cmp( "power", module.scene.powerSources.size(), dec.power );
cmp( "memcells", module.scene.memcells.size(), dec.memcells );
cmp( "launchers", module.scene.eventLaunchers.size(), dec.launchers );
cmp( "events", module.scene.events.size(), dec.events );
cmp( "sounds", module.scene.sounds.size(), dec.sounds );
cmp( "dynamics", module.scene.dynamics.size(), dec.dynamics );
cmp( "trainsets", module.scene.trainsets.size(), dec.trainsets );
outcome.verify_ok = pass;
outcome.message = report.str();
outcome.verify_ms = ms_since( verify_begin );
return outcome;
}
bool
verify_written_module(
std::filesystem::path const &path,
module_verify_spec const &spec,
bool const is_root,
std::size_t const pack_models,
std::string *message_out ) {
std::ifstream input { path, std::ios::binary };
if( !input ) {
if( message_out != nullptr ) {
*message_out = "nie mozna odczytac " + path.generic_string();
}
return false;
}
input.seekg( 0, std::ios::end );
const std::streamoff file_size { input.tellg() };
input.seekg( 0, std::ios::beg );
if( file_size <= 0 ) {
if( message_out != nullptr ) {
*message_out = "pusty plik " + path.generic_string();
}
return false;
}
std::vector<std::uint8_t> bytes( static_cast<std::size_t>( file_size ) );
input.read(
reinterpret_cast<char *>( bytes.data() ),
static_cast<std::streamsize>( bytes.size() ) );
decoded_counts const dec { decode_counts( bytes ) };
if( !dec.ok ) {
if( message_out != nullptr ) {
*message_out = "decode blad " + path.generic_string();
}
return false;
}
std::size_t src_models { spec.models };
if( is_root ) {
src_models += pack_models;
}
std::ostringstream report;
bool pass { dec.ok };
auto cmp { [&]( char const *label, std::size_t const src, std::size_t const got ) {
bool const ok { src == got };
pass = pass && ok;
report << " " << ( ok ? "ok " : "FAIL" ) << ' ' << label << " src=" << src
<< " eu7v2=" << got << '\n';
} };
cmp( "includes", spec.includes, dec.includes );
cmp( "placements", spec.placements, dec.placements );
cmp( "models", src_models, dec.instances );
cmp( "shapes", spec.shapes, dec.shapes );
cmp( "lines", spec.lines, dec.lines );
cmp( "tracks", spec.tracks, dec.tracks );
cmp( "traction", spec.traction, dec.traction );
cmp( "power", spec.power, dec.power );
cmp( "memcells", spec.memcells, dec.memcells );
cmp( "launchers", spec.launchers, dec.launchers );
cmp( "events", spec.events, dec.events );
cmp( "sounds", spec.sounds, dec.sounds );
cmp( "dynamics", spec.dynamics, dec.dynamics );
cmp( "trainsets", spec.trainsets, dec.trainsets );
if( message_out != nullptr ) {
*message_out = report.str();
}
return pass;
}
} // namespace eu7v2

View File

@@ -0,0 +1,99 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#pragma once
// ---------------------------------------------------------------------------
// RuntimeScene -> eu7v2 emitter. Lives in the eu07_bake target because it
// touches the parser's RuntimeModule types directly. It maps Runtime* records
// into the same dependency-free eu7v2 structs that scene_baker uses (one
// writer/reader per record), so the produced .eu7v2 matches what the engine
// loads via eu7v2::load_module. For the scenario root the flattened PACK model
// batches are deduplicated into prototypes + instances.
// ---------------------------------------------------------------------------
#include <eu07/scene/bake/module.hpp>
#include <eu07/scene/binary/runtime_codec.hpp>
#include <cstddef>
#include <cstdint>
#include <filesystem>
#include <string>
#include <vector>
namespace eu07::scene::bake {
class PackModelSpoolFile;
class ShapeSpoolFile;
}
namespace eu7v2 {
struct emit_outcome {
bool ok { true };
bool verified { false };
bool verify_ok { true };
std::string written_path;
std::string message;
std::size_t model_total { 0 };
std::size_t byte_size { 0 };
double build_ms { 0.0 };
double write_ms { 0.0 };
double verify_ms { 0.0 };
};
struct module_verify_spec {
std::size_t includes { 0 };
std::size_t placements { 0 };
std::size_t models { 0 };
std::size_t shapes { 0 };
std::size_t lines { 0 };
std::size_t tracks { 0 };
std::size_t traction { 0 };
std::size_t power { 0 };
std::size_t memcells { 0 };
std::size_t launchers { 0 };
std::size_t events { 0 };
std::size_t sounds { 0 };
std::size_t dynamics { 0 };
std::size_t trainsets { 0 };
};
// Builds the eu7v2 byte image for one baked module.
[[nodiscard]] std::vector<std::uint8_t>
emit_runtime_module_bytes(
eu07::scene::bake::RuntimeModule const &module,
bool is_root,
std::vector<eu07::scene::binary::codec::ModelSectionBatch> const *pack_batches,
eu07::scene::bake::PackModelSpoolFile const *pack_spool = nullptr,
eu07::scene::bake::ShapeSpoolFile const *shape_spool = nullptr,
std::filesystem::path const &text_path = {} );
// Emits the module, writes "<stem>.eu7v2" next to the text source, and (when
// verify is true) reloads the bytes and compares record counts against the
// source RuntimeModule, filling outcome.message with a PASS/FAIL report.
[[nodiscard]] emit_outcome
emit_runtime_module(
eu07::scene::bake::RuntimeModule const &module,
std::filesystem::path const &text_path,
bool is_root,
std::vector<eu07::scene::binary::codec::ModelSectionBatch> const *pack_batches,
bool verify,
eu07::scene::bake::PackModelSpoolFile const *pack_spool = nullptr,
eu07::scene::bake::ShapeSpoolFile const *shape_spool = nullptr );
// Porownanie zapisanej eu7v2 z oczekiwanymi licznikami (do odroczonego verify).
[[nodiscard]] bool
verify_written_module(
std::filesystem::path const &path,
module_verify_spec const &spec,
bool const is_root,
std::size_t const pack_models,
std::string *message_out = nullptr );
} // namespace eu7v2

383
scene/eu7/v2/eu7v2_format.h Normal file
View File

@@ -0,0 +1,383 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#pragma once
// ---------------------------------------------------------------------------
// eu7v2 - clean-slate compiled scenery format core (no backward compatibility
// with the legacy EU7B v4..v13 chunks).
//
// Design goals:
// * self-describing, skippable chunks: [u32 id][u64 size][payload]
// * explicit little-endian on disk so files are deterministic & portable
// * a central string table so payloads reference strings by u32 id
// * a separation of file kinds: sim core / reusable module / streamable tile
// * header-only and dependency-free so it can be unit tested in isolation
//
// This is the foundation layer only (iteration 1). Higher layers (baker,
// runtime loader, streaming) build on top of these primitives.
// ---------------------------------------------------------------------------
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <filesystem>
#include <stdexcept>
#include <string>
#include <string_view>
#include <unordered_map>
#include <vector>
namespace eu7v2 {
// FourCC packed little-endian: 'A' is the least significant byte so the bytes
// appear in source order when the file is viewed in a hex editor.
[[nodiscard]] constexpr std::uint32_t
fourcc( char const a, char const b, char const c, char const d ) noexcept {
return ( static_cast<std::uint32_t>( static_cast<unsigned char>( a ) ) )
| ( static_cast<std::uint32_t>( static_cast<unsigned char>( b ) ) << 8 )
| ( static_cast<std::uint32_t>( static_cast<unsigned char>( c ) ) << 16 )
| ( static_cast<std::uint32_t>( static_cast<unsigned char>( d ) ) << 24 );
}
// File magic + single current version. No legacy versions are recognised.
constexpr std::uint32_t kMagic { fourcc( 'E', 'U', '7', 'C' ) };
constexpr std::uint16_t kVersion { 1 };
enum class file_kind : std::uint16_t {
sim = 1, // global simulation core, loaded once
module = 2, // reusable module / prototype library (baked .inc)
tile = 3, // streamable 1km visual tile (terrain + instances)
manifest = 4, // index of tiles & modules for a map
};
// Chunk identifiers (FourCC). Kept small & explicit; grows as layers land.
namespace chunk {
constexpr std::uint32_t strs { fourcc( 'S', 'T', 'R', 'S' ) }; // string table
constexpr std::uint32_t meta { fourcc( 'M', 'E', 'T', 'A' ) }; // key/value metadata
constexpr std::uint32_t prot { fourcc( 'P', 'R', 'O', 'T' ) }; // model prototypes (module)
constexpr std::uint32_t inst { fourcc( 'I', 'N', 'S', 'T' ) }; // lean instances (tile)
constexpr std::uint32_t mesh { fourcc( 'M', 'E', 'S', 'H' ) }; // baked terrain mesh (tile)
constexpr std::uint32_t shpe { fourcc( 'S', 'H', 'P', 'E' ) }; // non-terrain shapes (triangles)
constexpr std::uint32_t line { fourcc( 'L', 'I', 'N', 'E' ) }; // line geometry nodes
constexpr std::uint32_t incl { fourcc( 'I', 'N', 'C', 'L' ) }; // module includes (recursion refs)
constexpr std::uint32_t plce { fourcc( 'P', 'L', 'C', 'E' ) }; // lean .inc placements (binary coords)
constexpr std::uint32_t trst { fourcc( 'T', 'R', 'S', 'T' ) }; // trainsets
constexpr std::uint32_t trgr { fourcc( 'T', 'R', 'G', 'R' ) }; // precomputed track graph (sim)
constexpr std::uint32_t sidx { fourcc( 'S', 'I', 'D', 'X' ) }; // spatial section index (sim/manifest)
// simulation record chunks (sim)
constexpr std::uint32_t trak { fourcc( 'T', 'R', 'A', 'K' ) }; // tracks
constexpr std::uint32_t trac { fourcc( 'T', 'R', 'A', 'C' ) }; // traction wires
constexpr std::uint32_t pwrs { fourcc( 'P', 'W', 'R', 'S' ) }; // traction power sources
constexpr std::uint32_t memc { fourcc( 'M', 'E', 'M', 'C' ) }; // memory cells
constexpr std::uint32_t laun { fourcc( 'L', 'A', 'U', 'N' ) }; // event launchers
constexpr std::uint32_t evnt { fourcc( 'E', 'V', 'N', 'T' ) }; // events
constexpr std::uint32_t sond { fourcc( 'S', 'O', 'N', 'D' ) }; // sounds
constexpr std::uint32_t dynm { fourcc( 'D', 'Y', 'N', 'M' ) }; // dynamic vehicles
} // namespace chunk
// ---------------------------------------------------------------------------
// Writer side
// ---------------------------------------------------------------------------
// Little-endian byte sink. All multi-byte values are written explicitly so the
// output does not depend on host endianness.
class byte_writer {
public:
void put_u8( std::uint8_t const v ) { m_data.push_back( v ); }
void put_u16( std::uint16_t const v ) {
put_u8( static_cast<std::uint8_t>( v ) );
put_u8( static_cast<std::uint8_t>( v >> 8 ) );
}
void put_u32( std::uint32_t const v ) {
put_u16( static_cast<std::uint16_t>( v ) );
put_u16( static_cast<std::uint16_t>( v >> 16 ) );
}
void put_u64( std::uint64_t const v ) {
put_u32( static_cast<std::uint32_t>( v ) );
put_u32( static_cast<std::uint32_t>( v >> 32 ) );
}
void put_i32( std::int32_t const v ) {
put_u32( static_cast<std::uint32_t>( v ) );
}
void put_f32( float const v ) {
std::uint32_t bits;
std::memcpy( &bits, &v, sizeof( bits ) );
put_u32( bits );
}
// doubles are stored as f32 on disk (positions excepted via put_f64 where
// the extra precision actually matters)
void put_f64( double const v ) {
std::uint64_t bits;
std::memcpy( &bits, &v, sizeof( bits ) );
put_u64( bits );
}
void put_vec3f( double const x, double const y, double const z ) {
put_f32( static_cast<float>( x ) );
put_f32( static_cast<float>( y ) );
put_f32( static_cast<float>( z ) );
}
void put_bytes( void const *data, std::size_t const size ) {
auto const *bytes { static_cast<std::uint8_t const *>( data ) };
m_data.insert( m_data.end(), bytes, bytes + size );
}
[[nodiscard]] std::vector<std::uint8_t> const &data() const noexcept { return m_data; }
[[nodiscard]] std::vector<std::uint8_t> &data() noexcept { return m_data; }
[[nodiscard]] std::size_t size() const noexcept { return m_data.size(); }
private:
std::vector<std::uint8_t> m_data;
};
// Deduplicating string table. Returns a stable u32 id per unique string.
class string_table {
public:
[[nodiscard]] std::uint32_t intern( std::string_view const s ) {
auto const key { std::string( s ) };
auto const it { m_lookup.find( key ) };
if( it != m_lookup.end() ) {
return it->second;
}
auto const id { static_cast<std::uint32_t>( m_strings.size() ) };
m_strings.emplace_back( key );
m_lookup.emplace( key, id );
return id;
}
[[nodiscard]] std::size_t size() const noexcept { return m_strings.size(); }
// Serialise as: [u32 count]( [u32 len][bytes] )*
void serialize( byte_writer &out ) const {
out.put_u32( static_cast<std::uint32_t>( m_strings.size() ) );
for( auto const &s : m_strings ) {
out.put_u32( static_cast<std::uint32_t>( s.size() ) );
out.put_bytes( s.data(), s.size() );
}
}
private:
std::vector<std::string> m_strings;
std::unordered_map<std::string, std::uint32_t> m_lookup;
};
// Top-level container writer: file header followed by [id][size][payload] chunks.
class container_writer {
public:
explicit container_writer( file_kind const kind ) : m_kind( kind ) {
m_out.put_u32( kMagic );
m_out.put_u16( kVersion );
m_out.put_u16( static_cast<std::uint16_t>( kind ) );
// reserved for flags / future use, keeps header 16-byte aligned
m_out.put_u32( 0 );
m_out.put_u32( 0 );
}
// Append a fully-built chunk payload under the given id.
void add_chunk( std::uint32_t const id, std::vector<std::uint8_t> const &payload ) {
m_out.put_u32( id );
m_out.put_u64( static_cast<std::uint64_t>( payload.size() ) );
m_out.put_bytes( payload.data(), payload.size() );
}
void add_chunk( std::uint32_t const id, byte_writer const &payload ) {
add_chunk( id, payload.data() );
}
[[nodiscard]] file_kind kind() const noexcept { return m_kind; }
[[nodiscard]] std::vector<std::uint8_t> const &data() const noexcept { return m_out.data(); }
private:
file_kind m_kind;
byte_writer m_out;
};
// ---------------------------------------------------------------------------
// Reader side (bounds-checked)
// ---------------------------------------------------------------------------
class parse_error : public std::runtime_error {
public:
explicit parse_error( std::string const &what ) : std::runtime_error( what ) {}
};
// Little-endian cursor over a byte span. Every read is bounds-checked and
// throws parse_error on truncation so a corrupt file can never read OOB.
class byte_reader {
public:
byte_reader( std::uint8_t const *data, std::size_t const size ) noexcept
: m_data( data ), m_size( size ) {}
[[nodiscard]] std::size_t remaining() const noexcept { return m_size - m_pos; }
[[nodiscard]] bool empty() const noexcept { return m_pos >= m_size; }
[[nodiscard]] std::size_t position() const noexcept { return m_pos; }
std::uint8_t get_u8() {
require( 1 );
return m_data[ m_pos++ ];
}
std::uint16_t get_u16() {
std::uint16_t const lo { get_u8() };
std::uint16_t const hi { get_u8() };
return static_cast<std::uint16_t>( lo | ( hi << 8 ) );
}
std::uint32_t get_u32() {
std::uint32_t const lo { get_u16() };
std::uint32_t const hi { get_u16() };
return lo | ( hi << 16 );
}
std::uint64_t get_u64() {
std::uint64_t const lo { get_u32() };
std::uint64_t const hi { get_u32() };
return lo | ( hi << 32 );
}
std::int32_t get_i32() { return static_cast<std::int32_t>( get_u32() ); }
float get_f32() {
std::uint32_t const bits { get_u32() };
float v;
std::memcpy( &v, &bits, sizeof( v ) );
return v;
}
double get_f64() {
std::uint64_t const bits { get_u64() };
double v;
std::memcpy( &v, &bits, sizeof( v ) );
return v;
}
void get_bytes( void *dst, std::size_t const size ) {
require( size );
std::memcpy( dst, m_data + m_pos, size );
m_pos += size;
}
[[nodiscard]] std::uint8_t const *take( std::size_t const size ) {
require( size );
auto const *ptr { m_data + m_pos };
m_pos += size;
return ptr;
}
private:
void require( std::size_t const n ) const {
if( m_pos + n > m_size ) {
throw parse_error( "eu7v2: unexpected end of data" );
}
}
std::uint8_t const *m_data;
std::size_t m_size;
std::size_t m_pos { 0 };
};
// Decoded string table: id -> string view into the owned backing store.
class string_pool {
public:
void deserialize( byte_reader &in ) {
auto const count { in.get_u32() };
m_strings.clear();
m_strings.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
auto const len { in.get_u32() };
std::string s;
s.resize( len );
if( len != 0 ) {
in.get_bytes( s.data(), len );
}
m_strings.emplace_back( std::move( s ) );
}
}
[[nodiscard]] std::string const &get( std::uint32_t const id ) const {
if( id >= m_strings.size() ) {
throw parse_error( "eu7v2: string id out of range" );
}
return m_strings[ id ];
}
[[nodiscard]] std::size_t size() const noexcept { return m_strings.size(); }
private:
std::vector<std::string> m_strings;
};
// One chunk located inside a container: its id and a view of its payload.
struct chunk_view {
std::uint32_t id { 0 };
std::uint8_t const *data { nullptr };
std::size_t size { 0 };
[[nodiscard]] byte_reader reader() const noexcept { return byte_reader( data, size ); }
};
// Reads the file header and iterates chunks without copying payloads.
class container_reader {
public:
container_reader( std::uint8_t const *data, std::size_t const size ) : m_cursor( data, size ) {
auto const magic { m_cursor.get_u32() };
if( magic != kMagic ) {
throw parse_error( "eu7v2: bad magic" );
}
m_version = m_cursor.get_u16();
if( m_version != kVersion ) {
throw parse_error( "eu7v2: unsupported version" );
}
m_kind = static_cast<file_kind>( m_cursor.get_u16() );
(void)m_cursor.get_u32(); // reserved
(void)m_cursor.get_u32(); // reserved
}
[[nodiscard]] file_kind kind() const noexcept { return m_kind; }
[[nodiscard]] std::uint16_t version() const noexcept { return m_version; }
// Returns false when there are no more chunks.
[[nodiscard]] bool next( chunk_view &out ) {
if( m_cursor.remaining() == 0 ) {
return false;
}
out.id = m_cursor.get_u32();
auto const size { m_cursor.get_u64() };
out.size = static_cast<std::size_t>( size );
out.data = m_cursor.take( out.size );
return true;
}
private:
byte_reader m_cursor;
file_kind m_kind { file_kind::sim };
std::uint16_t m_version { 0 };
};
// Maps a text scenery source to its compiled .eu7v2 path. Standard module
// extensions (.scm/.scn/.sbt/.inc) become "<stem>.eu7v2"; other extensions
// (.ctr, …) keep the source suffix so e.g. foo.scm and foo.ctr do not collide.
[[nodiscard]] inline std::filesystem::path
binary_path_from_text( std::filesystem::path const &text_path ) {
auto const ext { text_path.extension().string() };
if( ext == ".scm" || ext == ".scn" || ext == ".sbt" || ext == ".inc" ) {
return text_path.parent_path() / ( text_path.stem().string() + ".eu7v2" );
}
return text_path.parent_path() / ( text_path.filename().string() + ".eu7v2" );
}
} // namespace eu7v2

515
scene/eu7/v2/eu7v2_load.cpp Normal file
View File

@@ -0,0 +1,515 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#include "scene/eu7/v2/eu7v2_load.h"
#include "scene/eu7/v2/eu7v2_format.h"
#include "scene/eu7/v2/eu7v2_scene.h"
#include "scene/eu7/v2/eu7v2_records.h"
#include "scene/eu7/eu7_types.h"
#include <string>
namespace eu7v2 {
namespace {
// Reverse of scene_baker: turns decoded eu7v2 structs back into Eu7Scene.
class scene_loader {
public:
scene_loader( string_pool const &pool, scene::eu7::Eu7Scene &out )
: m_pool( pool ), m_out( out ) {}
[[nodiscard]] std::string str( std::uint32_t const id ) const {
return id == kNoString ? std::string() : m_pool.get( id );
}
void apply_node( scene::eu7::Eu7BasicNode &n, node_record const &r ) const {
n.name = str( r.name );
n.node_type = str( r.type );
n.area.center = { r.area_center.x, r.area_center.y, r.area_center.z };
n.area.radius = r.area_radius;
n.range_squared_min = r.range_sq_min;
n.range_squared_max = r.range_sq_max;
n.visible = r.visible;
}
void build_models(
std::vector<model_prototype> const &protos,
std::vector<model_instance> const &instances ) const {
m_out.models.reserve( instances.size() );
for( auto const &inst : instances ) {
if( inst.proto >= protos.size() ) {
continue;
}
auto const &proto { protos[ inst.proto ] };
scene::eu7::Eu7Model m;
m.location = { inst.x, inst.y, inst.z };
m.angles = { inst.ax, inst.ay, inst.az };
m.scale = { inst.sx, inst.sy, inst.sz };
m.model_file = str( proto.model_file );
m.texture_file =
inst.texture_override != kNoString ? str( inst.texture_override )
: str( proto.texture_file );
m.light_states = proto.light_states;
m.light_colors = proto.light_colors;
m.transition = ( proto.flags & proto_flag::transition ) != 0;
m.is_terrain = ( proto.flags & proto_flag::is_terrain ) != 0;
m.pack_flags =
( proto.flags & proto_flag::instanceable ) ? scene::eu7::kEu7PackFlagInstanceableHint : 0u;
m.baked_range_min = proto.range_min;
m.baked_range_max = proto.range_max;
m.pack_cell_id = inst.cell_id;
if( inst.has_node ) {
apply_node( m.node, inst.node );
}
else {
m.node.name = m.model_file;
m.node.node_type = "model";
m.node.area.center = m.location;
}
m_out.models.push_back( std::move( m ) );
}
}
static void apply_lighting(
scene::eu7::Eu7LightingData &dst, lighting_block const &src ) {
dst.diffuse = { src.diffuse[ 0 ], src.diffuse[ 1 ], src.diffuse[ 2 ], src.diffuse[ 3 ] };
dst.ambient = { src.ambient[ 0 ], src.ambient[ 1 ], src.ambient[ 2 ], src.ambient[ 3 ] };
dst.specular = { src.specular[ 0 ], src.specular[ 1 ], src.specular[ 2 ], src.specular[ 3 ] };
}
void build_shapes( std::vector<shape_record> const &shapes ) const {
m_out.shapes.reserve( shapes.size() );
for( auto const &r : shapes ) {
scene::eu7::Eu7Shape s;
apply_node( s.node, r.node );
s.translucent = r.translucent;
s.material_path = str( r.material );
apply_lighting( s.lighting, r.lighting );
s.origin = { r.ox, r.oy, r.oz };
s.vertices.reserve( r.vertices.size() );
for( auto const &v : r.vertices ) {
scene::eu7::Eu7WorldVertex wv;
wv.position = {
r.ox + static_cast<double>( v.px ),
r.oy + static_cast<double>( v.py ),
r.oz + static_cast<double>( v.pz ) };
wv.normal = { v.nx, v.ny, v.nz };
wv.u = v.u;
wv.v = v.v;
s.vertices.push_back( wv );
}
m_out.shapes.push_back( std::move( s ) );
}
}
void build_lines( std::vector<lines_record> const &items ) const {
m_out.lines.reserve( items.size() );
for( auto const &r : items ) {
scene::eu7::Eu7Lines l;
apply_node( l.node, r.node );
apply_lighting( l.lighting, r.lighting );
l.line_width = r.line_width;
l.origin = { r.ox, r.oy, r.oz };
l.vertices.reserve( r.vertices.size() );
for( auto const &v : r.vertices ) {
scene::eu7::Eu7WorldVertex wv;
wv.position = { v.x, v.y, v.z };
l.vertices.push_back( wv );
}
m_out.lines.push_back( std::move( l ) );
}
}
void build_trainsets( std::vector<trainset_record> const &items ) const {
m_out.trainsets.reserve( items.size() );
for( auto const &r : items ) {
scene::eu7::Eu7Trainset t;
t.name = str( r.name );
t.track = str( r.track );
t.offset = r.offset;
t.velocity = r.velocity;
for( auto const &kv : r.assignment ) {
t.assignment.emplace( str( kv.first ), str( kv.second ) );
}
t.vehicle_indices.reserve( r.vehicle_indices.size() );
for( auto const idx : r.vehicle_indices ) {
t.vehicle_indices.push_back( static_cast<std::size_t>( idx ) );
}
t.couplings.reserve( r.couplings.size() );
for( auto const c : r.couplings ) {
t.couplings.push_back( c );
}
t.driver_index =
r.driver_index == 0xffffffffu
? static_cast<std::size_t>( -1 )
: static_cast<std::size_t>( r.driver_index );
m_out.trainsets.push_back( std::move( t ) );
}
}
void build_terrain( std::vector<terrain_mesh> const &meshes ) const {
m_out.terrain_shapes.reserve( meshes.size() );
for( auto const &mesh : meshes ) {
scene::eu7::Eu7Shape s;
s.material_path = str( mesh.material );
s.translucent = mesh.translucent;
s.origin = { mesh.ox, mesh.oy, mesh.oz };
s.vertices.reserve( mesh.vertices.size() );
for( auto const &v : mesh.vertices ) {
scene::eu7::Eu7WorldVertex wv;
wv.position = {
mesh.ox + static_cast<double>( v.px ),
mesh.oy + static_cast<double>( v.py ),
mesh.oz + static_cast<double>( v.pz ) };
wv.normal = { v.nx, v.ny, v.nz };
wv.u = v.u;
wv.v = v.v;
s.vertices.push_back( wv );
}
m_out.terrain_shapes.push_back( std::move( s ) );
}
}
void build_tracks( std::vector<track_record> const &tracks ) const {
m_out.tracks.reserve( tracks.size() );
for( auto const &r : tracks ) {
scene::eu7::Eu7Track t;
apply_node( t.node, r.node );
t.track_type = static_cast<scene::eu7::Eu7TrackType>( r.track_type );
t.category = static_cast<scene::eu7::Eu7TrackCategory>( r.category );
t.length = r.length;
t.track_width = r.track_width;
t.friction = r.friction;
t.sound_distance = r.sound_distance;
t.quality_flag = r.quality_flag;
t.damage_flag = r.damage_flag;
t.environment = static_cast<scene::eu7::Eu7TrackEnvironment>( r.environment );
if( r.has_visibility ) {
scene::eu7::Eu7TrackVisibility vis;
vis.material1 = str( r.visibility.material1 );
vis.tex_length = r.visibility.tex_length;
vis.material2 = str( r.visibility.material2 );
vis.tex_height1 = r.visibility.tex_height1;
vis.tex_width = r.visibility.tex_width;
vis.tex_slope = r.visibility.tex_slope;
t.visibility = vis;
}
t.paths.reserve( r.paths.size() );
for( auto const &p : r.paths ) {
scene::eu7::Eu7SegmentPath sp;
sp.p_start = { p.p_start.x, p.p_start.y, p.p_start.z };
sp.roll_start = p.roll_start;
sp.cp_out = { p.cp_out.x, p.cp_out.y, p.cp_out.z };
sp.cp_in = { p.cp_in.x, p.cp_in.y, p.cp_in.z };
sp.p_end = { p.p_end.x, p.p_end.y, p.p_end.z };
sp.roll_end = p.roll_end;
sp.radius = p.radius;
t.paths.push_back( sp );
}
t.tail_keywords.reserve( r.tail_keywords.size() );
for( auto const &kv : r.tail_keywords ) {
t.tail_keywords.emplace_back( str( kv.first ), str( kv.second ) );
}
m_out.tracks.push_back( std::move( t ) );
}
}
void build_traction( std::vector<traction_record> const &items ) const {
m_out.traction.reserve( items.size() );
for( auto const &r : items ) {
scene::eu7::Eu7Traction t;
apply_node( t.node, r.node );
t.power_supply_name = str( r.power_supply_name );
t.nominal_voltage = r.nominal_voltage;
t.max_current = r.max_current;
t.resistivity_ohm_per_m = r.resistivity;
t.material = static_cast<scene::eu7::Eu7TractionWireMaterial>( r.material );
t.wire_thickness = r.wire_thickness;
t.damage_flag = r.damage_flag;
t.wire_p1 = { r.wire_p1.x, r.wire_p1.y, r.wire_p1.z };
t.wire_p2 = { r.wire_p2.x, r.wire_p2.y, r.wire_p2.z };
t.wire_p3 = { r.wire_p3.x, r.wire_p3.y, r.wire_p3.z };
t.wire_p4 = { r.wire_p4.x, r.wire_p4.y, r.wire_p4.z };
t.min_height = r.min_height;
t.segment_length = r.segment_length;
t.wire_count = r.wire_count;
t.wire_offset = r.wire_offset;
if( r.has_parallel ) {
t.parallel_name = str( r.parallel_name );
}
m_out.traction.push_back( std::move( t ) );
}
}
void build_power_sources( std::vector<power_source_record> const &items ) const {
m_out.power_sources.reserve( items.size() );
for( auto const &r : items ) {
scene::eu7::Eu7TractionPowerSource p;
apply_node( p.node, r.node );
p.position = { r.position.x, r.position.y, r.position.z };
p.nominal_voltage = r.nominal_voltage;
p.voltage_frequency = r.voltage_frequency;
p.internal_resistance = r.internal_resistance;
p.max_output_current = r.max_output_current;
p.fast_fuse_timeout = r.fast_fuse_timeout;
p.fast_fuse_repetition = r.fast_fuse_repetition;
p.slow_fuse_timeout = r.slow_fuse_timeout;
p.modifier = static_cast<scene::eu7::Eu7PowerSourceModifier>( r.modifier );
m_out.power_sources.push_back( std::move( p ) );
}
}
void build_memcells( std::vector<memcell_record> const &items ) const {
m_out.memcells.reserve( items.size() );
for( auto const &r : items ) {
scene::eu7::Eu7MemCell m;
apply_node( m.node, r.node );
m.text = str( r.text );
m.value1 = r.value1;
m.value2 = r.value2;
if( r.has_track ) {
m.track_name = str( r.track_name );
}
m_out.memcells.push_back( std::move( m ) );
}
}
void build_launchers( std::vector<launcher_record> const &items ) const {
m_out.event_launchers.reserve( items.size() );
for( auto const &r : items ) {
scene::eu7::Eu7EventLauncher l;
apply_node( l.node, r.node );
l.location = { r.location.x, r.location.y, r.location.z };
l.radius_squared = r.radius_squared;
l.activation_key = r.activation_key;
l.delta_time = r.delta_time;
l.event1_name = str( r.event1_name );
l.event2_name = str( r.event2_name );
if( r.has_condition ) {
scene::eu7::Eu7EventLauncherCondition c;
c.memcell_name = str( r.condition.memcell_name );
c.compare_text = str( r.condition.compare_text );
c.compare_value1 = r.condition.compare_value1;
c.compare_value2 = r.condition.compare_value2;
c.check_mask = r.condition.check_mask;
l.condition = c;
}
l.train_triggered = r.train_triggered;
l.launch_hour = r.launch_hour;
l.launch_minute = r.launch_minute;
m_out.event_launchers.push_back( std::move( l ) );
}
}
void build_events( std::vector<event_record> const &items ) const {
m_out.events.reserve( items.size() );
for( auto const &r : items ) {
scene::eu7::Eu7Event e;
e.name = str( r.name );
e.type = static_cast<scene::eu7::Eu7EventType>( r.type );
e.delay = r.delay;
e.delay_random = r.delay_random;
e.delay_departure = r.delay_departure;
e.ignored = r.ignored;
e.passive = r.passive;
e.targets.reserve( r.targets.size() );
for( auto const t : r.targets ) {
e.targets.push_back( str( t ) );
}
e.payload.reserve( r.payload.size() );
for( auto const &kv : r.payload ) {
e.payload.emplace_back( str( kv.first ), str( kv.second ) );
}
m_out.events.push_back( std::move( e ) );
}
}
void build_sounds( std::vector<sound_record> const &items ) const {
m_out.sounds.reserve( items.size() );
for( auto const &r : items ) {
scene::eu7::Eu7Sound s;
apply_node( s.node, r.node );
s.location = { r.location.x, r.location.y, r.location.z };
s.wav_file = str( r.wav_file );
m_out.sounds.push_back( std::move( s ) );
}
}
void build_dynamics( std::vector<dynamic_record> const &items ) const {
m_out.dynamics.reserve( items.size() );
for( auto const &r : items ) {
scene::eu7::Eu7Dynamic d;
apply_node( d.node, r.node );
d.data_folder = str( r.data_folder );
d.skin_file = str( r.skin_file );
d.mmd_file = str( r.mmd_file );
d.track_name = str( r.track_name );
d.offset = r.offset;
d.driver_type = str( r.driver_type );
d.coupling = r.coupling;
d.coupling_raw = str( r.coupling_raw );
d.coupling_params = str( r.coupling_params );
d.velocity = r.velocity;
d.load_count = r.load_count;
d.load_type = str( r.load_type );
if( r.has_destination ) {
d.destination = str( r.destination );
}
if( r.has_trainset ) {
d.trainset_index = static_cast<std::size_t>( r.trainset_index );
}
m_out.dynamics.push_back( std::move( d ) );
}
}
private:
string_pool const &m_pool;
scene::eu7::Eu7Scene &m_out;
};
} // namespace
namespace {
// Shared scan: reconstructs the Eu7Scene; when module != nullptr it also
// reconstructs includes / placement / flags / first_init_count.
[[nodiscard]] bool
load_into(
std::uint8_t const *data,
std::size_t const size,
scene::eu7::Eu7Scene &out,
scene::eu7::Eu7Module *module ) {
try {
container_reader reader( data, size );
if( reader.kind() != file_kind::sim && reader.kind() != file_kind::module ) {
return false;
}
string_pool pool;
std::vector<model_prototype> protos;
std::vector<model_instance> instances;
std::vector<include_record> includes;
std::vector<module_placement_record> placements;
bool saw_meta { false };
module_meta meta;
// STRS must be resolved before the records that reference it; the baker
// always writes it first, so a single pass over chunks is enough.
chunk_view chunk;
scene_loader loader( pool, out );
while( reader.next( chunk ) ) {
auto r { chunk.reader() };
switch( chunk.id ) {
case chunk::strs: pool.deserialize( r ); break;
case chunk::meta: meta = read_meta( r ); saw_meta = true; break;
case chunk::incl: includes = read_includes( r ); break;
case chunk::plce: placements = read_module_placements( r ); break;
case chunk::prot: protos = read_prototypes( r ); break;
case chunk::inst: instances = read_instances( r ); break;
case chunk::mesh: loader.build_terrain( read_terrain_meshes( r ) ); break;
case chunk::shpe: loader.build_shapes( read_shapes( r ) ); break;
case chunk::line: loader.build_lines( read_lines( r ) ); break;
case chunk::trak: loader.build_tracks( read_tracks( r ) ); break;
case chunk::trac: loader.build_traction( read_traction( r ) ); break;
case chunk::pwrs: loader.build_power_sources( read_power_sources( r ) ); break;
case chunk::memc: loader.build_memcells( read_memcells( r ) ); break;
case chunk::laun: loader.build_launchers( read_launchers( r ) ); break;
case chunk::evnt: loader.build_events( read_events( r ) ); break;
case chunk::sond: loader.build_sounds( read_sounds( r ) ); break;
case chunk::dynm: loader.build_dynamics( read_dynamics( r ) ); break;
case chunk::trst: loader.build_trainsets( read_trainsets( r ) ); break;
default: break; // unknown chunk: skip
}
}
// models depend on both PROT and INST, build them after the scan
loader.build_models( protos, instances );
if( saw_meta ) {
out.first_init_count = meta.first_init_count;
}
if( module != nullptr ) {
auto resolve { [&]( std::uint32_t const id ) {
return id == kNoString ? std::string() : pool.get( id );
} };
module->includes.reserve( includes.size() + placements.size() );
for( auto const &inc : includes ) {
scene::eu7::Eu7Include e;
e.source_line = inc.source_line;
e.source_path = resolve( inc.source_path );
e.binary_path = resolve( inc.binary_path );
e.parameters.reserve( inc.parameters.size() );
for( auto const p : inc.parameters ) {
e.parameters.push_back( resolve( p ) );
}
for( auto const &v : inc.site_transform.origin_stack ) {
e.site_transform.origin_stack.push_back( { v.x, v.y, v.z } );
}
for( auto const &v : inc.site_transform.scale_stack ) {
e.site_transform.scale_stack.push_back( { v.x, v.y, v.z } );
}
e.site_transform.rotation = {
inc.site_transform.rotation.x,
inc.site_transform.rotation.y,
inc.site_transform.rotation.z };
e.site_transform.group_depth =
static_cast<std::size_t>( inc.site_transform.group_depth );
module->includes.push_back( std::move( e ) );
}
for( auto const &p : placements ) {
scene::eu7::Eu7Include e;
e.binary_path = resolve( p.module_path );
if( !e.binary_path.empty() ) {
std::filesystem::path source { e.binary_path };
source.replace_extension( ".inc" );
e.source_path = source.generic_string();
}
if( p.texture_override != kNoString ) {
e.parameters.push_back( resolve( p.texture_override ) );
} else {
e.parameters.push_back( "none" );
}
e.parameters.push_back( std::to_string( p.x ) );
e.parameters.push_back( std::to_string( p.y ) );
e.parameters.push_back( std::to_string( p.z ) );
e.parameters.push_back( std::to_string( p.rotation_y ) );
module->includes.push_back( std::move( e ) );
}
if( saw_meta ) {
module->has_terrain_chunk = meta.has_terrain_chunk;
module->has_pack_chunk = meta.has_pack_chunk;
module->include_placement.origin_x_param = meta.placement_origin_x;
module->include_placement.origin_y_param = meta.placement_origin_y;
module->include_placement.origin_z_param = meta.placement_origin_z;
module->include_placement.rotation_y_param = meta.placement_rotation_y;
}
}
return true;
}
catch( parse_error const & ) {
return false;
}
}
} // namespace
bool
load_scene( std::uint8_t const *data, std::size_t const size, scene::eu7::Eu7Scene &out ) {
return load_into( data, size, out, nullptr );
}
bool
load_module( std::uint8_t const *data, std::size_t const size, scene::eu7::Eu7Module &out ) {
return load_into( data, size, out.scene, &out );
}
} // namespace eu7v2

36
scene/eu7/v2/eu7v2_load.h Normal file
View File

@@ -0,0 +1,36 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#pragma once
#include <cstdint>
#include <vector>
namespace scene::eu7 {
struct Eu7Scene;
struct Eu7Module;
} // namespace scene::eu7
namespace eu7v2 {
// Reconstructs a parsed scene from an eu7v2 container produced by bake_scene()
// / bake_module(). The reconstructed Eu7Scene can be fed to the existing apply
// path, so switching to the new format only swaps serialization and reuses the
// proven world-construction code. Returns false on a malformed container (the
// scene is left in a partially-populated but safe state).
[[nodiscard]] bool
load_scene( std::uint8_t const *data, std::size_t size, scene::eu7::Eu7Scene &out );
// Reconstructs a full module (scene + includes + placement + flags) from an
// eu7v2 container. The runtime needs the includes to recurse into child
// modules. Returns false on a malformed container.
[[nodiscard]] bool
load_module( std::uint8_t const *data, std::size_t size, scene::eu7::Eu7Module &out );
} // namespace eu7v2

View File

@@ -0,0 +1,816 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#pragma once
// ---------------------------------------------------------------------------
// eu7v2 simulation records (iteration 2b): the non-visual scene data that is
// loaded once into the sim core - tracks, traction, power sources, memory
// cells, event launchers, events, sounds and dynamic vehicles.
//
// All strings are referenced by string-table id; optional fields use a leading
// presence flag so absent data costs a single byte. Dependency-free so the
// encode/decode path stays unit testable with a standalone compiler.
// ---------------------------------------------------------------------------
#include "eu7v2_format.h"
#include "eu7v2_scene.h"
#include <cstdint>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
namespace eu7v2 {
// Shared helpers (put_strid/dvec3/node_record/...) live in eu7v2_scene.h so
// scene payloads and these sim records share a single definition.
// --- TRAK : tracks ---------------------------------------------------------
struct track_path {
dvec3 p_start;
double roll_start { 0.0 };
dvec3 cp_out;
dvec3 cp_in;
dvec3 p_end;
double roll_end { 0.0 };
double radius { 0.0 };
};
struct track_visibility {
std::uint32_t material1 { kNoString };
float tex_length { 4.f };
std::uint32_t material2 { kNoString };
float tex_height1 { 0.f };
float tex_width { 0.f };
float tex_slope { 0.f };
};
struct track_record {
node_record node;
std::uint8_t track_type { 0 };
std::uint8_t category { 1 };
float length { 0.f };
float track_width { 0.f };
float friction { 0.f };
float sound_distance { 0.f };
std::int32_t quality_flag { 0 };
std::int32_t damage_flag { 0 };
std::int8_t environment { -1 };
bool has_visibility { false };
track_visibility visibility;
std::vector<track_path> paths;
std::vector<std::pair<std::uint32_t, std::uint32_t>> tail_keywords; // (key strid, value strid)
};
inline void write_tracks( byte_writer &out, std::vector<track_record> const &tracks ) {
out.put_u32( static_cast<std::uint32_t>( tracks.size() ) );
for( auto const &t : tracks ) {
write_node( out, t.node );
out.put_u8( t.track_type );
out.put_u8( t.category );
out.put_f32( t.length );
out.put_f32( t.track_width );
out.put_f32( t.friction );
out.put_f32( t.sound_distance );
out.put_i32( t.quality_flag );
out.put_i32( t.damage_flag );
out.put_u8( static_cast<std::uint8_t>( t.environment ) );
out.put_u8( t.has_visibility ? 1u : 0u );
if( t.has_visibility ) {
out.put_u32( t.visibility.material1 );
out.put_f32( t.visibility.tex_length );
out.put_u32( t.visibility.material2 );
out.put_f32( t.visibility.tex_height1 );
out.put_f32( t.visibility.tex_width );
out.put_f32( t.visibility.tex_slope );
}
out.put_u32( static_cast<std::uint32_t>( t.paths.size() ) );
for( auto const &p : t.paths ) {
put_dvec3( out, p.p_start.x, p.p_start.y, p.p_start.z );
out.put_f64( p.roll_start );
put_dvec3( out, p.cp_out.x, p.cp_out.y, p.cp_out.z );
put_dvec3( out, p.cp_in.x, p.cp_in.y, p.cp_in.z );
put_dvec3( out, p.p_end.x, p.p_end.y, p.p_end.z );
out.put_f64( p.roll_end );
out.put_f64( p.radius );
}
out.put_u32( static_cast<std::uint32_t>( t.tail_keywords.size() ) );
for( auto const &kv : t.tail_keywords ) {
out.put_u32( kv.first );
out.put_u32( kv.second );
}
}
}
inline std::vector<track_record> read_tracks( byte_reader &in ) {
std::vector<track_record> tracks;
auto const count { in.get_u32() };
tracks.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
track_record t;
t.node = read_node( in );
t.track_type = in.get_u8();
t.category = in.get_u8();
t.length = in.get_f32();
t.track_width = in.get_f32();
t.friction = in.get_f32();
t.sound_distance = in.get_f32();
t.quality_flag = in.get_i32();
t.damage_flag = in.get_i32();
t.environment = static_cast<std::int8_t>( in.get_u8() );
t.has_visibility = in.get_u8() != 0;
if( t.has_visibility ) {
t.visibility.material1 = in.get_u32();
t.visibility.tex_length = in.get_f32();
t.visibility.material2 = in.get_u32();
t.visibility.tex_height1 = in.get_f32();
t.visibility.tex_width = in.get_f32();
t.visibility.tex_slope = in.get_f32();
}
auto const paths { in.get_u32() };
t.paths.reserve( paths );
for( std::uint32_t p { 0 }; p < paths; ++p ) {
track_path tp;
tp.p_start = get_dvec3( in );
tp.roll_start = in.get_f64();
tp.cp_out = get_dvec3( in );
tp.cp_in = get_dvec3( in );
tp.p_end = get_dvec3( in );
tp.roll_end = in.get_f64();
tp.radius = in.get_f64();
t.paths.push_back( tp );
}
auto const kws { in.get_u32() };
t.tail_keywords.reserve( kws );
for( std::uint32_t k { 0 }; k < kws; ++k ) {
auto const key { in.get_u32() };
auto const value { in.get_u32() };
t.tail_keywords.emplace_back( key, value );
}
tracks.push_back( std::move( t ) );
}
return tracks;
}
// --- TRAC : traction -------------------------------------------------------
struct traction_record {
node_record node;
std::uint32_t power_supply_name { kNoString };
float nominal_voltage { 0.f };
float max_current { 0.f };
float resistivity { 0.f };
std::uint8_t material { 1 };
float wire_thickness { 0.f };
std::int32_t damage_flag { 0 };
dvec3 wire_p1, wire_p2, wire_p3, wire_p4;
double min_height { 0.0 };
double segment_length { 0.0 };
std::int32_t wire_count { 0 };
float wire_offset { 0.f };
bool has_parallel { false };
std::uint32_t parallel_name { kNoString };
};
inline void write_traction( byte_writer &out, std::vector<traction_record> const &items ) {
out.put_u32( static_cast<std::uint32_t>( items.size() ) );
for( auto const &t : items ) {
write_node( out, t.node );
out.put_u32( t.power_supply_name );
out.put_f32( t.nominal_voltage );
out.put_f32( t.max_current );
out.put_f32( t.resistivity );
out.put_u8( t.material );
out.put_f32( t.wire_thickness );
out.put_i32( t.damage_flag );
put_dvec3( out, t.wire_p1.x, t.wire_p1.y, t.wire_p1.z );
put_dvec3( out, t.wire_p2.x, t.wire_p2.y, t.wire_p2.z );
put_dvec3( out, t.wire_p3.x, t.wire_p3.y, t.wire_p3.z );
put_dvec3( out, t.wire_p4.x, t.wire_p4.y, t.wire_p4.z );
out.put_f64( t.min_height );
out.put_f64( t.segment_length );
out.put_i32( t.wire_count );
out.put_f32( t.wire_offset );
put_opt_strid( out, t.has_parallel, t.parallel_name );
}
}
inline std::vector<traction_record> read_traction( byte_reader &in ) {
std::vector<traction_record> items;
auto const count { in.get_u32() };
items.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
traction_record t;
t.node = read_node( in );
t.power_supply_name = in.get_u32();
t.nominal_voltage = in.get_f32();
t.max_current = in.get_f32();
t.resistivity = in.get_f32();
t.material = in.get_u8();
t.wire_thickness = in.get_f32();
t.damage_flag = in.get_i32();
t.wire_p1 = get_dvec3( in );
t.wire_p2 = get_dvec3( in );
t.wire_p3 = get_dvec3( in );
t.wire_p4 = get_dvec3( in );
t.min_height = in.get_f64();
t.segment_length = in.get_f64();
t.wire_count = in.get_i32();
t.wire_offset = in.get_f32();
t.has_parallel = in.get_u8() != 0;
if( t.has_parallel ) {
t.parallel_name = in.get_u32();
}
items.push_back( t );
}
return items;
}
// --- PWRS : traction power sources -----------------------------------------
struct power_source_record {
node_record node;
dvec3 position;
float nominal_voltage { 0.f };
float voltage_frequency { 0.f };
float internal_resistance { 0.2f };
float max_output_current { 0.f };
float fast_fuse_timeout { 0.f };
float fast_fuse_repetition { 0.f };
float slow_fuse_timeout { 0.f };
std::uint8_t modifier { 0 };
};
inline void write_power_sources( byte_writer &out, std::vector<power_source_record> const &items ) {
out.put_u32( static_cast<std::uint32_t>( items.size() ) );
for( auto const &p : items ) {
write_node( out, p.node );
put_dvec3( out, p.position.x, p.position.y, p.position.z );
out.put_f32( p.nominal_voltage );
out.put_f32( p.voltage_frequency );
out.put_f32( p.internal_resistance );
out.put_f32( p.max_output_current );
out.put_f32( p.fast_fuse_timeout );
out.put_f32( p.fast_fuse_repetition );
out.put_f32( p.slow_fuse_timeout );
out.put_u8( p.modifier );
}
}
inline std::vector<power_source_record> read_power_sources( byte_reader &in ) {
std::vector<power_source_record> items;
auto const count { in.get_u32() };
items.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
power_source_record p;
p.node = read_node( in );
p.position = get_dvec3( in );
p.nominal_voltage = in.get_f32();
p.voltage_frequency = in.get_f32();
p.internal_resistance = in.get_f32();
p.max_output_current = in.get_f32();
p.fast_fuse_timeout = in.get_f32();
p.fast_fuse_repetition = in.get_f32();
p.slow_fuse_timeout = in.get_f32();
p.modifier = in.get_u8();
items.push_back( p );
}
return items;
}
// --- MEMC : memory cells ---------------------------------------------------
struct memcell_record {
node_record node;
std::uint32_t text { kNoString };
double value1 { 0.0 };
double value2 { 0.0 };
bool has_track { false };
std::uint32_t track_name { kNoString };
};
inline void write_memcells( byte_writer &out, std::vector<memcell_record> const &items ) {
out.put_u32( static_cast<std::uint32_t>( items.size() ) );
for( auto const &m : items ) {
write_node( out, m.node );
out.put_u32( m.text );
out.put_f64( m.value1 );
out.put_f64( m.value2 );
put_opt_strid( out, m.has_track, m.track_name );
}
}
inline std::vector<memcell_record> read_memcells( byte_reader &in ) {
std::vector<memcell_record> items;
auto const count { in.get_u32() };
items.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
memcell_record m;
m.node = read_node( in );
m.text = in.get_u32();
m.value1 = in.get_f64();
m.value2 = in.get_f64();
m.has_track = in.get_u8() != 0;
if( m.has_track ) {
m.track_name = in.get_u32();
}
items.push_back( m );
}
return items;
}
// --- LAUN : event launchers ------------------------------------------------
struct launcher_condition {
std::uint32_t memcell_name { kNoString };
std::uint32_t compare_text { kNoString };
double compare_value1 { 0.0 };
double compare_value2 { 0.0 };
std::int32_t check_mask { 0 };
};
struct launcher_record {
node_record node;
dvec3 location;
double radius_squared { 0.0 };
std::int32_t activation_key { 0 };
double delta_time { -1.0 };
std::uint32_t event1_name { kNoString };
std::uint32_t event2_name { kNoString };
bool has_condition { false };
launcher_condition condition;
bool train_triggered { false };
std::int32_t launch_hour { -1 };
std::int32_t launch_minute { -1 };
};
inline void write_launchers( byte_writer &out, std::vector<launcher_record> const &items ) {
out.put_u32( static_cast<std::uint32_t>( items.size() ) );
for( auto const &l : items ) {
write_node( out, l.node );
put_dvec3( out, l.location.x, l.location.y, l.location.z );
out.put_f64( l.radius_squared );
out.put_i32( l.activation_key );
out.put_f64( l.delta_time );
out.put_u32( l.event1_name );
out.put_u32( l.event2_name );
out.put_u8( l.has_condition ? 1u : 0u );
if( l.has_condition ) {
out.put_u32( l.condition.memcell_name );
out.put_u32( l.condition.compare_text );
out.put_f64( l.condition.compare_value1 );
out.put_f64( l.condition.compare_value2 );
out.put_i32( l.condition.check_mask );
}
out.put_u8( l.train_triggered ? 1u : 0u );
out.put_i32( l.launch_hour );
out.put_i32( l.launch_minute );
}
}
inline std::vector<launcher_record> read_launchers( byte_reader &in ) {
std::vector<launcher_record> items;
auto const count { in.get_u32() };
items.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
launcher_record l;
l.node = read_node( in );
l.location = get_dvec3( in );
l.radius_squared = in.get_f64();
l.activation_key = in.get_i32();
l.delta_time = in.get_f64();
l.event1_name = in.get_u32();
l.event2_name = in.get_u32();
l.has_condition = in.get_u8() != 0;
if( l.has_condition ) {
l.condition.memcell_name = in.get_u32();
l.condition.compare_text = in.get_u32();
l.condition.compare_value1 = in.get_f64();
l.condition.compare_value2 = in.get_f64();
l.condition.check_mask = in.get_i32();
}
l.train_triggered = in.get_u8() != 0;
l.launch_hour = in.get_i32();
l.launch_minute = in.get_i32();
items.push_back( l );
}
return items;
}
// --- EVNT : events ---------------------------------------------------------
struct event_record {
std::uint32_t name { kNoString };
std::uint8_t type { 0 };
double delay { 0.0 };
double delay_random { 0.0 };
double delay_departure { 0.0 };
bool ignored { false };
bool passive { false };
std::vector<std::uint32_t> targets; // string ids
std::vector<std::pair<std::uint32_t, std::uint32_t>> payload; // (key strid, value strid)
};
inline void write_events( byte_writer &out, std::vector<event_record> const &items ) {
out.put_u32( static_cast<std::uint32_t>( items.size() ) );
for( auto const &e : items ) {
out.put_u32( e.name );
out.put_u8( e.type );
out.put_f64( e.delay );
out.put_f64( e.delay_random );
out.put_f64( e.delay_departure );
out.put_u8( e.ignored ? 1u : 0u );
out.put_u8( e.passive ? 1u : 0u );
out.put_u32( static_cast<std::uint32_t>( e.targets.size() ) );
for( auto const t : e.targets ) {
out.put_u32( t );
}
out.put_u32( static_cast<std::uint32_t>( e.payload.size() ) );
for( auto const &kv : e.payload ) {
out.put_u32( kv.first );
out.put_u32( kv.second );
}
}
}
inline std::vector<event_record> read_events( byte_reader &in ) {
std::vector<event_record> items;
auto const count { in.get_u32() };
items.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
event_record e;
e.name = in.get_u32();
e.type = in.get_u8();
e.delay = in.get_f64();
e.delay_random = in.get_f64();
e.delay_departure = in.get_f64();
e.ignored = in.get_u8() != 0;
e.passive = in.get_u8() != 0;
auto const targets { in.get_u32() };
e.targets.reserve( targets );
for( std::uint32_t t { 0 }; t < targets; ++t ) {
e.targets.push_back( in.get_u32() );
}
auto const payload { in.get_u32() };
e.payload.reserve( payload );
for( std::uint32_t p { 0 }; p < payload; ++p ) {
auto const key { in.get_u32() };
auto const value { in.get_u32() };
e.payload.emplace_back( key, value );
}
items.push_back( std::move( e ) );
}
return items;
}
// --- SOND : sounds ---------------------------------------------------------
struct sound_record {
node_record node;
dvec3 location;
std::uint32_t wav_file { kNoString };
};
inline void write_sounds( byte_writer &out, std::vector<sound_record> const &items ) {
out.put_u32( static_cast<std::uint32_t>( items.size() ) );
for( auto const &s : items ) {
write_node( out, s.node );
put_dvec3( out, s.location.x, s.location.y, s.location.z );
out.put_u32( s.wav_file );
}
}
inline std::vector<sound_record> read_sounds( byte_reader &in ) {
std::vector<sound_record> items;
auto const count { in.get_u32() };
items.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
sound_record s;
s.node = read_node( in );
s.location = get_dvec3( in );
s.wav_file = in.get_u32();
items.push_back( s );
}
return items;
}
// --- DYNM : dynamic vehicles -----------------------------------------------
struct dynamic_record {
node_record node;
std::uint32_t data_folder { kNoString };
std::uint32_t skin_file { kNoString };
std::uint32_t mmd_file { kNoString };
std::uint32_t track_name { kNoString };
double offset { -1.0 };
std::uint32_t driver_type { kNoString };
std::int32_t coupling { 3 };
std::uint32_t coupling_raw { kNoString };
std::uint32_t coupling_params { kNoString };
float velocity { 0.f };
std::int32_t load_count { 0 };
std::uint32_t load_type { kNoString };
bool has_destination { false };
std::uint32_t destination { kNoString };
bool has_trainset { false };
std::uint32_t trainset_index { 0 };
};
inline void write_dynamics( byte_writer &out, std::vector<dynamic_record> const &items ) {
out.put_u32( static_cast<std::uint32_t>( items.size() ) );
for( auto const &d : items ) {
write_node( out, d.node );
out.put_u32( d.data_folder );
out.put_u32( d.skin_file );
out.put_u32( d.mmd_file );
out.put_u32( d.track_name );
out.put_f64( d.offset );
out.put_u32( d.driver_type );
out.put_i32( d.coupling );
out.put_u32( d.coupling_raw );
out.put_u32( d.coupling_params );
out.put_f32( d.velocity );
out.put_i32( d.load_count );
out.put_u32( d.load_type );
put_opt_strid( out, d.has_destination, d.destination );
out.put_u8( d.has_trainset ? 1u : 0u );
if( d.has_trainset ) {
out.put_u32( d.trainset_index );
}
}
}
inline std::vector<dynamic_record> read_dynamics( byte_reader &in ) {
std::vector<dynamic_record> items;
auto const count { in.get_u32() };
items.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
dynamic_record d;
d.node = read_node( in );
d.data_folder = in.get_u32();
d.skin_file = in.get_u32();
d.mmd_file = in.get_u32();
d.track_name = in.get_u32();
d.offset = in.get_f64();
d.driver_type = in.get_u32();
d.coupling = in.get_i32();
d.coupling_raw = in.get_u32();
d.coupling_params = in.get_u32();
d.velocity = in.get_f32();
d.load_count = in.get_i32();
d.load_type = in.get_u32();
d.has_destination = in.get_u8() != 0;
if( d.has_destination ) {
d.destination = in.get_u32();
}
d.has_trainset = in.get_u8() != 0;
if( d.has_trainset ) {
d.trainset_index = in.get_u32();
}
items.push_back( d );
}
return items;
}
// --- TRST : trainsets ------------------------------------------------------
struct trainset_record {
std::uint32_t name { kNoString };
std::uint32_t track { kNoString };
float offset { 0.f };
float velocity { 0.f };
std::vector<std::pair<std::uint32_t, std::uint32_t>> assignment; // (key strid, value strid)
std::vector<std::uint32_t> vehicle_indices;
std::vector<std::int32_t> couplings;
std::uint32_t driver_index { 0xffffffffu }; // (size_t)-1 sentinel
};
inline void write_trainsets( byte_writer &out, std::vector<trainset_record> const &items ) {
out.put_u32( static_cast<std::uint32_t>( items.size() ) );
for( auto const &t : items ) {
out.put_u32( t.name );
out.put_u32( t.track );
out.put_f32( t.offset );
out.put_f32( t.velocity );
out.put_u32( static_cast<std::uint32_t>( t.assignment.size() ) );
for( auto const &kv : t.assignment ) {
out.put_u32( kv.first );
out.put_u32( kv.second );
}
out.put_u32( static_cast<std::uint32_t>( t.vehicle_indices.size() ) );
for( auto const idx : t.vehicle_indices ) {
out.put_u32( idx );
}
out.put_u32( static_cast<std::uint32_t>( t.couplings.size() ) );
for( auto const c : t.couplings ) {
out.put_i32( c );
}
out.put_u32( t.driver_index );
}
}
inline std::vector<trainset_record> read_trainsets( byte_reader &in ) {
std::vector<trainset_record> items;
auto const count { in.get_u32() };
items.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
trainset_record t;
t.name = in.get_u32();
t.track = in.get_u32();
t.offset = in.get_f32();
t.velocity = in.get_f32();
auto const assign { in.get_u32() };
t.assignment.reserve( assign );
for( std::uint32_t a { 0 }; a < assign; ++a ) {
auto const key { in.get_u32() };
auto const value { in.get_u32() };
t.assignment.emplace_back( key, value );
}
auto const vehicles { in.get_u32() };
t.vehicle_indices.reserve( vehicles );
for( std::uint32_t v { 0 }; v < vehicles; ++v ) {
t.vehicle_indices.push_back( in.get_u32() );
}
auto const couplings { in.get_u32() };
t.couplings.reserve( couplings );
for( std::uint32_t c { 0 }; c < couplings; ++c ) {
t.couplings.push_back( in.get_i32() );
}
t.driver_index = in.get_u32();
items.push_back( std::move( t ) );
}
return items;
}
// --- PLCE : lean reusable-module placements (.inc with x/y/z/rot as f64/f32) -
struct module_placement_record {
std::uint32_t module_path { kNoString }; // e.g. scenery/grass_l61/20.eu7v2
std::uint32_t texture_override { kNoString };
double x { 0.0 }, y { 0.0 }, z { 0.0 };
float rotation_y { 0.f };
std::uint8_t cell_id { 0xffu };
};
inline void write_module_placements(
byte_writer &out,
std::vector<module_placement_record> const &items ) {
out.put_u32( static_cast<std::uint32_t>( items.size() ) );
for( auto const &p : items ) {
out.put_u32( p.module_path );
out.put_u32( p.texture_override );
out.put_f64( p.x );
out.put_f64( p.y );
out.put_f64( p.z );
out.put_f32( p.rotation_y );
out.put_u8( p.cell_id );
}
}
inline std::vector<module_placement_record> read_module_placements( byte_reader &in ) {
std::vector<module_placement_record> items;
auto const count { in.get_u32() };
items.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
module_placement_record p;
p.module_path = in.get_u32();
p.texture_override = in.get_u32();
p.x = in.get_f64();
p.y = in.get_f64();
p.z = in.get_f64();
p.rotation_y = in.get_f32();
p.cell_id = in.get_u8();
items.push_back( std::move( p ) );
}
return items;
}
// --- INCL : module includes (recursion references) -------------------------
// Snapshot of the origin/scale/rotation stacks at include site (detokenizer).
struct transform_record {
std::vector<dvec3> origin_stack;
std::vector<dvec3> scale_stack;
dvec3 rotation;
std::uint32_t group_depth { 0 };
};
inline void write_transform( byte_writer &out, transform_record const &t ) {
out.put_u32( static_cast<std::uint32_t>( t.origin_stack.size() ) );
for( auto const &v : t.origin_stack ) {
put_dvec3( out, v.x, v.y, v.z );
}
out.put_u32( static_cast<std::uint32_t>( t.scale_stack.size() ) );
for( auto const &v : t.scale_stack ) {
put_dvec3( out, v.x, v.y, v.z );
}
put_dvec3( out, t.rotation.x, t.rotation.y, t.rotation.z );
out.put_u32( t.group_depth );
}
inline transform_record read_transform( byte_reader &in ) {
transform_record t;
auto const origins { in.get_u32() };
t.origin_stack.reserve( origins );
for( std::uint32_t i { 0 }; i < origins; ++i ) {
t.origin_stack.push_back( get_dvec3( in ) );
}
auto const scales { in.get_u32() };
t.scale_stack.reserve( scales );
for( std::uint32_t i { 0 }; i < scales; ++i ) {
t.scale_stack.push_back( get_dvec3( in ) );
}
t.rotation = get_dvec3( in );
t.group_depth = in.get_u32();
return t;
}
struct include_record {
std::uint32_t source_line { 0 };
std::uint32_t source_path { kNoString };
std::uint32_t binary_path { kNoString }; // points at the .eu7v2 module
std::vector<std::uint32_t> parameters; // string ids
transform_record site_transform;
};
inline void write_includes( byte_writer &out, std::vector<include_record> const &items ) {
out.put_u32( static_cast<std::uint32_t>( items.size() ) );
for( auto const &inc : items ) {
out.put_u32( inc.source_line );
out.put_u32( inc.source_path );
out.put_u32( inc.binary_path );
out.put_u32( static_cast<std::uint32_t>( inc.parameters.size() ) );
for( auto const p : inc.parameters ) {
out.put_u32( p );
}
write_transform( out, inc.site_transform );
}
}
inline std::vector<include_record> read_includes( byte_reader &in ) {
std::vector<include_record> items;
auto const count { in.get_u32() };
items.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
include_record inc;
inc.source_line = in.get_u32();
inc.source_path = in.get_u32();
inc.binary_path = in.get_u32();
auto const params { in.get_u32() };
inc.parameters.reserve( params );
for( std::uint32_t p { 0 }; p < params; ++p ) {
inc.parameters.push_back( in.get_u32() );
}
inc.site_transform = read_transform( in );
items.push_back( std::move( inc ) );
}
return items;
}
// --- META : module-level metadata (flags, placement, counts) ---------------
struct module_meta {
std::uint32_t first_init_count { 0 };
bool has_terrain_chunk { false };
bool has_pack_chunk { false };
// include placement (parameter indices, 0 = unused)
std::uint8_t placement_origin_x { 0 };
std::uint8_t placement_origin_y { 0 };
std::uint8_t placement_origin_z { 0 };
std::uint8_t placement_rotation_y { 0 };
};
inline void write_meta( byte_writer &out, module_meta const &m ) {
out.put_u32( 1u ); // meta layout version (forward-tolerant)
out.put_u32( m.first_init_count );
out.put_u8( m.has_terrain_chunk ? 1u : 0u );
out.put_u8( m.has_pack_chunk ? 1u : 0u );
out.put_u8( m.placement_origin_x );
out.put_u8( m.placement_origin_y );
out.put_u8( m.placement_origin_z );
out.put_u8( m.placement_rotation_y );
}
inline module_meta read_meta( byte_reader &in ) {
module_meta m;
(void)in.get_u32(); // layout version
m.first_init_count = in.get_u32();
m.has_terrain_chunk = in.get_u8() != 0;
m.has_pack_chunk = in.get_u8() != 0;
m.placement_origin_x = in.get_u8();
m.placement_origin_y = in.get_u8();
m.placement_origin_z = in.get_u8();
m.placement_rotation_y = in.get_u8();
return m;
}
} // namespace eu7v2

472
scene/eu7/v2/eu7v2_scene.h Normal file
View File

@@ -0,0 +1,472 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#pragma once
// ---------------------------------------------------------------------------
// eu7v2 scene payloads (iteration 2 slice): the lean model + terrain records
// that fix the "heavy per-object" problem of the legacy streaming path.
//
// PROT - deduplicated model prototypes (mesh + material + LOD + lights),
// stored once per unique model, referenced by index.
// INST - lean instance: prototype index + world transform + optional
// per-instance texture override. No engine object is created here;
// the runtime resolves prototype -> shared mesh and renders via
// instancing.
// MESH - baked terrain mesh: origin (f64) + vertices relative to origin
// (f32) so a 1km tile keeps precision without paying f64 per vertex.
//
// Dependency-free on purpose so the whole encode/decode path is unit testable
// with a standalone compiler (see eu7v2_test.cpp), independent of the engine.
// ---------------------------------------------------------------------------
#include "eu7v2_format.h"
#include <cstdint>
#include <string>
#include <vector>
namespace eu7v2 {
constexpr std::uint32_t kNoString { 0xffffffffu };
// --- shared serialization helpers (used by scene payloads and sim records) -
inline void put_strid( byte_writer &out, std::uint32_t const id ) { out.put_u32( id ); }
inline std::uint32_t get_strid( byte_reader &in ) { return in.get_u32(); }
inline void put_dvec3( byte_writer &out, double const x, double const y, double const z ) {
out.put_f64( x );
out.put_f64( y );
out.put_f64( z );
}
struct dvec3 {
double x { 0.0 }, y { 0.0 }, z { 0.0 };
};
inline dvec3 get_dvec3( byte_reader &in ) {
dvec3 v;
v.x = in.get_f64();
v.y = in.get_f64();
v.z = in.get_f64();
return v;
}
inline void put_opt_strid( byte_writer &out, bool const present, std::uint32_t const id ) {
out.put_u8( present ? 1u : 0u );
if( present ) {
out.put_u32( id );
}
}
// Common node metadata kept after baking (world-space, transform already applied).
struct node_record {
std::uint32_t name { kNoString };
std::uint32_t type { kNoString };
dvec3 area_center;
float area_radius { -1.f };
double range_sq_min { 0.0 };
double range_sq_max { 0.0 };
bool visible { true };
};
inline void write_node( byte_writer &out, node_record const &n ) {
out.put_u32( n.name );
out.put_u32( n.type );
put_dvec3( out, n.area_center.x, n.area_center.y, n.area_center.z );
out.put_f32( n.area_radius );
out.put_f64( n.range_sq_min );
out.put_f64( n.range_sq_max );
out.put_u8( n.visible ? 1u : 0u );
}
inline node_record read_node( byte_reader &in ) {
node_record n;
n.name = in.get_u32();
n.type = in.get_u32();
n.area_center = get_dvec3( in );
n.area_radius = in.get_f32();
n.range_sq_min = in.get_f64();
n.range_sq_max = in.get_f64();
n.visible = in.get_u8() != 0;
return n;
}
// RGBA-ish lighting block stored as 12 floats (diffuse/ambient/specular vec4).
struct lighting_block {
float diffuse[ 4 ] { 0.8f, 0.8f, 0.8f, 1.f };
float ambient[ 4 ] { 0.2f, 0.2f, 0.2f, 1.f };
float specular[ 4 ] { 0.f, 0.f, 0.f, 1.f };
};
inline void write_lighting( byte_writer &out, lighting_block const &l ) {
for( auto const v : l.diffuse ) { out.put_f32( v ); }
for( auto const v : l.ambient ) { out.put_f32( v ); }
for( auto const v : l.specular ) { out.put_f32( v ); }
}
inline lighting_block read_lighting( byte_reader &in ) {
lighting_block l;
for( auto &v : l.diffuse ) { v = in.get_f32(); }
for( auto &v : l.ambient ) { v = in.get_f32(); }
for( auto &v : l.specular ) { v = in.get_f32(); }
return l;
}
// Prototype flags packed into a single byte.
namespace proto_flag {
constexpr std::uint8_t transition { 1u << 0 }; // model has LOD transition
constexpr std::uint8_t is_terrain { 1u << 1 }; // terrain-style submodel split
constexpr std::uint8_t instanceable { 1u << 2 }; // safe to GPU-instance
} // namespace proto_flag
struct model_prototype {
std::uint32_t model_file { kNoString }; // string id
std::uint32_t texture_file { kNoString }; // string id (default skin)
std::uint8_t flags { 0 };
float range_min { -1.f };
float range_max { -1.f };
std::vector<float> light_states;
std::vector<std::uint32_t> light_colors;
};
// Lean instance: which prototype, where, and an optional skin override.
struct model_instance {
std::uint32_t proto { 0 };
double x { 0.0 }, y { 0.0 }, z { 0.0 };
float ax { 0.f }, ay { 0.f }, az { 0.f }; // euler angles (deg), engine convention
float sx { 1.f }, sy { 1.f }, sz { 1.f }; // scale
std::uint32_t texture_override { kNoString };
std::uint8_t cell_id { 0xffu };
bool has_node { false }; // full node metadata present (lossless path)
node_record node;
};
struct mesh_vertex {
float px { 0.f }, py { 0.f }, pz { 0.f }; // position relative to mesh origin
float nx { 0.f }, ny { 0.f }, nz { 0.f }; // normal
float u { 0.f }, v { 0.f }; // texcoord
};
struct terrain_mesh {
std::uint32_t material { kNoString }; // string id
bool translucent { false };
double ox { 0.0 }, oy { 0.0 }, oz { 0.0 }; // world origin (f64)
std::vector<mesh_vertex> vertices;
};
// --- PROT ------------------------------------------------------------------
inline void
write_prototypes( byte_writer &out, std::vector<model_prototype> const &protos ) {
out.put_u32( static_cast<std::uint32_t>( protos.size() ) );
for( auto const &p : protos ) {
out.put_u32( p.model_file );
out.put_u32( p.texture_file );
out.put_u8( p.flags );
out.put_f32( p.range_min );
out.put_f32( p.range_max );
out.put_u32( static_cast<std::uint32_t>( p.light_states.size() ) );
for( auto const s : p.light_states ) {
out.put_f32( s );
}
out.put_u32( static_cast<std::uint32_t>( p.light_colors.size() ) );
for( auto const c : p.light_colors ) {
out.put_u32( c );
}
}
}
inline std::vector<model_prototype>
read_prototypes( byte_reader &in ) {
std::vector<model_prototype> protos;
auto const count { in.get_u32() };
protos.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
model_prototype p;
p.model_file = in.get_u32();
p.texture_file = in.get_u32();
p.flags = in.get_u8();
p.range_min = in.get_f32();
p.range_max = in.get_f32();
auto const states { in.get_u32() };
p.light_states.reserve( states );
for( std::uint32_t s { 0 }; s < states; ++s ) {
p.light_states.push_back( in.get_f32() );
}
auto const colors { in.get_u32() };
p.light_colors.reserve( colors );
for( std::uint32_t c { 0 }; c < colors; ++c ) {
p.light_colors.push_back( in.get_u32() );
}
protos.push_back( std::move( p ) );
}
return protos;
}
// --- INST ------------------------------------------------------------------
// Per-instance presence flags so common cases (unit scale, no override) stay
// compact instead of always paying for scale + override.
namespace inst_flag {
constexpr std::uint8_t has_scale { 1u << 0 };
constexpr std::uint8_t has_texture_override { 1u << 1 };
constexpr std::uint8_t has_node { 1u << 2 };
} // namespace inst_flag
inline void
write_instances( byte_writer &out, std::vector<model_instance> const &instances ) {
out.put_u32( static_cast<std::uint32_t>( instances.size() ) );
for( auto const &i : instances ) {
std::uint8_t flags { 0 };
bool const unit_scale { i.sx == 1.f && i.sy == 1.f && i.sz == 1.f };
if( !unit_scale ) {
flags |= inst_flag::has_scale;
}
if( i.texture_override != kNoString ) {
flags |= inst_flag::has_texture_override;
}
if( i.has_node ) {
flags |= inst_flag::has_node;
}
out.put_u8( flags );
out.put_u32( i.proto );
out.put_f64( i.x );
out.put_f64( i.y );
out.put_f64( i.z );
out.put_f32( i.ax );
out.put_f32( i.ay );
out.put_f32( i.az );
out.put_u8( i.cell_id );
if( flags & inst_flag::has_scale ) {
out.put_f32( i.sx );
out.put_f32( i.sy );
out.put_f32( i.sz );
}
if( flags & inst_flag::has_texture_override ) {
out.put_u32( i.texture_override );
}
if( flags & inst_flag::has_node ) {
write_node( out, i.node );
}
}
}
inline std::vector<model_instance>
read_instances( byte_reader &in ) {
std::vector<model_instance> instances;
auto const count { in.get_u32() };
instances.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
model_instance m;
auto const flags { in.get_u8() };
m.proto = in.get_u32();
m.x = in.get_f64();
m.y = in.get_f64();
m.z = in.get_f64();
m.ax = in.get_f32();
m.ay = in.get_f32();
m.az = in.get_f32();
m.cell_id = in.get_u8();
if( flags & inst_flag::has_scale ) {
m.sx = in.get_f32();
m.sy = in.get_f32();
m.sz = in.get_f32();
}
if( flags & inst_flag::has_texture_override ) {
m.texture_override = in.get_u32();
}
if( flags & inst_flag::has_node ) {
m.has_node = true;
m.node = read_node( in );
}
instances.push_back( m );
}
return instances;
}
// --- MESH ------------------------------------------------------------------
inline void
write_terrain_meshes( byte_writer &out, std::vector<terrain_mesh> const &meshes ) {
out.put_u32( static_cast<std::uint32_t>( meshes.size() ) );
for( auto const &m : meshes ) {
out.put_u32( m.material );
out.put_u8( m.translucent ? 1u : 0u );
out.put_f64( m.ox );
out.put_f64( m.oy );
out.put_f64( m.oz );
out.put_u32( static_cast<std::uint32_t>( m.vertices.size() ) );
for( auto const &v : m.vertices ) {
out.put_f32( v.px );
out.put_f32( v.py );
out.put_f32( v.pz );
out.put_f32( v.nx );
out.put_f32( v.ny );
out.put_f32( v.nz );
out.put_f32( v.u );
out.put_f32( v.v );
}
}
}
inline std::vector<terrain_mesh>
read_terrain_meshes( byte_reader &in ) {
std::vector<terrain_mesh> meshes;
auto const count { in.get_u32() };
meshes.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
terrain_mesh m;
m.material = in.get_u32();
m.translucent = in.get_u8() != 0;
m.ox = in.get_f64();
m.oy = in.get_f64();
m.oz = in.get_f64();
auto const verts { in.get_u32() };
m.vertices.reserve( verts );
for( std::uint32_t v { 0 }; v < verts; ++v ) {
mesh_vertex mv;
mv.px = in.get_f32();
mv.py = in.get_f32();
mv.pz = in.get_f32();
mv.nx = in.get_f32();
mv.ny = in.get_f32();
mv.nz = in.get_f32();
mv.u = in.get_f32();
mv.v = in.get_f32();
m.vertices.push_back( mv );
}
meshes.push_back( std::move( m ) );
}
return meshes;
}
// --- SHPE : non-terrain shape nodes (triangles/strip/fan, baked world-space) -
// Lossless counterpart of MESH: keeps node metadata, lighting and translucency.
// Vertex positions are stored relative to origin (f64) as f32, matching MESH.
struct shape_record {
node_record node;
bool translucent { false };
std::uint32_t material { kNoString };
lighting_block lighting;
double ox { 0.0 }, oy { 0.0 }, oz { 0.0 };
std::vector<mesh_vertex> vertices;
};
inline void write_shape_record( byte_writer &out, shape_record const &s ) {
write_node( out, s.node );
out.put_u8( s.translucent ? 1u : 0u );
out.put_u32( s.material );
write_lighting( out, s.lighting );
out.put_f64( s.ox );
out.put_f64( s.oy );
out.put_f64( s.oz );
out.put_u32( static_cast<std::uint32_t>( s.vertices.size() ) );
for( auto const &v : s.vertices ) {
out.put_f32( v.px );
out.put_f32( v.py );
out.put_f32( v.pz );
out.put_f32( v.nx );
out.put_f32( v.ny );
out.put_f32( v.nz );
out.put_f32( v.u );
out.put_f32( v.v );
}
}
inline void write_shapes( byte_writer &out, std::vector<shape_record> const &shapes ) {
out.put_u32( static_cast<std::uint32_t>( shapes.size() ) );
for( auto const &s : shapes ) {
write_shape_record( out, s );
}
}
inline std::vector<shape_record> read_shapes( byte_reader &in ) {
std::vector<shape_record> shapes;
auto const count { in.get_u32() };
shapes.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
shape_record s;
s.node = read_node( in );
s.translucent = in.get_u8() != 0;
s.material = in.get_u32();
s.lighting = read_lighting( in );
s.ox = in.get_f64();
s.oy = in.get_f64();
s.oz = in.get_f64();
auto const verts { in.get_u32() };
s.vertices.reserve( verts );
for( std::uint32_t v { 0 }; v < verts; ++v ) {
mesh_vertex mv;
mv.px = in.get_f32();
mv.py = in.get_f32();
mv.pz = in.get_f32();
mv.nx = in.get_f32();
mv.ny = in.get_f32();
mv.nz = in.get_f32();
mv.u = in.get_f32();
mv.v = in.get_f32();
s.vertices.push_back( mv );
}
shapes.push_back( std::move( s ) );
}
return shapes;
}
// --- LINE : line geometry nodes (only vertex positions are meaningful) -------
struct lines_record {
node_record node;
lighting_block lighting;
float line_width { 1.f };
double ox { 0.0 }, oy { 0.0 }, oz { 0.0 };
std::vector<dvec3> vertices; // world-space positions
};
inline void write_lines( byte_writer &out, std::vector<lines_record> const &items ) {
out.put_u32( static_cast<std::uint32_t>( items.size() ) );
for( auto const &l : items ) {
write_node( out, l.node );
write_lighting( out, l.lighting );
out.put_f32( l.line_width );
out.put_f64( l.ox );
out.put_f64( l.oy );
out.put_f64( l.oz );
out.put_u32( static_cast<std::uint32_t>( l.vertices.size() ) );
for( auto const &v : l.vertices ) {
put_dvec3( out, v.x, v.y, v.z );
}
}
}
inline std::vector<lines_record> read_lines( byte_reader &in ) {
std::vector<lines_record> items;
auto const count { in.get_u32() };
items.reserve( count );
for( std::uint32_t i { 0 }; i < count; ++i ) {
lines_record l;
l.node = read_node( in );
l.lighting = read_lighting( in );
l.line_width = in.get_f32();
l.ox = in.get_f64();
l.oy = in.get_f64();
l.oz = in.get_f64();
auto const verts { in.get_u32() };
l.vertices.reserve( verts );
for( std::uint32_t v { 0 }; v < verts; ++v ) {
l.vertices.push_back( get_dvec3( in ) );
}
items.push_back( std::move( l ) );
}
return items;
}
} // namespace eu7v2

673
scene/eu7/v2/eu7v2_test.cpp Normal file
View File

@@ -0,0 +1,673 @@
/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
// Standalone round-trip test for the eu7v2 format core. Builds in isolation
// (no engine dependencies) so the foundation can be validated without running
// the simulator:
// g++ -std=c++20 scene/eu7/v2/eu7v2_test.cpp -o eu7v2_test && ./eu7v2_test
#include "eu7v2_format.h"
#include "eu7v2_scene.h"
#include "eu7v2_records.h"
#include <cassert>
#include <cmath>
#include <cstdio>
#include <fstream>
namespace {
using namespace eu7v2;
int g_checks { 0 };
int g_failures { 0 };
void
check( bool const ok, char const *what ) {
++g_checks;
if( !ok ) {
++g_failures;
std::printf( " FAIL: %s\n", what );
}
}
// A lean instance record: prototype id + world transform + optional texture override.
struct sample_instance {
std::uint32_t proto_id;
double x, y, z;
float yaw;
std::uint32_t texture_override; // string id, 0xffffffff = none
};
// A precomputed track-graph edge: track index -> neighbour at an endpoint.
struct sample_edge {
std::uint32_t track;
std::uint32_t neighbour;
std::uint8_t end; // 0 = prev, 1 = next
std::uint8_t neighbour_end;
};
std::vector<std::uint8_t>
build_tile() {
string_table strings;
auto const tex_a { strings.intern( "textures/rail.dds" ) };
auto const tex_b { strings.intern( "textures/grass.dds" ) };
auto const proto_a { strings.intern( "models/sem.e3d" ) };
(void)tex_b;
(void)proto_a;
std::vector<sample_instance> const instances {
{ 0, 10.0, 0.0, -5.0, 1.5707963f, tex_a },
{ 1, 1234.5, 12.25, -6789.0, 0.0f, 0xffffffffu },
{ 2, -42.0, 3.0, 7.0, 3.14159f, tex_b },
};
byte_writer strs_payload;
strings.serialize( strs_payload );
byte_writer inst_payload;
inst_payload.put_u32( static_cast<std::uint32_t>( instances.size() ) );
for( auto const &i : instances ) {
inst_payload.put_u32( i.proto_id );
inst_payload.put_f64( i.x );
inst_payload.put_f64( i.y );
inst_payload.put_f64( i.z );
inst_payload.put_f32( i.yaw );
inst_payload.put_u32( i.texture_override );
}
container_writer writer( file_kind::tile );
writer.add_chunk( chunk::strs, strs_payload );
writer.add_chunk( chunk::inst, inst_payload );
return writer.data();
}
void
verify_tile( std::vector<std::uint8_t> const &bytes ) {
container_reader reader( bytes.data(), bytes.size() );
check( reader.kind() == file_kind::tile, "tile kind" );
check( reader.version() == kVersion, "tile version" );
string_pool pool;
bool saw_strs { false };
bool saw_inst { false };
chunk_view chunk;
while( reader.next( chunk ) ) {
if( chunk.id == chunk::strs ) {
saw_strs = true;
auto r { chunk.reader() };
pool.deserialize( r );
check( pool.size() == 3, "string count" );
check( pool.get( 0 ) == "textures/rail.dds", "string 0" );
check( pool.get( 2 ) == "models/sem.e3d", "string 2" );
}
else if( chunk.id == chunk::inst ) {
saw_inst = true;
auto r { chunk.reader() };
auto const count { r.get_u32() };
check( count == 3, "instance count" );
auto const p0 { r.get_u32() };
auto const x0 { r.get_f64() };
auto const y0 { r.get_f64() };
auto const z0 { r.get_f64() };
auto const yaw0 { r.get_f32() };
auto const tex0 { r.get_u32() };
check( p0 == 0, "inst0 proto" );
check( x0 == 10.0 && y0 == 0.0 && z0 == -5.0, "inst0 pos (f64 exact)" );
check( std::fabs( yaw0 - 1.5707963f ) < 1e-6f, "inst0 yaw" );
check( tex0 == 0, "inst0 tex override id" );
// second record: verify large f64 coordinate keeps full precision
auto const p1 { r.get_u32() };
auto const x1 { r.get_f64() };
(void)r.get_f64();
auto const z1 { r.get_f64() };
(void)r.get_f32();
auto const tex1 { r.get_u32() };
check( p1 == 1, "inst1 proto" );
check( x1 == 1234.5 && z1 == -6789.0, "inst1 large coord precision" );
check( tex1 == 0xffffffffu, "inst1 no tex override" );
}
}
check( saw_strs, "tile has STRS" );
check( saw_inst, "tile has INST" );
}
void
verify_trgr_roundtrip() {
std::vector<sample_edge> const edges {
{ 0, 1, 1, 0 },
{ 1, 2, 1, 0 },
{ 2, 0, 1, 0 },
};
byte_writer payload;
payload.put_u32( static_cast<std::uint32_t>( edges.size() ) );
for( auto const &e : edges ) {
payload.put_u32( e.track );
payload.put_u32( e.neighbour );
payload.put_u8( e.end );
payload.put_u8( e.neighbour_end );
}
container_writer writer( file_kind::sim );
writer.add_chunk( chunk::trgr, payload );
auto const bytes { writer.data() };
container_reader reader( bytes.data(), bytes.size() );
check( reader.kind() == file_kind::sim, "sim kind" );
chunk_view chunk;
bool ok { false };
while( reader.next( chunk ) ) {
if( chunk.id != chunk::trgr ) {
continue;
}
auto r { chunk.reader() };
auto const count { r.get_u32() };
check( count == edges.size(), "edge count" );
for( auto const &expected : edges ) {
auto const track { r.get_u32() };
auto const neighbour { r.get_u32() };
auto const end { r.get_u8() };
auto const nend { r.get_u8() };
check(
track == expected.track && neighbour == expected.neighbour &&
end == expected.end && nend == expected.neighbour_end,
"edge values" );
}
ok = true;
}
check( ok, "sim has TRGR" );
}
void
verify_truncation_is_rejected() {
auto bytes { build_tile() };
bytes.resize( bytes.size() - 4 ); // chop the tail of the last chunk
bool threw { false };
try {
container_reader reader( bytes.data(), bytes.size() );
chunk_view chunk;
while( reader.next( chunk ) ) {
auto r { chunk.reader() };
// force payload reads
while( r.remaining() > 0 ) {
(void)r.get_u8();
}
}
}
catch( parse_error const & ) {
threw = true;
}
check( threw, "truncated file rejected with parse_error" );
}
void
verify_scene_roundtrip() {
string_table strings;
auto const sem_model { strings.intern( "models/sem.e3d" ) };
auto const sem_tex { strings.intern( "textures/sem.dds" ) };
auto const grass { strings.intern( "textures/grass.dds" ) };
auto const tex_override { strings.intern( "textures/sem_winter.dds" ) };
std::vector<model_prototype> protos( 2 );
protos[ 0 ].model_file = sem_model;
protos[ 0 ].texture_file = sem_tex;
protos[ 0 ].flags = proto_flag::instanceable | proto_flag::transition;
protos[ 0 ].range_min = 0.f;
protos[ 0 ].range_max = 4000.f;
protos[ 0 ].light_states = { 1.f, 0.5f, 0.f };
protos[ 0 ].light_colors = { 0xff0000ffu, 0x00ff00ffu };
protos[ 1 ].model_file = strings.intern( "models/tree.e3d" );
protos[ 1 ].flags = proto_flag::instanceable;
std::vector<model_instance> instances( 3 );
instances[ 0 ] = { 0, 100.0, 0.0, -200.0, 0.f, 90.f, 0.f, 1.f, 1.f, 1.f, kNoString, 7 };
instances[ 1 ] = { 0, 999999.5, 1.25, -888888.0, 0.f, 0.f, 0.f, 2.f, 2.f, 2.f, tex_override, 3 };
instances[ 2 ] = { 1, -5.0, 0.0, 5.0, 0.f, 45.f, 0.f, 1.f, 1.f, 1.f, kNoString, 0xffu };
std::vector<terrain_mesh> meshes( 1 );
meshes[ 0 ].material = grass;
meshes[ 0 ].translucent = false;
meshes[ 0 ].ox = 500000.0;
meshes[ 0 ].oy = 0.0;
meshes[ 0 ].oz = -500000.0;
meshes[ 0 ].vertices = {
{ 0.f, 0.f, 0.f, 0.f, 1.f, 0.f, 0.f, 0.f },
{ 1000.f, 0.f, 0.f, 0.f, 1.f, 0.f, 1.f, 0.f },
{ 0.f, 0.f, -1000.f, 0.f, 1.f, 0.f, 0.f, 1.f },
};
byte_writer strs_payload;
strings.serialize( strs_payload );
byte_writer prot_payload;
write_prototypes( prot_payload, protos );
byte_writer inst_payload;
write_instances( inst_payload, instances );
byte_writer mesh_payload;
write_terrain_meshes( mesh_payload, meshes );
container_writer writer( file_kind::tile );
writer.add_chunk( chunk::strs, strs_payload );
writer.add_chunk( chunk::prot, prot_payload );
writer.add_chunk( chunk::inst, inst_payload );
writer.add_chunk( chunk::mesh, mesh_payload );
auto const bytes { writer.data() };
container_reader reader( bytes.data(), bytes.size() );
string_pool pool;
std::vector<model_prototype> rp;
std::vector<model_instance> ri;
std::vector<terrain_mesh> rm;
chunk_view chunk;
while( reader.next( chunk ) ) {
auto r { chunk.reader() };
if( chunk.id == chunk::strs ) {
pool.deserialize( r );
}
else if( chunk.id == chunk::prot ) {
rp = read_prototypes( r );
}
else if( chunk.id == chunk::inst ) {
ri = read_instances( r );
}
else if( chunk.id == chunk::mesh ) {
rm = read_terrain_meshes( r );
}
}
check( rp.size() == 2, "proto count" );
check( rp[ 0 ].flags == ( proto_flag::instanceable | proto_flag::transition ), "proto0 flags" );
check( rp[ 0 ].range_max == 4000.f, "proto0 range" );
check( rp[ 0 ].light_states.size() == 3 && rp[ 0 ].light_states[ 1 ] == 0.5f, "proto0 light states" );
check( rp[ 0 ].light_colors.size() == 2 && rp[ 0 ].light_colors[ 0 ] == 0xff0000ffu, "proto0 light colors" );
check( pool.get( rp[ 0 ].model_file ) == "models/sem.e3d", "proto0 model string" );
check( ri.size() == 3, "instance count" );
check( ri[ 0 ].proto == 0 && ri[ 0 ].cell_id == 7, "inst0 proto/cell" );
check( ri[ 0 ].sx == 1.f && ri[ 0 ].texture_override == kNoString, "inst0 defaults survive" );
check( ri[ 1 ].x == 999999.5 && ri[ 1 ].z == -888888.0, "inst1 large coord precision" );
check( ri[ 1 ].sx == 2.f && ri[ 1 ].sy == 2.f && ri[ 1 ].sz == 2.f, "inst1 scale" );
check( pool.get( ri[ 1 ].texture_override ) == "textures/sem_winter.dds", "inst1 override" );
check( ri[ 2 ].cell_id == 0xffu, "inst2 no cell" );
check( rm.size() == 1, "mesh count" );
check( rm[ 0 ].ox == 500000.0 && rm[ 0 ].oz == -500000.0, "mesh origin f64 precision" );
check( rm[ 0 ].vertices.size() == 3, "mesh vertex count" );
check( rm[ 0 ].vertices[ 1 ].px == 1000.f && rm[ 0 ].vertices[ 1 ].u == 1.f, "mesh vertex data" );
check( pool.get( rm[ 0 ].material ) == "textures/grass.dds", "mesh material string" );
}
void
verify_records_roundtrip() {
string_table s;
// --- tracks ---
std::vector<track_record> tracks( 1 );
tracks[ 0 ].node.name = s.intern( "track_42" );
tracks[ 0 ].node.type = s.intern( "track" );
tracks[ 0 ].node.area_center = { 123456.5, 0.0, -654321.0 };
tracks[ 0 ].node.range_sq_max = 1.0e12;
tracks[ 0 ].track_type = 2; // switch
tracks[ 0 ].category = 1;
tracks[ 0 ].length = 50.0f;
tracks[ 0 ].environment = 3;
tracks[ 0 ].has_visibility = true;
tracks[ 0 ].visibility.material1 = s.intern( "rail.mat" );
tracks[ 0 ].visibility.tex_length = 8.f;
tracks[ 0 ].paths.push_back( { { 1.0, 2.0, 3.0 }, 0.1, { 4.0, 5.0, 6.0 }, { 7.0, 8.0, 9.0 }, { 10.0, 11.0, 12.0 }, 0.2, 300.0 } );
tracks[ 0 ].tail_keywords.emplace_back( s.intern( "event0" ), s.intern( "ev_start" ) );
// --- traction ---
std::vector<traction_record> traction( 1 );
traction[ 0 ].node.name = s.intern( "trakcja1" );
traction[ 0 ].power_supply_name = s.intern( "psupply" );
traction[ 0 ].nominal_voltage = 3000.f;
traction[ 0 ].material = 1;
traction[ 0 ].wire_p1 = { 0.0, 6.0, 0.0 };
traction[ 0 ].wire_p2 = { 100.0, 6.0, 0.0 };
traction[ 0 ].wire_count = 2;
traction[ 0 ].has_parallel = true;
traction[ 0 ].parallel_name = s.intern( "trakcja0" );
// --- power sources ---
std::vector<power_source_record> power( 1 );
power[ 0 ].node.name = s.intern( "psupply" );
power[ 0 ].position = { 50.0, 0.0, -50.0 };
power[ 0 ].nominal_voltage = 3000.f;
power[ 0 ].max_output_current = 4000.f;
power[ 0 ].modifier = 2;
// --- memcells ---
std::vector<memcell_record> memcells( 1 );
memcells[ 0 ].node.name = s.intern( "mc_station" );
memcells[ 0 ].text = s.intern( "PASS depot 1" );
memcells[ 0 ].value1 = 12.5;
memcells[ 0 ].value2 = -7.0;
memcells[ 0 ].has_track = true;
memcells[ 0 ].track_name = s.intern( "track_42" );
// --- launchers ---
std::vector<launcher_record> launchers( 1 );
launchers[ 0 ].node.name = s.intern( "launch1" );
launchers[ 0 ].location = { 200.0, 1.0, -200.0 };
launchers[ 0 ].radius_squared = 400.0;
launchers[ 0 ].activation_key = 65;
launchers[ 0 ].event1_name = s.intern( "ev_start" );
launchers[ 0 ].has_condition = true;
launchers[ 0 ].condition.memcell_name = s.intern( "mc_station" );
launchers[ 0 ].condition.compare_value1 = 12.5;
launchers[ 0 ].condition.check_mask = 3;
launchers[ 0 ].launch_hour = 14;
launchers[ 0 ].launch_minute = 30;
// --- events ---
std::vector<event_record> events( 1 );
events[ 0 ].name = s.intern( "ev_start" );
events[ 0 ].type = 7; // multiple
events[ 0 ].delay = 2.5;
events[ 0 ].passive = true;
events[ 0 ].targets = { s.intern( "ev_a" ), s.intern( "ev_b" ) };
events[ 0 ].payload.emplace_back( s.intern( "k1" ), s.intern( "v1" ) );
// --- sounds ---
std::vector<sound_record> sounds( 1 );
sounds[ 0 ].node.name = s.intern( "snd1" );
sounds[ 0 ].location = { -10.0, 2.0, 10.0 };
sounds[ 0 ].wav_file = s.intern( "ambient.wav" );
// --- dynamics ---
std::vector<dynamic_record> dynamics( 1 );
dynamics[ 0 ].node.name = s.intern( "veh1" );
dynamics[ 0 ].data_folder = s.intern( "pkp/ep07" );
dynamics[ 0 ].mmd_file = s.intern( "ep07.mmd" );
dynamics[ 0 ].track_name = s.intern( "track_42" );
dynamics[ 0 ].offset = 25.0;
dynamics[ 0 ].coupling = 3;
dynamics[ 0 ].velocity = 40.f;
dynamics[ 0 ].has_destination = true;
dynamics[ 0 ].destination = s.intern( "Warszawa" );
dynamics[ 0 ].has_trainset = true;
dynamics[ 0 ].trainset_index = 5;
byte_writer strs_payload, trak_p, trac_p, pwrs_p, memc_p, laun_p, evnt_p, sond_p, dynm_p;
s.serialize( strs_payload );
write_tracks( trak_p, tracks );
write_traction( trac_p, traction );
write_power_sources( pwrs_p, power );
write_memcells( memc_p, memcells );
write_launchers( laun_p, launchers );
write_events( evnt_p, events );
write_sounds( sond_p, sounds );
write_dynamics( dynm_p, dynamics );
container_writer w( file_kind::sim );
w.add_chunk( chunk::strs, strs_payload );
w.add_chunk( chunk::trak, trak_p );
w.add_chunk( chunk::trac, trac_p );
w.add_chunk( chunk::pwrs, pwrs_p );
w.add_chunk( chunk::memc, memc_p );
w.add_chunk( chunk::laun, laun_p );
w.add_chunk( chunk::evnt, evnt_p );
w.add_chunk( chunk::sond, sond_p );
w.add_chunk( chunk::dynm, dynm_p );
auto const bytes { w.data() };
container_reader r( bytes.data(), bytes.size() );
check( r.kind() == file_kind::sim, "records sim kind" );
string_pool pool;
std::vector<track_record> rt;
std::vector<traction_record> rtr;
std::vector<power_source_record> rp;
std::vector<memcell_record> rmc;
std::vector<launcher_record> rl;
std::vector<event_record> re;
std::vector<sound_record> rs;
std::vector<dynamic_record> rd;
chunk_view c;
while( r.next( c ) ) {
auto rr { c.reader() };
switch( c.id ) {
case chunk::strs: pool.deserialize( rr ); break;
case chunk::trak: rt = read_tracks( rr ); break;
case chunk::trac: rtr = read_traction( rr ); break;
case chunk::pwrs: rp = read_power_sources( rr ); break;
case chunk::memc: rmc = read_memcells( rr ); break;
case chunk::laun: rl = read_launchers( rr ); break;
case chunk::evnt: re = read_events( rr ); break;
case chunk::sond: rs = read_sounds( rr ); break;
case chunk::dynm: rd = read_dynamics( rr ); break;
default: break;
}
}
check( rt.size() == 1 && pool.get( rt[ 0 ].node.name ) == "track_42", "track name" );
check( rt[ 0 ].node.area_center.x == 123456.5 && rt[ 0 ].node.area_center.z == -654321.0, "track center f64" );
check( rt[ 0 ].track_type == 2 && rt[ 0 ].environment == 3, "track type/env" );
check( rt[ 0 ].has_visibility && pool.get( rt[ 0 ].visibility.material1 ) == "rail.mat", "track visibility" );
check( rt[ 0 ].paths.size() == 1 && rt[ 0 ].paths[ 0 ].radius == 300.0, "track path" );
check( rt[ 0 ].paths[ 0 ].p_end.z == 12.0, "track path endpoint" );
check( rt[ 0 ].tail_keywords.size() == 1 && pool.get( rt[ 0 ].tail_keywords[ 0 ].second ) == "ev_start", "track keyword" );
check( rtr.size() == 1 && rtr[ 0 ].nominal_voltage == 3000.f, "traction voltage" );
check( rtr[ 0 ].wire_p2.x == 100.0 && rtr[ 0 ].wire_count == 2, "traction wire" );
check( rtr[ 0 ].has_parallel && pool.get( rtr[ 0 ].parallel_name ) == "trakcja0", "traction parallel" );
check( rp.size() == 1 && rp[ 0 ].max_output_current == 4000.f && rp[ 0 ].modifier == 2, "power source" );
check( rmc.size() == 1 && pool.get( rmc[ 0 ].text ) == "PASS depot 1", "memcell text" );
check( rmc[ 0 ].value1 == 12.5 && rmc[ 0 ].has_track, "memcell value/track" );
check( rl.size() == 1 && rl[ 0 ].activation_key == 65, "launcher key" );
check( rl[ 0 ].has_condition && rl[ 0 ].condition.check_mask == 3, "launcher condition" );
check( rl[ 0 ].launch_hour == 14 && rl[ 0 ].launch_minute == 30, "launcher time" );
check( re.size() == 1 && re[ 0 ].type == 7 && re[ 0 ].passive, "event type" );
check( re[ 0 ].targets.size() == 2 && pool.get( re[ 0 ].targets[ 1 ] ) == "ev_b", "event targets" );
check( re[ 0 ].payload.size() == 1 && pool.get( re[ 0 ].payload[ 0 ].first ) == "k1", "event payload" );
check( rs.size() == 1 && pool.get( rs[ 0 ].wav_file ) == "ambient.wav", "sound wav" );
check( rd.size() == 1 && pool.get( rd[ 0 ].data_folder ) == "pkp/ep07", "dynamic folder" );
check( rd[ 0 ].offset == 25.0 && rd[ 0 ].velocity == 40.f, "dynamic offset/vel" );
check( rd[ 0 ].has_destination && pool.get( rd[ 0 ].destination ) == "Warszawa", "dynamic destination" );
check( rd[ 0 ].has_trainset && rd[ 0 ].trainset_index == 5, "dynamic trainset" );
}
void
verify_extensions_roundtrip() {
string_table s;
// --- shapes (SHPE) ---
std::vector<shape_record> shapes( 1 );
shapes[ 0 ].node.name = s.intern( "shape0" );
shapes[ 0 ].node.type = s.intern( "triangles" );
shapes[ 0 ].translucent = true;
shapes[ 0 ].material = s.intern( "bricks.mat" );
shapes[ 0 ].lighting.diffuse[ 0 ] = 0.5f;
shapes[ 0 ].ox = 12345.5;
shapes[ 0 ].oy = 1.0;
shapes[ 0 ].oz = -54321.0;
shapes[ 0 ].vertices = {
{ 0.f, 0.f, 0.f, 0.f, 1.f, 0.f, 0.f, 0.f },
{ 10.f, 0.f, 0.f, 0.f, 1.f, 0.f, 1.f, 0.f },
{ 0.f, 5.f, 0.f, 0.f, 1.f, 0.f, 0.f, 1.f },
};
// --- lines (LINE) ---
std::vector<lines_record> lines( 1 );
lines[ 0 ].node.name = s.intern( "line0" );
lines[ 0 ].line_width = 2.f;
lines[ 0 ].ox = 100.0;
lines[ 0 ].vertices = { { 0.0, 0.0, 0.0 }, { 50.0, 0.0, -3.0 } };
// --- trainsets (TRST) ---
std::vector<trainset_record> trainsets( 1 );
trainsets[ 0 ].name = s.intern( "ts_warszawa" );
trainsets[ 0 ].track = s.intern( "track_42" );
trainsets[ 0 ].offset = 25.5f;
trainsets[ 0 ].velocity = 60.f;
trainsets[ 0 ].assignment.emplace_back( s.intern( "k" ), s.intern( "v" ) );
trainsets[ 0 ].vehicle_indices = { 0u, 1u, 2u };
trainsets[ 0 ].couplings = { 3, -1 };
trainsets[ 0 ].driver_index = 1u;
// --- includes (INCL) ---
std::vector<include_record> includes( 1 );
includes[ 0 ].source_line = 17;
includes[ 0 ].source_path = s.intern( "scenery/sub.inc" );
includes[ 0 ].binary_path = s.intern( "scenery/sub.eu7v2" );
includes[ 0 ].parameters = { s.intern( "p1" ), s.intern( "p2" ) };
includes[ 0 ].site_transform.origin_stack.push_back( { 10.0, 0.0, -20.0 } );
includes[ 0 ].site_transform.rotation = { 0.0, 90.0, 0.0 };
includes[ 0 ].site_transform.group_depth = 1;
// --- instance with full node ---
std::vector<model_prototype> protos( 1 );
protos[ 0 ].model_file = s.intern( "models/x.e3d" );
std::vector<model_instance> instances( 1 );
instances[ 0 ].proto = 0;
instances[ 0 ].x = 7.0;
instances[ 0 ].has_node = true;
instances[ 0 ].node.name = s.intern( "inst_named" );
instances[ 0 ].node.type = s.intern( "model" );
instances[ 0 ].node.range_sq_max = 1.0e9;
instances[ 0 ].node.visible = false;
// --- meta (META) ---
module_meta meta;
meta.first_init_count = 4;
meta.has_terrain_chunk = true;
meta.placement_origin_x = 1;
meta.placement_rotation_y = 4;
byte_writer strs_p, shpe_p, line_p, trst_p, incl_p, prot_p, inst_p, meta_p;
s.serialize( strs_p );
write_shapes( shpe_p, shapes );
write_lines( line_p, lines );
write_trainsets( trst_p, trainsets );
write_includes( incl_p, includes );
write_prototypes( prot_p, protos );
write_instances( inst_p, instances );
write_meta( meta_p, meta );
container_writer w( file_kind::module );
w.add_chunk( chunk::strs, strs_p );
w.add_chunk( chunk::meta, meta_p );
w.add_chunk( chunk::incl, incl_p );
w.add_chunk( chunk::shpe, shpe_p );
w.add_chunk( chunk::line, line_p );
w.add_chunk( chunk::trst, trst_p );
w.add_chunk( chunk::prot, prot_p );
w.add_chunk( chunk::inst, inst_p );
auto const bytes { w.data() };
container_reader r( bytes.data(), bytes.size() );
check( r.kind() == file_kind::module, "ext module kind" );
string_pool pool;
std::vector<shape_record> rsh;
std::vector<lines_record> rln;
std::vector<trainset_record> rts;
std::vector<include_record> rin;
std::vector<model_instance> ri;
module_meta rmeta;
chunk_view c;
while( r.next( c ) ) {
auto rr { c.reader() };
switch( c.id ) {
case chunk::strs: pool.deserialize( rr ); break;
case chunk::meta: rmeta = read_meta( rr ); break;
case chunk::incl: rin = read_includes( rr ); break;
case chunk::shpe: rsh = read_shapes( rr ); break;
case chunk::line: rln = read_lines( rr ); break;
case chunk::trst: rts = read_trainsets( rr ); break;
case chunk::inst: ri = read_instances( rr ); break;
default: break;
}
}
check( rsh.size() == 1 && rsh[ 0 ].translucent, "shape translucent" );
check( rsh[ 0 ].ox == 12345.5 && rsh[ 0 ].oz == -54321.0, "shape origin f64" );
check( rsh[ 0 ].vertices.size() == 3 && rsh[ 0 ].vertices[ 1 ].px == 10.f, "shape verts" );
check( pool.get( rsh[ 0 ].material ) == "bricks.mat", "shape material" );
check( rsh[ 0 ].lighting.diffuse[ 0 ] == 0.5f, "shape lighting" );
check( rln.size() == 1 && rln[ 0 ].line_width == 2.f, "line width" );
check( rln[ 0 ].vertices.size() == 2 && rln[ 0 ].vertices[ 1 ].x == 50.0, "line verts" );
check( rts.size() == 1 && pool.get( rts[ 0 ].name ) == "ts_warszawa", "trainset name" );
check( rts[ 0 ].vehicle_indices.size() == 3 && rts[ 0 ].vehicle_indices[ 2 ] == 2u, "trainset vehicles" );
check( rts[ 0 ].couplings.size() == 2 && rts[ 0 ].couplings[ 1 ] == -1, "trainset couplings" );
check( rts[ 0 ].driver_index == 1u && rts[ 0 ].offset == 25.5f, "trainset driver/offset" );
check( rin.size() == 1 && rin[ 0 ].source_line == 17, "include line" );
check( pool.get( rin[ 0 ].binary_path ) == "scenery/sub.eu7v2", "include binary path .eu7v2" );
check( rin[ 0 ].parameters.size() == 2, "include params" );
check( rin[ 0 ].site_transform.origin_stack.size() == 1 &&
rin[ 0 ].site_transform.origin_stack[ 0 ].z == -20.0, "include transform origin" );
check( rin[ 0 ].site_transform.rotation.y == 90.0 &&
rin[ 0 ].site_transform.group_depth == 1, "include transform rot/group" );
check( ri.size() == 1 && ri[ 0 ].has_node, "inst has node" );
check( pool.get( ri[ 0 ].node.name ) == "inst_named" && !ri[ 0 ].node.visible, "inst node fields" );
check( ri[ 0 ].node.range_sq_max == 1.0e9, "inst node range" );
check( rmeta.first_init_count == 4 && rmeta.has_terrain_chunk, "meta counts/flags" );
check( rmeta.placement_origin_x == 1 && rmeta.placement_rotation_y == 4, "meta placement" );
}
void
verify_file_io() {
auto const bytes { build_tile() };
char const *path { "eu7v2_roundtrip.bin" };
{
std::ofstream os( path, std::ios::binary );
os.write( reinterpret_cast<char const *>( bytes.data() ),
static_cast<std::streamsize>( bytes.size() ) );
}
std::ifstream is( path, std::ios::binary | std::ios::ate );
auto const size { static_cast<std::size_t>( is.tellg() ) };
is.seekg( 0 );
std::vector<std::uint8_t> loaded( size );
is.read( reinterpret_cast<char *>( loaded.data() ),
static_cast<std::streamsize>( size ) );
check( loaded == bytes, "file write/read byte-identical" );
verify_tile( loaded );
std::remove( path );
}
} // namespace
int
main() {
std::printf( "eu7v2 format core round-trip test\n" );
auto const tile { build_tile() };
verify_tile( tile );
verify_trgr_roundtrip();
verify_scene_roundtrip();
verify_records_roundtrip();
verify_extensions_roundtrip();
verify_truncation_is_rejected();
verify_file_io();
std::printf( "checks: %d, failures: %d\n", g_checks, g_failures );
if( g_failures == 0 ) {
std::printf( "OK\n" );
return 0;
}
std::printf( "FAILED\n" );
return 1;
}

View File

@@ -662,6 +662,18 @@ bool global_settings::ConfigParseSimulation(cParser& Parser, const std::string&
return true; return true;
} }
if (token == "eu7.bake.cpu.percent")
{
ParseOneClamped(Parser, eu7_bake_cpu_percent, 1, 100);
return true;
}
if (token == "eu7.v2.runtime")
{
ParseOne(Parser, eu7v2_runtime, 1, false);
return true;
}
if (token == "inactivepause") if (token == "inactivepause")
{ {
ParseOne(Parser, bInactivePause); ParseOne(Parser, bInactivePause);
@@ -1616,6 +1628,8 @@ global_settings::export_as_text( std::ostream &Output ) const {
export_as_text( Output, "eu7.pack.mesh.loader.workers", eu7_pack_mesh_loader_workers ); export_as_text( Output, "eu7.pack.mesh.loader.workers", eu7_pack_mesh_loader_workers );
export_as_text( Output, "eu7.auto.bake", eu7_auto_bake ); export_as_text( Output, "eu7.auto.bake", eu7_auto_bake );
export_as_text( Output, "eu7.bake.threads", eu7_bake_threads ); export_as_text( Output, "eu7.bake.threads", eu7_bake_threads );
export_as_text( Output, "eu7.bake.cpu.percent", eu7_bake_cpu_percent );
export_as_text( Output, "eu7.v2.runtime", eu7v2_runtime );
export_as_text( Output, "inactivepause", bInactivePause ); export_as_text( Output, "inactivepause", bInactivePause );
export_as_text( Output, "slowmotion", iSlowMotionMask ); export_as_text( Output, "slowmotion", iSlowMotionMask );
export_as_text( Output, "hideconsole", bHideConsole ); export_as_text( Output, "hideconsole", bHideConsole );

View File

@@ -86,6 +86,8 @@ struct global_settings {
int eu7_pack_mesh_loader_workers{ 0 }; // 0 = main-thread only (background mesh threads unsafe with Global) int eu7_pack_mesh_loader_workers{ 0 }; // 0 = main-thread only (background mesh threads unsafe with Global)
bool eu7_auto_bake{ true }; // przy ladowaniu mapy: bake SCM→EU7 gdy brak lub nieaktualny .eu7 bool eu7_auto_bake{ true }; // przy ladowaniu mapy: bake SCM→EU7 gdy brak lub nieaktualny .eu7
int eu7_bake_threads{ 0 }; // watki bake (0 = auto = hardware_concurrency) int eu7_bake_threads{ 0 }; // watki bake (0 = auto = hardware_concurrency)
int eu7_bake_cpu_percent{ 80 }; // bake: docelowy % rdzeni logicznych, gdy eu7_bake_threads=0 (auto)
bool eu7v2_runtime{ false }; // eksperyment: transkoduj/laduj moduly przez chudy format eu7v2 (.eu7.v2)
// logs // logs
bool priorityLoadText3D{false}; // ladowanie T3D priorytetowo bool priorityLoadText3D{false}; // ladowanie T3D priorytetowo
int iWriteLogEnabled{ 3 }; // maska bitowa: 1-zapis do pliku, 2-okienko, 4-nazwy torów int iWriteLogEnabled{ 3 }; // maska bitowa: 1-zapis do pliku, 2-okienko, 4-nazwy torów

View File

@@ -235,34 +235,40 @@ std::string cParser::readTokenFromStream(bool ToLower, const char *Break)
std::string token; std::string token;
token.reserve(64); token.reserve(64);
const auto breakTable = makeBreakTable(Break); // rebuild the separator lookup table only when the break set actually changes;
char c = 0; // callers overwhelmingly pass the same string literal, so this stays cached.
if (Break != m_breakTableKey) {
m_breakTable = makeBreakTable(Break);
m_breakTableKey = Break;
}
const auto &breakTable = m_breakTable;
int ci = 0;
while ((ci = mStream->get()) != EOF) {
char c = static_cast<char>(ci);
if (c == '\n') {
++mLine;
}
while (token.empty() && mStream->peek() != EOF) { const unsigned char uc = static_cast<unsigned char>(c);
while (mStream->peek() != EOF) { // idk why but with mStream->get(c) not all cars are loaded if (breakTable[uc]) {
c = static_cast<char>(mStream->get()); // separator ends token (or continues skipping if token empty)
if (c == '\n') { if (!token.empty())
++mLine; break;
} continue;
}
const unsigned char uc = static_cast<unsigned char>(c); if (ToLower) c = toLowerChar(c);
if (breakTable[uc]) { token.push_back(c);
// separator ends token (or continues skipping if token empty)
if (!token.empty())
break;
continue;
}
if (ToLower) c = toLowerChar(c); if (findQuotes(token)) {
token.push_back(c); continue; // glue quoted content
}
if (findQuotes(token)) { if (skipComments && trimComments(token)) {
continue; // glue quoted content // comment stripped: return the token if we already have one,
} // otherwise keep scanning for the next real token
if (skipComments && trimComments(token)) { if (!token.empty())
break; // don't glue tokens separated by comment break;
}
} }
} }

View File

@@ -14,10 +14,30 @@ http://mozilla.org/MPL/2.0/.
#include <fstream> #include <fstream>
#include <vector> #include <vector>
#include <map> #include <map>
#include <array>
#include <charconv>
#include <type_traits>
///////////////////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////////////////////
// cParser -- generic class for parsing text data, either from file or provided string // cParser -- generic class for parsing text data, either from file or provided string
namespace cparser_detail {
// Types for which std::from_chars matches the legacy stringstream extraction semantics.
// Character types are excluded on purpose: narrow-stream extraction of char/wchar_t reads a
// single character, whereas std::from_chars would parse an integer.
template<typename T>
inline constexpr bool use_from_chars_v =
( std::is_integral_v<T>
&& !std::is_same_v<T, bool>
&& !std::is_same_v<T, char>
&& !std::is_same_v<T, signed char>
&& !std::is_same_v<T, unsigned char>
&& !std::is_same_v<T, wchar_t>
&& !std::is_same_v<T, char16_t>
&& !std::is_same_v<T, char32_t> )
|| std::is_floating_point_v<T>;
} // namespace cparser_detail
class cParser //: public std::stringstream class cParser //: public std::stringstream
{ {
public: public:
@@ -126,6 +146,9 @@ class cParser //: public std::stringstream
commentmap mComments { commentmap mComments {
commentmap::value_type( "/*", "*/" ), commentmap::value_type( "/*", "*/" ),
commentmap::value_type( "//", "\n" ) }; commentmap::value_type( "//", "\n" ) };
// cached separator lookup table to avoid rebuilding it on every token read
char const *m_breakTableKey { nullptr };
std::array<bool, 256> m_breakTable {};
std::shared_ptr<cParser> mIncludeParser; // child class to handle include directives. std::shared_ptr<cParser> mIncludeParser; // child class to handle include directives.
std::vector<std::string> parameters; // parameter list for included file. std::vector<std::string> parameters; // parameter list for included file.
std::deque<std::string> tokens; std::deque<std::string> tokens;
@@ -143,11 +166,34 @@ cParser::operator>>( Type_ &Right ) {
if( true == this->tokens.empty() ) { return *this; } if( true == this->tokens.empty() ) { return *this; }
std::stringstream converter( this->tokens.front() ); if constexpr( cparser_detail::use_from_chars_v<Type_> ) {
converter >> Right; std::string const &token = this->tokens.front();
this->tokens.pop_front(); char const *first { token.data() };
char const *const last { first + token.size() };
// legacy stream extraction accepts a leading '+', std::from_chars does not
if( first != last && *first == '+' ) {
++first;
}
Type_ value {};
auto const result { std::from_chars( first, last, value ) };
if( result.ec == std::errc() ) {
Right = value;
}
else {
// fall back to the legacy path for inputs from_chars rejects (inf/nan/hex/...)
std::stringstream converter( token );
converter >> Right;
}
this->tokens.pop_front();
return *this;
}
else {
std::stringstream converter( this->tokens.front() );
converter >> Right;
this->tokens.pop_front();
return *this; return *this;
}
} }
template<> template<>