mirror of
https://github.com/MaSzyna-EU07/maszyna.git
synced 2026-11-19 00:35:59 +01:00
test: scenario tests that run the simulator and check a vehicle's physicslog
tests/scenarios/run.py builds a test scenery from a game install's scenery (own trainsets removed, case fragment added), runs eu07 in ghostview with physicslog on, and checks what the vehicle did: starts, max speed, brakes, stops. First case: EU07 on td.scn driven by AI on the td/rozklad timetable, with the Stacja1 exit route set by an event launcher instead of shift+4. eu07 gets a -ini <file> option: an extra settings file read after eu07.ini, so a test run can switch to a window and turn on physicslog without touching the user's config. Local only: scenery and vehicle data are not in the repo.
This commit is contained in:
@@ -1195,12 +1195,22 @@ int eu07_application::init_settings(int Argc, char *Argv[])
|
||||
WriteLog(std::format("seed given on the command line: {} -> {}", Argv[i], Global.random_seed));
|
||||
}
|
||||
}
|
||||
else if (token == "-ini")
|
||||
{
|
||||
// extra settings file read on top of eu07.ini, so a run can override a few keys
|
||||
// (window size, physicslog...) without touching the user's own config
|
||||
if (i + 1 < Argc)
|
||||
{
|
||||
Global.LoadIniFile(Argv[++i]);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
std::cout << "usage: " << std::string(Argv[0]) << " [-s sceneryfilepath]"
|
||||
<< " [-v vehiclename]"
|
||||
<< " [-edit [sceneryfilepath]]"
|
||||
<< " [-seed number|word]" << std::endl;
|
||||
<< " [-seed number|word]"
|
||||
<< " [-ini settingsfile]" << std::endl;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
2
tests/scenarios/.gitignore
vendored
Normal file
2
tests/scenarios/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
out/
|
||||
__pycache__/
|
||||
66
tests/scenarios/README.md
Normal file
66
tests/scenarios/README.md
Normal file
@@ -0,0 +1,66 @@
|
||||
# Scenario tests
|
||||
|
||||
End-to-end checks that run the real simulator on a scenery and look at what a vehicle
|
||||
did: did it start, did it keep to the speed limit, did it brake and stop.
|
||||
|
||||
Scenery and vehicle data are not in this repo, so these run against a local game install
|
||||
only, not in CI.
|
||||
|
||||
## Running
|
||||
|
||||
```
|
||||
python tests/scenarios/run.py --game-dir "E:/Program Files (x86)/Steam/steamapps/common/MaSzyna"
|
||||
```
|
||||
|
||||
Python 3.10+, no extra packages. Options:
|
||||
|
||||
- `EU07_GAME_DIR` can replace `--game-dir`.
|
||||
- `--exe path/to/eu07.exe` picks the build to test; the default is the newest
|
||||
`cmake-build-*/bin/Release/eu07*.exe`. The build needs the `-ini` command line option.
|
||||
- name one or more cases to run only those, e.g. `run.py ... eu07_td_timetable`.
|
||||
- `--timespeed 4` runs the simulation faster than real time.
|
||||
|
||||
Exit code is 0 when every case passed. Logs of each run (`log.txt`, `errors.txt`,
|
||||
the vehicle's physicslog) land in `tests/scenarios/out/<case>/`.
|
||||
|
||||
## How it works
|
||||
|
||||
For each case the runner:
|
||||
|
||||
1. writes `scenery/autotest_<case>.scn` into the game dir: the base scenery with its own
|
||||
trainsets removed, the start time changed if the case says so, and the case's `.scn`
|
||||
fragment added: its trainsets after `FirstInit`, everything else (event launchers standing
|
||||
in for keys the scenery expects a player to press, extra events) before it,
|
||||
2. starts the simulator with `-s autotest_<case>.scn -v ghostview -ini out/<case>/autotest.ini`;
|
||||
ghostview leaves every train to the AI, and the extra ini turns on `physicslog`,
|
||||
runs in a 1280x720 window and does not pause when the window loses focus,
|
||||
3. waits until `physicslog/<vehicle>.dat` covers `duration` seconds, then closes the simulator,
|
||||
4. runs the checks on the physicslog.
|
||||
|
||||
## Writing a case
|
||||
|
||||
`cases/<name>.json`:
|
||||
|
||||
| key | meaning |
|
||||
| --- | --- |
|
||||
| `base_scenery` | scenery file in the game's `scenery/` dir |
|
||||
| `strip_trainsets` | drop the base scenery's trainsets (default `true`) |
|
||||
| `time` | replace the scenery start time, `HH:MM` |
|
||||
| `fragment` | `.scn` file in `cases/` added to the scenery: trainsets after `FirstInit` (before it they derail on unlinked track), the rest before it (launchers read after it never fire) |
|
||||
| `vehicle` | node name of the vehicle whose physicslog is checked |
|
||||
| `duration` | seconds of simulation to record |
|
||||
| `timeout` | wall-clock seconds before giving up, loading included |
|
||||
| `fail_on_errors` | fail when `errors.txt` is not empty (default `false`) |
|
||||
| `checks` | list of checks, below |
|
||||
|
||||
Checks (speeds in km/h, times in seconds of physicslog time):
|
||||
|
||||
- `starts` `min_speed`, `within`: reaches `min_speed` within `within` seconds.
|
||||
- `max_speed` `limit`, `tolerance`: never goes faster than `limit + tolerance`.
|
||||
- `brakes` `min_pressure`: after moving off, brake cylinder pressure reaches `min_pressure` bar
|
||||
(physicslog pressures are in bar whatever its header says; the brake pipe runs at 5.0).
|
||||
- `stops` `moving_speed`, `stop_speed`, `hold`: after reaching `moving_speed`, stays at or
|
||||
below `stop_speed` for `hold` seconds.
|
||||
|
||||
New check types go in `CHECKS` in `run.py`; each gets the physicslog rows and the check's
|
||||
parameters and returns `(passed, detail)`.
|
||||
15
tests/scenarios/cases/eu07_td_timetable.json
Normal file
15
tests/scenarios/cases/eu07_td_timetable.json
Normal file
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"description": "EU07 on td.scn, AI driver on the td/rozklad timetable: starts from Stacja1, keeps to the speed limit, brakes to a stop at Przystanek.",
|
||||
"base_scenery": "td.scn",
|
||||
"time": "10:34",
|
||||
"fragment": "eu07_td_timetable.scn",
|
||||
"vehicle": "autotest_eu07",
|
||||
"duration": 240,
|
||||
"timeout": 600,
|
||||
"checks": [
|
||||
{ "type": "starts", "min_speed": 20, "within": 150 },
|
||||
{ "type": "max_speed", "limit": 90, "tolerance": 3 },
|
||||
{ "type": "brakes", "min_pressure": 0.5 },
|
||||
{ "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 }
|
||||
]
|
||||
}
|
||||
11
tests/scenarios/cases/eu07_td_timetable.scn
Normal file
11
tests/scenarios/cases/eu07_td_timetable.scn
Normal file
@@ -0,0 +1,11 @@
|
||||
// added by tests/scenarios/run.py to the base scenery, whose own trainsets were removed:
|
||||
// the trainset goes after FirstInit, the launcher before it.
|
||||
// the EU07 stands where td.scn keeps its timetable train and gets the same timetable.
|
||||
trainset td/rozklad tdo_n1734 0.0 0.1
|
||||
node -1 0 autotest_eu07 dynamic pkp\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic
|
||||
endtrainset
|
||||
|
||||
// td.scn sets routes from the keyboard (shift+1..4 run events keyctrl01..04). keyctrl04 is the
|
||||
// way out of the track the train stands on: switch zwr02 and the exit signal stacja1_j at 40.
|
||||
// a launcher with a positive time fires once at that hour (HHMM), radius -1 means anywhere.
|
||||
node -1 0 autotest_route eventlauncher 0 0 0 -1 none 1034 keyctrl04 end
|
||||
330
tests/scenarios/run.py
Normal file
330
tests/scenarios/run.py
Normal file
@@ -0,0 +1,330 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scenario tests: run the real simulator on a scenery and check what a vehicle did.
|
||||
|
||||
A case (cases/<name>.json) names a base scenery from the game install, a fragment
|
||||
appended to it (usually a trainset with an AI driver), and checks on the vehicle's
|
||||
physics log. The runner:
|
||||
|
||||
1. builds scenery/autotest_<case>.scn in the game dir: the base scenery with its own
|
||||
trainsets removed, optionally a different start time, and the case fragment appended,
|
||||
2. starts eu07 with that scenery in ghostview (every train driven by AI) and an extra
|
||||
settings file (-ini) that turns on physicslog and runs in a window,
|
||||
3. waits until the vehicle's physicslog covers `duration` seconds of simulation,
|
||||
then closes the simulator,
|
||||
4. evaluates the checks on the log and prints PASS/FAIL per check.
|
||||
|
||||
Scenery and vehicle data are not part of this repo, so this runs against a local game
|
||||
install only (--game-dir or EU07_GAME_DIR), not in CI.
|
||||
|
||||
Usage:
|
||||
python tests/scenarios/run.py --game-dir "E:/.../MaSzyna" # all cases
|
||||
python tests/scenarios/run.py --game-dir ... eu07_td_timetable # chosen cases
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
REPO = HERE.parents[1]
|
||||
CASES_DIR = HERE / "cases"
|
||||
OUT_DIR = HERE / "out"
|
||||
|
||||
# physicslog columns, see TController::PhysicsLog() in vehicle/Driver.cpp.
|
||||
# Velocity is in km/h and pressures in bar, despite the header saying m/s and MPa.
|
||||
COL_TIME = 0
|
||||
COL_VELOCITY = 1
|
||||
COL_BRAKE_PRESS = 8
|
||||
|
||||
# settings layered over the user's eu07.ini for a test run
|
||||
TEST_INI = """\
|
||||
fullscreen no
|
||||
fullscreenwindowed no
|
||||
width 1280
|
||||
height 720
|
||||
inactivepause no
|
||||
soundenabled no
|
||||
physicslog yes
|
||||
multiplelogs no
|
||||
timespeed {timespeed}
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- scenery
|
||||
|
||||
|
||||
def split_trainsets(text: str) -> tuple[str, str]:
|
||||
"""Splits text into (everything else, the trainset ... endtrainset blocks)."""
|
||||
rest, trainsets = [], []
|
||||
inside = False
|
||||
for line in text.splitlines(keepends=True):
|
||||
tokens = line.split("//", 1)[0].split()
|
||||
if not inside and tokens and tokens[0].lower() == "trainset":
|
||||
inside = True
|
||||
if inside:
|
||||
trainsets.append(line)
|
||||
if any(t.lower() == "endtrainset" for t in tokens):
|
||||
inside = False
|
||||
continue
|
||||
rest.append(line)
|
||||
return "".join(rest), "".join(trainsets)
|
||||
|
||||
|
||||
def set_start_time(text: str, hhmm: str) -> str:
|
||||
return re.sub(r"(?im)^(\s*time\s+)\d{1,2}:\d{2}", lambda m: m.group(1) + hhmm, text, count=1)
|
||||
|
||||
|
||||
def build_scenery(case: dict, game_dir: Path, case_name: str) -> str:
|
||||
"""Writes the case scenery into the game's scenery dir and returns its file name."""
|
||||
scenery_dir = game_dir / "scenery"
|
||||
# sceneries are cp1250; latin-1 round-trips every byte, so nothing gets mangled
|
||||
text = (scenery_dir / case["base_scenery"]).read_bytes().decode("latin-1")
|
||||
if case.get("strip_trainsets", True):
|
||||
text, _ = split_trainsets(text)
|
||||
if "time" in case:
|
||||
text = set_start_time(text, case["time"])
|
||||
fragment = (CASES_DIR / case["fragment"]).read_bytes().decode("latin-1")
|
||||
# FirstInit binds event launchers to their events and links the tracks. so launchers and
|
||||
# events of the case go in before it (read after it they never fire), trainsets after it
|
||||
# (placed before it they stand on unlinked track and derail at once)
|
||||
before, trainsets = split_trainsets(fragment)
|
||||
header = f"\n// ---- autotest: {case_name} ----\n"
|
||||
firstinit = re.search(r"(?im)^\s*firstinit\b", text)
|
||||
if firstinit:
|
||||
text = text[: firstinit.start()] + header + before + "\n" + text[firstinit.start():]
|
||||
else:
|
||||
text += header + before + "\n"
|
||||
text += header + trainsets + "\n"
|
||||
name = f"autotest_{case_name}.scn"
|
||||
(scenery_dir / name).write_bytes(text.encode("latin-1"))
|
||||
return name
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- run
|
||||
|
||||
|
||||
def read_physicslog(path: Path) -> list[list[float]]:
|
||||
"""Returns rows of numeric columns up to BrakePress; skips the header and partial lines."""
|
||||
rows = []
|
||||
try:
|
||||
lines = path.read_text(errors="replace").splitlines()
|
||||
except FileNotFoundError:
|
||||
return rows
|
||||
for line in lines[1:]:
|
||||
parts = line.split()
|
||||
if len(parts) <= COL_BRAKE_PRESS:
|
||||
continue # line still being written
|
||||
try:
|
||||
rows.append([float(p) for p in parts[: COL_BRAKE_PRESS + 1]])
|
||||
except ValueError:
|
||||
continue
|
||||
return rows
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunResult:
|
||||
rows: list[list[float]] = field(default_factory=list)
|
||||
exit_code: int | None = None # None: still running when we closed it (the normal end)
|
||||
timed_out: bool = False
|
||||
wall_seconds: float = 0.0
|
||||
|
||||
|
||||
def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog: Path,
|
||||
duration: float, timeout: float) -> RunResult:
|
||||
physicslog.unlink(missing_ok=True)
|
||||
cmd = [str(exe), "-s", scenery, "-v", "ghostview", "-ini", str(ini)]
|
||||
print(" $", " ".join(f'"{c}"' if " " in c else c for c in cmd))
|
||||
started = time.monotonic()
|
||||
proc = subprocess.Popen(cmd, cwd=game_dir)
|
||||
result = RunResult()
|
||||
last_report = 0.0
|
||||
try:
|
||||
while True:
|
||||
time.sleep(1.0)
|
||||
result.rows = read_physicslog(physicslog)
|
||||
sim_t = result.rows[-1][COL_TIME] if result.rows else 0.0
|
||||
wall = time.monotonic() - started
|
||||
if wall - last_report >= 15:
|
||||
v = result.rows[-1][COL_VELOCITY] if result.rows else 0.0
|
||||
print(f" ... {wall:5.0f}s wall, sim {sim_t:6.1f}s, v={v:6.1f} km/h")
|
||||
last_report = wall
|
||||
if proc.poll() is not None:
|
||||
result.exit_code = proc.returncode
|
||||
break
|
||||
if sim_t >= duration:
|
||||
break
|
||||
if wall > timeout:
|
||||
result.timed_out = True
|
||||
break
|
||||
finally:
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
result.wall_seconds = time.monotonic() - started
|
||||
result.rows = read_physicslog(physicslog)
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- checks
|
||||
|
||||
|
||||
def check_starts(rows, min_speed, within):
|
||||
for r in rows:
|
||||
if r[COL_TIME] > within:
|
||||
break
|
||||
if r[COL_VELOCITY] >= min_speed:
|
||||
return True, f"reached {min_speed} km/h at t={r[COL_TIME]:.1f}s"
|
||||
top = max((r[COL_VELOCITY] for r in rows if r[COL_TIME] <= within), default=0.0)
|
||||
return False, f"top speed in the first {within}s was {top:.1f} km/h, wanted {min_speed}"
|
||||
|
||||
|
||||
def check_max_speed(rows, limit, tolerance=0.0):
|
||||
worst = max(rows, key=lambda r: r[COL_VELOCITY])
|
||||
v, t = worst[COL_VELOCITY], worst[COL_TIME]
|
||||
ok = v <= limit + tolerance
|
||||
return ok, f"top speed {v:.1f} km/h at t={t:.1f}s (limit {limit} +{tolerance})"
|
||||
|
||||
|
||||
def _first_moving(rows, moving_speed):
|
||||
return next((i for i, r in enumerate(rows) if r[COL_VELOCITY] >= moving_speed), None)
|
||||
|
||||
|
||||
def check_brakes(rows, min_pressure, moving_speed=20):
|
||||
start = _first_moving(rows, moving_speed)
|
||||
if start is None:
|
||||
return False, "never moved, so never braked"
|
||||
for r in rows[start:]:
|
||||
if r[COL_BRAKE_PRESS] >= min_pressure:
|
||||
return True, f"brake cylinder {r[COL_BRAKE_PRESS]:.2f} bar at t={r[COL_TIME]:.1f}s, v={r[COL_VELOCITY]:.1f} km/h"
|
||||
return False, f"brake cylinder pressure never reached {min_pressure} bar after moving off"
|
||||
|
||||
|
||||
def check_stops(rows, moving_speed, stop_speed, hold):
|
||||
start = _first_moving(rows, moving_speed)
|
||||
if start is None:
|
||||
return False, f"never reached {moving_speed} km/h"
|
||||
stopped_at = None
|
||||
for r in rows[start:]:
|
||||
if r[COL_VELOCITY] <= stop_speed:
|
||||
if stopped_at is None:
|
||||
stopped_at = r[COL_TIME]
|
||||
if r[COL_TIME] - stopped_at >= hold:
|
||||
return True, f"came to a stop at t={stopped_at:.1f}s"
|
||||
else:
|
||||
stopped_at = None
|
||||
return False, f"did not stay below {stop_speed} km/h for {hold}s after moving off"
|
||||
|
||||
|
||||
CHECKS = {
|
||||
"starts": check_starts,
|
||||
"max_speed": check_max_speed,
|
||||
"brakes": check_brakes,
|
||||
"stops": check_stops,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- main
|
||||
|
||||
|
||||
def find_exe(explicit: str | None) -> Path:
|
||||
if explicit:
|
||||
return Path(explicit).resolve()
|
||||
candidates = sorted(REPO.glob("cmake-build-*/bin/Release/eu07*.exe"), key=lambda p: p.stat().st_mtime)
|
||||
if not candidates:
|
||||
sys.exit("no eu07 executable found under cmake-build-*/bin/Release, pass --exe")
|
||||
return candidates[-1]
|
||||
|
||||
|
||||
def count_lines(path: Path) -> int:
|
||||
try:
|
||||
return sum(1 for line in path.read_text(errors="replace").splitlines() if line.strip())
|
||||
except FileNotFoundError:
|
||||
return 0
|
||||
|
||||
|
||||
def run_case(name: str, exe: Path, game_dir: Path, timespeed: float) -> bool:
|
||||
case = json.loads((CASES_DIR / f"{name}.json").read_text(encoding="utf-8"))
|
||||
print(f"\n=== {name}: {case.get('description', '')}")
|
||||
out = OUT_DIR / name
|
||||
shutil.rmtree(out, ignore_errors=True)
|
||||
out.mkdir(parents=True)
|
||||
|
||||
scenery = build_scenery(case, game_dir, name)
|
||||
ini = out / "autotest.ini"
|
||||
ini.write_text(TEST_INI.format(timespeed=timespeed), encoding="ascii")
|
||||
physicslog = game_dir / "physicslog" / f"{case['vehicle']}.dat"
|
||||
|
||||
result = run_simulator(exe, game_dir, scenery, ini, physicslog,
|
||||
float(case["duration"]), float(case.get("timeout", 600)))
|
||||
|
||||
for f in ("log.txt", "errors.txt"):
|
||||
if (game_dir / f).exists():
|
||||
shutil.copy2(game_dir / f, out / f)
|
||||
if physicslog.exists():
|
||||
shutil.copy2(physicslog, out / physicslog.name)
|
||||
|
||||
ok = True
|
||||
sim_t = result.rows[-1][COL_TIME] if result.rows else 0.0
|
||||
print(f" ran {result.wall_seconds:.0f}s wall, {sim_t:.1f}s of physicslog, results in {out.relative_to(REPO)}")
|
||||
if result.exit_code is not None:
|
||||
print(f" FAIL simulator exited on its own with code {result.exit_code} (crash?)")
|
||||
ok = False
|
||||
if result.timed_out:
|
||||
print(f" FAIL timed out before {case['duration']}s of simulation")
|
||||
ok = False
|
||||
if not result.rows:
|
||||
print(f" FAIL no physicslog for '{case['vehicle']}' ({physicslog})")
|
||||
return False
|
||||
|
||||
for check in case["checks"]:
|
||||
params = {k: v for k, v in check.items() if k != "type"}
|
||||
passed, detail = CHECKS[check["type"]](result.rows, **params)
|
||||
print(f" {'PASS' if passed else 'FAIL'} {check['type']}: {detail}")
|
||||
ok &= passed
|
||||
|
||||
errors = count_lines(out / "errors.txt")
|
||||
if errors:
|
||||
print(f" note errors.txt has {errors} line(s)")
|
||||
if case.get("fail_on_errors") and errors:
|
||||
ok = False
|
||||
return ok
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("cases", nargs="*", help="case names from cases/ (default: all)")
|
||||
ap.add_argument("--game-dir", default=os.environ.get("EU07_GAME_DIR"),
|
||||
help="game install with scenery/ and dynamic/ (or EU07_GAME_DIR)")
|
||||
ap.add_argument("--exe", help="simulator executable (default: newest cmake-build-*/bin/Release/eu07*.exe)")
|
||||
ap.add_argument("--timespeed", type=float, default=1.0, help="simulation time multiplier")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not args.game_dir:
|
||||
sys.exit("pass --game-dir or set EU07_GAME_DIR")
|
||||
game_dir = Path(args.game_dir).resolve()
|
||||
exe = find_exe(args.exe)
|
||||
print(f"exe: {exe}\ngame: {game_dir}")
|
||||
|
||||
names = args.cases or sorted(p.stem for p in CASES_DIR.glob("*.json"))
|
||||
results = {n: run_case(n, exe, game_dir, args.timespeed) for n in names}
|
||||
|
||||
print("\n=== summary")
|
||||
for n, ok in results.items():
|
||||
print(f" {'PASS' if ok else 'FAIL'} {n}")
|
||||
return 0 if all(results.values()) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user