From 798fd89dd0fc616967485a06530f9b40a88791e9 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Wed, 30 Sep 2026 19:16:46 +0200 Subject: [PATCH 01/28] test: scenario tests that run the simulator and check a vehicle's physicslog tests/scenarios/run.py builds a test scenery from a game install's scenery (own trainsets removed, case fragment added), runs eu07 in ghostview with physicslog on, and checks what the vehicle did: starts, max speed, brakes, stops. First case: EU07 on td.scn driven by AI on the td/rozklad timetable, with the Stacja1 exit route set by an event launcher instead of shift+4. eu07 gets a -ini option: an extra settings file read after eu07.ini, so a test run can switch to a window and turn on physicslog without touching the user's config. Local only: scenery and vehicle data are not in the repo. --- application/application.cpp | 12 +- tests/scenarios/.gitignore | 2 + tests/scenarios/README.md | 66 ++++ tests/scenarios/cases/eu07_td_timetable.json | 15 + tests/scenarios/cases/eu07_td_timetable.scn | 11 + tests/scenarios/run.py | 330 +++++++++++++++++++ 6 files changed, 435 insertions(+), 1 deletion(-) create mode 100644 tests/scenarios/.gitignore create mode 100644 tests/scenarios/README.md create mode 100644 tests/scenarios/cases/eu07_td_timetable.json create mode 100644 tests/scenarios/cases/eu07_td_timetable.scn create mode 100644 tests/scenarios/run.py diff --git a/application/application.cpp b/application/application.cpp index 9b7b8285..ee381f98 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -1195,12 +1195,22 @@ int eu07_application::init_settings(int Argc, char *Argv[]) WriteLog(std::format("seed given on the command line: {} -> {}", Argv[i], Global.random_seed)); } } + else if (token == "-ini") + { + // extra settings file read on top of eu07.ini, so a run can override a few keys + // (window size, physicslog...) without touching the user's own config + if (i + 1 < Argc) + { + Global.LoadIniFile(Argv[++i]); + } + } else { std::cout << "usage: " << std::string(Argv[0]) << " [-s sceneryfilepath]" << " [-v vehiclename]" << " [-edit [sceneryfilepath]]" - << " [-seed number|word]" << std::endl; + << " [-seed number|word]" + << " [-ini settingsfile]" << std::endl; return -1; } } diff --git a/tests/scenarios/.gitignore b/tests/scenarios/.gitignore new file mode 100644 index 00000000..ea93687f --- /dev/null +++ b/tests/scenarios/.gitignore @@ -0,0 +1,2 @@ +out/ +__pycache__/ diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md new file mode 100644 index 00000000..77edb5a9 --- /dev/null +++ b/tests/scenarios/README.md @@ -0,0 +1,66 @@ +# Scenario tests + +End-to-end checks that run the real simulator on a scenery and look at what a vehicle +did: did it start, did it keep to the speed limit, did it brake and stop. + +Scenery and vehicle data are not in this repo, so these run against a local game install +only, not in CI. + +## Running + +``` +python tests/scenarios/run.py --game-dir "E:/Program Files (x86)/Steam/steamapps/common/MaSzyna" +``` + +Python 3.10+, no extra packages. Options: + +- `EU07_GAME_DIR` can replace `--game-dir`. +- `--exe path/to/eu07.exe` picks the build to test; the default is the newest + `cmake-build-*/bin/Release/eu07*.exe`. The build needs the `-ini` command line option. +- name one or more cases to run only those, e.g. `run.py ... eu07_td_timetable`. +- `--timespeed 4` runs the simulation faster than real time. + +Exit code is 0 when every case passed. Logs of each run (`log.txt`, `errors.txt`, +the vehicle's physicslog) land in `tests/scenarios/out//`. + +## How it works + +For each case the runner: + +1. writes `scenery/autotest_.scn` into the game dir: the base scenery with its own + trainsets removed, the start time changed if the case says so, and the case's `.scn` + fragment added: its trainsets after `FirstInit`, everything else (event launchers standing + in for keys the scenery expects a player to press, extra events) before it, +2. starts the simulator with `-s autotest_.scn -v ghostview -ini out//autotest.ini`; + ghostview leaves every train to the AI, and the extra ini turns on `physicslog`, + runs in a 1280x720 window and does not pause when the window loses focus, +3. waits until `physicslog/.dat` covers `duration` seconds, then closes the simulator, +4. runs the checks on the physicslog. + +## Writing a case + +`cases/.json`: + +| key | meaning | +| --- | --- | +| `base_scenery` | scenery file in the game's `scenery/` dir | +| `strip_trainsets` | drop the base scenery's trainsets (default `true`) | +| `time` | replace the scenery start time, `HH:MM` | +| `fragment` | `.scn` file in `cases/` added to the scenery: trainsets after `FirstInit` (before it they derail on unlinked track), the rest before it (launchers read after it never fire) | +| `vehicle` | node name of the vehicle whose physicslog is checked | +| `duration` | seconds of simulation to record | +| `timeout` | wall-clock seconds before giving up, loading included | +| `fail_on_errors` | fail when `errors.txt` is not empty (default `false`) | +| `checks` | list of checks, below | + +Checks (speeds in km/h, times in seconds of physicslog time): + +- `starts` `min_speed`, `within`: reaches `min_speed` within `within` seconds. +- `max_speed` `limit`, `tolerance`: never goes faster than `limit + tolerance`. +- `brakes` `min_pressure`: after moving off, brake cylinder pressure reaches `min_pressure` bar + (physicslog pressures are in bar whatever its header says; the brake pipe runs at 5.0). +- `stops` `moving_speed`, `stop_speed`, `hold`: after reaching `moving_speed`, stays at or + below `stop_speed` for `hold` seconds. + +New check types go in `CHECKS` in `run.py`; each gets the physicslog rows and the check's +parameters and returns `(passed, detail)`. diff --git a/tests/scenarios/cases/eu07_td_timetable.json b/tests/scenarios/cases/eu07_td_timetable.json new file mode 100644 index 00000000..c12b8ae7 --- /dev/null +++ b/tests/scenarios/cases/eu07_td_timetable.json @@ -0,0 +1,15 @@ +{ + "description": "EU07 on td.scn, AI driver on the td/rozklad timetable: starts from Stacja1, keeps to the speed limit, brakes to a stop at Przystanek.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "eu07_td_timetable.scn", + "vehicle": "autotest_eu07", + "duration": 240, + "timeout": 600, + "checks": [ + { "type": "starts", "min_speed": 20, "within": 150 }, + { "type": "max_speed", "limit": 90, "tolerance": 3 }, + { "type": "brakes", "min_pressure": 0.5 }, + { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + ] +} diff --git a/tests/scenarios/cases/eu07_td_timetable.scn b/tests/scenarios/cases/eu07_td_timetable.scn new file mode 100644 index 00000000..c8044e46 --- /dev/null +++ b/tests/scenarios/cases/eu07_td_timetable.scn @@ -0,0 +1,11 @@ +// added by tests/scenarios/run.py to the base scenery, whose own trainsets were removed: +// the trainset goes after FirstInit, the launcher before it. +// the EU07 stands where td.scn keeps its timetable train and gets the same timetable. +trainset td/rozklad tdo_n1734 0.0 0.1 +node -1 0 autotest_eu07 dynamic pkp\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic +endtrainset + +// td.scn sets routes from the keyboard (shift+1..4 run events keyctrl01..04). keyctrl04 is the +// way out of the track the train stands on: switch zwr02 and the exit signal stacja1_j at 40. +// a launcher with a positive time fires once at that hour (HHMM), radius -1 means anywhere. +node -1 0 autotest_route eventlauncher 0 0 0 -1 none 1034 keyctrl04 end diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py new file mode 100644 index 00000000..ad1f51ee --- /dev/null +++ b/tests/scenarios/run.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +"""Scenario tests: run the real simulator on a scenery and check what a vehicle did. + +A case (cases/.json) names a base scenery from the game install, a fragment +appended to it (usually a trainset with an AI driver), and checks on the vehicle's +physics log. The runner: + + 1. builds scenery/autotest_.scn in the game dir: the base scenery with its own + trainsets removed, optionally a different start time, and the case fragment appended, + 2. starts eu07 with that scenery in ghostview (every train driven by AI) and an extra + settings file (-ini) that turns on physicslog and runs in a window, + 3. waits until the vehicle's physicslog covers `duration` seconds of simulation, + then closes the simulator, + 4. evaluates the checks on the log and prints PASS/FAIL per check. + +Scenery and vehicle data are not part of this repo, so this runs against a local game +install only (--game-dir or EU07_GAME_DIR), not in CI. + +Usage: + python tests/scenarios/run.py --game-dir "E:/.../MaSzyna" # all cases + python tests/scenarios/run.py --game-dir ... eu07_td_timetable # chosen cases +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[1] +CASES_DIR = HERE / "cases" +OUT_DIR = HERE / "out" + +# physicslog columns, see TController::PhysicsLog() in vehicle/Driver.cpp. +# Velocity is in km/h and pressures in bar, despite the header saying m/s and MPa. +COL_TIME = 0 +COL_VELOCITY = 1 +COL_BRAKE_PRESS = 8 + +# settings layered over the user's eu07.ini for a test run +TEST_INI = """\ +fullscreen no +fullscreenwindowed no +width 1280 +height 720 +inactivepause no +soundenabled no +physicslog yes +multiplelogs no +timespeed {timespeed} +""" + + +# ---------------------------------------------------------------- scenery + + +def split_trainsets(text: str) -> tuple[str, str]: + """Splits text into (everything else, the trainset ... endtrainset blocks).""" + rest, trainsets = [], [] + inside = False + for line in text.splitlines(keepends=True): + tokens = line.split("//", 1)[0].split() + if not inside and tokens and tokens[0].lower() == "trainset": + inside = True + if inside: + trainsets.append(line) + if any(t.lower() == "endtrainset" for t in tokens): + inside = False + continue + rest.append(line) + return "".join(rest), "".join(trainsets) + + +def set_start_time(text: str, hhmm: str) -> str: + return re.sub(r"(?im)^(\s*time\s+)\d{1,2}:\d{2}", lambda m: m.group(1) + hhmm, text, count=1) + + +def build_scenery(case: dict, game_dir: Path, case_name: str) -> str: + """Writes the case scenery into the game's scenery dir and returns its file name.""" + scenery_dir = game_dir / "scenery" + # sceneries are cp1250; latin-1 round-trips every byte, so nothing gets mangled + text = (scenery_dir / case["base_scenery"]).read_bytes().decode("latin-1") + if case.get("strip_trainsets", True): + text, _ = split_trainsets(text) + if "time" in case: + text = set_start_time(text, case["time"]) + fragment = (CASES_DIR / case["fragment"]).read_bytes().decode("latin-1") + # FirstInit binds event launchers to their events and links the tracks. so launchers and + # events of the case go in before it (read after it they never fire), trainsets after it + # (placed before it they stand on unlinked track and derail at once) + before, trainsets = split_trainsets(fragment) + header = f"\n// ---- autotest: {case_name} ----\n" + firstinit = re.search(r"(?im)^\s*firstinit\b", text) + if firstinit: + text = text[: firstinit.start()] + header + before + "\n" + text[firstinit.start():] + else: + text += header + before + "\n" + text += header + trainsets + "\n" + name = f"autotest_{case_name}.scn" + (scenery_dir / name).write_bytes(text.encode("latin-1")) + return name + + +# ---------------------------------------------------------------- run + + +def read_physicslog(path: Path) -> list[list[float]]: + """Returns rows of numeric columns up to BrakePress; skips the header and partial lines.""" + rows = [] + try: + lines = path.read_text(errors="replace").splitlines() + except FileNotFoundError: + return rows + for line in lines[1:]: + parts = line.split() + if len(parts) <= COL_BRAKE_PRESS: + continue # line still being written + try: + rows.append([float(p) for p in parts[: COL_BRAKE_PRESS + 1]]) + except ValueError: + continue + return rows + + +@dataclass +class RunResult: + rows: list[list[float]] = field(default_factory=list) + exit_code: int | None = None # None: still running when we closed it (the normal end) + timed_out: bool = False + wall_seconds: float = 0.0 + + +def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog: Path, + duration: float, timeout: float) -> RunResult: + physicslog.unlink(missing_ok=True) + cmd = [str(exe), "-s", scenery, "-v", "ghostview", "-ini", str(ini)] + print(" $", " ".join(f'"{c}"' if " " in c else c for c in cmd)) + started = time.monotonic() + proc = subprocess.Popen(cmd, cwd=game_dir) + result = RunResult() + last_report = 0.0 + try: + while True: + time.sleep(1.0) + result.rows = read_physicslog(physicslog) + sim_t = result.rows[-1][COL_TIME] if result.rows else 0.0 + wall = time.monotonic() - started + if wall - last_report >= 15: + v = result.rows[-1][COL_VELOCITY] if result.rows else 0.0 + print(f" ... {wall:5.0f}s wall, sim {sim_t:6.1f}s, v={v:6.1f} km/h") + last_report = wall + if proc.poll() is not None: + result.exit_code = proc.returncode + break + if sim_t >= duration: + break + if wall > timeout: + result.timed_out = True + break + finally: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + result.wall_seconds = time.monotonic() - started + result.rows = read_physicslog(physicslog) + return result + + +# ---------------------------------------------------------------- checks + + +def check_starts(rows, min_speed, within): + for r in rows: + if r[COL_TIME] > within: + break + if r[COL_VELOCITY] >= min_speed: + return True, f"reached {min_speed} km/h at t={r[COL_TIME]:.1f}s" + top = max((r[COL_VELOCITY] for r in rows if r[COL_TIME] <= within), default=0.0) + return False, f"top speed in the first {within}s was {top:.1f} km/h, wanted {min_speed}" + + +def check_max_speed(rows, limit, tolerance=0.0): + worst = max(rows, key=lambda r: r[COL_VELOCITY]) + v, t = worst[COL_VELOCITY], worst[COL_TIME] + ok = v <= limit + tolerance + return ok, f"top speed {v:.1f} km/h at t={t:.1f}s (limit {limit} +{tolerance})" + + +def _first_moving(rows, moving_speed): + return next((i for i, r in enumerate(rows) if r[COL_VELOCITY] >= moving_speed), None) + + +def check_brakes(rows, min_pressure, moving_speed=20): + start = _first_moving(rows, moving_speed) + if start is None: + return False, "never moved, so never braked" + for r in rows[start:]: + if r[COL_BRAKE_PRESS] >= min_pressure: + return True, f"brake cylinder {r[COL_BRAKE_PRESS]:.2f} bar at t={r[COL_TIME]:.1f}s, v={r[COL_VELOCITY]:.1f} km/h" + return False, f"brake cylinder pressure never reached {min_pressure} bar after moving off" + + +def check_stops(rows, moving_speed, stop_speed, hold): + start = _first_moving(rows, moving_speed) + if start is None: + return False, f"never reached {moving_speed} km/h" + stopped_at = None + for r in rows[start:]: + if r[COL_VELOCITY] <= stop_speed: + if stopped_at is None: + stopped_at = r[COL_TIME] + if r[COL_TIME] - stopped_at >= hold: + return True, f"came to a stop at t={stopped_at:.1f}s" + else: + stopped_at = None + return False, f"did not stay below {stop_speed} km/h for {hold}s after moving off" + + +CHECKS = { + "starts": check_starts, + "max_speed": check_max_speed, + "brakes": check_brakes, + "stops": check_stops, +} + + +# ---------------------------------------------------------------- main + + +def find_exe(explicit: str | None) -> Path: + if explicit: + return Path(explicit).resolve() + candidates = sorted(REPO.glob("cmake-build-*/bin/Release/eu07*.exe"), key=lambda p: p.stat().st_mtime) + if not candidates: + sys.exit("no eu07 executable found under cmake-build-*/bin/Release, pass --exe") + return candidates[-1] + + +def count_lines(path: Path) -> int: + try: + return sum(1 for line in path.read_text(errors="replace").splitlines() if line.strip()) + except FileNotFoundError: + return 0 + + +def run_case(name: str, exe: Path, game_dir: Path, timespeed: float) -> bool: + case = json.loads((CASES_DIR / f"{name}.json").read_text(encoding="utf-8")) + print(f"\n=== {name}: {case.get('description', '')}") + out = OUT_DIR / name + shutil.rmtree(out, ignore_errors=True) + out.mkdir(parents=True) + + scenery = build_scenery(case, game_dir, name) + ini = out / "autotest.ini" + ini.write_text(TEST_INI.format(timespeed=timespeed), encoding="ascii") + physicslog = game_dir / "physicslog" / f"{case['vehicle']}.dat" + + result = run_simulator(exe, game_dir, scenery, ini, physicslog, + float(case["duration"]), float(case.get("timeout", 600))) + + for f in ("log.txt", "errors.txt"): + if (game_dir / f).exists(): + shutil.copy2(game_dir / f, out / f) + if physicslog.exists(): + shutil.copy2(physicslog, out / physicslog.name) + + ok = True + sim_t = result.rows[-1][COL_TIME] if result.rows else 0.0 + print(f" ran {result.wall_seconds:.0f}s wall, {sim_t:.1f}s of physicslog, results in {out.relative_to(REPO)}") + if result.exit_code is not None: + print(f" FAIL simulator exited on its own with code {result.exit_code} (crash?)") + ok = False + if result.timed_out: + print(f" FAIL timed out before {case['duration']}s of simulation") + ok = False + if not result.rows: + print(f" FAIL no physicslog for '{case['vehicle']}' ({physicslog})") + return False + + for check in case["checks"]: + params = {k: v for k, v in check.items() if k != "type"} + passed, detail = CHECKS[check["type"]](result.rows, **params) + print(f" {'PASS' if passed else 'FAIL'} {check['type']}: {detail}") + ok &= passed + + errors = count_lines(out / "errors.txt") + if errors: + print(f" note errors.txt has {errors} line(s)") + if case.get("fail_on_errors") and errors: + ok = False + return ok + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("cases", nargs="*", help="case names from cases/ (default: all)") + ap.add_argument("--game-dir", default=os.environ.get("EU07_GAME_DIR"), + help="game install with scenery/ and dynamic/ (or EU07_GAME_DIR)") + ap.add_argument("--exe", help="simulator executable (default: newest cmake-build-*/bin/Release/eu07*.exe)") + ap.add_argument("--timespeed", type=float, default=1.0, help="simulation time multiplier") + args = ap.parse_args() + + if not args.game_dir: + sys.exit("pass --game-dir or set EU07_GAME_DIR") + game_dir = Path(args.game_dir).resolve() + exe = find_exe(args.exe) + print(f"exe: {exe}\ngame: {game_dir}") + + names = args.cases or sorted(p.stem for p in CASES_DIR.glob("*.json")) + results = {n: run_case(n, exe, game_dir, args.timespeed) for n in names} + + print("\n=== summary") + for n, ok in results.items(): + print(f" {'PASS' if ok else 'FAIL'} {n}") + return 0 if all(results.values()) else 1 + + +if __name__ == "__main__": + sys.exit(main()) From 6cf8934acb65aec779aea3d0f7a9604f9ef7ec15 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Wed, 30 Sep 2026 19:29:10 +0200 Subject: [PATCH 02/28] test: run scenario tests at timespeed 20 by default eu07_td_timetable now takes about 20 s instead of 4 min. Its speed profile at 20 and 60 matched a real-time run within a few km/h and seconds; the physicslog keeps its 0.05 s rows. The runner polls the log every 0.25 s so it does not overshoot the case duration by a minute of simulation. --- tests/scenarios/README.md | 5 ++++- tests/scenarios/run.py | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index 77edb5a9..c233154d 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -18,7 +18,10 @@ Python 3.10+, no extra packages. Options: - `--exe path/to/eu07.exe` picks the build to test; the default is the newest `cmake-build-*/bin/Release/eu07*.exe`. The build needs the `-ini` command line option. - name one or more cases to run only those, e.g. `run.py ... eu07_td_timetable`. -- `--timespeed 4` runs the simulation faster than real time. +- `--timespeed N` sets the simulation time multiplier, default 20: a 4-minute case runs in + about 20 s. Physics still steps finely (the physicslog keeps its 0.05 s rows) and the speed + profile at 20 and 60 matched a real-time run within a few km/h and seconds. Use + `--timespeed 1` to watch a run. Exit code is 0 when every case passed. Logs of each run (`log.txt`, `errors.txt`, the vehicle's physicslog) land in `tests/scenarios/out//`. diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index ad1f51ee..b3736a92 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -149,7 +149,7 @@ def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog last_report = 0.0 try: while True: - time.sleep(1.0) + time.sleep(0.25) # at timespeed 20 that is 5 s of simulation result.rows = read_physicslog(physicslog) sim_t = result.rows[-1][COL_TIME] if result.rows else 0.0 wall = time.monotonic() - started @@ -308,7 +308,10 @@ def main() -> int: ap.add_argument("--game-dir", default=os.environ.get("EU07_GAME_DIR"), help="game install with scenery/ and dynamic/ (or EU07_GAME_DIR)") ap.add_argument("--exe", help="simulator executable (default: newest cmake-build-*/bin/Release/eu07*.exe)") - ap.add_argument("--timespeed", type=float, default=1.0, help="simulation time multiplier") + # 20 gave the same speed profile as 1 within a few km/h and seconds on eu07_td_timetable, + # in 20 s instead of 4 min; 1 is for watching a run + ap.add_argument("--timespeed", type=float, default=20.0, + help="simulation time multiplier, the simulator offers 1, 5, 20 and 60 (default 20)") args = ap.parse_args() if not args.game_dir: From cb0a1223835009fa8e82726170d4cc0afe992a41 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Wed, 30 Sep 2026 21:55:16 +0200 Subject: [PATCH 03/28] test: red signal case, seven vehicles on the td.scn timetable, parallel runs - eu07_td_red_signal: no route set, the train has to stop before the exit signal and stay there (new checks stays_stopped and distance; distance is integrated from the speed, the physicslog has no position) - en57/ep09/et22/sm42/sn61/vectron/eu07_td_timetable share one fragment template, td_timetable.scn; a case sets vars.consist and the lead vehicle is named {{vehicle}} = autotest_ - -j N runs cases in parallel: each case has its own physicslog and, with multiplelogs, its own log files, which are moved to out// - a run stops at once on Bad vehicle / Bad trainset instead of waiting for the timeout, and fails after 120 s without physicslog rows - the .sbt geometry cache of a test scenery is dropped when the base scenery is newer All 8 cases pass in about a minute with -j 4. --- tests/scenarios/README.md | 35 ++- tests/scenarios/cases/en57_td_timetable.json | 21 ++ tests/scenarios/cases/ep09_td_timetable.json | 17 ++ tests/scenarios/cases/et22_td_timetable.json | 17 ++ tests/scenarios/cases/eu07_td_red_signal.json | 18 ++ tests/scenarios/cases/eu07_td_timetable.json | 6 +- tests/scenarios/cases/sm42_td_timetable.json | 17 ++ tests/scenarios/cases/sn61_td_timetable.json | 17 ++ tests/scenarios/cases/td_red_signal.scn | 6 + ...eu07_td_timetable.scn => td_timetable.scn} | 9 +- .../scenarios/cases/vectron_td_timetable.json | 17 ++ tests/scenarios/run.py | 201 ++++++++++++++---- 12 files changed, 332 insertions(+), 49 deletions(-) create mode 100644 tests/scenarios/cases/en57_td_timetable.json create mode 100644 tests/scenarios/cases/ep09_td_timetable.json create mode 100644 tests/scenarios/cases/et22_td_timetable.json create mode 100644 tests/scenarios/cases/eu07_td_red_signal.json create mode 100644 tests/scenarios/cases/sm42_td_timetable.json create mode 100644 tests/scenarios/cases/sn61_td_timetable.json create mode 100644 tests/scenarios/cases/td_red_signal.scn rename tests/scenarios/cases/{eu07_td_timetable.scn => td_timetable.scn} (51%) create mode 100644 tests/scenarios/cases/vectron_td_timetable.json diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index c233154d..23f79301 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -9,7 +9,7 @@ only, not in CI. ## Running ``` -python tests/scenarios/run.py --game-dir "E:/Program Files (x86)/Steam/steamapps/common/MaSzyna" +python tests/scenarios/run.py --game-dir "E:/Program Files (x86)/Steam/steamapps/common/MaSzyna" -j 4 ``` Python 3.10+, no extra packages. Options: @@ -18,13 +18,15 @@ Python 3.10+, no extra packages. Options: - `--exe path/to/eu07.exe` picks the build to test; the default is the newest `cmake-build-*/bin/Release/eu07*.exe`. The build needs the `-ini` command line option. - name one or more cases to run only those, e.g. `run.py ... eu07_td_timetable`. +- `-j N` runs N cases at once, each in its own simulator window. Each case prints its + results when it is done. All 8 cases take about a minute with `-j 4`. - `--timespeed N` sets the simulation time multiplier, default 20: a 4-minute case runs in about 20 s. Physics still steps finely (the physicslog keeps its 0.05 s rows) and the speed profile at 20 and 60 matched a real-time run within a few km/h and seconds. Use `--timespeed 1` to watch a run. Exit code is 0 when every case passed. Logs of each run (`log.txt`, `errors.txt`, -the vehicle's physicslog) land in `tests/scenarios/out//`. +the vehicle's physicslog) are moved to `tests/scenarios/out//`. ## How it works @@ -35,11 +37,19 @@ For each case the runner: fragment added: its trainsets after `FirstInit`, everything else (event launchers standing in for keys the scenery expects a player to press, extra events) before it, 2. starts the simulator with `-s autotest_.scn -v ghostview -ini out//autotest.ini`; - ghostview leaves every train to the AI, and the extra ini turns on `physicslog`, - runs in a 1280x720 window and does not pause when the window loses focus, -3. waits until `physicslog/.dat` covers `duration` seconds, then closes the simulator, + ghostview leaves every train to the AI, and the extra ini turns on `physicslog`, runs in a + 1280x720 window, does not pause when the window loses focus, and writes the logs to + `logs/log (autotest_) .txt` rather than the shared `log.txt` (`multiplelogs`), +3. waits until `physicslog/autotest_.dat` covers `duration` seconds, then closes the + simulator. It gives up early when the log reports `Bad vehicle` / `Bad trainset`, or when + the scenario has loaded and no physicslog appeared, 4. runs the checks on the physicslog. +The lead vehicle is named `autotest_`, so every case has its own physicslog and log +files and cases can run in parallel. The `autotest_*` sceneries (and the simulator's `.sbt` +geometry cache of them) stay in the game's `scenery/` dir; the cache is dropped when the base +scenery is newer. + ## Writing a case `cases/.json`: @@ -50,13 +60,19 @@ For each case the runner: | `strip_trainsets` | drop the base scenery's trainsets (default `true`) | | `time` | replace the scenery start time, `HH:MM` | | `fragment` | `.scn` file in `cases/` added to the scenery: trainsets after `FirstInit` (before it they derail on unlinked track), the rest before it (launchers read after it never fire) | -| `vehicle` | node name of the vehicle whose physicslog is checked | +| `vars` | values for `{{name}}` in the fragment; a list becomes one line per item | | `duration` | seconds of simulation to record | | `timeout` | wall-clock seconds before giving up, loading included | | `fail_on_errors` | fail when `errors.txt` is not empty (default `false`) | | `checks` | list of checks, below | -Checks (speeds in km/h, times in seconds of physicslog time): +`{{vehicle}}` is always set, also inside `vars` values: the node name of the lead vehicle, +the one whose physicslog is checked. Other vehicles of the consist take names made from it, +e.g. `{{vehicle}}_s`. The td.scn cases share two fragments, `td_timetable.scn` (route set, +train runs to Przystanek) and `td_red_signal.scn` (no route, exit signal at stop), and differ +only in `vars.consist`. + +Checks (speeds in km/h, times in seconds of physicslog time, distances in metres): - `starts` `min_speed`, `within`: reaches `min_speed` within `within` seconds. - `max_speed` `limit`, `tolerance`: never goes faster than `limit + tolerance`. @@ -64,6 +80,11 @@ Checks (speeds in km/h, times in seconds of physicslog time): (physicslog pressures are in bar whatever its header says; the brake pipe runs at 5.0). - `stops` `moving_speed`, `stop_speed`, `hold`: after reaching `moving_speed`, stays at or below `stop_speed` for `hold` seconds. +- `stays_stopped` `moving_speed`, `stop_speed`: after moving off and stopping, never goes + above `stop_speed` again. +- `distance` `min`, `max`: total distance travelled. The physicslog has no position, so the + distance comes from integrating the speed; on td.scn every vehicle's first stop after + Stacja1 came out at 1400 m (+-3), and the red signal stop at 140 m. New check types go in `CHECKS` in `run.py`; each gets the physicslog rows and the check's parameters and returns `(passed, detail)`. diff --git a/tests/scenarios/cases/en57_td_timetable.json b/tests/scenarios/cases/en57_td_timetable.json new file mode 100644 index 00000000..ddd52af3 --- /dev/null +++ b/tests/scenarios/cases/en57_td_timetable.json @@ -0,0 +1,21 @@ +{ + "description": "EN57 EMU (three cars) on td.scn, AI driver on the td/rozklad timetable: starts from Stacja1, keeps to the speed limit, brakes to a stop at Przystanek.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_timetable.scn", + "vars": { + "consist": [ + "node -1 0 {{vehicle}} dynamic pkp\\en57_v2 EN57-1000ra EN57RA_GSO_WKM1_KM_NS_N 0 headdriver 255 0 enddynamic", + "node -1 0 {{vehicle}}_s dynamic pkp\\en57_v2 EN57-1000s EN57S_WKM1_KM_NS_S 0 nobody 255 0 enddynamic", + "node -1 0 {{vehicle}}_rb dynamic pkp\\en57_v2 EN57-1000rb EN57RB_GSO_WKM1_KM_NS 0 nobody 0 0 enddynamic" + ] + }, + "duration": 240, + "timeout": 600, + "checks": [ + { "type": "starts", "min_speed": 20, "within": 150 }, + { "type": "max_speed", "limit": 90, "tolerance": 3 }, + { "type": "brakes", "min_pressure": 0.5 }, + { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + ] +} diff --git a/tests/scenarios/cases/ep09_td_timetable.json b/tests/scenarios/cases/ep09_td_timetable.json new file mode 100644 index 00000000..dd130be3 --- /dev/null +++ b/tests/scenarios/cases/ep09_td_timetable.json @@ -0,0 +1,17 @@ +{ + "description": "EP09 on td.scn, AI driver on the td/rozklad timetable: starts from Stacja1, keeps to the speed limit, brakes to a stop at Przystanek.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_timetable.scn", + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\ep09_v1 104e-005 104e_3 0 headdriver 0 0 enddynamic" + }, + "duration": 240, + "timeout": 600, + "checks": [ + { "type": "starts", "min_speed": 20, "within": 150 }, + { "type": "max_speed", "limit": 90, "tolerance": 3 }, + { "type": "brakes", "min_pressure": 0.5 }, + { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + ] +} diff --git a/tests/scenarios/cases/et22_td_timetable.json b/tests/scenarios/cases/et22_td_timetable.json new file mode 100644 index 00000000..004f5354 --- /dev/null +++ b/tests/scenarios/cases/et22_td_timetable.json @@ -0,0 +1,17 @@ +{ + "description": "ET22 on td.scn, AI driver on the td/rozklad timetable: starts from Stacja1, keeps to the speed limit, brakes to a stop at Przystanek.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_timetable.scn", + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\et22_v3 201E-1150 201E 0 headdriver 0 0 enddynamic" + }, + "duration": 240, + "timeout": 600, + "checks": [ + { "type": "starts", "min_speed": 20, "within": 150 }, + { "type": "max_speed", "limit": 90, "tolerance": 3 }, + { "type": "brakes", "min_pressure": 0.5 }, + { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + ] +} diff --git a/tests/scenarios/cases/eu07_td_red_signal.json b/tests/scenarios/cases/eu07_td_red_signal.json new file mode 100644 index 00000000..b02acdc8 --- /dev/null +++ b/tests/scenarios/cases/eu07_td_red_signal.json @@ -0,0 +1,18 @@ +{ + "description": "EU07 on td.scn with no route set: pulls off, stops before the exit signal stacja1_j at stop, and stays there.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_red_signal.scn", + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic" + }, + "duration": 240, + "timeout": 600, + "checks": [ + { "type": "starts", "min_speed": 5, "within": 150 }, + { "type": "max_speed", "limit": 40 }, + { "type": "stops", "moving_speed": 5, "stop_speed": 0.5, "hold": 5 }, + { "type": "stays_stopped", "moving_speed": 5, "stop_speed": 0.5 }, + { "type": "distance", "max": 150 } + ] +} diff --git a/tests/scenarios/cases/eu07_td_timetable.json b/tests/scenarios/cases/eu07_td_timetable.json index c12b8ae7..2f23019f 100644 --- a/tests/scenarios/cases/eu07_td_timetable.json +++ b/tests/scenarios/cases/eu07_td_timetable.json @@ -2,8 +2,10 @@ "description": "EU07 on td.scn, AI driver on the td/rozklad timetable: starts from Stacja1, keeps to the speed limit, brakes to a stop at Przystanek.", "base_scenery": "td.scn", "time": "10:34", - "fragment": "eu07_td_timetable.scn", - "vehicle": "autotest_eu07", + "fragment": "td_timetable.scn", + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic" + }, "duration": 240, "timeout": 600, "checks": [ diff --git a/tests/scenarios/cases/sm42_td_timetable.json b/tests/scenarios/cases/sm42_td_timetable.json new file mode 100644 index 00000000..fc89be0f --- /dev/null +++ b/tests/scenarios/cases/sm42_td_timetable.json @@ -0,0 +1,17 @@ +{ + "description": "SM42 diesel shunter on td.scn, AI driver on the td/rozklad timetable: starts from Stacja1, keeps to the speed limit, brakes to a stop at Przystanek.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_timetable.scn", + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\sm42_v1 6D-468 6D 0 headdriver 0 0 enddynamic" + }, + "duration": 240, + "timeout": 600, + "checks": [ + { "type": "starts", "min_speed": 20, "within": 150 }, + { "type": "max_speed", "limit": 90, "tolerance": 3 }, + { "type": "brakes", "min_pressure": 0.5 }, + { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + ] +} diff --git a/tests/scenarios/cases/sn61_td_timetable.json b/tests/scenarios/cases/sn61_td_timetable.json new file mode 100644 index 00000000..1db425c8 --- /dev/null +++ b/tests/scenarios/cases/sn61_td_timetable.json @@ -0,0 +1,17 @@ +{ + "description": "SN61 diesel railcar on td.scn, AI driver on the td/rozklad timetable: starts from Stacja1, keeps to the speed limit, brakes to a stop at Przystanek.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_timetable.scn", + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\sn61_v2 SN61-13 SN61_v1 0 headdriver 0 0 enddynamic" + }, + "duration": 240, + "timeout": 600, + "checks": [ + { "type": "starts", "min_speed": 20, "within": 150 }, + { "type": "max_speed", "limit": 90, "tolerance": 3 }, + { "type": "brakes", "min_pressure": 0.5 }, + { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + ] +} diff --git a/tests/scenarios/cases/td_red_signal.scn b/tests/scenarios/cases/td_red_signal.scn new file mode 100644 index 00000000..98170708 --- /dev/null +++ b/tests/scenarios/cases/td_red_signal.scn @@ -0,0 +1,6 @@ +// added by tests/scenarios/run.py to td.scn, whose own trainsets were removed. +// same start as td_timetable.scn, but nobody sets the route: the exit signal stacja1_j, +// about 150 m ahead of the train, stays at stop. +trainset td/rozklad tdo_n1734 0.0 0.1 +{{consist}} +endtrainset diff --git a/tests/scenarios/cases/eu07_td_timetable.scn b/tests/scenarios/cases/td_timetable.scn similarity index 51% rename from tests/scenarios/cases/eu07_td_timetable.scn rename to tests/scenarios/cases/td_timetable.scn index c8044e46..e67dcf9a 100644 --- a/tests/scenarios/cases/eu07_td_timetable.scn +++ b/tests/scenarios/cases/td_timetable.scn @@ -1,11 +1,12 @@ -// added by tests/scenarios/run.py to the base scenery, whose own trainsets were removed: +// added by tests/scenarios/run.py to td.scn, whose own trainsets were removed: // the trainset goes after FirstInit, the launcher before it. -// the EU07 stands where td.scn keeps its timetable train and gets the same timetable. +// the case's consist stands where td.scn keeps its timetable train (track 1 of Stacja1) +// and gets the same timetable: Stacja1 10:35 -> Przystanek -> Stacja2. trainset td/rozklad tdo_n1734 0.0 0.1 -node -1 0 autotest_eu07 dynamic pkp\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic +{{consist}} endtrainset // td.scn sets routes from the keyboard (shift+1..4 run events keyctrl01..04). keyctrl04 is the -// way out of the track the train stands on: switch zwr02 and the exit signal stacja1_j at 40. +// way out of track 1: switch zwr02 and the exit signal stacja1_j at 40. // a launcher with a positive time fires once at that hour (HHMM), radius -1 means anywhere. node -1 0 autotest_route eventlauncher 0 0 0 -1 none 1034 keyctrl04 end diff --git a/tests/scenarios/cases/vectron_td_timetable.json b/tests/scenarios/cases/vectron_td_timetable.json new file mode 100644 index 00000000..9948a09e --- /dev/null +++ b/tests/scenarios/cases/vectron_td_timetable.json @@ -0,0 +1,17 @@ +{ + "description": "Vectron 1293-107 on td.scn, AI driver on the td/rozklad timetable: starts from Stacja1, keeps to the speed limit, brakes to a stop at Przystanek.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_timetable.scn", + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\vectron_v1 1293-107 wariant_7w 0 headdriver 0 0 enddynamic" + }, + "duration": 240, + "timeout": 600, + "checks": [ + { "type": "starts", "min_speed": 20, "within": 150 }, + { "type": "max_speed", "limit": 90, "tolerance": 3 }, + { "type": "brakes", "min_pressure": 0.5 }, + { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + ] +} diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index b3736a92..41990670 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -2,11 +2,11 @@ """Scenario tests: run the real simulator on a scenery and check what a vehicle did. A case (cases/.json) names a base scenery from the game install, a fragment -appended to it (usually a trainset with an AI driver), and checks on the vehicle's +added to it (usually a trainset with an AI driver), and checks on the vehicle's physics log. The runner: 1. builds scenery/autotest_.scn in the game dir: the base scenery with its own - trainsets removed, optionally a different start time, and the case fragment appended, + trainsets removed, optionally a different start time, and the case fragment added, 2. starts eu07 with that scenery in ghostview (every train driven by AI) and an extra settings file (-ini) that turns on physicslog and runs in a window, 3. waits until the vehicle's physicslog covers `duration` seconds of simulation, @@ -14,10 +14,10 @@ physics log. The runner: 4. evaluates the checks on the log and prints PASS/FAIL per check. Scenery and vehicle data are not part of this repo, so this runs against a local game -install only (--game-dir or EU07_GAME_DIR), not in CI. +install only (--game-dir or EU07_GAME_DIR), not in CI. See README.md. Usage: - python tests/scenarios/run.py --game-dir "E:/.../MaSzyna" # all cases + python tests/scenarios/run.py --game-dir "E:/.../MaSzyna" -j 4 # all cases python tests/scenarios/run.py --game-dir ... eu07_td_timetable # chosen cases """ @@ -31,6 +31,7 @@ import shutil import subprocess import sys import time +from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from pathlib import Path @@ -54,9 +55,11 @@ height 720 inactivepause no soundenabled no physicslog yes -multiplelogs no +multiplelogs yes timespeed {timespeed} """ +# multiplelogs: log and errors go to logs/log () .txt instead of the shared +# log.txt, so runs of different cases can go in parallel # ---------------------------------------------------------------- scenery @@ -83,7 +86,19 @@ def set_start_time(text: str, hhmm: str) -> str: return re.sub(r"(?im)^(\s*time\s+)\d{1,2}:\d{2}", lambda m: m.group(1) + hhmm, text, count=1) -def build_scenery(case: dict, game_dir: Path, case_name: str) -> str: +def fill_vars(fragment: str, values: dict, keep=frozenset()) -> str: + """Replaces {{name}} with the case's value; a list value becomes one line per item.""" + def value(m): + if m.group(1) in keep: + return m.group(0) + v = values.get(m.group(1)) + if v is None: + sys.exit(f"fragment uses {{{{{m.group(1)}}}}} but the case sets no such var") + return "\n".join(v) if isinstance(v, list) else str(v) + return re.sub(r"\{\{(\w+)\}\}", value, fragment) + + +def build_scenery(case: dict, game_dir: Path, case_name: str, vehicle: str) -> str: """Writes the case scenery into the game's scenery dir and returns its file name.""" scenery_dir = game_dir / "scenery" # sceneries are cp1250; latin-1 round-trips every byte, so nothing gets mangled @@ -93,6 +108,9 @@ def build_scenery(case: dict, game_dir: Path, case_name: str) -> str: if "time" in case: text = set_start_time(text, case["time"]) fragment = (CASES_DIR / case["fragment"]).read_bytes().decode("latin-1") + # vars first: {{vehicle}} inside a var's value gets filled in the second pass + fragment = fill_vars(fragment, case.get("vars", {}), keep={"vehicle"}) + fragment = fill_vars(fragment, {"vehicle": vehicle}) # FirstInit binds event launchers to their events and links the tracks. so launchers and # events of the case go in before it (read after it they never fire), trainsets after it # (placed before it they stand on unlinked track and derail at once) @@ -106,6 +124,12 @@ def build_scenery(case: dict, game_dir: Path, case_name: str) -> str: text += header + trainsets + "\n" name = f"autotest_{case_name}.scn" (scenery_dir / name).write_bytes(text.encode("latin-1")) + # the simulator caches the scenery's static geometry in .sbt and reads it instead of + # the terrain in the .scn (tracks, events and trainsets always come from the .scn), so + # drop the cache once the base scenery has changed + cache = scenery_dir / f"autotest_{case_name}.sbt" + if cache.exists() and cache.stat().st_mtime < (scenery_dir / case["base_scenery"]).stat().st_mtime: + cache.unlink() return name @@ -130,19 +154,47 @@ def read_physicslog(path: Path) -> list[list[float]]: return rows +def find_logs(game_dir: Path, scenery: str, since: float) -> dict[str, Path]: + """The log.txt / errors.txt of one run; with multiplelogs each scenery gets its own files.""" + stem = scenery.rsplit(".", 1)[0] + found = {} + for kind in ("log", "errors"): + files = [p for p in (game_dir / "logs").glob(f"{kind} ({stem}) *.txt") if p.stat().st_mtime >= since] + if files: + found[kind] = max(files, key=lambda p: p.stat().st_mtime) + return found + + +def read_text(path: Path | None) -> str: + try: + return path.read_text(errors="replace") if path else "" + except FileNotFoundError: + return "" + + +# load problems that leave the case without its vehicle; no point waiting for the timeout +FATAL_LOG_LINES = ("Bad vehicle:", "Bad trainset:") +# wall seconds from start to the first physicslog row before giving up. "Scenario loading +# time" in the log is not the end of loading (the trainsets after FirstInit still follow), +# and with several simulators loading at once the first rows took well over 15 s +STARTUP_TIMEOUT = 120.0 + + @dataclass class RunResult: rows: list[list[float]] = field(default_factory=list) exit_code: int | None = None # None: still running when we closed it (the normal end) - timed_out: bool = False + failure: str = "" # why the run was cut short wall_seconds: float = 0.0 + logs: dict[str, Path] = field(default_factory=dict) def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog: Path, - duration: float, timeout: float) -> RunResult: + duration: float, timeout: float, say) -> RunResult: physicslog.unlink(missing_ok=True) cmd = [str(exe), "-s", scenery, "-v", "ghostview", "-ini", str(ini)] - print(" $", " ".join(f'"{c}"' if " " in c else c for c in cmd)) + say(" $ " + " ".join(f'"{c}"' if " " in c else c for c in cmd), progress=True) + since = time.time() - 1 started = time.monotonic() proc = subprocess.Popen(cmd, cwd=game_dir) result = RunResult() @@ -150,12 +202,12 @@ def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog try: while True: time.sleep(0.25) # at timespeed 20 that is 5 s of simulation + wall = time.monotonic() - started result.rows = read_physicslog(physicslog) sim_t = result.rows[-1][COL_TIME] if result.rows else 0.0 - wall = time.monotonic() - started if wall - last_report >= 15: v = result.rows[-1][COL_VELOCITY] if result.rows else 0.0 - print(f" ... {wall:5.0f}s wall, sim {sim_t:6.1f}s, v={v:6.1f} km/h") + say(f" ... {wall:5.0f}s wall, sim {sim_t:6.1f}s, v={v:6.1f} km/h", progress=True) last_report = wall if proc.poll() is not None: result.exit_code = proc.returncode @@ -163,8 +215,18 @@ def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog if sim_t >= duration: break if wall > timeout: - result.timed_out = True + result.failure = f"timed out after {timeout:.0f}s before {duration}s of simulation" break + if not result.rows: + result.logs = find_logs(game_dir, scenery, since) + fatal = [l.strip() for l in read_text(result.logs.get("errors")).splitlines() + if any(k in l for k in FATAL_LOG_LINES)] + if fatal: + result.failure = "vehicle failed to load: " + " | ".join(fatal) + break + if wall > STARTUP_TIMEOUT: + result.failure = f"no physicslog rows after {STARTUP_TIMEOUT:.0f}s" + break finally: if proc.poll() is None: proc.terminate() @@ -174,6 +236,7 @@ def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog proc.kill() result.wall_seconds = time.monotonic() - started result.rows = read_physicslog(physicslog) + result.logs = find_logs(game_dir, scenery, since) return result @@ -201,6 +264,15 @@ def _first_moving(rows, moving_speed): return next((i for i, r in enumerate(rows) if r[COL_VELOCITY] >= moving_speed), None) +def _distances(rows): + """Metres travelled up to each row, from the speed; the physicslog has no position.""" + out, d = [0.0], 0.0 + for a, b in zip(rows, rows[1:]): + d += (a[COL_VELOCITY] + b[COL_VELOCITY]) / 2 / 3.6 * (b[COL_TIME] - a[COL_TIME]) + out.append(d) + return out + + def check_brakes(rows, min_pressure, moving_speed=20): start = _first_moving(rows, moving_speed) if start is None: @@ -215,23 +287,49 @@ def check_stops(rows, moving_speed, stop_speed, hold): start = _first_moving(rows, moving_speed) if start is None: return False, f"never reached {moving_speed} km/h" + dist = _distances(rows) stopped_at = None - for r in rows[start:]: + for i in range(start, len(rows)): + r = rows[i] if r[COL_VELOCITY] <= stop_speed: if stopped_at is None: - stopped_at = r[COL_TIME] - if r[COL_TIME] - stopped_at >= hold: - return True, f"came to a stop at t={stopped_at:.1f}s" + stopped_at = i + if r[COL_TIME] - rows[stopped_at][COL_TIME] >= hold: + return True, f"came to a stop at t={rows[stopped_at][COL_TIME]:.1f}s after {dist[stopped_at]:.0f} m" else: stopped_at = None return False, f"did not stay below {stop_speed} km/h for {hold}s after moving off" +def check_stays_stopped(rows, moving_speed, stop_speed): + """Once stopped after moving off, never moves again (e.g. waiting at a red signal).""" + start = _first_moving(rows, moving_speed) + if start is None: + return False, f"never reached {moving_speed} km/h" + stop = next((i for i in range(start, len(rows)) if rows[i][COL_VELOCITY] <= stop_speed), None) + if stop is None: + return False, f"never came down to {stop_speed} km/h" + moved = next((r for r in rows[stop:] if r[COL_VELOCITY] > stop_speed), None) + if moved is not None: + return False, f"stopped at t={rows[stop][COL_TIME]:.1f}s but moved again at t={moved[COL_TIME]:.1f}s, v={moved[COL_VELOCITY]:.1f} km/h" + return True, f"stopped at t={rows[stop][COL_TIME]:.1f}s and stayed until t={rows[-1][COL_TIME]:.1f}s" + + +def check_distance(rows, max=None, min=None): + """Total distance travelled, in metres.""" + d = _distances(rows)[-1] + ok = (max is None or d <= max) and (min is None or d >= min) + bounds = " ".join(s for s in (f">= {min}" if min is not None else "", f"<= {max}" if max is not None else "") if s) + return ok, f"travelled {d:.0f} m (wanted {bounds})" + + CHECKS = { "starts": check_starts, "max_speed": check_max_speed, "brakes": check_brakes, "stops": check_stops, + "stays_stopped": check_stays_stopped, + "distance": check_distance, } @@ -254,49 +352,52 @@ def count_lines(path: Path) -> int: return 0 -def run_case(name: str, exe: Path, game_dir: Path, timespeed: float) -> bool: +def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, say) -> bool: case = json.loads((CASES_DIR / f"{name}.json").read_text(encoding="utf-8")) - print(f"\n=== {name}: {case.get('description', '')}") + say(f"\n=== {name}: {case.get('description', '')}") out = OUT_DIR / name shutil.rmtree(out, ignore_errors=True) out.mkdir(parents=True) - scenery = build_scenery(case, game_dir, name) + # the lead vehicle is named after the case, so parallel runs never share a physicslog + vehicle = f"autotest_{name}" + scenery = build_scenery(case, game_dir, name, vehicle) ini = out / "autotest.ini" ini.write_text(TEST_INI.format(timespeed=timespeed), encoding="ascii") - physicslog = game_dir / "physicslog" / f"{case['vehicle']}.dat" + physicslog = game_dir / "physicslog" / f"{vehicle}.dat" result = run_simulator(exe, game_dir, scenery, ini, physicslog, - float(case["duration"]), float(case.get("timeout", 600))) + float(case["duration"]), float(case.get("timeout", 600)), say) - for f in ("log.txt", "errors.txt"): - if (game_dir / f).exists(): - shutil.copy2(game_dir / f, out / f) + # the logs are ours alone (named after the case scenery), move them out of the game dir + for kind, path in result.logs.items(): + shutil.move(path, out / f"{kind}.txt") if physicslog.exists(): - shutil.copy2(physicslog, out / physicslog.name) + shutil.move(physicslog, out / physicslog.name) ok = True sim_t = result.rows[-1][COL_TIME] if result.rows else 0.0 - print(f" ran {result.wall_seconds:.0f}s wall, {sim_t:.1f}s of physicslog, results in {out.relative_to(REPO)}") + say(f" ran {result.wall_seconds:.0f}s wall, {sim_t:.1f}s of physicslog, results in {out.relative_to(REPO)}") if result.exit_code is not None: - print(f" FAIL simulator exited on its own with code {result.exit_code} (crash?)") + say(f" FAIL simulator exited on its own with code {result.exit_code}") ok = False - if result.timed_out: - print(f" FAIL timed out before {case['duration']}s of simulation") + if result.failure: + say(f" FAIL {result.failure}") ok = False if not result.rows: - print(f" FAIL no physicslog for '{case['vehicle']}' ({physicslog})") + if not result.failure: + say(f" FAIL no physicslog for '{vehicle}'") return False for check in case["checks"]: params = {k: v for k, v in check.items() if k != "type"} passed, detail = CHECKS[check["type"]](result.rows, **params) - print(f" {'PASS' if passed else 'FAIL'} {check['type']}: {detail}") + say(f" {'PASS' if passed else 'FAIL'} {check['type']}: {detail}") ok &= passed errors = count_lines(out / "errors.txt") if errors: - print(f" note errors.txt has {errors} line(s)") + say(f" note errors.txt has {errors} line(s)") if case.get("fail_on_errors") and errors: ok = False return ok @@ -312,6 +413,8 @@ def main() -> int: # in 20 s instead of 4 min; 1 is for watching a run ap.add_argument("--timespeed", type=float, default=20.0, help="simulation time multiplier, the simulator offers 1, 5, 20 and 60 (default 20)") + ap.add_argument("-j", "--jobs", type=int, default=1, + help="cases run at once, each in its own simulator window (default 1)") args = ap.parse_args() if not args.game_dir: @@ -321,11 +424,37 @@ def main() -> int: print(f"exe: {exe}\ngame: {game_dir}") names = args.cases or sorted(p.stem for p in CASES_DIR.glob("*.json")) - results = {n: run_case(n, exe, game_dir, args.timespeed) for n in names} + missing = [n for n in names if not (CASES_DIR / f"{n}.json").exists()] + if missing: + sys.exit("no such case: " + ", ".join(missing)) - print("\n=== summary") - for n, ok in results.items(): - print(f" {'PASS' if ok else 'FAIL'} {n}") + started = time.monotonic() + results = {} + if args.jobs <= 1: + def say(line, progress=False): + print(line, flush=True) + for n in names: + results[n] = run_case(n, exe, game_dir, args.timespeed, say) + else: + # each case prints its block when it is done, so parallel output does not interleave + def job(n): + lines = [] + def say(line, progress=False): + if not progress: + lines.append(line) + ok = run_case(n, exe, game_dir, args.timespeed, say) + return n, ok, lines + with ThreadPoolExecutor(max_workers=args.jobs) as pool: + for n in names: + print(f"started {n}", flush=True) + for fut in as_completed([pool.submit(job, n) for n in names]): + n, ok, lines = fut.result() + print("\n".join(lines), flush=True) + results[n] = ok + + print(f"\n=== summary ({time.monotonic() - started:.0f}s)") + for n in names: + print(f" {'PASS' if results[n] else 'FAIL'} {n}") return 0 if all(results.values()) else 1 From 219836b4648020de36f1ccf7dc7d7edc71b8e2a6 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Wed, 14 Oct 2026 13:55:06 +0200 Subject: [PATCH 04/28] test: track_friction case key for slippery rail scenarios --- tests/scenarios/README.md | 1 + tests/scenarios/run.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index 23f79301..463086cc 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -59,6 +59,7 @@ scenery is newer. | `base_scenery` | scenery file in the game's `scenery/` dir | | `strip_trainsets` | drop the base scenery's trainsets (default `true`) | | `time` | replace the scenery start time, `HH:MM` | +| `track_friction` | set the friction of every track of the base scenery (default in the sceneries is 0.15), e.g. for slippery rail | | `fragment` | `.scn` file in `cases/` added to the scenery: trainsets after `FirstInit` (before it they derail on unlinked track), the rest before it (launchers read after it never fire) | | `vars` | values for `{{name}}` in the fragment; a list becomes one line per item | | `duration` | seconds of simulation to record | diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index 41990670..32be5dfc 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -86,6 +86,12 @@ def set_start_time(text: str, hhmm: str) -> str: return re.sub(r"(?im)^(\s*time\s+)\d{1,2}:\d{2}", lambda m: m.group(1) + hhmm, text, count=1) +def set_track_friction(text: str, friction: float) -> str: + """Sets the friction of every track: node track ...""" + return re.sub(r"(?im)^(\s*node\s+\S+\s+\S+\s+\S+\s+track\s+\S+\s+\S+\s+\S+\s+)[-\d.]+", + lambda m: m.group(1) + f"{friction}", text) + + def fill_vars(fragment: str, values: dict, keep=frozenset()) -> str: """Replaces {{name}} with the case's value; a list value becomes one line per item.""" def value(m): @@ -107,6 +113,8 @@ def build_scenery(case: dict, game_dir: Path, case_name: str, vehicle: str) -> s text, _ = split_trainsets(text) if "time" in case: text = set_start_time(text, case["time"]) + if "track_friction" in case: + text = set_track_friction(text, case["track_friction"]) fragment = (CASES_DIR / case["fragment"]).read_bytes().decode("latin-1") # vars first: {{vehicle}} inside a var's value gets filled in the second pass fragment = fill_vars(fragment, case.get("vars", {}), keep={"vehicle"}) From fe9e2f77c24f71d965fd3d6c11478840c6f659e8 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Wed, 14 Oct 2026 14:11:54 +0200 Subject: [PATCH 05/28] feat(log): vehicle speed column in the physicslog The Velocity column is the wheel speed, which stays at 0 while the wheels are locked in a skid, so a vehicle sliding on locked wheels looked as if it stood still. Add the signed vehicle speed in km/h as the last column. --- vehicle/Driver.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vehicle/Driver.cpp b/vehicle/Driver.cpp index 08f0c0d2..c5f5f51e 100644 --- a/vehicle/Driver.cpp +++ b/vehicle/Driver.cpp @@ -1880,7 +1880,8 @@ TController::TController(bool AI, TDynamicObject *NewControll, bool InitPsyche, << "BrakePress[MPa] PipePress[MPa] MotorCurrent[A] " << "MCP SCP BCP LBP Direction Command CVal1 CVal2 " << "Security Wheelslip " - << "EngineTemp[Deg] OilTemp[Deg] WaterTemp[Deg] WaterAuxTemp[Deg]" + << "EngineTemp[Deg] OilTemp[Deg] WaterTemp[Deg] WaterAuxTemp[Deg] " + << "VehicleVelocity[km/h]" << "\r\n"; LogFile << std::fixed << std::setprecision( 4 ); #endif @@ -4817,7 +4818,9 @@ void TController::PhysicsLog() << mvControlling->dizel_heat.Ts << " " << mvControlling->dizel_heat.To << " " << mvControlling->dizel_heat.temperatura1 << " " - << mvControlling->dizel_heat.temperatura2 + << mvControlling->dizel_heat.temperatura2 << " " + // vehicle speed [km/h], the second column is the wheel speed, which stays at 0 while the wheels are locked in a skid + << mvOccupied->V * 3.6 << "\r\n"; #endif #if LOGPRESS == 1 From 6209e165583b841b61d4ec9a99b2d69b2e0db5c2 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Wed, 14 Oct 2026 14:11:54 +0200 Subject: [PATCH 06/28] test: slippery rail skid cases, wheelslip and vehicle speed checks - physicslog rows keep all columns (text ones as None), checks can read the wheelslip flag and the new vehicle speed column - slips: the lead vehicle's wheels slip at some point (makes sure a slippery rail case does what it's meant to) - slip_clears: once stopped, the wheels stop slipping within N seconds - holds_still: once stopped, the vehicle doesn't creep, by vehicle speed - td_red_signal_moving.scn: the red signal case starting already moving - eu07_slippery_red_signal / eu07_slippery_push_red_signal: skid on braking for the red signal with track friction 0.03; the pushing one fails without the fix for eu07/maszyna#479 (stays skidding, creeps) - sn61_td_timetable: stop hold 5 -> 2 s, the railcar arrives 5-7 s before its departure time, so 5 s made the case flaky --- tests/scenarios/README.md | 14 +++- .../cases/eu07_slippery_push_red_signal.json | 36 ++++++++++ .../cases/eu07_slippery_red_signal.json | 36 ++++++++++ tests/scenarios/cases/sn61_td_timetable.json | 2 +- .../scenarios/cases/td_red_signal_moving.scn | 6 ++ tests/scenarios/run.py | 68 ++++++++++++++++++- 6 files changed, 156 insertions(+), 6 deletions(-) create mode 100644 tests/scenarios/cases/eu07_slippery_push_red_signal.json create mode 100644 tests/scenarios/cases/eu07_slippery_red_signal.json create mode 100644 tests/scenarios/cases/td_red_signal_moving.scn diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index 463086cc..4600a357 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -71,9 +71,13 @@ scenery is newer. the one whose physicslog is checked. Other vehicles of the consist take names made from it, e.g. `{{vehicle}}_s`. The td.scn cases share two fragments, `td_timetable.scn` (route set, train runs to Przystanek) and `td_red_signal.scn` (no route, exit signal at stop), and differ -only in `vars.consist`. +only in `vars.consist`. `td_red_signal_moving.scn` is the red signal one with the train +starting at `vars.velocity` km/h; with `track_friction` 0.03 it locks the wheels when braking. -Checks (speeds in km/h, times in seconds of physicslog time, distances in metres): +Checks (speeds in km/h, times in seconds of physicslog time, distances in metres). Most use +the physicslog Velocity column, which is the wheel speed: it stays at 0 while the wheels are +locked in a skid. The ones marked *vehicle speed* use the vehicle speed column at the end of +the row instead, logged by builds from this branch on. - `starts` `min_speed`, `within`: reaches `min_speed` within `within` seconds. - `max_speed` `limit`, `tolerance`: never goes faster than `limit + tolerance`. @@ -86,6 +90,12 @@ Checks (speeds in km/h, times in seconds of physicslog time, distances in metres - `distance` `min`, `max`: total distance travelled. The physicslog has no position, so the distance comes from integrating the speed; on td.scn every vehicle's first stop after Stacja1 came out at 1400 m (+-3), and the red signal stop at 140 m. +- `slips` `within`: the lead vehicle's wheels slip or lock at some point (within `within` + seconds if given), so a slippery rail case does what it's meant to. +- `slip_clears` `moving_speed`, `stop_speed`, `within` (*vehicle speed*): once stopped after + moving off, the wheels stop slipping within `within` seconds. +- `holds_still` `moving_speed`, `stop_speed`, `max_distance` (*vehicle speed*): once stopped + after moving off, moves no more than `max_distance` metres, e.g. no creeping on locked wheels. New check types go in `CHECKS` in `run.py`; each gets the physicslog rows and the check's parameters and returns `(passed, detail)`. diff --git a/tests/scenarios/cases/eu07_slippery_push_red_signal.json b/tests/scenarios/cases/eu07_slippery_push_red_signal.json new file mode 100644 index 00000000..f7ae10bb --- /dev/null +++ b/tests/scenarios/cases/eu07_slippery_push_red_signal.json @@ -0,0 +1,36 @@ +{ + "description": "EU07 pushing three coaches on td.scn with slippery rail and no route set, starting at 30 km/h: brakes for the exit signal at stop and skids. Pushed by the train it doesn't roll back at the stop, so it used to stay skidding on locked wheels with the brakes capped by the sliding adhesion and creep (eu07/maszyna#479).", + "base_scenery": "td.scn", + "time": "10:34", + "track_friction": 0.03, + "fragment": "td_red_signal_moving.scn", + "vars": { + "velocity": "30", + "consist": [ + "node -1 0 {{vehicle}}_w1 dynamic pkp\\bdhpumn_v3 bdhpumn_50512608068-6 BDHPUMN_TAPICERKA_1 0 nobody 115 0 enddynamic", + "node -1 0 {{vehicle}}_w2 dynamic pkp\\bdhpumn_v3 bdhpumn_50512608011-6 BDHPUMN_NG_1 0 nobody 115 0 enddynamic", + "node -1 0 {{vehicle}}_w3 dynamic pkp\\bdhpumn_v3 bdhpumn_50512608068-6 BDHPUMN_TAPICERKA_1 0 nobody 115 0 enddynamic", + "node -1 0 {{vehicle}} dynamic pkp\\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic" + ] + }, + "duration": 240, + "timeout": 900, + "checks": [ + { + "type": "slips", + "within": 60 + }, + { + "type": "slip_clears", + "moving_speed": 5, + "stop_speed": 0.05, + "within": 5 + }, + { + "type": "holds_still", + "moving_speed": 5, + "stop_speed": 0.05, + "max_distance": 0.01 + } + ] +} diff --git a/tests/scenarios/cases/eu07_slippery_red_signal.json b/tests/scenarios/cases/eu07_slippery_red_signal.json new file mode 100644 index 00000000..40b5944e --- /dev/null +++ b/tests/scenarios/cases/eu07_slippery_red_signal.json @@ -0,0 +1,36 @@ +{ + "description": "EU07 with three coaches on td.scn with slippery rail and no route set, starting at 30 km/h: brakes for the exit signal at stop and skids, then the skid ends and the train holds still.", + "base_scenery": "td.scn", + "time": "10:34", + "track_friction": 0.03, + "fragment": "td_red_signal_moving.scn", + "vars": { + "velocity": "30", + "consist": [ + "node -1 0 {{vehicle}} dynamic pkp\\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic", + "node -1 0 {{vehicle}}_w1 dynamic pkp\\bdhpumn_v3 bdhpumn_50512608068-6 BDHPUMN_TAPICERKA_1 0 nobody 115 0 enddynamic", + "node -1 0 {{vehicle}}_w2 dynamic pkp\\bdhpumn_v3 bdhpumn_50512608011-6 BDHPUMN_NG_1 0 nobody 115 0 enddynamic", + "node -1 0 {{vehicle}}_w3 dynamic pkp\\bdhpumn_v3 bdhpumn_50512608068-6 BDHPUMN_TAPICERKA_1 0 nobody 115 0 enddynamic" + ] + }, + "duration": 240, + "timeout": 900, + "checks": [ + { + "type": "slips", + "within": 60 + }, + { + "type": "slip_clears", + "moving_speed": 5, + "stop_speed": 0.05, + "within": 5 + }, + { + "type": "holds_still", + "moving_speed": 5, + "stop_speed": 0.05, + "max_distance": 0.01 + } + ] +} diff --git a/tests/scenarios/cases/sn61_td_timetable.json b/tests/scenarios/cases/sn61_td_timetable.json index 1db425c8..9fa4b18d 100644 --- a/tests/scenarios/cases/sn61_td_timetable.json +++ b/tests/scenarios/cases/sn61_td_timetable.json @@ -12,6 +12,6 @@ { "type": "starts", "min_speed": 20, "within": 150 }, { "type": "max_speed", "limit": 90, "tolerance": 3 }, { "type": "brakes", "min_pressure": 0.5 }, - { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 2 } ] } diff --git a/tests/scenarios/cases/td_red_signal_moving.scn b/tests/scenarios/cases/td_red_signal_moving.scn new file mode 100644 index 00000000..648e5d90 --- /dev/null +++ b/tests/scenarios/cases/td_red_signal_moving.scn @@ -0,0 +1,6 @@ +// added by tests/scenarios/run.py to td.scn, whose own trainsets were removed. +// like td_red_signal.scn (no route, the exit signal stacja1_j about 150 m ahead stays at stop), +// but the train starts already moving at {{velocity}} km/h, so it has to brake right away. +trainset td/rozklad tdo_n1734 0.0 {{velocity}} +{{consist}} +endtrainset diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index 32be5dfc..2cd754a0 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -43,8 +43,11 @@ OUT_DIR = HERE / "out" # physicslog columns, see TController::PhysicsLog() in vehicle/Driver.cpp. # Velocity is in km/h and pressures in bar, despite the header saying m/s and MPa. COL_TIME = 0 -COL_VELOCITY = 1 +COL_VELOCITY = 1 # wheel speed: stays at 0 while the wheels are locked in a skid COL_BRAKE_PRESS = 8 +COL_COMMAND = 16 # text, read as None +COL_WHEELSLIP = 20 +COL_VEHICLE_VELOCITY = 25 # signed vehicle speed, km/h; only in builds which log it # settings layered over the user's eu07.ini for a test run TEST_INI = """\ @@ -145,7 +148,7 @@ def build_scenery(case: dict, game_dir: Path, case_name: str, vehicle: str) -> s def read_physicslog(path: Path) -> list[list[float]]: - """Returns rows of numeric columns up to BrakePress; skips the header and partial lines.""" + """Returns rows of columns, None for the text ones; skips the header and partial lines.""" rows = [] try: lines = path.read_text(errors="replace").splitlines() @@ -156,9 +159,15 @@ def read_physicslog(path: Path) -> list[list[float]]: if len(parts) <= COL_BRAKE_PRESS: continue # line still being written try: - rows.append([float(p) for p in parts[: COL_BRAKE_PRESS + 1]]) + row = [float(p) for p in parts[: COL_BRAKE_PRESS + 1]] except ValueError: continue + for p in parts[COL_BRAKE_PRESS + 1:]: + try: + row.append(float(p)) + except ValueError: + row.append(None) + rows.append(row) return rows @@ -331,7 +340,60 @@ def check_distance(rows, max=None, min=None): return ok, f"travelled {d:.0f} m (wanted {bounds})" +def check_slips(rows, within=None): + """The lead vehicle's wheels slip (or lock in a skid) at some point, e.g. to make sure a slippery rail case does what it's meant to.""" + for r in rows: + if within is not None and r[COL_TIME] > within: + break + if len(r) > COL_WHEELSLIP and r[COL_WHEELSLIP]: + return True, f"wheels slipping at t={r[COL_TIME]:.1f}s, v={r[COL_VELOCITY]:.1f} km/h" + return False, "the wheels never slipped" + + +def check_holds_still(rows, moving_speed, stop_speed, max_distance): + """Once stopped after moving off, the vehicle doesn't creep. Uses the vehicle speed rather than the + wheel speed, so it also catches a vehicle sliding on locked wheels.""" + if not rows or len(rows[0]) <= COL_VEHICLE_VELOCITY or rows[0][COL_VEHICLE_VELOCITY] is None: + return False, "the physicslog has no vehicle speed column, the build is too old" + speed = [abs(r[COL_VEHICLE_VELOCITY]) for r in rows] + start = next((i for i, v in enumerate(speed) if v >= moving_speed), None) + if start is None: + return False, f"never reached {moving_speed} km/h" + stop = next((i for i in range(start, len(rows)) if speed[i] <= stop_speed), None) + if stop is None: + return False, f"never came down to {stop_speed} km/h" + d, top, top_t = 0.0, 0.0, rows[stop][COL_TIME] + for i in range(stop + 1, len(rows)): + d += (speed[i - 1] + speed[i]) / 2 / 3.6 * (rows[i][COL_TIME] - rows[i - 1][COL_TIME]) + if speed[i] > top: + top, top_t = speed[i], rows[i][COL_TIME] + held = rows[-1][COL_TIME] - rows[stop][COL_TIME] + detail = f"stopped at t={rows[stop][COL_TIME]:.1f}s, then moved {d:.3f} m in {held:.0f}s, top {top:.4f} km/h at t={top_t:.1f}s" + return d <= max_distance, detail + + +def check_slip_clears(rows, moving_speed, stop_speed, within): + """Once the vehicle has stopped after moving off, its wheels stop slipping within `within` seconds + (a vehicle left skidding on locked wheels keeps its brakes capped by the sliding adhesion).""" + if not rows or len(rows[0]) <= COL_VEHICLE_VELOCITY or rows[0][COL_VEHICLE_VELOCITY] is None: + return False, "the physicslog has no vehicle speed column, the build is too old" + start = next((i for i, r in enumerate(rows) if abs(r[COL_VEHICLE_VELOCITY]) >= moving_speed), None) + if start is None: + return False, f"never reached {moving_speed} km/h" + stop = next((i for i in range(start, len(rows)) if abs(rows[i][COL_VEHICLE_VELOCITY]) <= stop_speed), None) + if stop is None: + return False, f"never came down to {stop_speed} km/h" + deadline = rows[stop][COL_TIME] + within + late = [r for r in rows[stop:] if r[COL_TIME] > deadline and r[COL_WHEELSLIP]] + if late: + return False, f"stopped at t={rows[stop][COL_TIME]:.1f}s, wheels still slipping at t={late[0][COL_TIME]:.1f}s ({len(late)} rows after the {within}s grace)" + return True, f"stopped at t={rows[stop][COL_TIME]:.1f}s, no slipping after t={deadline:.1f}s" + + CHECKS = { + "slips": check_slips, + "slip_clears": check_slip_clears, + "holds_still": check_holds_still, "starts": check_starts, "max_speed": check_max_speed, "brakes": check_brakes, From 02e29f69712d1e02e960c14eba32feb1a3245ab3 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sat, 7 Nov 2026 13:17:26 +0100 Subject: [PATCH 07/28] feat(simulation): test mode, --test runs a scenery unattended and exits with the result (eu07/maszyna#541) eu07 --test -s [--test-timeout ] runs the scenery in ghostview with the null renderer (models loaded, no textures), a hidden window, no sound, network or python, and in fixed steps of 0.03 s of simulation time per frame (three 10 ms physics steps) as fast as it can. Without -seed it uses a fixed seed, so two runs give byte-identical physicslogs. The scenery reports its result to the __test.result memory cell, which the simulator always creates: value 1 passes once every vehicle has stood still for 1 s, value 0 fails at once. The run ends with exit code 0 pass, 1 fail, 2 a train passed a signal at stop (SetVelocity 0 0, as seen in its driver's speed table; SetVelocity 0 x on automatic block doesn't count), 3 timeout (test.timeout, default 3600 s of simulation time), 4 standstill (every vehicle still for test.standstill s, default 300, with no result; a pass with test.pass_on_standstill yes), 5 error (no -s, the scenery didn't load or has no vehicles, an exception). The test.* keys go in the scenery's config block; test.fail_if_stop_overran no only logs a signal passed at danger. Outside the test mode a signal passed at danger is logged. The null renderer gets a ui backend which draws nothing, hands out the null texture and material instead of throwing, and ends the ui frame. A GL context is still made for the hidden window, without asking for a version, as parts of the ui create textures with gl calls of their own. No crash dialog or message box in test mode; the window code no longer needs a monitor. --- CMakeLists.txt | 1 + EU07.cpp | 17 +++- application/application.cpp | 82 +++++++++++++--- application/drivermode.cpp | 13 +++ application/scenarioloadermode.cpp | 10 ++ rendering/nullrenderer.cpp | 19 ++++ rendering/nullrenderer.h | 30 ++++-- simulation/simulation.cpp | 3 + simulation/simulationtest.cpp | 149 +++++++++++++++++++++++++++++ simulation/simulationtest.h | 66 +++++++++++++ utilities/Globals.cpp | 37 +++++++ utilities/Globals.h | 10 ++ utilities/Timer.cpp | 13 ++- utilities/Timer.h | 3 + vehicle/Driver.cpp | 29 +++++- vehicle/Driver.h | 2 + 16 files changed, 458 insertions(+), 26 deletions(-) create mode 100644 simulation/simulationtest.cpp create mode 100644 simulation/simulationtest.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f34ad61..8a4e1d6e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -160,6 +160,7 @@ set(SOURCES "world/station.cpp" "application/application.cpp" "simulation/simulationtime.cpp" +"simulation/simulationtest.cpp" "scene/sceneeditor.cpp" "rendering/screenshot.cpp" "input/driverkeyboardinput.cpp" diff --git a/EU07.cpp b/EU07.cpp index e0361829..aac7cae1 100644 --- a/EU07.cpp +++ b/EU07.cpp @@ -19,6 +19,8 @@ Stele, firleju, szociu, hunter, ZiomalCl, OLI_EU and others #include "stdafx.h" #include "application/application.h" +#include "simulation/simulationtest.h" +#include "utilities/Globals.h" #include "utilities/Logs.h" #include #ifdef WITHDUMPGEN @@ -148,9 +150,14 @@ int main(int argc, char *argv[]) #ifdef _WIN32 catch (std::runtime_error const &Error) { - std::string msg = "Simulator crash occured :(\n"; - msg += Error.what(); - MessageBoxA(nullptr, msg.c_str(), "Simulator crashed :(", MB_ICONERROR); + ErrorLog("Critical error: " + std::string(Error.what())); + // in test mode nobody is there to close the box, the exit code tells of the error + if (false == Global.test.enabled) + { + std::string msg = "Simulator crash occured :(\n"; + msg += Error.what(); + MessageBoxA(nullptr, msg.c_str(), "Simulator crashed :(", MB_ICONERROR); + } } #endif } @@ -158,5 +165,7 @@ int main(int argc, char *argv[]) fflush(stdout); fflush(stderr); #endif - std::_Exit(0); // skip destructors, there are ordering errors which causes segfaults + // skip destructors, there are ordering errors which causes segfaults + // a test run (--test) says its result in the exit code + std::_Exit(Global.test.enabled ? simulation::Test.exit_code() : 0); } diff --git a/application/application.cpp b/application/application.cpp index ee381f98..37afff67 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -378,7 +378,7 @@ int eu07_application::init(int Argc, char *Argv[]) return result; } - if (crashreport_is_pending()) + if (crashreport_is_pending() && false == Global.test.enabled) { // run crashgui as early as possible if ((result = run_crashgui()) != 0) { @@ -951,14 +951,17 @@ GLFWwindow *eu07_application::window(int const Windowindex, bool visible, int wi } // for index -1 create a new child window - auto const *vmode{glfwGetVideoMode(monitor ? monitor : glfwGetPrimaryMonitor())}; - - // match requested video mode to current to allow for - // fullwindow creation when resolution is the same - glfwWindowHint(GLFW_RED_BITS, vmode->redBits); - glfwWindowHint(GLFW_GREEN_BITS, vmode->greenBits); - glfwWindowHint(GLFW_BLUE_BITS, vmode->blueBits); - glfwWindowHint(GLFW_REFRESH_RATE, vmode->refreshRate); + auto *videomonitor{monitor ? monitor : glfwGetPrimaryMonitor()}; + // a machine without a display (e.g. a test runner) has no monitor to match + if (auto const *vmode{videomonitor ? glfwGetVideoMode(videomonitor) : nullptr}) + { + // match requested video mode to current to allow for + // fullwindow creation when resolution is the same + glfwWindowHint(GLFW_RED_BITS, vmode->redBits); + glfwWindowHint(GLFW_GREEN_BITS, vmode->greenBits); + glfwWindowHint(GLFW_BLUE_BITS, vmode->blueBits); + glfwWindowHint(GLFW_REFRESH_RATE, vmode->refreshRate); + } glfwWindowHint(GLFW_VISIBLE, visible); @@ -1195,6 +1198,19 @@ int eu07_application::init_settings(int Argc, char *Argv[]) WriteLog(std::format("seed given on the command line: {} -> {}", Argv[i], Global.random_seed)); } } + else if (token == "--test") + { + // integration test: the scenery runs unattended and the exit code is the result (eu07/maszyna#541) + Global.test.enabled = true; + } + else if (token == "--test-timeout") + { + if (i + 1 < Argc) + { + Global.test.timeout = std::max(0.0, std::atof(Argv[++i])); + Global.test.timeout_from_command_line = true; + } + } else if (token == "-ini") { // extra settings file read on top of eu07.ini, so a run can override a few keys @@ -1210,13 +1226,45 @@ int eu07_application::init_settings(int Argc, char *Argv[]) << " [-v vehiclename]" << " [-edit [sceneryfilepath]]" << " [-seed number|word]" - << " [-ini settingsfile]" << std::endl; + << " [-ini settingsfile]" + << " [--test [--test-timeout seconds]]" << std::endl; return -1; } } if (editwithoutscenery && false == sceneryfromcommandline) Global.SceneryFile.clear(); + if (Global.test.enabled) + { + if (false == sceneryfromcommandline || Global.SceneryFile.empty()) + { + ErrorLog("test: --test needs a scenery given with -s"); + return -1; + } + // nobody watches nor listens: no player vehicle, nothing drawn, no sound, no window in the way + Global.editor_session = false; + Global.local_start_vehicle = "ghostview"; + Global.GfxRenderer = "null"; + Global.NvRenderer = false; + Global.bSoundEnabled = false; + Global.python_enabled = false; + Global.bInactivePause = false; + Global.bFullScreen = false; + Global.fullscreen_windowed = false; + Global.VSync = false; + Global.iMultisampling = 0; + Global.gfx_usegles = false; + Global.minframetime = std::chrono::duration(0.0f); + Global.extra_viewports.clear(); + Global.headtrack_conf.joy.clear(); + // a test runs on its own + Global.network_servers.clear(); + Global.network_client.reset(); + // and the same way every time, unless given another seed (-seed, or randomseed in the settings) + if (!Global.random_seed) + Global.random_seed = seed_of("test"); + } + return 0; } @@ -1290,6 +1338,12 @@ int eu07_application::init_glfw() { glfwWindowHint(GLFW_CLIENT_API, GLFW_NO_API); } + else if (Global.test.enabled) + { + // the null renderer draws nothing, but parts of the ui make textures with gl calls of their own. + // any context will do for them, so none is asked for in particular: a machine without a graphics + // driver (a test runner) has only the basic one of the system + } else { if (!Global.LegacyRenderer) @@ -1341,7 +1395,8 @@ int eu07_application::init_glfw() Global.bFullScreen = true; } - auto *mainwindow = window(-1, true, Global.window_size.x, Global.window_size.y, Global.bFullScreen ? monitor : nullptr, true, false); + // test mode keeps its window hidden: glfw, the ui and the input still need one, but nothing is shown in it + auto *mainwindow = window(-1, false == Global.test.enabled, Global.window_size.x, Global.window_size.y, Global.bFullScreen ? monitor : nullptr, true, false); if (mainwindow == nullptr) { @@ -1428,6 +1483,11 @@ int eu07_application::init_gfx() // default render path GfxRenderer = gfx_renderer_factory::get_instance()->create("modern"); } + else if (Global.GfxRenderer == "null") + { + // draws nothing and loads no textures (test mode) + GfxRenderer = gfx_renderer_factory::get_instance()->create("null"); + } else if (Global.GfxRenderer == "experimental") { GfxRenderer = gfx_renderer_factory::get_instance()->create(Global.GfxRenderer); diff --git a/application/drivermode.cpp b/application/drivermode.cpp index b55d09c2..1b3a44d6 100644 --- a/application/drivermode.cpp +++ b/application/drivermode.cpp @@ -17,6 +17,7 @@ http://mozilla.org/MPL/2.0/. #include "simulation/simulation.h" #include "simulation/simulationtime.h" #include "simulation/simulationenvironment.h" +#include "simulation/simulationtest.h" #include "scene/scene.h" #include "rendering/lightarray.h" #include "rendering/particles.h" @@ -305,6 +306,11 @@ bool driver_mode::update() simulation::Events.update(); simulation::Region->update_events(); simulation::Lights.update(); + + if (Global.test.enabled && simulation::is_ready) + { + simulation::Test.update(deltatime); + } } // render time routines follow: @@ -478,6 +484,13 @@ void driver_mode::enter() Timer::ResetTimers(); + if (Global.test.enabled) + { + // the test runs as fast as it can compute, in steps of fixed length, so that runs on different machines agree + Timer::set_fixed_step(simulation::test_manager::frame_step); + simulation::Test.start(); + } + set_picking(!Global.captureonstart); m_editorpending = Global.editor_session; diff --git a/application/scenarioloadermode.cpp b/application/scenarioloadermode.cpp index f8f06ac9..b84c1912 100644 --- a/application/scenarioloadermode.cpp +++ b/application/scenarioloadermode.cpp @@ -14,6 +14,7 @@ http://mozilla.org/MPL/2.0/. #include "simulation/simulation.h" #include "simulation/simulationtime.h" #include "simulation/simulationenvironment.h" +#include "simulation/simulationtest.h" #include "application/application.h" #include "application/scenarioloaderuilayer.h" #include "rendering/renderer.h" @@ -58,6 +59,10 @@ bool scenarioloader_mode::update() { catch (std::runtime_error const &e) { ErrorLog(std::format("Bad init: scenario \"{}\" couldn't be opened ({})", Global.SceneryFile, e.what())); state.reset(); + if (Global.test.enabled) { + simulation::Test.finish(simulation::test_result::error, "scenario couldn't be opened"); + return true; + } Global.SceneryFile.clear(); Application.pop_mode(); Application.push_mode(eu07_application::mode::launcher); @@ -73,6 +78,11 @@ bool scenarioloader_mode::update() { // back to the launcher as when the scenario can't be opened, not on to the driver mode with the scenery half loaded ErrorLog( "Bad init: scenario loading failed" ); state.reset(); + if( Global.test.enabled ) { + // there's nothing to test; the exit code says the run went wrong + simulation::Test.finish( simulation::test_result::error, "scenario loading failed" ); + return true; + } Global.SceneryFile.clear(); Application.pop_mode(); Application.push_mode(eu07_application::mode::launcher); diff --git a/rendering/nullrenderer.cpp b/rendering/nullrenderer.cpp index 332d357c..f193ad3e 100644 --- a/rendering/nullrenderer.cpp +++ b/rendering/nullrenderer.cpp @@ -1,6 +1,25 @@ #include "stdafx.h" #include "rendering/nullrenderer.h" +#include "application/application.h" + +void null_imgui_renderer::BeginFrame() +{ + auto &io { ImGui::GetIO() }; + if( false == io.Fonts->IsBuilt() ) { + unsigned char *pixels; + int width, height; + io.Fonts->GetTexDataAsAlpha8( &pixels, &width, &height ); + } +} + +bool null_renderer::Render() +{ + // the ui frame begun by the application has to be ended, or imgui won't start the next one + Application.render_ui(); + return true; +} + std::unique_ptr null_renderer::create_func() { return std::make_unique(); diff --git a/rendering/nullrenderer.h b/rendering/nullrenderer.h index 0b21580b..2a7ef039 100644 --- a/rendering/nullrenderer.h +++ b/rendering/nullrenderer.h @@ -35,7 +35,18 @@ private: release_() override { /* null implementation, nothing to do */ } }; +// user interface backend which draws nothing: imgui still builds its frames, so the ui code runs as usual +class null_imgui_renderer : public imgui_renderer { +public: + bool Init() override { return true; } + void Shutdown() override { /* null implementation, nothing to do */ } + // imgui wants its font atlas built before a frame starts; that's done on the cpu, no texture is made of it + void BeginFrame() override; + void Render() override { /* null implementation, nothing to do */ } +}; + // bare-bones render controller, in lack of anything better yet +// it's the renderer of the test mode (--test): models are loaded for the physics, textures aren't class null_renderer : public gfx_renderer { public: @@ -49,7 +60,7 @@ public: Init( GLFWwindow *Window ) override { return true; } // main draw call. returns false on error bool - Render() override { return true; } + Render() override; void SwapBuffers() override { /* null implementation, nothing to do */ } inline @@ -105,20 +116,22 @@ public: void Bind_Material( material_handle const Material, TSubModel const *sm = nullptr, lighting_data const *lighting = nullptr ) override { /* null implementation, nothing to do */ } IMaterial const * - Material( material_handle const Material ) const override { return m_materials.at(Material - 1).get(); } + Material( material_handle const Material ) const override { + return ( Material > 0 && static_cast( Material ) <= m_materials.size() ? m_materials[ Material - 1 ].get() : IMaterial::null_material() ); } // shader methods auto Fetch_Shader( std::string const &name ) -> std::shared_ptr override { throw std::runtime_error("not impl"); } // texture methods + // no textures are loaded; whoever asks gets the null texture texture_handle - Fetch_Texture( std::string const &Filename, bool const Loadnow = true, GLint format_hint = GL_SRGB_ALPHA ) override { throw std::runtime_error("not impl"); } + Fetch_Texture( std::string const &Filename, bool const Loadnow = true, GLint format_hint = GL_SRGB_ALPHA ) override { return null_handle; } void Bind_Texture( texture_handle const Texture ) override { /* null implementation, nothing to do */ } void Bind_Texture( std::size_t const Unit, texture_handle const Texture ) override { /* null implementation, nothing to do */ } - opengl_texture & - Texture( texture_handle const Texture ) override { throw std::runtime_error("not impl"); } - opengl_texture const & - Texture( texture_handle const Texture ) const override { throw std::runtime_error("not impl"); } + ITexture & + Texture( texture_handle const Texture ) override { return *ITexture::null_texture(); } + ITexture const & + Texture( texture_handle const Texture ) const override { return *ITexture::null_texture(); } // utility methods void Pick_Control_Callback( std::function Callback ) override { /* null implementation, nothing to do */ } @@ -152,11 +165,12 @@ public: imgui_renderer *GetImguiRenderer() override { - return nullptr; + return &m_imgui; } private: std::string empty_str; + null_imgui_renderer m_imgui; gfx::geometrybank_manager m_geometry; std::vector> m_materials; }; diff --git a/simulation/simulation.cpp b/simulation/simulation.cpp index ea217717..59635c38 100644 --- a/simulation/simulation.cpp +++ b/simulation/simulation.cpp @@ -11,6 +11,7 @@ http://mozilla.org/MPL/2.0/. #include "simulation/simulation.h" #include "simulation/simulationtime.h" #include "simulation/simulationenvironment.h" +#include "simulation/simulationtest.h" #include "utilities/Globals.h" #include "world/Event.h" @@ -109,6 +110,8 @@ state_manager::init_scripting_interface() { simulation::Memory.insert( memorycell ); simulation::Region->insert( memorycell ); } + // the cell a test scenery reports its result to + simulation::Test.init_result_cell(); } // legacy method, calculates changes in simulation state over specified time diff --git a/simulation/simulationtest.cpp b/simulation/simulationtest.cpp new file mode 100644 index 00000000..4dbe4c24 --- /dev/null +++ b/simulation/simulationtest.cpp @@ -0,0 +1,149 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#include "stdafx.h" +#include "simulation/simulationtest.h" + +#include "application/application.h" +#include "scene/scene.h" +#include "simulation/simulation.h" +#include "utilities/Globals.h" +#include "utilities/Logs.h" +#include "vehicle/DynObj.h" +#include "world/Event.h" +#include "world/MemCell.h" + +namespace simulation { + +test_manager Test; + +std::string_view to_string(test_result const Result) +{ + switch (Result) + { + case test_result::pass: return "pass"; + case test_result::fail: return "fail"; + case test_result::signal_passed_at_danger: return "signal passed at danger"; + case test_result::timeout: return "timeout"; + case test_result::standstill: return "standstill"; + case test_result::error: return "error"; + } + return "unknown"; +} + +void test_manager::init_result_cell() +{ + auto *cell = new TMemCell({0, -1, std::string(result_cell), "memcell"}); + // neither of the reported values, so that a cell nobody wrote to says nothing + cell->UpdateValues("", -1.0, 0.0, basic_event::flags::value1 | basic_event::flags::value2); + simulation::Memory.insert(cell); + simulation::Region->insert(cell); +} + +void test_manager::start() +{ + if (m_started) + return; + m_started = true; + m_time = 0.0; + m_resttime = 0.0; + if (simulation::Vehicles.sequence().empty()) + { + // most likely the scenery wasn't found, or it isn't a test scenery + finish(test_result::error, "the scenery has no vehicles"); + return; + } + // a pause set by the scenery or the settings would never be lifted, nobody is there to press the key + Global.iPause = 0; + WriteLog(std::format("test: started, timeout {:.0f} s, standstill {:.0f} s{}{}", Global.test.timeout, Global.test.standstill, + Global.test.pass_on_standstill ? ", standstill passes" : "", + Global.test.fail_if_stop_overran ? "" : ", signals passed at danger allowed")); +} + +void test_manager::update(double const Deltatime) +{ + if (false == m_started || m_finished) + return; + + m_time += Deltatime; + + if (auto const *cell = simulation::Memory.find(std::string(result_cell))) + { + auto const value { cell->Value1() }; + auto const reason { cell->Text().empty() ? std::string() : " (" + cell->Text() + ")" }; + if (value == 0.0) + { + finish(test_result::fail, "__test.result set to 0" + reason); + return; + } + if (value == 1.0 && false == m_passrequested) + { + m_passrequested = true; + WriteLog(std::format("test: pass requested at {:.1f} s{}, waiting for every vehicle to stop", m_time, reason)); + } + } + + auto const atrest = std::none_of(simulation::Vehicles.sequence().begin(), simulation::Vehicles.sequence().end(), + [](TDynamicObject const *Vehicle) { return std::abs(Vehicle->GetVelocity()) > 0.01; }); + m_resttime = atrest ? m_resttime + Deltatime : 0.0; + + if (m_passrequested && m_resttime >= pass_settle_time) + { + finish(test_result::pass, "__test.result set to 1 and every vehicle stopped"); + return; + } + if (Global.test.standstill > 0.0 && m_resttime >= Global.test.standstill) + { + auto const reason { std::format("every vehicle stood still for {:.0f} s", Global.test.standstill) }; + finish(Global.test.pass_on_standstill ? test_result::pass : test_result::standstill, reason); + return; + } + if (Global.test.timeout > 0.0 && m_time >= Global.test.timeout) + { + finish(test_result::timeout, std::format("no result after {:.0f} s", Global.test.timeout)); + return; + } +} + +void test_manager::signal_passed_at_danger(std::string const &Vehicle, std::string const &Signal) +{ + auto const reason { Vehicle + " passed signal " + Signal + " at danger" }; + if (false == Global.test.enabled) + { + WriteLog(reason, logtype::generic); + return; + } + if (false == Global.test.fail_if_stop_overran) + { + WriteLog(std::format("test: {} at {:.1f} s, allowed by test.fail_if_stop_overran", reason, m_time)); + return; + } + finish(test_result::signal_passed_at_danger, reason); +} + +void test_manager::finish(test_result const Result, std::string const &Reason) +{ + if (m_finished || false == Global.test.enabled) + return; + m_finished = true; + m_result = Result; + + auto const line { std::format("test: {} (exit code {}) after {:.1f} s of simulation time: {}", to_string(Result), static_cast(Result), m_time, Reason) }; + if (Result == test_result::pass) + WriteLog(line); + else + ErrorLog(line); + // the process has no console of its own; whoever started it with a redirected output gets the verdict there too + std::fprintf(stdout, "%s\n", line.c_str()); + std::fflush(stdout); + + Application.queue_quit(true); +} + +} // namespace simulation diff --git a/simulation/simulationtest.h b/simulation/simulationtest.h new file mode 100644 index 00000000..c9bc5437 --- /dev/null +++ b/simulation/simulationtest.h @@ -0,0 +1,66 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +#include + +// integration test mode (eu07 --test -s , eu07/maszyna#541): the scenery runs unattended in a tight +// fixed-step loop and the simulator quits with an exit code saying how the test went +namespace simulation { + +// exit codes of a test run; public, scripts and CI read them +enum class test_result : int { + pass = 0, // __test.result set to 1, then every vehicle came to rest (or test.pass_on_standstill) + fail = 1, // __test.result set to 0 + signal_passed_at_danger = 2, // a train went past a stop signal (SetVelocity 0 0), unless test.fail_if_stop_overran no + timeout = 3, // test.timeout seconds of simulation time went by without a result + standstill = 4, // every vehicle stood still for test.standstill seconds without a result + error = 5, // the run could not start (no -s, the scenery failed to load) or the simulator threw +}; + +std::string_view to_string(test_result Result); + +class test_manager { + +public: + // name of the memory cell a scenery reports its result to + static constexpr std::string_view result_cell { "__test.result" }; + // how long everything has to stand before a requested pass is granted [s] + static constexpr double pass_settle_time { 1.0 }; + // fixed simulation time step of one frame in test mode [s]: 3 physics steps of 10 ms + static constexpr double frame_step { 0.03 }; + + // creates the result cell; called with the other scripting interface cells, before the scenery is read + void init_result_cell(); + // the simulation is ready: start the clocks + void start(); + // checks the end conditions after a simulation frame of Deltatime seconds + void update(double Deltatime); + // reported by a driver whose train went past a signal showing stop + void signal_passed_at_danger(std::string const &Vehicle, std::string const &Signal); + // ends the test with given result; the first call wins + void finish(test_result Result, std::string const &Reason); + + bool finished() const { return m_finished; } + // exit code of the process: the result, or error while the test hasn't finished + int exit_code() const { return static_cast(m_finished ? m_result : test_result::error); } + +private: + bool m_started { false }; + bool m_finished { false }; + test_result m_result { test_result::error }; + bool m_passrequested { false }; + double m_time { 0.0 }; // simulation time since the start [s] + double m_resttime { 0.0 }; // how long every vehicle has been standing [s] +}; + +extern test_manager Test; + +} // namespace simulation diff --git a/utilities/Globals.cpp b/utilities/Globals.cpp index b4f2e14b..e3622317 100644 --- a/utilities/Globals.cpp +++ b/utilities/Globals.cpp @@ -456,6 +456,40 @@ bool global_settings::ConfigParseInput(cParser& Parser, const std::string& token return false; } +bool global_settings::ConfigParseTest(cParser& Parser, const std::string& token) +{ + // integration test limits, usually given in the config block of a test scenery + if (token == "test.timeout") + { + double value{ test.timeout }; + ParseOne(Parser, value); + if (false == test.timeout_from_command_line) + test.timeout = std::max(0.0, value); + return true; + } + + if (token == "test.standstill") + { + ParseOne(Parser, test.standstill); + test.standstill = std::max(0.0, test.standstill); + return true; + } + + if (token == "test.pass_on_standstill") + { + ParseOne(Parser, test.pass_on_standstill); + return true; + } + + if (token == "test.fail_if_stop_overran") + { + ParseOne(Parser, test.fail_if_stop_overran); + return true; + } + + return false; +} + bool global_settings::ConfigParseSimulation(cParser& Parser, const std::string& token) { if (token == "async.trainThreads") @@ -1316,6 +1350,9 @@ void global_settings::ConfigParse(cParser& Parser) if (ConfigParseDebug(Parser, token)) continue; + if (ConfigParseTest(Parser, token)) + continue; + // WriteLog(std::format("Unknown config token: {}", token), logtype::warning, false); } while (true); diff --git a/utilities/Globals.h b/utilities/Globals.h index ad216cd9..8fa04e9c 100644 --- a/utilities/Globals.h +++ b/utilities/Globals.h @@ -86,6 +86,15 @@ struct global_settings { std::string local_start_vehicle{ "EU07-424" }; std::string editor_enter_vehicle; // vehicle placed in the editor, to be entered when the driving goes on bool editor_session{ false }; // scenery was opened for editing (-edit): loader keeps track of scenery layers and hands over to the editor + // integration test mode (--test, eu07/maszyna#541); the limits can be set in the scenery's config block (test.*) + struct test_settings { + bool enabled{ false }; + double timeout{ 3600.0 }; // [s] of simulation time; no result by then fails the test + double standstill{ 300.0 }; // [s] every vehicle standing still this long ends the test; 0 = never + bool pass_on_standstill{ false }; // the standstill end passes the test instead of failing it + bool fail_if_stop_overran{ true }; // a train passing a signal at stop ends the test as failed; with false it's only logged + bool timeout_from_command_line{ false }; // --test-timeout given, the scenery doesn't override it + } test; bool editor_tracks{ false }; // F3 track workspace: draw and pick tracks without the scenery draw-range cap int iConvertModels{ 0 }; // tworzenie plików binarnych int iConvertIndexRange{ 1000 }; // range of duplicate vertex scan @@ -377,6 +386,7 @@ struct global_settings { bool ConfigParseNetwork(cParser& Parser, const std::string& token); bool ConfigParseHardware(cParser& Parser, const std::string& token); bool ConfigParseDebug(cParser& Parser, const std::string& token); + bool ConfigParseTest(cParser& Parser, const std::string& token); bool ConfigParse_gfx( cParser &parser, std::string_view const Token ); // sends basic content of the class in legacy (text) format to provided stream void diff --git a/utilities/Timer.cpp b/utilities/Timer.cpp index 0b877f2b..388330bb 100644 --- a/utilities/Timer.cpp +++ b/utilities/Timer.cpp @@ -26,6 +26,7 @@ double fRenderTime{ 0.0 }; double fSoundTimer{ 0.0 }; double fSinceStart{ 0.0 }; double override_delta = -1.0f; +double fixed_step = 0.0; double GetTime() { @@ -53,6 +54,11 @@ void set_delta_override(double t) override_delta = t; } +void set_fixed_step(double t) +{ + fixed_step = t; +} + void ResetTimers() { UpdateTimers( Global.iPause != 0 ); @@ -76,10 +82,15 @@ void UpdateTimers(bool pause) fr = 1000000000; #endif DeltaRenderTime = double(count - oldCount) / double(fr); + if (fixed_step > 0.0) + { + // the frame lasts exactly as long as said, however long it took to compute; the time speed doesn't apply + DeltaRenderTime = fixed_step; + } fRenderTime += DeltaRenderTime; if (!pause) { - DeltaTime = Global.fTimeSpeed * DeltaRenderTime; + DeltaTime = (fixed_step > 0.0 ? 1.0 : Global.fTimeSpeed) * DeltaRenderTime; fSoundTimer += DeltaTime; if (fSoundTimer > 0.1) fSoundTimer = 0.0; diff --git a/utilities/Timer.h b/utilities/Timer.h index 32e170a4..15fdd4f6 100644 --- a/utilities/Timer.h +++ b/utilities/Timer.h @@ -19,6 +19,9 @@ double GetDeltaRenderTime(); void set_delta_override(double v); +// every frame advances the clocks by this many seconds, whatever the real time; 0 = real time (test mode) +void set_fixed_step(double v); + void ResetTimers(); void UpdateTimers(bool pause); diff --git a/vehicle/Driver.cpp b/vehicle/Driver.cpp index c5f5f51e..5f1bde20 100644 --- a/vehicle/Driver.cpp +++ b/vehicle/Driver.cpp @@ -26,6 +26,7 @@ http://mozilla.org/MPL/2.0/. #include "world/MemCell.h" #include "simulation/simulation.h" #include "simulation/simulationtime.h" +#include "simulation/simulationtest.h" #include "world/Track.h" #include "world/station.h" #include "input/keyboardinput.h" @@ -767,7 +768,12 @@ void TController::TableCheck(double fDistance) { // aktualizacja rekordów z wyjątkiem ostatniego if (sSpeedTable[i].iFlags & spEnabled) // jeśli pozycja istotna { - if (sSpeedTable[i].Update()) + auto const waspassed { TestFlag( sSpeedTable[ i ].iFlags, spElapsed ) }; + auto const switchchanged { sSpeedTable[ i ].Update() }; + if( false == waspassed ) { + CheckSignalPassedAtDanger( sSpeedTable[ i ] ); + } + if (switchchanged) { if( Global.iWriteLogEnabled & 8 ) { WriteLog( "Speed table for " + OwnerName() + " detected switch change at " + sSpeedTable[ i ].trTrack->name() + " (generating fresh trace)" ); @@ -816,7 +822,12 @@ void TController::TableCheck(double fDistance) } } } - sSpeedTable[iLast].Update(); // aktualizacja ostatniego + // aktualizacja ostatniego; a signal at stop often is the last, the scan doesn't go past it + auto const lastwaspassed { TestFlag( sSpeedTable[ iLast ].iFlags, spElapsed ) }; + sSpeedTable[iLast].Update(); + if( false == lastwaspassed ) { + CheckSignalPassedAtDanger( sSpeedTable[ iLast ] ); + } // WriteLog("TableCheck: Upate last track. Dist=" + AnsiString(sSpeedTable[iLast].fDist)); if( sSpeedTable[ iLast ].fDist < fDistance ) { TableTraceRoute( fDistance, pVehicles[ end::rear ] ); // doskanowanie dalszego odcinka @@ -827,6 +838,20 @@ void TController::TableCheck(double fDistance) } }; +void TController::CheckSignalPassedAtDanger( TSpeedPos const &Point ) const +{ + if( false == TestFlag( Point.iFlags, spElapsed ) + || false == TestFlag( Point.iFlags, spEvent ) + || false == TestFlag( Point.iFlags, spSemaphor ) + || true == TestFlag( Point.iFlags, spStopOnSBL ) // SetVelocity 0 x on automatic block: may go on after a stop + || Point.fVelNext != 0.0 + || Point.evEvent == nullptr + || ( mvOccupied->CategoryFlag & 1 ) == 0 ) { // railway only + return; + } + simulation::Test.signal_passed_at_danger( OwnerName(), Point.evEvent->m_name ); +} + auto const passengerstopmaxdistance { 400.0 }; // maximum allowed distance between passenger stop point and consist head TCommandType TController::TableUpdate(double &fVelDes, double &fDist, double &fNext, double &fAcc) diff --git a/vehicle/Driver.h b/vehicle/Driver.h index f88e115a..8c12087f 100644 --- a/vehicle/Driver.h +++ b/vehicle/Driver.h @@ -519,6 +519,8 @@ private: bool TableNotFound( basic_event const *Event, double const Distance ) const; void TableTraceRoute( double fDistance, TDynamicObject *pVehicle ); void TableCheck( double fDistance ); + // reports the train going past a signal showing stop (SetVelocity 0 0); called once the head passed Point + void CheckSignalPassedAtDanger( TSpeedPos const &Point ) const; TCommandType TableUpdate( double &fVelDes, double &fDist, double &fNext, double &fAcc ); bool TableUpdateStopPoint( TCommandType &Command, TSpeedPos &Point, double const Signaldistance ); bool TableUpdateEvent( double &Velocity, TCommandType &Command, TSpeedPos &Point, double &Signaldistance, int const Pointindex ); From 6a538d202b6437e27e189e7c63a0fa413da65167 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sat, 7 Nov 2026 13:17:27 +0100 Subject: [PATCH 08/28] test: scenario tests run in the simulator's test mode and check its exit code run.py starts eu07 with --test (and --test-timeout for cases that record a fixed time) and compares the result from the exit code with the case's expect key, before the physicslog checks. The 14 cases take about 30 s with -j 4 instead of minutes. --watch keeps the old windowed run. New case keys: expect, test_config (lines of a config block), track_events (lines added to a track of the base scenery, e.g. event0 test_pass). New cases show each end of the test mode: test_mode_pass, _signal_passed_at_danger, _timeout and _standstill. The slippery rail cases slide past the red signal, which now ends a test, so they set test.fail_if_stop_overran no. README documents the test mode for scenery authors. --- tests/scenarios/README.md | 130 ++++++++++++++---- .../cases/eu07_slippery_push_red_signal.json | 3 + .../cases/eu07_slippery_red_signal.json | 3 + tests/scenarios/cases/td_red_signal_pass.scn | 8 ++ tests/scenarios/cases/test_mode_pass.json | 31 +++++ .../test_mode_signal_passed_at_danger.json | 22 +++ .../scenarios/cases/test_mode_standstill.json | 26 ++++ tests/scenarios/cases/test_mode_timeout.json | 21 +++ tests/scenarios/run.py | 122 ++++++++++++---- 9 files changed, 315 insertions(+), 51 deletions(-) create mode 100644 tests/scenarios/cases/td_red_signal_pass.scn create mode 100644 tests/scenarios/cases/test_mode_pass.json create mode 100644 tests/scenarios/cases/test_mode_signal_passed_at_danger.json create mode 100644 tests/scenarios/cases/test_mode_standstill.json create mode 100644 tests/scenarios/cases/test_mode_timeout.json diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index 4600a357..1ebd84c7 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -1,49 +1,121 @@ # Scenario tests -End-to-end checks that run the real simulator on a scenery and look at what a vehicle -did: did it start, did it keep to the speed limit, did it brake and stop. +End-to-end checks that run the real simulator on a scenery and look at how the run ended +and what a vehicle did: did it start, did it keep to the speed limit, did it brake and stop, +did it pass a signal at stop. Scenery and vehicle data are not in this repo, so these run against a local game install only, not in CI. +## The simulator's test mode + +The runner drives the simulator's test mode (eu07/maszyna#541), which works without the +runner too and is meant for scenery authors as well: + +``` +eu07 --test -s [--test-timeout ] [-seed ] [-ini ] +``` + +With `--test` the simulator: + +- needs `-s`; `-v` is ignored and every train is driven by AI (ghostview), +- draws nothing: the null renderer loads the 3D models (the physics needs some of their + data) but no textures, and the window stays hidden, +- has no sound, no network, no python, no pause (not even one set by the scenery), and + no crash dialog, +- runs the simulation as fast as it can in fixed steps of 0.03 s of simulation time per + frame (three 10 ms physics steps), whatever the real time, the time speed or the machine, +- seeds its random numbers with a fixed seed unless `-seed` or `randomseed` gives one, so two + runs of a scenery come out the same (the physicslogs of two runs are byte for byte equal), +- quits when the test ends, and says how it ended in the exit code and in a `test: ...` + line on stdout and in the log. + +A scenery reports its result to the memory cell `__test.result`, which the simulator creates +(in every run, so a test scenery loads normally too). Write value 1 to it for a pass and 0 +for a failure, with an `UpdateValues` event; other values are ignored, and the text, if +given, goes to the log as the reason: + +``` +event test_pass UpdateValues 0 __test.result reached_the_stop 1 * endevent +event test_fail UpdateValues 0 __test.result wrong_track 0 * endevent +``` + +(`PutValues` isn't the event for it: it hands a command to a vehicle, it doesn't write a cell.) +A good place for the pass event is `event0` of a track the train should stop on: it runs +while a train stands there, and the length of the track is the margin of the stop. + +The test ends with the first of these: + +| exit code | result | when | +| --- | --- | --- | +| 0 | pass | `__test.result` was set to 1 and then every vehicle stood still for 1 s, so a train still moving towards a signal at stop has to stop first. Also the standstill end with `test.pass_on_standstill yes` | +| 1 | fail | `__test.result` was set to 0; ends at once | +| 2 | signal passed at danger | a train went past a signal showing stop (`SetVelocity 0 0`; `SetVelocity 0 x` on automatic block, which may be passed after a stop, doesn't count); ends at once, unless `test.fail_if_stop_overran no` | +| 3 | timeout | `test.timeout` seconds of simulation time went by (default 3600) | +| 4 | standstill | every vehicle stood still for `test.standstill` seconds (default 300) and nothing reported a result. Trains waiting at a station or for a signal count as standing, so the limit has to be longer than such waits | +| 5 | error | the run could not start: no `-s`, the scenery failed to load or has no vehicles, or the simulator threw | + +A crash ends with the system's code instead (on Windows e.g. `0xC0000005`, -1073741819). + +The limits go in a `config` block of the scenery: + +``` +config +test.timeout 600 +test.standstill 120 +test.pass_on_standstill no +test.fail_if_stop_overran yes +endconfig +``` + +`--test-timeout` on the command line wins over `test.timeout` of the scenery. + +A signal is passed at danger when the head of a train goes past a signal which its driver +has in its speed table, showing stop. So it's the driver's view that counts: a train without +a driver isn't watched, and neither is a signal the driver never scanned (a train placed at +100 km/h 150 m before a signal at stop got past it before its AI first looked ahead; at +60 km/h it was caught). Outside the test mode the same is written to the log, nothing more. + ## Running ``` -python tests/scenarios/run.py --game-dir "E:/Program Files (x86)/Steam/steamapps/common/MaSzyna" -j 4 +python tests/scenarios/run.py --game-dir "C:/Steam/steamapps/common/MaSzyna" -j 4 ``` Python 3.10+, no extra packages. Options: - `EU07_GAME_DIR` can replace `--game-dir`. - `--exe path/to/eu07.exe` picks the build to test; the default is the newest - `cmake-build-*/bin/Release/eu07*.exe`. The build needs the `-ini` command line option. + `cmake-build-*/bin/Release/eu07*.exe`. The build needs the `--test` and `-ini` command + line options. - name one or more cases to run only those, e.g. `run.py ... eu07_td_timetable`. -- `-j N` runs N cases at once, each in its own simulator window. Each case prints its - results when it is done. All 8 cases take about a minute with `-j 4`. -- `--timespeed N` sets the simulation time multiplier, default 20: a 4-minute case runs in - about 20 s. Physics still steps finely (the physicslog keeps its 0.05 s rows) and the speed - profile at 20 and 60 matched a real-time run within a few km/h and seconds. Use - `--timespeed 1` to watch a run. +- `-j N` runs N cases at once. Each case prints its results when it is done. All 14 cases + take about 30 s with `-j 4`, loading included. +- `--watch` runs a case the old way, in a visible window in real time (or faster with + `--timespeed N`), closed after the case's `duration`; the exit code isn't checked then. For + watching what a case does. Exit code is 0 when every case passed. Logs of each run (`log.txt`, `errors.txt`, -the vehicle's physicslog) are moved to `tests/scenarios/out//`. +`stdout.txt`, the vehicle's physicslog) are moved to `tests/scenarios/out//`. ## How it works For each case the runner: 1. writes `scenery/autotest_.scn` into the game dir: the base scenery with its own - trainsets removed, the start time changed if the case says so, and the case's `.scn` - fragment added: its trainsets after `FirstInit`, everything else (event launchers standing - in for keys the scenery expects a player to press, extra events) before it, -2. starts the simulator with `-s autotest_.scn -v ghostview -ini out//autotest.ini`; - ghostview leaves every train to the AI, and the extra ini turns on `physicslog`, runs in a - 1280x720 window, does not pause when the window loses focus, and writes the logs to - `logs/log (autotest_) .txt` rather than the shared `log.txt` (`multiplelogs`), -3. waits until `physicslog/autotest_.dat` covers `duration` seconds, then closes the - simulator. It gives up early when the log reports `Bad vehicle` / `Bad trainset`, or when - the scenario has loaded and no physicslog appeared, -4. runs the checks on the physicslog. + trainsets removed, the start time changed if the case says so, the case's track events + added, and the case's `.scn` fragment added: its trainsets after `FirstInit`, everything + else (the case's `test_config`, event launchers standing in for keys the scenery expects a + player to press, extra events) before it, +2. starts the simulator with `--test -s autotest_.scn -ini out//autotest.ini`, + plus `--test-timeout ` when the case gives a `duration`; the extra ini turns on + `physicslog` and writes the logs to `logs/log (autotest_) .txt` rather than the + shared `log.txt` (`multiplelogs`), +3. waits until the simulator quits by itself. It gives up early when the log reports + `Bad vehicle` / `Bad trainset`, when the scenario has loaded and no physicslog appeared, or + after the case's `timeout` of wall-clock time, +4. compares the result from the exit code with the case's `expect`, then runs the checks on + the physicslog. The lead vehicle is named `autotest_`, so every case has its own physicslog and log files and cases can run in parallel. The `autotest_*` sceneries (and the simulator's `.sbt` @@ -60,9 +132,12 @@ scenery is newer. | `strip_trainsets` | drop the base scenery's trainsets (default `true`) | | `time` | replace the scenery start time, `HH:MM` | | `track_friction` | set the friction of every track of the base scenery (default in the sceneries is 0.15), e.g. for slippery rail | +| `track_events` | lines added to tracks of the base scenery, by track name, e.g. `{ "tor1k": "event0 test_pass" }`; only tracks in the scenery file itself, not in its includes | +| `test_config` | lines of a `config` block added to the scenery, e.g. `["test.timeout 90"]` | | `fragment` | `.scn` file in `cases/` added to the scenery: trainsets after `FirstInit` (before it they derail on unlinked track), the rest before it (launchers read after it never fire) | | `vars` | values for `{{name}}` in the fragment; a list becomes one line per item | -| `duration` | seconds of simulation to record | +| `expect` | the result the test mode should end with: `pass`, `fail`, `signal_passed_at_danger`, `timeout`, `standstill` or `error`. Default `timeout` for a case with `duration`, else `pass` | +| `duration` | seconds of simulation to record: the test timeout of the run (`--test-timeout`) | | `timeout` | wall-clock seconds before giving up, loading included | | `fail_on_errors` | fail when `errors.txt` is not empty (default `false`) | | `checks` | list of checks, below | @@ -72,7 +147,14 @@ the one whose physicslog is checked. Other vehicles of the consist take names ma e.g. `{{vehicle}}_s`. The td.scn cases share two fragments, `td_timetable.scn` (route set, train runs to Przystanek) and `td_red_signal.scn` (no route, exit signal at stop), and differ only in `vars.consist`. `td_red_signal_moving.scn` is the red signal one with the train -starting at `vars.velocity` km/h; with `track_friction` 0.03 it locks the wheels when braking. +starting at `vars.velocity` km/h; with `track_friction` 0.03 it locks the wheels when braking +and slides past the signal, so those cases set `test.fail_if_stop_overran no`. +`td_red_signal_pass.scn` adds the `test_pass` event, which `test_mode_pass` puts as `event0` +on `tor1k`, the track in front of the signal. + +The `test_mode_*` cases show the ends of the test mode: `test_mode_pass` (exit 0, the pass +event), `test_mode_signal_passed_at_danger` (exit 2), `test_mode_timeout` (exit 3, from +`test.timeout`) and `test_mode_standstill` (exit 4, from `test.standstill`). Checks (speeds in km/h, times in seconds of physicslog time, distances in metres). Most use the physicslog Velocity column, which is the wheel speed: it stays at 0 while the wheels are diff --git a/tests/scenarios/cases/eu07_slippery_push_red_signal.json b/tests/scenarios/cases/eu07_slippery_push_red_signal.json index f7ae10bb..64d6ba88 100644 --- a/tests/scenarios/cases/eu07_slippery_push_red_signal.json +++ b/tests/scenarios/cases/eu07_slippery_push_red_signal.json @@ -4,6 +4,9 @@ "time": "10:34", "track_friction": 0.03, "fragment": "td_red_signal_moving.scn", + "test_config": [ + "test.fail_if_stop_overran no" + ], "vars": { "velocity": "30", "consist": [ diff --git a/tests/scenarios/cases/eu07_slippery_red_signal.json b/tests/scenarios/cases/eu07_slippery_red_signal.json index 40b5944e..c2f60f22 100644 --- a/tests/scenarios/cases/eu07_slippery_red_signal.json +++ b/tests/scenarios/cases/eu07_slippery_red_signal.json @@ -4,6 +4,9 @@ "time": "10:34", "track_friction": 0.03, "fragment": "td_red_signal_moving.scn", + "test_config": [ + "test.fail_if_stop_overran no" + ], "vars": { "velocity": "30", "consist": [ diff --git a/tests/scenarios/cases/td_red_signal_pass.scn b/tests/scenarios/cases/td_red_signal_pass.scn new file mode 100644 index 00000000..72405032 --- /dev/null +++ b/tests/scenarios/cases/td_red_signal_pass.scn @@ -0,0 +1,8 @@ +// added by tests/scenarios/run.py to td.scn, whose own trainsets were removed. +// like td_red_signal.scn (no route, the exit signal stacja1_j about 150 m ahead stays at stop), +// with the test mode's pass event: the case puts it as event0 on tor1k, the track in front of +// the signal, so it runs once the train stands there. the test passes when every vehicle stopped. +event test_pass UpdateValues 0 __test.result stopped_before_stacja1_j 1 * endevent +trainset td/rozklad tdo_n1734 0.0 0.1 +{{consist}} +endtrainset diff --git a/tests/scenarios/cases/test_mode_pass.json b/tests/scenarios/cases/test_mode_pass.json new file mode 100644 index 00000000..686b7d16 --- /dev/null +++ b/tests/scenarios/cases/test_mode_pass.json @@ -0,0 +1,31 @@ +{ + "description": "Test mode, pass: EU07 on td.scn with no route set stops before the exit signal stacja1_j; event0 of the track there reports the pass, and the test ends as passed once the train stands.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_red_signal_pass.scn", + "track_events": { + "tor1k": "event0 test_pass" + }, + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic" + }, + "expect": "pass", + "timeout": 600, + "checks": [ + { + "type": "starts", + "min_speed": 5, + "within": 150 + }, + { + "type": "stops", + "moving_speed": 5, + "stop_speed": 0.5, + "hold": 0.5 + }, + { + "type": "distance", + "max": 150 + } + ] +} diff --git a/tests/scenarios/cases/test_mode_signal_passed_at_danger.json b/tests/scenarios/cases/test_mode_signal_passed_at_danger.json new file mode 100644 index 00000000..338e7e6e --- /dev/null +++ b/tests/scenarios/cases/test_mode_signal_passed_at_danger.json @@ -0,0 +1,22 @@ +{ + "description": "Test mode, signal passed at danger: EU07 on td.scn with no route set starts at 60 km/h about 150 m before the exit signal stacja1_j at stop. Its AI driver is still preparing the engine and doesn't brake, and the run ends when the train passes the signal.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_red_signal_moving.scn", + "vars": { + "velocity": "60", + "consist": "node -1 0 {{vehicle}} dynamic pkp\\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic" + }, + "expect": "signal_passed_at_danger", + "timeout": 600, + "checks": [ + { + "type": "max_speed", + "limit": 61 + }, + { + "type": "distance", + "min": 140 + } + ] +} diff --git a/tests/scenarios/cases/test_mode_standstill.json b/tests/scenarios/cases/test_mode_standstill.json new file mode 100644 index 00000000..0d96585e --- /dev/null +++ b/tests/scenarios/cases/test_mode_standstill.json @@ -0,0 +1,26 @@ +{ + "description": "Test mode, standstill: EU07 on td.scn with no route set stops before the exit signal stacja1_j at stop and stays; nothing reports a result, so the test ends once every vehicle stood still for test.standstill 120 seconds (the train waits about 70 s before it pulls off, so less would end it before the start).", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_red_signal.scn", + "test_config": [ + "test.standstill 120" + ], + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic" + }, + "expect": "standstill", + "timeout": 600, + "checks": [ + { + "type": "stops", + "moving_speed": 5, + "stop_speed": 0.5, + "hold": 5 + }, + { + "type": "distance", + "max": 150 + } + ] +} diff --git a/tests/scenarios/cases/test_mode_timeout.json b/tests/scenarios/cases/test_mode_timeout.json new file mode 100644 index 00000000..46899e89 --- /dev/null +++ b/tests/scenarios/cases/test_mode_timeout.json @@ -0,0 +1,21 @@ +{ + "description": "Test mode, timeout: EU07 on td.scn with no route set pulls off towards the exit signal stacja1_j at stop; nothing reports a result and test.timeout 90 from the scenery's config block runs out first.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_red_signal.scn", + "test_config": [ + "test.timeout 90" + ], + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic" + }, + "expect": "timeout", + "timeout": 600, + "checks": [ + { + "type": "starts", + "min_speed": 5, + "within": 90 + } + ] +} diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index 2cd754a0..770069d1 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -2,16 +2,19 @@ """Scenario tests: run the real simulator on a scenery and check what a vehicle did. A case (cases/.json) names a base scenery from the game install, a fragment -added to it (usually a trainset with an AI driver), and checks on the vehicle's -physics log. The runner: +added to it (usually a trainset with an AI driver), the result the simulator's test +mode should end with, and checks on the vehicle's physics log. The runner: 1. builds scenery/autotest_.scn in the game dir: the base scenery with its own trainsets removed, optionally a different start time, and the case fragment added, - 2. starts eu07 with that scenery in ghostview (every train driven by AI) and an extra - settings file (-ini) that turns on physicslog and runs in a window, - 3. waits until the vehicle's physicslog covers `duration` seconds of simulation, - then closes the simulator, - 4. evaluates the checks on the log and prints PASS/FAIL per check. + 2. starts eu07 in test mode (--test: every train driven by AI, nothing drawn, fixed + time steps as fast as it can) with that scenery and an extra settings file (-ini) + that turns on physicslog, + 3. waits until the simulator quits by itself and reads the test result from its exit + code: pass, fail, signal passed at danger, timeout (after `duration` seconds of + simulation, if the case gives one), standstill or error, + 4. compares the result with the one the case expects, evaluates the checks on the + physicslog and prints PASS/FAIL per check. Scenery and vehicle data are not part of this repo, so this runs against a local game install only (--game-dir or EU07_GAME_DIR), not in CI. See README.md. @@ -49,7 +52,18 @@ COL_COMMAND = 16 # text, read as None COL_WHEELSLIP = 20 COL_VEHICLE_VELOCITY = 25 # signed vehicle speed, km/h; only in builds which log it -# settings layered over the user's eu07.ini for a test run +# exit codes of the simulator's test mode (eu07 --test), see simulation/simulationtest.h +EXIT_RESULTS = { + 0: "pass", + 1: "fail", + 2: "signal_passed_at_danger", + 3: "timeout", + 4: "standstill", + 5: "error", +} + +# settings layered over the user's eu07.ini for a test run; the window ones only matter +# with --watch, test mode keeps its window hidden and turns the sound off by itself TEST_INI = """\ fullscreen no fullscreenwindowed no @@ -89,6 +103,17 @@ def set_start_time(text: str, hhmm: str) -> str: return re.sub(r"(?im)^(\s*time\s+)\d{1,2}:\d{2}", lambda m: m.group(1) + hhmm, text, count=1) +def add_track_events(text: str, events: dict) -> str: + """Adds lines (e.g. "event0 test_pass") to tracks named in events, before their endtrack.""" + for track, lines in events.items(): + lines = [lines] if isinstance(lines, str) else lines + m = re.search(rf"(?im)^\s*node\s+\S+\s+\S+\s+{re.escape(track)}\s+track\b.*?^(\s*endtrack\b)", text, re.S) + if not m: + sys.exit(f"track_events: no track {track} in the base scenery itself (not in its includes)") + text = text[: m.start(1)] + "".join(line + "\n" for line in lines) + text[m.start(1):] + return text + + def set_track_friction(text: str, friction: float) -> str: """Sets the friction of every track: node track ...""" return re.sub(r"(?im)^(\s*node\s+\S+\s+\S+\s+\S+\s+track\s+\S+\s+\S+\s+\S+\s+)[-\d.]+", @@ -118,6 +143,8 @@ def build_scenery(case: dict, game_dir: Path, case_name: str, vehicle: str) -> s text = set_start_time(text, case["time"]) if "track_friction" in case: text = set_track_friction(text, case["track_friction"]) + if "track_events" in case: + text = add_track_events(text, case["track_events"]) fragment = (CASES_DIR / case["fragment"]).read_bytes().decode("latin-1") # vars first: {{vehicle}} inside a var's value gets filled in the second pass fragment = fill_vars(fragment, case.get("vars", {}), keep={"vehicle"}) @@ -126,6 +153,9 @@ def build_scenery(case: dict, game_dir: Path, case_name: str, vehicle: str) -> s # events of the case go in before it (read after it they never fire), trainsets after it # (placed before it they stand on unlinked track and derail at once) before, trainsets = split_trainsets(fragment) + if case.get("test_config"): + # settings of the test mode (test.timeout and so on), read like any other config block + before = "config\n" + "".join(line + "\n" for line in case["test_config"]) + "endconfig\n" + before header = f"\n// ---- autotest: {case_name} ----\n" firstinit = re.search(r"(?im)^\s*firstinit\b", text) if firstinit: @@ -200,20 +230,30 @@ STARTUP_TIMEOUT = 120.0 @dataclass class RunResult: rows: list[list[float]] = field(default_factory=list) - exit_code: int | None = None # None: still running when we closed it (the normal end) + exit_code: int | None = None # None: still running when we closed it (the normal end with --watch) + verdict: str = "" # the simulator's own line about the test result failure: str = "" # why the run was cut short wall_seconds: float = 0.0 logs: dict[str, Path] = field(default_factory=dict) def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog: Path, - duration: float, timeout: float, say) -> RunResult: + duration: float | None, timeout: float, watch: bool, say) -> RunResult: + """Runs the simulator on the scenery. In test mode it quits by itself and the exit code + is the result; with watch it runs in a window and gets closed after `duration`.""" physicslog.unlink(missing_ok=True) - cmd = [str(exe), "-s", scenery, "-v", "ghostview", "-ini", str(ini)] + if watch: + cmd = [str(exe), "-s", scenery, "-v", "ghostview", "-ini", str(ini)] + else: + cmd = [str(exe), "--test", "-s", scenery, "-ini", str(ini)] + if duration is not None: + cmd += ["--test-timeout", f"{duration:g}"] say(" $ " + " ".join(f'"{c}"' if " " in c else c for c in cmd), progress=True) since = time.time() - 1 started = time.monotonic() - proc = subprocess.Popen(cmd, cwd=game_dir) + # the test result line goes to stdout; a file, not a pipe, so a chatty run can't block on it + stdout = (ini.parent / "stdout.txt").open("w+b") + proc = subprocess.Popen(cmd, cwd=game_dir, stdout=stdout, stderr=subprocess.STDOUT) result = RunResult() last_report = 0.0 try: @@ -229,7 +269,7 @@ def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog if proc.poll() is not None: result.exit_code = proc.returncode break - if sim_t >= duration: + if watch and sim_t >= duration: break if wall > timeout: result.failure = f"timed out after {timeout:.0f}s before {duration}s of simulation" @@ -252,6 +292,10 @@ def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog except subprocess.TimeoutExpired: proc.kill() result.wall_seconds = time.monotonic() - started + stdout.seek(0) + result.verdict = next((line.strip() for line in stdout.read().decode(errors="replace").splitlines() + if line.startswith("test:")), "") + stdout.close() result.rows = read_physicslog(physicslog) result.logs = find_logs(game_dir, scenery, since) return result @@ -422,7 +466,14 @@ def count_lines(path: Path) -> int: return 0 -def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, say) -> bool: +def describe_exit(code: int | None) -> str: + if code is None: + return "none, still running" + name = EXIT_RESULTS.get(code) + return f"{code} ({name})" if name else f"{code} (0x{code & 0xFFFFFFFF:08X}, a crash?)" + + +def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, watch: bool, say) -> bool: case = json.loads((CASES_DIR / f"{name}.json").read_text(encoding="utf-8")) say(f"\n=== {name}: {case.get('description', '')}") out = OUT_DIR / name @@ -436,8 +487,16 @@ def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, say) -> boo ini.write_text(TEST_INI.format(timespeed=timespeed), encoding="ascii") physicslog = game_dir / "physicslog" / f"{vehicle}.dat" + duration = float(case["duration"]) if "duration" in case else None + # a case that only records `duration` seconds ends on the test timeout; one that says + # how the test ends (a pass event, a signal passed at danger...) expects that result + expect = case.get("expect", "timeout" if duration is not None else "pass") + if expect not in EXIT_RESULTS.values(): + sys.exit(f"{name}: expect must be one of {', '.join(EXIT_RESULTS.values())}") + if watch and duration is None: + sys.exit(f"{name}: --watch needs a case with duration") result = run_simulator(exe, game_dir, scenery, ini, physicslog, - float(case["duration"]), float(case.get("timeout", 600)), say) + duration, float(case.get("timeout", 600)), watch, say) # the logs are ours alone (named after the case scenery), move them out of the game dir for kind, path in result.logs.items(): @@ -448,18 +507,26 @@ def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, say) -> boo ok = True sim_t = result.rows[-1][COL_TIME] if result.rows else 0.0 say(f" ran {result.wall_seconds:.0f}s wall, {sim_t:.1f}s of physicslog, results in {out.relative_to(REPO)}") - if result.exit_code is not None: - say(f" FAIL simulator exited on its own with code {result.exit_code}") - ok = False + if watch: + if result.exit_code is not None: + say(f" FAIL simulator exited on its own with code {result.exit_code}") + ok = False + else: + if result.verdict: + say(f" {result.verdict}") + got = EXIT_RESULTS.get(result.exit_code) if result.exit_code is not None else None + passed = got == expect + say(f" {'PASS' if passed else 'FAIL'} exit code {describe_exit(result.exit_code)}, expected {expect}") + ok &= passed if result.failure: say(f" FAIL {result.failure}") ok = False - if not result.rows: + if not result.rows and case.get("checks"): if not result.failure: say(f" FAIL no physicslog for '{vehicle}'") return False - for check in case["checks"]: + for check in case.get("checks", []): params = {k: v for k, v in check.items() if k != "type"} passed, detail = CHECKS[check["type"]](result.rows, **params) say(f" {'PASS' if passed else 'FAIL'} {check['type']}: {detail}") @@ -479,12 +546,13 @@ def main() -> int: ap.add_argument("--game-dir", default=os.environ.get("EU07_GAME_DIR"), help="game install with scenery/ and dynamic/ (or EU07_GAME_DIR)") ap.add_argument("--exe", help="simulator executable (default: newest cmake-build-*/bin/Release/eu07*.exe)") - # 20 gave the same speed profile as 1 within a few km/h and seconds on eu07_td_timetable, - # in 20 s instead of 4 min; 1 is for watching a run - ap.add_argument("--timespeed", type=float, default=20.0, - help="simulation time multiplier, the simulator offers 1, 5, 20 and 60 (default 20)") + ap.add_argument("--watch", action="store_true", + help="run in a visible window in real time (see --timespeed) instead of the test mode, " + "closed after the case's duration; the exit code isn't checked") + ap.add_argument("--timespeed", type=float, default=1.0, + help="with --watch: simulation time multiplier, the simulator offers 1, 5, 20 and 60 (default 1)") ap.add_argument("-j", "--jobs", type=int, default=1, - help="cases run at once, each in its own simulator window (default 1)") + help="cases run at once, each in a simulator of its own (default 1)") args = ap.parse_args() if not args.game_dir: @@ -504,7 +572,7 @@ def main() -> int: def say(line, progress=False): print(line, flush=True) for n in names: - results[n] = run_case(n, exe, game_dir, args.timespeed, say) + results[n] = run_case(n, exe, game_dir, args.timespeed, args.watch, say) else: # each case prints its block when it is done, so parallel output does not interleave def job(n): @@ -512,7 +580,7 @@ def main() -> int: def say(line, progress=False): if not progress: lines.append(line) - ok = run_case(n, exe, game_dir, args.timespeed, say) + ok = run_case(n, exe, game_dir, args.timespeed, args.watch, say) return n, ok, lines with ThreadPoolExecutor(max_workers=args.jobs) as pool: for n in names: From 441caeed641b1a2257b382916f5dd6431b3369bc Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sat, 7 Nov 2026 17:12:11 +0100 Subject: [PATCH 09/28] refactor(simulation): tidy the test mode code The test manager is reached through simulation::test() instead of a global, the result cell is owned by a unique_ptr until the memory table takes it, the cell value is compared with a margin, strings are built with std::format and the --test settings moved to apply_test_mode_settings(). The driver's signal check is folded into UpdateSpeedPosition(), which updates a speed table entry and reports a signal at stop the head of the train just passed. The test.* keys are listed with the other settings at the start of the log. --- EU07.cpp | 5 +-- application/application.cpp | 55 ++++++++++++++++++------------ application/drivermode.cpp | 4 +-- application/scenarioloadermode.cpp | 4 +-- rendering/nullrenderer.cpp | 5 +-- simulation/simulation.cpp | 2 +- simulation/simulationtest.cpp | 47 ++++++++++++++++++------- simulation/simulationtest.h | 8 +++-- utilities/Globals.cpp | 4 +++ utilities/Timer.cpp | 21 +++++++++--- vehicle/Driver.cpp | 46 +++++++++++++------------ vehicle/Driver.h | 5 +-- 12 files changed, 131 insertions(+), 75 deletions(-) diff --git a/EU07.cpp b/EU07.cpp index aac7cae1..ec3e972e 100644 --- a/EU07.cpp +++ b/EU07.cpp @@ -23,6 +23,7 @@ Stele, firleju, szociu, hunter, ZiomalCl, OLI_EU and others #include "utilities/Globals.h" #include "utilities/Logs.h" #include +#include #ifdef WITHDUMPGEN #ifdef _WIN32 #include @@ -150,7 +151,7 @@ int main(int argc, char *argv[]) #ifdef _WIN32 catch (std::runtime_error const &Error) { - ErrorLog("Critical error: " + std::string(Error.what())); + ErrorLog(std::format("Critical error: {}", Error.what())); // in test mode nobody is there to close the box, the exit code tells of the error if (false == Global.test.enabled) { @@ -167,5 +168,5 @@ int main(int argc, char *argv[]) #endif // skip destructors, there are ordering errors which causes segfaults // a test run (--test) says its result in the exit code - std::_Exit(Global.test.enabled ? simulation::Test.exit_code() : 0); + std::_Exit(Global.test.enabled ? simulation::test().exit_code() : 0); } diff --git a/application/application.cpp b/application/application.cpp index 37afff67..439c01d9 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -1137,6 +1137,38 @@ void eu07_application::init_files() } namespace fs = std::filesystem; +namespace +{ + +// the settings of the test mode (--test), over whatever the settings files say +void apply_test_mode_settings() +{ + // nobody watches nor listens: no player vehicle, nothing drawn, no sound, no window in the way + Global.editor_session = false; + Global.local_start_vehicle = "ghostview"; + Global.GfxRenderer = "null"; + Global.NvRenderer = false; + Global.bSoundEnabled = false; + Global.python_enabled = false; + Global.bInactivePause = false; + Global.bFullScreen = false; + Global.fullscreen_windowed = false; + Global.VSync = false; + Global.iMultisampling = 0; + Global.gfx_usegles = false; + Global.minframetime = std::chrono::duration(0.0f); + Global.extra_viewports.clear(); + Global.headtrack_conf.joy.clear(); + // a test runs on its own + Global.network_servers.clear(); + Global.network_client.reset(); + // and the same way every time, unless given another seed (-seed, or randomseed in the settings) + if (!Global.random_seed) + Global.random_seed = seed_of("test"); +} + +} // namespace + int eu07_application::init_settings(int Argc, char *Argv[]) { Global.asVersion = VERSION_INFO; @@ -1241,28 +1273,7 @@ int eu07_application::init_settings(int Argc, char *Argv[]) ErrorLog("test: --test needs a scenery given with -s"); return -1; } - // nobody watches nor listens: no player vehicle, nothing drawn, no sound, no window in the way - Global.editor_session = false; - Global.local_start_vehicle = "ghostview"; - Global.GfxRenderer = "null"; - Global.NvRenderer = false; - Global.bSoundEnabled = false; - Global.python_enabled = false; - Global.bInactivePause = false; - Global.bFullScreen = false; - Global.fullscreen_windowed = false; - Global.VSync = false; - Global.iMultisampling = 0; - Global.gfx_usegles = false; - Global.minframetime = std::chrono::duration(0.0f); - Global.extra_viewports.clear(); - Global.headtrack_conf.joy.clear(); - // a test runs on its own - Global.network_servers.clear(); - Global.network_client.reset(); - // and the same way every time, unless given another seed (-seed, or randomseed in the settings) - if (!Global.random_seed) - Global.random_seed = seed_of("test"); + apply_test_mode_settings(); } return 0; diff --git a/application/drivermode.cpp b/application/drivermode.cpp index 1b3a44d6..deb906a4 100644 --- a/application/drivermode.cpp +++ b/application/drivermode.cpp @@ -309,7 +309,7 @@ bool driver_mode::update() if (Global.test.enabled && simulation::is_ready) { - simulation::Test.update(deltatime); + simulation::test().update(deltatime); } } @@ -488,7 +488,7 @@ void driver_mode::enter() { // the test runs as fast as it can compute, in steps of fixed length, so that runs on different machines agree Timer::set_fixed_step(simulation::test_manager::frame_step); - simulation::Test.start(); + simulation::test().start(); } set_picking(!Global.captureonstart); diff --git a/application/scenarioloadermode.cpp b/application/scenarioloadermode.cpp index b84c1912..20fa46d3 100644 --- a/application/scenarioloadermode.cpp +++ b/application/scenarioloadermode.cpp @@ -60,7 +60,7 @@ bool scenarioloader_mode::update() { ErrorLog(std::format("Bad init: scenario \"{}\" couldn't be opened ({})", Global.SceneryFile, e.what())); state.reset(); if (Global.test.enabled) { - simulation::Test.finish(simulation::test_result::error, "scenario couldn't be opened"); + simulation::test().finish(simulation::test_result::error, "scenario couldn't be opened"); return true; } Global.SceneryFile.clear(); @@ -80,7 +80,7 @@ bool scenarioloader_mode::update() { state.reset(); if( Global.test.enabled ) { // there's nothing to test; the exit code says the run went wrong - simulation::Test.finish( simulation::test_result::error, "scenario loading failed" ); + simulation::test().finish( simulation::test_result::error, "scenario loading failed" ); return true; } Global.SceneryFile.clear(); diff --git a/rendering/nullrenderer.cpp b/rendering/nullrenderer.cpp index f193ad3e..bc466e97 100644 --- a/rendering/nullrenderer.cpp +++ b/rendering/nullrenderer.cpp @@ -7,8 +7,9 @@ void null_imgui_renderer::BeginFrame() { auto &io { ImGui::GetIO() }; if( false == io.Fonts->IsBuilt() ) { - unsigned char *pixels; - int width, height; + unsigned char *pixels { nullptr }; + int width { 0 }; + int height { 0 }; io.Fonts->GetTexDataAsAlpha8( &pixels, &width, &height ); } } diff --git a/simulation/simulation.cpp b/simulation/simulation.cpp index 59635c38..d10d555a 100644 --- a/simulation/simulation.cpp +++ b/simulation/simulation.cpp @@ -111,7 +111,7 @@ state_manager::init_scripting_interface() { simulation::Region->insert( memorycell ); } // the cell a test scenery reports its result to - simulation::Test.init_result_cell(); + simulation::test().init_result_cell(); } // legacy method, calculates changes in simulation state over specified time diff --git a/simulation/simulationtest.cpp b/simulation/simulationtest.cpp index 4dbe4c24..439f295f 100644 --- a/simulation/simulationtest.cpp +++ b/simulation/simulationtest.cpp @@ -19,9 +19,29 @@ http://mozilla.org/MPL/2.0/. #include "world/Event.h" #include "world/MemCell.h" +#include +#include +#include +#include +#include + +namespace { + +// the cell holds what the scenery wrote, so the values are whole numbers; the margin only keeps a stray fraction out +bool holds(double const Value, double const Expected) +{ + return std::abs(Value - Expected) < 1e-6; +} + +} // namespace + namespace simulation { -test_manager Test; +test_manager &test() +{ + static test_manager instance; + return instance; +} std::string_view to_string(test_result const Result) { @@ -39,11 +59,12 @@ std::string_view to_string(test_result const Result) void test_manager::init_result_cell() { - auto *cell = new TMemCell({0, -1, std::string(result_cell), "memcell"}); + auto cell { std::make_unique(scene::node_data{0, -1, std::string(result_cell), "memcell"}) }; // neither of the reported values, so that a cell nobody wrote to says nothing cell->UpdateValues("", -1.0, 0.0, basic_event::flags::value1 | basic_event::flags::value2); - simulation::Memory.insert(cell); - simulation::Region->insert(cell); + simulation::Region->insert(cell.get()); + // the table owns its cells + simulation::Memory.insert(cell.release()); } void test_manager::start() @@ -76,21 +97,21 @@ void test_manager::update(double const Deltatime) if (auto const *cell = simulation::Memory.find(std::string(result_cell))) { auto const value { cell->Value1() }; - auto const reason { cell->Text().empty() ? std::string() : " (" + cell->Text() + ")" }; - if (value == 0.0) + auto const reason { cell->Text().empty() ? std::string() : std::format(" ({})", cell->Text()) }; + if (holds(value, 0.0)) { - finish(test_result::fail, "__test.result set to 0" + reason); + finish(test_result::fail, std::format("__test.result set to 0{}", reason)); return; } - if (value == 1.0 && false == m_passrequested) + if (holds(value, 1.0) && false == m_passrequested) { m_passrequested = true; WriteLog(std::format("test: pass requested at {:.1f} s{}, waiting for every vehicle to stop", m_time, reason)); } } - auto const atrest = std::none_of(simulation::Vehicles.sequence().begin(), simulation::Vehicles.sequence().end(), - [](TDynamicObject const *Vehicle) { return std::abs(Vehicle->GetVelocity()) > 0.01; }); + auto const atrest { std::ranges::none_of(simulation::Vehicles.sequence(), + [](TDynamicObject const *Vehicle) { return std::abs(Vehicle->GetVelocity()) > 0.01; }) }; m_resttime = atrest ? m_resttime + Deltatime : 0.0; if (m_passrequested && m_resttime >= pass_settle_time) @@ -111,9 +132,9 @@ void test_manager::update(double const Deltatime) } } -void test_manager::signal_passed_at_danger(std::string const &Vehicle, std::string const &Signal) +void test_manager::signal_passed_at_danger(std::string_view const Vehicle, std::string_view const Signal) { - auto const reason { Vehicle + " passed signal " + Signal + " at danger" }; + auto const reason { std::format("{} passed signal {} at danger", Vehicle, Signal) }; if (false == Global.test.enabled) { WriteLog(reason, logtype::generic); @@ -127,7 +148,7 @@ void test_manager::signal_passed_at_danger(std::string const &Vehicle, std::stri finish(test_result::signal_passed_at_danger, reason); } -void test_manager::finish(test_result const Result, std::string const &Reason) +void test_manager::finish(test_result const Result, std::string_view const Reason) { if (m_finished || false == Global.test.enabled) return; diff --git a/simulation/simulationtest.h b/simulation/simulationtest.h index c9bc5437..f47f408b 100644 --- a/simulation/simulationtest.h +++ b/simulation/simulationtest.h @@ -10,6 +10,7 @@ http://mozilla.org/MPL/2.0/. #pragma once #include +#include // integration test mode (eu07 --test -s , eu07/maszyna#541): the scenery runs unattended in a tight // fixed-step loop and the simulator quits with an exit code saying how the test went @@ -44,9 +45,9 @@ public: // checks the end conditions after a simulation frame of Deltatime seconds void update(double Deltatime); // reported by a driver whose train went past a signal showing stop - void signal_passed_at_danger(std::string const &Vehicle, std::string const &Signal); + void signal_passed_at_danger(std::string_view Vehicle, std::string_view Signal); // ends the test with given result; the first call wins - void finish(test_result Result, std::string const &Reason); + void finish(test_result Result, std::string_view Reason); bool finished() const { return m_finished; } // exit code of the process: the result, or error while the test hasn't finished @@ -61,6 +62,7 @@ private: double m_resttime { 0.0 }; // how long every vehicle has been standing [s] }; -extern test_manager Test; +// the test of the run +test_manager &test(); } // namespace simulation diff --git a/utilities/Globals.cpp b/utilities/Globals.cpp index e3622317..5cdb3ef2 100644 --- a/utilities/Globals.cpp +++ b/utilities/Globals.cpp @@ -1779,6 +1779,10 @@ global_settings::export_as_text( std::ostream &Output ) const { export_as_text( Output, "gfx.shadowmap.enabled", gfx_shadowmap_enabled ); // TODO: export fpslimit export_as_text( Output, "randomseed", Global.random_seed ); + export_as_text( Output, "test.timeout", test.timeout ); + export_as_text( Output, "test.standstill", test.standstill ); + export_as_text( Output, "test.pass_on_standstill", test.pass_on_standstill ); + export_as_text( Output, "test.fail_if_stop_overran", test.fail_if_stop_overran ); export_as_text( Output, "gfx.envmap.enabled", gfx_envmap_enabled ); export_as_text( Output, "gfx.postfx.motionblur.enabled", gfx_postfx_motionblur_enabled ); export_as_text( Output, "gfx.postfx.motionblur.shutter", gfx_postfx_motionblur_shutter ); diff --git a/utilities/Timer.cpp b/utilities/Timer.cpp index 388330bb..eacd2ba0 100644 --- a/utilities/Timer.cpp +++ b/utilities/Timer.cpp @@ -26,7 +26,17 @@ double fRenderTime{ 0.0 }; double fSoundTimer{ 0.0 }; double fSinceStart{ 0.0 }; double override_delta = -1.0f; -double fixed_step = 0.0; + +namespace { + +// when set (test mode), every frame lasts this long, see set_fixed_step() +double &fixed_step() +{ + static double step{0.0}; + return step; +} + +} // namespace double GetTime() { @@ -56,7 +66,7 @@ void set_delta_override(double t) void set_fixed_step(double t) { - fixed_step = t; + fixed_step() = t; } void ResetTimers() @@ -82,15 +92,16 @@ void UpdateTimers(bool pause) fr = 1000000000; #endif DeltaRenderTime = double(count - oldCount) / double(fr); - if (fixed_step > 0.0) + auto const fixedstep{fixed_step()}; + if (fixedstep > 0.0) { // the frame lasts exactly as long as said, however long it took to compute; the time speed doesn't apply - DeltaRenderTime = fixed_step; + DeltaRenderTime = fixedstep; } fRenderTime += DeltaRenderTime; if (!pause) { - DeltaTime = (fixed_step > 0.0 ? 1.0 : Global.fTimeSpeed) * DeltaRenderTime; + DeltaTime = (fixedstep > 0.0 ? 1.0 : Global.fTimeSpeed) * DeltaRenderTime; fSoundTimer += DeltaTime; if (fSoundTimer > 0.1) fSoundTimer = 0.0; diff --git a/vehicle/Driver.cpp b/vehicle/Driver.cpp index 5f1bde20..c0614923 100644 --- a/vehicle/Driver.cpp +++ b/vehicle/Driver.cpp @@ -768,12 +768,7 @@ void TController::TableCheck(double fDistance) { // aktualizacja rekordów z wyjątkiem ostatniego if (sSpeedTable[i].iFlags & spEnabled) // jeśli pozycja istotna { - auto const waspassed { TestFlag( sSpeedTable[ i ].iFlags, spElapsed ) }; - auto const switchchanged { sSpeedTable[ i ].Update() }; - if( false == waspassed ) { - CheckSignalPassedAtDanger( sSpeedTable[ i ] ); - } - if (switchchanged) + if (UpdateSpeedPosition(sSpeedTable[i])) { if( Global.iWriteLogEnabled & 8 ) { WriteLog( "Speed table for " + OwnerName() + " detected switch change at " + sSpeedTable[ i ].trTrack->name() + " (generating fresh trace)" ); @@ -823,11 +818,7 @@ void TController::TableCheck(double fDistance) } } // aktualizacja ostatniego; a signal at stop often is the last, the scan doesn't go past it - auto const lastwaspassed { TestFlag( sSpeedTable[ iLast ].iFlags, spElapsed ) }; - sSpeedTable[iLast].Update(); - if( false == lastwaspassed ) { - CheckSignalPassedAtDanger( sSpeedTable[ iLast ] ); - } + UpdateSpeedPosition(sSpeedTable[iLast]); // WriteLog("TableCheck: Upate last track. Dist=" + AnsiString(sSpeedTable[iLast].fDist)); if( sSpeedTable[ iLast ].fDist < fDistance ) { TableTraceRoute( fDistance, pVehicles[ end::rear ] ); // doskanowanie dalszego odcinka @@ -838,18 +829,31 @@ void TController::TableCheck(double fDistance) } }; -void TController::CheckSignalPassedAtDanger( TSpeedPos const &Point ) const +namespace { + +// the position is a signal showing stop (SetVelocity 0 0). SetVelocity 0 x on automatic block isn't: it may be passed after a stop +bool is_signal_at_stop( TSpeedPos const &Point ) { - if( false == TestFlag( Point.iFlags, spElapsed ) - || false == TestFlag( Point.iFlags, spEvent ) - || false == TestFlag( Point.iFlags, spSemaphor ) - || true == TestFlag( Point.iFlags, spStopOnSBL ) // SetVelocity 0 x on automatic block: may go on after a stop - || Point.fVelNext != 0.0 - || Point.evEvent == nullptr - || ( mvOccupied->CategoryFlag & 1 ) == 0 ) { // railway only - return; + return TestFlag( Point.iFlags, spEvent ) + && TestFlag( Point.iFlags, spSemaphor ) + && false == TestFlag( Point.iFlags, spStopOnSBL ) + && std::abs( Point.fVelNext ) <= 0.0 + && Point.evEvent != nullptr; +} + +} // namespace + +bool TController::UpdateSpeedPosition( TSpeedPos &Point ) const +{ + auto const passedbefore { TestFlag( Point.iFlags, spElapsed ) }; + auto const switchchanged { Point.Update() }; + if( false == passedbefore + && TestFlag( Point.iFlags, spElapsed ) + && is_signal_at_stop( Point ) + && ( mvOccupied->CategoryFlag & 1 ) != 0 ) { // railway only + simulation::test().signal_passed_at_danger( OwnerName(), Point.evEvent->m_name ); } - simulation::Test.signal_passed_at_danger( OwnerName(), Point.evEvent->m_name ); + return switchchanged; } auto const passengerstopmaxdistance { 400.0 }; // maximum allowed distance between passenger stop point and consist head diff --git a/vehicle/Driver.h b/vehicle/Driver.h index 8c12087f..8f2015f2 100644 --- a/vehicle/Driver.h +++ b/vehicle/Driver.h @@ -519,8 +519,9 @@ private: bool TableNotFound( basic_event const *Event, double const Distance ) const; void TableTraceRoute( double fDistance, TDynamicObject *pVehicle ); void TableCheck( double fDistance ); - // reports the train going past a signal showing stop (SetVelocity 0 0); called once the head passed Point - void CheckSignalPassedAtDanger( TSpeedPos const &Point ) const; + // updates a position of the speed table, returns true when a switch on it changed (TSpeedPos::Update). + // reports the head of the train going past it, if it's a signal showing stop + bool UpdateSpeedPosition( TSpeedPos &Point ) const; TCommandType TableUpdate( double &fVelDes, double &fDist, double &fNext, double &fAcc ); bool TableUpdateStopPoint( TCommandType &Command, TSpeedPos &Point, double const Signaldistance ); bool TableUpdateEvent( double &Velocity, TCommandType &Command, TSpeedPos &Point, double &Signaldistance, int const Pointindex ); From dd019d88ba4f45c711e37abe00782c27207ed5be Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sat, 7 Nov 2026 17:38:37 +0100 Subject: [PATCH 10/28] feat(simulation): the test mode takes no input from outside (eu07/maszyna#541) With --test the gamepad, mouse picking, UART, ZeroMQ and motion telemetry are off, the hidden window gets no key, mouse, scroll or char callbacks, and the window procedure that takes WM_COPYDATA commands from other programs isn't installed, so nothing but the scenery drives a test run. --- application/application.cpp | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/application/application.cpp b/application/application.cpp index 439c01d9..eaf60c7c 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -1159,9 +1159,16 @@ void apply_test_mode_settings() Global.minframetime = std::chrono::duration(0.0f); Global.extra_viewports.clear(); Global.headtrack_conf.joy.clear(); - // a test runs on its own + // a test runs on its own: no network, and no input from outside (eu07/maszyna#541) Global.network_servers.clear(); Global.network_client.reset(); + Global.InputGamepad = false; + Global.InputMouse = false; + Global.uart_conf.enable = false; + Global.motiontelemetry_conf.enable = false; +#ifdef WITH_ZMQ + Global.zmq_address.clear(); +#endif // and the same way every time, unless given another seed (-seed, or randomseed in the settings) if (!Global.random_seed) Global.random_seed = seed_of("test"); @@ -1431,7 +1438,9 @@ int eu07_application::init_glfw() #ifdef _WIN32 // setup wrapper for base glfw window proc, to handle copydata messages Hwnd = glfwGetWin32Window(mainwindow); - BaseWindowProc = (WNDPROC)::SetWindowLongPtr(Hwnd, GWLP_WNDPROC, (LONG_PTR)WndProc); + // commands sent by other programs are input from outside, which a test doesn't take + if (false == Global.test.enabled) + BaseWindowProc = (WNDPROC)::SetWindowLongPtr(Hwnd, GWLP_WNDPROC, (LONG_PTR)WndProc); // switch off the topmost flag ::SetWindowPos(Hwnd, HWND_NOTOPMOST, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE); #endif @@ -1445,13 +1454,18 @@ void eu07_application::init_callbacks() auto *window{m_windows.front()}; glfwSetWindowSizeCallback(window, window_resize_callback); glfwSetFramebufferSizeCallback(window, framebuffer_resize_callback); + glfwSetWindowFocusCallback(window, focus_callback); + glfwSetCursorEnterCallback(window, cursor_enter_callback); + if (Global.test.enabled) + { + // a test takes no input: keys and clicks sent to its hidden window would change the run + return; + } glfwSetCursorPosCallback(window, cursor_pos_callback); glfwSetMouseButtonCallback(window, mouse_button_callback); glfwSetKeyCallback(window, key_callback); glfwSetScrollCallback(window, scroll_callback); glfwSetCharCallback(window, char_callback); - glfwSetWindowFocusCallback(window, focus_callback); - glfwSetCursorEnterCallback(window, cursor_enter_callback); } int eu07_application::init_ogl() From 6ccf4b14bf8a02fb7835dd20837db54ed5a5edcb Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sat, 7 Nov 2026 17:38:38 +0100 Subject: [PATCH 11/28] feat(simulation): test.no_movement_before_pass, a pass that doesn't wait for a stop The config key from eu07/maszyna#541. By default (yes) a pass reported to __test.result waits until every vehicle has stood still for 1 s; with no it ends the test at once. The key is listed with the other test.* settings in the log and in the test start line. Scenario tests: new case test_mode_pass_while_moving reports the pass from event2 of tor1k as the train runs onto it and checks the run ended with the train still moving (new check ends_moving). The README lists the test.* keys with their defaults and the input the test mode ignores. --- simulation/simulationtest.cpp | 10 +++++-- tests/scenarios/README.md | 18 +++++++++++- .../cases/test_mode_pass_while_moving.json | 28 +++++++++++++++++++ tests/scenarios/run.py | 10 +++++++ utilities/Globals.cpp | 7 +++++ utilities/Globals.h | 1 + 6 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 tests/scenarios/cases/test_mode_pass_while_moving.json diff --git a/simulation/simulationtest.cpp b/simulation/simulationtest.cpp index 439f295f..8d57505f 100644 --- a/simulation/simulationtest.cpp +++ b/simulation/simulationtest.cpp @@ -82,9 +82,10 @@ void test_manager::start() } // a pause set by the scenery or the settings would never be lifted, nobody is there to press the key Global.iPause = 0; - WriteLog(std::format("test: started, timeout {:.0f} s, standstill {:.0f} s{}{}", Global.test.timeout, Global.test.standstill, + WriteLog(std::format("test: started, timeout {:.0f} s, standstill {:.0f} s{}{}{}", Global.test.timeout, Global.test.standstill, Global.test.pass_on_standstill ? ", standstill passes" : "", - Global.test.fail_if_stop_overran ? "" : ", signals passed at danger allowed")); + Global.test.fail_if_stop_overran ? "" : ", signals passed at danger allowed", + Global.test.no_movement_before_pass ? "" : ", pass without waiting for a stop")); } void test_manager::update(double const Deltatime) @@ -103,6 +104,11 @@ void test_manager::update(double const Deltatime) finish(test_result::fail, std::format("__test.result set to 0{}", reason)); return; } + if (holds(value, 1.0) && false == Global.test.no_movement_before_pass) + { + finish(test_result::pass, std::format("__test.result set to 1{}", reason)); + return; + } if (holds(value, 1.0) && false == m_passrequested) { m_passrequested = true; diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index 1ebd84c7..b57fc4a0 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -23,6 +23,9 @@ With `--test` the simulator: data) but no textures, and the window stays hidden, - has no sound, no network, no python, no pause (not even one set by the scenery), and no crash dialog, +- takes no input from outside: keys and mouse sent to its window, gamepad, UART (a cab + desk), ZeroMQ and commands sent by other programs (`WM_COPYDATA`) are all ignored, and + motion telemetry is off, - runs the simulation as fast as it can in fixed steps of 0.03 s of simulation time per frame (three 10 ms physics steps), whatever the real time, the time speed or the machine, - seeds its random numbers with a fixed seed unless `-seed` or `randomseed` gives one, so two @@ -48,7 +51,7 @@ The test ends with the first of these: | exit code | result | when | | --- | --- | --- | -| 0 | pass | `__test.result` was set to 1 and then every vehicle stood still for 1 s, so a train still moving towards a signal at stop has to stop first. Also the standstill end with `test.pass_on_standstill yes` | +| 0 | pass | `__test.result` was set to 1 and then every vehicle stood still for 1 s, so a train still moving towards a signal at stop has to stop first (with `test.no_movement_before_pass no` the pass ends the test at once). Also the standstill end with `test.pass_on_standstill yes` | | 1 | fail | `__test.result` was set to 0; ends at once | | 2 | signal passed at danger | a train went past a signal showing stop (`SetVelocity 0 0`; `SetVelocity 0 x` on automatic block, which may be passed after a stop, doesn't count); ends at once, unless `test.fail_if_stop_overran no` | | 3 | timeout | `test.timeout` seconds of simulation time went by (default 3600) | @@ -65,9 +68,18 @@ test.timeout 600 test.standstill 120 test.pass_on_standstill no test.fail_if_stop_overran yes +test.no_movement_before_pass yes endconfig ``` +| key | default | meaning | +| --- | --- | --- | +| `test.timeout` | 3600 | seconds of simulation time before the test ends as `timeout`; 0 = never | +| `test.standstill` | 300 | seconds with every vehicle still and no result before the test ends as `standstill`; 0 = never | +| `test.pass_on_standstill` | no | the standstill end passes the test instead | +| `test.fail_if_stop_overran` | yes | a signal passed at danger ends the test; with `no` it's only logged | +| `test.no_movement_before_pass` | yes | a pass waits until every vehicle stands still; with `no` it ends the test at once | + `--test-timeout` on the command line wins over `test.timeout` of the scenery. A signal is passed at danger when the head of a train goes past a signal which its driver @@ -155,6 +167,8 @@ on `tor1k`, the track in front of the signal. The `test_mode_*` cases show the ends of the test mode: `test_mode_pass` (exit 0, the pass event), `test_mode_signal_passed_at_danger` (exit 2), `test_mode_timeout` (exit 3, from `test.timeout`) and `test_mode_standstill` (exit 4, from `test.standstill`). +`test_mode_pass_while_moving` reports the pass from `event2` of `tor1k` (the direction the train runs), as the train runs onto +it, with `test.no_movement_before_pass no`, so the test ends with the train still moving. Checks (speeds in km/h, times in seconds of physicslog time, distances in metres). Most use the physicslog Velocity column, which is the wheel speed: it stays at 0 while the wheels are @@ -178,6 +192,8 @@ the row instead, logged by builds from this branch on. moving off, the wheels stop slipping within `within` seconds. - `holds_still` `moving_speed`, `stop_speed`, `max_distance` (*vehicle speed*): once stopped after moving off, moves no more than `max_distance` metres, e.g. no creeping on locked wheels. +- `ends_moving` `min_speed`: the last row of the physicslog is at `min_speed` or faster, e.g. + the test ended without waiting for the train to stop. New check types go in `CHECKS` in `run.py`; each gets the physicslog rows and the check's parameters and returns `(passed, detail)`. diff --git a/tests/scenarios/cases/test_mode_pass_while_moving.json b/tests/scenarios/cases/test_mode_pass_while_moving.json new file mode 100644 index 00000000..cdf798f1 --- /dev/null +++ b/tests/scenarios/cases/test_mode_pass_while_moving.json @@ -0,0 +1,28 @@ +{ + "description": "Test mode, pass without waiting for a stop: like test_mode_pass, but event2 of the track in front of the exit signal reports the pass as the train runs onto it, and with test.no_movement_before_pass no the test ends at once, the train still moving.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_red_signal_pass.scn", + "track_events": { + "tor1k": "event2 test_pass" + }, + "test_config": [ + "test.no_movement_before_pass no" + ], + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic" + }, + "expect": "pass", + "timeout": 600, + "checks": [ + { + "type": "starts", + "min_speed": 5, + "within": 150 + }, + { + "type": "ends_moving", + "min_speed": 3 + } + ] +} diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index 770069d1..4c338308 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -416,6 +416,15 @@ def check_holds_still(rows, moving_speed, stop_speed, max_distance): return d <= max_distance, detail +def check_ends_moving(rows, min_speed): + """The run ended with the vehicle still moving, e.g. a pass that didn't wait for the trains to stop.""" + if not rows: + return False, "the physicslog is empty" + last = rows[-1] + ok = last[COL_VELOCITY] >= min_speed + return ok, f"last row t={last[COL_TIME]:.1f}s, v={last[COL_VELOCITY]:.1f} km/h (wanted >= {min_speed})" + + def check_slip_clears(rows, moving_speed, stop_speed, within): """Once the vehicle has stopped after moving off, its wheels stop slipping within `within` seconds (a vehicle left skidding on locked wheels keeps its brakes capped by the sliding adhesion).""" @@ -444,6 +453,7 @@ CHECKS = { "stops": check_stops, "stays_stopped": check_stays_stopped, "distance": check_distance, + "ends_moving": check_ends_moving, } diff --git a/utilities/Globals.cpp b/utilities/Globals.cpp index 5cdb3ef2..2516da29 100644 --- a/utilities/Globals.cpp +++ b/utilities/Globals.cpp @@ -487,6 +487,12 @@ bool global_settings::ConfigParseTest(cParser& Parser, const std::string& token) return true; } + if (token == "test.no_movement_before_pass") + { + ParseOne(Parser, test.no_movement_before_pass); + return true; + } + return false; } @@ -1783,6 +1789,7 @@ global_settings::export_as_text( std::ostream &Output ) const { export_as_text( Output, "test.standstill", test.standstill ); export_as_text( Output, "test.pass_on_standstill", test.pass_on_standstill ); export_as_text( Output, "test.fail_if_stop_overran", test.fail_if_stop_overran ); + export_as_text( Output, "test.no_movement_before_pass", test.no_movement_before_pass ); export_as_text( Output, "gfx.envmap.enabled", gfx_envmap_enabled ); export_as_text( Output, "gfx.postfx.motionblur.enabled", gfx_postfx_motionblur_enabled ); export_as_text( Output, "gfx.postfx.motionblur.shutter", gfx_postfx_motionblur_shutter ); diff --git a/utilities/Globals.h b/utilities/Globals.h index 8fa04e9c..a6a9e122 100644 --- a/utilities/Globals.h +++ b/utilities/Globals.h @@ -93,6 +93,7 @@ struct global_settings { double standstill{ 300.0 }; // [s] every vehicle standing still this long ends the test; 0 = never bool pass_on_standstill{ false }; // the standstill end passes the test instead of failing it bool fail_if_stop_overran{ true }; // a train passing a signal at stop ends the test as failed; with false it's only logged + bool no_movement_before_pass{ true }; // a pass waits until every vehicle stands still; with false it ends the test at once bool timeout_from_command_line{ false }; // --test-timeout given, the scenery doesn't override it } test; bool editor_tracks{ false }; // F3 track workspace: draw and pick tracks without the scenery draw-range cap From 547706b54d769865bc9fb8239cdbdbe64f292dc8 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 09:35:59 +0100 Subject: [PATCH 12/28] refactor(simulation): fix the Sonar remarks on the test mode code - TController::TableCheck() calls TSpeedPos::Update() as it did before the test mode: the check for a signal passed at danger is a call of its own after it (CheckSignalPassedAtDanger), so the update lines nest no deeper than on master (S134). A switch is a track, so a signal's update never takes the switch-change branch and the check runs in the same place as before; the physicslogs of all 15 scenario cases are byte for byte the same. - null_imgui_renderer is final (S1235), to_string(test_result) uses `using enum` (S6177), init_result_cell() is const (S5817), ConfigParseTest() takes a std::string_view (S6009) and the test settings struct is declared apart from its member (S3646). --- rendering/nullrenderer.h | 2 +- simulation/simulationtest.cpp | 15 ++++++++------- simulation/simulationtest.h | 2 +- utilities/Globals.cpp | 12 ++++++------ utilities/Globals.h | 5 +++-- vehicle/Driver.cpp | 18 ++++++++++-------- vehicle/Driver.h | 4 ++-- 7 files changed, 31 insertions(+), 27 deletions(-) diff --git a/rendering/nullrenderer.h b/rendering/nullrenderer.h index 2a7ef039..7ecb57bf 100644 --- a/rendering/nullrenderer.h +++ b/rendering/nullrenderer.h @@ -36,7 +36,7 @@ private: }; // user interface backend which draws nothing: imgui still builds its frames, so the ui code runs as usual -class null_imgui_renderer : public imgui_renderer { +class null_imgui_renderer final : public imgui_renderer { public: bool Init() override { return true; } void Shutdown() override { /* null implementation, nothing to do */ } diff --git a/simulation/simulationtest.cpp b/simulation/simulationtest.cpp index 8d57505f..45aa513f 100644 --- a/simulation/simulationtest.cpp +++ b/simulation/simulationtest.cpp @@ -45,19 +45,20 @@ test_manager &test() std::string_view to_string(test_result const Result) { + using enum test_result; switch (Result) { - case test_result::pass: return "pass"; - case test_result::fail: return "fail"; - case test_result::signal_passed_at_danger: return "signal passed at danger"; - case test_result::timeout: return "timeout"; - case test_result::standstill: return "standstill"; - case test_result::error: return "error"; + case pass: return "pass"; + case fail: return "fail"; + case signal_passed_at_danger: return "signal passed at danger"; + case timeout: return "timeout"; + case standstill: return "standstill"; + case error: return "error"; } return "unknown"; } -void test_manager::init_result_cell() +void test_manager::init_result_cell() const { auto cell { std::make_unique(scene::node_data{0, -1, std::string(result_cell), "memcell"}) }; // neither of the reported values, so that a cell nobody wrote to says nothing diff --git a/simulation/simulationtest.h b/simulation/simulationtest.h index f47f408b..a58e9ae2 100644 --- a/simulation/simulationtest.h +++ b/simulation/simulationtest.h @@ -39,7 +39,7 @@ public: static constexpr double frame_step { 0.03 }; // creates the result cell; called with the other scripting interface cells, before the scenery is read - void init_result_cell(); + void init_result_cell() const; // the simulation is ready: start the clocks void start(); // checks the end conditions after a simulation frame of Deltatime seconds diff --git a/utilities/Globals.cpp b/utilities/Globals.cpp index 2516da29..3a9e1014 100644 --- a/utilities/Globals.cpp +++ b/utilities/Globals.cpp @@ -456,10 +456,10 @@ bool global_settings::ConfigParseInput(cParser& Parser, const std::string& token return false; } -bool global_settings::ConfigParseTest(cParser& Parser, const std::string& token) +bool global_settings::ConfigParseTest(cParser& Parser, std::string_view const Token) { // integration test limits, usually given in the config block of a test scenery - if (token == "test.timeout") + if (Token == "test.timeout") { double value{ test.timeout }; ParseOne(Parser, value); @@ -468,26 +468,26 @@ bool global_settings::ConfigParseTest(cParser& Parser, const std::string& token) return true; } - if (token == "test.standstill") + if (Token == "test.standstill") { ParseOne(Parser, test.standstill); test.standstill = std::max(0.0, test.standstill); return true; } - if (token == "test.pass_on_standstill") + if (Token == "test.pass_on_standstill") { ParseOne(Parser, test.pass_on_standstill); return true; } - if (token == "test.fail_if_stop_overran") + if (Token == "test.fail_if_stop_overran") { ParseOne(Parser, test.fail_if_stop_overran); return true; } - if (token == "test.no_movement_before_pass") + if (Token == "test.no_movement_before_pass") { ParseOne(Parser, test.no_movement_before_pass); return true; diff --git a/utilities/Globals.h b/utilities/Globals.h index a6a9e122..11d14e34 100644 --- a/utilities/Globals.h +++ b/utilities/Globals.h @@ -95,7 +95,8 @@ struct global_settings { bool fail_if_stop_overran{ true }; // a train passing a signal at stop ends the test as failed; with false it's only logged bool no_movement_before_pass{ true }; // a pass waits until every vehicle stands still; with false it ends the test at once bool timeout_from_command_line{ false }; // --test-timeout given, the scenery doesn't override it - } test; + }; + test_settings test; bool editor_tracks{ false }; // F3 track workspace: draw and pick tracks without the scenery draw-range cap int iConvertModels{ 0 }; // tworzenie plików binarnych int iConvertIndexRange{ 1000 }; // range of duplicate vertex scan @@ -387,7 +388,7 @@ struct global_settings { bool ConfigParseNetwork(cParser& Parser, const std::string& token); bool ConfigParseHardware(cParser& Parser, const std::string& token); bool ConfigParseDebug(cParser& Parser, const std::string& token); - bool ConfigParseTest(cParser& Parser, const std::string& token); + bool ConfigParseTest(cParser& Parser, std::string_view Token); bool ConfigParse_gfx( cParser &parser, std::string_view const Token ); // sends basic content of the class in legacy (text) format to provided stream void diff --git a/vehicle/Driver.cpp b/vehicle/Driver.cpp index c0614923..fe09fb24 100644 --- a/vehicle/Driver.cpp +++ b/vehicle/Driver.cpp @@ -768,7 +768,8 @@ void TController::TableCheck(double fDistance) { // aktualizacja rekordów z wyjątkiem ostatniego if (sSpeedTable[i].iFlags & spEnabled) // jeśli pozycja istotna { - if (UpdateSpeedPosition(sSpeedTable[i])) + auto const passedbefore { TestFlag( sSpeedTable[ i ].iFlags, spElapsed ) }; + if (sSpeedTable[i].Update()) { if( Global.iWriteLogEnabled & 8 ) { WriteLog( "Speed table for " + OwnerName() + " detected switch change at " + sSpeedTable[ i ].trTrack->name() + " (generating fresh trace)" ); @@ -794,6 +795,8 @@ void TController::TableCheck(double fDistance) // nie kontynuujemy pętli, trzeba doskanować ciąg dalszy break; } + // a switch is a track, the update of a signal never ends the loop above + CheckSignalPassedAtDanger( sSpeedTable[ i ], passedbefore ); if (sSpeedTable[i].iFlags & spTrack) // jeśli odcinek { if( sSpeedTable[ i ].fDist + sSpeedTable[ i ].trTrack->Length() < -fLength ) // a skład wyjechał całą długością poza @@ -817,8 +820,10 @@ void TController::TableCheck(double fDistance) } } } - // aktualizacja ostatniego; a signal at stop often is the last, the scan doesn't go past it - UpdateSpeedPosition(sSpeedTable[iLast]); + auto const lastpassedbefore { TestFlag( sSpeedTable[ iLast ].iFlags, spElapsed ) }; + sSpeedTable[iLast].Update(); // aktualizacja ostatniego + // a signal at stop often is the last, the scan doesn't go past it + CheckSignalPassedAtDanger( sSpeedTable[ iLast ], lastpassedbefore ); // WriteLog("TableCheck: Upate last track. Dist=" + AnsiString(sSpeedTable[iLast].fDist)); if( sSpeedTable[ iLast ].fDist < fDistance ) { TableTraceRoute( fDistance, pVehicles[ end::rear ] ); // doskanowanie dalszego odcinka @@ -843,17 +848,14 @@ bool is_signal_at_stop( TSpeedPos const &Point ) } // namespace -bool TController::UpdateSpeedPosition( TSpeedPos &Point ) const +void TController::CheckSignalPassedAtDanger( TSpeedPos const &Point, bool const Passedbefore ) const { - auto const passedbefore { TestFlag( Point.iFlags, spElapsed ) }; - auto const switchchanged { Point.Update() }; - if( false == passedbefore + if( false == Passedbefore && TestFlag( Point.iFlags, spElapsed ) && is_signal_at_stop( Point ) && ( mvOccupied->CategoryFlag & 1 ) != 0 ) { // railway only simulation::test().signal_passed_at_danger( OwnerName(), Point.evEvent->m_name ); } - return switchchanged; } auto const passengerstopmaxdistance { 400.0 }; // maximum allowed distance between passenger stop point and consist head diff --git a/vehicle/Driver.h b/vehicle/Driver.h index 8f2015f2..dfa63c08 100644 --- a/vehicle/Driver.h +++ b/vehicle/Driver.h @@ -519,9 +519,9 @@ private: bool TableNotFound( basic_event const *Event, double const Distance ) const; void TableTraceRoute( double fDistance, TDynamicObject *pVehicle ); void TableCheck( double fDistance ); - // updates a position of the speed table, returns true when a switch on it changed (TSpeedPos::Update). + // called after TSpeedPos::Update() of a position of the speed table, with its spElapsed flag from before it: // reports the head of the train going past it, if it's a signal showing stop - bool UpdateSpeedPosition( TSpeedPos &Point ) const; + void CheckSignalPassedAtDanger( TSpeedPos const &Point, bool Passedbefore ) const; TCommandType TableUpdate( double &fVelDes, double &fDist, double &fNext, double &fAcc ); bool TableUpdateStopPoint( TCommandType &Command, TSpeedPos &Point, double const Signaldistance ); bool TableUpdateEvent( double &Velocity, TCommandType &Command, TSpeedPos &Point, double &Signaldistance, int const Pointindex ); From 4356e7c3fd8beb83c9c22e0ae55b0ac34df7705e Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 09:36:28 +0100 Subject: [PATCH 13/28] fix(simulation): a test run that can't start ends with the error code and says why - `--test` is looked for before the rest of the command line is read. An unknown option before it (e.g. `-s x.scn --test-timout 90 --test`) used to end the run with exit code 0, which a caller reads as a pass. - `--test-timeout` without a value, or with one that isn't a number of seconds, fails the run instead of leaving the default or, through atof, turning the timeout off. - These, a missing -s, a failed start and an exception caught in main() end the test through test_manager::conclude(), which writes the usual `test: error (exit code 5) ... : ` line to stdout and the log. Before, `eu07 --test` alone exited with 5 and nothing anywhere: the error went to the log queue and the process ended before the log thread wrote it. In test mode main() now stops the log thread and waits for it before _Exit(). Outside the test mode nothing changes: conclude() does nothing there. --- EU07.cpp | 21 +++++++++++++++++++++ application/application.cpp | 34 ++++++++++++++++++++++++++++++---- simulation/simulationtest.cpp | 11 ++++++++--- simulation/simulationtest.h | 5 ++++- 4 files changed, 63 insertions(+), 8 deletions(-) diff --git a/EU07.cpp b/EU07.cpp index ec3e972e..790f6ffb 100644 --- a/EU07.cpp +++ b/EU07.cpp @@ -84,6 +84,20 @@ LONG WINAPI CrashHandler(EXCEPTION_POINTERS *ExceptionInfo) #endif +namespace +{ + +// the log is written by a thread of its own, which the closing of the application stops and waits for. on the ways +// out that skip the closing (a failed start, an exception) what was logged last would be lost with the process +void finish_log() +{ + StopLogService(); + if (auto const logservice{Global.threads.find("LogService")}; logservice != Global.threads.end() && logservice->second.joinable()) + logservice->second.join(); +} + +} // namespace + int main(int argc, char *argv[]) { #ifdef WITHDUMPGEN @@ -153,6 +167,7 @@ int main(int argc, char *argv[]) { ErrorLog(std::format("Critical error: {}", Error.what())); // in test mode nobody is there to close the box, the exit code tells of the error + simulation::test().conclude(simulation::test_result::error, std::format("critical error: {}", Error.what())); if (false == Global.test.enabled) { std::string msg = "Simulator crash occured :(\n"; @@ -166,6 +181,12 @@ int main(int argc, char *argv[]) fflush(stdout); fflush(stderr); #endif + if (Global.test.enabled) + { + // a run that stopped before its test ended (a failed start, an exception) says so like any other end + simulation::test().conclude(simulation::test_result::error, "the simulator stopped before the test ended"); + finish_log(); + } // skip destructors, there are ordering errors which causes segfaults // a test run (--test) says its result in the exit code std::_Exit(Global.test.enabled ? simulation::test().exit_code() : 0); diff --git a/application/application.cpp b/application/application.cpp index eaf60c7c..f8e20bed 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -30,8 +30,15 @@ http://mozilla.org/MPL/2.0/. #include "utilities/Timer.h" #include "utilities/dictionary.h" #include "version_info.h" +#include "simulation/simulationtest.h" +#include +#include #include +#include #include +#include +#include +#include #include "utilities/translation.h" #if WITH_DISCORD_RPC @@ -1174,6 +1181,16 @@ void apply_test_mode_settings() Global.random_seed = seed_of("test"); } +// seconds given on the command line: a number, not negative. returns nothing for anything else +std::optional parse_seconds(std::string_view const Text) +{ + auto const *const last{Text.data() + Text.size()}; + double value{0.0}; + if (auto const [parsed, status]{std::from_chars(Text.data(), last, value)}; status != std::errc{} || parsed != last || false == std::isfinite(value) || value < 0.0) + return std::nullopt; + return value; +} + } // namespace int eu07_application::init_settings(int Argc, char *Argv[]) @@ -1191,6 +1208,10 @@ int eu07_application::init_settings(int Argc, char *Argv[]) Global.LoadIniFile("eu07.ini"); } + // a test run is known before the rest of the command line is read, so a command line it can't take + // ends with the error code of the test, not with 0, which would pass it + Global.test.enabled = std::ranges::any_of(std::span(Argv, static_cast(Argc)), [](char const *Argument) { return std::string_view(Argument) == "--test"; }); + // process command line arguments bool sceneryfromcommandline{false}; bool editwithoutscenery{false}; @@ -1244,11 +1265,15 @@ int eu07_application::init_settings(int Argc, char *Argv[]) } else if (token == "--test-timeout") { - if (i + 1 < Argc) + auto const timeout{i + 1 < Argc ? parse_seconds(Argv[++i]) : std::nullopt}; + if (false == timeout.has_value()) { - Global.test.timeout = std::max(0.0, std::atof(Argv[++i])); - Global.test.timeout_from_command_line = true; + // a typo would otherwise turn the timeout off + simulation::test().conclude(simulation::test_result::error, "--test-timeout needs a number of seconds"); + return -1; } + Global.test.timeout = *timeout; + Global.test.timeout_from_command_line = true; } else if (token == "-ini") { @@ -1267,6 +1292,7 @@ int eu07_application::init_settings(int Argc, char *Argv[]) << " [-seed number|word]" << " [-ini settingsfile]" << " [--test [--test-timeout seconds]]" << std::endl; + simulation::test().conclude(simulation::test_result::error, std::format("unknown command line argument {}", token)); return -1; } } @@ -1277,7 +1303,7 @@ int eu07_application::init_settings(int Argc, char *Argv[]) { if (false == sceneryfromcommandline || Global.SceneryFile.empty()) { - ErrorLog("test: --test needs a scenery given with -s"); + simulation::test().conclude(simulation::test_result::error, "--test needs a scenery given with -s"); return -1; } apply_test_mode_settings(); diff --git a/simulation/simulationtest.cpp b/simulation/simulationtest.cpp index 45aa513f..214d7e46 100644 --- a/simulation/simulationtest.cpp +++ b/simulation/simulationtest.cpp @@ -156,9 +156,15 @@ void test_manager::signal_passed_at_danger(std::string_view const Vehicle, std:: } void test_manager::finish(test_result const Result, std::string_view const Reason) +{ + if (conclude(Result, Reason)) + Application.queue_quit(true); +} + +bool test_manager::conclude(test_result const Result, std::string_view const Reason) { if (m_finished || false == Global.test.enabled) - return; + return false; m_finished = true; m_result = Result; @@ -170,8 +176,7 @@ void test_manager::finish(test_result const Result, std::string_view const Reaso // the process has no console of its own; whoever started it with a redirected output gets the verdict there too std::fprintf(stdout, "%s\n", line.c_str()); std::fflush(stdout); - - Application.queue_quit(true); + return true; } } // namespace simulation diff --git a/simulation/simulationtest.h b/simulation/simulationtest.h index a58e9ae2..2b5012e9 100644 --- a/simulation/simulationtest.h +++ b/simulation/simulationtest.h @@ -46,8 +46,11 @@ public: void update(double Deltatime); // reported by a driver whose train went past a signal showing stop void signal_passed_at_danger(std::string_view Vehicle, std::string_view Signal); - // ends the test with given result; the first call wins + // ends the test with given result and closes the application; the first call wins void finish(test_result Result, std::string_view Reason); + // ends the test with given result, without closing the application: for the ways out that close it anyway + // (a bad command line, a failed start, an exception). returns false if the test had already ended + bool conclude(test_result Result, std::string_view Reason); bool finished() const { return m_finished; } // exit code of the process: the result, or error while the test hasn't finished From ced4e06b35b7945789460efed5af22c205fa0b98 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 09:36:35 +0100 Subject: [PATCH 14/28] fix(simulation): no crash message box in test mode The crash handler of WITHDUMPGEN builds (on by default) shows a message box before it writes the dump. It also gets the C++ exceptions nothing catches, since it replaces the CRT's unhandled exception filter. In test mode nobody is there to close the box, so a crashing test waited for it until whatever started it gave up (run.py after the case's timeout, 600 s by default). The dump is still written, the exit code is the exception's. --- EU07.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/EU07.cpp b/EU07.cpp index 790f6ffb..0fbabba5 100644 --- a/EU07.cpp +++ b/EU07.cpp @@ -73,7 +73,9 @@ LONG WINAPI CrashHandler(EXCEPTION_POINTERS *ExceptionInfo) MINIDUMP_TYPE dumpType = MINIDUMP_TYPE(MiniDumpWithFullMemory | MiniDumpWithHandleData | MiniDumpWithThreadInfo | MiniDumpWithUnloadedModules | MiniDumpWithIndirectlyReferencedMemory | MiniDumpWithFullMemoryInfo | MiniDumpWithTokenInformation); - MessageBoxA(nullptr, "Simulator crash occured :(\n", "Simulator crashed :(", MB_ICONERROR); + // in test mode nobody is there to close the box, and the run would wait for it until killed + if (false == Global.test.enabled) + MessageBoxA(nullptr, "Simulator crash occured :(\n", "Simulator crashed :(", MB_ICONERROR); MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(), hFile, dumpType, &dumpInfo, nullptr, nullptr); CloseHandle(hFile); From 93e2b85d08cca5d4d374a3965c747292d59656d9 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 09:36:51 +0100 Subject: [PATCH 15/28] fix(simulation): the test mode gives nothing to the outside and leaves the player's files alone The test mode turned off what comes in from outside, but a few settings of the user's eu07.ini still reached out of a test run: - feedbackmode: 1 and 2 toggle Caps Lock / Scroll Lock by injecting key events into the whole system, 3 and 4 drive an LPT port or a PoKeys desk (and 4 reads the desk's switches); - multiplayer: events are reported to a multiplayer server, and some events are held back for it; - execonexit: a program is run with system() at the end of each run, which also waits for it; - showsystemconsole: a console of the run's own pops up and stdout is reopened on it, so whoever started the test with a redirected output doesn't get the `test:` line (and a console the run was started from would be hidden without it); - imgui.ini in the game folder was written by every run, and by several at once with run.py -j. All of these are off in test mode. Nothing changes outside it. --- application/application.cpp | 18 ++++++++++++++++++ application/uilayer.cpp | 5 +++++ 2 files changed, 23 insertions(+) diff --git a/application/application.cpp b/application/application.cpp index f8e20bed..d4aed9d8 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -31,6 +31,9 @@ http://mozilla.org/MPL/2.0/. #include "utilities/dictionary.h" #include "version_info.h" #include "simulation/simulationtest.h" +#ifdef _WIN32 +#include "Console.h" +#endif #include #include #include @@ -1048,6 +1051,13 @@ void eu07_application::init_debug() void eu07_application::init_console() { #ifdef _WIN32 + if (Global.test.enabled) + { + // a test is started by whoever waits for its result, often reading its output: no console of its own + // (showsystemconsole) pops up and takes stdout away from them, and a console it was started from isn't hidden + return; + } + HWND consoleWnd = ::GetConsoleWindow(); const bool hadConsole = consoleWnd != nullptr; @@ -1176,6 +1186,14 @@ void apply_test_mode_settings() #ifdef WITH_ZMQ Global.zmq_address.clear(); #endif + // nor anything to the outside: no keyboard leds, LPT or PoKeys desk (feedbackmode), no messages to a + // multiplayer server (multiplayer, which also holds some events back for it), no program run at the end + Global.iFeedbackMode = 0; +#ifdef _WIN32 + Console::ModeSet(Global.iFeedbackMode, Global.iFeedbackPort); +#endif + Global.iMultiplayer = 0; + Global.exec_on_exit.clear(); // and the same way every time, unless given another seed (-seed, or randomseed in the settings) if (!Global.random_seed) Global.random_seed = seed_of("test"); diff --git a/application/uilayer.cpp b/application/uilayer.cpp index 5443d158..855bfbad 100644 --- a/application/uilayer.cpp +++ b/application/uilayer.cpp @@ -288,6 +288,11 @@ bool ui_layer::init(GLFWwindow *Window) IMGUI_CHECKVERSION(); ImGui::CreateContext(); m_imguiio = &ImGui::GetIO(); + if (Global.test.enabled) + { + // a test leaves the player's window layout alone (imgui.ini in the game folder), runs in parallel included + m_imguiio->IniFilename = nullptr; + } ImGui::GetCurrentContext()->ConfigNavWindowingKeyNext = ImGuiKey_None; ImGui::GetCurrentContext()->ConfigNavWindowingKeyPrev = ImGuiKey_None; // imgui.ini is read with the first frame, which may be of another mode than the editor From 1b373bcc385e800e3a6717fc37f1fbda75f5de7c Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 09:36:56 +0100 Subject: [PATCH 16/28] docs(test): the test mode README on the new ends, outside settings and Windows shells - what the test mode turns off towards the outside (feedbackmode, multiplayer, execonexit, showsystemconsole, imgui.ini), - a command line it can't take or a failed start ends with `error`, - cmd and PowerShell don't wait for the (windowed) simulator nor show its output: start /wait, Start-Process -Wait, - 15 cases, not 14. --- tests/scenarios/README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index b57fc4a0..1d63ebef 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -26,12 +26,21 @@ With `--test` the simulator: - takes no input from outside: keys and mouse sent to its window, gamepad, UART (a cab desk), ZeroMQ and commands sent by other programs (`WM_COPYDATA`) are all ignored, and motion telemetry is off, +- gives nothing to the outside either: no keyboard leds, LPT or PoKeys (`feedbackmode`), no + messages to a multiplayer server (`multiplayer`), no program run at the end (`execonexit`), + no console window of its own (`showsystemconsole`), and the game's `imgui.ini` stays as it is, - runs the simulation as fast as it can in fixed steps of 0.03 s of simulation time per frame (three 10 ms physics steps), whatever the real time, the time speed or the machine, - seeds its random numbers with a fixed seed unless `-seed` or `randomseed` gives one, so two runs of a scenery come out the same (the physicslogs of two runs are byte for byte equal), - quits when the test ends, and says how it ended in the exit code and in a `test: ...` - line on stdout and in the log. + line on stdout and in the log. A command line it can't take, or a start that fails, ends + the same way with the `error` code. + +On Windows the simulator is a windowed program, so `cmd` and PowerShell don't wait for it nor +show its output by themselves: run it with `start /wait eu07 --test ... > result.txt` and read +`%errorlevel%` in `cmd`, or `(Start-Process eu07 '--test ...' -Wait -PassThru -RedirectStandardOutput result.txt).ExitCode` +in PowerShell. Scripts (Python's `subprocess`, bash) wait for it as for any other program. A scenery reports its result to the memory cell `__test.result`, which the simulator creates (in every run, so a test scenery loads normally too). Write value 1 to it for a pass and 0 @@ -101,7 +110,7 @@ Python 3.10+, no extra packages. Options: `cmake-build-*/bin/Release/eu07*.exe`. The build needs the `--test` and `-ini` command line options. - name one or more cases to run only those, e.g. `run.py ... eu07_td_timetable`. -- `-j N` runs N cases at once. Each case prints its results when it is done. All 14 cases +- `-j N` runs N cases at once. Each case prints its results when it is done. All 15 cases take about 30 s with `-j 4`, loading included. - `--watch` runs a case the old way, in a visible window in real time (or faster with `--timespeed N`), closed after the case's `duration`; the exit code isn't checked then. For From b2331ad2d185a2b4e52529b1900e90eeeecf4be7 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 09:37:32 +0100 Subject: [PATCH 17/28] docs(test): what waits for a test run in cmd, checked start /wait gives the exit code but not the output; a .bat file waits for the windowed simulator and passes its output on. --- tests/scenarios/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index 1d63ebef..8ad7e747 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -37,10 +37,11 @@ With `--test` the simulator: line on stdout and in the log. A command line it can't take, or a start that fails, ends the same way with the `error` code. -On Windows the simulator is a windowed program, so `cmd` and PowerShell don't wait for it nor -show its output by themselves: run it with `start /wait eu07 --test ... > result.txt` and read -`%errorlevel%` in `cmd`, or `(Start-Process eu07 '--test ...' -Wait -PassThru -RedirectStandardOutput result.txt).ExitCode` -in PowerShell. Scripts (Python's `subprocess`, bash) wait for it as for any other program. +On Windows the simulator is a windowed program, which the `cmd` prompt and PowerShell don't wait +for. In a `.bat` file `eu07 --test ... > result.txt` waits and sets `%errorlevel%`; at the prompt +`start /wait eu07 --test ...` waits too but doesn't pass the output on. In PowerShell use +`(Start-Process eu07 '--test ...' -Wait -PassThru -RedirectStandardOutput result.txt).ExitCode`. +Scripts (Python's `subprocess`, bash) wait for it as for any other program. A scenery reports its result to the memory cell `__test.result`, which the simulator creates (in every run, so a test scenery loads normally too). Write value 1 to it for a pass and 0 From 62526e9d6a1b87fcb0f67cb36ba674a91da94f9c Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 11:16:33 +0100 Subject: [PATCH 18/28] test(scenarios): the timetable cases check where the train stops, not how long Przystanek on td.scn is a technical stop of the td/rozklad timetable (pt): the AI doesn't wait there for the departure time but sets off as soon as it counts the train as standing at the W4, below 1 km/h. How long the speed then stays under 0.5 km/h is only how fast the brake releases. With the AI braking of #264 the trains creep up to the W4 at about 10 km/h on a light application and stood there for 1-5 s; the SN61 turns back at 0.36 km/h without stopping. The hold of 5 s (2 s for the SN61) failed for EP09, SN61 and Vectron and had well under a second to spare for others. The cases now check that the train comes down to 1 km/h, the AI's own standstill at a W4, 1380-1420 m from the start; every vehicle stops at 1397-1401 m, before and after #264. `stops` takes min_distance and max_distance for that. --- tests/scenarios/README.md | 14 ++++++++++---- tests/scenarios/cases/en57_td_timetable.json | 2 +- tests/scenarios/cases/ep09_td_timetable.json | 2 +- tests/scenarios/cases/et22_td_timetable.json | 2 +- tests/scenarios/cases/eu07_td_timetable.json | 2 +- tests/scenarios/cases/sm42_td_timetable.json | 2 +- tests/scenarios/cases/sn61_td_timetable.json | 2 +- tests/scenarios/cases/vectron_td_timetable.json | 2 +- tests/scenarios/run.py | 8 ++++++-- 9 files changed, 23 insertions(+), 13 deletions(-) diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index 8ad7e747..8bd6f1a4 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -168,7 +168,12 @@ scenery is newer. the one whose physicslog is checked. Other vehicles of the consist take names made from it, e.g. `{{vehicle}}_s`. The td.scn cases share two fragments, `td_timetable.scn` (route set, train runs to Przystanek) and `td_red_signal.scn` (no route, exit signal at stop), and differ -only in `vars.consist`. `td_red_signal_moving.scn` is the red signal one with the train +only in `vars.consist`. Przystanek is a technical stop in that timetable (`pt`), where the AI +doesn't wait for the departure time: it sets off as soon as it counts the train as standing at +the W4, which is below 1 km/h, so the train stands there for a few seconds at most, or doesn't +quite stop (since the AI braking of #264 the SN61 turns back at 0.4 km/h). The +timetable cases therefore check that the train comes down to 1 km/h, with no hold, 1380-1420 m +from the start (every vehicle stops at 1397-1401 m). `td_red_signal_moving.scn` is the red signal one with the train starting at `vars.velocity` km/h; with `track_friction` 0.03 it locks the wheels when braking and slides past the signal, so those cases set `test.fail_if_stop_overran no`. `td_red_signal_pass.scn` adds the `test_pass` event, which `test_mode_pass` puts as `event0` @@ -189,13 +194,14 @@ the row instead, logged by builds from this branch on. - `max_speed` `limit`, `tolerance`: never goes faster than `limit + tolerance`. - `brakes` `min_pressure`: after moving off, brake cylinder pressure reaches `min_pressure` bar (physicslog pressures are in bar whatever its header says; the brake pipe runs at 5.0). -- `stops` `moving_speed`, `stop_speed`, `hold`: after reaching `moving_speed`, stays at or - below `stop_speed` for `hold` seconds. +- `stops` `moving_speed`, `stop_speed`, `hold`, optional `min_distance`, `max_distance`: after + reaching `moving_speed`, stays at or below `stop_speed` for `hold` seconds; with the distances, + that stop has to be between them (metres travelled, as in `distance`). - `stays_stopped` `moving_speed`, `stop_speed`: after moving off and stopping, never goes above `stop_speed` again. - `distance` `min`, `max`: total distance travelled. The physicslog has no position, so the distance comes from integrating the speed; on td.scn every vehicle's first stop after - Stacja1 came out at 1400 m (+-3), and the red signal stop at 140 m. + Stacja1 came out at 1400 m (+-4), and the red signal stop at 140 m. - `slips` `within`: the lead vehicle's wheels slip or lock at some point (within `within` seconds if given), so a slippery rail case does what it's meant to. - `slip_clears` `moving_speed`, `stop_speed`, `within` (*vehicle speed*): once stopped after diff --git a/tests/scenarios/cases/en57_td_timetable.json b/tests/scenarios/cases/en57_td_timetable.json index ddd52af3..d4c4f26f 100644 --- a/tests/scenarios/cases/en57_td_timetable.json +++ b/tests/scenarios/cases/en57_td_timetable.json @@ -16,6 +16,6 @@ { "type": "starts", "min_speed": 20, "within": 150 }, { "type": "max_speed", "limit": 90, "tolerance": 3 }, { "type": "brakes", "min_pressure": 0.5 }, - { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + { "type": "stops", "moving_speed": 20, "stop_speed": 1.0, "hold": 0, "min_distance": 1380, "max_distance": 1420 } ] } diff --git a/tests/scenarios/cases/ep09_td_timetable.json b/tests/scenarios/cases/ep09_td_timetable.json index dd130be3..851b4dbf 100644 --- a/tests/scenarios/cases/ep09_td_timetable.json +++ b/tests/scenarios/cases/ep09_td_timetable.json @@ -12,6 +12,6 @@ { "type": "starts", "min_speed": 20, "within": 150 }, { "type": "max_speed", "limit": 90, "tolerance": 3 }, { "type": "brakes", "min_pressure": 0.5 }, - { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + { "type": "stops", "moving_speed": 20, "stop_speed": 1.0, "hold": 0, "min_distance": 1380, "max_distance": 1420 } ] } diff --git a/tests/scenarios/cases/et22_td_timetable.json b/tests/scenarios/cases/et22_td_timetable.json index 004f5354..47622c62 100644 --- a/tests/scenarios/cases/et22_td_timetable.json +++ b/tests/scenarios/cases/et22_td_timetable.json @@ -12,6 +12,6 @@ { "type": "starts", "min_speed": 20, "within": 150 }, { "type": "max_speed", "limit": 90, "tolerance": 3 }, { "type": "brakes", "min_pressure": 0.5 }, - { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + { "type": "stops", "moving_speed": 20, "stop_speed": 1.0, "hold": 0, "min_distance": 1380, "max_distance": 1420 } ] } diff --git a/tests/scenarios/cases/eu07_td_timetable.json b/tests/scenarios/cases/eu07_td_timetable.json index 2f23019f..3774242c 100644 --- a/tests/scenarios/cases/eu07_td_timetable.json +++ b/tests/scenarios/cases/eu07_td_timetable.json @@ -12,6 +12,6 @@ { "type": "starts", "min_speed": 20, "within": 150 }, { "type": "max_speed", "limit": 90, "tolerance": 3 }, { "type": "brakes", "min_pressure": 0.5 }, - { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + { "type": "stops", "moving_speed": 20, "stop_speed": 1.0, "hold": 0, "min_distance": 1380, "max_distance": 1420 } ] } diff --git a/tests/scenarios/cases/sm42_td_timetable.json b/tests/scenarios/cases/sm42_td_timetable.json index fc89be0f..994f94c0 100644 --- a/tests/scenarios/cases/sm42_td_timetable.json +++ b/tests/scenarios/cases/sm42_td_timetable.json @@ -12,6 +12,6 @@ { "type": "starts", "min_speed": 20, "within": 150 }, { "type": "max_speed", "limit": 90, "tolerance": 3 }, { "type": "brakes", "min_pressure": 0.5 }, - { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + { "type": "stops", "moving_speed": 20, "stop_speed": 1.0, "hold": 0, "min_distance": 1380, "max_distance": 1420 } ] } diff --git a/tests/scenarios/cases/sn61_td_timetable.json b/tests/scenarios/cases/sn61_td_timetable.json index 9fa4b18d..1a03c31f 100644 --- a/tests/scenarios/cases/sn61_td_timetable.json +++ b/tests/scenarios/cases/sn61_td_timetable.json @@ -12,6 +12,6 @@ { "type": "starts", "min_speed": 20, "within": 150 }, { "type": "max_speed", "limit": 90, "tolerance": 3 }, { "type": "brakes", "min_pressure": 0.5 }, - { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 2 } + { "type": "stops", "moving_speed": 20, "stop_speed": 1.0, "hold": 0, "min_distance": 1380, "max_distance": 1420 } ] } diff --git a/tests/scenarios/cases/vectron_td_timetable.json b/tests/scenarios/cases/vectron_td_timetable.json index 9948a09e..04656a00 100644 --- a/tests/scenarios/cases/vectron_td_timetable.json +++ b/tests/scenarios/cases/vectron_td_timetable.json @@ -12,6 +12,6 @@ { "type": "starts", "min_speed": 20, "within": 150 }, { "type": "max_speed", "limit": 90, "tolerance": 3 }, { "type": "brakes", "min_pressure": 0.5 }, - { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + { "type": "stops", "moving_speed": 20, "stop_speed": 1.0, "hold": 0, "min_distance": 1380, "max_distance": 1420 } ] } diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index 4c338308..210d6c01 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -344,7 +344,7 @@ def check_brakes(rows, min_pressure, moving_speed=20): return False, f"brake cylinder pressure never reached {min_pressure} bar after moving off" -def check_stops(rows, moving_speed, stop_speed, hold): +def check_stops(rows, moving_speed, stop_speed, hold, min_distance=None, max_distance=None): start = _first_moving(rows, moving_speed) if start is None: return False, f"never reached {moving_speed} km/h" @@ -356,7 +356,11 @@ def check_stops(rows, moving_speed, stop_speed, hold): if stopped_at is None: stopped_at = i if r[COL_TIME] - rows[stopped_at][COL_TIME] >= hold: - return True, f"came to a stop at t={rows[stopped_at][COL_TIME]:.1f}s after {dist[stopped_at]:.0f} m" + where = dist[stopped_at] + detail = f"came to a stop at t={rows[stopped_at][COL_TIME]:.1f}s after {where:.0f} m" + if (min_distance is not None and where < min_distance) or (max_distance is not None and where > max_distance): + return False, f"{detail}, wanted between {min_distance} and {max_distance} m" + return True, detail else: stopped_at = None return False, f"did not stay below {stop_speed} km/h for {hold}s after moving off" From 486a714f521ce6d28aca9f9adc475f03817306dc Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 11:20:52 +0100 Subject: [PATCH 19/28] refactor(simulation): the rules of the test mode in a file of their own, with unit tests The command line options of the test mode and the end conditions checked after each frame are now plain functions in simulation/simulationtestrules.cpp, which needs nothing of the running simulation: parse_test_command_line() and evaluate_frame(). The file is part of the core library, so the unit tests check them (tests/unit/test_test_mode.cpp): the --test-timeout value, values of other options not taken for options, a pass that waits for the stop, the standstill, the timeout. test_manager only gathers what a frame brought and acts on the verdict. No change in behaviour. --- CMakeLists.txt | 3 +- application/application.cpp | 45 +++---- simulation/simulationtest.cpp | 77 +++--------- simulation/simulationtest.h | 21 +--- simulation/simulationtestrules.cpp | 139 ++++++++++++++++++++ simulation/simulationtestrules.h | 82 ++++++++++++ tests/unit/test_test_mode.cpp | 195 +++++++++++++++++++++++++++++ 7 files changed, 452 insertions(+), 110 deletions(-) create mode 100644 simulation/simulationtestrules.cpp create mode 100644 simulation/simulationtestrules.h create mode 100644 tests/unit/test_test_mode.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a4e1d6e..483dbc14 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -454,7 +454,7 @@ endif() add_library(eu07_settings INTERFACE) # the parts of the simulation the game shares with the unit tests: the vehicle physics, the vehicle -# definition parser and the helpers they use. built once, and tested as the game uses them +# definition parser, the helpers they use and the rules of the test mode. built once, and tested as the game uses them set(CORE_SOURCES "McZapkie/friction.cpp" "McZapkie/hamulce.cpp" @@ -463,6 +463,7 @@ set(CORE_SOURCES "utilities/parser.cpp" "utilities/utilities.cpp" "scripting/ladderlogic.cpp" +"simulation/simulationtestrules.cpp" "vehicle/handover.cpp" "vehicle/horn.cpp" "vehicle/smoke.cpp" diff --git a/application/application.cpp b/application/application.cpp index d4aed9d8..f3c3fe46 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -34,14 +34,9 @@ http://mozilla.org/MPL/2.0/. #ifdef _WIN32 #include "Console.h" #endif -#include -#include #include -#include #include -#include -#include -#include +#include #include "utilities/translation.h" #if WITH_DISCORD_RPC @@ -1199,16 +1194,6 @@ void apply_test_mode_settings() Global.random_seed = seed_of("test"); } -// seconds given on the command line: a number, not negative. returns nothing for anything else -std::optional parse_seconds(std::string_view const Text) -{ - auto const *const last{Text.data() + Text.size()}; - double value{0.0}; - if (auto const [parsed, status]{std::from_chars(Text.data(), last, value)}; status != std::errc{} || parsed != last || false == std::isfinite(value) || value < 0.0) - return std::nullopt; - return value; -} - } // namespace int eu07_application::init_settings(int Argc, char *Argv[]) @@ -1228,7 +1213,18 @@ int eu07_application::init_settings(int Argc, char *Argv[]) // a test run is known before the rest of the command line is read, so a command line it can't take // ends with the error code of the test, not with 0, which would pass it - Global.test.enabled = std::ranges::any_of(std::span(Argv, static_cast(Argc)), [](char const *Argument) { return std::string_view(Argument) == "--test"; }); + auto const testcommandline{simulation::parse_test_command_line(std::vector(Argv, Argv + Argc))}; + Global.test.enabled = testcommandline.enabled; + if (false == testcommandline.error.empty()) + { + simulation::test().conclude(simulation::test_result::error, testcommandline.error); + return -1; + } + if (testcommandline.timeout.has_value()) + { + Global.test.timeout = *testcommandline.timeout; + Global.test.timeout_from_command_line = true; + } // process command line arguments bool sceneryfromcommandline{false}; @@ -1278,20 +1274,13 @@ int eu07_application::init_settings(int Argc, char *Argv[]) } else if (token == "--test") { - // integration test: the scenery runs unattended and the exit code is the result (eu07/maszyna#541) - Global.test.enabled = true; + // integration test: the scenery runs unattended and the exit code is the result (eu07/maszyna#541), + // taken from the command line above } else if (token == "--test-timeout") { - auto const timeout{i + 1 < Argc ? parse_seconds(Argv[++i]) : std::nullopt}; - if (false == timeout.has_value()) - { - // a typo would otherwise turn the timeout off - simulation::test().conclude(simulation::test_result::error, "--test-timeout needs a number of seconds"); - return -1; - } - Global.test.timeout = *timeout; - Global.test.timeout_from_command_line = true; + // taken from the command line above, with its value + ++i; } else if (token == "-ini") { diff --git a/simulation/simulationtest.cpp b/simulation/simulationtest.cpp index 214d7e46..76aa1be3 100644 --- a/simulation/simulationtest.cpp +++ b/simulation/simulationtest.cpp @@ -25,16 +25,6 @@ http://mozilla.org/MPL/2.0/. #include #include -namespace { - -// the cell holds what the scenery wrote, so the values are whole numbers; the margin only keeps a stray fraction out -bool holds(double const Value, double const Expected) -{ - return std::abs(Value - Expected) < 1e-6; -} - -} // namespace - namespace simulation { test_manager &test() @@ -43,21 +33,6 @@ test_manager &test() return instance; } -std::string_view to_string(test_result const Result) -{ - using enum test_result; - switch (Result) - { - case pass: return "pass"; - case fail: return "fail"; - case signal_passed_at_danger: return "signal passed at danger"; - case timeout: return "timeout"; - case standstill: return "standstill"; - case error: return "error"; - } - return "unknown"; -} - void test_manager::init_result_cell() const { auto cell { std::make_unique(scene::node_data{0, -1, std::string(result_cell), "memcell"}) }; @@ -73,8 +48,7 @@ void test_manager::start() if (m_started) return; m_started = true; - m_time = 0.0; - m_resttime = 0.0; + m_progress = test_progress {}; if (simulation::Vehicles.sequence().empty()) { // most likely the scenery wasn't found, or it isn't a test scenery @@ -94,48 +68,25 @@ void test_manager::update(double const Deltatime) if (false == m_started || m_finished) return; - m_time += Deltatime; - + test_frame frame { Deltatime, std::nullopt, {}, + std::ranges::none_of(simulation::Vehicles.sequence(), [](TDynamicObject const *Vehicle) { return std::abs(Vehicle->GetVelocity()) > 0.01; }) }; if (auto const *cell = simulation::Memory.find(std::string(result_cell))) { - auto const value { cell->Value1() }; - auto const reason { cell->Text().empty() ? std::string() : std::format(" ({})", cell->Text()) }; - if (holds(value, 0.0)) - { - finish(test_result::fail, std::format("__test.result set to 0{}", reason)); - return; - } - if (holds(value, 1.0) && false == Global.test.no_movement_before_pass) - { - finish(test_result::pass, std::format("__test.result set to 1{}", reason)); - return; - } - if (holds(value, 1.0) && false == m_passrequested) - { - m_passrequested = true; - WriteLog(std::format("test: pass requested at {:.1f} s{}, waiting for every vehicle to stop", m_time, reason)); - } + frame.reported = cell->Value1(); + frame.reason = cell->Text(); } - auto const atrest { std::ranges::none_of(simulation::Vehicles.sequence(), - [](TDynamicObject const *Vehicle) { return std::abs(Vehicle->GetVelocity()) > 0.01; }) }; - m_resttime = atrest ? m_resttime + Deltatime : 0.0; - - if (m_passrequested && m_resttime >= pass_settle_time) + auto const requestedbefore { m_progress.passrequested }; + auto const verdict { evaluate_frame(m_progress, frame, test_limits { Global.test.timeout, Global.test.standstill, Global.test.pass_on_standstill, Global.test.no_movement_before_pass }) }; + if (verdict.has_value()) { - finish(test_result::pass, "__test.result set to 1 and every vehicle stopped"); + finish(verdict->result, verdict->reason); return; } - if (Global.test.standstill > 0.0 && m_resttime >= Global.test.standstill) + if (m_progress.passrequested && false == requestedbefore) { - auto const reason { std::format("every vehicle stood still for {:.0f} s", Global.test.standstill) }; - finish(Global.test.pass_on_standstill ? test_result::pass : test_result::standstill, reason); - return; - } - if (Global.test.timeout > 0.0 && m_time >= Global.test.timeout) - { - finish(test_result::timeout, std::format("no result after {:.0f} s", Global.test.timeout)); - return; + WriteLog(std::format("test: pass requested at {:.1f} s{}, waiting for every vehicle to stop", m_progress.time, + frame.reason.empty() ? std::string() : std::format(" ({})", frame.reason))); } } @@ -149,7 +100,7 @@ void test_manager::signal_passed_at_danger(std::string_view const Vehicle, std:: } if (false == Global.test.fail_if_stop_overran) { - WriteLog(std::format("test: {} at {:.1f} s, allowed by test.fail_if_stop_overran", reason, m_time)); + WriteLog(std::format("test: {} at {:.1f} s, allowed by test.fail_if_stop_overran", reason, m_progress.time)); return; } finish(test_result::signal_passed_at_danger, reason); @@ -168,7 +119,7 @@ bool test_manager::conclude(test_result const Result, std::string_view const Rea m_finished = true; m_result = Result; - auto const line { std::format("test: {} (exit code {}) after {:.1f} s of simulation time: {}", to_string(Result), static_cast(Result), m_time, Reason) }; + auto const line { std::format("test: {} (exit code {}) after {:.1f} s of simulation time: {}", to_string(Result), static_cast(Result), m_progress.time, Reason) }; if (Result == test_result::pass) WriteLog(line); else diff --git a/simulation/simulationtest.h b/simulation/simulationtest.h index 2b5012e9..66384b6e 100644 --- a/simulation/simulationtest.h +++ b/simulation/simulationtest.h @@ -9,32 +9,19 @@ http://mozilla.org/MPL/2.0/. #pragma once -#include +#include "simulation/simulationtestrules.h" + #include // integration test mode (eu07 --test -s , eu07/maszyna#541): the scenery runs unattended in a tight // fixed-step loop and the simulator quits with an exit code saying how the test went namespace simulation { -// exit codes of a test run; public, scripts and CI read them -enum class test_result : int { - pass = 0, // __test.result set to 1, then every vehicle came to rest (or test.pass_on_standstill) - fail = 1, // __test.result set to 0 - signal_passed_at_danger = 2, // a train went past a stop signal (SetVelocity 0 0), unless test.fail_if_stop_overran no - timeout = 3, // test.timeout seconds of simulation time went by without a result - standstill = 4, // every vehicle stood still for test.standstill seconds without a result - error = 5, // the run could not start (no -s, the scenery failed to load) or the simulator threw -}; - -std::string_view to_string(test_result Result); - class test_manager { public: // name of the memory cell a scenery reports its result to static constexpr std::string_view result_cell { "__test.result" }; - // how long everything has to stand before a requested pass is granted [s] - static constexpr double pass_settle_time { 1.0 }; // fixed simulation time step of one frame in test mode [s]: 3 physics steps of 10 ms static constexpr double frame_step { 0.03 }; @@ -60,9 +47,7 @@ private: bool m_started { false }; bool m_finished { false }; test_result m_result { test_result::error }; - bool m_passrequested { false }; - double m_time { 0.0 }; // simulation time since the start [s] - double m_resttime { 0.0 }; // how long every vehicle has been standing [s] + test_progress m_progress; }; // the test of the run diff --git a/simulation/simulationtestrules.cpp b/simulation/simulationtestrules.cpp new file mode 100644 index 00000000..44f740f1 --- /dev/null +++ b/simulation/simulationtestrules.cpp @@ -0,0 +1,139 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#include "stdafx.h" +#include "simulation/simulationtestrules.h" + +#include +#include +#include + +namespace { + +// the cell holds what the scenery wrote, so the values are whole numbers; the margin only keeps a stray fraction out +bool holds(double const Value, double const Expected) +{ + return std::abs(Value - Expected) < 1e-6; +} + +// the text of the result cell, as said after the result +std::string reason_suffix(std::string_view const Reason) +{ + return Reason.empty() ? std::string() : std::format(" ({})", Reason); +} + +// options of the simulator which take a value, see eu07_application::init_settings() +bool takes_value(std::string_view const Option) +{ + return Option == "-s" || Option == "-v" || Option == "-seed" || Option == "-ini"; +} + +// a number of seconds given to Option, at Index of Arguments; the index is moved past the value +std::optional option_seconds(std::span const Arguments, std::size_t &Index) +{ + if (Index + 1 >= Arguments.size()) + return std::nullopt; + ++Index; + return simulation::parse_seconds(Arguments[Index]); +} + +} // namespace + +namespace simulation { + +std::string_view to_string(test_result const Result) +{ + using enum test_result; + switch (Result) + { + case pass: return "pass"; + case fail: return "fail"; + case signal_passed_at_danger: return "signal passed at danger"; + case timeout: return "timeout"; + case standstill: return "standstill"; + case error: return "error"; + } + return "unknown"; +} + +std::optional parse_seconds(std::string_view const Text) +{ + auto const *const last{Text.data() + Text.size()}; + double value{0.0}; + if (auto const [parsed, status]{std::from_chars(Text.data(), last, value)}; status != std::errc{} || parsed != last || false == std::isfinite(value) || value < 0.0) + return std::nullopt; + return value; +} + +test_command_line parse_test_command_line(std::span const Arguments) +{ + test_command_line result; + for (std::size_t i = 1; i < Arguments.size(); ++i) + { + std::string_view const option{Arguments[i]}; + if (option == "--test") + { + result.enabled = true; + } + else if (option == "--test-timeout") + { + auto const seconds{option_seconds(Arguments, i)}; + if (false == seconds.has_value()) + { + // a typo would otherwise turn the limit off + if (result.error.empty()) + result.error = std::format("{} needs a number of seconds", option); + continue; + } + result.timeout = seconds; + } + else if (takes_value(option)) + { + ++i; + } + else if (option == "-edit" && i + 1 < Arguments.size() && Arguments[i + 1][0] != '-') + { + // the scenery to edit is optional + ++i; + } + } + return result; +} + +std::optional evaluate_frame(test_progress &Progress, test_frame const &Frame, test_limits const &Limits) +{ + using enum test_result; + + Progress.time += Frame.deltatime; + + if (Frame.reported.has_value()) + { + auto const value{*Frame.reported}; + if (holds(value, 0.0)) + return test_verdict{fail, std::format("__test.result set to 0{}", reason_suffix(Frame.reason))}; + if (holds(value, 1.0)) + { + if (false == Limits.no_movement_before_pass) + return test_verdict{pass, std::format("__test.result set to 1{}", reason_suffix(Frame.reason))}; + Progress.passrequested = true; + } + } + + Progress.resttime = Frame.atrest ? Progress.resttime + Frame.deltatime : 0.0; + + if (Progress.passrequested && Progress.resttime >= pass_settle_time) + return test_verdict{pass, "__test.result set to 1 and every vehicle stopped"}; + if (Limits.standstill > 0.0 && Progress.resttime >= Limits.standstill) + return test_verdict{Limits.pass_on_standstill ? pass : standstill, std::format("every vehicle stood still for {:.0f} s", Limits.standstill)}; + if (Limits.timeout > 0.0 && Progress.time >= Limits.timeout) + return test_verdict{timeout, std::format("no result after {:.0f} s", Limits.timeout)}; + return std::nullopt; +} + +} // namespace simulation diff --git a/simulation/simulationtestrules.h b/simulation/simulationtestrules.h new file mode 100644 index 00000000..5d966c3c --- /dev/null +++ b/simulation/simulationtestrules.h @@ -0,0 +1,82 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +#include +#include +#include +#include + +// the rules of the test mode (eu07 --test, eu07/maszyna#541) which need nothing of the running simulation: the exit +// codes, the command line and the end conditions. kept apart so the unit tests can check them +namespace simulation { + +// exit codes of a test run; public, scripts and CI read them +enum class test_result : int { + pass = 0, // __test.result set to 1, then every vehicle came to rest (or test.pass_on_standstill) + fail = 1, // __test.result set to 0 + signal_passed_at_danger = 2, // a train went past a stop signal (SetVelocity 0 0), unless test.fail_if_stop_overran no + timeout = 3, // test.timeout seconds of simulation time went by without a result + standstill = 4, // every vehicle stood still for test.standstill seconds without a result + error = 5, // the run could not start (a bad command line, the scenery failed to load) or the simulator threw +}; + +std::string_view to_string(test_result Result); + +// how a test ended, and why +struct test_verdict { + test_result result{test_result::error}; + std::string reason; +}; + +// the test mode options of the command line +struct test_command_line { + bool enabled{false}; // --test + std::optional timeout; // --test-timeout + std::string error; // what's wrong with them; empty if nothing +}; + +// seconds given on the command line: a number, not negative. nothing for anything else +std::optional parse_seconds(std::string_view Text); + +// picks the test mode options out of a command line (the first argument is the program). the values of the other +// options are skipped, so a scenery named like an option isn't taken for one; options it doesn't know are left alone +test_command_line parse_test_command_line(std::span Arguments); + +// the limits and switches the end conditions go by +struct test_limits { + double timeout{3600.0}; // [s] of simulation time; 0 = never + double standstill{300.0}; // [s] of every vehicle standing without a result; 0 = never + bool pass_on_standstill{false}; + bool no_movement_before_pass{true}; +}; + +// what a frame of the simulation brought +struct test_frame { + double deltatime{0.0}; // [s] of simulation time + std::optional reported; // value of the result cell, if there is one + std::string_view reason; // text of the result cell + bool atrest{false}; // every vehicle stands +}; + +// the state of a test between frames +struct test_progress { + double time{0.0}; // simulation time since the start [s] + double resttime{0.0}; // how long every vehicle has been standing [s] + bool passrequested{false}; // __test.result was set to 1, the pass waits for the vehicles to stop +}; + +// how long everything has to stand before a requested pass is granted [s] +inline constexpr double pass_settle_time{1.0}; + +// checks the end conditions after a frame and moves the progress on. returns the verdict once the test is over +std::optional evaluate_frame(test_progress &Progress, test_frame const &Frame, test_limits const &Limits); + +} // namespace simulation diff --git a/tests/unit/test_test_mode.cpp b/tests/unit/test_test_mode.cpp new file mode 100644 index 00000000..fec475c5 --- /dev/null +++ b/tests/unit/test_test_mode.cpp @@ -0,0 +1,195 @@ +#include "stdafx.h" +#include "doctest.h" + +#include "simulation/simulationtestrules.h" + +#include + +// the test mode of the simulator (eu07 --test, eu07/maszyna#541): its command line and end conditions + +namespace { + +using simulation::test_result; + +simulation::test_command_line parse(std::vector Arguments) +{ + Arguments.insert(Arguments.begin(), "eu07"); + return simulation::parse_test_command_line(Arguments); +} + +// a frame of the default 0.03 s +simulation::test_frame frame(bool const Atrest, std::optional const Reported = std::nullopt, std::string_view const Reason = {}) +{ + return simulation::test_frame{0.03, Reported, Reason, Atrest}; +} + +// runs frames until the test ends or Seconds of simulation time go by +std::optional run(simulation::test_progress &Progress, simulation::test_limits const &Limits, double const Seconds, + simulation::test_frame const &Frame) +{ + while (Progress.time < Seconds) + { + if (auto verdict{simulation::evaluate_frame(Progress, Frame, Limits)}) + return verdict; + } + return std::nullopt; +} + +} // namespace + +TEST_CASE("test mode: seconds on the command line are a number, not negative") +{ + CHECK(simulation::parse_seconds("240") == 240.0); + CHECK(simulation::parse_seconds("0.5") == 0.5); + CHECK(simulation::parse_seconds("0") == 0.0); + CHECK_FALSE(simulation::parse_seconds("").has_value()); + CHECK_FALSE(simulation::parse_seconds("-1").has_value()); + CHECK_FALSE(simulation::parse_seconds("60s").has_value()); + CHECK_FALSE(simulation::parse_seconds("abc").has_value()); + CHECK_FALSE(simulation::parse_seconds("inf").has_value()); + CHECK_FALSE(simulation::parse_seconds("nan").has_value()); +} + +TEST_CASE("test mode: the command line") +{ + SUBCASE("without --test there is no test") + { + auto const commandline{parse({"-s", "td.scn", "-v", "ep07-424"})}; + CHECK_FALSE(commandline.enabled); + CHECK(commandline.error.empty()); + } + SUBCASE("--test and its limit, in any order") + { + auto const commandline{parse({"-s", "td.scn", "--test-timeout", "240", "--test"})}; + CHECK(commandline.enabled); + CHECK(commandline.timeout == 240.0); + CHECK(commandline.error.empty()); + } + SUBCASE("a limit not given stays unset") + { + auto const commandline{parse({"--test", "-s", "td.scn"})}; + CHECK_FALSE(commandline.timeout.has_value()); + } + SUBCASE("a limit which isn't a number is an error, not a limit turned off") + { + auto const commandline{parse({"--test", "-s", "td.scn", "--test-timeout", "4m"})}; + CHECK(commandline.enabled); + CHECK_FALSE(commandline.timeout.has_value()); + CHECK(commandline.error == "--test-timeout needs a number of seconds"); + } + SUBCASE("a limit without its value is an error") + { + auto const commandline{parse({"--test", "-s", "td.scn", "--test-timeout"})}; + CHECK(commandline.error == "--test-timeout needs a number of seconds"); + } + SUBCASE("values of other options aren't taken for options") + { + auto const commandline{parse({"-s", "--test", "-ini", "--test-timeout", "-v", "--test", "-seed", "--test"})}; + CHECK_FALSE(commandline.enabled); + CHECK(commandline.error.empty()); + } + SUBCASE("the scenery of -edit is optional") + { + CHECK(parse({"-edit", "--test", "-s", "td.scn"}).enabled); + CHECK_FALSE(parse({"-edit", "td.scn"}).enabled); + } +} + +TEST_CASE("test mode: the end conditions") +{ + simulation::test_limits limits; + simulation::test_progress progress; + + SUBCASE("nothing happens, nothing ends") + { + CHECK_FALSE(simulation::evaluate_frame(progress, frame(false), limits).has_value()); + CHECK(progress.time == doctest::Approx(0.03)); + CHECK(progress.resttime == 0.0); + } + SUBCASE("a cell nobody wrote to says nothing") + { + CHECK_FALSE(simulation::evaluate_frame(progress, frame(false, -1.0), limits).has_value()); + CHECK_FALSE(simulation::evaluate_frame(progress, frame(false, 2.0), limits).has_value()); + CHECK_FALSE(progress.passrequested); + } + SUBCASE("0 fails at once, with the text of the cell") + { + auto const verdict{simulation::evaluate_frame(progress, frame(false, 0.0, "wrong_track"), limits)}; + REQUIRE(verdict.has_value()); + CHECK(verdict->result == test_result::fail); + CHECK(verdict->reason == "__test.result set to 0 (wrong_track)"); + } + SUBCASE("1 passes once every vehicle has stood for a second") + { + CHECK_FALSE(simulation::evaluate_frame(progress, frame(false, 1.0), limits).has_value()); + CHECK(progress.passrequested); + // still moving: the pass waits + CHECK_FALSE(run(progress, limits, 30.0, frame(false, 1.0)).has_value()); + auto const stoppedat{progress.time}; + auto const verdict{run(progress, limits, 60.0, frame(true, 1.0))}; + REQUIRE(verdict.has_value()); + CHECK(verdict->result == test_result::pass); + CHECK(progress.time - stoppedat == doctest::Approx(simulation::pass_settle_time).epsilon(0.05)); + } + SUBCASE("the pass stays requested when the cell changes back") + { + CHECK_FALSE(simulation::evaluate_frame(progress, frame(false, 1.0), limits).has_value()); + auto const verdict{run(progress, limits, 60.0, frame(true, -1.0))}; + REQUIRE(verdict.has_value()); + CHECK(verdict->result == test_result::pass); + } + SUBCASE("1 with test.no_movement_before_pass no passes at once") + { + limits.no_movement_before_pass = false; + auto const verdict{simulation::evaluate_frame(progress, frame(false, 1.0, "reached_the_stop"), limits)}; + REQUIRE(verdict.has_value()); + CHECK(verdict->result == test_result::pass); + CHECK(verdict->reason == "__test.result set to 1 (reached_the_stop)"); + } + SUBCASE("standing still without a result ends after test.standstill") + { + limits.standstill = 120.0; + auto const verdict{run(progress, limits, 1000.0, frame(true))}; + REQUIRE(verdict.has_value()); + CHECK(verdict->result == test_result::standstill); + CHECK(progress.time == doctest::Approx(120.0).epsilon(0.001)); + } + SUBCASE("a move starts the standstill over") + { + limits.standstill = 120.0; + CHECK_FALSE(run(progress, limits, 100.0, frame(true)).has_value()); + CHECK_FALSE(simulation::evaluate_frame(progress, frame(false), limits).has_value()); + CHECK(progress.resttime == 0.0); + CHECK_FALSE(run(progress, limits, 200.0, frame(true)).has_value()); + } + SUBCASE("test.pass_on_standstill makes the standstill a pass") + { + limits.standstill = 10.0; + limits.pass_on_standstill = true; + auto const verdict{run(progress, limits, 1000.0, frame(true))}; + REQUIRE(verdict.has_value()); + CHECK(verdict->result == test_result::pass); + } + SUBCASE("test.standstill 0 never ends the test") + { + limits.standstill = 0.0; + limits.timeout = 0.0; + CHECK_FALSE(run(progress, limits, 5000.0, frame(true)).has_value()); + } + SUBCASE("no result by test.timeout") + { + limits.timeout = 240.0; + auto const verdict{run(progress, limits, 1000.0, frame(false))}; + REQUIRE(verdict.has_value()); + CHECK(verdict->result == test_result::timeout); + CHECK(verdict->reason == "no result after 240 s"); + CHECK(progress.time == doctest::Approx(240.0).epsilon(0.001)); + } + SUBCASE("a result in the last frame wins over the timeout") + { + limits.timeout = 0.03; + auto const verdict{simulation::evaluate_frame(progress, frame(false, 0.0), limits)}; + REQUIRE(verdict.has_value()); + CHECK(verdict->result == test_result::fail); + } +} From baf6bc2a2080a2c6a691b4e0e119e346a15d6603 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 11:21:30 +0100 Subject: [PATCH 20/28] feat(simulation): the test mode's exit codes keep clear of 1 and 3 1 is what a process killed with TerminateProcess (and the plain failure of most programs) ends with, 3 what abort() ends with on Windows, so a fail or a timeout couldn't be told from a run that was killed or blew up. The codes other than pass now are 10 fail, 11 signal passed at danger, 12 timeout, 13 standstill and 14 error: none of them a code a process gets on its own, neither the shells' 126-127 nor 128 + a signal number on Linux. pass stays 0. --- simulation/simulationtestrules.h | 14 ++++++++------ tests/scenarios/README.md | 19 +++++++++++-------- tests/scenarios/run.py | 12 ++++++------ tests/unit/test_test_mode.cpp | 24 +++++++++++++++++++++++- 4 files changed, 48 insertions(+), 21 deletions(-) diff --git a/simulation/simulationtestrules.h b/simulation/simulationtestrules.h index 5d966c3c..99d35db7 100644 --- a/simulation/simulationtestrules.h +++ b/simulation/simulationtestrules.h @@ -18,14 +18,16 @@ http://mozilla.org/MPL/2.0/. // codes, the command line and the end conditions. kept apart so the unit tests can check them namespace simulation { -// exit codes of a test run; public, scripts and CI read them +// exit codes of a test run; public, scripts and CI read them. none of them is a code a process commonly ends with +// when it goes wrong on its own: 1 (killed with TerminateProcess, or a plain failure of most programs), 3 (abort() +// on Windows), 126-127 (a shell that can't run the program) or 128 + a signal number (killed, on Linux) enum class test_result : int { pass = 0, // __test.result set to 1, then every vehicle came to rest (or test.pass_on_standstill) - fail = 1, // __test.result set to 0 - signal_passed_at_danger = 2, // a train went past a stop signal (SetVelocity 0 0), unless test.fail_if_stop_overran no - timeout = 3, // test.timeout seconds of simulation time went by without a result - standstill = 4, // every vehicle stood still for test.standstill seconds without a result - error = 5, // the run could not start (a bad command line, the scenery failed to load) or the simulator threw + fail = 10, // __test.result set to 0 + signal_passed_at_danger = 11, // a train went past a stop signal (SetVelocity 0 0), unless test.fail_if_stop_overran no + timeout = 12, // test.timeout seconds of simulation time went by without a result + standstill = 13, // every vehicle stood still for test.standstill seconds without a result + error = 14, // the run could not start (a bad command line, the scenery failed to load) or the simulator threw }; std::string_view to_string(test_result Result); diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index 8bd6f1a4..acab07e7 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -62,13 +62,16 @@ The test ends with the first of these: | exit code | result | when | | --- | --- | --- | | 0 | pass | `__test.result` was set to 1 and then every vehicle stood still for 1 s, so a train still moving towards a signal at stop has to stop first (with `test.no_movement_before_pass no` the pass ends the test at once). Also the standstill end with `test.pass_on_standstill yes` | -| 1 | fail | `__test.result` was set to 0; ends at once | -| 2 | signal passed at danger | a train went past a signal showing stop (`SetVelocity 0 0`; `SetVelocity 0 x` on automatic block, which may be passed after a stop, doesn't count); ends at once, unless `test.fail_if_stop_overran no` | -| 3 | timeout | `test.timeout` seconds of simulation time went by (default 3600) | -| 4 | standstill | every vehicle stood still for `test.standstill` seconds (default 300) and nothing reported a result. Trains waiting at a station or for a signal count as standing, so the limit has to be longer than such waits | -| 5 | error | the run could not start: no `-s`, the scenery failed to load or has no vehicles, or the simulator threw | +| 10 | fail | `__test.result` was set to 0; ends at once | +| 11 | signal passed at danger | a train went past a signal showing stop (`SetVelocity 0 0`; `SetVelocity 0 x` on automatic block, which may be passed after a stop, doesn't count); ends at once, unless `test.fail_if_stop_overran no` | +| 12 | timeout | `test.timeout` seconds of simulation time went by (default 3600) | +| 13 | standstill | every vehicle stood still for `test.standstill` seconds (default 300) and nothing reported a result. Trains waiting at a station or for a signal count as standing, so the limit has to be longer than such waits | +| 14 | error | the run could not start: no `-s`, the scenery failed to load or has no vehicles, or the simulator threw | -A crash ends with the system's code instead (on Windows e.g. `0xC0000005`, -1073741819). +A crash ends with the system's code instead (on Windows e.g. `0xC0000005`, -1073741819). The +codes of the test mode keep clear of the ones a process gets when it goes wrong on its own: 1 +(killed with `TerminateProcess`, or a plain failure of most programs), 3 (`abort()` on Windows), +126-127 (a shell that can't run the program) and 128 + a signal number (killed, on Linux). The limits go in a `config` block of the scenery: @@ -180,8 +183,8 @@ and slides past the signal, so those cases set `test.fail_if_stop_overran no`. on `tor1k`, the track in front of the signal. The `test_mode_*` cases show the ends of the test mode: `test_mode_pass` (exit 0, the pass -event), `test_mode_signal_passed_at_danger` (exit 2), `test_mode_timeout` (exit 3, from -`test.timeout`) and `test_mode_standstill` (exit 4, from `test.standstill`). +event), `test_mode_signal_passed_at_danger` (exit 11), `test_mode_timeout` (exit 12, from +`test.timeout`) and `test_mode_standstill` (exit 13, from `test.standstill`). `test_mode_pass_while_moving` reports the pass from `event2` of `tor1k` (the direction the train runs), as the train runs onto it, with `test.no_movement_before_pass no`, so the test ends with the train still moving. diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index 210d6c01..886e2e90 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -52,14 +52,14 @@ COL_COMMAND = 16 # text, read as None COL_WHEELSLIP = 20 COL_VEHICLE_VELOCITY = 25 # signed vehicle speed, km/h; only in builds which log it -# exit codes of the simulator's test mode (eu07 --test), see simulation/simulationtest.h +# exit codes of the simulator's test mode (eu07 --test), see simulation/simulationtestrules.h EXIT_RESULTS = { 0: "pass", - 1: "fail", - 2: "signal_passed_at_danger", - 3: "timeout", - 4: "standstill", - 5: "error", + 10: "fail", + 11: "signal_passed_at_danger", + 12: "timeout", + 13: "standstill", + 14: "error", } # settings layered over the user's eu07.ini for a test run; the window ones only matter diff --git a/tests/unit/test_test_mode.cpp b/tests/unit/test_test_mode.cpp index fec475c5..609b16ff 100644 --- a/tests/unit/test_test_mode.cpp +++ b/tests/unit/test_test_mode.cpp @@ -3,9 +3,10 @@ #include "simulation/simulationtestrules.h" +#include #include -// the test mode of the simulator (eu07 --test, eu07/maszyna#541): its command line and end conditions +// the test mode of the simulator (eu07 --test, eu07/maszyna#541): its exit codes, command line and end conditions namespace { @@ -37,6 +38,27 @@ std::optional run(simulation::test_progress &Progress, } // namespace +TEST_CASE("test mode: the exit codes keep clear of the ones of a killed or aborted process") +{ + std::array const results{test_result::pass, test_result::fail, test_result::signal_passed_at_danger, test_result::timeout, test_result::standstill, test_result::error}; + for (auto const result : results) + { + auto const code{static_cast(result)}; + CAPTURE(code); + CHECK(code != 1); // TerminateProcess, most programs' failure + CHECK(code != 3); // abort() on Windows + CHECK(code < 126); // shells, signals on Linux + CHECK(simulation::to_string(result) != "unknown"); + } + // the codes are public: scripts and CI read them + CHECK(static_cast(test_result::pass) == 0); + CHECK(static_cast(test_result::fail) == 10); + CHECK(static_cast(test_result::signal_passed_at_danger) == 11); + CHECK(static_cast(test_result::timeout) == 12); + CHECK(static_cast(test_result::standstill) == 13); + CHECK(static_cast(test_result::error) == 14); +} + TEST_CASE("test mode: seconds on the command line are a number, not negative") { CHECK(simulation::parse_seconds("240") == 240.0); From 7f8892cda6f02facd364d093992c48060bd1db09 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 11:21:57 +0100 Subject: [PATCH 21/28] fix(simulation): a test doesn't read the player's own eu07.ini The simulator reads eu07.ini from %APPDATA%/MaSzyna (~/.config/MaSzyna on Linux) when there is one, in place of the game folder's. A test run then depended on whose machine it ran on. With --test only the game folder's eu07.ini is read, then the files given with -ini. The test mode options are taken from the command line before the settings for that; --test-timeout still wins over test.timeout of any settings file. --- application/application.cpp | 24 +++++++++++++----------- tests/scenarios/README.md | 3 +++ tests/scenarios/run.py | 3 ++- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/application/application.cpp b/application/application.cpp index f3c3fe46..638eacdc 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -1200,17 +1200,6 @@ int eu07_application::init_settings(int Argc, char *Argv[]) { Global.asVersion = VERSION_INFO; - fs::path iniPath = user_config_path("eu07.ini"); - - if (!iniPath.empty() && fs::exists(iniPath)) - { - Global.LoadIniFile(iniPath.string().c_str()); - } - else - { - Global.LoadIniFile("eu07.ini"); - } - // a test run is known before the rest of the command line is read, so a command line it can't take // ends with the error code of the test, not with 0, which would pass it auto const testcommandline{simulation::parse_test_command_line(std::vector(Argv, Argv + Argc))}; @@ -1226,6 +1215,19 @@ int eu07_application::init_settings(int Argc, char *Argv[]) Global.test.timeout_from_command_line = true; } + fs::path iniPath = user_config_path("eu07.ini"); + + // a test reads the settings of the game folder only (and those given with -ini), not the player's own + // (%APPDATA%/MaSzyna, ~/.config/MaSzyna), so it runs the same for whoever runs it + if (false == Global.test.enabled && !iniPath.empty() && fs::exists(iniPath)) + { + Global.LoadIniFile(iniPath.string().c_str()); + } + else + { + Global.LoadIniFile("eu07.ini"); + } + // process command line arguments bool sceneryfromcommandline{false}; bool editwithoutscenery{false}; diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index acab07e7..ca45dc34 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -19,6 +19,9 @@ eu07 --test -s [--test-timeout ] [-seed ] [- With `--test` the simulator: - needs `-s`; `-v` is ignored and every train is driven by AI (ghostview), +- reads the settings of the game folder's `eu07.ini` and of the files given with `-ini` (in that + order), not the player's own `eu07.ini` in `%APPDATA%\MaSzyna` (`~/.config/MaSzyna` on Linux), + so a test runs the same for whoever runs it, - draws nothing: the null renderer loads the 3D models (the physics needs some of their data) but no textures, and the window stays hidden, - has no sound, no network, no python, no pause (not even one set by the scenery), and diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index 886e2e90..9cc6f84a 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -62,7 +62,8 @@ EXIT_RESULTS = { 14: "error", } -# settings layered over the user's eu07.ini for a test run; the window ones only matter +# settings layered over the game's eu07.ini for a test run (test mode doesn't read the player's own +# in %APPDATA%); the window ones only matter # with --watch, test mode keeps its window hidden and turns the sound off by itself TEST_INI = """\ fullscreen no From 1626df2bc49945680f596395f06f81256edd2130 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 11:22:37 +0100 Subject: [PATCH 22/28] fix(simulation): a scenery's config block can't turn back on what a test keeps off The config block of a scenery is read like the settings files, after the test mode settings were applied, so a scenery with soundenabled, multiplayer, execonexit, feedbackmode or the like in it got them back in a test run. The test mode settings now are applied again after each config block of the scenery; apply_test_mode_settings() moved to simulationtest.cpp for that. --- application/application.cpp | 52 +----------------------- simulation/simulationstateserializer.cpp | 5 +++ simulation/simulationtest.cpp | 46 +++++++++++++++++++++ simulation/simulationtest.h | 5 +++ tests/scenarios/README.md | 2 + 5 files changed, 59 insertions(+), 51 deletions(-) diff --git a/application/application.cpp b/application/application.cpp index 638eacdc..d0ef6a01 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -31,9 +31,6 @@ http://mozilla.org/MPL/2.0/. #include "utilities/dictionary.h" #include "version_info.h" #include "simulation/simulationtest.h" -#ifdef _WIN32 -#include "Console.h" -#endif #include #include #include @@ -1149,53 +1146,6 @@ void eu07_application::init_files() } namespace fs = std::filesystem; -namespace -{ - -// the settings of the test mode (--test), over whatever the settings files say -void apply_test_mode_settings() -{ - // nobody watches nor listens: no player vehicle, nothing drawn, no sound, no window in the way - Global.editor_session = false; - Global.local_start_vehicle = "ghostview"; - Global.GfxRenderer = "null"; - Global.NvRenderer = false; - Global.bSoundEnabled = false; - Global.python_enabled = false; - Global.bInactivePause = false; - Global.bFullScreen = false; - Global.fullscreen_windowed = false; - Global.VSync = false; - Global.iMultisampling = 0; - Global.gfx_usegles = false; - Global.minframetime = std::chrono::duration(0.0f); - Global.extra_viewports.clear(); - Global.headtrack_conf.joy.clear(); - // a test runs on its own: no network, and no input from outside (eu07/maszyna#541) - Global.network_servers.clear(); - Global.network_client.reset(); - Global.InputGamepad = false; - Global.InputMouse = false; - Global.uart_conf.enable = false; - Global.motiontelemetry_conf.enable = false; -#ifdef WITH_ZMQ - Global.zmq_address.clear(); -#endif - // nor anything to the outside: no keyboard leds, LPT or PoKeys desk (feedbackmode), no messages to a - // multiplayer server (multiplayer, which also holds some events back for it), no program run at the end - Global.iFeedbackMode = 0; -#ifdef _WIN32 - Console::ModeSet(Global.iFeedbackMode, Global.iFeedbackPort); -#endif - Global.iMultiplayer = 0; - Global.exec_on_exit.clear(); - // and the same way every time, unless given another seed (-seed, or randomseed in the settings) - if (!Global.random_seed) - Global.random_seed = seed_of("test"); -} - -} // namespace - int eu07_application::init_settings(int Argc, char *Argv[]) { Global.asVersion = VERSION_INFO; @@ -1315,7 +1265,7 @@ int eu07_application::init_settings(int Argc, char *Argv[]) simulation::test().conclude(simulation::test_result::error, "--test needs a scenery given with -s"); return -1; } - apply_test_mode_settings(); + simulation::apply_test_mode_settings(); } return 0; diff --git a/simulation/simulationstateserializer.cpp b/simulation/simulationstateserializer.cpp index 664f3b95..24470007 100644 --- a/simulation/simulationstateserializer.cpp +++ b/simulation/simulationstateserializer.cpp @@ -15,6 +15,7 @@ http://mozilla.org/MPL/2.0/. #include "simulation/simulationtime.h" #include "simulation/simulationsounds.h" #include "simulation/simulationenvironment.h" +#include "simulation/simulationtest.h" #include "scene/scenenodegroups.h" #include "scene/scenelayers.h" #include "scene/sceneterrain.h" @@ -363,6 +364,10 @@ state_serializer::deserialize_config( cParser &Input, scene::scratch_data &Scrat // config parameters (re)definition Global.ConfigParse( Input ); + if( Global.test.enabled ) { + // a scenery doesn't get to turn on what a test keeps off + simulation::apply_test_mode_settings(); + } } void diff --git a/simulation/simulationtest.cpp b/simulation/simulationtest.cpp index 76aa1be3..67e05ad3 100644 --- a/simulation/simulationtest.cpp +++ b/simulation/simulationtest.cpp @@ -15,11 +15,16 @@ http://mozilla.org/MPL/2.0/. #include "simulation/simulation.h" #include "utilities/Globals.h" #include "utilities/Logs.h" +#include "utilities/utilities.h" #include "vehicle/DynObj.h" #include "world/Event.h" #include "world/MemCell.h" +#ifdef _WIN32 +#include "Console.h" +#endif #include +#include #include #include #include @@ -33,6 +38,47 @@ test_manager &test() return instance; } +void apply_test_mode_settings() +{ + // nobody watches nor listens: no player vehicle, nothing drawn, no sound, no window in the way + Global.editor_session = false; + Global.local_start_vehicle = "ghostview"; + Global.GfxRenderer = "null"; + Global.NvRenderer = false; + Global.bSoundEnabled = false; + Global.python_enabled = false; + Global.bInactivePause = false; + Global.bFullScreen = false; + Global.fullscreen_windowed = false; + Global.VSync = false; + Global.iMultisampling = 0; + Global.gfx_usegles = false; + Global.minframetime = std::chrono::duration(0.0f); + Global.extra_viewports.clear(); + Global.headtrack_conf.joy.clear(); + // a test runs on its own: no network, and no input from outside (eu07/maszyna#541) + Global.network_servers.clear(); + Global.network_client.reset(); + Global.InputGamepad = false; + Global.InputMouse = false; + Global.uart_conf.enable = false; + Global.motiontelemetry_conf.enable = false; +#ifdef WITH_ZMQ + Global.zmq_address.clear(); +#endif + // nor anything to the outside: no keyboard leds, LPT or PoKeys desk (feedbackmode), no messages to a + // multiplayer server (multiplayer, which also holds some events back for it), no program run at the end + Global.iFeedbackMode = 0; +#ifdef _WIN32 + Console::ModeSet(Global.iFeedbackMode, Global.iFeedbackPort); +#endif + Global.iMultiplayer = 0; + Global.exec_on_exit.clear(); + // and the same way every time, unless given another seed (-seed, or randomseed in the settings) + if (!Global.random_seed) + Global.random_seed = seed_of("test"); +} + void test_manager::init_result_cell() const { auto cell { std::make_unique(scene::node_data{0, -1, std::string(result_cell), "memcell"}) }; diff --git a/simulation/simulationtest.h b/simulation/simulationtest.h index 66384b6e..e67b2ada 100644 --- a/simulation/simulationtest.h +++ b/simulation/simulationtest.h @@ -53,4 +53,9 @@ private: // the test of the run test_manager &test(); +// the settings of the test mode, over whatever the settings files and the scenery say: applied once the command +// line is read, and again after each config block of the scenery, which could turn the sound, the network or the +// program run at the end back on +void apply_test_mode_settings(); + } // namespace simulation diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index ca45dc34..006a0fc6 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -32,6 +32,8 @@ With `--test` the simulator: - gives nothing to the outside either: no keyboard leds, LPT or PoKeys (`feedbackmode`), no messages to a multiplayer server (`multiplayer`), no program run at the end (`execonexit`), no console window of its own (`showsystemconsole`), and the game's `imgui.ini` stays as it is, +- keeps all of the above whatever the `config` block of the scenery says: the settings of the + test mode are applied again after it, - runs the simulation as fast as it can in fixed steps of 0.03 s of simulation time per frame (three 10 ms physics steps), whatever the real time, the time speed or the machine, - seeds its random numbers with a fixed seed unless `-seed` or `randomseed` gives one, so two From 14d25520f3e056b9279c0458e7a1487bc4fb2bea Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 11:23:02 +0100 Subject: [PATCH 23/28] fix(simulation): a test's standard output carries its result only The log was printed in colour to stdout too (debuglog bit 2), so a script reading the verdict of a test got the whole log with it. In test mode the log goes to the log file only and stdout has the `test: ...` line alone; the usage text of a bad command line goes to stderr. Whether the run is a test is now known before the log service starts. --- application/application.cpp | 11 +++++++---- tests/scenarios/README.md | 3 ++- utilities/Logs.cpp | 4 ++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/application/application.cpp b/application/application.cpp index d0ef6a01..4fe0911a 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -322,6 +322,9 @@ int eu07_application::init(int Argc, char *Argv[]) { int result{0}; + // a test run is known before anything is logged: whoever started it reads its result on the standard output, + // which the log isn't written to then (eu07/maszyna#541) + Global.test.enabled = simulation::parse_test_command_line(std::vector(Argv, Argv + Argc)).enabled; // start logging service std::thread sLoggingService(LogService); Global.threads.emplace("LogService", std::move(sLoggingService)); @@ -1150,10 +1153,9 @@ int eu07_application::init_settings(int Argc, char *Argv[]) { Global.asVersion = VERSION_INFO; - // a test run is known before the rest of the command line is read, so a command line it can't take - // ends with the error code of the test, not with 0, which would pass it + // a test run is known before the rest of the command line is read (Global.test.enabled, see init()), so a + // command line it can't take ends with the error code of the test, not with 0, which would pass it auto const testcommandline{simulation::parse_test_command_line(std::vector(Argv, Argv + Argc))}; - Global.test.enabled = testcommandline.enabled; if (false == testcommandline.error.empty()) { simulation::test().conclude(simulation::test_result::error, testcommandline.error); @@ -1245,7 +1247,8 @@ int eu07_application::init_settings(int Argc, char *Argv[]) } else { - std::cout << "usage: " << std::string(Argv[0]) << " [-s sceneryfilepath]" + // a test's standard output carries its result only + (Global.test.enabled ? std::cerr : std::cout) << "usage: " << std::string(Argv[0]) << " [-s sceneryfilepath]" << " [-v vehiclename]" << " [-edit [sceneryfilepath]]" << " [-seed number|word]" diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index 006a0fc6..d3f25678 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -40,7 +40,8 @@ With `--test` the simulator: runs of a scenery come out the same (the physicslogs of two runs are byte for byte equal), - quits when the test ends, and says how it ended in the exit code and in a `test: ...` line on stdout and in the log. A command line it can't take, or a start that fails, ends - the same way with the `error` code. + the same way with the `error` code. That line is all there is on stdout: the log goes to + the log file only (`debuglog` doesn't print it on the console in test mode). On Windows the simulator is a windowed program, which the `cmd` prompt and PowerShell don't wait for. In a `.bat` file `eu07 --test ... > result.txt` waits and sets `%errorlevel%`; at the prompt diff --git a/utilities/Logs.cpp b/utilities/Logs.cpp index 38094d72..e0949aba 100644 --- a/utilities/Logs.cpp +++ b/utilities/Logs.cpp @@ -125,8 +125,8 @@ void LogService() if (log_scrollback.size() > 200) log_scrollback.pop_front(); - // log to console - if (Global.iWriteLogEnabled & 2) + // log to console; not in test mode, where the standard output carries the result of the test only + if ((Global.iWriteLogEnabled & 2) && false == Global.test.enabled) { if (isError) printf("\033[1;37;41m%s\033[0m\n", msg.c_str()); From 03a417c7b0b4307546bec78a9cf0321abd573f17 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 11:33:36 +0100 Subject: [PATCH 24/28] feat(simulation): a real time watchdog ends a test run that hangs A test run that hung, in the loading or anywhere else, waited until whoever started it killed it. A watchdog thread now ends a run still going test.wall_timeout seconds of real time after the start of the simulator, loading included: it says `test: timeout ... still running after N s of real time` and quits with the timeout code, whatever the main thread is doing, with no dialog. A test that has ended but whose simulator doesn't close ends with its own result. The limit is test.wall_timeout (settings, the scenery's config block) or --test-wall-timeout, which wins. Not given, it is test.timeout + 600 s: ten minutes for the loading, and a simulation that runs at least as fast as real time is never cut short. test.timeout 0 means no watchdog unless the limit is given; test.wall_timeout 0 turns it off. run.py passes the case's timeout as --test-wall-timeout and kills a simulator only 30 s after it. A timeout from the watchdog counts as wall_timeout there, so a run that hung doesn't pass a case expecting the simulation time to run out; test_mode_wall_timeout ends while loading on a limit of 2 s. --- application/application.cpp | 9 ++- simulation/simulationtest.cpp | 73 +++++++++++++++++-- simulation/simulationtest.h | 17 ++++- simulation/simulationtestrules.cpp | 14 +++- simulation/simulationtestrules.h | 13 +++- tests/scenarios/README.md | 36 ++++++--- .../cases/test_mode_wall_timeout.json | 11 +++ tests/scenarios/run.py | 23 ++++-- tests/unit/test_test_mode.cpp | 34 ++++++++- utilities/Globals.cpp | 12 +++ utilities/Globals.h | 3 + 11 files changed, 210 insertions(+), 35 deletions(-) create mode 100644 tests/scenarios/cases/test_mode_wall_timeout.json diff --git a/application/application.cpp b/application/application.cpp index 4fe0911a..860b8a23 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -1166,6 +1166,11 @@ int eu07_application::init_settings(int Argc, char *Argv[]) Global.test.timeout = *testcommandline.timeout; Global.test.timeout_from_command_line = true; } + if (testcommandline.wall_timeout.has_value()) + { + Global.test.wall_timeout = testcommandline.wall_timeout; + Global.test.wall_timeout_from_command_line = true; + } fs::path iniPath = user_config_path("eu07.ini"); @@ -1231,7 +1236,7 @@ int eu07_application::init_settings(int Argc, char *Argv[]) // integration test: the scenery runs unattended and the exit code is the result (eu07/maszyna#541), // taken from the command line above } - else if (token == "--test-timeout") + else if (token == "--test-timeout" || token == "--test-wall-timeout") { // taken from the command line above, with its value ++i; @@ -1253,7 +1258,7 @@ int eu07_application::init_settings(int Argc, char *Argv[]) << " [-edit [sceneryfilepath]]" << " [-seed number|word]" << " [-ini settingsfile]" - << " [--test [--test-timeout seconds]]" << std::endl; + << " [--test [--test-timeout seconds] [--test-wall-timeout seconds]]" << std::endl; simulation::test().conclude(simulation::test_result::error, std::format("unknown command line argument {}", token)); return -1; } diff --git a/simulation/simulationtest.cpp b/simulation/simulationtest.cpp index 67e05ad3..15cdf0e3 100644 --- a/simulation/simulationtest.cpp +++ b/simulation/simulationtest.cpp @@ -27,9 +27,20 @@ http://mozilla.org/MPL/2.0/. #include #include #include +#include #include #include +namespace { + +// [s] of real time since the start of the simulator +double elapsed_wall_time() +{ + return std::chrono::duration(std::chrono::steady_clock::now() - Global.startTimestamp).count(); +} + +} // namespace + namespace simulation { test_manager &test() @@ -77,6 +88,8 @@ void apply_test_mode_settings() // and the same way every time, unless given another seed (-seed, or randomseed in the settings) if (!Global.random_seed) Global.random_seed = seed_of("test"); + // the real time limit, which the scenery may set too + test().watch_wall_clock(wall_clock_limit(Global.test.wall_timeout, Global.test.timeout)); } void test_manager::init_result_cell() const @@ -94,7 +107,10 @@ void test_manager::start() if (m_started) return; m_started = true; - m_progress = test_progress {}; + { + std::scoped_lock lock { m_lock }; + m_progress = test_progress {}; + } if (simulation::Vehicles.sequence().empty()) { // most likely the scenery wasn't found, or it isn't a test scenery @@ -103,7 +119,9 @@ void test_manager::start() } // a pause set by the scenery or the settings would never be lifted, nobody is there to press the key Global.iPause = 0; - WriteLog(std::format("test: started, timeout {:.0f} s, standstill {:.0f} s{}{}{}", Global.test.timeout, Global.test.standstill, + auto const walllimit { m_walllimit.load() }; + WriteLog(std::format("test: started, timeout {:.0f} s, real time limit {}, standstill {:.0f} s{}{}{}", Global.test.timeout, + walllimit > 0.0 ? std::format("{:.0f} s", walllimit) : std::string("none"), Global.test.standstill, Global.test.pass_on_standstill ? ", standstill passes" : "", Global.test.fail_if_stop_overran ? "" : ", signals passed at danger allowed", Global.test.no_movement_before_pass ? "" : ", pass without waiting for a stop")); @@ -122,14 +140,21 @@ void test_manager::update(double const Deltatime) frame.reason = cell->Text(); } - auto const requestedbefore { m_progress.passrequested }; - auto const verdict { evaluate_frame(m_progress, frame, test_limits { Global.test.timeout, Global.test.standstill, Global.test.pass_on_standstill, Global.test.no_movement_before_pass }) }; + auto const limits { test_limits { Global.test.timeout, Global.test.standstill, Global.test.pass_on_standstill, Global.test.no_movement_before_pass } }; + std::optional verdict; + auto passrequested { false }; + { + std::scoped_lock lock { m_lock }; + auto const requestedbefore { m_progress.passrequested }; + verdict = evaluate_frame(m_progress, frame, limits); + passrequested = m_progress.passrequested && false == requestedbefore; + } if (verdict.has_value()) { finish(verdict->result, verdict->reason); return; } - if (m_progress.passrequested && false == requestedbefore) + if (passrequested) { WriteLog(std::format("test: pass requested at {:.1f} s{}, waiting for every vehicle to stop", m_progress.time, frame.reason.empty() ? std::string() : std::format(" ({})", frame.reason))); @@ -160,10 +185,11 @@ void test_manager::finish(test_result const Result, std::string_view const Reaso bool test_manager::conclude(test_result const Result, std::string_view const Reason) { + std::scoped_lock lock { m_lock }; if (m_finished || false == Global.test.enabled) return false; - m_finished = true; m_result = Result; + m_finished = true; auto const line { std::format("test: {} (exit code {}) after {:.1f} s of simulation time: {}", to_string(Result), static_cast(Result), m_progress.time, Reason) }; if (Result == test_result::pass) @@ -176,4 +202,39 @@ bool test_manager::conclude(test_result const Result, std::string_view const Rea return true; } +int test_manager::exit_code() const +{ + std::scoped_lock lock { m_lock }; + return static_cast(m_finished ? m_result : test_result::error); +} + +void test_manager::watch_wall_clock(double const Limit) +{ + m_walllimit = Limit; + if (Limit <= 0.0 || m_watchdog.joinable()) + return; + m_watchdog = std::jthread([this](std::stop_token const &Stop) { + while (false == Stop.stop_requested()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + check_wall_clock(elapsed_wall_time()); + } + }); +} + +void test_manager::check_wall_clock(double const Elapsed) +{ + auto const limit { m_walllimit.load() }; + if (limit <= 0.0 || Elapsed < limit) + return; + // the run hangs, or is too slow: it ends here, whatever the main thread is doing. a run whose test has ended + // but which doesn't close ends with its result + if (false == conclude(test_result::timeout, std::format("still running after {:.0f} s of real time (test.wall_timeout)", limit))) + ErrorLog(std::format("test: the simulator was still running {:.0f} s of real time after the start, ended", limit)); + // what was logged last gets written, unless the log is held up by a hung thread + StopLogService(); + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + std::_Exit(exit_code()); +} + } // namespace simulation diff --git a/simulation/simulationtest.h b/simulation/simulationtest.h index e67b2ada..a59ec87b 100644 --- a/simulation/simulationtest.h +++ b/simulation/simulationtest.h @@ -11,7 +11,10 @@ http://mozilla.org/MPL/2.0/. #include "simulation/simulationtestrules.h" +#include +#include #include +#include // integration test mode (eu07 --test -s , eu07/maszyna#541): the scenery runs unattended in a tight // fixed-step loop and the simulator quits with an exit code saying how the test went @@ -38,16 +41,26 @@ public: // ends the test with given result, without closing the application: for the ways out that close it anyway // (a bad command line, a failed start, an exception). returns false if the test had already ended bool conclude(test_result Result, std::string_view Reason); + // (re)sets the real time limit of the run [s] since the start of the simulator, 0 = none. the first limit + // starts the watchdog, which ends a run still going at the limit with the timeout code, whatever it's doing + void watch_wall_clock(double Limit); bool finished() const { return m_finished; } // exit code of the process: the result, or error while the test hasn't finished - int exit_code() const { return static_cast(m_finished ? m_result : test_result::error); } + int exit_code() const; private: + // the watchdog's check, at Elapsed seconds of real time since the start of the simulator + void check_wall_clock(double Elapsed); + + // the watchdog may end the test from its own thread + mutable std::mutex m_lock; bool m_started { false }; - bool m_finished { false }; + std::atomic m_finished { false }; test_result m_result { test_result::error }; test_progress m_progress; + std::atomic m_walllimit { 0.0 }; // [s] since the start of the simulator, 0 = none + std::jthread m_watchdog; }; // the test of the run diff --git a/simulation/simulationtestrules.cpp b/simulation/simulationtestrules.cpp index 44f740f1..eba1d7dd 100644 --- a/simulation/simulationtestrules.cpp +++ b/simulation/simulationtestrules.cpp @@ -81,7 +81,7 @@ test_command_line parse_test_command_line(std::span const Arg { result.enabled = true; } - else if (option == "--test-timeout") + else if (option == "--test-timeout" || option == "--test-wall-timeout") { auto const seconds{option_seconds(Arguments, i)}; if (false == seconds.has_value()) @@ -91,7 +91,10 @@ test_command_line parse_test_command_line(std::span const Arg result.error = std::format("{} needs a number of seconds", option); continue; } - result.timeout = seconds; + if (option == "--test-timeout") + result.timeout = seconds; + else + result.wall_timeout = seconds; } else if (takes_value(option)) { @@ -106,6 +109,13 @@ test_command_line parse_test_command_line(std::span const Arg return result; } +double wall_clock_limit(std::optional const Given, double const Simulationtimeout) +{ + if (Given.has_value()) + return *Given; + return Simulationtimeout > 0.0 ? Simulationtimeout + wall_timeout_loading_allowance : 0.0; +} + std::optional evaluate_frame(test_progress &Progress, test_frame const &Frame, test_limits const &Limits) { using enum test_result; diff --git a/simulation/simulationtestrules.h b/simulation/simulationtestrules.h index 99d35db7..52adf27a 100644 --- a/simulation/simulationtestrules.h +++ b/simulation/simulationtestrules.h @@ -15,7 +15,7 @@ http://mozilla.org/MPL/2.0/. #include // the rules of the test mode (eu07 --test, eu07/maszyna#541) which need nothing of the running simulation: the exit -// codes, the command line and the end conditions. kept apart so the unit tests can check them +// codes, the command line, the end conditions and the real time limit. kept apart so the unit tests can check them namespace simulation { // exit codes of a test run; public, scripts and CI read them. none of them is a code a process commonly ends with @@ -25,7 +25,7 @@ enum class test_result : int { pass = 0, // __test.result set to 1, then every vehicle came to rest (or test.pass_on_standstill) fail = 10, // __test.result set to 0 signal_passed_at_danger = 11, // a train went past a stop signal (SetVelocity 0 0), unless test.fail_if_stop_overran no - timeout = 12, // test.timeout seconds of simulation time went by without a result + timeout = 12, // test.timeout seconds of simulation time, or test.wall_timeout seconds of real time went by standstill = 13, // every vehicle stood still for test.standstill seconds without a result error = 14, // the run could not start (a bad command line, the scenery failed to load) or the simulator threw }; @@ -42,6 +42,7 @@ struct test_verdict { struct test_command_line { bool enabled{false}; // --test std::optional timeout; // --test-timeout + std::optional wall_timeout; // --test-wall-timeout std::string error; // what's wrong with them; empty if nothing }; @@ -52,6 +53,14 @@ std::optional parse_seconds(std::string_view Text); // options are skipped, so a scenery named like an option isn't taken for one; options it doesn't know are left alone test_command_line parse_test_command_line(std::span Arguments); +// time allowed for loading the scenery, on top of the simulation time, when the real time limit isn't given [s] +inline constexpr double wall_timeout_loading_allowance{600.0}; + +// the real time limit of a test run [s], 0 = none: the one given, else the simulation time limit plus time for the +// loading, so a simulation that runs at least as fast as real time is never cut short. no simulation time limit +// (test.timeout 0) means none in real time either, unless given +double wall_clock_limit(std::optional Given, double Simulationtimeout); + // the limits and switches the end conditions go by struct test_limits { double timeout{3600.0}; // [s] of simulation time; 0 = never diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index d3f25678..5eed983e 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -13,7 +13,7 @@ The runner drives the simulator's test mode (eu07/maszyna#541), which works with runner too and is meant for scenery authors as well: ``` -eu07 --test -s [--test-timeout ] [-seed ] [-ini ] +eu07 --test -s [--test-timeout ] [--test-wall-timeout ] [-seed ] [-ini ] ``` With `--test` the simulator: @@ -70,7 +70,7 @@ The test ends with the first of these: | 0 | pass | `__test.result` was set to 1 and then every vehicle stood still for 1 s, so a train still moving towards a signal at stop has to stop first (with `test.no_movement_before_pass no` the pass ends the test at once). Also the standstill end with `test.pass_on_standstill yes` | | 10 | fail | `__test.result` was set to 0; ends at once | | 11 | signal passed at danger | a train went past a signal showing stop (`SetVelocity 0 0`; `SetVelocity 0 x` on automatic block, which may be passed after a stop, doesn't count); ends at once, unless `test.fail_if_stop_overran no` | -| 12 | timeout | `test.timeout` seconds of simulation time went by (default 3600) | +| 12 | timeout | `test.timeout` seconds of simulation time went by (default 3600), or `test.wall_timeout` seconds of real time since the start (see below) | | 13 | standstill | every vehicle stood still for `test.standstill` seconds (default 300) and nothing reported a result. Trains waiting at a station or for a signal count as standing, so the limit has to be longer than such waits | | 14 | error | the run could not start: no `-s`, the scenery failed to load or has no vehicles, or the simulator threw | @@ -84,6 +84,7 @@ The limits go in a `config` block of the scenery: ``` config test.timeout 600 +test.wall_timeout 900 test.standstill 120 test.pass_on_standstill no test.fail_if_stop_overran yes @@ -94,12 +95,22 @@ endconfig | key | default | meaning | | --- | --- | --- | | `test.timeout` | 3600 | seconds of simulation time before the test ends as `timeout`; 0 = never | +| `test.wall_timeout` | `test.timeout` + 600 | seconds of real time since the start before the test ends as `timeout`; 0 = never | | `test.standstill` | 300 | seconds with every vehicle still and no result before the test ends as `standstill`; 0 = never | | `test.pass_on_standstill` | no | the standstill end passes the test instead | | `test.fail_if_stop_overran` | yes | a signal passed at danger ends the test; with `no` it's only logged | | `test.no_movement_before_pass` | yes | a pass waits until every vehicle stands still; with `no` it ends the test at once | -`--test-timeout` on the command line wins over `test.timeout` of the scenery. +`--test-timeout` and `--test-wall-timeout` on the command line win over `test.timeout` and +`test.wall_timeout` of the scenery and the settings files. + +`test.wall_timeout` is a watchdog in real time, for a run that hangs or crawls: when the +simulator still runs that many seconds after its start, loading included, it says `test: timeout` +and quits with the timeout code, whatever it is doing at the moment, with no dialog. Not given, +it is `test.timeout` + 600 s: ten minutes for the loading, and a simulation that runs at least as +fast as real time is never cut short by it. With `test.timeout 0` there is no watchdog unless +`test.wall_timeout` is given; `test.wall_timeout 0` turns it off. A test that has ended but whose +simulator doesn't close is ended by the watchdog too, with its own result. A signal is passed at danger when the head of a train goes past a signal which its driver has in its speed table, showing stop. So it's the driver's view that counts: a train without @@ -120,7 +131,7 @@ Python 3.10+, no extra packages. Options: `cmake-build-*/bin/Release/eu07*.exe`. The build needs the `--test` and `-ini` command line options. - name one or more cases to run only those, e.g. `run.py ... eu07_td_timetable`. -- `-j N` runs N cases at once. Each case prints its results when it is done. All 15 cases +- `-j N` runs N cases at once. Each case prints its results when it is done. All 16 cases take about 30 s with `-j 4`, loading included. - `--watch` runs a case the old way, in a visible window in real time (or faster with `--timespeed N`), closed after the case's `duration`; the exit code isn't checked then. For @@ -138,13 +149,15 @@ For each case the runner: added, and the case's `.scn` fragment added: its trainsets after `FirstInit`, everything else (the case's `test_config`, event launchers standing in for keys the scenery expects a player to press, extra events) before it, -2. starts the simulator with `--test -s autotest_.scn -ini out//autotest.ini`, - plus `--test-timeout ` when the case gives a `duration`; the extra ini turns on +2. starts the simulator with `--test -s autotest_.scn -ini out//autotest.ini + --test-wall-timeout `, plus `--test-timeout ` when the case gives a + `duration`; the extra ini turns on `physicslog` and writes the logs to `logs/log (autotest_) .txt` rather than the shared `log.txt` (`multiplelogs`), 3. waits until the simulator quits by itself. It gives up early when the log reports - `Bad vehicle` / `Bad trainset`, when the scenario has loaded and no physicslog appeared, or - after the case's `timeout` of wall-clock time, + `Bad vehicle` / `Bad trainset`, or when the scenario has loaded and no physicslog appeared. + After the case's `timeout` of wall-clock time the simulator ends itself with the timeout code; + one that doesn't is killed 30 s later, 4. compares the result from the exit code with the case's `expect`, then runs the checks on the physicslog. @@ -167,9 +180,9 @@ scenery is newer. | `test_config` | lines of a `config` block added to the scenery, e.g. `["test.timeout 90"]` | | `fragment` | `.scn` file in `cases/` added to the scenery: trainsets after `FirstInit` (before it they derail on unlinked track), the rest before it (launchers read after it never fire) | | `vars` | values for `{{name}}` in the fragment; a list becomes one line per item | -| `expect` | the result the test mode should end with: `pass`, `fail`, `signal_passed_at_danger`, `timeout`, `standstill` or `error`. Default `timeout` for a case with `duration`, else `pass` | +| `expect` | the result the test mode should end with: `pass`, `fail`, `signal_passed_at_danger`, `timeout`, `standstill` or `error`, or `wall_timeout` for the timeout of the real time watchdog (the timeout code with a verdict saying it was the real time; a case expecting `timeout` fails on it). Default `timeout` for a case with `duration`, else `pass` | | `duration` | seconds of simulation to record: the test timeout of the run (`--test-timeout`) | -| `timeout` | wall-clock seconds before giving up, loading included | +| `timeout` | wall-clock seconds before giving up, loading included: the simulator's `--test-wall-timeout` (default 600) | | `fail_on_errors` | fail when `errors.txt` is not empty (default `false`) | | `checks` | list of checks, below | @@ -190,7 +203,8 @@ on `tor1k`, the track in front of the signal. The `test_mode_*` cases show the ends of the test mode: `test_mode_pass` (exit 0, the pass event), `test_mode_signal_passed_at_danger` (exit 11), `test_mode_timeout` (exit 12, from -`test.timeout`) and `test_mode_standstill` (exit 13, from `test.standstill`). +`test.timeout`), `test_mode_standstill` (exit 13, from `test.standstill`) and +`test_mode_wall_timeout` (exit 12 from the real time watchdog, which ends it while it loads). `test_mode_pass_while_moving` reports the pass from `event2` of `tor1k` (the direction the train runs), as the train runs onto it, with `test.no_movement_before_pass no`, so the test ends with the train still moving. diff --git a/tests/scenarios/cases/test_mode_wall_timeout.json b/tests/scenarios/cases/test_mode_wall_timeout.json new file mode 100644 index 00000000..078ccca9 --- /dev/null +++ b/tests/scenarios/cases/test_mode_wall_timeout.json @@ -0,0 +1,11 @@ +{ + "description": "Test mode, real time limit: EU07 on td.scn, with a limit of 2 s of real time (the case's timeout, which the runner passes as --test-wall-timeout). The scenery takes longer to load, so the simulator's watchdog ends the run while it loads, with the timeout code and a verdict saying it was the real time.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_red_signal.scn", + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic" + }, + "expect": "wall_timeout", + "timeout": 2 +} diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index 9cc6f84a..62cb7d13 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -61,6 +61,10 @@ EXIT_RESULTS = { 13: "standstill", 14: "error", } +# the timeout of the simulator's real time watchdog (test.wall_timeout) has the timeout code too; told +# apart by its verdict, so a run that hung doesn't pass a case that expects the simulation time to run out +WALL_TIMEOUT = "wall_timeout" +RESULTS = (*EXIT_RESULTS.values(), WALL_TIMEOUT) # settings layered over the game's eu07.ini for a test run (test mode doesn't read the player's own # in %APPDATA%); the window ones only matter @@ -226,6 +230,8 @@ FATAL_LOG_LINES = ("Bad vehicle:", "Bad trainset:") # time" in the log is not the end of loading (the trainsets after FirstInit still follow), # and with several simulators loading at once the first rows took well over 15 s STARTUP_TIMEOUT = 120.0 +# wall seconds past a case's timeout before the runner kills a simulator that didn't end itself +KILL_GRACE = 30.0 @dataclass @@ -246,7 +252,8 @@ def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog if watch: cmd = [str(exe), "-s", scenery, "-v", "ghostview", "-ini", str(ini)] else: - cmd = [str(exe), "--test", "-s", scenery, "-ini", str(ini)] + # the simulator's own watchdog ends a run that takes too long, with the timeout code and its verdict + cmd = [str(exe), "--test", "-s", scenery, "-ini", str(ini), "--test-wall-timeout", f"{timeout:g}"] if duration is not None: cmd += ["--test-timeout", f"{duration:g}"] say(" $ " + " ".join(f'"{c}"' if " " in c else c for c in cmd), progress=True) @@ -272,8 +279,9 @@ def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog break if watch and sim_t >= duration: break - if wall > timeout: - result.failure = f"timed out after {timeout:.0f}s before {duration}s of simulation" + # in test mode a little later than the simulator's own limit, for a run that can't even end itself + if wall > timeout + (0 if watch else KILL_GRACE): + result.failure = f"timed out after {wall:.0f}s before {duration}s of simulation" break if not result.rows: result.logs = find_logs(game_dir, scenery, since) @@ -506,8 +514,8 @@ def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, watch: bool # a case that only records `duration` seconds ends on the test timeout; one that says # how the test ends (a pass event, a signal passed at danger...) expects that result expect = case.get("expect", "timeout" if duration is not None else "pass") - if expect not in EXIT_RESULTS.values(): - sys.exit(f"{name}: expect must be one of {', '.join(EXIT_RESULTS.values())}") + if expect not in RESULTS: + sys.exit(f"{name}: expect must be one of {', '.join(RESULTS)}") if watch and duration is None: sys.exit(f"{name}: --watch needs a case with duration") result = run_simulator(exe, game_dir, scenery, ini, physicslog, @@ -530,8 +538,11 @@ def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, watch: bool if result.verdict: say(f" {result.verdict}") got = EXIT_RESULTS.get(result.exit_code) if result.exit_code is not None else None + if got == "timeout" and "of real time" in result.verdict: + got = WALL_TIMEOUT passed = got == expect - say(f" {'PASS' if passed else 'FAIL'} exit code {describe_exit(result.exit_code)}, expected {expect}") + watchdog = ", from the real time watchdog" if got == WALL_TIMEOUT else "" + say(f" {'PASS' if passed else 'FAIL'} exit code {describe_exit(result.exit_code)}{watchdog}, expected {expect}") ok &= passed if result.failure: say(f" FAIL {result.failure}") diff --git a/tests/unit/test_test_mode.cpp b/tests/unit/test_test_mode.cpp index 609b16ff..55a90387 100644 --- a/tests/unit/test_test_mode.cpp +++ b/tests/unit/test_test_mode.cpp @@ -80,17 +80,19 @@ TEST_CASE("test mode: the command line") CHECK_FALSE(commandline.enabled); CHECK(commandline.error.empty()); } - SUBCASE("--test and its limit, in any order") + SUBCASE("--test and its limits, in any order") { - auto const commandline{parse({"-s", "td.scn", "--test-timeout", "240", "--test"})}; + auto const commandline{parse({"--test-wall-timeout", "900", "-s", "td.scn", "--test", "--test-timeout", "240"})}; CHECK(commandline.enabled); CHECK(commandline.timeout == 240.0); + CHECK(commandline.wall_timeout == 900.0); CHECK(commandline.error.empty()); } SUBCASE("a limit not given stays unset") { auto const commandline{parse({"--test", "-s", "td.scn"})}; CHECK_FALSE(commandline.timeout.has_value()); + CHECK_FALSE(commandline.wall_timeout.has_value()); } SUBCASE("a limit which isn't a number is an error, not a limit turned off") { @@ -101,12 +103,17 @@ TEST_CASE("test mode: the command line") } SUBCASE("a limit without its value is an error") { - auto const commandline{parse({"--test", "-s", "td.scn", "--test-timeout"})}; + auto const commandline{parse({"--test", "-s", "td.scn", "--test-wall-timeout"})}; + CHECK(commandline.error == "--test-wall-timeout needs a number of seconds"); + } + SUBCASE("the first error is the one told") + { + auto const commandline{parse({"--test", "--test-timeout", "x", "--test-wall-timeout", "y"})}; CHECK(commandline.error == "--test-timeout needs a number of seconds"); } SUBCASE("values of other options aren't taken for options") { - auto const commandline{parse({"-s", "--test", "-ini", "--test-timeout", "-v", "--test", "-seed", "--test"})}; + auto const commandline{parse({"-s", "--test", "-ini", "--test-timeout", "-v", "--test-wall-timeout", "-seed", "--test"})}; CHECK_FALSE(commandline.enabled); CHECK(commandline.error.empty()); } @@ -117,6 +124,25 @@ TEST_CASE("test mode: the command line") } } +TEST_CASE("test mode: the real time limit") +{ + SUBCASE("given, it is the limit, 0 turns it off") + { + CHECK(simulation::wall_clock_limit(120.0, 3600.0) == 120.0); + CHECK(simulation::wall_clock_limit(0.0, 3600.0) == 0.0); + } + SUBCASE("not given, it is the simulation time limit and time for the loading") + { + CHECK(simulation::wall_clock_limit(std::nullopt, 240.0) == 240.0 + simulation::wall_timeout_loading_allowance); + CHECK(simulation::wall_clock_limit(std::nullopt, 3600.0) == 3600.0 + simulation::wall_timeout_loading_allowance); + } + SUBCASE("no simulation time limit, no real time limit unless given") + { + CHECK(simulation::wall_clock_limit(std::nullopt, 0.0) == 0.0); + CHECK(simulation::wall_clock_limit(300.0, 0.0) == 300.0); + } +} + TEST_CASE("test mode: the end conditions") { simulation::test_limits limits; diff --git a/utilities/Globals.cpp b/utilities/Globals.cpp index 3a9e1014..217374cb 100644 --- a/utilities/Globals.cpp +++ b/utilities/Globals.cpp @@ -468,6 +468,15 @@ bool global_settings::ConfigParseTest(cParser& Parser, std::string_view const To return true; } + if (Token == "test.wall_timeout") + { + double value{ 0.0 }; + ParseOne(Parser, value); + if (false == test.wall_timeout_from_command_line) + test.wall_timeout = std::max(0.0, value); + return true; + } + if (Token == "test.standstill") { ParseOne(Parser, test.standstill); @@ -1786,6 +1795,9 @@ global_settings::export_as_text( std::ostream &Output ) const { // TODO: export fpslimit export_as_text( Output, "randomseed", Global.random_seed ); export_as_text( Output, "test.timeout", test.timeout ); + if( test.wall_timeout.has_value() ) { + export_as_text( Output, "test.wall_timeout", *test.wall_timeout ); + } export_as_text( Output, "test.standstill", test.standstill ); export_as_text( Output, "test.pass_on_standstill", test.pass_on_standstill ); export_as_text( Output, "test.fail_if_stop_overran", test.fail_if_stop_overran ); diff --git a/utilities/Globals.h b/utilities/Globals.h index 11d14e34..81bc1e47 100644 --- a/utilities/Globals.h +++ b/utilities/Globals.h @@ -16,6 +16,7 @@ http://mozilla.org/MPL/2.0/. #include "utilities/utilities.h" #include "utilities/motiontelemetry.h" #include +#include #ifdef WITH_UART #include "utilities/uart.h" @@ -95,6 +96,8 @@ struct global_settings { bool fail_if_stop_overran{ true }; // a train passing a signal at stop ends the test as failed; with false it's only logged bool no_movement_before_pass{ true }; // a pass waits until every vehicle stands still; with false it ends the test at once bool timeout_from_command_line{ false }; // --test-timeout given, the scenery doesn't override it + std::optional wall_timeout; // [s] of real time since the start; none = test.timeout + time for the loading, 0 = no limit + bool wall_timeout_from_command_line{ false }; // --test-wall-timeout given, the scenery doesn't override it }; test_settings test; bool editor_tracks{ false }; // F3 track workspace: draw and pick tracks without the scenery draw-range cap From e5aafd154f058ae801093fe2032dbe56f3751e3e Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 11:39:52 +0100 Subject: [PATCH 25/28] feat(simulation): the pass and the standstill wait for the train under test only Both waited for every vehicle of the scenery to stand still, so on a scenery with AI traffic a test could pass only when the traffic happened to stand too, and the standstill end hardly ever came. A test scenery names the train under test with test.vehicle in its config block (once for each train, if several): the pass and the standstill wait for the consists of these vehicles as they are at the moment, so a train the tested one couples to counts with it. A name the scenery doesn't have ends the test with the error code. Without test.vehicle every vehicle counts, as before. A signal passed at danger counts for every train either way. run.py puts the case's lead vehicle into test.vehicle. test_mode_standstill_traffic has the train under test standing on track 2 of Stacja1 while AI traffic leaves track 1 on the timetable: it ends on its 100 s standstill, where counting every vehicle runs into the 300 s timeout. Physicslogs of the other trains of a case (_) are moved out of the game dir too. --- simulation/simulationtest.cpp | 38 +++++++++++++++++-- simulation/simulationtest.h | 2 + simulation/simulationtestrules.cpp | 4 +- simulation/simulationtestrules.h | 14 +++---- tests/scenarios/README.md | 28 +++++++++----- tests/scenarios/cases/td_traffic.scn | 13 +++++++ .../scenarios/cases/test_mode_standstill.json | 2 +- .../cases/test_mode_standstill_traffic.json | 18 +++++++++ tests/scenarios/run.py | 10 +++-- tests/unit/test_test_mode.cpp | 2 +- utilities/Globals.cpp | 14 +++++++ utilities/Globals.h | 5 ++- 12 files changed, 120 insertions(+), 30 deletions(-) create mode 100644 tests/scenarios/cases/td_traffic.scn create mode 100644 tests/scenarios/cases/test_mode_standstill_traffic.json diff --git a/simulation/simulationtest.cpp b/simulation/simulationtest.cpp index 15cdf0e3..89a8a104 100644 --- a/simulation/simulationtest.cpp +++ b/simulation/simulationtest.cpp @@ -33,6 +33,11 @@ http://mozilla.org/MPL/2.0/. namespace { +bool is_moving(TDynamicObject const *Vehicle) +{ + return std::abs(Vehicle->GetVelocity()) > 0.01; +} + // [s] of real time since the start of the simulator double elapsed_wall_time() { @@ -117,10 +122,25 @@ void test_manager::start() finish(test_result::error, "the scenery has no vehicles"); return; } + for (auto const &name : Global.test.vehicles) + { + if (simulation::Vehicles.find(name) == nullptr) + { + finish(test_result::error, std::format("the scenery has no vehicle {} (test.vehicle)", name)); + return; + } + } // a pause set by the scenery or the settings would never be lifted, nobody is there to press the key Global.iPause = 0; + auto tested { std::string("every vehicle") }; + if (false == Global.test.vehicles.empty()) + { + tested = "the consists of"; + for (auto const &name : Global.test.vehicles) + tested += " " + name; + } auto const walllimit { m_walllimit.load() }; - WriteLog(std::format("test: started, timeout {:.0f} s, real time limit {}, standstill {:.0f} s{}{}{}", Global.test.timeout, + WriteLog(std::format("test: started, testing {}, timeout {:.0f} s, real time limit {}, standstill {:.0f} s{}{}{}", tested, Global.test.timeout, walllimit > 0.0 ? std::format("{:.0f} s", walllimit) : std::string("none"), Global.test.standstill, Global.test.pass_on_standstill ? ", standstill passes" : "", Global.test.fail_if_stop_overran ? "" : ", signals passed at danger allowed", @@ -132,8 +152,7 @@ void test_manager::update(double const Deltatime) if (false == m_started || m_finished) return; - test_frame frame { Deltatime, std::nullopt, {}, - std::ranges::none_of(simulation::Vehicles.sequence(), [](TDynamicObject const *Vehicle) { return std::abs(Vehicle->GetVelocity()) > 0.01; }) }; + test_frame frame { Deltatime, std::nullopt, {}, tested_trains_at_rest() }; if (auto const *cell = simulation::Memory.find(std::string(result_cell))) { frame.reported = cell->Value1(); @@ -156,11 +175,22 @@ void test_manager::update(double const Deltatime) } if (passrequested) { - WriteLog(std::format("test: pass requested at {:.1f} s{}, waiting for every vehicle to stop", m_progress.time, + WriteLog(std::format("test: pass requested at {:.1f} s{}, waiting for the tested trains to stop", m_progress.time, frame.reason.empty() ? std::string() : std::format(" ({})", frame.reason))); } } +bool test_manager::tested_trains_at_rest() const +{ + if (Global.test.vehicles.empty()) + return std::ranges::none_of(simulation::Vehicles.sequence(), is_moving); + // the consists as they are now: a train the tested one couples to joins it, one it leaves behind doesn't + return std::ranges::none_of(Global.test.vehicles, [](std::string const &Name) { + auto *vehicle { simulation::Vehicles.find(Name) }; + return vehicle != nullptr && vehicle->find_vehicle(coupling::coupler, is_moving) != nullptr; + }); +} + void test_manager::signal_passed_at_danger(std::string_view const Vehicle, std::string_view const Signal) { auto const reason { std::format("{} passed signal {} at danger", Vehicle, Signal) }; diff --git a/simulation/simulationtest.h b/simulation/simulationtest.h index a59ec87b..784cd39e 100644 --- a/simulation/simulationtest.h +++ b/simulation/simulationtest.h @@ -52,6 +52,8 @@ public: private: // the watchdog's check, at Elapsed seconds of real time since the start of the simulator void check_wall_clock(double Elapsed); + // every vehicle of the tested trains stands: the consists of test.vehicle, or every vehicle without it + bool tested_trains_at_rest() const; // the watchdog may end the test from its own thread mutable std::mutex m_lock; diff --git a/simulation/simulationtestrules.cpp b/simulation/simulationtestrules.cpp index eba1d7dd..cdfea0e3 100644 --- a/simulation/simulationtestrules.cpp +++ b/simulation/simulationtestrules.cpp @@ -138,9 +138,9 @@ std::optional evaluate_frame(test_progress &Progress, test_frame c Progress.resttime = Frame.atrest ? Progress.resttime + Frame.deltatime : 0.0; if (Progress.passrequested && Progress.resttime >= pass_settle_time) - return test_verdict{pass, "__test.result set to 1 and every vehicle stopped"}; + return test_verdict{pass, "__test.result set to 1 and the tested trains stopped"}; if (Limits.standstill > 0.0 && Progress.resttime >= Limits.standstill) - return test_verdict{Limits.pass_on_standstill ? pass : standstill, std::format("every vehicle stood still for {:.0f} s", Limits.standstill)}; + return test_verdict{Limits.pass_on_standstill ? pass : standstill, std::format("the tested trains stood still for {:.0f} s", Limits.standstill)}; if (Limits.timeout > 0.0 && Progress.time >= Limits.timeout) return test_verdict{timeout, std::format("no result after {:.0f} s", Limits.timeout)}; return std::nullopt; diff --git a/simulation/simulationtestrules.h b/simulation/simulationtestrules.h index 52adf27a..153dc8a4 100644 --- a/simulation/simulationtestrules.h +++ b/simulation/simulationtestrules.h @@ -22,11 +22,11 @@ namespace simulation { // when it goes wrong on its own: 1 (killed with TerminateProcess, or a plain failure of most programs), 3 (abort() // on Windows), 126-127 (a shell that can't run the program) or 128 + a signal number (killed, on Linux) enum class test_result : int { - pass = 0, // __test.result set to 1, then every vehicle came to rest (or test.pass_on_standstill) + pass = 0, // __test.result set to 1, then the tested trains came to rest (or test.pass_on_standstill) fail = 10, // __test.result set to 0 signal_passed_at_danger = 11, // a train went past a stop signal (SetVelocity 0 0), unless test.fail_if_stop_overran no timeout = 12, // test.timeout seconds of simulation time, or test.wall_timeout seconds of real time went by - standstill = 13, // every vehicle stood still for test.standstill seconds without a result + standstill = 13, // the tested trains stood still for test.standstill seconds without a result error = 14, // the run could not start (a bad command line, the scenery failed to load) or the simulator threw }; @@ -64,7 +64,7 @@ double wall_clock_limit(std::optional Given, double Simulationtimeout); // the limits and switches the end conditions go by struct test_limits { double timeout{3600.0}; // [s] of simulation time; 0 = never - double standstill{300.0}; // [s] of every vehicle standing without a result; 0 = never + double standstill{300.0}; // [s] of the tested trains standing without a result; 0 = never bool pass_on_standstill{false}; bool no_movement_before_pass{true}; }; @@ -74,17 +74,17 @@ struct test_frame { double deltatime{0.0}; // [s] of simulation time std::optional reported; // value of the result cell, if there is one std::string_view reason; // text of the result cell - bool atrest{false}; // every vehicle stands + bool atrest{false}; // every vehicle of the tested trains stands (test.vehicle; every vehicle without it) }; // the state of a test between frames struct test_progress { double time{0.0}; // simulation time since the start [s] - double resttime{0.0}; // how long every vehicle has been standing [s] - bool passrequested{false}; // __test.result was set to 1, the pass waits for the vehicles to stop + double resttime{0.0}; // how long the tested trains have been standing [s] + bool passrequested{false}; // __test.result was set to 1, the pass waits for the tested trains to stop }; -// how long everything has to stand before a requested pass is granted [s] +// how long the tested trains have to stand before a requested pass is granted [s] inline constexpr double pass_settle_time{1.0}; // checks the end conditions after a frame and moves the progress on. returns the verdict once the test is over diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index 5eed983e..c1226e63 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -67,11 +67,11 @@ The test ends with the first of these: | exit code | result | when | | --- | --- | --- | -| 0 | pass | `__test.result` was set to 1 and then every vehicle stood still for 1 s, so a train still moving towards a signal at stop has to stop first (with `test.no_movement_before_pass no` the pass ends the test at once). Also the standstill end with `test.pass_on_standstill yes` | +| 0 | pass | `__test.result` was set to 1 and then the tested trains stood still for 1 s, so a train still moving towards a signal at stop has to stop first (with `test.no_movement_before_pass no` the pass ends the test at once). Also the standstill end with `test.pass_on_standstill yes` | | 10 | fail | `__test.result` was set to 0; ends at once | | 11 | signal passed at danger | a train went past a signal showing stop (`SetVelocity 0 0`; `SetVelocity 0 x` on automatic block, which may be passed after a stop, doesn't count); ends at once, unless `test.fail_if_stop_overran no` | | 12 | timeout | `test.timeout` seconds of simulation time went by (default 3600), or `test.wall_timeout` seconds of real time since the start (see below) | -| 13 | standstill | every vehicle stood still for `test.standstill` seconds (default 300) and nothing reported a result. Trains waiting at a station or for a signal count as standing, so the limit has to be longer than such waits | +| 13 | standstill | the tested trains stood still for `test.standstill` seconds (default 300) and nothing reported a result. Trains waiting at a station or for a signal count as standing, so the limit has to be longer than such waits | | 14 | error | the run could not start: no `-s`, the scenery failed to load or has no vehicles, or the simulator threw | A crash ends with the system's code instead (on Windows e.g. `0xC0000005`, -1073741819). The @@ -89,6 +89,7 @@ test.standstill 120 test.pass_on_standstill no test.fail_if_stop_overran yes test.no_movement_before_pass yes +test.vehicle ep07-424 endconfig ``` @@ -96,10 +97,11 @@ endconfig | --- | --- | --- | | `test.timeout` | 3600 | seconds of simulation time before the test ends as `timeout`; 0 = never | | `test.wall_timeout` | `test.timeout` + 600 | seconds of real time since the start before the test ends as `timeout`; 0 = never | -| `test.standstill` | 300 | seconds with every vehicle still and no result before the test ends as `standstill`; 0 = never | +| `test.standstill` | 300 | seconds with the tested trains still and no result before the test ends as `standstill`; 0 = never | | `test.pass_on_standstill` | no | the standstill end passes the test instead | | `test.fail_if_stop_overran` | yes | a signal passed at danger ends the test; with `no` it's only logged | -| `test.no_movement_before_pass` | yes | a pass waits until every vehicle stands still; with `no` it ends the test at once | +| `test.no_movement_before_pass` | yes | a pass waits until the tested trains stand still; with `no` it ends the test at once | +| `test.vehicle` | none | a vehicle of a train under test, by its name in the scenery (once for each train under test). The tested trains are the consists of these vehicles as they are at the moment (a train the tested one couples to joins it), and only they count for the pass and the standstill: other trains, AI traffic, may run on. A name the scenery doesn't have ends the test with `error`. Not given, every vehicle of the scenery counts. A signal passed at danger counts for every train either way | `--test-timeout` and `--test-wall-timeout` on the command line win over `test.timeout` and `test.wall_timeout` of the scenery and the settings files. @@ -131,14 +133,15 @@ Python 3.10+, no extra packages. Options: `cmake-build-*/bin/Release/eu07*.exe`. The build needs the `--test` and `-ini` command line options. - name one or more cases to run only those, e.g. `run.py ... eu07_td_timetable`. -- `-j N` runs N cases at once. Each case prints its results when it is done. All 16 cases +- `-j N` runs N cases at once. Each case prints its results when it is done. All 17 cases take about 30 s with `-j 4`, loading included. - `--watch` runs a case the old way, in a visible window in real time (or faster with `--timespeed N`), closed after the case's `duration`; the exit code isn't checked then. For watching what a case does. Exit code is 0 when every case passed. Logs of each run (`log.txt`, `errors.txt`, -`stdout.txt`, the vehicle's physicslog) are moved to `tests/scenarios/out//`. +`stdout.txt`, the physicslogs of the vehicle and of other trains of the case) are moved to +`tests/scenarios/out//`. ## How it works @@ -177,7 +180,7 @@ scenery is newer. | `time` | replace the scenery start time, `HH:MM` | | `track_friction` | set the friction of every track of the base scenery (default in the sceneries is 0.15), e.g. for slippery rail | | `track_events` | lines added to tracks of the base scenery, by track name, e.g. `{ "tor1k": "event0 test_pass" }`; only tracks in the scenery file itself, not in its includes | -| `test_config` | lines of a `config` block added to the scenery, e.g. `["test.timeout 90"]` | +| `test_config` | lines of a `config` block added to the scenery, e.g. `["test.timeout 90"]`; the block always has `test.vehicle` with the case's lead vehicle, so only its train counts for the pass and the standstill | | `fragment` | `.scn` file in `cases/` added to the scenery: trainsets after `FirstInit` (before it they derail on unlinked track), the rest before it (launchers read after it never fire) | | `vars` | values for `{{name}}` in the fragment; a list becomes one line per item | | `expect` | the result the test mode should end with: `pass`, `fail`, `signal_passed_at_danger`, `timeout`, `standstill` or `error`, or `wall_timeout` for the timeout of the real time watchdog (the timeout code with a verdict saying it was the real time; a case expecting `timeout` fails on it). Default `timeout` for a case with `duration`, else `pass` | @@ -187,15 +190,16 @@ scenery is newer. | `checks` | list of checks, below | `{{vehicle}}` is always set, also inside `vars` values: the node name of the lead vehicle, -the one whose physicslog is checked. Other vehicles of the consist take names made from it, -e.g. `{{vehicle}}_s`. The td.scn cases share two fragments, `td_timetable.scn` (route set, +the one whose physicslog is checked, and the train under test (`test.vehicle`). Other vehicles +of the consist take names made from it, e.g. `{{vehicle}}_s`; another train with a driver is named +`_{{vehicle}}`, so its physicslog is moved out of the game dir too. The td.scn cases share two fragments, `td_timetable.scn` (route set, train runs to Przystanek) and `td_red_signal.scn` (no route, exit signal at stop), and differ only in `vars.consist`. Przystanek is a technical stop in that timetable (`pt`), where the AI doesn't wait for the departure time: it sets off as soon as it counts the train as standing at the W4, which is below 1 km/h, so the train stands there for a few seconds at most, or doesn't quite stop (since the AI braking of #264 the SN61 turns back at 0.4 km/h). The timetable cases therefore check that the train comes down to 1 km/h, with no hold, 1380-1420 m -from the start (every vehicle stops at 1397-1401 m). `td_red_signal_moving.scn` is the red signal one with the train +from the start (each of the seven trains stops at 1397-1401 m). `td_red_signal_moving.scn` is the red signal one with the train starting at `vars.velocity` km/h; with `track_friction` 0.03 it locks the wheels when braking and slides past the signal, so those cases set `test.fail_if_stop_overran no`. `td_red_signal_pass.scn` adds the `test_pass` event, which `test_mode_pass` puts as `event0` @@ -205,6 +209,10 @@ The `test_mode_*` cases show the ends of the test mode: `test_mode_pass` (exit 0 event), `test_mode_signal_passed_at_danger` (exit 11), `test_mode_timeout` (exit 12, from `test.timeout`), `test_mode_standstill` (exit 13, from `test.standstill`) and `test_mode_wall_timeout` (exit 12 from the real time watchdog, which ends it while it loads). +`test_mode_standstill_traffic` (fragment `td_traffic.scn`) has a train under test standing on +track 2 of Stacja1 and AI traffic, `traffic_{{vehicle}}`, leaving track 1 on the timetable: the +standstill of 100 s ends the test although the traffic runs on (counting every vehicle, it runs +into the test timeout). `test_mode_pass_while_moving` reports the pass from `event2` of `tor1k` (the direction the train runs), as the train runs onto it, with `test.no_movement_before_pass no`, so the test ends with the train still moving. diff --git a/tests/scenarios/cases/td_traffic.scn b/tests/scenarios/cases/td_traffic.scn new file mode 100644 index 00000000..49720a0b --- /dev/null +++ b/tests/scenarios/cases/td_traffic.scn @@ -0,0 +1,13 @@ +// added by tests/scenarios/run.py to td.scn, whose own trainsets were removed: two trains at Stacja1. +// the train under test (test.vehicle) stands on track 2, where its driver has nothing to do; the other one, +// AI traffic, has the td/rozklad timetable on track 1 and leaves for Przystanek at 10:35. the standstill of +// the test waits for the train under test only, so the traffic running on doesn't hold it up. +trainset none tor2k 30.0 0.0 +node -1 0 {{vehicle}} dynamic pkp\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic +endtrainset +trainset td/rozklad tdo_n1734 0.0 0.1 +node -1 0 traffic_{{vehicle}} dynamic pkp\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic +endtrainset + +// the way out of track 1 for the traffic, see td_timetable.scn +node -1 0 autotest_route eventlauncher 0 0 0 -1 none 1034 keyctrl04 end diff --git a/tests/scenarios/cases/test_mode_standstill.json b/tests/scenarios/cases/test_mode_standstill.json index 0d96585e..4dd891cf 100644 --- a/tests/scenarios/cases/test_mode_standstill.json +++ b/tests/scenarios/cases/test_mode_standstill.json @@ -1,5 +1,5 @@ { - "description": "Test mode, standstill: EU07 on td.scn with no route set stops before the exit signal stacja1_j at stop and stays; nothing reports a result, so the test ends once every vehicle stood still for test.standstill 120 seconds (the train waits about 70 s before it pulls off, so less would end it before the start).", + "description": "Test mode, standstill: EU07 on td.scn with no route set stops before the exit signal stacja1_j at stop and stays; nothing reports a result, so the test ends once the train under test stood still for test.standstill 120 seconds (the train waits about 70 s before it pulls off, so less would end it before the start).", "base_scenery": "td.scn", "time": "10:34", "fragment": "td_red_signal.scn", diff --git a/tests/scenarios/cases/test_mode_standstill_traffic.json b/tests/scenarios/cases/test_mode_standstill_traffic.json new file mode 100644 index 00000000..8b545e2a --- /dev/null +++ b/tests/scenarios/cases/test_mode_standstill_traffic.json @@ -0,0 +1,18 @@ +{ + "description": "Test mode, standstill with traffic: an EU07 stands on track 2 of Stacja1 on td.scn while another EU07, AI traffic, leaves track 1 on the td/rozklad timetable at 10:35 and runs on. Only the train under test (test.vehicle, the case's vehicle) counts, so the test ends on test.standstill 100 although the traffic is moving; counting every vehicle, it would run into the 300 s test timeout.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_traffic.scn", + "test_config": [ + "test.standstill 100" + ], + "expect": "standstill", + "duration": 300, + "timeout": 600, + "checks": [ + { + "type": "max_speed", + "limit": 0.5 + } + ] +} diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index 62cb7d13..3757a824 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -158,9 +158,10 @@ def build_scenery(case: dict, game_dir: Path, case_name: str, vehicle: str) -> s # events of the case go in before it (read after it they never fire), trainsets after it # (placed before it they stand on unlinked track and derail at once) before, trainsets = split_trainsets(fragment) - if case.get("test_config"): - # settings of the test mode (test.timeout and so on), read like any other config block - before = "config\n" + "".join(line + "\n" for line in case["test_config"]) + "endconfig\n" + before + # settings of the test mode, read like any other config block: the case's vehicle is the train under test, + # whose consist the pass and the standstill wait for, and the case's own (test.timeout and so on) + config = [f"test.vehicle {vehicle}", *case.get("test_config", [])] + before = "config\n" + "".join(line + "\n" for line in config) + "endconfig\n" + before header = f"\n// ---- autotest: {case_name} ----\n" firstinit = re.search(r"(?im)^\s*firstinit\b", text) if firstinit: @@ -526,6 +527,9 @@ def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, watch: bool shutil.move(path, out / f"{kind}.txt") if physicslog.exists(): shutil.move(physicslog, out / physicslog.name) + # other trains of the case with a driver, named _, log too + for other in physicslog.parent.glob(f"*_{vehicle}.dat"): + shutil.move(other, out / other.name) ok = True sim_t = result.rows[-1][COL_TIME] if result.rows else 0.0 diff --git a/tests/unit/test_test_mode.cpp b/tests/unit/test_test_mode.cpp index 55a90387..27e8c0d9 100644 --- a/tests/unit/test_test_mode.cpp +++ b/tests/unit/test_test_mode.cpp @@ -167,7 +167,7 @@ TEST_CASE("test mode: the end conditions") CHECK(verdict->result == test_result::fail); CHECK(verdict->reason == "__test.result set to 0 (wrong_track)"); } - SUBCASE("1 passes once every vehicle has stood for a second") + SUBCASE("1 passes once the tested trains have stood for a second") { CHECK_FALSE(simulation::evaluate_frame(progress, frame(false, 1.0), limits).has_value()); CHECK(progress.passrequested); diff --git a/utilities/Globals.cpp b/utilities/Globals.cpp index 217374cb..8957a4a0 100644 --- a/utilities/Globals.cpp +++ b/utilities/Globals.cpp @@ -477,6 +477,17 @@ bool global_settings::ConfigParseTest(cParser& Parser, std::string_view const To return true; } + if (Token == "test.vehicle") + { + // a vehicle of a train under test, by its name in the scenery; the pass and the standstill wait for its + // consist only, not for other trains (eu07/maszyna#541). given more than once, for each of them + std::string name; + ParseOne(Parser, name, 1, true); + if (false == name.empty() && std::ranges::find(test.vehicles, name) == test.vehicles.end()) + test.vehicles.emplace_back(name); + return true; + } + if (Token == "test.standstill") { ParseOne(Parser, test.standstill); @@ -1802,6 +1813,9 @@ global_settings::export_as_text( std::ostream &Output ) const { export_as_text( Output, "test.pass_on_standstill", test.pass_on_standstill ); export_as_text( Output, "test.fail_if_stop_overran", test.fail_if_stop_overran ); export_as_text( Output, "test.no_movement_before_pass", test.no_movement_before_pass ); + for( auto const &vehicle : test.vehicles ) { + export_as_text( Output, "test.vehicle", vehicle ); + } export_as_text( Output, "gfx.envmap.enabled", gfx_envmap_enabled ); export_as_text( Output, "gfx.postfx.motionblur.enabled", gfx_postfx_motionblur_enabled ); export_as_text( Output, "gfx.postfx.motionblur.shutter", gfx_postfx_motionblur_shutter ); diff --git a/utilities/Globals.h b/utilities/Globals.h index 81bc1e47..50e42085 100644 --- a/utilities/Globals.h +++ b/utilities/Globals.h @@ -91,10 +91,11 @@ struct global_settings { struct test_settings { bool enabled{ false }; double timeout{ 3600.0 }; // [s] of simulation time; no result by then fails the test - double standstill{ 300.0 }; // [s] every vehicle standing still this long ends the test; 0 = never + double standstill{ 300.0 }; // [s] the tested trains standing still this long ends the test; 0 = never bool pass_on_standstill{ false }; // the standstill end passes the test instead of failing it bool fail_if_stop_overran{ true }; // a train passing a signal at stop ends the test as failed; with false it's only logged - bool no_movement_before_pass{ true }; // a pass waits until every vehicle stands still; with false it ends the test at once + bool no_movement_before_pass{ true }; // a pass waits until the tested trains stand still; with false it ends the test at once + std::vector vehicles; // test.vehicle: the trains under test are the consists of these; none = every vehicle bool timeout_from_command_line{ false }; // --test-timeout given, the scenery doesn't override it std::optional wall_timeout; // [s] of real time since the start; none = test.timeout + time for the loading, 0 = no limit bool wall_timeout_from_command_line{ false }; // --test-wall-timeout given, the scenery doesn't override it From 349ffb1caf0ec40a8c350a29e36ca7cce383e38a Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 11:40:13 +0100 Subject: [PATCH 26/28] fix(simulation): on Linux an exception ends a test run with its verdict too The runtime_error the simulator throws on a fatal problem was caught on Windows only. On Linux a test run aborted on it, with the code of abort() and no `test:` line. It is caught there too in test mode and ends the test with the error code and the reason; outside a test it is thrown on as before. --- EU07.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/EU07.cpp b/EU07.cpp index 0fbabba5..4bc45386 100644 --- a/EU07.cpp +++ b/EU07.cpp @@ -164,20 +164,25 @@ int main(int argc, char *argv[]) { ErrorLog("Critical error, memory allocation failure: " + std::string(Error.what())); } -#ifdef _WIN32 catch (std::runtime_error const &Error) { +#ifndef _WIN32 + // outside a test the error ends the program as it always did there, uncaught + if (false == Global.test.enabled) + throw; +#endif ErrorLog(std::format("Critical error: {}", Error.what())); // in test mode nobody is there to close the box, the exit code tells of the error simulation::test().conclude(simulation::test_result::error, std::format("critical error: {}", Error.what())); +#ifdef _WIN32 if (false == Global.test.enabled) { std::string msg = "Simulator crash occured :(\n"; msg += Error.what(); MessageBoxA(nullptr, msg.c_str(), "Simulator crashed :(", MB_ICONERROR); } - } #endif + } } #ifndef _WIN32 fflush(stdout); From cea92f017011389d37deecf72926b61bbd291080 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 11:57:03 +0100 Subject: [PATCH 27/28] test(unit): the test mode tests compare seconds with a tolerance Floating point values of the command line, the real time limit and the standstill clock are checked with doctest::Approx rather than ==. --- tests/unit/test_test_mode.cpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/unit/test_test_mode.cpp b/tests/unit/test_test_mode.cpp index 27e8c0d9..a85d73e5 100644 --- a/tests/unit/test_test_mode.cpp +++ b/tests/unit/test_test_mode.cpp @@ -61,9 +61,9 @@ TEST_CASE("test mode: the exit codes keep clear of the ones of a killed or abort TEST_CASE("test mode: seconds on the command line are a number, not negative") { - CHECK(simulation::parse_seconds("240") == 240.0); - CHECK(simulation::parse_seconds("0.5") == 0.5); - CHECK(simulation::parse_seconds("0") == 0.0); + CHECK(simulation::parse_seconds("240").value_or(-1.0) == doctest::Approx(240.0)); + CHECK(simulation::parse_seconds("0.5").value_or(-1.0) == doctest::Approx(0.5)); + CHECK(simulation::parse_seconds("0").value_or(-1.0) == doctest::Approx(0.0)); CHECK_FALSE(simulation::parse_seconds("").has_value()); CHECK_FALSE(simulation::parse_seconds("-1").has_value()); CHECK_FALSE(simulation::parse_seconds("60s").has_value()); @@ -84,8 +84,8 @@ TEST_CASE("test mode: the command line") { auto const commandline{parse({"--test-wall-timeout", "900", "-s", "td.scn", "--test", "--test-timeout", "240"})}; CHECK(commandline.enabled); - CHECK(commandline.timeout == 240.0); - CHECK(commandline.wall_timeout == 900.0); + CHECK(commandline.timeout.value_or(-1.0) == doctest::Approx(240.0)); + CHECK(commandline.wall_timeout.value_or(-1.0) == doctest::Approx(900.0)); CHECK(commandline.error.empty()); } SUBCASE("a limit not given stays unset") @@ -128,18 +128,18 @@ TEST_CASE("test mode: the real time limit") { SUBCASE("given, it is the limit, 0 turns it off") { - CHECK(simulation::wall_clock_limit(120.0, 3600.0) == 120.0); - CHECK(simulation::wall_clock_limit(0.0, 3600.0) == 0.0); + CHECK(simulation::wall_clock_limit(120.0, 3600.0) == doctest::Approx(120.0)); + CHECK(simulation::wall_clock_limit(0.0, 3600.0) == doctest::Approx(0.0)); } SUBCASE("not given, it is the simulation time limit and time for the loading") { - CHECK(simulation::wall_clock_limit(std::nullopt, 240.0) == 240.0 + simulation::wall_timeout_loading_allowance); - CHECK(simulation::wall_clock_limit(std::nullopt, 3600.0) == 3600.0 + simulation::wall_timeout_loading_allowance); + CHECK(simulation::wall_clock_limit(std::nullopt, 240.0) == doctest::Approx(240.0 + simulation::wall_timeout_loading_allowance)); + CHECK(simulation::wall_clock_limit(std::nullopt, 3600.0) == doctest::Approx(3600.0 + simulation::wall_timeout_loading_allowance)); } SUBCASE("no simulation time limit, no real time limit unless given") { - CHECK(simulation::wall_clock_limit(std::nullopt, 0.0) == 0.0); - CHECK(simulation::wall_clock_limit(300.0, 0.0) == 300.0); + CHECK(simulation::wall_clock_limit(std::nullopt, 0.0) == doctest::Approx(0.0)); + CHECK(simulation::wall_clock_limit(300.0, 0.0) == doctest::Approx(300.0)); } } @@ -152,7 +152,7 @@ TEST_CASE("test mode: the end conditions") { CHECK_FALSE(simulation::evaluate_frame(progress, frame(false), limits).has_value()); CHECK(progress.time == doctest::Approx(0.03)); - CHECK(progress.resttime == 0.0); + CHECK(progress.resttime == doctest::Approx(0.0)); } SUBCASE("a cell nobody wrote to says nothing") { @@ -207,7 +207,7 @@ TEST_CASE("test mode: the end conditions") limits.standstill = 120.0; CHECK_FALSE(run(progress, limits, 100.0, frame(true)).has_value()); CHECK_FALSE(simulation::evaluate_frame(progress, frame(false), limits).has_value()); - CHECK(progress.resttime == 0.0); + CHECK(progress.resttime == doctest::Approx(0.0)); CHECK_FALSE(run(progress, limits, 200.0, frame(true)).has_value()); } SUBCASE("test.pass_on_standstill makes the standstill a pass") From 8fb596f2186cdbac2ed26d7b25a56f0f57da0ad0 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 19:46:16 +0100 Subject: [PATCH 28/28] fix(simulation): any exception ends a test run with error, other runs as on master main() tells a test run (--test) apart before anything runs. Only a test run catches every exception and installs a std::terminate handler, both ending the run with the error code and a test: line; any other run keeps master's exception handling, so on Linux an uncaught runtime_error aborts at the throw site again. A test run doesn't start the Discord RPC thread: it published the test scenery as the player's presence and held the closing up to 5 s. run.py drops a case scenery's .sbt cache after a run cut short (wall_timeout, error, a crash, killed by run.py), which could have been writing it. The -edit branch of the test command line parser had no effect (a token that isn't an option is ignored either way) and is gone; its test checks what can be seen instead. --- EU07.cpp | 78 +++++++++++++++++++----------- application/application.cpp | 8 ++- simulation/simulationtestrules.cpp | 23 +++++++-- simulation/simulationtestrules.h | 5 ++ tests/scenarios/README.md | 11 +++-- tests/scenarios/run.py | 12 ++++- tests/unit/test_test_mode.cpp | 29 ++++++++++- 7 files changed, 125 insertions(+), 41 deletions(-) diff --git a/EU07.cpp b/EU07.cpp index 4bc45386..20f24aa5 100644 --- a/EU07.cpp +++ b/EU07.cpp @@ -23,7 +23,10 @@ Stele, firleju, szociu, hunter, ZiomalCl, OLI_EU and others #include "utilities/Globals.h" #include "utilities/Logs.h" #include -#include +#include +#include +#include +#include #ifdef WITHDUMPGEN #ifdef _WIN32 #include @@ -94,10 +97,45 @@ namespace void finish_log() { StopLogService(); - if (auto const logservice{Global.threads.find("LogService")}; logservice != Global.threads.end() && logservice->second.joinable()) + if (auto const logservice{Global.threads.find("LogService")}; + logservice != Global.threads.end() && logservice->second.joinable() && logservice->second.get_id() != std::this_thread::get_id()) logservice->second.join(); } +// ends a test run with its verdict on the standard output and in the exit code. a test the simulator didn't end +// (a failed start, an exception) ends with the error and Reason +[[noreturn]] void end_test_run(std::string_view const Reason) +{ + simulation::test().conclude(simulation::test_result::error, Reason); + finish_log(); + fflush(stdout); + fflush(stderr); + // skip destructors, there are ordering errors which causes segfaults + std::_Exit(simulation::test().exit_code()); +} + +// a test run (--test): whatever stops it, it ends with its verdict. any other run never gets here and ends as +// it always has +[[noreturn]] void run_test(int argc, char *argv[]) +{ + // for what doesn't reach the catch below: an exception leaving a noexcept function, or on Linux one leaving + // a thread (on Windows the handler is the main thread's only) + std::set_terminate([] { end_test_run(simulation::describe_exception(std::current_exception())); }); + try + { + if (Application.init(argc, argv) == 0) + { + Application.run(); + Application.exit(); + } + } + catch (...) + { + end_test_run(simulation::describe_exception(std::current_exception())); + } + end_test_run("the simulator stopped before the test ended"); +} + } // namespace int main(int argc, char *argv[]) @@ -149,6 +187,11 @@ int main(int argc, char *argv[]) fflush(stderr); std::_Exit(result); } + // unattended test run of a scenery, which says its result in the exit code (eu07/maszyna#541) + else if (simulation::parse_test_command_line(std::vector(argv, argv + argc)).enabled) + { + run_test(argc, argv); + } else { try @@ -164,37 +207,18 @@ int main(int argc, char *argv[]) { ErrorLog("Critical error, memory allocation failure: " + std::string(Error.what())); } +#ifdef _WIN32 catch (std::runtime_error const &Error) { -#ifndef _WIN32 - // outside a test the error ends the program as it always did there, uncaught - if (false == Global.test.enabled) - throw; -#endif - ErrorLog(std::format("Critical error: {}", Error.what())); - // in test mode nobody is there to close the box, the exit code tells of the error - simulation::test().conclude(simulation::test_result::error, std::format("critical error: {}", Error.what())); -#ifdef _WIN32 - if (false == Global.test.enabled) - { - std::string msg = "Simulator crash occured :(\n"; - msg += Error.what(); - MessageBoxA(nullptr, msg.c_str(), "Simulator crashed :(", MB_ICONERROR); - } -#endif + std::string msg = "Simulator crash occured :(\n"; + msg += Error.what(); + MessageBoxA(nullptr, msg.c_str(), "Simulator crashed :(", MB_ICONERROR); } +#endif } #ifndef _WIN32 fflush(stdout); fflush(stderr); #endif - if (Global.test.enabled) - { - // a run that stopped before its test ended (a failed start, an exception) says so like any other end - simulation::test().conclude(simulation::test_result::error, "the simulator stopped before the test ended"); - finish_log(); - } - // skip destructors, there are ordering errors which causes segfaults - // a test run (--test) says its result in the exit code - std::_Exit(Global.test.enabled ? simulation::test().exit_code() : 0); + std::_Exit(0); // skip destructors, there are ordering errors which causes segfaults } diff --git a/application/application.cpp b/application/application.cpp index 860b8a23..75ecde77 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -433,8 +433,12 @@ int eu07_application::init(int Argc, char *Argv[]) #if WITH_DISCORD_RPC // Run DiscordRPC service - std::thread sDiscordRPC(&eu07_application::DiscordRPCService, this); - Global.threads.emplace("DiscordRPC", std::move(sDiscordRPC)); + // not for a test run: it would show the test scenery as the player's presence, and hold up the closing + if (false == Global.test.enabled) + { + std::thread sDiscordRPC(&eu07_application::DiscordRPCService, this); + Global.threads.emplace("DiscordRPC", std::move(sDiscordRPC)); + } #endif if (!init_network()) diff --git a/simulation/simulationtestrules.cpp b/simulation/simulationtestrules.cpp index cdfea0e3..fcedaed7 100644 --- a/simulation/simulationtestrules.cpp +++ b/simulation/simulationtestrules.cpp @@ -100,15 +100,28 @@ test_command_line parse_test_command_line(std::span const Arg { ++i; } - else if (option == "-edit" && i + 1 < Arguments.size() && Arguments[i + 1][0] != '-') - { - // the scenery to edit is optional - ++i; - } } return result; } +std::string describe_exception(std::exception_ptr const &Exception) +{ + if (nullptr == Exception) + return "the simulator was terminated"; + try + { + std::rethrow_exception(Exception); + } + catch (std::exception const &Error) + { + return std::format("critical error: {}", Error.what()); + } + catch (...) + { + return "critical error: an exception of unknown type"; + } +} + double wall_clock_limit(std::optional const Given, double const Simulationtimeout) { if (Given.has_value()) diff --git a/simulation/simulationtestrules.h b/simulation/simulationtestrules.h index 153dc8a4..684b90cd 100644 --- a/simulation/simulationtestrules.h +++ b/simulation/simulationtestrules.h @@ -9,6 +9,7 @@ http://mozilla.org/MPL/2.0/. #pragma once +#include #include #include #include @@ -53,6 +54,10 @@ std::optional parse_seconds(std::string_view Text); // options are skipped, so a scenery named like an option isn't taken for one; options it doesn't know are left alone test_command_line parse_test_command_line(std::span Arguments); +// what ended a test run that the simulator didn't end itself: the exception caught (or being handled when the +// program was terminated), or nothing +std::string describe_exception(std::exception_ptr const &Exception); + // time allowed for loading the scenery, on top of the simulation time, when the real time limit isn't given [s] inline constexpr double wall_timeout_loading_allowance{600.0}; diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index c1226e63..0a11c6a0 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -31,7 +31,8 @@ With `--test` the simulator: motion telemetry is off, - gives nothing to the outside either: no keyboard leds, LPT or PoKeys (`feedbackmode`), no messages to a multiplayer server (`multiplayer`), no program run at the end (`execonexit`), - no console window of its own (`showsystemconsole`), and the game's `imgui.ini` stays as it is, + no console window of its own (`showsystemconsole`), no Discord presence, and the game's `imgui.ini` + stays as it is, - keeps all of the above whatever the `config` block of the scenery says: the settings of the test mode are applied again after it, - runs the simulation as fast as it can in fixed steps of 0.03 s of simulation time per @@ -39,8 +40,9 @@ With `--test` the simulator: - seeds its random numbers with a fixed seed unless `-seed` or `randomseed` gives one, so two runs of a scenery come out the same (the physicslogs of two runs are byte for byte equal), - quits when the test ends, and says how it ended in the exit code and in a `test: ...` - line on stdout and in the log. A command line it can't take, or a start that fails, ends - the same way with the `error` code. That line is all there is on stdout: the log goes to + line on stdout and in the log. A command line it can't take, a start that fails, or an + exception thrown by the simulator (of any type, on Linux too) ends the same way, with the + `error` code. That line is all there is on stdout: the log goes to the log file only (`debuglog` doesn't print it on the console in test mode). On Windows the simulator is a windowed program, which the `cmd` prompt and PowerShell don't wait @@ -167,7 +169,8 @@ For each case the runner: The lead vehicle is named `autotest_`, so every case has its own physicslog and log files and cases can run in parallel. The `autotest_*` sceneries (and the simulator's `.sbt` geometry cache of them) stay in the game's `scenery/` dir; the cache is dropped when the base -scenery is newer. +scenery is newer, and after a run cut short (`wall_timeout`, `error`, a crash, or killed by run.py), +which could have been writing it. ## Writing a case diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index 3757a824..18b3e3f2 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -137,6 +137,11 @@ def fill_vars(fragment: str, values: dict, keep=frozenset()) -> str: return re.sub(r"\{\{(\w+)\}\}", value, fragment) +def scenery_cache(game_dir: Path, case_name: str) -> Path: + """The simulator's geometry cache of the case scenery.""" + return game_dir / "scenery" / f"autotest_{case_name}.sbt" + + def build_scenery(case: dict, game_dir: Path, case_name: str, vehicle: str) -> str: """Writes the case scenery into the game's scenery dir and returns its file name.""" scenery_dir = game_dir / "scenery" @@ -174,7 +179,7 @@ def build_scenery(case: dict, game_dir: Path, case_name: str, vehicle: str) -> s # the simulator caches the scenery's static geometry in .sbt and reads it instead of # the terrain in the .scn (tracks, events and trainsets always come from the .scn), so # drop the cache once the base scenery has changed - cache = scenery_dir / f"autotest_{case_name}.sbt" + cache = scenery_cache(game_dir, case_name) if cache.exists() and cache.stat().st_mtime < (scenery_dir / case["base_scenery"]).stat().st_mtime: cache.unlink() return name @@ -548,6 +553,11 @@ def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, watch: bool watchdog = ", from the real time watchdog" if got == WALL_TIMEOUT else "" say(f" {'PASS' if passed else 'FAIL'} exit code {describe_exit(result.exit_code)}{watchdog}, expected {expect}") ok &= passed + # a run cut short (the real time watchdog, an error, a crash, or closed by us) could have been writing + # the scenery's geometry cache; the simulator checks only the header of it, so the next runs would read + # a truncated one. it is written again on the next run + if result.failure or (not watch and got in (None, "error", WALL_TIMEOUT)): + scenery_cache(game_dir, name).unlink(missing_ok=True) if result.failure: say(f" FAIL {result.failure}") ok = False diff --git a/tests/unit/test_test_mode.cpp b/tests/unit/test_test_mode.cpp index a85d73e5..eba64109 100644 --- a/tests/unit/test_test_mode.cpp +++ b/tests/unit/test_test_mode.cpp @@ -4,6 +4,9 @@ #include "simulation/simulationtestrules.h" #include +#include +#include +#include #include // the test mode of the simulator (eu07 --test, eu07/maszyna#541): its exit codes, command line and end conditions @@ -117,10 +120,13 @@ TEST_CASE("test mode: the command line") CHECK_FALSE(commandline.enabled); CHECK(commandline.error.empty()); } - SUBCASE("the scenery of -edit is optional") + SUBCASE("-edit takes no option for its scenery, which is optional") { + // a scenery after -edit or none: the option after it is read either way CHECK(parse({"-edit", "--test", "-s", "td.scn"}).enabled); - CHECK_FALSE(parse({"-edit", "td.scn"}).enabled); + CHECK(parse({"-edit", "td.scn", "--test"}).enabled); + // -edit leaves -s alone, which takes --test for its scenery, as the simulator reads it + CHECK_FALSE(parse({"-edit", "-s", "--test"}).enabled); } } @@ -241,3 +247,22 @@ TEST_CASE("test mode: the end conditions") CHECK(verdict->result == test_result::fail); } } + +TEST_CASE("test mode: what ended a run the simulator didn't end") +{ + SUBCASE("an exception says what it was") + { + CHECK(simulation::describe_exception(std::make_exception_ptr(std::runtime_error("no scenery"))) == "critical error: no scenery"); + // e.g. std::stof on a token of a scenery which isn't a number + CHECK(simulation::describe_exception(std::make_exception_ptr(std::invalid_argument("stof"))) == "critical error: stof"); + CHECK(simulation::describe_exception(std::make_exception_ptr(std::bad_alloc())).starts_with("critical error: ")); + } + SUBCASE("something thrown which isn't a std::exception") + { + CHECK(simulation::describe_exception(std::make_exception_ptr(42)) == "critical error: an exception of unknown type"); + } + SUBCASE("terminated without an exception") + { + CHECK(simulation::describe_exception(nullptr) == "the simulator was terminated"); + } +}