diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index c233154d6..23f793018 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -9,7 +9,7 @@ only, not in CI. ## Running ``` -python tests/scenarios/run.py --game-dir "E:/Program Files (x86)/Steam/steamapps/common/MaSzyna" +python tests/scenarios/run.py --game-dir "E:/Program Files (x86)/Steam/steamapps/common/MaSzyna" -j 4 ``` Python 3.10+, no extra packages. Options: @@ -18,13 +18,15 @@ Python 3.10+, no extra packages. Options: - `--exe path/to/eu07.exe` picks the build to test; the default is the newest `cmake-build-*/bin/Release/eu07*.exe`. The build needs the `-ini` command line option. - name one or more cases to run only those, e.g. `run.py ... eu07_td_timetable`. +- `-j N` runs N cases at once, each in its own simulator window. Each case prints its + results when it is done. All 8 cases take about a minute with `-j 4`. - `--timespeed N` sets the simulation time multiplier, default 20: a 4-minute case runs in about 20 s. Physics still steps finely (the physicslog keeps its 0.05 s rows) and the speed profile at 20 and 60 matched a real-time run within a few km/h and seconds. Use `--timespeed 1` to watch a run. Exit code is 0 when every case passed. Logs of each run (`log.txt`, `errors.txt`, -the vehicle's physicslog) land in `tests/scenarios/out//`. +the vehicle's physicslog) are moved to `tests/scenarios/out//`. ## How it works @@ -35,11 +37,19 @@ For each case the runner: fragment added: its trainsets after `FirstInit`, everything else (event launchers standing in for keys the scenery expects a player to press, extra events) before it, 2. starts the simulator with `-s autotest_.scn -v ghostview -ini out//autotest.ini`; - ghostview leaves every train to the AI, and the extra ini turns on `physicslog`, - runs in a 1280x720 window and does not pause when the window loses focus, -3. waits until `physicslog/.dat` covers `duration` seconds, then closes the simulator, + ghostview leaves every train to the AI, and the extra ini turns on `physicslog`, runs in a + 1280x720 window, does not pause when the window loses focus, and writes the logs to + `logs/log (autotest_) .txt` rather than the shared `log.txt` (`multiplelogs`), +3. waits until `physicslog/autotest_.dat` covers `duration` seconds, then closes the + simulator. It gives up early when the log reports `Bad vehicle` / `Bad trainset`, or when + the scenario has loaded and no physicslog appeared, 4. runs the checks on the physicslog. +The lead vehicle is named `autotest_`, so every case has its own physicslog and log +files and cases can run in parallel. The `autotest_*` sceneries (and the simulator's `.sbt` +geometry cache of them) stay in the game's `scenery/` dir; the cache is dropped when the base +scenery is newer. + ## Writing a case `cases/.json`: @@ -50,13 +60,19 @@ For each case the runner: | `strip_trainsets` | drop the base scenery's trainsets (default `true`) | | `time` | replace the scenery start time, `HH:MM` | | `fragment` | `.scn` file in `cases/` added to the scenery: trainsets after `FirstInit` (before it they derail on unlinked track), the rest before it (launchers read after it never fire) | -| `vehicle` | node name of the vehicle whose physicslog is checked | +| `vars` | values for `{{name}}` in the fragment; a list becomes one line per item | | `duration` | seconds of simulation to record | | `timeout` | wall-clock seconds before giving up, loading included | | `fail_on_errors` | fail when `errors.txt` is not empty (default `false`) | | `checks` | list of checks, below | -Checks (speeds in km/h, times in seconds of physicslog time): +`{{vehicle}}` is always set, also inside `vars` values: the node name of the lead vehicle, +the one whose physicslog is checked. Other vehicles of the consist take names made from it, +e.g. `{{vehicle}}_s`. The td.scn cases share two fragments, `td_timetable.scn` (route set, +train runs to Przystanek) and `td_red_signal.scn` (no route, exit signal at stop), and differ +only in `vars.consist`. + +Checks (speeds in km/h, times in seconds of physicslog time, distances in metres): - `starts` `min_speed`, `within`: reaches `min_speed` within `within` seconds. - `max_speed` `limit`, `tolerance`: never goes faster than `limit + tolerance`. @@ -64,6 +80,11 @@ Checks (speeds in km/h, times in seconds of physicslog time): (physicslog pressures are in bar whatever its header says; the brake pipe runs at 5.0). - `stops` `moving_speed`, `stop_speed`, `hold`: after reaching `moving_speed`, stays at or below `stop_speed` for `hold` seconds. +- `stays_stopped` `moving_speed`, `stop_speed`: after moving off and stopping, never goes + above `stop_speed` again. +- `distance` `min`, `max`: total distance travelled. The physicslog has no position, so the + distance comes from integrating the speed; on td.scn every vehicle's first stop after + Stacja1 came out at 1400 m (+-3), and the red signal stop at 140 m. New check types go in `CHECKS` in `run.py`; each gets the physicslog rows and the check's parameters and returns `(passed, detail)`. diff --git a/tests/scenarios/cases/en57_td_timetable.json b/tests/scenarios/cases/en57_td_timetable.json new file mode 100644 index 000000000..ddd52af37 --- /dev/null +++ b/tests/scenarios/cases/en57_td_timetable.json @@ -0,0 +1,21 @@ +{ + "description": "EN57 EMU (three cars) on td.scn, AI driver on the td/rozklad timetable: starts from Stacja1, keeps to the speed limit, brakes to a stop at Przystanek.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_timetable.scn", + "vars": { + "consist": [ + "node -1 0 {{vehicle}} dynamic pkp\\en57_v2 EN57-1000ra EN57RA_GSO_WKM1_KM_NS_N 0 headdriver 255 0 enddynamic", + "node -1 0 {{vehicle}}_s dynamic pkp\\en57_v2 EN57-1000s EN57S_WKM1_KM_NS_S 0 nobody 255 0 enddynamic", + "node -1 0 {{vehicle}}_rb dynamic pkp\\en57_v2 EN57-1000rb EN57RB_GSO_WKM1_KM_NS 0 nobody 0 0 enddynamic" + ] + }, + "duration": 240, + "timeout": 600, + "checks": [ + { "type": "starts", "min_speed": 20, "within": 150 }, + { "type": "max_speed", "limit": 90, "tolerance": 3 }, + { "type": "brakes", "min_pressure": 0.5 }, + { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + ] +} diff --git a/tests/scenarios/cases/ep09_td_timetable.json b/tests/scenarios/cases/ep09_td_timetable.json new file mode 100644 index 000000000..dd130be3c --- /dev/null +++ b/tests/scenarios/cases/ep09_td_timetable.json @@ -0,0 +1,17 @@ +{ + "description": "EP09 on td.scn, AI driver on the td/rozklad timetable: starts from Stacja1, keeps to the speed limit, brakes to a stop at Przystanek.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_timetable.scn", + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\ep09_v1 104e-005 104e_3 0 headdriver 0 0 enddynamic" + }, + "duration": 240, + "timeout": 600, + "checks": [ + { "type": "starts", "min_speed": 20, "within": 150 }, + { "type": "max_speed", "limit": 90, "tolerance": 3 }, + { "type": "brakes", "min_pressure": 0.5 }, + { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + ] +} diff --git a/tests/scenarios/cases/et22_td_timetable.json b/tests/scenarios/cases/et22_td_timetable.json new file mode 100644 index 000000000..004f5354d --- /dev/null +++ b/tests/scenarios/cases/et22_td_timetable.json @@ -0,0 +1,17 @@ +{ + "description": "ET22 on td.scn, AI driver on the td/rozklad timetable: starts from Stacja1, keeps to the speed limit, brakes to a stop at Przystanek.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_timetable.scn", + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\et22_v3 201E-1150 201E 0 headdriver 0 0 enddynamic" + }, + "duration": 240, + "timeout": 600, + "checks": [ + { "type": "starts", "min_speed": 20, "within": 150 }, + { "type": "max_speed", "limit": 90, "tolerance": 3 }, + { "type": "brakes", "min_pressure": 0.5 }, + { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + ] +} diff --git a/tests/scenarios/cases/eu07_td_red_signal.json b/tests/scenarios/cases/eu07_td_red_signal.json new file mode 100644 index 000000000..b02acdc81 --- /dev/null +++ b/tests/scenarios/cases/eu07_td_red_signal.json @@ -0,0 +1,18 @@ +{ + "description": "EU07 on td.scn with no route set: pulls off, stops before the exit signal stacja1_j at stop, and stays there.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_red_signal.scn", + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic" + }, + "duration": 240, + "timeout": 600, + "checks": [ + { "type": "starts", "min_speed": 5, "within": 150 }, + { "type": "max_speed", "limit": 40 }, + { "type": "stops", "moving_speed": 5, "stop_speed": 0.5, "hold": 5 }, + { "type": "stays_stopped", "moving_speed": 5, "stop_speed": 0.5 }, + { "type": "distance", "max": 150 } + ] +} diff --git a/tests/scenarios/cases/eu07_td_timetable.json b/tests/scenarios/cases/eu07_td_timetable.json index c12b8ae71..2f23019fe 100644 --- a/tests/scenarios/cases/eu07_td_timetable.json +++ b/tests/scenarios/cases/eu07_td_timetable.json @@ -2,8 +2,10 @@ "description": "EU07 on td.scn, AI driver on the td/rozklad timetable: starts from Stacja1, keeps to the speed limit, brakes to a stop at Przystanek.", "base_scenery": "td.scn", "time": "10:34", - "fragment": "eu07_td_timetable.scn", - "vehicle": "autotest_eu07", + "fragment": "td_timetable.scn", + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic" + }, "duration": 240, "timeout": 600, "checks": [ diff --git a/tests/scenarios/cases/sm42_td_timetable.json b/tests/scenarios/cases/sm42_td_timetable.json new file mode 100644 index 000000000..fc89be0fa --- /dev/null +++ b/tests/scenarios/cases/sm42_td_timetable.json @@ -0,0 +1,17 @@ +{ + "description": "SM42 diesel shunter on td.scn, AI driver on the td/rozklad timetable: starts from Stacja1, keeps to the speed limit, brakes to a stop at Przystanek.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_timetable.scn", + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\sm42_v1 6D-468 6D 0 headdriver 0 0 enddynamic" + }, + "duration": 240, + "timeout": 600, + "checks": [ + { "type": "starts", "min_speed": 20, "within": 150 }, + { "type": "max_speed", "limit": 90, "tolerance": 3 }, + { "type": "brakes", "min_pressure": 0.5 }, + { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + ] +} diff --git a/tests/scenarios/cases/sn61_td_timetable.json b/tests/scenarios/cases/sn61_td_timetable.json new file mode 100644 index 000000000..1db425c80 --- /dev/null +++ b/tests/scenarios/cases/sn61_td_timetable.json @@ -0,0 +1,17 @@ +{ + "description": "SN61 diesel railcar on td.scn, AI driver on the td/rozklad timetable: starts from Stacja1, keeps to the speed limit, brakes to a stop at Przystanek.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_timetable.scn", + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\sn61_v2 SN61-13 SN61_v1 0 headdriver 0 0 enddynamic" + }, + "duration": 240, + "timeout": 600, + "checks": [ + { "type": "starts", "min_speed": 20, "within": 150 }, + { "type": "max_speed", "limit": 90, "tolerance": 3 }, + { "type": "brakes", "min_pressure": 0.5 }, + { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + ] +} diff --git a/tests/scenarios/cases/td_red_signal.scn b/tests/scenarios/cases/td_red_signal.scn new file mode 100644 index 000000000..981707085 --- /dev/null +++ b/tests/scenarios/cases/td_red_signal.scn @@ -0,0 +1,6 @@ +// added by tests/scenarios/run.py to td.scn, whose own trainsets were removed. +// same start as td_timetable.scn, but nobody sets the route: the exit signal stacja1_j, +// about 150 m ahead of the train, stays at stop. +trainset td/rozklad tdo_n1734 0.0 0.1 +{{consist}} +endtrainset diff --git a/tests/scenarios/cases/eu07_td_timetable.scn b/tests/scenarios/cases/td_timetable.scn similarity index 51% rename from tests/scenarios/cases/eu07_td_timetable.scn rename to tests/scenarios/cases/td_timetable.scn index c8044e467..e67dcf9ad 100644 --- a/tests/scenarios/cases/eu07_td_timetable.scn +++ b/tests/scenarios/cases/td_timetable.scn @@ -1,11 +1,12 @@ -// added by tests/scenarios/run.py to the base scenery, whose own trainsets were removed: +// added by tests/scenarios/run.py to td.scn, whose own trainsets were removed: // the trainset goes after FirstInit, the launcher before it. -// the EU07 stands where td.scn keeps its timetable train and gets the same timetable. +// the case's consist stands where td.scn keeps its timetable train (track 1 of Stacja1) +// and gets the same timetable: Stacja1 10:35 -> Przystanek -> Stacja2. trainset td/rozklad tdo_n1734 0.0 0.1 -node -1 0 autotest_eu07 dynamic pkp\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic +{{consist}} endtrainset // td.scn sets routes from the keyboard (shift+1..4 run events keyctrl01..04). keyctrl04 is the -// way out of the track the train stands on: switch zwr02 and the exit signal stacja1_j at 40. +// way out of track 1: switch zwr02 and the exit signal stacja1_j at 40. // a launcher with a positive time fires once at that hour (HHMM), radius -1 means anywhere. node -1 0 autotest_route eventlauncher 0 0 0 -1 none 1034 keyctrl04 end diff --git a/tests/scenarios/cases/vectron_td_timetable.json b/tests/scenarios/cases/vectron_td_timetable.json new file mode 100644 index 000000000..9948a09ef --- /dev/null +++ b/tests/scenarios/cases/vectron_td_timetable.json @@ -0,0 +1,17 @@ +{ + "description": "Vectron 1293-107 on td.scn, AI driver on the td/rozklad timetable: starts from Stacja1, keeps to the speed limit, brakes to a stop at Przystanek.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_timetable.scn", + "vars": { + "consist": "node -1 0 {{vehicle}} dynamic pkp\\vectron_v1 1293-107 wariant_7w 0 headdriver 0 0 enddynamic" + }, + "duration": 240, + "timeout": 600, + "checks": [ + { "type": "starts", "min_speed": 20, "within": 150 }, + { "type": "max_speed", "limit": 90, "tolerance": 3 }, + { "type": "brakes", "min_pressure": 0.5 }, + { "type": "stops", "moving_speed": 20, "stop_speed": 0.5, "hold": 5 } + ] +} diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index b3736a929..419906700 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -2,11 +2,11 @@ """Scenario tests: run the real simulator on a scenery and check what a vehicle did. A case (cases/.json) names a base scenery from the game install, a fragment -appended to it (usually a trainset with an AI driver), and checks on the vehicle's +added to it (usually a trainset with an AI driver), and checks on the vehicle's physics log. The runner: 1. builds scenery/autotest_.scn in the game dir: the base scenery with its own - trainsets removed, optionally a different start time, and the case fragment appended, + trainsets removed, optionally a different start time, and the case fragment added, 2. starts eu07 with that scenery in ghostview (every train driven by AI) and an extra settings file (-ini) that turns on physicslog and runs in a window, 3. waits until the vehicle's physicslog covers `duration` seconds of simulation, @@ -14,10 +14,10 @@ physics log. The runner: 4. evaluates the checks on the log and prints PASS/FAIL per check. Scenery and vehicle data are not part of this repo, so this runs against a local game -install only (--game-dir or EU07_GAME_DIR), not in CI. +install only (--game-dir or EU07_GAME_DIR), not in CI. See README.md. Usage: - python tests/scenarios/run.py --game-dir "E:/.../MaSzyna" # all cases + python tests/scenarios/run.py --game-dir "E:/.../MaSzyna" -j 4 # all cases python tests/scenarios/run.py --game-dir ... eu07_td_timetable # chosen cases """ @@ -31,6 +31,7 @@ import shutil import subprocess import sys import time +from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from pathlib import Path @@ -54,9 +55,11 @@ height 720 inactivepause no soundenabled no physicslog yes -multiplelogs no +multiplelogs yes timespeed {timespeed} """ +# multiplelogs: log and errors go to logs/log () .txt instead of the shared +# log.txt, so runs of different cases can go in parallel # ---------------------------------------------------------------- scenery @@ -83,7 +86,19 @@ def set_start_time(text: str, hhmm: str) -> str: return re.sub(r"(?im)^(\s*time\s+)\d{1,2}:\d{2}", lambda m: m.group(1) + hhmm, text, count=1) -def build_scenery(case: dict, game_dir: Path, case_name: str) -> str: +def fill_vars(fragment: str, values: dict, keep=frozenset()) -> str: + """Replaces {{name}} with the case's value; a list value becomes one line per item.""" + def value(m): + if m.group(1) in keep: + return m.group(0) + v = values.get(m.group(1)) + if v is None: + sys.exit(f"fragment uses {{{{{m.group(1)}}}}} but the case sets no such var") + return "\n".join(v) if isinstance(v, list) else str(v) + return re.sub(r"\{\{(\w+)\}\}", value, fragment) + + +def build_scenery(case: dict, game_dir: Path, case_name: str, vehicle: str) -> str: """Writes the case scenery into the game's scenery dir and returns its file name.""" scenery_dir = game_dir / "scenery" # sceneries are cp1250; latin-1 round-trips every byte, so nothing gets mangled @@ -93,6 +108,9 @@ def build_scenery(case: dict, game_dir: Path, case_name: str) -> str: if "time" in case: text = set_start_time(text, case["time"]) fragment = (CASES_DIR / case["fragment"]).read_bytes().decode("latin-1") + # vars first: {{vehicle}} inside a var's value gets filled in the second pass + fragment = fill_vars(fragment, case.get("vars", {}), keep={"vehicle"}) + fragment = fill_vars(fragment, {"vehicle": vehicle}) # FirstInit binds event launchers to their events and links the tracks. so launchers and # events of the case go in before it (read after it they never fire), trainsets after it # (placed before it they stand on unlinked track and derail at once) @@ -106,6 +124,12 @@ def build_scenery(case: dict, game_dir: Path, case_name: str) -> str: text += header + trainsets + "\n" name = f"autotest_{case_name}.scn" (scenery_dir / name).write_bytes(text.encode("latin-1")) + # the simulator caches the scenery's static geometry in .sbt and reads it instead of + # the terrain in the .scn (tracks, events and trainsets always come from the .scn), so + # drop the cache once the base scenery has changed + cache = scenery_dir / f"autotest_{case_name}.sbt" + if cache.exists() and cache.stat().st_mtime < (scenery_dir / case["base_scenery"]).stat().st_mtime: + cache.unlink() return name @@ -130,19 +154,47 @@ def read_physicslog(path: Path) -> list[list[float]]: return rows +def find_logs(game_dir: Path, scenery: str, since: float) -> dict[str, Path]: + """The log.txt / errors.txt of one run; with multiplelogs each scenery gets its own files.""" + stem = scenery.rsplit(".", 1)[0] + found = {} + for kind in ("log", "errors"): + files = [p for p in (game_dir / "logs").glob(f"{kind} ({stem}) *.txt") if p.stat().st_mtime >= since] + if files: + found[kind] = max(files, key=lambda p: p.stat().st_mtime) + return found + + +def read_text(path: Path | None) -> str: + try: + return path.read_text(errors="replace") if path else "" + except FileNotFoundError: + return "" + + +# load problems that leave the case without its vehicle; no point waiting for the timeout +FATAL_LOG_LINES = ("Bad vehicle:", "Bad trainset:") +# wall seconds from start to the first physicslog row before giving up. "Scenario loading +# time" in the log is not the end of loading (the trainsets after FirstInit still follow), +# and with several simulators loading at once the first rows took well over 15 s +STARTUP_TIMEOUT = 120.0 + + @dataclass class RunResult: rows: list[list[float]] = field(default_factory=list) exit_code: int | None = None # None: still running when we closed it (the normal end) - timed_out: bool = False + failure: str = "" # why the run was cut short wall_seconds: float = 0.0 + logs: dict[str, Path] = field(default_factory=dict) def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog: Path, - duration: float, timeout: float) -> RunResult: + duration: float, timeout: float, say) -> RunResult: physicslog.unlink(missing_ok=True) cmd = [str(exe), "-s", scenery, "-v", "ghostview", "-ini", str(ini)] - print(" $", " ".join(f'"{c}"' if " " in c else c for c in cmd)) + say(" $ " + " ".join(f'"{c}"' if " " in c else c for c in cmd), progress=True) + since = time.time() - 1 started = time.monotonic() proc = subprocess.Popen(cmd, cwd=game_dir) result = RunResult() @@ -150,12 +202,12 @@ def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog try: while True: time.sleep(0.25) # at timespeed 20 that is 5 s of simulation + wall = time.monotonic() - started result.rows = read_physicslog(physicslog) sim_t = result.rows[-1][COL_TIME] if result.rows else 0.0 - wall = time.monotonic() - started if wall - last_report >= 15: v = result.rows[-1][COL_VELOCITY] if result.rows else 0.0 - print(f" ... {wall:5.0f}s wall, sim {sim_t:6.1f}s, v={v:6.1f} km/h") + say(f" ... {wall:5.0f}s wall, sim {sim_t:6.1f}s, v={v:6.1f} km/h", progress=True) last_report = wall if proc.poll() is not None: result.exit_code = proc.returncode @@ -163,8 +215,18 @@ def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog if sim_t >= duration: break if wall > timeout: - result.timed_out = True + result.failure = f"timed out after {timeout:.0f}s before {duration}s of simulation" break + if not result.rows: + result.logs = find_logs(game_dir, scenery, since) + fatal = [l.strip() for l in read_text(result.logs.get("errors")).splitlines() + if any(k in l for k in FATAL_LOG_LINES)] + if fatal: + result.failure = "vehicle failed to load: " + " | ".join(fatal) + break + if wall > STARTUP_TIMEOUT: + result.failure = f"no physicslog rows after {STARTUP_TIMEOUT:.0f}s" + break finally: if proc.poll() is None: proc.terminate() @@ -174,6 +236,7 @@ def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog proc.kill() result.wall_seconds = time.monotonic() - started result.rows = read_physicslog(physicslog) + result.logs = find_logs(game_dir, scenery, since) return result @@ -201,6 +264,15 @@ def _first_moving(rows, moving_speed): return next((i for i, r in enumerate(rows) if r[COL_VELOCITY] >= moving_speed), None) +def _distances(rows): + """Metres travelled up to each row, from the speed; the physicslog has no position.""" + out, d = [0.0], 0.0 + for a, b in zip(rows, rows[1:]): + d += (a[COL_VELOCITY] + b[COL_VELOCITY]) / 2 / 3.6 * (b[COL_TIME] - a[COL_TIME]) + out.append(d) + return out + + def check_brakes(rows, min_pressure, moving_speed=20): start = _first_moving(rows, moving_speed) if start is None: @@ -215,23 +287,49 @@ def check_stops(rows, moving_speed, stop_speed, hold): start = _first_moving(rows, moving_speed) if start is None: return False, f"never reached {moving_speed} km/h" + dist = _distances(rows) stopped_at = None - for r in rows[start:]: + for i in range(start, len(rows)): + r = rows[i] if r[COL_VELOCITY] <= stop_speed: if stopped_at is None: - stopped_at = r[COL_TIME] - if r[COL_TIME] - stopped_at >= hold: - return True, f"came to a stop at t={stopped_at:.1f}s" + stopped_at = i + if r[COL_TIME] - rows[stopped_at][COL_TIME] >= hold: + return True, f"came to a stop at t={rows[stopped_at][COL_TIME]:.1f}s after {dist[stopped_at]:.0f} m" else: stopped_at = None return False, f"did not stay below {stop_speed} km/h for {hold}s after moving off" +def check_stays_stopped(rows, moving_speed, stop_speed): + """Once stopped after moving off, never moves again (e.g. waiting at a red signal).""" + start = _first_moving(rows, moving_speed) + if start is None: + return False, f"never reached {moving_speed} km/h" + stop = next((i for i in range(start, len(rows)) if rows[i][COL_VELOCITY] <= stop_speed), None) + if stop is None: + return False, f"never came down to {stop_speed} km/h" + moved = next((r for r in rows[stop:] if r[COL_VELOCITY] > stop_speed), None) + if moved is not None: + return False, f"stopped at t={rows[stop][COL_TIME]:.1f}s but moved again at t={moved[COL_TIME]:.1f}s, v={moved[COL_VELOCITY]:.1f} km/h" + return True, f"stopped at t={rows[stop][COL_TIME]:.1f}s and stayed until t={rows[-1][COL_TIME]:.1f}s" + + +def check_distance(rows, max=None, min=None): + """Total distance travelled, in metres.""" + d = _distances(rows)[-1] + ok = (max is None or d <= max) and (min is None or d >= min) + bounds = " ".join(s for s in (f">= {min}" if min is not None else "", f"<= {max}" if max is not None else "") if s) + return ok, f"travelled {d:.0f} m (wanted {bounds})" + + CHECKS = { "starts": check_starts, "max_speed": check_max_speed, "brakes": check_brakes, "stops": check_stops, + "stays_stopped": check_stays_stopped, + "distance": check_distance, } @@ -254,49 +352,52 @@ def count_lines(path: Path) -> int: return 0 -def run_case(name: str, exe: Path, game_dir: Path, timespeed: float) -> bool: +def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, say) -> bool: case = json.loads((CASES_DIR / f"{name}.json").read_text(encoding="utf-8")) - print(f"\n=== {name}: {case.get('description', '')}") + say(f"\n=== {name}: {case.get('description', '')}") out = OUT_DIR / name shutil.rmtree(out, ignore_errors=True) out.mkdir(parents=True) - scenery = build_scenery(case, game_dir, name) + # the lead vehicle is named after the case, so parallel runs never share a physicslog + vehicle = f"autotest_{name}" + scenery = build_scenery(case, game_dir, name, vehicle) ini = out / "autotest.ini" ini.write_text(TEST_INI.format(timespeed=timespeed), encoding="ascii") - physicslog = game_dir / "physicslog" / f"{case['vehicle']}.dat" + physicslog = game_dir / "physicslog" / f"{vehicle}.dat" result = run_simulator(exe, game_dir, scenery, ini, physicslog, - float(case["duration"]), float(case.get("timeout", 600))) + float(case["duration"]), float(case.get("timeout", 600)), say) - for f in ("log.txt", "errors.txt"): - if (game_dir / f).exists(): - shutil.copy2(game_dir / f, out / f) + # the logs are ours alone (named after the case scenery), move them out of the game dir + for kind, path in result.logs.items(): + shutil.move(path, out / f"{kind}.txt") if physicslog.exists(): - shutil.copy2(physicslog, out / physicslog.name) + shutil.move(physicslog, out / physicslog.name) ok = True sim_t = result.rows[-1][COL_TIME] if result.rows else 0.0 - print(f" ran {result.wall_seconds:.0f}s wall, {sim_t:.1f}s of physicslog, results in {out.relative_to(REPO)}") + say(f" ran {result.wall_seconds:.0f}s wall, {sim_t:.1f}s of physicslog, results in {out.relative_to(REPO)}") if result.exit_code is not None: - print(f" FAIL simulator exited on its own with code {result.exit_code} (crash?)") + say(f" FAIL simulator exited on its own with code {result.exit_code}") ok = False - if result.timed_out: - print(f" FAIL timed out before {case['duration']}s of simulation") + if result.failure: + say(f" FAIL {result.failure}") ok = False if not result.rows: - print(f" FAIL no physicslog for '{case['vehicle']}' ({physicslog})") + if not result.failure: + say(f" FAIL no physicslog for '{vehicle}'") return False for check in case["checks"]: params = {k: v for k, v in check.items() if k != "type"} passed, detail = CHECKS[check["type"]](result.rows, **params) - print(f" {'PASS' if passed else 'FAIL'} {check['type']}: {detail}") + say(f" {'PASS' if passed else 'FAIL'} {check['type']}: {detail}") ok &= passed errors = count_lines(out / "errors.txt") if errors: - print(f" note errors.txt has {errors} line(s)") + say(f" note errors.txt has {errors} line(s)") if case.get("fail_on_errors") and errors: ok = False return ok @@ -312,6 +413,8 @@ def main() -> int: # in 20 s instead of 4 min; 1 is for watching a run ap.add_argument("--timespeed", type=float, default=20.0, help="simulation time multiplier, the simulator offers 1, 5, 20 and 60 (default 20)") + ap.add_argument("-j", "--jobs", type=int, default=1, + help="cases run at once, each in its own simulator window (default 1)") args = ap.parse_args() if not args.game_dir: @@ -321,11 +424,37 @@ def main() -> int: print(f"exe: {exe}\ngame: {game_dir}") names = args.cases or sorted(p.stem for p in CASES_DIR.glob("*.json")) - results = {n: run_case(n, exe, game_dir, args.timespeed) for n in names} + missing = [n for n in names if not (CASES_DIR / f"{n}.json").exists()] + if missing: + sys.exit("no such case: " + ", ".join(missing)) - print("\n=== summary") - for n, ok in results.items(): - print(f" {'PASS' if ok else 'FAIL'} {n}") + started = time.monotonic() + results = {} + if args.jobs <= 1: + def say(line, progress=False): + print(line, flush=True) + for n in names: + results[n] = run_case(n, exe, game_dir, args.timespeed, say) + else: + # each case prints its block when it is done, so parallel output does not interleave + def job(n): + lines = [] + def say(line, progress=False): + if not progress: + lines.append(line) + ok = run_case(n, exe, game_dir, args.timespeed, say) + return n, ok, lines + with ThreadPoolExecutor(max_workers=args.jobs) as pool: + for n in names: + print(f"started {n}", flush=True) + for fut in as_completed([pool.submit(job, n) for n in names]): + n, ok, lines = fut.result() + print("\n".join(lines), flush=True) + results[n] = ok + + print(f"\n=== summary ({time.monotonic() - started:.0f}s)") + for n in names: + print(f" {'PASS' if results[n] else 'FAIL'} {n}") return 0 if all(results.values()) else 1