mirror of
https://github.com/MaSzyna-EU07/maszyna.git
synced 2026-11-20 12:44:59 +01:00
The checks of en57_2000_automatic_coupler_couples said only that nothing collided nor derailed, so the case passed just as well with the units buffering, bouncing apart or never meeting. The simulator writes nothing to the log when automatic couplers catch, but the black box of the run (physicslog/<scenery>_<date>.blackbox, written along with the physicslogs the runner turns on) gets a coupling record with the two vehicles. run.py reads the vehicle and the coupling records of it and gets a check on them, couples: the two named vehicles couple after the start and are still coupled at the end. It fails when the black box has no coupling of that pair, when they came apart again, when they were coupled from the start, and when the run left no black box. The case names the vehicles that meet: the last car of the standing unit and the first of the rolling one. The black box is moved to out/<case>/ with the other files of the run.
799 lines
36 KiB
Python
799 lines
36 KiB
Python
#!/usr/bin/env python3
|
|
"""Scenario tests: run the real simulator on a scenery and check what a vehicle did.
|
|
|
|
A case (cases/<name>.json) names a base scenery from the game install, a fragment
|
|
added to it (usually a trainset with an AI driver), the result the simulator's test
|
|
mode should end with, and checks on the vehicle's physics log. The runner:
|
|
|
|
1. builds scenery/autotest_<case>.scn in the game dir: the base scenery with its own
|
|
trainsets removed, optionally a different start time, and the case fragment added,
|
|
2. starts eu07 in test mode (--test: every train driven by AI, nothing drawn, fixed
|
|
time steps as fast as it can) with that scenery and an extra settings file (-ini)
|
|
that turns on physicslog,
|
|
3. waits until the simulator quits by itself and reads the test result from its exit
|
|
code: pass, fail, signal passed at danger, timeout (after `duration` seconds of
|
|
simulation, if the case gives one), standstill or error,
|
|
4. compares the result with the one the case expects, evaluates the checks on the
|
|
physicslog and prints PASS/FAIL per check.
|
|
|
|
Scenery and vehicle data are not part of this repo, so this runs against a local game
|
|
install only (--game-dir or EU07_GAME_DIR), not in CI. See README.md.
|
|
|
|
Usage:
|
|
python tests/scenarios/run.py --game-dir "E:/.../MaSzyna" -j 4 # all cases
|
|
python tests/scenarios/run.py --game-dir ... eu07_td_timetable # chosen cases
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
REPO = HERE.parents[1]
|
|
CASES_DIR = HERE / "cases"
|
|
OUT_DIR = HERE / "out"
|
|
|
|
# physicslog columns, see TController::PhysicsLog() in vehicle/Driver.cpp.
|
|
# Velocity is in km/h and pressures in bar, despite the header saying m/s and MPa.
|
|
COL_TIME = 0
|
|
COL_VELOCITY = 1 # wheel speed: stays at 0 while the wheels are locked in a skid
|
|
COL_BRAKE_PRESS = 8
|
|
COL_COMMAND = 16 # text, read as None
|
|
COL_WHEELSLIP = 20
|
|
COL_VEHICLE_VELOCITY = 25 # signed vehicle speed, km/h; only in builds which log it
|
|
|
|
# exit codes of the simulator's test mode (eu07 --test), see simulation/simulationtestrules.h
|
|
EXIT_RESULTS = {
|
|
0: "pass",
|
|
10: "fail",
|
|
11: "signal_passed_at_danger",
|
|
12: "timeout",
|
|
13: "standstill",
|
|
14: "error",
|
|
}
|
|
# the timeout of the simulator's real time watchdog (test.wall_timeout) has the timeout code too; told
|
|
# apart by its verdict, so a run that hung doesn't pass a case that expects the simulation time to run out
|
|
WALL_TIMEOUT = "wall_timeout"
|
|
RESULTS = (*EXIT_RESULTS.values(), WALL_TIMEOUT)
|
|
|
|
# settings layered over the game's eu07.ini for a test run (test mode doesn't read the player's own
|
|
# in %APPDATA%); the window ones only matter
|
|
# with --watch, test mode keeps its window hidden and turns the sound off by itself
|
|
TEST_INI = """\
|
|
fullscreen no
|
|
fullscreenwindowed no
|
|
width 1280
|
|
height 720
|
|
inactivepause no
|
|
soundenabled no
|
|
physicslog yes
|
|
multiplelogs yes
|
|
timespeed {timespeed}
|
|
"""
|
|
# multiplelogs: log and errors go to logs/log (<scenery>) <date>.txt instead of the shared
|
|
# log.txt, so runs of different cases can go in parallel
|
|
|
|
|
|
# ---------------------------------------------------------------- scenery
|
|
|
|
|
|
def split_trainsets(text: str) -> tuple[str, str]:
|
|
"""Splits text into (everything else, the trainset ... endtrainset blocks)."""
|
|
rest, trainsets = [], []
|
|
inside = False
|
|
for line in text.splitlines(keepends=True):
|
|
tokens = line.split("//", 1)[0].split()
|
|
if not inside and tokens and tokens[0].lower() == "trainset":
|
|
inside = True
|
|
if inside:
|
|
trainsets.append(line)
|
|
if any(t.lower() == "endtrainset" for t in tokens):
|
|
inside = False
|
|
continue
|
|
rest.append(line)
|
|
return "".join(rest), "".join(trainsets)
|
|
|
|
|
|
def set_start_time(text: str, hhmm: str) -> str:
|
|
return re.sub(r"(?im)^(\s*time\s+)\d{1,2}:\d{2}", lambda m: m.group(1) + hhmm, text, count=1)
|
|
|
|
|
|
def add_track_events(text: str, events: dict) -> str:
|
|
"""Adds lines (e.g. "event0 test_pass") to tracks named in events, before their endtrack."""
|
|
for track, lines in events.items():
|
|
lines = [lines] if isinstance(lines, str) else lines
|
|
m = re.search(rf"(?im)^\s*node\s+\S+\s+\S+\s+{re.escape(track)}\s+track\b.*?^(\s*endtrack\b)", text, re.S)
|
|
if not m:
|
|
sys.exit(f"track_events: no track {track} in the base scenery itself (not in its includes)")
|
|
text = text[: m.start(1)] + "".join(line + "\n" for line in lines) + text[m.start(1):]
|
|
return text
|
|
|
|
|
|
def set_track_friction(text: str, friction: float) -> str:
|
|
"""Sets the friction of every track: node <range> <range> <name> track <type> <length> <gauge> <friction> ..."""
|
|
return re.sub(r"(?im)^(\s*node\s+\S+\s+\S+\s+\S+\s+track\s+\S+\s+\S+\s+\S+\s+)[-\d.]+",
|
|
lambda m: m.group(1) + f"{friction}", text)
|
|
|
|
|
|
def fill_vars(fragment: str, values: dict, keep=frozenset()) -> str:
|
|
"""Replaces {{name}} with the case's value; a list value becomes one line per item."""
|
|
def value(m):
|
|
if m.group(1) in keep:
|
|
return m.group(0)
|
|
v = values.get(m.group(1))
|
|
if v is None:
|
|
sys.exit(f"fragment uses {{{{{m.group(1)}}}}} but the case sets no such var")
|
|
return "\n".join(v) if isinstance(v, list) else str(v)
|
|
return re.sub(r"\{\{(\w+)\}\}", value, fragment)
|
|
|
|
|
|
def scenery_cache(game_dir: Path, case_name: str) -> Path:
|
|
"""The simulator's geometry cache of the case scenery."""
|
|
return game_dir / "scenery" / f"autotest_{case_name}.sbt"
|
|
|
|
|
|
def build_scenery(case: dict, game_dir: Path, case_name: str, vehicle: str) -> str:
|
|
"""Writes the case scenery into the game's scenery dir and returns its file name."""
|
|
scenery_dir = game_dir / "scenery"
|
|
# sceneries are cp1250; latin-1 round-trips every byte, so nothing gets mangled
|
|
text = (scenery_dir / case["base_scenery"]).read_bytes().decode("latin-1")
|
|
if case.get("strip_trainsets", True):
|
|
text, _ = split_trainsets(text)
|
|
if "time" in case:
|
|
text = set_start_time(text, case["time"])
|
|
if "track_friction" in case:
|
|
text = set_track_friction(text, case["track_friction"])
|
|
if "track_events" in case:
|
|
text = add_track_events(text, case["track_events"])
|
|
fragment = (CASES_DIR / case["fragment"]).read_bytes().decode("latin-1")
|
|
# vars first: {{vehicle}} inside a var's value gets filled in the second pass
|
|
fragment = fill_vars(fragment, case.get("vars", {}), keep={"vehicle"})
|
|
fragment = fill_vars(fragment, {"vehicle": vehicle})
|
|
# FirstInit binds event launchers to their events and links the tracks. so launchers and
|
|
# events of the case go in before it (read after it they never fire), trainsets after it
|
|
# (placed before it they stand on unlinked track and derail at once)
|
|
before, trainsets = split_trainsets(fragment)
|
|
# settings of the test mode, read like any other config block: the case's vehicle is the train under test,
|
|
# whose consist the pass and the standstill wait for, and the case's own (test.timeout and so on)
|
|
config = [f"test.vehicle {vehicle}", *case.get("test_config", [])]
|
|
before = "config\n" + "".join(line + "\n" for line in config) + "endconfig\n" + before
|
|
header = f"\n// ---- autotest: {case_name} ----\n"
|
|
firstinit = re.search(r"(?im)^\s*firstinit\b", text)
|
|
if firstinit:
|
|
text = text[: firstinit.start()] + header + before + "\n" + text[firstinit.start():]
|
|
else:
|
|
text += header + before + "\n"
|
|
text += header + trainsets + "\n"
|
|
name = f"autotest_{case_name}.scn"
|
|
(scenery_dir / name).write_bytes(text.encode("latin-1"))
|
|
# the simulator caches the scenery's static geometry in <name>.sbt and reads it instead of
|
|
# the terrain in the .scn (tracks, events and trainsets always come from the .scn), so
|
|
# drop the cache once the base scenery has changed
|
|
cache = scenery_cache(game_dir, case_name)
|
|
if cache.exists() and cache.stat().st_mtime < (scenery_dir / case["base_scenery"]).stat().st_mtime:
|
|
cache.unlink()
|
|
return name
|
|
|
|
|
|
# ---------------------------------------------------------------- run
|
|
|
|
|
|
def read_physicslog(path: Path) -> list[list[float]]:
|
|
"""Returns rows of columns, None for the text ones; skips the header and partial lines."""
|
|
rows = []
|
|
try:
|
|
lines = path.read_text(errors="replace").splitlines()
|
|
except FileNotFoundError:
|
|
return rows
|
|
for line in lines[1:]:
|
|
parts = line.split()
|
|
if len(parts) <= COL_BRAKE_PRESS:
|
|
continue # line still being written
|
|
try:
|
|
row = [float(p) for p in parts[: COL_BRAKE_PRESS + 1]]
|
|
except ValueError:
|
|
continue
|
|
for p in parts[COL_BRAKE_PRESS + 1:]:
|
|
try:
|
|
row.append(float(p))
|
|
except ValueError:
|
|
row.append(None)
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def find_logs(game_dir: Path, scenery: str, since: float) -> dict[str, Path]:
|
|
"""The log.txt / errors.txt of one run; with multiplelogs each scenery gets its own files."""
|
|
stem = scenery.rsplit(".", 1)[0]
|
|
found = {}
|
|
for kind in ("log", "errors"):
|
|
files = [p for p in (game_dir / "logs").glob(f"{kind} ({stem}) *.txt") if p.stat().st_mtime >= since]
|
|
if files:
|
|
found[kind] = max(files, key=lambda p: p.stat().st_mtime)
|
|
return found
|
|
|
|
|
|
def read_text(path: Path | None) -> str:
|
|
try:
|
|
return path.read_text(errors="replace") if path else ""
|
|
except FileNotFoundError:
|
|
return ""
|
|
|
|
|
|
# the black box of a run, see blackbox_doc.md: written along with the physicslogs, to
|
|
# physicslog/<scenery>_<date>.blackbox. A header, then records: a byte of the kind, the length, the content
|
|
BLACKBOX_MAGIC = b"EU07BBX\0"
|
|
BLACKBOX_HEADER = 12 # the magic, the version of the format, two bytes reserved
|
|
BLACKBOX_VEHICLE = 0x12 # a vehicle: its number and its name
|
|
BLACKBOX_COUPLING = 0x37 # a coupler of a vehicle coupled, uncoupled, or its hoses and cables changed
|
|
COUPLING_COUPLER = 0x01 # of the bits telling what is coupled: the coupler itself
|
|
|
|
|
|
@dataclass
|
|
class Coupling:
|
|
"""A coupling record: what a coupler of a vehicle is coupled to from `time` on."""
|
|
time: float
|
|
vehicle: int # vehicles by their numbers, 0 for none
|
|
side: int # 0 the front coupler, 1 the rear one
|
|
other: int
|
|
other_side: int
|
|
flags: int # what is coupled, 0 nothing
|
|
previous_flags: int
|
|
previous_other: int
|
|
|
|
|
|
@dataclass
|
|
class Blackbox:
|
|
vehicles: dict[int, str] = field(default_factory=dict) # names by number
|
|
couplings: list[Coupling] = field(default_factory=list) # in the order written
|
|
|
|
|
|
def find_blackbox(game_dir: Path, scenery: str, since: float) -> Path | None:
|
|
"""The black box of one run: the simulator names it after the scenery and the time it started."""
|
|
stem = re.sub(r"[^0-9A-Za-z_-]", "_", scenery.rsplit(".", 1)[0])
|
|
# the whole name, so a case whose name another one starts with doesn't take that one's file
|
|
name = re.compile(re.escape(stem) + r"_\d{4}-\d\d-\d\d_\d\d-\d\d-\d\d\.blackbox")
|
|
files = [p for p in (game_dir / "physicslog").glob(f"{stem}_*.blackbox")
|
|
if name.fullmatch(p.name) and p.stat().st_mtime >= since]
|
|
return max(files, key=lambda p: p.stat().st_mtime, default=None)
|
|
|
|
|
|
def read_blackbox(path: Path | None) -> Blackbox | None:
|
|
"""The vehicles and the coupling records of a black box; None when there is none or it can't be read."""
|
|
try:
|
|
data = path.read_bytes() if path else b""
|
|
except FileNotFoundError:
|
|
return None
|
|
if data[:len(BLACKBOX_MAGIC)] != BLACKBOX_MAGIC:
|
|
return None
|
|
box = Blackbox()
|
|
at = BLACKBOX_HEADER
|
|
try:
|
|
while at + 5 <= len(data):
|
|
kind, length = struct.unpack_from("<BI", data, at)
|
|
content = data[at + 5: at + 5 + length]
|
|
if len(content) < length:
|
|
break # a record still being written when the run ended
|
|
at += 5 + length
|
|
if kind == BLACKBOX_VEHICLE:
|
|
number, = struct.unpack_from("<I", content)
|
|
# a text: its length in 7 bit groups, low first, the high bit set on all but the last
|
|
size, shift, start = 0, 0, 4
|
|
while True:
|
|
byte = content[start]
|
|
start += 1
|
|
size |= (byte & 0x7F) << shift
|
|
shift += 7
|
|
if byte < 0x80:
|
|
break
|
|
box.vehicles[number] = content[start: start + size].decode("utf-8", errors="replace")
|
|
elif kind == BLACKBOX_COUPLING:
|
|
# the position of the coupler follows
|
|
box.couplings.append(Coupling(*struct.unpack_from("<dIBIBBBI", content)))
|
|
except (struct.error, IndexError):
|
|
return None # a record shorter than its kind is
|
|
return box
|
|
|
|
|
|
# load problems that leave the case without its vehicle; no point waiting for the timeout
|
|
FATAL_LOG_LINES = ("Bad vehicle:", "Bad trainset:")
|
|
# wall seconds from start to the first physicslog row before giving up. "Scenario loading
|
|
# time" in the log is not the end of loading (the trainsets after FirstInit still follow),
|
|
# and with several simulators loading at once the first rows took well over 15 s
|
|
STARTUP_TIMEOUT = 120.0
|
|
# wall seconds past a case's timeout before the runner kills a simulator that didn't end itself
|
|
KILL_GRACE = 30.0
|
|
|
|
|
|
@dataclass
|
|
class RunResult:
|
|
rows: list[list[float]] = field(default_factory=list)
|
|
exit_code: int | None = None # None: still running when we closed it (the normal end with --watch)
|
|
verdict: str = "" # the simulator's own line about the test result
|
|
failure: str = "" # why the run was cut short
|
|
wall_seconds: float = 0.0
|
|
logs: dict[str, Path] = field(default_factory=dict)
|
|
blackbox: Path | None = None
|
|
|
|
|
|
def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog: Path,
|
|
duration: float | None, timeout: float, watch: bool, say) -> RunResult:
|
|
"""Runs the simulator on the scenery. In test mode it quits by itself and the exit code
|
|
is the result; with watch it runs in a window and gets closed after `duration`."""
|
|
physicslog.unlink(missing_ok=True)
|
|
if watch:
|
|
cmd = [str(exe), "-s", scenery, "-v", "ghostview", "-ini", str(ini)]
|
|
else:
|
|
# the simulator's own watchdog ends a run that takes too long, with the timeout code and its verdict
|
|
cmd = [str(exe), "--test", "-s", scenery, "-ini", str(ini), "--test-wall-timeout", f"{timeout:g}"]
|
|
if duration is not None:
|
|
cmd += ["--test-timeout", f"{duration:g}"]
|
|
say(" $ " + " ".join(f'"{c}"' if " " in c else c for c in cmd), progress=True)
|
|
since = time.time() - 1
|
|
started = time.monotonic()
|
|
# the test result line goes to stdout; a file, not a pipe, so a chatty run can't block on it
|
|
stdout = (ini.parent / "stdout.txt").open("w+b")
|
|
proc = subprocess.Popen(cmd, cwd=game_dir, stdout=stdout, stderr=subprocess.STDOUT)
|
|
result = RunResult()
|
|
last_report = 0.0
|
|
try:
|
|
while True:
|
|
time.sleep(0.25) # at timespeed 20 that is 5 s of simulation
|
|
wall = time.monotonic() - started
|
|
result.rows = read_physicslog(physicslog)
|
|
sim_t = result.rows[-1][COL_TIME] if result.rows else 0.0
|
|
if wall - last_report >= 15:
|
|
v = result.rows[-1][COL_VELOCITY] if result.rows else 0.0
|
|
say(f" ... {wall:5.0f}s wall, sim {sim_t:6.1f}s, v={v:6.1f} km/h", progress=True)
|
|
last_report = wall
|
|
if proc.poll() is not None:
|
|
result.exit_code = proc.returncode
|
|
break
|
|
if watch and sim_t >= duration:
|
|
break
|
|
# in test mode a little later than the simulator's own limit, for a run that can't even end itself
|
|
if wall > timeout + (0 if watch else KILL_GRACE):
|
|
result.failure = f"timed out after {wall:.0f}s before {duration}s of simulation"
|
|
break
|
|
if not result.rows:
|
|
result.logs = find_logs(game_dir, scenery, since)
|
|
fatal = [l.strip() for l in read_text(result.logs.get("errors")).splitlines()
|
|
if any(k in l for k in FATAL_LOG_LINES)]
|
|
if fatal:
|
|
result.failure = "vehicle failed to load: " + " | ".join(fatal)
|
|
break
|
|
if wall > STARTUP_TIMEOUT:
|
|
result.failure = f"no physicslog rows after {STARTUP_TIMEOUT:.0f}s"
|
|
break
|
|
finally:
|
|
if proc.poll() is None:
|
|
proc.terminate()
|
|
try:
|
|
proc.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
proc.kill()
|
|
result.wall_seconds = time.monotonic() - started
|
|
stdout.seek(0)
|
|
result.verdict = next((line.strip() for line in stdout.read().decode(errors="replace").splitlines()
|
|
if line.startswith("test:")), "")
|
|
stdout.close()
|
|
result.rows = read_physicslog(physicslog)
|
|
result.logs = find_logs(game_dir, scenery, since)
|
|
result.blackbox = find_blackbox(game_dir, scenery, since)
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------- checks
|
|
|
|
|
|
def check_starts(rows, min_speed, within):
|
|
for r in rows:
|
|
if r[COL_TIME] > within:
|
|
break
|
|
if r[COL_VELOCITY] >= min_speed:
|
|
return True, f"reached {min_speed} km/h at t={r[COL_TIME]:.1f}s"
|
|
top = max((r[COL_VELOCITY] for r in rows if r[COL_TIME] <= within), default=0.0)
|
|
return False, f"top speed in the first {within}s was {top:.1f} km/h, wanted {min_speed}"
|
|
|
|
|
|
def check_max_speed(rows, limit, tolerance=0.0):
|
|
worst = max(rows, key=lambda r: r[COL_VELOCITY])
|
|
v, t = worst[COL_VELOCITY], worst[COL_TIME]
|
|
ok = v <= limit + tolerance
|
|
return ok, f"top speed {v:.1f} km/h at t={t:.1f}s (limit {limit} +{tolerance})"
|
|
|
|
|
|
def _first_moving(rows, moving_speed):
|
|
return next((i for i, r in enumerate(rows) if r[COL_VELOCITY] >= moving_speed), None)
|
|
|
|
|
|
def _distances(rows):
|
|
"""Metres travelled up to each row, from the speed; the physicslog has no position."""
|
|
out, d = [0.0], 0.0
|
|
for a, b in zip(rows, rows[1:]):
|
|
d += (a[COL_VELOCITY] + b[COL_VELOCITY]) / 2 / 3.6 * (b[COL_TIME] - a[COL_TIME])
|
|
out.append(d)
|
|
return out
|
|
|
|
|
|
def check_brakes(rows, min_pressure, moving_speed=20):
|
|
start = _first_moving(rows, moving_speed)
|
|
if start is None:
|
|
return False, "never moved, so never braked"
|
|
for r in rows[start:]:
|
|
if r[COL_BRAKE_PRESS] >= min_pressure:
|
|
return True, f"brake cylinder {r[COL_BRAKE_PRESS]:.2f} bar at t={r[COL_TIME]:.1f}s, v={r[COL_VELOCITY]:.1f} km/h"
|
|
return False, f"brake cylinder pressure never reached {min_pressure} bar after moving off"
|
|
|
|
|
|
def check_stops(rows, moving_speed, stop_speed, hold, min_distance=None, max_distance=None):
|
|
start = _first_moving(rows, moving_speed)
|
|
if start is None:
|
|
return False, f"never reached {moving_speed} km/h"
|
|
dist = _distances(rows)
|
|
stopped_at = None
|
|
for i in range(start, len(rows)):
|
|
r = rows[i]
|
|
if r[COL_VELOCITY] <= stop_speed:
|
|
if stopped_at is None:
|
|
stopped_at = i
|
|
if r[COL_TIME] - rows[stopped_at][COL_TIME] >= hold:
|
|
where = dist[stopped_at]
|
|
detail = f"came to a stop at t={rows[stopped_at][COL_TIME]:.1f}s after {where:.0f} m"
|
|
if (min_distance is not None and where < min_distance) or (max_distance is not None and where > max_distance):
|
|
return False, f"{detail}, wanted between {min_distance} and {max_distance} m"
|
|
return True, detail
|
|
else:
|
|
stopped_at = None
|
|
return False, f"did not stay below {stop_speed} km/h for {hold}s after moving off"
|
|
|
|
|
|
def check_stays_stopped(rows, moving_speed, stop_speed):
|
|
"""Once stopped after moving off, never moves again (e.g. waiting at a red signal)."""
|
|
start = _first_moving(rows, moving_speed)
|
|
if start is None:
|
|
return False, f"never reached {moving_speed} km/h"
|
|
stop = next((i for i in range(start, len(rows)) if rows[i][COL_VELOCITY] <= stop_speed), None)
|
|
if stop is None:
|
|
return False, f"never came down to {stop_speed} km/h"
|
|
moved = next((r for r in rows[stop:] if r[COL_VELOCITY] > stop_speed), None)
|
|
if moved is not None:
|
|
return False, f"stopped at t={rows[stop][COL_TIME]:.1f}s but moved again at t={moved[COL_TIME]:.1f}s, v={moved[COL_VELOCITY]:.1f} km/h"
|
|
return True, f"stopped at t={rows[stop][COL_TIME]:.1f}s and stayed until t={rows[-1][COL_TIME]:.1f}s"
|
|
|
|
|
|
def check_distance(rows, max=None, min=None):
|
|
"""Total distance travelled, in metres."""
|
|
d = _distances(rows)[-1]
|
|
ok = (max is None or d <= max) and (min is None or d >= min)
|
|
bounds = " ".join(s for s in (f">= {min}" if min is not None else "", f"<= {max}" if max is not None else "") if s)
|
|
return ok, f"travelled {d:.0f} m (wanted {bounds})"
|
|
|
|
|
|
def check_slips(rows, within=None):
|
|
"""The lead vehicle's wheels slip (or lock in a skid) at some point, e.g. to make sure a slippery rail case does what it's meant to."""
|
|
for r in rows:
|
|
if within is not None and r[COL_TIME] > within:
|
|
break
|
|
if len(r) > COL_WHEELSLIP and r[COL_WHEELSLIP]:
|
|
return True, f"wheels slipping at t={r[COL_TIME]:.1f}s, v={r[COL_VELOCITY]:.1f} km/h"
|
|
return False, "the wheels never slipped"
|
|
|
|
|
|
def check_holds_still(rows, moving_speed, stop_speed, max_distance):
|
|
"""Once stopped after moving off, the vehicle doesn't creep. Uses the vehicle speed rather than the
|
|
wheel speed, so it also catches a vehicle sliding on locked wheels."""
|
|
if not rows or len(rows[0]) <= COL_VEHICLE_VELOCITY or rows[0][COL_VEHICLE_VELOCITY] is None:
|
|
return False, "the physicslog has no vehicle speed column, the build is too old"
|
|
speed = [abs(r[COL_VEHICLE_VELOCITY]) for r in rows]
|
|
start = next((i for i, v in enumerate(speed) if v >= moving_speed), None)
|
|
if start is None:
|
|
return False, f"never reached {moving_speed} km/h"
|
|
stop = next((i for i in range(start, len(rows)) if speed[i] <= stop_speed), None)
|
|
if stop is None:
|
|
return False, f"never came down to {stop_speed} km/h"
|
|
d, top, top_t = 0.0, 0.0, rows[stop][COL_TIME]
|
|
for i in range(stop + 1, len(rows)):
|
|
d += (speed[i - 1] + speed[i]) / 2 / 3.6 * (rows[i][COL_TIME] - rows[i - 1][COL_TIME])
|
|
if speed[i] > top:
|
|
top, top_t = speed[i], rows[i][COL_TIME]
|
|
held = rows[-1][COL_TIME] - rows[stop][COL_TIME]
|
|
detail = f"stopped at t={rows[stop][COL_TIME]:.1f}s, then moved {d:.3f} m in {held:.0f}s, top {top:.4f} km/h at t={top_t:.1f}s"
|
|
return d <= max_distance, detail
|
|
|
|
|
|
def check_ends_moving(rows, min_speed):
|
|
"""The run ended with the vehicle still moving, e.g. a pass that didn't wait for the trains to stop."""
|
|
if not rows:
|
|
return False, "the physicslog is empty"
|
|
last = rows[-1]
|
|
ok = last[COL_VELOCITY] >= min_speed
|
|
return ok, f"last row t={last[COL_TIME]:.1f}s, v={last[COL_VELOCITY]:.1f} km/h (wanted >= {min_speed})"
|
|
|
|
|
|
def check_slip_clears(rows, moving_speed, stop_speed, within):
|
|
"""Once the vehicle has stopped after moving off, its wheels stop slipping within `within` seconds
|
|
(a vehicle left skidding on locked wheels keeps its brakes capped by the sliding adhesion)."""
|
|
if not rows or len(rows[0]) <= COL_VEHICLE_VELOCITY or rows[0][COL_VEHICLE_VELOCITY] is None:
|
|
return False, "the physicslog has no vehicle speed column, the build is too old"
|
|
start = next((i for i, r in enumerate(rows) if abs(r[COL_VEHICLE_VELOCITY]) >= moving_speed), None)
|
|
if start is None:
|
|
return False, f"never reached {moving_speed} km/h"
|
|
stop = next((i for i in range(start, len(rows)) if abs(rows[i][COL_VEHICLE_VELOCITY]) <= stop_speed), None)
|
|
if stop is None:
|
|
return False, f"never came down to {stop_speed} km/h"
|
|
deadline = rows[stop][COL_TIME] + within
|
|
late = [r for r in rows[stop:] if r[COL_TIME] > deadline and r[COL_WHEELSLIP]]
|
|
if late:
|
|
return False, f"stopped at t={rows[stop][COL_TIME]:.1f}s, wheels still slipping at t={late[0][COL_TIME]:.1f}s ({len(late)} rows after the {within}s grace)"
|
|
return True, f"stopped at t={rows[stop][COL_TIME]:.1f}s, no slipping after t={deadline:.1f}s"
|
|
|
|
|
|
CHECKS = {
|
|
"slips": check_slips,
|
|
"slip_clears": check_slip_clears,
|
|
"holds_still": check_holds_still,
|
|
"starts": check_starts,
|
|
"max_speed": check_max_speed,
|
|
"brakes": check_brakes,
|
|
"stops": check_stops,
|
|
"stays_stopped": check_stays_stopped,
|
|
"distance": check_distance,
|
|
"ends_moving": check_ends_moving,
|
|
}
|
|
|
|
|
|
def check_log_contains(log, pattern):
|
|
"""A line of the run's log.txt matches the regex, e.g. the "Bad driving" message of a collision."""
|
|
found = next((line.strip() for line in log.splitlines() if re.search(pattern, line)), None)
|
|
if found is None:
|
|
return False, f"no log line matches /{pattern}/"
|
|
return True, f"log: {found}"
|
|
|
|
|
|
def check_log_lacks(log, pattern):
|
|
"""No line of the run's log.txt matches the regex."""
|
|
found = [line.strip() for line in log.splitlines() if re.search(pattern, line)]
|
|
if found:
|
|
return False, f"{len(found)} log line(s) match /{pattern}/, first: {found[0]}"
|
|
return True, f"no log line matches /{pattern}/"
|
|
|
|
|
|
# checks on the text of the run's log.txt rather than on the physicslog rows
|
|
LOG_CHECKS = {
|
|
"log_contains": check_log_contains,
|
|
"log_lacks": check_log_lacks,
|
|
}
|
|
|
|
|
|
def check_couples(box, vehicle, other):
|
|
"""The two vehicles couple in the run and stay coupled to its end; coupled from the start doesn't count."""
|
|
numbers = {name: number for number, name in box.vehicles.items()}
|
|
unknown = [name for name in (vehicle, other) if name not in numbers]
|
|
if unknown:
|
|
return False, f"the black box has no vehicle {' nor '.join(unknown)}"
|
|
pair = {numbers[vehicle], numbers[other]}
|
|
# of the two vehicles of a coupling the one with the lower number tells it, each change of that coupler
|
|
# under the same vehicle and side: the record which coupled the two, for as long as they stay coupled
|
|
made, parted = {}, None
|
|
for c in box.couplings:
|
|
coupler = (c.vehicle, c.side)
|
|
if {c.vehicle, c.other} == pair and c.flags & COUPLING_COUPLER:
|
|
made.setdefault(coupler, c)
|
|
elif coupler in made:
|
|
parted = (made.pop(coupler), c)
|
|
if not made:
|
|
if parted:
|
|
return False, f"{vehicle} and {other} coupled at t={parted[0].time:.1f}s but came apart at t={parted[1].time:.1f}s"
|
|
return False, f"none of the {len(box.couplings)} coupling record(s) of the black box couples {vehicle} and {other}"
|
|
c = min(made.values(), key=lambda c: c.time)
|
|
if c.time <= 0:
|
|
return False, f"{vehicle} and {other} were coupled when the recording started"
|
|
ends = ("front", "rear")
|
|
return True, (f"at t={c.time:.1f}s the {ends[c.side]} coupler of {box.vehicles[c.vehicle]} coupled to the "
|
|
f"{ends[c.other_side]} one of {box.vehicles[c.other]}, coupling 0x{c.flags:02X}")
|
|
|
|
|
|
# checks on the black box of the run rather than on the physicslog rows
|
|
BLACKBOX_CHECKS = {
|
|
"couples": check_couples,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------- main
|
|
|
|
|
|
def find_exe(explicit: str | None) -> Path:
|
|
if explicit:
|
|
return Path(explicit).resolve()
|
|
candidates = sorted(REPO.glob("cmake-build-*/bin/Release/eu07*.exe"), key=lambda p: p.stat().st_mtime)
|
|
if not candidates:
|
|
sys.exit("no eu07 executable found under cmake-build-*/bin/Release, pass --exe")
|
|
return candidates[-1]
|
|
|
|
|
|
def count_lines(path: Path) -> int:
|
|
try:
|
|
return sum(1 for line in path.read_text(errors="replace").splitlines() if line.strip())
|
|
except FileNotFoundError:
|
|
return 0
|
|
|
|
|
|
def describe_exit(code: int | None) -> str:
|
|
if code is None:
|
|
return "none, still running"
|
|
name = EXIT_RESULTS.get(code)
|
|
return f"{code} ({name})" if name else f"{code} (0x{code & 0xFFFFFFFF:08X}, a crash?)"
|
|
|
|
|
|
def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, watch: bool, say) -> bool:
|
|
case = json.loads((CASES_DIR / f"{name}.json").read_text(encoding="utf-8"))
|
|
say(f"\n=== {name}: {case.get('description', '')}")
|
|
out = OUT_DIR / name
|
|
shutil.rmtree(out, ignore_errors=True)
|
|
out.mkdir(parents=True)
|
|
|
|
# the lead vehicle is named after the case, so parallel runs never share a physicslog
|
|
vehicle = f"autotest_{name}"
|
|
scenery = build_scenery(case, game_dir, name, vehicle)
|
|
ini = out / "autotest.ini"
|
|
# the case's own settings go last, over the common ones
|
|
ini.write_text(TEST_INI.format(timespeed=timespeed) + "".join(line + "\n" for line in case.get("ini", [])),
|
|
encoding="ascii")
|
|
physicslog = game_dir / "physicslog" / f"{vehicle}.dat"
|
|
|
|
duration = float(case["duration"]) if "duration" in case else None
|
|
# a case that only records `duration` seconds ends on the test timeout; one that says
|
|
# how the test ends (a pass event, a signal passed at danger...) expects that result
|
|
expect = case.get("expect", "timeout" if duration is not None else "pass")
|
|
if expect not in RESULTS:
|
|
sys.exit(f"{name}: expect must be one of {', '.join(RESULTS)}")
|
|
if watch and duration is None:
|
|
sys.exit(f"{name}: --watch needs a case with duration")
|
|
result = run_simulator(exe, game_dir, scenery, ini, physicslog,
|
|
duration, float(case.get("timeout", 600)), watch, say)
|
|
|
|
# the logs are ours alone (named after the case scenery), move them out of the game dir
|
|
for kind, path in result.logs.items():
|
|
shutil.move(path, out / f"{kind}.txt")
|
|
if physicslog.exists():
|
|
shutil.move(physicslog, out / physicslog.name)
|
|
# other trains of the case with a driver, named <something>_<vehicle>, log too
|
|
for other in physicslog.parent.glob(f"*_{vehicle}.dat"):
|
|
shutil.move(other, out / other.name)
|
|
# the black box is named after the case scenery too
|
|
if result.blackbox:
|
|
result.blackbox = Path(shutil.move(result.blackbox, out / f"{vehicle}.blackbox"))
|
|
|
|
ok = True
|
|
sim_t = result.rows[-1][COL_TIME] if result.rows else 0.0
|
|
say(f" ran {result.wall_seconds:.0f}s wall, {sim_t:.1f}s of physicslog, results in {out.relative_to(REPO)}")
|
|
if watch:
|
|
if result.exit_code is not None:
|
|
say(f" FAIL simulator exited on its own with code {result.exit_code}")
|
|
ok = False
|
|
else:
|
|
if result.verdict:
|
|
say(f" {result.verdict}")
|
|
got = EXIT_RESULTS.get(result.exit_code) if result.exit_code is not None else None
|
|
if got == "timeout" and "of real time" in result.verdict:
|
|
got = WALL_TIMEOUT
|
|
passed = got == expect
|
|
watchdog = ", from the real time watchdog" if got == WALL_TIMEOUT else ""
|
|
say(f" {'PASS' if passed else 'FAIL'} exit code {describe_exit(result.exit_code)}{watchdog}, expected {expect}")
|
|
ok &= passed
|
|
# a run cut short (the real time watchdog, an error, a crash, or closed by us) could have been writing
|
|
# the scenery's geometry cache; the simulator checks only the header of it, so the next runs would read
|
|
# a truncated one. it is written again on the next run
|
|
if result.failure or (not watch and got in (None, "error", WALL_TIMEOUT)):
|
|
scenery_cache(game_dir, name).unlink(missing_ok=True)
|
|
if result.failure:
|
|
say(f" FAIL {result.failure}")
|
|
ok = False
|
|
if not result.rows and case.get("checks"):
|
|
if not result.failure:
|
|
say(f" FAIL no physicslog for '{vehicle}'")
|
|
return False
|
|
|
|
log = read_text(out / "log.txt")
|
|
box = read_blackbox(result.blackbox)
|
|
for check in case.get("checks", []):
|
|
params = {k: v for k, v in check.items() if k != "type"}
|
|
if check["type"] in LOG_CHECKS:
|
|
# with no log text a log check proves nothing, log_lacks would pass vacuously
|
|
if log.strip():
|
|
passed, detail = LOG_CHECKS[check["type"]](log, **params)
|
|
else:
|
|
passed, detail = False, "log.txt is missing or empty"
|
|
elif check["type"] in BLACKBOX_CHECKS:
|
|
if box is not None:
|
|
# vehicles are named as in the fragment
|
|
params = {k: v.replace("{{vehicle}}", vehicle) if isinstance(v, str) else v for k, v in params.items()}
|
|
passed, detail = BLACKBOX_CHECKS[check["type"]](box, **params)
|
|
elif result.blackbox:
|
|
passed, detail = False, f"{result.blackbox.name} is not a black box, or is damaged"
|
|
else:
|
|
passed, detail = False, "the run left no black box (physicslog/*.blackbox); the build is too old, or has blackbox no"
|
|
else:
|
|
passed, detail = CHECKS[check["type"]](result.rows, **params)
|
|
say(f" {'PASS' if passed else 'FAIL'} {check['type']}: {detail}")
|
|
ok &= passed
|
|
|
|
errors = count_lines(out / "errors.txt")
|
|
if errors:
|
|
say(f" note errors.txt has {errors} line(s)")
|
|
if case.get("fail_on_errors") and errors:
|
|
ok = False
|
|
return ok
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("cases", nargs="*", help="case names from cases/ (default: all)")
|
|
ap.add_argument("--game-dir", default=os.environ.get("EU07_GAME_DIR"),
|
|
help="game install with scenery/ and dynamic/ (or EU07_GAME_DIR)")
|
|
ap.add_argument("--exe", help="simulator executable (default: newest cmake-build-*/bin/Release/eu07*.exe)")
|
|
ap.add_argument("--watch", action="store_true",
|
|
help="run in a visible window in real time (see --timespeed) instead of the test mode, "
|
|
"closed after the case's duration; the exit code isn't checked")
|
|
ap.add_argument("--timespeed", type=float, default=1.0,
|
|
help="with --watch: simulation time multiplier, the simulator offers 1, 5, 20 and 60 (default 1)")
|
|
ap.add_argument("-j", "--jobs", type=int, default=1,
|
|
help="cases run at once, each in a simulator of its own (default 1)")
|
|
args = ap.parse_args()
|
|
|
|
if not args.game_dir:
|
|
sys.exit("pass --game-dir or set EU07_GAME_DIR")
|
|
game_dir = Path(args.game_dir).resolve()
|
|
exe = find_exe(args.exe)
|
|
print(f"exe: {exe}\ngame: {game_dir}")
|
|
|
|
names = args.cases or sorted(p.stem for p in CASES_DIR.glob("*.json"))
|
|
missing = [n for n in names if not (CASES_DIR / f"{n}.json").exists()]
|
|
if missing:
|
|
sys.exit("no such case: " + ", ".join(missing))
|
|
|
|
started = time.monotonic()
|
|
results = {}
|
|
if args.jobs <= 1:
|
|
def say(line, progress=False):
|
|
print(line, flush=True)
|
|
for n in names:
|
|
results[n] = run_case(n, exe, game_dir, args.timespeed, args.watch, say)
|
|
else:
|
|
# each case prints its block when it is done, so parallel output does not interleave
|
|
def job(n):
|
|
lines = []
|
|
def say(line, progress=False):
|
|
if not progress:
|
|
lines.append(line)
|
|
ok = run_case(n, exe, game_dir, args.timespeed, args.watch, say)
|
|
return n, ok, lines
|
|
with ThreadPoolExecutor(max_workers=args.jobs) as pool:
|
|
for n in names:
|
|
print(f"started {n}", flush=True)
|
|
for fut in as_completed([pool.submit(job, n) for n in names]):
|
|
n, ok, lines = fut.result()
|
|
print("\n".join(lines), flush=True)
|
|
results[n] = ok
|
|
|
|
print(f"\n=== summary ({time.monotonic() - started:.0f}s)")
|
|
for n in names:
|
|
print(f" {'PASS' if results[n] else 'FAIL'} {n}")
|
|
return 0 if all(results.values()) else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|