diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index c35d1d951..ea5ebc89b 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -142,8 +142,8 @@ Python 3.10+, no extra packages. Options: watching what a case does. Exit code is 0 when every case passed. Logs of each run (`log.txt`, `errors.txt`, -`stdout.txt`, the physicslogs of the vehicle and of other trains of the case) are moved to -`tests/scenarios/out//`. +`stdout.txt`, the physicslogs of the vehicle and of other trains of the case, the black box as +`autotest_.blackbox`) are moved to `tests/scenarios/out//`. ## How it works @@ -157,14 +157,15 @@ For each case the runner: 2. starts the simulator with `--test -s autotest_.scn -ini out//autotest.ini --test-wall-timeout `, plus `--test-timeout ` when the case gives a `duration`; the extra ini turns on - `physicslog` and writes the logs to `logs/log (autotest_) .txt` rather than the + `physicslog` (and with it the black box of the run, `physicslog/autotest__.blackbox`) + and writes the logs to `logs/log (autotest_) .txt` rather than the shared `log.txt` (`multiplelogs`), 3. waits until the simulator quits by itself. It gives up early when the log reports `Bad vehicle` / `Bad trainset`, or when the scenario has loaded and no physicslog appeared. After the case's `timeout` of wall-clock time the simulator ends itself with the timeout code; one that doesn't is killed 30 s later, 4. compares the result from the exit code with the case's `expect`, then runs the checks on - the physicslog. + the physicslog, the log and the black box. The lead vehicle is named `autotest_`, so every case has its own physicslog and log files and cases can run in parallel. The `autotest_*` sceneries (and the simulator's `.sbt` @@ -224,7 +225,10 @@ The `en57_2000_automatic_coupler_*` cases (issue #86) share `td_automatic_couple EN57-2000 unit without a driver runs at `vars.velocity` km/h into a standing one, from `35.6 - vars.offset` m behind it, with `crashdamage yes` in `ini`. At 50 km/h the automatic couplers mustn't catch and the log has to show the collision and the derailment; at 4 km/h they -couple and neither shows up. Nothing reports a result, so both end on the 60 s test timeout. +couple and neither shows up: the black box has to have the last car of the standing unit, +`{{vehicle}}_rb`, and the first of the rolling one, `{{vehicle}}_hit_ra`, coupled (the simulator +writes nothing to the log when couplers catch). Nothing reports a result, so both end on the 60 s +test timeout. Checks (speeds in km/h, times in seconds of physicslog time, distances in metres). Most use the physicslog Velocity column, which is the wheel speed: it stays at 0 while the wheels are @@ -254,7 +258,16 @@ the row instead, logged by builds from this branch on. - `log_contains` `pattern`: a line of the run's `log.txt` matches the regex, e.g. the `Bad driving: ... collided` message of a collision. - `log_lacks` `pattern`: no line of the run's `log.txt` matches the regex. +- `couples` `vehicle`, `other`: the black box of the run has the two vehicles coupled to each + other (the coupler itself, whatever else of the hoses and the cables) at some time after the + start, and still coupled at the end. The vehicles go by their node names, with `{{vehicle}}` + filled in as in the fragment, e.g. `{{vehicle}}_rb`. Fails when the two were coupled from the + start, came apart again, or the run left no black box. The black box is looked at once a + second, so the time given is that of the first look after the couplers caught. New check types go in `CHECKS` in `run.py`; each gets the physicslog rows and the check's parameters and returns `(passed, detail)`. Checks on the log go in `LOG_CHECKS` and get the -text of `log.txt` instead of the rows. +text of `log.txt` instead of the rows. Checks on the black box (`blackbox_doc.md` in the root of +the repo; the simulator writes it along with the physicslogs) go in `BLACKBOX_CHECKS` and get +what `read_blackbox()` read of it: the names of the vehicles and the coupling records. A check +on another kind of record has to add it there. diff --git a/tests/scenarios/cases/en57_2000_automatic_coupler_couples.json b/tests/scenarios/cases/en57_2000_automatic_coupler_couples.json index 00917005d..375a08a96 100644 --- a/tests/scenarios/cases/en57_2000_automatic_coupler_couples.json +++ b/tests/scenarios/cases/en57_2000_automatic_coupler_couples.json @@ -1,5 +1,5 @@ { - "description": "Issue #86, the control case: an EN57-2000 unit without a driver rolls at 4 km/h into a standing one on td.scn. Their automatic couplers catch, nothing collides nor derails; nothing reports a result, so the run ends on the 60 s test timeout.", + "description": "Issue #86, the control case: an EN57-2000 unit without a driver rolls at 4 km/h into a standing one on td.scn. Their automatic couplers catch: the black box of the run has the last car of the standing unit and the first of the rolling one coupled. Nothing collides nor derails; nothing reports a result, so the run ends on the 60 s test timeout.", "base_scenery": "td.scn", "ini": ["crashdamage yes"], "fragment": "td_automatic_coupler_hit.scn", @@ -11,6 +11,7 @@ "duration": 60, "timeout": 300, "checks": [ + { "type": "couples", "vehicle": "{{vehicle}}_rb", "other": "{{vehicle}}_hit_ra" }, { "type": "log_lacks", "pattern": "collided" }, { "type": "log_lacks", "pattern": "derailed" } ] diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index f984564b1..51fdc3c66 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -31,6 +31,7 @@ import json import os import re import shutil +import struct import subprocess import sys import time @@ -230,6 +231,81 @@ def read_text(path: Path | None) -> str: return "" +# the black box of a run, see blackbox_doc.md: written along with the physicslogs, to +# physicslog/_.blackbox. A header, then records: a byte of the kind, the length, the content +BLACKBOX_MAGIC = b"EU07BBX\0" +BLACKBOX_HEADER = 12 # the magic, the version of the format, two bytes reserved +BLACKBOX_VEHICLE = 0x12 # a vehicle: its number and its name +BLACKBOX_COUPLING = 0x37 # a coupler of a vehicle coupled, uncoupled, or its hoses and cables changed +COUPLING_COUPLER = 0x01 # of the bits telling what is coupled: the coupler itself + + +@dataclass +class Coupling: + """A coupling record: what a coupler of a vehicle is coupled to from `time` on.""" + time: float + vehicle: int # vehicles by their numbers, 0 for none + side: int # 0 the front coupler, 1 the rear one + other: int + other_side: int + flags: int # what is coupled, 0 nothing + previous_flags: int + previous_other: int + + +@dataclass +class Blackbox: + vehicles: dict[int, str] = field(default_factory=dict) # names by number + couplings: list[Coupling] = field(default_factory=list) # in the order written + + +def find_blackbox(game_dir: Path, scenery: str, since: float) -> Path | None: + """The black box of one run: the simulator names it after the scenery and the time it started.""" + stem = re.sub(r"[^0-9A-Za-z_-]", "_", scenery.rsplit(".", 1)[0]) + # the whole name, so a case whose name another one starts with doesn't take that one's file + name = re.compile(re.escape(stem) + r"_\d{4}-\d\d-\d\d_\d\d-\d\d-\d\d\.blackbox") + files = [p for p in (game_dir / "physicslog").glob(f"{stem}_*.blackbox") + if name.fullmatch(p.name) and p.stat().st_mtime >= since] + return max(files, key=lambda p: p.stat().st_mtime, default=None) + + +def read_blackbox(path: Path | None) -> Blackbox | None: + """The vehicles and the coupling records of a black box; None when there is none or it can't be read.""" + try: + data = path.read_bytes() if path else b"" + except FileNotFoundError: + return None + if data[:len(BLACKBOX_MAGIC)] != BLACKBOX_MAGIC: + return None + box = Blackbox() + at = BLACKBOX_HEADER + try: + while at + 5 <= len(data): + kind, length = struct.unpack_from("_, log too for other in physicslog.parent.glob(f"*_{vehicle}.dat"): shutil.move(other, out / other.name) + # the black box is named after the case scenery too + if result.blackbox: + result.blackbox = Path(shutil.move(result.blackbox, out / f"{vehicle}.blackbox")) ok = True sim_t = result.rows[-1][COL_TIME] if result.rows else 0.0 @@ -592,6 +707,7 @@ def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, watch: bool return False log = read_text(out / "log.txt") + box = read_blackbox(result.blackbox) for check in case.get("checks", []): params = {k: v for k, v in check.items() if k != "type"} if check["type"] in LOG_CHECKS: @@ -600,6 +716,15 @@ def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, watch: bool passed, detail = LOG_CHECKS[check["type"]](log, **params) else: passed, detail = False, "log.txt is missing or empty" + elif check["type"] in BLACKBOX_CHECKS: + if box is not None: + # vehicles are named as in the fragment + params = {k: v.replace("{{vehicle}}", vehicle) if isinstance(v, str) else v for k, v in params.items()} + passed, detail = BLACKBOX_CHECKS[check["type"]](box, **params) + elif result.blackbox: + passed, detail = False, f"{result.blackbox.name} is not a black box, or is damaged" + else: + passed, detail = False, "the run left no black box (physicslog/*.blackbox); the build is too old, or has blackbox no" else: passed, detail = CHECKS[check["type"]](result.rows, **params) say(f" {'PASS' if passed else 'FAIL'} {check['type']}: {detail}")