From 85119cc110baa9dee58406d98d44a42a9eca6fe7 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Sun, 8 Nov 2026 21:06:11 +0100 Subject: [PATCH 1/2] test(scenarios): automatic coupler cases (#86) Re-lands #288, which was merged into feat/exe-test-mode after that branch had already gone to master, so master never got it. Two EN57-2000 units on td.scn, one standing, the other rolling into it without a driver. At 50 km/h their automatic couplers mustn't catch: the log has to show the collision and the derailment. At 4 km/h they couple and nothing collides. Nothing reports a result, so both cases end on the 60 s test timeout. run.py gets log_contains / log_lacks checks on the run's log.txt, and an "ini" case key for extra eu07.ini lines (here crashdamage yes). The log checks fail when log.txt is missing or empty: with no log text log_lacks would pass without checking anything. --- tests/scenarios/README.md | 15 +++++++- .../en57_2000_automatic_coupler_couples.json | 17 +++++++++ .../en57_2000_automatic_coupler_crash.json | 17 +++++++++ .../cases/td_automatic_coupler_hit.scn | 19 ++++++++++ tests/scenarios/run.py | 37 ++++++++++++++++++- 5 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 tests/scenarios/cases/en57_2000_automatic_coupler_couples.json create mode 100644 tests/scenarios/cases/en57_2000_automatic_coupler_crash.json create mode 100644 tests/scenarios/cases/td_automatic_coupler_hit.scn diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index 0a11c6a0a..c35d1d951 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -135,7 +135,7 @@ Python 3.10+, no extra packages. Options: `cmake-build-*/bin/Release/eu07*.exe`. The build needs the `--test` and `-ini` command line options. - name one or more cases to run only those, e.g. `run.py ... eu07_td_timetable`. -- `-j N` runs N cases at once. Each case prints its results when it is done. All 17 cases +- `-j N` runs N cases at once. Each case prints its results when it is done. All 19 cases take about 30 s with `-j 4`, loading included. - `--watch` runs a case the old way, in a visible window in real time (or faster with `--timespeed N`), closed after the case's `duration`; the exit code isn't checked then. For @@ -184,6 +184,7 @@ which could have been writing it. | `track_friction` | set the friction of every track of the base scenery (default in the sceneries is 0.15), e.g. for slippery rail | | `track_events` | lines added to tracks of the base scenery, by track name, e.g. `{ "tor1k": "event0 test_pass" }`; only tracks in the scenery file itself, not in its includes | | `test_config` | lines of a `config` block added to the scenery, e.g. `["test.timeout 90"]`; the block always has `test.vehicle` with the case's lead vehicle, so only its train counts for the pass and the standstill | +| `ini` | extra `eu07.ini` lines for the run, after the common ones of the runner's `-ini` file, e.g. `["crashdamage yes"]` | | `fragment` | `.scn` file in `cases/` added to the scenery: trainsets after `FirstInit` (before it they derail on unlinked track), the rest before it (launchers read after it never fire) | | `vars` | values for `{{name}}` in the fragment; a list becomes one line per item | | `expect` | the result the test mode should end with: `pass`, `fail`, `signal_passed_at_danger`, `timeout`, `standstill` or `error`, or `wall_timeout` for the timeout of the real time watchdog (the timeout code with a verdict saying it was the real time; a case expecting `timeout` fails on it). Default `timeout` for a case with `duration`, else `pass` | @@ -219,6 +220,12 @@ into the test timeout). `test_mode_pass_while_moving` reports the pass from `event2` of `tor1k` (the direction the train runs), as the train runs onto it, with `test.no_movement_before_pass no`, so the test ends with the train still moving. +The `en57_2000_automatic_coupler_*` cases (issue #86) share `td_automatic_coupler_hit.scn`: an +EN57-2000 unit without a driver runs at `vars.velocity` km/h into a standing one, from +`35.6 - vars.offset` m behind it, with `crashdamage yes` in `ini`. At 50 km/h the automatic +couplers mustn't catch and the log has to show the collision and the derailment; at 4 km/h they +couple and neither shows up. Nothing reports a result, so both end on the 60 s test timeout. + Checks (speeds in km/h, times in seconds of physicslog time, distances in metres). Most use the physicslog Velocity column, which is the wheel speed: it stays at 0 while the wheels are locked in a skid. The ones marked *vehicle speed* use the vehicle speed column at the end of @@ -244,6 +251,10 @@ the row instead, logged by builds from this branch on. after moving off, moves no more than `max_distance` metres, e.g. no creeping on locked wheels. - `ends_moving` `min_speed`: the last row of the physicslog is at `min_speed` or faster, e.g. the test ended without waiting for the train to stop. +- `log_contains` `pattern`: a line of the run's `log.txt` matches the regex, e.g. the + `Bad driving: ... collided` message of a collision. +- `log_lacks` `pattern`: no line of the run's `log.txt` matches the regex. New check types go in `CHECKS` in `run.py`; each gets the physicslog rows and the check's -parameters and returns `(passed, detail)`. +parameters and returns `(passed, detail)`. Checks on the log go in `LOG_CHECKS` and get the +text of `log.txt` instead of the rows. diff --git a/tests/scenarios/cases/en57_2000_automatic_coupler_couples.json b/tests/scenarios/cases/en57_2000_automatic_coupler_couples.json new file mode 100644 index 000000000..00917005d --- /dev/null +++ b/tests/scenarios/cases/en57_2000_automatic_coupler_couples.json @@ -0,0 +1,17 @@ +{ + "description": "Issue #86, the control case: an EN57-2000 unit without a driver rolls at 4 km/h into a standing one on td.scn. Their automatic couplers catch, nothing collides nor derails; nothing reports a result, so the run ends on the 60 s test timeout.", + "base_scenery": "td.scn", + "ini": ["crashdamage yes"], + "fragment": "td_automatic_coupler_hit.scn", + "vars": { + "velocity": "4", + "offset": "30" + }, + "expect": "timeout", + "duration": 60, + "timeout": 300, + "checks": [ + { "type": "log_lacks", "pattern": "collided" }, + { "type": "log_lacks", "pattern": "derailed" } + ] +} diff --git a/tests/scenarios/cases/en57_2000_automatic_coupler_crash.json b/tests/scenarios/cases/en57_2000_automatic_coupler_crash.json new file mode 100644 index 000000000..26c1e7f2b --- /dev/null +++ b/tests/scenarios/cases/en57_2000_automatic_coupler_crash.json @@ -0,0 +1,17 @@ +{ + "description": "Issue #86: an EN57-2000 unit without a driver runs at 50 km/h into a standing one on td.scn. Their automatic couplers mustn't catch at that speed: the units collide and derail; the run ends on the 60 s test timeout.", + "base_scenery": "td.scn", + "ini": ["crashdamage yes"], + "fragment": "td_automatic_coupler_hit.scn", + "vars": { + "velocity": "50", + "offset": "0" + }, + "expect": "timeout", + "duration": 60, + "timeout": 300, + "checks": [ + { "type": "log_contains", "pattern": "Bad driving: autotest_\\S+ and autotest_\\S+ collided" }, + { "type": "log_contains", "pattern": "Bad driving: autotest_\\S+ derailed" } + ] +} diff --git a/tests/scenarios/cases/td_automatic_coupler_hit.scn b/tests/scenarios/cases/td_automatic_coupler_hit.scn new file mode 100644 index 000000000..835a4b8b6 --- /dev/null +++ b/tests/scenarios/cases/td_automatic_coupler_hit.scn @@ -0,0 +1,19 @@ +// added by tests/scenarios/run.py to td.scn, whose own trainsets were removed. +// two EN57-2000 units, which have automatic couplers at their cab ends, on the straight track 1 +// of Stacja1: one stands where td.scn keeps its timetable train, with an AI driver and no +// timetable, so it stays put and logs its physics; the other has no driver and rolls into it at +// {{velocity}} km/h, its front coupler hitting the rear one of the first. +// a trainset's first vehicle stands {{offset}} m past point 1 of its track, the rest towards point 1, +// and a positive velocity runs towards point 2. tdo_n1734 is the 100 m track after tdo_n1725, a unit +// is 64.4 m long: the first one reaches back to 35.6 m into tdo_n1725, so the gap is 35.6 - offset m. +trainset none tdo_n1734 0.0 0.0 +node -1 0 {{vehicle}} dynamic pkp\en57-2000_v1 en57-2040_ra 6ba 0 headdriver 255 0 enddynamic +node -1 0 {{vehicle}}_s dynamic pkp\en57-2000_v1 en57-2040_s 6bs 0 nobody 255 0 enddynamic +node -1 0 {{vehicle}}_rb dynamic pkp\en57-2000_v1 en57-2040_rb 6bb 0 nobody 0 0 enddynamic +endtrainset + +trainset none tdo_n1725 {{offset}} {{velocity}} +node -1 0 {{vehicle}}_hit_ra dynamic pkp\en57-2000_v1 en57-2040_ra 6ba 0 nobody 255 0 enddynamic +node -1 0 {{vehicle}}_hit_s dynamic pkp\en57-2000_v1 en57-2040_s 6bs 0 nobody 255 0 enddynamic +node -1 0 {{vehicle}}_hit_rb dynamic pkp\en57-2000_v1 en57-2040_rb 6bb 0 nobody 0 0 enddynamic +endtrainset diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index 18b3e3f2e..f984564b1 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -476,6 +476,29 @@ CHECKS = { } +def check_log_contains(log, pattern): + """A line of the run's log.txt matches the regex, e.g. the "Bad driving" message of a collision.""" + found = next((line.strip() for line in log.splitlines() if re.search(pattern, line)), None) + if found is None: + return False, f"no log line matches /{pattern}/" + return True, f"log: {found}" + + +def check_log_lacks(log, pattern): + """No line of the run's log.txt matches the regex.""" + found = [line.strip() for line in log.splitlines() if re.search(pattern, line)] + if found: + return False, f"{len(found)} log line(s) match /{pattern}/, first: {found[0]}" + return True, f"no log line matches /{pattern}/" + + +# checks on the text of the run's log.txt rather than on the physicslog rows +LOG_CHECKS = { + "log_contains": check_log_contains, + "log_lacks": check_log_lacks, +} + + # ---------------------------------------------------------------- main @@ -513,7 +536,9 @@ def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, watch: bool vehicle = f"autotest_{name}" scenery = build_scenery(case, game_dir, name, vehicle) ini = out / "autotest.ini" - ini.write_text(TEST_INI.format(timespeed=timespeed), encoding="ascii") + # the case's own settings go last, over the common ones + ini.write_text(TEST_INI.format(timespeed=timespeed) + "".join(line + "\n" for line in case.get("ini", [])), + encoding="ascii") physicslog = game_dir / "physicslog" / f"{vehicle}.dat" duration = float(case["duration"]) if "duration" in case else None @@ -566,9 +591,17 @@ def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, watch: bool say(f" FAIL no physicslog for '{vehicle}'") return False + log = read_text(out / "log.txt") for check in case.get("checks", []): params = {k: v for k, v in check.items() if k != "type"} - passed, detail = CHECKS[check["type"]](result.rows, **params) + if check["type"] in LOG_CHECKS: + # with no log text a log check proves nothing, log_lacks would pass vacuously + if log.strip(): + passed, detail = LOG_CHECKS[check["type"]](log, **params) + else: + passed, detail = False, "log.txt is missing or empty" + else: + passed, detail = CHECKS[check["type"]](result.rows, **params) say(f" {'PASS' if passed else 'FAIL'} {check['type']}: {detail}") ok &= passed From 75c8ac61490cb018399c2e12d23856e6e746cc12 Mon Sep 17 00:00:00 2001 From: jerrrrycho Date: Wed, 11 Nov 2026 21:25:19 +0100 Subject: [PATCH 2/2] test(scenarios): the coupler control case checks that the units coupled The checks of en57_2000_automatic_coupler_couples said only that nothing collided nor derailed, so the case passed just as well with the units buffering, bouncing apart or never meeting. The simulator writes nothing to the log when automatic couplers catch, but the black box of the run (physicslog/_.blackbox, written along with the physicslogs the runner turns on) gets a coupling record with the two vehicles. run.py reads the vehicle and the coupling records of it and gets a check on them, couples: the two named vehicles couple after the start and are still coupled at the end. It fails when the black box has no coupling of that pair, when they came apart again, when they were coupled from the start, and when the run left no black box. The case names the vehicles that meet: the last car of the standing unit and the first of the rolling one. The black box is moved to out// with the other files of the run. --- tests/scenarios/README.md | 25 +++- .../en57_2000_automatic_coupler_couples.json | 3 +- tests/scenarios/run.py | 125 ++++++++++++++++++ 3 files changed, 146 insertions(+), 7 deletions(-) diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index c35d1d951..ea5ebc89b 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -142,8 +142,8 @@ Python 3.10+, no extra packages. Options: watching what a case does. Exit code is 0 when every case passed. Logs of each run (`log.txt`, `errors.txt`, -`stdout.txt`, the physicslogs of the vehicle and of other trains of the case) are moved to -`tests/scenarios/out//`. +`stdout.txt`, the physicslogs of the vehicle and of other trains of the case, the black box as +`autotest_.blackbox`) are moved to `tests/scenarios/out//`. ## How it works @@ -157,14 +157,15 @@ For each case the runner: 2. starts the simulator with `--test -s autotest_.scn -ini out//autotest.ini --test-wall-timeout `, plus `--test-timeout ` when the case gives a `duration`; the extra ini turns on - `physicslog` and writes the logs to `logs/log (autotest_) .txt` rather than the + `physicslog` (and with it the black box of the run, `physicslog/autotest__.blackbox`) + and writes the logs to `logs/log (autotest_) .txt` rather than the shared `log.txt` (`multiplelogs`), 3. waits until the simulator quits by itself. It gives up early when the log reports `Bad vehicle` / `Bad trainset`, or when the scenario has loaded and no physicslog appeared. After the case's `timeout` of wall-clock time the simulator ends itself with the timeout code; one that doesn't is killed 30 s later, 4. compares the result from the exit code with the case's `expect`, then runs the checks on - the physicslog. + the physicslog, the log and the black box. The lead vehicle is named `autotest_`, so every case has its own physicslog and log files and cases can run in parallel. The `autotest_*` sceneries (and the simulator's `.sbt` @@ -224,7 +225,10 @@ The `en57_2000_automatic_coupler_*` cases (issue #86) share `td_automatic_couple EN57-2000 unit without a driver runs at `vars.velocity` km/h into a standing one, from `35.6 - vars.offset` m behind it, with `crashdamage yes` in `ini`. At 50 km/h the automatic couplers mustn't catch and the log has to show the collision and the derailment; at 4 km/h they -couple and neither shows up. Nothing reports a result, so both end on the 60 s test timeout. +couple and neither shows up: the black box has to have the last car of the standing unit, +`{{vehicle}}_rb`, and the first of the rolling one, `{{vehicle}}_hit_ra`, coupled (the simulator +writes nothing to the log when couplers catch). Nothing reports a result, so both end on the 60 s +test timeout. Checks (speeds in km/h, times in seconds of physicslog time, distances in metres). Most use the physicslog Velocity column, which is the wheel speed: it stays at 0 while the wheels are @@ -254,7 +258,16 @@ the row instead, logged by builds from this branch on. - `log_contains` `pattern`: a line of the run's `log.txt` matches the regex, e.g. the `Bad driving: ... collided` message of a collision. - `log_lacks` `pattern`: no line of the run's `log.txt` matches the regex. +- `couples` `vehicle`, `other`: the black box of the run has the two vehicles coupled to each + other (the coupler itself, whatever else of the hoses and the cables) at some time after the + start, and still coupled at the end. The vehicles go by their node names, with `{{vehicle}}` + filled in as in the fragment, e.g. `{{vehicle}}_rb`. Fails when the two were coupled from the + start, came apart again, or the run left no black box. The black box is looked at once a + second, so the time given is that of the first look after the couplers caught. New check types go in `CHECKS` in `run.py`; each gets the physicslog rows and the check's parameters and returns `(passed, detail)`. Checks on the log go in `LOG_CHECKS` and get the -text of `log.txt` instead of the rows. +text of `log.txt` instead of the rows. Checks on the black box (`blackbox_doc.md` in the root of +the repo; the simulator writes it along with the physicslogs) go in `BLACKBOX_CHECKS` and get +what `read_blackbox()` read of it: the names of the vehicles and the coupling records. A check +on another kind of record has to add it there. diff --git a/tests/scenarios/cases/en57_2000_automatic_coupler_couples.json b/tests/scenarios/cases/en57_2000_automatic_coupler_couples.json index 00917005d..375a08a96 100644 --- a/tests/scenarios/cases/en57_2000_automatic_coupler_couples.json +++ b/tests/scenarios/cases/en57_2000_automatic_coupler_couples.json @@ -1,5 +1,5 @@ { - "description": "Issue #86, the control case: an EN57-2000 unit without a driver rolls at 4 km/h into a standing one on td.scn. Their automatic couplers catch, nothing collides nor derails; nothing reports a result, so the run ends on the 60 s test timeout.", + "description": "Issue #86, the control case: an EN57-2000 unit without a driver rolls at 4 km/h into a standing one on td.scn. Their automatic couplers catch: the black box of the run has the last car of the standing unit and the first of the rolling one coupled. Nothing collides nor derails; nothing reports a result, so the run ends on the 60 s test timeout.", "base_scenery": "td.scn", "ini": ["crashdamage yes"], "fragment": "td_automatic_coupler_hit.scn", @@ -11,6 +11,7 @@ "duration": 60, "timeout": 300, "checks": [ + { "type": "couples", "vehicle": "{{vehicle}}_rb", "other": "{{vehicle}}_hit_ra" }, { "type": "log_lacks", "pattern": "collided" }, { "type": "log_lacks", "pattern": "derailed" } ] diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index f984564b1..51fdc3c66 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -31,6 +31,7 @@ import json import os import re import shutil +import struct import subprocess import sys import time @@ -230,6 +231,81 @@ def read_text(path: Path | None) -> str: return "" +# the black box of a run, see blackbox_doc.md: written along with the physicslogs, to +# physicslog/_.blackbox. A header, then records: a byte of the kind, the length, the content +BLACKBOX_MAGIC = b"EU07BBX\0" +BLACKBOX_HEADER = 12 # the magic, the version of the format, two bytes reserved +BLACKBOX_VEHICLE = 0x12 # a vehicle: its number and its name +BLACKBOX_COUPLING = 0x37 # a coupler of a vehicle coupled, uncoupled, or its hoses and cables changed +COUPLING_COUPLER = 0x01 # of the bits telling what is coupled: the coupler itself + + +@dataclass +class Coupling: + """A coupling record: what a coupler of a vehicle is coupled to from `time` on.""" + time: float + vehicle: int # vehicles by their numbers, 0 for none + side: int # 0 the front coupler, 1 the rear one + other: int + other_side: int + flags: int # what is coupled, 0 nothing + previous_flags: int + previous_other: int + + +@dataclass +class Blackbox: + vehicles: dict[int, str] = field(default_factory=dict) # names by number + couplings: list[Coupling] = field(default_factory=list) # in the order written + + +def find_blackbox(game_dir: Path, scenery: str, since: float) -> Path | None: + """The black box of one run: the simulator names it after the scenery and the time it started.""" + stem = re.sub(r"[^0-9A-Za-z_-]", "_", scenery.rsplit(".", 1)[0]) + # the whole name, so a case whose name another one starts with doesn't take that one's file + name = re.compile(re.escape(stem) + r"_\d{4}-\d\d-\d\d_\d\d-\d\d-\d\d\.blackbox") + files = [p for p in (game_dir / "physicslog").glob(f"{stem}_*.blackbox") + if name.fullmatch(p.name) and p.stat().st_mtime >= since] + return max(files, key=lambda p: p.stat().st_mtime, default=None) + + +def read_blackbox(path: Path | None) -> Blackbox | None: + """The vehicles and the coupling records of a black box; None when there is none or it can't be read.""" + try: + data = path.read_bytes() if path else b"" + except FileNotFoundError: + return None + if data[:len(BLACKBOX_MAGIC)] != BLACKBOX_MAGIC: + return None + box = Blackbox() + at = BLACKBOX_HEADER + try: + while at + 5 <= len(data): + kind, length = struct.unpack_from("_, log too for other in physicslog.parent.glob(f"*_{vehicle}.dat"): shutil.move(other, out / other.name) + # the black box is named after the case scenery too + if result.blackbox: + result.blackbox = Path(shutil.move(result.blackbox, out / f"{vehicle}.blackbox")) ok = True sim_t = result.rows[-1][COL_TIME] if result.rows else 0.0 @@ -592,6 +707,7 @@ def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, watch: bool return False log = read_text(out / "log.txt") + box = read_blackbox(result.blackbox) for check in case.get("checks", []): params = {k: v for k, v in check.items() if k != "type"} if check["type"] in LOG_CHECKS: @@ -600,6 +716,15 @@ def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, watch: bool passed, detail = LOG_CHECKS[check["type"]](log, **params) else: passed, detail = False, "log.txt is missing or empty" + elif check["type"] in BLACKBOX_CHECKS: + if box is not None: + # vehicles are named as in the fragment + params = {k: v.replace("{{vehicle}}", vehicle) if isinstance(v, str) else v for k, v in params.items()} + passed, detail = BLACKBOX_CHECKS[check["type"]](box, **params) + elif result.blackbox: + passed, detail = False, f"{result.blackbox.name} is not a black box, or is damaged" + else: + passed, detail = False, "the run left no black box (physicslog/*.blackbox); the build is too old, or has blackbox no" else: passed, detail = CHECKS[check["type"]](result.rows, **params) say(f" {'PASS' if passed else 'FAIL'} {check['type']}: {detail}")