diff --git a/simulation/simulationtest.cpp b/simulation/simulationtest.cpp index 15cdf0e3..89a8a104 100644 --- a/simulation/simulationtest.cpp +++ b/simulation/simulationtest.cpp @@ -33,6 +33,11 @@ http://mozilla.org/MPL/2.0/. namespace { +bool is_moving(TDynamicObject const *Vehicle) +{ + return std::abs(Vehicle->GetVelocity()) > 0.01; +} + // [s] of real time since the start of the simulator double elapsed_wall_time() { @@ -117,10 +122,25 @@ void test_manager::start() finish(test_result::error, "the scenery has no vehicles"); return; } + for (auto const &name : Global.test.vehicles) + { + if (simulation::Vehicles.find(name) == nullptr) + { + finish(test_result::error, std::format("the scenery has no vehicle {} (test.vehicle)", name)); + return; + } + } // a pause set by the scenery or the settings would never be lifted, nobody is there to press the key Global.iPause = 0; + auto tested { std::string("every vehicle") }; + if (false == Global.test.vehicles.empty()) + { + tested = "the consists of"; + for (auto const &name : Global.test.vehicles) + tested += " " + name; + } auto const walllimit { m_walllimit.load() }; - WriteLog(std::format("test: started, timeout {:.0f} s, real time limit {}, standstill {:.0f} s{}{}{}", Global.test.timeout, + WriteLog(std::format("test: started, testing {}, timeout {:.0f} s, real time limit {}, standstill {:.0f} s{}{}{}", tested, Global.test.timeout, walllimit > 0.0 ? std::format("{:.0f} s", walllimit) : std::string("none"), Global.test.standstill, Global.test.pass_on_standstill ? ", standstill passes" : "", Global.test.fail_if_stop_overran ? "" : ", signals passed at danger allowed", @@ -132,8 +152,7 @@ void test_manager::update(double const Deltatime) if (false == m_started || m_finished) return; - test_frame frame { Deltatime, std::nullopt, {}, - std::ranges::none_of(simulation::Vehicles.sequence(), [](TDynamicObject const *Vehicle) { return std::abs(Vehicle->GetVelocity()) > 0.01; }) }; + test_frame frame { Deltatime, std::nullopt, {}, tested_trains_at_rest() }; if (auto const *cell = simulation::Memory.find(std::string(result_cell))) { frame.reported = cell->Value1(); @@ -156,11 +175,22 @@ void test_manager::update(double const Deltatime) } if (passrequested) { - WriteLog(std::format("test: pass requested at {:.1f} s{}, waiting for every vehicle to stop", m_progress.time, + WriteLog(std::format("test: pass requested at {:.1f} s{}, waiting for the tested trains to stop", m_progress.time, frame.reason.empty() ? std::string() : std::format(" ({})", frame.reason))); } } +bool test_manager::tested_trains_at_rest() const +{ + if (Global.test.vehicles.empty()) + return std::ranges::none_of(simulation::Vehicles.sequence(), is_moving); + // the consists as they are now: a train the tested one couples to joins it, one it leaves behind doesn't + return std::ranges::none_of(Global.test.vehicles, [](std::string const &Name) { + auto *vehicle { simulation::Vehicles.find(Name) }; + return vehicle != nullptr && vehicle->find_vehicle(coupling::coupler, is_moving) != nullptr; + }); +} + void test_manager::signal_passed_at_danger(std::string_view const Vehicle, std::string_view const Signal) { auto const reason { std::format("{} passed signal {} at danger", Vehicle, Signal) }; diff --git a/simulation/simulationtest.h b/simulation/simulationtest.h index a59ec87b..784cd39e 100644 --- a/simulation/simulationtest.h +++ b/simulation/simulationtest.h @@ -52,6 +52,8 @@ public: private: // the watchdog's check, at Elapsed seconds of real time since the start of the simulator void check_wall_clock(double Elapsed); + // every vehicle of the tested trains stands: the consists of test.vehicle, or every vehicle without it + bool tested_trains_at_rest() const; // the watchdog may end the test from its own thread mutable std::mutex m_lock; diff --git a/simulation/simulationtestrules.cpp b/simulation/simulationtestrules.cpp index eba1d7dd..cdfea0e3 100644 --- a/simulation/simulationtestrules.cpp +++ b/simulation/simulationtestrules.cpp @@ -138,9 +138,9 @@ std::optional evaluate_frame(test_progress &Progress, test_frame c Progress.resttime = Frame.atrest ? Progress.resttime + Frame.deltatime : 0.0; if (Progress.passrequested && Progress.resttime >= pass_settle_time) - return test_verdict{pass, "__test.result set to 1 and every vehicle stopped"}; + return test_verdict{pass, "__test.result set to 1 and the tested trains stopped"}; if (Limits.standstill > 0.0 && Progress.resttime >= Limits.standstill) - return test_verdict{Limits.pass_on_standstill ? pass : standstill, std::format("every vehicle stood still for {:.0f} s", Limits.standstill)}; + return test_verdict{Limits.pass_on_standstill ? pass : standstill, std::format("the tested trains stood still for {:.0f} s", Limits.standstill)}; if (Limits.timeout > 0.0 && Progress.time >= Limits.timeout) return test_verdict{timeout, std::format("no result after {:.0f} s", Limits.timeout)}; return std::nullopt; diff --git a/simulation/simulationtestrules.h b/simulation/simulationtestrules.h index 52adf27a..153dc8a4 100644 --- a/simulation/simulationtestrules.h +++ b/simulation/simulationtestrules.h @@ -22,11 +22,11 @@ namespace simulation { // when it goes wrong on its own: 1 (killed with TerminateProcess, or a plain failure of most programs), 3 (abort() // on Windows), 126-127 (a shell that can't run the program) or 128 + a signal number (killed, on Linux) enum class test_result : int { - pass = 0, // __test.result set to 1, then every vehicle came to rest (or test.pass_on_standstill) + pass = 0, // __test.result set to 1, then the tested trains came to rest (or test.pass_on_standstill) fail = 10, // __test.result set to 0 signal_passed_at_danger = 11, // a train went past a stop signal (SetVelocity 0 0), unless test.fail_if_stop_overran no timeout = 12, // test.timeout seconds of simulation time, or test.wall_timeout seconds of real time went by - standstill = 13, // every vehicle stood still for test.standstill seconds without a result + standstill = 13, // the tested trains stood still for test.standstill seconds without a result error = 14, // the run could not start (a bad command line, the scenery failed to load) or the simulator threw }; @@ -64,7 +64,7 @@ double wall_clock_limit(std::optional Given, double Simulationtimeout); // the limits and switches the end conditions go by struct test_limits { double timeout{3600.0}; // [s] of simulation time; 0 = never - double standstill{300.0}; // [s] of every vehicle standing without a result; 0 = never + double standstill{300.0}; // [s] of the tested trains standing without a result; 0 = never bool pass_on_standstill{false}; bool no_movement_before_pass{true}; }; @@ -74,17 +74,17 @@ struct test_frame { double deltatime{0.0}; // [s] of simulation time std::optional reported; // value of the result cell, if there is one std::string_view reason; // text of the result cell - bool atrest{false}; // every vehicle stands + bool atrest{false}; // every vehicle of the tested trains stands (test.vehicle; every vehicle without it) }; // the state of a test between frames struct test_progress { double time{0.0}; // simulation time since the start [s] - double resttime{0.0}; // how long every vehicle has been standing [s] - bool passrequested{false}; // __test.result was set to 1, the pass waits for the vehicles to stop + double resttime{0.0}; // how long the tested trains have been standing [s] + bool passrequested{false}; // __test.result was set to 1, the pass waits for the tested trains to stop }; -// how long everything has to stand before a requested pass is granted [s] +// how long the tested trains have to stand before a requested pass is granted [s] inline constexpr double pass_settle_time{1.0}; // checks the end conditions after a frame and moves the progress on. returns the verdict once the test is over diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index 5eed983e..c1226e63 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -67,11 +67,11 @@ The test ends with the first of these: | exit code | result | when | | --- | --- | --- | -| 0 | pass | `__test.result` was set to 1 and then every vehicle stood still for 1 s, so a train still moving towards a signal at stop has to stop first (with `test.no_movement_before_pass no` the pass ends the test at once). Also the standstill end with `test.pass_on_standstill yes` | +| 0 | pass | `__test.result` was set to 1 and then the tested trains stood still for 1 s, so a train still moving towards a signal at stop has to stop first (with `test.no_movement_before_pass no` the pass ends the test at once). Also the standstill end with `test.pass_on_standstill yes` | | 10 | fail | `__test.result` was set to 0; ends at once | | 11 | signal passed at danger | a train went past a signal showing stop (`SetVelocity 0 0`; `SetVelocity 0 x` on automatic block, which may be passed after a stop, doesn't count); ends at once, unless `test.fail_if_stop_overran no` | | 12 | timeout | `test.timeout` seconds of simulation time went by (default 3600), or `test.wall_timeout` seconds of real time since the start (see below) | -| 13 | standstill | every vehicle stood still for `test.standstill` seconds (default 300) and nothing reported a result. Trains waiting at a station or for a signal count as standing, so the limit has to be longer than such waits | +| 13 | standstill | the tested trains stood still for `test.standstill` seconds (default 300) and nothing reported a result. Trains waiting at a station or for a signal count as standing, so the limit has to be longer than such waits | | 14 | error | the run could not start: no `-s`, the scenery failed to load or has no vehicles, or the simulator threw | A crash ends with the system's code instead (on Windows e.g. `0xC0000005`, -1073741819). The @@ -89,6 +89,7 @@ test.standstill 120 test.pass_on_standstill no test.fail_if_stop_overran yes test.no_movement_before_pass yes +test.vehicle ep07-424 endconfig ``` @@ -96,10 +97,11 @@ endconfig | --- | --- | --- | | `test.timeout` | 3600 | seconds of simulation time before the test ends as `timeout`; 0 = never | | `test.wall_timeout` | `test.timeout` + 600 | seconds of real time since the start before the test ends as `timeout`; 0 = never | -| `test.standstill` | 300 | seconds with every vehicle still and no result before the test ends as `standstill`; 0 = never | +| `test.standstill` | 300 | seconds with the tested trains still and no result before the test ends as `standstill`; 0 = never | | `test.pass_on_standstill` | no | the standstill end passes the test instead | | `test.fail_if_stop_overran` | yes | a signal passed at danger ends the test; with `no` it's only logged | -| `test.no_movement_before_pass` | yes | a pass waits until every vehicle stands still; with `no` it ends the test at once | +| `test.no_movement_before_pass` | yes | a pass waits until the tested trains stand still; with `no` it ends the test at once | +| `test.vehicle` | none | a vehicle of a train under test, by its name in the scenery (once for each train under test). The tested trains are the consists of these vehicles as they are at the moment (a train the tested one couples to joins it), and only they count for the pass and the standstill: other trains, AI traffic, may run on. A name the scenery doesn't have ends the test with `error`. Not given, every vehicle of the scenery counts. A signal passed at danger counts for every train either way | `--test-timeout` and `--test-wall-timeout` on the command line win over `test.timeout` and `test.wall_timeout` of the scenery and the settings files. @@ -131,14 +133,15 @@ Python 3.10+, no extra packages. Options: `cmake-build-*/bin/Release/eu07*.exe`. The build needs the `--test` and `-ini` command line options. - name one or more cases to run only those, e.g. `run.py ... eu07_td_timetable`. -- `-j N` runs N cases at once. Each case prints its results when it is done. All 16 cases +- `-j N` runs N cases at once. Each case prints its results when it is done. All 17 cases take about 30 s with `-j 4`, loading included. - `--watch` runs a case the old way, in a visible window in real time (or faster with `--timespeed N`), closed after the case's `duration`; the exit code isn't checked then. For watching what a case does. Exit code is 0 when every case passed. Logs of each run (`log.txt`, `errors.txt`, -`stdout.txt`, the vehicle's physicslog) are moved to `tests/scenarios/out//`. +`stdout.txt`, the physicslogs of the vehicle and of other trains of the case) are moved to +`tests/scenarios/out//`. ## How it works @@ -177,7 +180,7 @@ scenery is newer. | `time` | replace the scenery start time, `HH:MM` | | `track_friction` | set the friction of every track of the base scenery (default in the sceneries is 0.15), e.g. for slippery rail | | `track_events` | lines added to tracks of the base scenery, by track name, e.g. `{ "tor1k": "event0 test_pass" }`; only tracks in the scenery file itself, not in its includes | -| `test_config` | lines of a `config` block added to the scenery, e.g. `["test.timeout 90"]` | +| `test_config` | lines of a `config` block added to the scenery, e.g. `["test.timeout 90"]`; the block always has `test.vehicle` with the case's lead vehicle, so only its train counts for the pass and the standstill | | `fragment` | `.scn` file in `cases/` added to the scenery: trainsets after `FirstInit` (before it they derail on unlinked track), the rest before it (launchers read after it never fire) | | `vars` | values for `{{name}}` in the fragment; a list becomes one line per item | | `expect` | the result the test mode should end with: `pass`, `fail`, `signal_passed_at_danger`, `timeout`, `standstill` or `error`, or `wall_timeout` for the timeout of the real time watchdog (the timeout code with a verdict saying it was the real time; a case expecting `timeout` fails on it). Default `timeout` for a case with `duration`, else `pass` | @@ -187,15 +190,16 @@ scenery is newer. | `checks` | list of checks, below | `{{vehicle}}` is always set, also inside `vars` values: the node name of the lead vehicle, -the one whose physicslog is checked. Other vehicles of the consist take names made from it, -e.g. `{{vehicle}}_s`. The td.scn cases share two fragments, `td_timetable.scn` (route set, +the one whose physicslog is checked, and the train under test (`test.vehicle`). Other vehicles +of the consist take names made from it, e.g. `{{vehicle}}_s`; another train with a driver is named +`_{{vehicle}}`, so its physicslog is moved out of the game dir too. The td.scn cases share two fragments, `td_timetable.scn` (route set, train runs to Przystanek) and `td_red_signal.scn` (no route, exit signal at stop), and differ only in `vars.consist`. Przystanek is a technical stop in that timetable (`pt`), where the AI doesn't wait for the departure time: it sets off as soon as it counts the train as standing at the W4, which is below 1 km/h, so the train stands there for a few seconds at most, or doesn't quite stop (since the AI braking of #264 the SN61 turns back at 0.4 km/h). The timetable cases therefore check that the train comes down to 1 km/h, with no hold, 1380-1420 m -from the start (every vehicle stops at 1397-1401 m). `td_red_signal_moving.scn` is the red signal one with the train +from the start (each of the seven trains stops at 1397-1401 m). `td_red_signal_moving.scn` is the red signal one with the train starting at `vars.velocity` km/h; with `track_friction` 0.03 it locks the wheels when braking and slides past the signal, so those cases set `test.fail_if_stop_overran no`. `td_red_signal_pass.scn` adds the `test_pass` event, which `test_mode_pass` puts as `event0` @@ -205,6 +209,10 @@ The `test_mode_*` cases show the ends of the test mode: `test_mode_pass` (exit 0 event), `test_mode_signal_passed_at_danger` (exit 11), `test_mode_timeout` (exit 12, from `test.timeout`), `test_mode_standstill` (exit 13, from `test.standstill`) and `test_mode_wall_timeout` (exit 12 from the real time watchdog, which ends it while it loads). +`test_mode_standstill_traffic` (fragment `td_traffic.scn`) has a train under test standing on +track 2 of Stacja1 and AI traffic, `traffic_{{vehicle}}`, leaving track 1 on the timetable: the +standstill of 100 s ends the test although the traffic runs on (counting every vehicle, it runs +into the test timeout). `test_mode_pass_while_moving` reports the pass from `event2` of `tor1k` (the direction the train runs), as the train runs onto it, with `test.no_movement_before_pass no`, so the test ends with the train still moving. diff --git a/tests/scenarios/cases/td_traffic.scn b/tests/scenarios/cases/td_traffic.scn new file mode 100644 index 00000000..49720a0b --- /dev/null +++ b/tests/scenarios/cases/td_traffic.scn @@ -0,0 +1,13 @@ +// added by tests/scenarios/run.py to td.scn, whose own trainsets were removed: two trains at Stacja1. +// the train under test (test.vehicle) stands on track 2, where its driver has nothing to do; the other one, +// AI traffic, has the td/rozklad timetable on track 1 and leaves for Przystanek at 10:35. the standstill of +// the test waits for the train under test only, so the traffic running on doesn't hold it up. +trainset none tor2k 30.0 0.0 +node -1 0 {{vehicle}} dynamic pkp\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic +endtrainset +trainset td/rozklad tdo_n1734 0.0 0.1 +node -1 0 traffic_{{vehicle}} dynamic pkp\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic +endtrainset + +// the way out of track 1 for the traffic, see td_timetable.scn +node -1 0 autotest_route eventlauncher 0 0 0 -1 none 1034 keyctrl04 end diff --git a/tests/scenarios/cases/test_mode_standstill.json b/tests/scenarios/cases/test_mode_standstill.json index 0d96585e..4dd891cf 100644 --- a/tests/scenarios/cases/test_mode_standstill.json +++ b/tests/scenarios/cases/test_mode_standstill.json @@ -1,5 +1,5 @@ { - "description": "Test mode, standstill: EU07 on td.scn with no route set stops before the exit signal stacja1_j at stop and stays; nothing reports a result, so the test ends once every vehicle stood still for test.standstill 120 seconds (the train waits about 70 s before it pulls off, so less would end it before the start).", + "description": "Test mode, standstill: EU07 on td.scn with no route set stops before the exit signal stacja1_j at stop and stays; nothing reports a result, so the test ends once the train under test stood still for test.standstill 120 seconds (the train waits about 70 s before it pulls off, so less would end it before the start).", "base_scenery": "td.scn", "time": "10:34", "fragment": "td_red_signal.scn", diff --git a/tests/scenarios/cases/test_mode_standstill_traffic.json b/tests/scenarios/cases/test_mode_standstill_traffic.json new file mode 100644 index 00000000..8b545e2a --- /dev/null +++ b/tests/scenarios/cases/test_mode_standstill_traffic.json @@ -0,0 +1,18 @@ +{ + "description": "Test mode, standstill with traffic: an EU07 stands on track 2 of Stacja1 on td.scn while another EU07, AI traffic, leaves track 1 on the td/rozklad timetable at 10:35 and runs on. Only the train under test (test.vehicle, the case's vehicle) counts, so the test ends on test.standstill 100 although the traffic is moving; counting every vehicle, it would run into the 300 s test timeout.", + "base_scenery": "td.scn", + "time": "10:34", + "fragment": "td_traffic.scn", + "test_config": [ + "test.standstill 100" + ], + "expect": "standstill", + "duration": 300, + "timeout": 600, + "checks": [ + { + "type": "max_speed", + "limit": 0.5 + } + ] +} diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index 62cb7d13..3757a824 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -158,9 +158,10 @@ def build_scenery(case: dict, game_dir: Path, case_name: str, vehicle: str) -> s # events of the case go in before it (read after it they never fire), trainsets after it # (placed before it they stand on unlinked track and derail at once) before, trainsets = split_trainsets(fragment) - if case.get("test_config"): - # settings of the test mode (test.timeout and so on), read like any other config block - before = "config\n" + "".join(line + "\n" for line in case["test_config"]) + "endconfig\n" + before + # settings of the test mode, read like any other config block: the case's vehicle is the train under test, + # whose consist the pass and the standstill wait for, and the case's own (test.timeout and so on) + config = [f"test.vehicle {vehicle}", *case.get("test_config", [])] + before = "config\n" + "".join(line + "\n" for line in config) + "endconfig\n" + before header = f"\n// ---- autotest: {case_name} ----\n" firstinit = re.search(r"(?im)^\s*firstinit\b", text) if firstinit: @@ -526,6 +527,9 @@ def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, watch: bool shutil.move(path, out / f"{kind}.txt") if physicslog.exists(): shutil.move(physicslog, out / physicslog.name) + # other trains of the case with a driver, named _, log too + for other in physicslog.parent.glob(f"*_{vehicle}.dat"): + shutil.move(other, out / other.name) ok = True sim_t = result.rows[-1][COL_TIME] if result.rows else 0.0 diff --git a/tests/unit/test_test_mode.cpp b/tests/unit/test_test_mode.cpp index 55a90387..27e8c0d9 100644 --- a/tests/unit/test_test_mode.cpp +++ b/tests/unit/test_test_mode.cpp @@ -167,7 +167,7 @@ TEST_CASE("test mode: the end conditions") CHECK(verdict->result == test_result::fail); CHECK(verdict->reason == "__test.result set to 0 (wrong_track)"); } - SUBCASE("1 passes once every vehicle has stood for a second") + SUBCASE("1 passes once the tested trains have stood for a second") { CHECK_FALSE(simulation::evaluate_frame(progress, frame(false, 1.0), limits).has_value()); CHECK(progress.passrequested); diff --git a/utilities/Globals.cpp b/utilities/Globals.cpp index 217374cb..8957a4a0 100644 --- a/utilities/Globals.cpp +++ b/utilities/Globals.cpp @@ -477,6 +477,17 @@ bool global_settings::ConfigParseTest(cParser& Parser, std::string_view const To return true; } + if (Token == "test.vehicle") + { + // a vehicle of a train under test, by its name in the scenery; the pass and the standstill wait for its + // consist only, not for other trains (eu07/maszyna#541). given more than once, for each of them + std::string name; + ParseOne(Parser, name, 1, true); + if (false == name.empty() && std::ranges::find(test.vehicles, name) == test.vehicles.end()) + test.vehicles.emplace_back(name); + return true; + } + if (Token == "test.standstill") { ParseOne(Parser, test.standstill); @@ -1802,6 +1813,9 @@ global_settings::export_as_text( std::ostream &Output ) const { export_as_text( Output, "test.pass_on_standstill", test.pass_on_standstill ); export_as_text( Output, "test.fail_if_stop_overran", test.fail_if_stop_overran ); export_as_text( Output, "test.no_movement_before_pass", test.no_movement_before_pass ); + for( auto const &vehicle : test.vehicles ) { + export_as_text( Output, "test.vehicle", vehicle ); + } export_as_text( Output, "gfx.envmap.enabled", gfx_envmap_enabled ); export_as_text( Output, "gfx.postfx.motionblur.enabled", gfx_postfx_motionblur_enabled ); export_as_text( Output, "gfx.postfx.motionblur.shutter", gfx_postfx_motionblur_shutter ); diff --git a/utilities/Globals.h b/utilities/Globals.h index 81bc1e47..50e42085 100644 --- a/utilities/Globals.h +++ b/utilities/Globals.h @@ -91,10 +91,11 @@ struct global_settings { struct test_settings { bool enabled{ false }; double timeout{ 3600.0 }; // [s] of simulation time; no result by then fails the test - double standstill{ 300.0 }; // [s] every vehicle standing still this long ends the test; 0 = never + double standstill{ 300.0 }; // [s] the tested trains standing still this long ends the test; 0 = never bool pass_on_standstill{ false }; // the standstill end passes the test instead of failing it bool fail_if_stop_overran{ true }; // a train passing a signal at stop ends the test as failed; with false it's only logged - bool no_movement_before_pass{ true }; // a pass waits until every vehicle stands still; with false it ends the test at once + bool no_movement_before_pass{ true }; // a pass waits until the tested trains stand still; with false it ends the test at once + std::vector vehicles; // test.vehicle: the trains under test are the consists of these; none = every vehicle bool timeout_from_command_line{ false }; // --test-timeout given, the scenery doesn't override it std::optional wall_timeout; // [s] of real time since the start; none = test.timeout + time for the loading, 0 = no limit bool wall_timeout_from_command_line{ false }; // --test-wall-timeout given, the scenery doesn't override it