diff --git a/simulation/simulationtestrules.h b/simulation/simulationtestrules.h index 5d966c3c2..99d35db7b 100644 --- a/simulation/simulationtestrules.h +++ b/simulation/simulationtestrules.h @@ -18,14 +18,16 @@ http://mozilla.org/MPL/2.0/. // codes, the command line and the end conditions. kept apart so the unit tests can check them namespace simulation { -// exit codes of a test run; public, scripts and CI read them +// exit codes of a test run; public, scripts and CI read them. none of them is a code a process commonly ends with +// when it goes wrong on its own: 1 (killed with TerminateProcess, or a plain failure of most programs), 3 (abort() +// on Windows), 126-127 (a shell that can't run the program) or 128 + a signal number (killed, on Linux) enum class test_result : int { pass = 0, // __test.result set to 1, then every vehicle came to rest (or test.pass_on_standstill) - fail = 1, // __test.result set to 0 - signal_passed_at_danger = 2, // a train went past a stop signal (SetVelocity 0 0), unless test.fail_if_stop_overran no - timeout = 3, // test.timeout seconds of simulation time went by without a result - standstill = 4, // every vehicle stood still for test.standstill seconds without a result - error = 5, // the run could not start (a bad command line, the scenery failed to load) or the simulator threw + fail = 10, // __test.result set to 0 + signal_passed_at_danger = 11, // a train went past a stop signal (SetVelocity 0 0), unless test.fail_if_stop_overran no + timeout = 12, // test.timeout seconds of simulation time went by without a result + standstill = 13, // every vehicle stood still for test.standstill seconds without a result + error = 14, // the run could not start (a bad command line, the scenery failed to load) or the simulator threw }; std::string_view to_string(test_result Result); diff --git a/tests/scenarios/README.md b/tests/scenarios/README.md index 8bd6f1a4a..acab07e79 100644 --- a/tests/scenarios/README.md +++ b/tests/scenarios/README.md @@ -62,13 +62,16 @@ The test ends with the first of these: | exit code | result | when | | --- | --- | --- | | 0 | pass | `__test.result` was set to 1 and then every vehicle stood still for 1 s, so a train still moving towards a signal at stop has to stop first (with `test.no_movement_before_pass no` the pass ends the test at once). Also the standstill end with `test.pass_on_standstill yes` | -| 1 | fail | `__test.result` was set to 0; ends at once | -| 2 | signal passed at danger | a train went past a signal showing stop (`SetVelocity 0 0`; `SetVelocity 0 x` on automatic block, which may be passed after a stop, doesn't count); ends at once, unless `test.fail_if_stop_overran no` | -| 3 | timeout | `test.timeout` seconds of simulation time went by (default 3600) | -| 4 | standstill | every vehicle stood still for `test.standstill` seconds (default 300) and nothing reported a result. Trains waiting at a station or for a signal count as standing, so the limit has to be longer than such waits | -| 5 | error | the run could not start: no `-s`, the scenery failed to load or has no vehicles, or the simulator threw | +| 10 | fail | `__test.result` was set to 0; ends at once | +| 11 | signal passed at danger | a train went past a signal showing stop (`SetVelocity 0 0`; `SetVelocity 0 x` on automatic block, which may be passed after a stop, doesn't count); ends at once, unless `test.fail_if_stop_overran no` | +| 12 | timeout | `test.timeout` seconds of simulation time went by (default 3600) | +| 13 | standstill | every vehicle stood still for `test.standstill` seconds (default 300) and nothing reported a result. Trains waiting at a station or for a signal count as standing, so the limit has to be longer than such waits | +| 14 | error | the run could not start: no `-s`, the scenery failed to load or has no vehicles, or the simulator threw | -A crash ends with the system's code instead (on Windows e.g. `0xC0000005`, -1073741819). +A crash ends with the system's code instead (on Windows e.g. `0xC0000005`, -1073741819). The +codes of the test mode keep clear of the ones a process gets when it goes wrong on its own: 1 +(killed with `TerminateProcess`, or a plain failure of most programs), 3 (`abort()` on Windows), +126-127 (a shell that can't run the program) and 128 + a signal number (killed, on Linux). The limits go in a `config` block of the scenery: @@ -180,8 +183,8 @@ and slides past the signal, so those cases set `test.fail_if_stop_overran no`. on `tor1k`, the track in front of the signal. The `test_mode_*` cases show the ends of the test mode: `test_mode_pass` (exit 0, the pass -event), `test_mode_signal_passed_at_danger` (exit 2), `test_mode_timeout` (exit 3, from -`test.timeout`) and `test_mode_standstill` (exit 4, from `test.standstill`). +event), `test_mode_signal_passed_at_danger` (exit 11), `test_mode_timeout` (exit 12, from +`test.timeout`) and `test_mode_standstill` (exit 13, from `test.standstill`). `test_mode_pass_while_moving` reports the pass from `event2` of `tor1k` (the direction the train runs), as the train runs onto it, with `test.no_movement_before_pass no`, so the test ends with the train still moving. diff --git a/tests/scenarios/run.py b/tests/scenarios/run.py index 210d6c013..886e2e90d 100644 --- a/tests/scenarios/run.py +++ b/tests/scenarios/run.py @@ -52,14 +52,14 @@ COL_COMMAND = 16 # text, read as None COL_WHEELSLIP = 20 COL_VEHICLE_VELOCITY = 25 # signed vehicle speed, km/h; only in builds which log it -# exit codes of the simulator's test mode (eu07 --test), see simulation/simulationtest.h +# exit codes of the simulator's test mode (eu07 --test), see simulation/simulationtestrules.h EXIT_RESULTS = { 0: "pass", - 1: "fail", - 2: "signal_passed_at_danger", - 3: "timeout", - 4: "standstill", - 5: "error", + 10: "fail", + 11: "signal_passed_at_danger", + 12: "timeout", + 13: "standstill", + 14: "error", } # settings layered over the user's eu07.ini for a test run; the window ones only matter diff --git a/tests/unit/test_test_mode.cpp b/tests/unit/test_test_mode.cpp index fec475c5c..609b16ff0 100644 --- a/tests/unit/test_test_mode.cpp +++ b/tests/unit/test_test_mode.cpp @@ -3,9 +3,10 @@ #include "simulation/simulationtestrules.h" +#include #include -// the test mode of the simulator (eu07 --test, eu07/maszyna#541): its command line and end conditions +// the test mode of the simulator (eu07 --test, eu07/maszyna#541): its exit codes, command line and end conditions namespace { @@ -37,6 +38,27 @@ std::optional run(simulation::test_progress &Progress, } // namespace +TEST_CASE("test mode: the exit codes keep clear of the ones of a killed or aborted process") +{ + std::array const results{test_result::pass, test_result::fail, test_result::signal_passed_at_danger, test_result::timeout, test_result::standstill, test_result::error}; + for (auto const result : results) + { + auto const code{static_cast(result)}; + CAPTURE(code); + CHECK(code != 1); // TerminateProcess, most programs' failure + CHECK(code != 3); // abort() on Windows + CHECK(code < 126); // shells, signals on Linux + CHECK(simulation::to_string(result) != "unknown"); + } + // the codes are public: scripts and CI read them + CHECK(static_cast(test_result::pass) == 0); + CHECK(static_cast(test_result::fail) == 10); + CHECK(static_cast(test_result::signal_passed_at_danger) == 11); + CHECK(static_cast(test_result::timeout) == 12); + CHECK(static_cast(test_result::standstill) == 13); + CHECK(static_cast(test_result::error) == 14); +} + TEST_CASE("test mode: seconds on the command line are a number, not negative") { CHECK(simulation::parse_seconds("240") == 240.0);