18
0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-11-19 20:24:59 +01:00

feat(simulation): a real time watchdog ends a test run that hangs

A test run that hung, in the loading or anywhere else, waited until whoever
started it killed it. A watchdog thread now ends a run still going test.wall_timeout
seconds of real time after the start of the simulator, loading included: it says
`test: timeout ... still running after N s of real time` and quits with the
timeout code, whatever the main thread is doing, with no dialog. A test that has
ended but whose simulator doesn't close ends with its own result.

The limit is test.wall_timeout (settings, the scenery's config block) or
--test-wall-timeout, which wins. Not given, it is test.timeout + 600 s: ten
minutes for the loading, and a simulation that runs at least as fast as real
time is never cut short. test.timeout 0 means no watchdog unless the limit is
given; test.wall_timeout 0 turns it off.

run.py passes the case's timeout as --test-wall-timeout and kills a simulator
only 30 s after it. A timeout from the watchdog counts as wall_timeout there, so
a run that hung doesn't pass a case expecting the simulation time to run out;
test_mode_wall_timeout ends while loading on a limit of 2 s.
This commit is contained in:
jerrrrycho
2026-11-08 11:33:36 +01:00
parent 14d25520f3
commit 03a417c7b0
11 changed files with 210 additions and 35 deletions

View File

@@ -1166,6 +1166,11 @@ int eu07_application::init_settings(int Argc, char *Argv[])
Global.test.timeout = *testcommandline.timeout; Global.test.timeout = *testcommandline.timeout;
Global.test.timeout_from_command_line = true; Global.test.timeout_from_command_line = true;
} }
if (testcommandline.wall_timeout.has_value())
{
Global.test.wall_timeout = testcommandline.wall_timeout;
Global.test.wall_timeout_from_command_line = true;
}
fs::path iniPath = user_config_path("eu07.ini"); fs::path iniPath = user_config_path("eu07.ini");
@@ -1231,7 +1236,7 @@ int eu07_application::init_settings(int Argc, char *Argv[])
// integration test: the scenery runs unattended and the exit code is the result (eu07/maszyna#541), // integration test: the scenery runs unattended and the exit code is the result (eu07/maszyna#541),
// taken from the command line above // taken from the command line above
} }
else if (token == "--test-timeout") else if (token == "--test-timeout" || token == "--test-wall-timeout")
{ {
// taken from the command line above, with its value // taken from the command line above, with its value
++i; ++i;
@@ -1253,7 +1258,7 @@ int eu07_application::init_settings(int Argc, char *Argv[])
<< " [-edit [sceneryfilepath]]" << " [-edit [sceneryfilepath]]"
<< " [-seed number|word]" << " [-seed number|word]"
<< " [-ini settingsfile]" << " [-ini settingsfile]"
<< " [--test [--test-timeout seconds]]" << std::endl; << " [--test [--test-timeout seconds] [--test-wall-timeout seconds]]" << std::endl;
simulation::test().conclude(simulation::test_result::error, std::format("unknown command line argument {}", token)); simulation::test().conclude(simulation::test_result::error, std::format("unknown command line argument {}", token));
return -1; return -1;
} }

View File

@@ -27,9 +27,20 @@ http://mozilla.org/MPL/2.0/.
#include <chrono> #include <chrono>
#include <cmath> #include <cmath>
#include <cstdio> #include <cstdio>
#include <cstdlib>
#include <format> #include <format>
#include <memory> #include <memory>
namespace {
// [s] of real time since the start of the simulator
double elapsed_wall_time()
{
return std::chrono::duration<double>(std::chrono::steady_clock::now() - Global.startTimestamp).count();
}
} // namespace
namespace simulation { namespace simulation {
test_manager &test() test_manager &test()
@@ -77,6 +88,8 @@ void apply_test_mode_settings()
// and the same way every time, unless given another seed (-seed, or randomseed in the settings) // and the same way every time, unless given another seed (-seed, or randomseed in the settings)
if (!Global.random_seed) if (!Global.random_seed)
Global.random_seed = seed_of("test"); Global.random_seed = seed_of("test");
// the real time limit, which the scenery may set too
test().watch_wall_clock(wall_clock_limit(Global.test.wall_timeout, Global.test.timeout));
} }
void test_manager::init_result_cell() const void test_manager::init_result_cell() const
@@ -94,7 +107,10 @@ void test_manager::start()
if (m_started) if (m_started)
return; return;
m_started = true; m_started = true;
m_progress = test_progress {}; {
std::scoped_lock lock { m_lock };
m_progress = test_progress {};
}
if (simulation::Vehicles.sequence().empty()) if (simulation::Vehicles.sequence().empty())
{ {
// most likely the scenery wasn't found, or it isn't a test scenery // most likely the scenery wasn't found, or it isn't a test scenery
@@ -103,7 +119,9 @@ void test_manager::start()
} }
// a pause set by the scenery or the settings would never be lifted, nobody is there to press the key // a pause set by the scenery or the settings would never be lifted, nobody is there to press the key
Global.iPause = 0; Global.iPause = 0;
WriteLog(std::format("test: started, timeout {:.0f} s, standstill {:.0f} s{}{}{}", Global.test.timeout, Global.test.standstill, auto const walllimit { m_walllimit.load() };
WriteLog(std::format("test: started, timeout {:.0f} s, real time limit {}, standstill {:.0f} s{}{}{}", Global.test.timeout,
walllimit > 0.0 ? std::format("{:.0f} s", walllimit) : std::string("none"), Global.test.standstill,
Global.test.pass_on_standstill ? ", standstill passes" : "", Global.test.pass_on_standstill ? ", standstill passes" : "",
Global.test.fail_if_stop_overran ? "" : ", signals passed at danger allowed", Global.test.fail_if_stop_overran ? "" : ", signals passed at danger allowed",
Global.test.no_movement_before_pass ? "" : ", pass without waiting for a stop")); Global.test.no_movement_before_pass ? "" : ", pass without waiting for a stop"));
@@ -122,14 +140,21 @@ void test_manager::update(double const Deltatime)
frame.reason = cell->Text(); frame.reason = cell->Text();
} }
auto const requestedbefore { m_progress.passrequested }; auto const limits { test_limits { Global.test.timeout, Global.test.standstill, Global.test.pass_on_standstill, Global.test.no_movement_before_pass } };
auto const verdict { evaluate_frame(m_progress, frame, test_limits { Global.test.timeout, Global.test.standstill, Global.test.pass_on_standstill, Global.test.no_movement_before_pass }) }; std::optional<test_verdict> verdict;
auto passrequested { false };
{
std::scoped_lock lock { m_lock };
auto const requestedbefore { m_progress.passrequested };
verdict = evaluate_frame(m_progress, frame, limits);
passrequested = m_progress.passrequested && false == requestedbefore;
}
if (verdict.has_value()) if (verdict.has_value())
{ {
finish(verdict->result, verdict->reason); finish(verdict->result, verdict->reason);
return; return;
} }
if (m_progress.passrequested && false == requestedbefore) if (passrequested)
{ {
WriteLog(std::format("test: pass requested at {:.1f} s{}, waiting for every vehicle to stop", m_progress.time, WriteLog(std::format("test: pass requested at {:.1f} s{}, waiting for every vehicle to stop", m_progress.time,
frame.reason.empty() ? std::string() : std::format(" ({})", frame.reason))); frame.reason.empty() ? std::string() : std::format(" ({})", frame.reason)));
@@ -160,10 +185,11 @@ void test_manager::finish(test_result const Result, std::string_view const Reaso
bool test_manager::conclude(test_result const Result, std::string_view const Reason) bool test_manager::conclude(test_result const Result, std::string_view const Reason)
{ {
std::scoped_lock lock { m_lock };
if (m_finished || false == Global.test.enabled) if (m_finished || false == Global.test.enabled)
return false; return false;
m_finished = true;
m_result = Result; m_result = Result;
m_finished = true;
auto const line { std::format("test: {} (exit code {}) after {:.1f} s of simulation time: {}", to_string(Result), static_cast<int>(Result), m_progress.time, Reason) }; auto const line { std::format("test: {} (exit code {}) after {:.1f} s of simulation time: {}", to_string(Result), static_cast<int>(Result), m_progress.time, Reason) };
if (Result == test_result::pass) if (Result == test_result::pass)
@@ -176,4 +202,39 @@ bool test_manager::conclude(test_result const Result, std::string_view const Rea
return true; return true;
} }
int test_manager::exit_code() const
{
std::scoped_lock lock { m_lock };
return static_cast<int>(m_finished ? m_result : test_result::error);
}
void test_manager::watch_wall_clock(double const Limit)
{
m_walllimit = Limit;
if (Limit <= 0.0 || m_watchdog.joinable())
return;
m_watchdog = std::jthread([this](std::stop_token const &Stop) {
while (false == Stop.stop_requested())
{
std::this_thread::sleep_for(std::chrono::milliseconds(100));
check_wall_clock(elapsed_wall_time());
}
});
}
void test_manager::check_wall_clock(double const Elapsed)
{
auto const limit { m_walllimit.load() };
if (limit <= 0.0 || Elapsed < limit)
return;
// the run hangs, or is too slow: it ends here, whatever the main thread is doing. a run whose test has ended
// but which doesn't close ends with its result
if (false == conclude(test_result::timeout, std::format("still running after {:.0f} s of real time (test.wall_timeout)", limit)))
ErrorLog(std::format("test: the simulator was still running {:.0f} s of real time after the start, ended", limit));
// what was logged last gets written, unless the log is held up by a hung thread
StopLogService();
std::this_thread::sleep_for(std::chrono::milliseconds(250));
std::_Exit(exit_code());
}
} // namespace simulation } // namespace simulation

View File

@@ -11,7 +11,10 @@ http://mozilla.org/MPL/2.0/.
#include "simulation/simulationtestrules.h" #include "simulation/simulationtestrules.h"
#include <atomic>
#include <mutex>
#include <string_view> #include <string_view>
#include <thread>
// integration test mode (eu07 --test -s <scenery>, eu07/maszyna#541): the scenery runs unattended in a tight // integration test mode (eu07 --test -s <scenery>, eu07/maszyna#541): the scenery runs unattended in a tight
// fixed-step loop and the simulator quits with an exit code saying how the test went // fixed-step loop and the simulator quits with an exit code saying how the test went
@@ -38,16 +41,26 @@ public:
// ends the test with given result, without closing the application: for the ways out that close it anyway // ends the test with given result, without closing the application: for the ways out that close it anyway
// (a bad command line, a failed start, an exception). returns false if the test had already ended // (a bad command line, a failed start, an exception). returns false if the test had already ended
bool conclude(test_result Result, std::string_view Reason); bool conclude(test_result Result, std::string_view Reason);
// (re)sets the real time limit of the run [s] since the start of the simulator, 0 = none. the first limit
// starts the watchdog, which ends a run still going at the limit with the timeout code, whatever it's doing
void watch_wall_clock(double Limit);
bool finished() const { return m_finished; } bool finished() const { return m_finished; }
// exit code of the process: the result, or error while the test hasn't finished // exit code of the process: the result, or error while the test hasn't finished
int exit_code() const { return static_cast<int>(m_finished ? m_result : test_result::error); } int exit_code() const;
private: private:
// the watchdog's check, at Elapsed seconds of real time since the start of the simulator
void check_wall_clock(double Elapsed);
// the watchdog may end the test from its own thread
mutable std::mutex m_lock;
bool m_started { false }; bool m_started { false };
bool m_finished { false }; std::atomic<bool> m_finished { false };
test_result m_result { test_result::error }; test_result m_result { test_result::error };
test_progress m_progress; test_progress m_progress;
std::atomic<double> m_walllimit { 0.0 }; // [s] since the start of the simulator, 0 = none
std::jthread m_watchdog;
}; };
// the test of the run // the test of the run

View File

@@ -81,7 +81,7 @@ test_command_line parse_test_command_line(std::span<char const *const> const Arg
{ {
result.enabled = true; result.enabled = true;
} }
else if (option == "--test-timeout") else if (option == "--test-timeout" || option == "--test-wall-timeout")
{ {
auto const seconds{option_seconds(Arguments, i)}; auto const seconds{option_seconds(Arguments, i)};
if (false == seconds.has_value()) if (false == seconds.has_value())
@@ -91,7 +91,10 @@ test_command_line parse_test_command_line(std::span<char const *const> const Arg
result.error = std::format("{} needs a number of seconds", option); result.error = std::format("{} needs a number of seconds", option);
continue; continue;
} }
result.timeout = seconds; if (option == "--test-timeout")
result.timeout = seconds;
else
result.wall_timeout = seconds;
} }
else if (takes_value(option)) else if (takes_value(option))
{ {
@@ -106,6 +109,13 @@ test_command_line parse_test_command_line(std::span<char const *const> const Arg
return result; return result;
} }
double wall_clock_limit(std::optional<double> const Given, double const Simulationtimeout)
{
if (Given.has_value())
return *Given;
return Simulationtimeout > 0.0 ? Simulationtimeout + wall_timeout_loading_allowance : 0.0;
}
std::optional<test_verdict> evaluate_frame(test_progress &Progress, test_frame const &Frame, test_limits const &Limits) std::optional<test_verdict> evaluate_frame(test_progress &Progress, test_frame const &Frame, test_limits const &Limits)
{ {
using enum test_result; using enum test_result;

View File

@@ -15,7 +15,7 @@ http://mozilla.org/MPL/2.0/.
#include <string_view> #include <string_view>
// the rules of the test mode (eu07 --test, eu07/maszyna#541) which need nothing of the running simulation: the exit // the rules of the test mode (eu07 --test, eu07/maszyna#541) which need nothing of the running simulation: the exit
// codes, the command line and the end conditions. kept apart so the unit tests can check them // codes, the command line, the end conditions and the real time limit. kept apart so the unit tests can check them
namespace simulation { namespace simulation {
// exit codes of a test run; public, scripts and CI read them. none of them is a code a process commonly ends with // exit codes of a test run; public, scripts and CI read them. none of them is a code a process commonly ends with
@@ -25,7 +25,7 @@ enum class test_result : int {
pass = 0, // __test.result set to 1, then every vehicle came to rest (or test.pass_on_standstill) pass = 0, // __test.result set to 1, then every vehicle came to rest (or test.pass_on_standstill)
fail = 10, // __test.result set to 0 fail = 10, // __test.result set to 0
signal_passed_at_danger = 11, // a train went past a stop signal (SetVelocity 0 0), unless test.fail_if_stop_overran no signal_passed_at_danger = 11, // a train went past a stop signal (SetVelocity 0 0), unless test.fail_if_stop_overran no
timeout = 12, // test.timeout seconds of simulation time went by without a result timeout = 12, // test.timeout seconds of simulation time, or test.wall_timeout seconds of real time went by
standstill = 13, // every vehicle stood still for test.standstill seconds without a result standstill = 13, // every vehicle stood still for test.standstill seconds without a result
error = 14, // the run could not start (a bad command line, the scenery failed to load) or the simulator threw error = 14, // the run could not start (a bad command line, the scenery failed to load) or the simulator threw
}; };
@@ -42,6 +42,7 @@ struct test_verdict {
struct test_command_line { struct test_command_line {
bool enabled{false}; // --test bool enabled{false}; // --test
std::optional<double> timeout; // --test-timeout <seconds of simulation time> std::optional<double> timeout; // --test-timeout <seconds of simulation time>
std::optional<double> wall_timeout; // --test-wall-timeout <seconds of real time>
std::string error; // what's wrong with them; empty if nothing std::string error; // what's wrong with them; empty if nothing
}; };
@@ -52,6 +53,14 @@ std::optional<double> parse_seconds(std::string_view Text);
// options are skipped, so a scenery named like an option isn't taken for one; options it doesn't know are left alone // options are skipped, so a scenery named like an option isn't taken for one; options it doesn't know are left alone
test_command_line parse_test_command_line(std::span<char const *const> Arguments); test_command_line parse_test_command_line(std::span<char const *const> Arguments);
// time allowed for loading the scenery, on top of the simulation time, when the real time limit isn't given [s]
inline constexpr double wall_timeout_loading_allowance{600.0};
// the real time limit of a test run [s], 0 = none: the one given, else the simulation time limit plus time for the
// loading, so a simulation that runs at least as fast as real time is never cut short. no simulation time limit
// (test.timeout 0) means none in real time either, unless given
double wall_clock_limit(std::optional<double> Given, double Simulationtimeout);
// the limits and switches the end conditions go by // the limits and switches the end conditions go by
struct test_limits { struct test_limits {
double timeout{3600.0}; // [s] of simulation time; 0 = never double timeout{3600.0}; // [s] of simulation time; 0 = never

View File

@@ -13,7 +13,7 @@ The runner drives the simulator's test mode (eu07/maszyna#541), which works with
runner too and is meant for scenery authors as well: runner too and is meant for scenery authors as well:
``` ```
eu07 --test -s <scenery.scn> [--test-timeout <seconds>] [-seed <number|word>] [-ini <settings file>] eu07 --test -s <scenery.scn> [--test-timeout <seconds>] [--test-wall-timeout <seconds>] [-seed <number|word>] [-ini <settings file>]
``` ```
With `--test` the simulator: With `--test` the simulator:
@@ -70,7 +70,7 @@ The test ends with the first of these:
| 0 | pass | `__test.result` was set to 1 and then every vehicle stood still for 1 s, so a train still moving towards a signal at stop has to stop first (with `test.no_movement_before_pass no` the pass ends the test at once). Also the standstill end with `test.pass_on_standstill yes` | | 0 | pass | `__test.result` was set to 1 and then every vehicle stood still for 1 s, so a train still moving towards a signal at stop has to stop first (with `test.no_movement_before_pass no` the pass ends the test at once). Also the standstill end with `test.pass_on_standstill yes` |
| 10 | fail | `__test.result` was set to 0; ends at once | | 10 | fail | `__test.result` was set to 0; ends at once |
| 11 | signal passed at danger | a train went past a signal showing stop (`SetVelocity 0 0`; `SetVelocity 0 x` on automatic block, which may be passed after a stop, doesn't count); ends at once, unless `test.fail_if_stop_overran no` | | 11 | signal passed at danger | a train went past a signal showing stop (`SetVelocity 0 0`; `SetVelocity 0 x` on automatic block, which may be passed after a stop, doesn't count); ends at once, unless `test.fail_if_stop_overran no` |
| 12 | timeout | `test.timeout` seconds of simulation time went by (default 3600) | | 12 | timeout | `test.timeout` seconds of simulation time went by (default 3600), or `test.wall_timeout` seconds of real time since the start (see below) |
| 13 | standstill | every vehicle stood still for `test.standstill` seconds (default 300) and nothing reported a result. Trains waiting at a station or for a signal count as standing, so the limit has to be longer than such waits | | 13 | standstill | every vehicle stood still for `test.standstill` seconds (default 300) and nothing reported a result. Trains waiting at a station or for a signal count as standing, so the limit has to be longer than such waits |
| 14 | error | the run could not start: no `-s`, the scenery failed to load or has no vehicles, or the simulator threw | | 14 | error | the run could not start: no `-s`, the scenery failed to load or has no vehicles, or the simulator threw |
@@ -84,6 +84,7 @@ The limits go in a `config` block of the scenery:
``` ```
config config
test.timeout 600 test.timeout 600
test.wall_timeout 900
test.standstill 120 test.standstill 120
test.pass_on_standstill no test.pass_on_standstill no
test.fail_if_stop_overran yes test.fail_if_stop_overran yes
@@ -94,12 +95,22 @@ endconfig
| key | default | meaning | | key | default | meaning |
| --- | --- | --- | | --- | --- | --- |
| `test.timeout` | 3600 | seconds of simulation time before the test ends as `timeout`; 0 = never | | `test.timeout` | 3600 | seconds of simulation time before the test ends as `timeout`; 0 = never |
| `test.wall_timeout` | `test.timeout` + 600 | seconds of real time since the start before the test ends as `timeout`; 0 = never |
| `test.standstill` | 300 | seconds with every vehicle still and no result before the test ends as `standstill`; 0 = never | | `test.standstill` | 300 | seconds with every vehicle still and no result before the test ends as `standstill`; 0 = never |
| `test.pass_on_standstill` | no | the standstill end passes the test instead | | `test.pass_on_standstill` | no | the standstill end passes the test instead |
| `test.fail_if_stop_overran` | yes | a signal passed at danger ends the test; with `no` it's only logged | | `test.fail_if_stop_overran` | yes | a signal passed at danger ends the test; with `no` it's only logged |
| `test.no_movement_before_pass` | yes | a pass waits until every vehicle stands still; with `no` it ends the test at once | | `test.no_movement_before_pass` | yes | a pass waits until every vehicle stands still; with `no` it ends the test at once |
`--test-timeout` on the command line wins over `test.timeout` of the scenery. `--test-timeout` and `--test-wall-timeout` on the command line win over `test.timeout` and
`test.wall_timeout` of the scenery and the settings files.
`test.wall_timeout` is a watchdog in real time, for a run that hangs or crawls: when the
simulator still runs that many seconds after its start, loading included, it says `test: timeout`
and quits with the timeout code, whatever it is doing at the moment, with no dialog. Not given,
it is `test.timeout` + 600 s: ten minutes for the loading, and a simulation that runs at least as
fast as real time is never cut short by it. With `test.timeout 0` there is no watchdog unless
`test.wall_timeout` is given; `test.wall_timeout 0` turns it off. A test that has ended but whose
simulator doesn't close is ended by the watchdog too, with its own result.
A signal is passed at danger when the head of a train goes past a signal which its driver A signal is passed at danger when the head of a train goes past a signal which its driver
has in its speed table, showing stop. So it's the driver's view that counts: a train without has in its speed table, showing stop. So it's the driver's view that counts: a train without
@@ -120,7 +131,7 @@ Python 3.10+, no extra packages. Options:
`cmake-build-*/bin/Release/eu07*.exe`. The build needs the `--test` and `-ini` command `cmake-build-*/bin/Release/eu07*.exe`. The build needs the `--test` and `-ini` command
line options. line options.
- name one or more cases to run only those, e.g. `run.py ... eu07_td_timetable`. - name one or more cases to run only those, e.g. `run.py ... eu07_td_timetable`.
- `-j N` runs N cases at once. Each case prints its results when it is done. All 15 cases - `-j N` runs N cases at once. Each case prints its results when it is done. All 16 cases
take about 30 s with `-j 4`, loading included. take about 30 s with `-j 4`, loading included.
- `--watch` runs a case the old way, in a visible window in real time (or faster with - `--watch` runs a case the old way, in a visible window in real time (or faster with
`--timespeed N`), closed after the case's `duration`; the exit code isn't checked then. For `--timespeed N`), closed after the case's `duration`; the exit code isn't checked then. For
@@ -138,13 +149,15 @@ For each case the runner:
added, and the case's `.scn` fragment added: its trainsets after `FirstInit`, everything added, and the case's `.scn` fragment added: its trainsets after `FirstInit`, everything
else (the case's `test_config`, event launchers standing in for keys the scenery expects a else (the case's `test_config`, event launchers standing in for keys the scenery expects a
player to press, extra events) before it, player to press, extra events) before it,
2. starts the simulator with `--test -s autotest_<case>.scn -ini out/<case>/autotest.ini`, 2. starts the simulator with `--test -s autotest_<case>.scn -ini out/<case>/autotest.ini
plus `--test-timeout <duration>` when the case gives a `duration`; the extra ini turns on --test-wall-timeout <timeout>`, plus `--test-timeout <duration>` when the case gives a
`duration`; the extra ini turns on
`physicslog` and writes the logs to `logs/log (autotest_<case>) <date>.txt` rather than the `physicslog` and writes the logs to `logs/log (autotest_<case>) <date>.txt` rather than the
shared `log.txt` (`multiplelogs`), shared `log.txt` (`multiplelogs`),
3. waits until the simulator quits by itself. It gives up early when the log reports 3. waits until the simulator quits by itself. It gives up early when the log reports
`Bad vehicle` / `Bad trainset`, when the scenario has loaded and no physicslog appeared, or `Bad vehicle` / `Bad trainset`, or when the scenario has loaded and no physicslog appeared.
after the case's `timeout` of wall-clock time, After the case's `timeout` of wall-clock time the simulator ends itself with the timeout code;
one that doesn't is killed 30 s later,
4. compares the result from the exit code with the case's `expect`, then runs the checks on 4. compares the result from the exit code with the case's `expect`, then runs the checks on
the physicslog. the physicslog.
@@ -167,9 +180,9 @@ scenery is newer.
| `test_config` | lines of a `config` block added to the scenery, e.g. `["test.timeout 90"]` | | `test_config` | lines of a `config` block added to the scenery, e.g. `["test.timeout 90"]` |
| `fragment` | `.scn` file in `cases/` added to the scenery: trainsets after `FirstInit` (before it they derail on unlinked track), the rest before it (launchers read after it never fire) | | `fragment` | `.scn` file in `cases/` added to the scenery: trainsets after `FirstInit` (before it they derail on unlinked track), the rest before it (launchers read after it never fire) |
| `vars` | values for `{{name}}` in the fragment; a list becomes one line per item | | `vars` | values for `{{name}}` in the fragment; a list becomes one line per item |
| `expect` | the result the test mode should end with: `pass`, `fail`, `signal_passed_at_danger`, `timeout`, `standstill` or `error`. Default `timeout` for a case with `duration`, else `pass` | | `expect` | the result the test mode should end with: `pass`, `fail`, `signal_passed_at_danger`, `timeout`, `standstill` or `error`, or `wall_timeout` for the timeout of the real time watchdog (the timeout code with a verdict saying it was the real time; a case expecting `timeout` fails on it). Default `timeout` for a case with `duration`, else `pass` |
| `duration` | seconds of simulation to record: the test timeout of the run (`--test-timeout`) | | `duration` | seconds of simulation to record: the test timeout of the run (`--test-timeout`) |
| `timeout` | wall-clock seconds before giving up, loading included | | `timeout` | wall-clock seconds before giving up, loading included: the simulator's `--test-wall-timeout` (default 600) |
| `fail_on_errors` | fail when `errors.txt` is not empty (default `false`) | | `fail_on_errors` | fail when `errors.txt` is not empty (default `false`) |
| `checks` | list of checks, below | | `checks` | list of checks, below |
@@ -190,7 +203,8 @@ on `tor1k`, the track in front of the signal.
The `test_mode_*` cases show the ends of the test mode: `test_mode_pass` (exit 0, the pass The `test_mode_*` cases show the ends of the test mode: `test_mode_pass` (exit 0, the pass
event), `test_mode_signal_passed_at_danger` (exit 11), `test_mode_timeout` (exit 12, from event), `test_mode_signal_passed_at_danger` (exit 11), `test_mode_timeout` (exit 12, from
`test.timeout`) and `test_mode_standstill` (exit 13, from `test.standstill`). `test.timeout`), `test_mode_standstill` (exit 13, from `test.standstill`) and
`test_mode_wall_timeout` (exit 12 from the real time watchdog, which ends it while it loads).
`test_mode_pass_while_moving` reports the pass from `event2` of `tor1k` (the direction the train runs), as the train runs onto `test_mode_pass_while_moving` reports the pass from `event2` of `tor1k` (the direction the train runs), as the train runs onto
it, with `test.no_movement_before_pass no`, so the test ends with the train still moving. it, with `test.no_movement_before_pass no`, so the test ends with the train still moving.

View File

@@ -0,0 +1,11 @@
{
"description": "Test mode, real time limit: EU07 on td.scn, with a limit of 2 s of real time (the case's timeout, which the runner passes as --test-wall-timeout). The scenery takes longer to load, so the simulator's watchdog ends the run while it loads, with the timeout code and a verdict saying it was the real time.",
"base_scenery": "td.scn",
"time": "10:34",
"fragment": "td_red_signal.scn",
"vars": {
"consist": "node -1 0 {{vehicle}} dynamic pkp\\4e_v1 4e-004 4e 0 headdriver 0 0 enddynamic"
},
"expect": "wall_timeout",
"timeout": 2
}

View File

@@ -61,6 +61,10 @@ EXIT_RESULTS = {
13: "standstill", 13: "standstill",
14: "error", 14: "error",
} }
# the timeout of the simulator's real time watchdog (test.wall_timeout) has the timeout code too; told
# apart by its verdict, so a run that hung doesn't pass a case that expects the simulation time to run out
WALL_TIMEOUT = "wall_timeout"
RESULTS = (*EXIT_RESULTS.values(), WALL_TIMEOUT)
# settings layered over the game's eu07.ini for a test run (test mode doesn't read the player's own # settings layered over the game's eu07.ini for a test run (test mode doesn't read the player's own
# in %APPDATA%); the window ones only matter # in %APPDATA%); the window ones only matter
@@ -226,6 +230,8 @@ FATAL_LOG_LINES = ("Bad vehicle:", "Bad trainset:")
# time" in the log is not the end of loading (the trainsets after FirstInit still follow), # time" in the log is not the end of loading (the trainsets after FirstInit still follow),
# and with several simulators loading at once the first rows took well over 15 s # and with several simulators loading at once the first rows took well over 15 s
STARTUP_TIMEOUT = 120.0 STARTUP_TIMEOUT = 120.0
# wall seconds past a case's timeout before the runner kills a simulator that didn't end itself
KILL_GRACE = 30.0
@dataclass @dataclass
@@ -246,7 +252,8 @@ def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog
if watch: if watch:
cmd = [str(exe), "-s", scenery, "-v", "ghostview", "-ini", str(ini)] cmd = [str(exe), "-s", scenery, "-v", "ghostview", "-ini", str(ini)]
else: else:
cmd = [str(exe), "--test", "-s", scenery, "-ini", str(ini)] # the simulator's own watchdog ends a run that takes too long, with the timeout code and its verdict
cmd = [str(exe), "--test", "-s", scenery, "-ini", str(ini), "--test-wall-timeout", f"{timeout:g}"]
if duration is not None: if duration is not None:
cmd += ["--test-timeout", f"{duration:g}"] cmd += ["--test-timeout", f"{duration:g}"]
say(" $ " + " ".join(f'"{c}"' if " " in c else c for c in cmd), progress=True) say(" $ " + " ".join(f'"{c}"' if " " in c else c for c in cmd), progress=True)
@@ -272,8 +279,9 @@ def run_simulator(exe: Path, game_dir: Path, scenery: str, ini: Path, physicslog
break break
if watch and sim_t >= duration: if watch and sim_t >= duration:
break break
if wall > timeout: # in test mode a little later than the simulator's own limit, for a run that can't even end itself
result.failure = f"timed out after {timeout:.0f}s before {duration}s of simulation" if wall > timeout + (0 if watch else KILL_GRACE):
result.failure = f"timed out after {wall:.0f}s before {duration}s of simulation"
break break
if not result.rows: if not result.rows:
result.logs = find_logs(game_dir, scenery, since) result.logs = find_logs(game_dir, scenery, since)
@@ -506,8 +514,8 @@ def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, watch: bool
# a case that only records `duration` seconds ends on the test timeout; one that says # a case that only records `duration` seconds ends on the test timeout; one that says
# how the test ends (a pass event, a signal passed at danger...) expects that result # how the test ends (a pass event, a signal passed at danger...) expects that result
expect = case.get("expect", "timeout" if duration is not None else "pass") expect = case.get("expect", "timeout" if duration is not None else "pass")
if expect not in EXIT_RESULTS.values(): if expect not in RESULTS:
sys.exit(f"{name}: expect must be one of {', '.join(EXIT_RESULTS.values())}") sys.exit(f"{name}: expect must be one of {', '.join(RESULTS)}")
if watch and duration is None: if watch and duration is None:
sys.exit(f"{name}: --watch needs a case with duration") sys.exit(f"{name}: --watch needs a case with duration")
result = run_simulator(exe, game_dir, scenery, ini, physicslog, result = run_simulator(exe, game_dir, scenery, ini, physicslog,
@@ -530,8 +538,11 @@ def run_case(name: str, exe: Path, game_dir: Path, timespeed: float, watch: bool
if result.verdict: if result.verdict:
say(f" {result.verdict}") say(f" {result.verdict}")
got = EXIT_RESULTS.get(result.exit_code) if result.exit_code is not None else None got = EXIT_RESULTS.get(result.exit_code) if result.exit_code is not None else None
if got == "timeout" and "of real time" in result.verdict:
got = WALL_TIMEOUT
passed = got == expect passed = got == expect
say(f" {'PASS' if passed else 'FAIL'} exit code {describe_exit(result.exit_code)}, expected {expect}") watchdog = ", from the real time watchdog" if got == WALL_TIMEOUT else ""
say(f" {'PASS' if passed else 'FAIL'} exit code {describe_exit(result.exit_code)}{watchdog}, expected {expect}")
ok &= passed ok &= passed
if result.failure: if result.failure:
say(f" FAIL {result.failure}") say(f" FAIL {result.failure}")

View File

@@ -80,17 +80,19 @@ TEST_CASE("test mode: the command line")
CHECK_FALSE(commandline.enabled); CHECK_FALSE(commandline.enabled);
CHECK(commandline.error.empty()); CHECK(commandline.error.empty());
} }
SUBCASE("--test and its limit, in any order") SUBCASE("--test and its limits, in any order")
{ {
auto const commandline{parse({"-s", "td.scn", "--test-timeout", "240", "--test"})}; auto const commandline{parse({"--test-wall-timeout", "900", "-s", "td.scn", "--test", "--test-timeout", "240"})};
CHECK(commandline.enabled); CHECK(commandline.enabled);
CHECK(commandline.timeout == 240.0); CHECK(commandline.timeout == 240.0);
CHECK(commandline.wall_timeout == 900.0);
CHECK(commandline.error.empty()); CHECK(commandline.error.empty());
} }
SUBCASE("a limit not given stays unset") SUBCASE("a limit not given stays unset")
{ {
auto const commandline{parse({"--test", "-s", "td.scn"})}; auto const commandline{parse({"--test", "-s", "td.scn"})};
CHECK_FALSE(commandline.timeout.has_value()); CHECK_FALSE(commandline.timeout.has_value());
CHECK_FALSE(commandline.wall_timeout.has_value());
} }
SUBCASE("a limit which isn't a number is an error, not a limit turned off") SUBCASE("a limit which isn't a number is an error, not a limit turned off")
{ {
@@ -101,12 +103,17 @@ TEST_CASE("test mode: the command line")
} }
SUBCASE("a limit without its value is an error") SUBCASE("a limit without its value is an error")
{ {
auto const commandline{parse({"--test", "-s", "td.scn", "--test-timeout"})}; auto const commandline{parse({"--test", "-s", "td.scn", "--test-wall-timeout"})};
CHECK(commandline.error == "--test-wall-timeout needs a number of seconds");
}
SUBCASE("the first error is the one told")
{
auto const commandline{parse({"--test", "--test-timeout", "x", "--test-wall-timeout", "y"})};
CHECK(commandline.error == "--test-timeout needs a number of seconds"); CHECK(commandline.error == "--test-timeout needs a number of seconds");
} }
SUBCASE("values of other options aren't taken for options") SUBCASE("values of other options aren't taken for options")
{ {
auto const commandline{parse({"-s", "--test", "-ini", "--test-timeout", "-v", "--test", "-seed", "--test"})}; auto const commandline{parse({"-s", "--test", "-ini", "--test-timeout", "-v", "--test-wall-timeout", "-seed", "--test"})};
CHECK_FALSE(commandline.enabled); CHECK_FALSE(commandline.enabled);
CHECK(commandline.error.empty()); CHECK(commandline.error.empty());
} }
@@ -117,6 +124,25 @@ TEST_CASE("test mode: the command line")
} }
} }
TEST_CASE("test mode: the real time limit")
{
SUBCASE("given, it is the limit, 0 turns it off")
{
CHECK(simulation::wall_clock_limit(120.0, 3600.0) == 120.0);
CHECK(simulation::wall_clock_limit(0.0, 3600.0) == 0.0);
}
SUBCASE("not given, it is the simulation time limit and time for the loading")
{
CHECK(simulation::wall_clock_limit(std::nullopt, 240.0) == 240.0 + simulation::wall_timeout_loading_allowance);
CHECK(simulation::wall_clock_limit(std::nullopt, 3600.0) == 3600.0 + simulation::wall_timeout_loading_allowance);
}
SUBCASE("no simulation time limit, no real time limit unless given")
{
CHECK(simulation::wall_clock_limit(std::nullopt, 0.0) == 0.0);
CHECK(simulation::wall_clock_limit(300.0, 0.0) == 300.0);
}
}
TEST_CASE("test mode: the end conditions") TEST_CASE("test mode: the end conditions")
{ {
simulation::test_limits limits; simulation::test_limits limits;

View File

@@ -468,6 +468,15 @@ bool global_settings::ConfigParseTest(cParser& Parser, std::string_view const To
return true; return true;
} }
if (Token == "test.wall_timeout")
{
double value{ 0.0 };
ParseOne(Parser, value);
if (false == test.wall_timeout_from_command_line)
test.wall_timeout = std::max(0.0, value);
return true;
}
if (Token == "test.standstill") if (Token == "test.standstill")
{ {
ParseOne(Parser, test.standstill); ParseOne(Parser, test.standstill);
@@ -1786,6 +1795,9 @@ global_settings::export_as_text( std::ostream &Output ) const {
// TODO: export fpslimit // TODO: export fpslimit
export_as_text( Output, "randomseed", Global.random_seed ); export_as_text( Output, "randomseed", Global.random_seed );
export_as_text( Output, "test.timeout", test.timeout ); export_as_text( Output, "test.timeout", test.timeout );
if( test.wall_timeout.has_value() ) {
export_as_text( Output, "test.wall_timeout", *test.wall_timeout );
}
export_as_text( Output, "test.standstill", test.standstill ); export_as_text( Output, "test.standstill", test.standstill );
export_as_text( Output, "test.pass_on_standstill", test.pass_on_standstill ); export_as_text( Output, "test.pass_on_standstill", test.pass_on_standstill );
export_as_text( Output, "test.fail_if_stop_overran", test.fail_if_stop_overran ); export_as_text( Output, "test.fail_if_stop_overran", test.fail_if_stop_overran );

View File

@@ -16,6 +16,7 @@ http://mozilla.org/MPL/2.0/.
#include "utilities/utilities.h" #include "utilities/utilities.h"
#include "utilities/motiontelemetry.h" #include "utilities/motiontelemetry.h"
#include <map> #include <map>
#include <optional>
#ifdef WITH_UART #ifdef WITH_UART
#include "utilities/uart.h" #include "utilities/uart.h"
@@ -95,6 +96,8 @@ struct global_settings {
bool fail_if_stop_overran{ true }; // a train passing a signal at stop ends the test as failed; with false it's only logged bool fail_if_stop_overran{ true }; // a train passing a signal at stop ends the test as failed; with false it's only logged
bool no_movement_before_pass{ true }; // a pass waits until every vehicle stands still; with false it ends the test at once bool no_movement_before_pass{ true }; // a pass waits until every vehicle stands still; with false it ends the test at once
bool timeout_from_command_line{ false }; // --test-timeout given, the scenery doesn't override it bool timeout_from_command_line{ false }; // --test-timeout given, the scenery doesn't override it
std::optional<double> wall_timeout; // [s] of real time since the start; none = test.timeout + time for the loading, 0 = no limit
bool wall_timeout_from_command_line{ false }; // --test-wall-timeout given, the scenery doesn't override it
}; };
test_settings test; test_settings test;
bool editor_tracks{ false }; // F3 track workspace: draw and pick tracks without the scenery draw-range cap bool editor_tracks{ false }; // F3 track workspace: draw and pick tracks without the scenery draw-range cap