18
0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-11-19 07:36:02 +01:00

feat(simulation): a real time watchdog ends a test run that hangs

A test run that hung, in the loading or anywhere else, waited until whoever
started it killed it. A watchdog thread now ends a run still going test.wall_timeout
seconds of real time after the start of the simulator, loading included: it says
`test: timeout ... still running after N s of real time` and quits with the
timeout code, whatever the main thread is doing, with no dialog. A test that has
ended but whose simulator doesn't close ends with its own result.

The limit is test.wall_timeout (settings, the scenery's config block) or
--test-wall-timeout, which wins. Not given, it is test.timeout + 600 s: ten
minutes for the loading, and a simulation that runs at least as fast as real
time is never cut short. test.timeout 0 means no watchdog unless the limit is
given; test.wall_timeout 0 turns it off.

run.py passes the case's timeout as --test-wall-timeout and kills a simulator
only 30 s after it. A timeout from the watchdog counts as wall_timeout there, so
a run that hung doesn't pass a case expecting the simulation time to run out;
test_mode_wall_timeout ends while loading on a limit of 2 s.
This commit is contained in:
jerrrrycho
2026-11-08 11:33:36 +01:00
parent 14d25520f3
commit 03a417c7b0
11 changed files with 210 additions and 35 deletions

View File

@@ -27,9 +27,20 @@ http://mozilla.org/MPL/2.0/.
#include <chrono>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <format>
#include <memory>
namespace {
// [s] of real time since the start of the simulator
double elapsed_wall_time()
{
return std::chrono::duration<double>(std::chrono::steady_clock::now() - Global.startTimestamp).count();
}
} // namespace
namespace simulation {
test_manager &test()
@@ -77,6 +88,8 @@ void apply_test_mode_settings()
// and the same way every time, unless given another seed (-seed, or randomseed in the settings)
if (!Global.random_seed)
Global.random_seed = seed_of("test");
// the real time limit, which the scenery may set too
test().watch_wall_clock(wall_clock_limit(Global.test.wall_timeout, Global.test.timeout));
}
void test_manager::init_result_cell() const
@@ -94,7 +107,10 @@ void test_manager::start()
if (m_started)
return;
m_started = true;
m_progress = test_progress {};
{
std::scoped_lock lock { m_lock };
m_progress = test_progress {};
}
if (simulation::Vehicles.sequence().empty())
{
// most likely the scenery wasn't found, or it isn't a test scenery
@@ -103,7 +119,9 @@ void test_manager::start()
}
// a pause set by the scenery or the settings would never be lifted, nobody is there to press the key
Global.iPause = 0;
WriteLog(std::format("test: started, timeout {:.0f} s, standstill {:.0f} s{}{}{}", Global.test.timeout, Global.test.standstill,
auto const walllimit { m_walllimit.load() };
WriteLog(std::format("test: started, timeout {:.0f} s, real time limit {}, standstill {:.0f} s{}{}{}", Global.test.timeout,
walllimit > 0.0 ? std::format("{:.0f} s", walllimit) : std::string("none"), Global.test.standstill,
Global.test.pass_on_standstill ? ", standstill passes" : "",
Global.test.fail_if_stop_overran ? "" : ", signals passed at danger allowed",
Global.test.no_movement_before_pass ? "" : ", pass without waiting for a stop"));
@@ -122,14 +140,21 @@ void test_manager::update(double const Deltatime)
frame.reason = cell->Text();
}
auto const requestedbefore { m_progress.passrequested };
auto const verdict { evaluate_frame(m_progress, frame, test_limits { Global.test.timeout, Global.test.standstill, Global.test.pass_on_standstill, Global.test.no_movement_before_pass }) };
auto const limits { test_limits { Global.test.timeout, Global.test.standstill, Global.test.pass_on_standstill, Global.test.no_movement_before_pass } };
std::optional<test_verdict> verdict;
auto passrequested { false };
{
std::scoped_lock lock { m_lock };
auto const requestedbefore { m_progress.passrequested };
verdict = evaluate_frame(m_progress, frame, limits);
passrequested = m_progress.passrequested && false == requestedbefore;
}
if (verdict.has_value())
{
finish(verdict->result, verdict->reason);
return;
}
if (m_progress.passrequested && false == requestedbefore)
if (passrequested)
{
WriteLog(std::format("test: pass requested at {:.1f} s{}, waiting for every vehicle to stop", m_progress.time,
frame.reason.empty() ? std::string() : std::format(" ({})", frame.reason)));
@@ -160,10 +185,11 @@ void test_manager::finish(test_result const Result, std::string_view const Reaso
bool test_manager::conclude(test_result const Result, std::string_view const Reason)
{
std::scoped_lock lock { m_lock };
if (m_finished || false == Global.test.enabled)
return false;
m_finished = true;
m_result = Result;
m_finished = true;
auto const line { std::format("test: {} (exit code {}) after {:.1f} s of simulation time: {}", to_string(Result), static_cast<int>(Result), m_progress.time, Reason) };
if (Result == test_result::pass)
@@ -176,4 +202,39 @@ bool test_manager::conclude(test_result const Result, std::string_view const Rea
return true;
}
int test_manager::exit_code() const
{
std::scoped_lock lock { m_lock };
return static_cast<int>(m_finished ? m_result : test_result::error);
}
void test_manager::watch_wall_clock(double const Limit)
{
m_walllimit = Limit;
if (Limit <= 0.0 || m_watchdog.joinable())
return;
m_watchdog = std::jthread([this](std::stop_token const &Stop) {
while (false == Stop.stop_requested())
{
std::this_thread::sleep_for(std::chrono::milliseconds(100));
check_wall_clock(elapsed_wall_time());
}
});
}
void test_manager::check_wall_clock(double const Elapsed)
{
auto const limit { m_walllimit.load() };
if (limit <= 0.0 || Elapsed < limit)
return;
// the run hangs, or is too slow: it ends here, whatever the main thread is doing. a run whose test has ended
// but which doesn't close ends with its result
if (false == conclude(test_result::timeout, std::format("still running after {:.0f} s of real time (test.wall_timeout)", limit)))
ErrorLog(std::format("test: the simulator was still running {:.0f} s of real time after the start, ended", limit));
// what was logged last gets written, unless the log is held up by a hung thread
StopLogService();
std::this_thread::sleep_for(std::chrono::milliseconds(250));
std::_Exit(exit_code());
}
} // namespace simulation

View File

@@ -11,7 +11,10 @@ http://mozilla.org/MPL/2.0/.
#include "simulation/simulationtestrules.h"
#include <atomic>
#include <mutex>
#include <string_view>
#include <thread>
// integration test mode (eu07 --test -s <scenery>, eu07/maszyna#541): the scenery runs unattended in a tight
// fixed-step loop and the simulator quits with an exit code saying how the test went
@@ -38,16 +41,26 @@ public:
// ends the test with given result, without closing the application: for the ways out that close it anyway
// (a bad command line, a failed start, an exception). returns false if the test had already ended
bool conclude(test_result Result, std::string_view Reason);
// (re)sets the real time limit of the run [s] since the start of the simulator, 0 = none. the first limit
// starts the watchdog, which ends a run still going at the limit with the timeout code, whatever it's doing
void watch_wall_clock(double Limit);
bool finished() const { return m_finished; }
// exit code of the process: the result, or error while the test hasn't finished
int exit_code() const { return static_cast<int>(m_finished ? m_result : test_result::error); }
int exit_code() const;
private:
// the watchdog's check, at Elapsed seconds of real time since the start of the simulator
void check_wall_clock(double Elapsed);
// the watchdog may end the test from its own thread
mutable std::mutex m_lock;
bool m_started { false };
bool m_finished { false };
std::atomic<bool> m_finished { false };
test_result m_result { test_result::error };
test_progress m_progress;
std::atomic<double> m_walllimit { 0.0 }; // [s] since the start of the simulator, 0 = none
std::jthread m_watchdog;
};
// the test of the run

View File

@@ -81,7 +81,7 @@ test_command_line parse_test_command_line(std::span<char const *const> const Arg
{
result.enabled = true;
}
else if (option == "--test-timeout")
else if (option == "--test-timeout" || option == "--test-wall-timeout")
{
auto const seconds{option_seconds(Arguments, i)};
if (false == seconds.has_value())
@@ -91,7 +91,10 @@ test_command_line parse_test_command_line(std::span<char const *const> const Arg
result.error = std::format("{} needs a number of seconds", option);
continue;
}
result.timeout = seconds;
if (option == "--test-timeout")
result.timeout = seconds;
else
result.wall_timeout = seconds;
}
else if (takes_value(option))
{
@@ -106,6 +109,13 @@ test_command_line parse_test_command_line(std::span<char const *const> const Arg
return result;
}
double wall_clock_limit(std::optional<double> const Given, double const Simulationtimeout)
{
if (Given.has_value())
return *Given;
return Simulationtimeout > 0.0 ? Simulationtimeout + wall_timeout_loading_allowance : 0.0;
}
std::optional<test_verdict> evaluate_frame(test_progress &Progress, test_frame const &Frame, test_limits const &Limits)
{
using enum test_result;

View File

@@ -15,7 +15,7 @@ http://mozilla.org/MPL/2.0/.
#include <string_view>
// the rules of the test mode (eu07 --test, eu07/maszyna#541) which need nothing of the running simulation: the exit
// codes, the command line and the end conditions. kept apart so the unit tests can check them
// codes, the command line, the end conditions and the real time limit. kept apart so the unit tests can check them
namespace simulation {
// exit codes of a test run; public, scripts and CI read them. none of them is a code a process commonly ends with
@@ -25,7 +25,7 @@ enum class test_result : int {
pass = 0, // __test.result set to 1, then every vehicle came to rest (or test.pass_on_standstill)
fail = 10, // __test.result set to 0
signal_passed_at_danger = 11, // a train went past a stop signal (SetVelocity 0 0), unless test.fail_if_stop_overran no
timeout = 12, // test.timeout seconds of simulation time went by without a result
timeout = 12, // test.timeout seconds of simulation time, or test.wall_timeout seconds of real time went by
standstill = 13, // every vehicle stood still for test.standstill seconds without a result
error = 14, // the run could not start (a bad command line, the scenery failed to load) or the simulator threw
};
@@ -42,6 +42,7 @@ struct test_verdict {
struct test_command_line {
bool enabled{false}; // --test
std::optional<double> timeout; // --test-timeout <seconds of simulation time>
std::optional<double> wall_timeout; // --test-wall-timeout <seconds of real time>
std::string error; // what's wrong with them; empty if nothing
};
@@ -52,6 +53,14 @@ std::optional<double> parse_seconds(std::string_view Text);
// options are skipped, so a scenery named like an option isn't taken for one; options it doesn't know are left alone
test_command_line parse_test_command_line(std::span<char const *const> Arguments);
// time allowed for loading the scenery, on top of the simulation time, when the real time limit isn't given [s]
inline constexpr double wall_timeout_loading_allowance{600.0};
// the real time limit of a test run [s], 0 = none: the one given, else the simulation time limit plus time for the
// loading, so a simulation that runs at least as fast as real time is never cut short. no simulation time limit
// (test.timeout 0) means none in real time either, unless given
double wall_clock_limit(std::optional<double> Given, double Simulationtimeout);
// the limits and switches the end conditions go by
struct test_limits {
double timeout{3600.0}; // [s] of simulation time; 0 = never