mirror of
https://github.com/MaSzyna-EU07/maszyna.git
synced 2026-11-19 07:36:02 +01:00
feat(simulation): a real time watchdog ends a test run that hangs
A test run that hung, in the loading or anywhere else, waited until whoever started it killed it. A watchdog thread now ends a run still going test.wall_timeout seconds of real time after the start of the simulator, loading included: it says `test: timeout ... still running after N s of real time` and quits with the timeout code, whatever the main thread is doing, with no dialog. A test that has ended but whose simulator doesn't close ends with its own result. The limit is test.wall_timeout (settings, the scenery's config block) or --test-wall-timeout, which wins. Not given, it is test.timeout + 600 s: ten minutes for the loading, and a simulation that runs at least as fast as real time is never cut short. test.timeout 0 means no watchdog unless the limit is given; test.wall_timeout 0 turns it off. run.py passes the case's timeout as --test-wall-timeout and kills a simulator only 30 s after it. A timeout from the watchdog counts as wall_timeout there, so a run that hung doesn't pass a case expecting the simulation time to run out; test_mode_wall_timeout ends while loading on a limit of 2 s.
This commit is contained in:
@@ -27,9 +27,20 @@ http://mozilla.org/MPL/2.0/.
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <format>
|
||||
#include <memory>
|
||||
|
||||
namespace {
|
||||
|
||||
// [s] of real time since the start of the simulator
|
||||
double elapsed_wall_time()
|
||||
{
|
||||
return std::chrono::duration<double>(std::chrono::steady_clock::now() - Global.startTimestamp).count();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace simulation {
|
||||
|
||||
test_manager &test()
|
||||
@@ -77,6 +88,8 @@ void apply_test_mode_settings()
|
||||
// and the same way every time, unless given another seed (-seed, or randomseed in the settings)
|
||||
if (!Global.random_seed)
|
||||
Global.random_seed = seed_of("test");
|
||||
// the real time limit, which the scenery may set too
|
||||
test().watch_wall_clock(wall_clock_limit(Global.test.wall_timeout, Global.test.timeout));
|
||||
}
|
||||
|
||||
void test_manager::init_result_cell() const
|
||||
@@ -94,7 +107,10 @@ void test_manager::start()
|
||||
if (m_started)
|
||||
return;
|
||||
m_started = true;
|
||||
m_progress = test_progress {};
|
||||
{
|
||||
std::scoped_lock lock { m_lock };
|
||||
m_progress = test_progress {};
|
||||
}
|
||||
if (simulation::Vehicles.sequence().empty())
|
||||
{
|
||||
// most likely the scenery wasn't found, or it isn't a test scenery
|
||||
@@ -103,7 +119,9 @@ void test_manager::start()
|
||||
}
|
||||
// a pause set by the scenery or the settings would never be lifted, nobody is there to press the key
|
||||
Global.iPause = 0;
|
||||
WriteLog(std::format("test: started, timeout {:.0f} s, standstill {:.0f} s{}{}{}", Global.test.timeout, Global.test.standstill,
|
||||
auto const walllimit { m_walllimit.load() };
|
||||
WriteLog(std::format("test: started, timeout {:.0f} s, real time limit {}, standstill {:.0f} s{}{}{}", Global.test.timeout,
|
||||
walllimit > 0.0 ? std::format("{:.0f} s", walllimit) : std::string("none"), Global.test.standstill,
|
||||
Global.test.pass_on_standstill ? ", standstill passes" : "",
|
||||
Global.test.fail_if_stop_overran ? "" : ", signals passed at danger allowed",
|
||||
Global.test.no_movement_before_pass ? "" : ", pass without waiting for a stop"));
|
||||
@@ -122,14 +140,21 @@ void test_manager::update(double const Deltatime)
|
||||
frame.reason = cell->Text();
|
||||
}
|
||||
|
||||
auto const requestedbefore { m_progress.passrequested };
|
||||
auto const verdict { evaluate_frame(m_progress, frame, test_limits { Global.test.timeout, Global.test.standstill, Global.test.pass_on_standstill, Global.test.no_movement_before_pass }) };
|
||||
auto const limits { test_limits { Global.test.timeout, Global.test.standstill, Global.test.pass_on_standstill, Global.test.no_movement_before_pass } };
|
||||
std::optional<test_verdict> verdict;
|
||||
auto passrequested { false };
|
||||
{
|
||||
std::scoped_lock lock { m_lock };
|
||||
auto const requestedbefore { m_progress.passrequested };
|
||||
verdict = evaluate_frame(m_progress, frame, limits);
|
||||
passrequested = m_progress.passrequested && false == requestedbefore;
|
||||
}
|
||||
if (verdict.has_value())
|
||||
{
|
||||
finish(verdict->result, verdict->reason);
|
||||
return;
|
||||
}
|
||||
if (m_progress.passrequested && false == requestedbefore)
|
||||
if (passrequested)
|
||||
{
|
||||
WriteLog(std::format("test: pass requested at {:.1f} s{}, waiting for every vehicle to stop", m_progress.time,
|
||||
frame.reason.empty() ? std::string() : std::format(" ({})", frame.reason)));
|
||||
@@ -160,10 +185,11 @@ void test_manager::finish(test_result const Result, std::string_view const Reaso
|
||||
|
||||
bool test_manager::conclude(test_result const Result, std::string_view const Reason)
|
||||
{
|
||||
std::scoped_lock lock { m_lock };
|
||||
if (m_finished || false == Global.test.enabled)
|
||||
return false;
|
||||
m_finished = true;
|
||||
m_result = Result;
|
||||
m_finished = true;
|
||||
|
||||
auto const line { std::format("test: {} (exit code {}) after {:.1f} s of simulation time: {}", to_string(Result), static_cast<int>(Result), m_progress.time, Reason) };
|
||||
if (Result == test_result::pass)
|
||||
@@ -176,4 +202,39 @@ bool test_manager::conclude(test_result const Result, std::string_view const Rea
|
||||
return true;
|
||||
}
|
||||
|
||||
int test_manager::exit_code() const
|
||||
{
|
||||
std::scoped_lock lock { m_lock };
|
||||
return static_cast<int>(m_finished ? m_result : test_result::error);
|
||||
}
|
||||
|
||||
void test_manager::watch_wall_clock(double const Limit)
|
||||
{
|
||||
m_walllimit = Limit;
|
||||
if (Limit <= 0.0 || m_watchdog.joinable())
|
||||
return;
|
||||
m_watchdog = std::jthread([this](std::stop_token const &Stop) {
|
||||
while (false == Stop.stop_requested())
|
||||
{
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
check_wall_clock(elapsed_wall_time());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void test_manager::check_wall_clock(double const Elapsed)
|
||||
{
|
||||
auto const limit { m_walllimit.load() };
|
||||
if (limit <= 0.0 || Elapsed < limit)
|
||||
return;
|
||||
// the run hangs, or is too slow: it ends here, whatever the main thread is doing. a run whose test has ended
|
||||
// but which doesn't close ends with its result
|
||||
if (false == conclude(test_result::timeout, std::format("still running after {:.0f} s of real time (test.wall_timeout)", limit)))
|
||||
ErrorLog(std::format("test: the simulator was still running {:.0f} s of real time after the start, ended", limit));
|
||||
// what was logged last gets written, unless the log is held up by a hung thread
|
||||
StopLogService();
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(250));
|
||||
std::_Exit(exit_code());
|
||||
}
|
||||
|
||||
} // namespace simulation
|
||||
|
||||
@@ -11,7 +11,10 @@ http://mozilla.org/MPL/2.0/.
|
||||
|
||||
#include "simulation/simulationtestrules.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
#include <string_view>
|
||||
#include <thread>
|
||||
|
||||
// integration test mode (eu07 --test -s <scenery>, eu07/maszyna#541): the scenery runs unattended in a tight
|
||||
// fixed-step loop and the simulator quits with an exit code saying how the test went
|
||||
@@ -38,16 +41,26 @@ public:
|
||||
// ends the test with given result, without closing the application: for the ways out that close it anyway
|
||||
// (a bad command line, a failed start, an exception). returns false if the test had already ended
|
||||
bool conclude(test_result Result, std::string_view Reason);
|
||||
// (re)sets the real time limit of the run [s] since the start of the simulator, 0 = none. the first limit
|
||||
// starts the watchdog, which ends a run still going at the limit with the timeout code, whatever it's doing
|
||||
void watch_wall_clock(double Limit);
|
||||
|
||||
bool finished() const { return m_finished; }
|
||||
// exit code of the process: the result, or error while the test hasn't finished
|
||||
int exit_code() const { return static_cast<int>(m_finished ? m_result : test_result::error); }
|
||||
int exit_code() const;
|
||||
|
||||
private:
|
||||
// the watchdog's check, at Elapsed seconds of real time since the start of the simulator
|
||||
void check_wall_clock(double Elapsed);
|
||||
|
||||
// the watchdog may end the test from its own thread
|
||||
mutable std::mutex m_lock;
|
||||
bool m_started { false };
|
||||
bool m_finished { false };
|
||||
std::atomic<bool> m_finished { false };
|
||||
test_result m_result { test_result::error };
|
||||
test_progress m_progress;
|
||||
std::atomic<double> m_walllimit { 0.0 }; // [s] since the start of the simulator, 0 = none
|
||||
std::jthread m_watchdog;
|
||||
};
|
||||
|
||||
// the test of the run
|
||||
|
||||
@@ -81,7 +81,7 @@ test_command_line parse_test_command_line(std::span<char const *const> const Arg
|
||||
{
|
||||
result.enabled = true;
|
||||
}
|
||||
else if (option == "--test-timeout")
|
||||
else if (option == "--test-timeout" || option == "--test-wall-timeout")
|
||||
{
|
||||
auto const seconds{option_seconds(Arguments, i)};
|
||||
if (false == seconds.has_value())
|
||||
@@ -91,7 +91,10 @@ test_command_line parse_test_command_line(std::span<char const *const> const Arg
|
||||
result.error = std::format("{} needs a number of seconds", option);
|
||||
continue;
|
||||
}
|
||||
result.timeout = seconds;
|
||||
if (option == "--test-timeout")
|
||||
result.timeout = seconds;
|
||||
else
|
||||
result.wall_timeout = seconds;
|
||||
}
|
||||
else if (takes_value(option))
|
||||
{
|
||||
@@ -106,6 +109,13 @@ test_command_line parse_test_command_line(std::span<char const *const> const Arg
|
||||
return result;
|
||||
}
|
||||
|
||||
double wall_clock_limit(std::optional<double> const Given, double const Simulationtimeout)
|
||||
{
|
||||
if (Given.has_value())
|
||||
return *Given;
|
||||
return Simulationtimeout > 0.0 ? Simulationtimeout + wall_timeout_loading_allowance : 0.0;
|
||||
}
|
||||
|
||||
std::optional<test_verdict> evaluate_frame(test_progress &Progress, test_frame const &Frame, test_limits const &Limits)
|
||||
{
|
||||
using enum test_result;
|
||||
|
||||
@@ -15,7 +15,7 @@ http://mozilla.org/MPL/2.0/.
|
||||
#include <string_view>
|
||||
|
||||
// the rules of the test mode (eu07 --test, eu07/maszyna#541) which need nothing of the running simulation: the exit
|
||||
// codes, the command line and the end conditions. kept apart so the unit tests can check them
|
||||
// codes, the command line, the end conditions and the real time limit. kept apart so the unit tests can check them
|
||||
namespace simulation {
|
||||
|
||||
// exit codes of a test run; public, scripts and CI read them. none of them is a code a process commonly ends with
|
||||
@@ -25,7 +25,7 @@ enum class test_result : int {
|
||||
pass = 0, // __test.result set to 1, then every vehicle came to rest (or test.pass_on_standstill)
|
||||
fail = 10, // __test.result set to 0
|
||||
signal_passed_at_danger = 11, // a train went past a stop signal (SetVelocity 0 0), unless test.fail_if_stop_overran no
|
||||
timeout = 12, // test.timeout seconds of simulation time went by without a result
|
||||
timeout = 12, // test.timeout seconds of simulation time, or test.wall_timeout seconds of real time went by
|
||||
standstill = 13, // every vehicle stood still for test.standstill seconds without a result
|
||||
error = 14, // the run could not start (a bad command line, the scenery failed to load) or the simulator threw
|
||||
};
|
||||
@@ -42,6 +42,7 @@ struct test_verdict {
|
||||
struct test_command_line {
|
||||
bool enabled{false}; // --test
|
||||
std::optional<double> timeout; // --test-timeout <seconds of simulation time>
|
||||
std::optional<double> wall_timeout; // --test-wall-timeout <seconds of real time>
|
||||
std::string error; // what's wrong with them; empty if nothing
|
||||
};
|
||||
|
||||
@@ -52,6 +53,14 @@ std::optional<double> parse_seconds(std::string_view Text);
|
||||
// options are skipped, so a scenery named like an option isn't taken for one; options it doesn't know are left alone
|
||||
test_command_line parse_test_command_line(std::span<char const *const> Arguments);
|
||||
|
||||
// time allowed for loading the scenery, on top of the simulation time, when the real time limit isn't given [s]
|
||||
inline constexpr double wall_timeout_loading_allowance{600.0};
|
||||
|
||||
// the real time limit of a test run [s], 0 = none: the one given, else the simulation time limit plus time for the
|
||||
// loading, so a simulation that runs at least as fast as real time is never cut short. no simulation time limit
|
||||
// (test.timeout 0) means none in real time either, unless given
|
||||
double wall_clock_limit(std::optional<double> Given, double Simulationtimeout);
|
||||
|
||||
// the limits and switches the end conditions go by
|
||||
struct test_limits {
|
||||
double timeout{3600.0}; // [s] of simulation time; 0 = never
|
||||
|
||||
Reference in New Issue
Block a user