diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a4e1d6e..483dbc14 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -454,7 +454,7 @@ endif() add_library(eu07_settings INTERFACE) # the parts of the simulation the game shares with the unit tests: the vehicle physics, the vehicle -# definition parser and the helpers they use. built once, and tested as the game uses them +# definition parser, the helpers they use and the rules of the test mode. built once, and tested as the game uses them set(CORE_SOURCES "McZapkie/friction.cpp" "McZapkie/hamulce.cpp" @@ -463,6 +463,7 @@ set(CORE_SOURCES "utilities/parser.cpp" "utilities/utilities.cpp" "scripting/ladderlogic.cpp" +"simulation/simulationtestrules.cpp" "vehicle/handover.cpp" "vehicle/horn.cpp" "vehicle/smoke.cpp" diff --git a/application/application.cpp b/application/application.cpp index d4aed9d8..f3c3fe46 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -34,14 +34,9 @@ http://mozilla.org/MPL/2.0/. #ifdef _WIN32 #include "Console.h" #endif -#include -#include #include -#include #include -#include -#include -#include +#include #include "utilities/translation.h" #if WITH_DISCORD_RPC @@ -1199,16 +1194,6 @@ void apply_test_mode_settings() Global.random_seed = seed_of("test"); } -// seconds given on the command line: a number, not negative. returns nothing for anything else -std::optional parse_seconds(std::string_view const Text) -{ - auto const *const last{Text.data() + Text.size()}; - double value{0.0}; - if (auto const [parsed, status]{std::from_chars(Text.data(), last, value)}; status != std::errc{} || parsed != last || false == std::isfinite(value) || value < 0.0) - return std::nullopt; - return value; -} - } // namespace int eu07_application::init_settings(int Argc, char *Argv[]) @@ -1228,7 +1213,18 @@ int eu07_application::init_settings(int Argc, char *Argv[]) // a test run is known before the rest of the command line is read, so a command line it can't take // ends with the error code of the test, not with 0, which would pass it - Global.test.enabled = std::ranges::any_of(std::span(Argv, static_cast(Argc)), [](char const *Argument) { return std::string_view(Argument) == "--test"; }); + auto const testcommandline{simulation::parse_test_command_line(std::vector(Argv, Argv + Argc))}; + Global.test.enabled = testcommandline.enabled; + if (false == testcommandline.error.empty()) + { + simulation::test().conclude(simulation::test_result::error, testcommandline.error); + return -1; + } + if (testcommandline.timeout.has_value()) + { + Global.test.timeout = *testcommandline.timeout; + Global.test.timeout_from_command_line = true; + } // process command line arguments bool sceneryfromcommandline{false}; @@ -1278,20 +1274,13 @@ int eu07_application::init_settings(int Argc, char *Argv[]) } else if (token == "--test") { - // integration test: the scenery runs unattended and the exit code is the result (eu07/maszyna#541) - Global.test.enabled = true; + // integration test: the scenery runs unattended and the exit code is the result (eu07/maszyna#541), + // taken from the command line above } else if (token == "--test-timeout") { - auto const timeout{i + 1 < Argc ? parse_seconds(Argv[++i]) : std::nullopt}; - if (false == timeout.has_value()) - { - // a typo would otherwise turn the timeout off - simulation::test().conclude(simulation::test_result::error, "--test-timeout needs a number of seconds"); - return -1; - } - Global.test.timeout = *timeout; - Global.test.timeout_from_command_line = true; + // taken from the command line above, with its value + ++i; } else if (token == "-ini") { diff --git a/simulation/simulationtest.cpp b/simulation/simulationtest.cpp index 214d7e46..76aa1be3 100644 --- a/simulation/simulationtest.cpp +++ b/simulation/simulationtest.cpp @@ -25,16 +25,6 @@ http://mozilla.org/MPL/2.0/. #include #include -namespace { - -// the cell holds what the scenery wrote, so the values are whole numbers; the margin only keeps a stray fraction out -bool holds(double const Value, double const Expected) -{ - return std::abs(Value - Expected) < 1e-6; -} - -} // namespace - namespace simulation { test_manager &test() @@ -43,21 +33,6 @@ test_manager &test() return instance; } -std::string_view to_string(test_result const Result) -{ - using enum test_result; - switch (Result) - { - case pass: return "pass"; - case fail: return "fail"; - case signal_passed_at_danger: return "signal passed at danger"; - case timeout: return "timeout"; - case standstill: return "standstill"; - case error: return "error"; - } - return "unknown"; -} - void test_manager::init_result_cell() const { auto cell { std::make_unique(scene::node_data{0, -1, std::string(result_cell), "memcell"}) }; @@ -73,8 +48,7 @@ void test_manager::start() if (m_started) return; m_started = true; - m_time = 0.0; - m_resttime = 0.0; + m_progress = test_progress {}; if (simulation::Vehicles.sequence().empty()) { // most likely the scenery wasn't found, or it isn't a test scenery @@ -94,48 +68,25 @@ void test_manager::update(double const Deltatime) if (false == m_started || m_finished) return; - m_time += Deltatime; - + test_frame frame { Deltatime, std::nullopt, {}, + std::ranges::none_of(simulation::Vehicles.sequence(), [](TDynamicObject const *Vehicle) { return std::abs(Vehicle->GetVelocity()) > 0.01; }) }; if (auto const *cell = simulation::Memory.find(std::string(result_cell))) { - auto const value { cell->Value1() }; - auto const reason { cell->Text().empty() ? std::string() : std::format(" ({})", cell->Text()) }; - if (holds(value, 0.0)) - { - finish(test_result::fail, std::format("__test.result set to 0{}", reason)); - return; - } - if (holds(value, 1.0) && false == Global.test.no_movement_before_pass) - { - finish(test_result::pass, std::format("__test.result set to 1{}", reason)); - return; - } - if (holds(value, 1.0) && false == m_passrequested) - { - m_passrequested = true; - WriteLog(std::format("test: pass requested at {:.1f} s{}, waiting for every vehicle to stop", m_time, reason)); - } + frame.reported = cell->Value1(); + frame.reason = cell->Text(); } - auto const atrest { std::ranges::none_of(simulation::Vehicles.sequence(), - [](TDynamicObject const *Vehicle) { return std::abs(Vehicle->GetVelocity()) > 0.01; }) }; - m_resttime = atrest ? m_resttime + Deltatime : 0.0; - - if (m_passrequested && m_resttime >= pass_settle_time) + auto const requestedbefore { m_progress.passrequested }; + auto const verdict { evaluate_frame(m_progress, frame, test_limits { Global.test.timeout, Global.test.standstill, Global.test.pass_on_standstill, Global.test.no_movement_before_pass }) }; + if (verdict.has_value()) { - finish(test_result::pass, "__test.result set to 1 and every vehicle stopped"); + finish(verdict->result, verdict->reason); return; } - if (Global.test.standstill > 0.0 && m_resttime >= Global.test.standstill) + if (m_progress.passrequested && false == requestedbefore) { - auto const reason { std::format("every vehicle stood still for {:.0f} s", Global.test.standstill) }; - finish(Global.test.pass_on_standstill ? test_result::pass : test_result::standstill, reason); - return; - } - if (Global.test.timeout > 0.0 && m_time >= Global.test.timeout) - { - finish(test_result::timeout, std::format("no result after {:.0f} s", Global.test.timeout)); - return; + WriteLog(std::format("test: pass requested at {:.1f} s{}, waiting for every vehicle to stop", m_progress.time, + frame.reason.empty() ? std::string() : std::format(" ({})", frame.reason))); } } @@ -149,7 +100,7 @@ void test_manager::signal_passed_at_danger(std::string_view const Vehicle, std:: } if (false == Global.test.fail_if_stop_overran) { - WriteLog(std::format("test: {} at {:.1f} s, allowed by test.fail_if_stop_overran", reason, m_time)); + WriteLog(std::format("test: {} at {:.1f} s, allowed by test.fail_if_stop_overran", reason, m_progress.time)); return; } finish(test_result::signal_passed_at_danger, reason); @@ -168,7 +119,7 @@ bool test_manager::conclude(test_result const Result, std::string_view const Rea m_finished = true; m_result = Result; - auto const line { std::format("test: {} (exit code {}) after {:.1f} s of simulation time: {}", to_string(Result), static_cast(Result), m_time, Reason) }; + auto const line { std::format("test: {} (exit code {}) after {:.1f} s of simulation time: {}", to_string(Result), static_cast(Result), m_progress.time, Reason) }; if (Result == test_result::pass) WriteLog(line); else diff --git a/simulation/simulationtest.h b/simulation/simulationtest.h index 2b5012e9..66384b6e 100644 --- a/simulation/simulationtest.h +++ b/simulation/simulationtest.h @@ -9,32 +9,19 @@ http://mozilla.org/MPL/2.0/. #pragma once -#include +#include "simulation/simulationtestrules.h" + #include // integration test mode (eu07 --test -s , eu07/maszyna#541): the scenery runs unattended in a tight // fixed-step loop and the simulator quits with an exit code saying how the test went namespace simulation { -// exit codes of a test run; public, scripts and CI read them -enum class test_result : int { - pass = 0, // __test.result set to 1, then every vehicle came to rest (or test.pass_on_standstill) - fail = 1, // __test.result set to 0 - signal_passed_at_danger = 2, // a train went past a stop signal (SetVelocity 0 0), unless test.fail_if_stop_overran no - timeout = 3, // test.timeout seconds of simulation time went by without a result - standstill = 4, // every vehicle stood still for test.standstill seconds without a result - error = 5, // the run could not start (no -s, the scenery failed to load) or the simulator threw -}; - -std::string_view to_string(test_result Result); - class test_manager { public: // name of the memory cell a scenery reports its result to static constexpr std::string_view result_cell { "__test.result" }; - // how long everything has to stand before a requested pass is granted [s] - static constexpr double pass_settle_time { 1.0 }; // fixed simulation time step of one frame in test mode [s]: 3 physics steps of 10 ms static constexpr double frame_step { 0.03 }; @@ -60,9 +47,7 @@ private: bool m_started { false }; bool m_finished { false }; test_result m_result { test_result::error }; - bool m_passrequested { false }; - double m_time { 0.0 }; // simulation time since the start [s] - double m_resttime { 0.0 }; // how long every vehicle has been standing [s] + test_progress m_progress; }; // the test of the run diff --git a/simulation/simulationtestrules.cpp b/simulation/simulationtestrules.cpp new file mode 100644 index 00000000..44f740f1 --- /dev/null +++ b/simulation/simulationtestrules.cpp @@ -0,0 +1,139 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#include "stdafx.h" +#include "simulation/simulationtestrules.h" + +#include +#include +#include + +namespace { + +// the cell holds what the scenery wrote, so the values are whole numbers; the margin only keeps a stray fraction out +bool holds(double const Value, double const Expected) +{ + return std::abs(Value - Expected) < 1e-6; +} + +// the text of the result cell, as said after the result +std::string reason_suffix(std::string_view const Reason) +{ + return Reason.empty() ? std::string() : std::format(" ({})", Reason); +} + +// options of the simulator which take a value, see eu07_application::init_settings() +bool takes_value(std::string_view const Option) +{ + return Option == "-s" || Option == "-v" || Option == "-seed" || Option == "-ini"; +} + +// a number of seconds given to Option, at Index of Arguments; the index is moved past the value +std::optional option_seconds(std::span const Arguments, std::size_t &Index) +{ + if (Index + 1 >= Arguments.size()) + return std::nullopt; + ++Index; + return simulation::parse_seconds(Arguments[Index]); +} + +} // namespace + +namespace simulation { + +std::string_view to_string(test_result const Result) +{ + using enum test_result; + switch (Result) + { + case pass: return "pass"; + case fail: return "fail"; + case signal_passed_at_danger: return "signal passed at danger"; + case timeout: return "timeout"; + case standstill: return "standstill"; + case error: return "error"; + } + return "unknown"; +} + +std::optional parse_seconds(std::string_view const Text) +{ + auto const *const last{Text.data() + Text.size()}; + double value{0.0}; + if (auto const [parsed, status]{std::from_chars(Text.data(), last, value)}; status != std::errc{} || parsed != last || false == std::isfinite(value) || value < 0.0) + return std::nullopt; + return value; +} + +test_command_line parse_test_command_line(std::span const Arguments) +{ + test_command_line result; + for (std::size_t i = 1; i < Arguments.size(); ++i) + { + std::string_view const option{Arguments[i]}; + if (option == "--test") + { + result.enabled = true; + } + else if (option == "--test-timeout") + { + auto const seconds{option_seconds(Arguments, i)}; + if (false == seconds.has_value()) + { + // a typo would otherwise turn the limit off + if (result.error.empty()) + result.error = std::format("{} needs a number of seconds", option); + continue; + } + result.timeout = seconds; + } + else if (takes_value(option)) + { + ++i; + } + else if (option == "-edit" && i + 1 < Arguments.size() && Arguments[i + 1][0] != '-') + { + // the scenery to edit is optional + ++i; + } + } + return result; +} + +std::optional evaluate_frame(test_progress &Progress, test_frame const &Frame, test_limits const &Limits) +{ + using enum test_result; + + Progress.time += Frame.deltatime; + + if (Frame.reported.has_value()) + { + auto const value{*Frame.reported}; + if (holds(value, 0.0)) + return test_verdict{fail, std::format("__test.result set to 0{}", reason_suffix(Frame.reason))}; + if (holds(value, 1.0)) + { + if (false == Limits.no_movement_before_pass) + return test_verdict{pass, std::format("__test.result set to 1{}", reason_suffix(Frame.reason))}; + Progress.passrequested = true; + } + } + + Progress.resttime = Frame.atrest ? Progress.resttime + Frame.deltatime : 0.0; + + if (Progress.passrequested && Progress.resttime >= pass_settle_time) + return test_verdict{pass, "__test.result set to 1 and every vehicle stopped"}; + if (Limits.standstill > 0.0 && Progress.resttime >= Limits.standstill) + return test_verdict{Limits.pass_on_standstill ? pass : standstill, std::format("every vehicle stood still for {:.0f} s", Limits.standstill)}; + if (Limits.timeout > 0.0 && Progress.time >= Limits.timeout) + return test_verdict{timeout, std::format("no result after {:.0f} s", Limits.timeout)}; + return std::nullopt; +} + +} // namespace simulation diff --git a/simulation/simulationtestrules.h b/simulation/simulationtestrules.h new file mode 100644 index 00000000..5d966c3c --- /dev/null +++ b/simulation/simulationtestrules.h @@ -0,0 +1,82 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +#include +#include +#include +#include + +// the rules of the test mode (eu07 --test, eu07/maszyna#541) which need nothing of the running simulation: the exit +// codes, the command line and the end conditions. kept apart so the unit tests can check them +namespace simulation { + +// exit codes of a test run; public, scripts and CI read them +enum class test_result : int { + pass = 0, // __test.result set to 1, then every vehicle came to rest (or test.pass_on_standstill) + fail = 1, // __test.result set to 0 + signal_passed_at_danger = 2, // a train went past a stop signal (SetVelocity 0 0), unless test.fail_if_stop_overran no + timeout = 3, // test.timeout seconds of simulation time went by without a result + standstill = 4, // every vehicle stood still for test.standstill seconds without a result + error = 5, // the run could not start (a bad command line, the scenery failed to load) or the simulator threw +}; + +std::string_view to_string(test_result Result); + +// how a test ended, and why +struct test_verdict { + test_result result{test_result::error}; + std::string reason; +}; + +// the test mode options of the command line +struct test_command_line { + bool enabled{false}; // --test + std::optional timeout; // --test-timeout + std::string error; // what's wrong with them; empty if nothing +}; + +// seconds given on the command line: a number, not negative. nothing for anything else +std::optional parse_seconds(std::string_view Text); + +// picks the test mode options out of a command line (the first argument is the program). the values of the other +// options are skipped, so a scenery named like an option isn't taken for one; options it doesn't know are left alone +test_command_line parse_test_command_line(std::span Arguments); + +// the limits and switches the end conditions go by +struct test_limits { + double timeout{3600.0}; // [s] of simulation time; 0 = never + double standstill{300.0}; // [s] of every vehicle standing without a result; 0 = never + bool pass_on_standstill{false}; + bool no_movement_before_pass{true}; +}; + +// what a frame of the simulation brought +struct test_frame { + double deltatime{0.0}; // [s] of simulation time + std::optional reported; // value of the result cell, if there is one + std::string_view reason; // text of the result cell + bool atrest{false}; // every vehicle stands +}; + +// the state of a test between frames +struct test_progress { + double time{0.0}; // simulation time since the start [s] + double resttime{0.0}; // how long every vehicle has been standing [s] + bool passrequested{false}; // __test.result was set to 1, the pass waits for the vehicles to stop +}; + +// how long everything has to stand before a requested pass is granted [s] +inline constexpr double pass_settle_time{1.0}; + +// checks the end conditions after a frame and moves the progress on. returns the verdict once the test is over +std::optional evaluate_frame(test_progress &Progress, test_frame const &Frame, test_limits const &Limits); + +} // namespace simulation diff --git a/tests/unit/test_test_mode.cpp b/tests/unit/test_test_mode.cpp new file mode 100644 index 00000000..fec475c5 --- /dev/null +++ b/tests/unit/test_test_mode.cpp @@ -0,0 +1,195 @@ +#include "stdafx.h" +#include "doctest.h" + +#include "simulation/simulationtestrules.h" + +#include + +// the test mode of the simulator (eu07 --test, eu07/maszyna#541): its command line and end conditions + +namespace { + +using simulation::test_result; + +simulation::test_command_line parse(std::vector Arguments) +{ + Arguments.insert(Arguments.begin(), "eu07"); + return simulation::parse_test_command_line(Arguments); +} + +// a frame of the default 0.03 s +simulation::test_frame frame(bool const Atrest, std::optional const Reported = std::nullopt, std::string_view const Reason = {}) +{ + return simulation::test_frame{0.03, Reported, Reason, Atrest}; +} + +// runs frames until the test ends or Seconds of simulation time go by +std::optional run(simulation::test_progress &Progress, simulation::test_limits const &Limits, double const Seconds, + simulation::test_frame const &Frame) +{ + while (Progress.time < Seconds) + { + if (auto verdict{simulation::evaluate_frame(Progress, Frame, Limits)}) + return verdict; + } + return std::nullopt; +} + +} // namespace + +TEST_CASE("test mode: seconds on the command line are a number, not negative") +{ + CHECK(simulation::parse_seconds("240") == 240.0); + CHECK(simulation::parse_seconds("0.5") == 0.5); + CHECK(simulation::parse_seconds("0") == 0.0); + CHECK_FALSE(simulation::parse_seconds("").has_value()); + CHECK_FALSE(simulation::parse_seconds("-1").has_value()); + CHECK_FALSE(simulation::parse_seconds("60s").has_value()); + CHECK_FALSE(simulation::parse_seconds("abc").has_value()); + CHECK_FALSE(simulation::parse_seconds("inf").has_value()); + CHECK_FALSE(simulation::parse_seconds("nan").has_value()); +} + +TEST_CASE("test mode: the command line") +{ + SUBCASE("without --test there is no test") + { + auto const commandline{parse({"-s", "td.scn", "-v", "ep07-424"})}; + CHECK_FALSE(commandline.enabled); + CHECK(commandline.error.empty()); + } + SUBCASE("--test and its limit, in any order") + { + auto const commandline{parse({"-s", "td.scn", "--test-timeout", "240", "--test"})}; + CHECK(commandline.enabled); + CHECK(commandline.timeout == 240.0); + CHECK(commandline.error.empty()); + } + SUBCASE("a limit not given stays unset") + { + auto const commandline{parse({"--test", "-s", "td.scn"})}; + CHECK_FALSE(commandline.timeout.has_value()); + } + SUBCASE("a limit which isn't a number is an error, not a limit turned off") + { + auto const commandline{parse({"--test", "-s", "td.scn", "--test-timeout", "4m"})}; + CHECK(commandline.enabled); + CHECK_FALSE(commandline.timeout.has_value()); + CHECK(commandline.error == "--test-timeout needs a number of seconds"); + } + SUBCASE("a limit without its value is an error") + { + auto const commandline{parse({"--test", "-s", "td.scn", "--test-timeout"})}; + CHECK(commandline.error == "--test-timeout needs a number of seconds"); + } + SUBCASE("values of other options aren't taken for options") + { + auto const commandline{parse({"-s", "--test", "-ini", "--test-timeout", "-v", "--test", "-seed", "--test"})}; + CHECK_FALSE(commandline.enabled); + CHECK(commandline.error.empty()); + } + SUBCASE("the scenery of -edit is optional") + { + CHECK(parse({"-edit", "--test", "-s", "td.scn"}).enabled); + CHECK_FALSE(parse({"-edit", "td.scn"}).enabled); + } +} + +TEST_CASE("test mode: the end conditions") +{ + simulation::test_limits limits; + simulation::test_progress progress; + + SUBCASE("nothing happens, nothing ends") + { + CHECK_FALSE(simulation::evaluate_frame(progress, frame(false), limits).has_value()); + CHECK(progress.time == doctest::Approx(0.03)); + CHECK(progress.resttime == 0.0); + } + SUBCASE("a cell nobody wrote to says nothing") + { + CHECK_FALSE(simulation::evaluate_frame(progress, frame(false, -1.0), limits).has_value()); + CHECK_FALSE(simulation::evaluate_frame(progress, frame(false, 2.0), limits).has_value()); + CHECK_FALSE(progress.passrequested); + } + SUBCASE("0 fails at once, with the text of the cell") + { + auto const verdict{simulation::evaluate_frame(progress, frame(false, 0.0, "wrong_track"), limits)}; + REQUIRE(verdict.has_value()); + CHECK(verdict->result == test_result::fail); + CHECK(verdict->reason == "__test.result set to 0 (wrong_track)"); + } + SUBCASE("1 passes once every vehicle has stood for a second") + { + CHECK_FALSE(simulation::evaluate_frame(progress, frame(false, 1.0), limits).has_value()); + CHECK(progress.passrequested); + // still moving: the pass waits + CHECK_FALSE(run(progress, limits, 30.0, frame(false, 1.0)).has_value()); + auto const stoppedat{progress.time}; + auto const verdict{run(progress, limits, 60.0, frame(true, 1.0))}; + REQUIRE(verdict.has_value()); + CHECK(verdict->result == test_result::pass); + CHECK(progress.time - stoppedat == doctest::Approx(simulation::pass_settle_time).epsilon(0.05)); + } + SUBCASE("the pass stays requested when the cell changes back") + { + CHECK_FALSE(simulation::evaluate_frame(progress, frame(false, 1.0), limits).has_value()); + auto const verdict{run(progress, limits, 60.0, frame(true, -1.0))}; + REQUIRE(verdict.has_value()); + CHECK(verdict->result == test_result::pass); + } + SUBCASE("1 with test.no_movement_before_pass no passes at once") + { + limits.no_movement_before_pass = false; + auto const verdict{simulation::evaluate_frame(progress, frame(false, 1.0, "reached_the_stop"), limits)}; + REQUIRE(verdict.has_value()); + CHECK(verdict->result == test_result::pass); + CHECK(verdict->reason == "__test.result set to 1 (reached_the_stop)"); + } + SUBCASE("standing still without a result ends after test.standstill") + { + limits.standstill = 120.0; + auto const verdict{run(progress, limits, 1000.0, frame(true))}; + REQUIRE(verdict.has_value()); + CHECK(verdict->result == test_result::standstill); + CHECK(progress.time == doctest::Approx(120.0).epsilon(0.001)); + } + SUBCASE("a move starts the standstill over") + { + limits.standstill = 120.0; + CHECK_FALSE(run(progress, limits, 100.0, frame(true)).has_value()); + CHECK_FALSE(simulation::evaluate_frame(progress, frame(false), limits).has_value()); + CHECK(progress.resttime == 0.0); + CHECK_FALSE(run(progress, limits, 200.0, frame(true)).has_value()); + } + SUBCASE("test.pass_on_standstill makes the standstill a pass") + { + limits.standstill = 10.0; + limits.pass_on_standstill = true; + auto const verdict{run(progress, limits, 1000.0, frame(true))}; + REQUIRE(verdict.has_value()); + CHECK(verdict->result == test_result::pass); + } + SUBCASE("test.standstill 0 never ends the test") + { + limits.standstill = 0.0; + limits.timeout = 0.0; + CHECK_FALSE(run(progress, limits, 5000.0, frame(true)).has_value()); + } + SUBCASE("no result by test.timeout") + { + limits.timeout = 240.0; + auto const verdict{run(progress, limits, 1000.0, frame(false))}; + REQUIRE(verdict.has_value()); + CHECK(verdict->result == test_result::timeout); + CHECK(verdict->reason == "no result after 240 s"); + CHECK(progress.time == doctest::Approx(240.0).epsilon(0.001)); + } + SUBCASE("a result in the last frame wins over the timeout") + { + limits.timeout = 0.03; + auto const verdict{simulation::evaluate_frame(progress, frame(false, 0.0), limits)}; + REQUIRE(verdict.has_value()); + CHECK(verdict->result == test_result::fail); + } +}