mirror of
https://github.com/MaSzyna-EU07/maszyna.git
synced 2026-11-19 05:16:01 +01:00
Merge pull request #266 from MaSzyna-EU07/feat/exe-test-mode
Test mode: `--test` runs a scenery unattended and exits with the result (#541)
This commit is contained in:
268
tests/unit/test_test_mode.cpp
Normal file
268
tests/unit/test_test_mode.cpp
Normal file
@@ -0,0 +1,268 @@
|
||||
#include "stdafx.h"
|
||||
#include "doctest.h"
|
||||
|
||||
#include "simulation/simulationtestrules.h"
|
||||
|
||||
#include <array>
|
||||
#include <exception>
|
||||
#include <new>
|
||||
#include <stdexcept>
|
||||
#include <vector>
|
||||
|
||||
// the test mode of the simulator (eu07 --test, eu07/maszyna#541): its exit codes, command line and end conditions
|
||||
|
||||
namespace {
|
||||
|
||||
using simulation::test_result;
|
||||
|
||||
simulation::test_command_line parse(std::vector<char const *> Arguments)
|
||||
{
|
||||
Arguments.insert(Arguments.begin(), "eu07");
|
||||
return simulation::parse_test_command_line(Arguments);
|
||||
}
|
||||
|
||||
// a frame of the default 0.03 s
|
||||
simulation::test_frame frame(bool const Atrest, std::optional<double> const Reported = std::nullopt, std::string_view const Reason = {})
|
||||
{
|
||||
return simulation::test_frame{0.03, Reported, Reason, Atrest};
|
||||
}
|
||||
|
||||
// runs frames until the test ends or Seconds of simulation time go by
|
||||
std::optional<simulation::test_verdict> run(simulation::test_progress &Progress, simulation::test_limits const &Limits, double const Seconds,
|
||||
simulation::test_frame const &Frame)
|
||||
{
|
||||
while (Progress.time < Seconds)
|
||||
{
|
||||
if (auto verdict{simulation::evaluate_frame(Progress, Frame, Limits)})
|
||||
return verdict;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("test mode: the exit codes keep clear of the ones of a killed or aborted process")
|
||||
{
|
||||
std::array const results{test_result::pass, test_result::fail, test_result::signal_passed_at_danger, test_result::timeout, test_result::standstill, test_result::error};
|
||||
for (auto const result : results)
|
||||
{
|
||||
auto const code{static_cast<int>(result)};
|
||||
CAPTURE(code);
|
||||
CHECK(code != 1); // TerminateProcess, most programs' failure
|
||||
CHECK(code != 3); // abort() on Windows
|
||||
CHECK(code < 126); // shells, signals on Linux
|
||||
CHECK(simulation::to_string(result) != "unknown");
|
||||
}
|
||||
// the codes are public: scripts and CI read them
|
||||
CHECK(static_cast<int>(test_result::pass) == 0);
|
||||
CHECK(static_cast<int>(test_result::fail) == 10);
|
||||
CHECK(static_cast<int>(test_result::signal_passed_at_danger) == 11);
|
||||
CHECK(static_cast<int>(test_result::timeout) == 12);
|
||||
CHECK(static_cast<int>(test_result::standstill) == 13);
|
||||
CHECK(static_cast<int>(test_result::error) == 14);
|
||||
}
|
||||
|
||||
TEST_CASE("test mode: seconds on the command line are a number, not negative")
|
||||
{
|
||||
CHECK(simulation::parse_seconds("240").value_or(-1.0) == doctest::Approx(240.0));
|
||||
CHECK(simulation::parse_seconds("0.5").value_or(-1.0) == doctest::Approx(0.5));
|
||||
CHECK(simulation::parse_seconds("0").value_or(-1.0) == doctest::Approx(0.0));
|
||||
CHECK_FALSE(simulation::parse_seconds("").has_value());
|
||||
CHECK_FALSE(simulation::parse_seconds("-1").has_value());
|
||||
CHECK_FALSE(simulation::parse_seconds("60s").has_value());
|
||||
CHECK_FALSE(simulation::parse_seconds("abc").has_value());
|
||||
CHECK_FALSE(simulation::parse_seconds("inf").has_value());
|
||||
CHECK_FALSE(simulation::parse_seconds("nan").has_value());
|
||||
}
|
||||
|
||||
TEST_CASE("test mode: the command line")
|
||||
{
|
||||
SUBCASE("without --test there is no test")
|
||||
{
|
||||
auto const commandline{parse({"-s", "td.scn", "-v", "ep07-424"})};
|
||||
CHECK_FALSE(commandline.enabled);
|
||||
CHECK(commandline.error.empty());
|
||||
}
|
||||
SUBCASE("--test and its limits, in any order")
|
||||
{
|
||||
auto const commandline{parse({"--test-wall-timeout", "900", "-s", "td.scn", "--test", "--test-timeout", "240"})};
|
||||
CHECK(commandline.enabled);
|
||||
CHECK(commandline.timeout.value_or(-1.0) == doctest::Approx(240.0));
|
||||
CHECK(commandline.wall_timeout.value_or(-1.0) == doctest::Approx(900.0));
|
||||
CHECK(commandline.error.empty());
|
||||
}
|
||||
SUBCASE("a limit not given stays unset")
|
||||
{
|
||||
auto const commandline{parse({"--test", "-s", "td.scn"})};
|
||||
CHECK_FALSE(commandline.timeout.has_value());
|
||||
CHECK_FALSE(commandline.wall_timeout.has_value());
|
||||
}
|
||||
SUBCASE("a limit which isn't a number is an error, not a limit turned off")
|
||||
{
|
||||
auto const commandline{parse({"--test", "-s", "td.scn", "--test-timeout", "4m"})};
|
||||
CHECK(commandline.enabled);
|
||||
CHECK_FALSE(commandline.timeout.has_value());
|
||||
CHECK(commandline.error == "--test-timeout needs a number of seconds");
|
||||
}
|
||||
SUBCASE("a limit without its value is an error")
|
||||
{
|
||||
auto const commandline{parse({"--test", "-s", "td.scn", "--test-wall-timeout"})};
|
||||
CHECK(commandline.error == "--test-wall-timeout needs a number of seconds");
|
||||
}
|
||||
SUBCASE("the first error is the one told")
|
||||
{
|
||||
auto const commandline{parse({"--test", "--test-timeout", "x", "--test-wall-timeout", "y"})};
|
||||
CHECK(commandline.error == "--test-timeout needs a number of seconds");
|
||||
}
|
||||
SUBCASE("values of other options aren't taken for options")
|
||||
{
|
||||
auto const commandline{parse({"-s", "--test", "-ini", "--test-timeout", "-v", "--test-wall-timeout", "-seed", "--test"})};
|
||||
CHECK_FALSE(commandline.enabled);
|
||||
CHECK(commandline.error.empty());
|
||||
}
|
||||
SUBCASE("-edit takes no option for its scenery, which is optional")
|
||||
{
|
||||
// a scenery after -edit or none: the option after it is read either way
|
||||
CHECK(parse({"-edit", "--test", "-s", "td.scn"}).enabled);
|
||||
CHECK(parse({"-edit", "td.scn", "--test"}).enabled);
|
||||
// -edit leaves -s alone, which takes --test for its scenery, as the simulator reads it
|
||||
CHECK_FALSE(parse({"-edit", "-s", "--test"}).enabled);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("test mode: the real time limit")
|
||||
{
|
||||
SUBCASE("given, it is the limit, 0 turns it off")
|
||||
{
|
||||
CHECK(simulation::wall_clock_limit(120.0, 3600.0) == doctest::Approx(120.0));
|
||||
CHECK(simulation::wall_clock_limit(0.0, 3600.0) == doctest::Approx(0.0));
|
||||
}
|
||||
SUBCASE("not given, it is the simulation time limit and time for the loading")
|
||||
{
|
||||
CHECK(simulation::wall_clock_limit(std::nullopt, 240.0) == doctest::Approx(240.0 + simulation::wall_timeout_loading_allowance));
|
||||
CHECK(simulation::wall_clock_limit(std::nullopt, 3600.0) == doctest::Approx(3600.0 + simulation::wall_timeout_loading_allowance));
|
||||
}
|
||||
SUBCASE("no simulation time limit, no real time limit unless given")
|
||||
{
|
||||
CHECK(simulation::wall_clock_limit(std::nullopt, 0.0) == doctest::Approx(0.0));
|
||||
CHECK(simulation::wall_clock_limit(300.0, 0.0) == doctest::Approx(300.0));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("test mode: the end conditions")
|
||||
{
|
||||
simulation::test_limits limits;
|
||||
simulation::test_progress progress;
|
||||
|
||||
SUBCASE("nothing happens, nothing ends")
|
||||
{
|
||||
CHECK_FALSE(simulation::evaluate_frame(progress, frame(false), limits).has_value());
|
||||
CHECK(progress.time == doctest::Approx(0.03));
|
||||
CHECK(progress.resttime == doctest::Approx(0.0));
|
||||
}
|
||||
SUBCASE("a cell nobody wrote to says nothing")
|
||||
{
|
||||
CHECK_FALSE(simulation::evaluate_frame(progress, frame(false, -1.0), limits).has_value());
|
||||
CHECK_FALSE(simulation::evaluate_frame(progress, frame(false, 2.0), limits).has_value());
|
||||
CHECK_FALSE(progress.passrequested);
|
||||
}
|
||||
SUBCASE("0 fails at once, with the text of the cell")
|
||||
{
|
||||
auto const verdict{simulation::evaluate_frame(progress, frame(false, 0.0, "wrong_track"), limits)};
|
||||
REQUIRE(verdict.has_value());
|
||||
CHECK(verdict->result == test_result::fail);
|
||||
CHECK(verdict->reason == "__test.result set to 0 (wrong_track)");
|
||||
}
|
||||
SUBCASE("1 passes once the tested trains have stood for a second")
|
||||
{
|
||||
CHECK_FALSE(simulation::evaluate_frame(progress, frame(false, 1.0), limits).has_value());
|
||||
CHECK(progress.passrequested);
|
||||
// still moving: the pass waits
|
||||
CHECK_FALSE(run(progress, limits, 30.0, frame(false, 1.0)).has_value());
|
||||
auto const stoppedat{progress.time};
|
||||
auto const verdict{run(progress, limits, 60.0, frame(true, 1.0))};
|
||||
REQUIRE(verdict.has_value());
|
||||
CHECK(verdict->result == test_result::pass);
|
||||
CHECK(progress.time - stoppedat == doctest::Approx(simulation::pass_settle_time).epsilon(0.05));
|
||||
}
|
||||
SUBCASE("the pass stays requested when the cell changes back")
|
||||
{
|
||||
CHECK_FALSE(simulation::evaluate_frame(progress, frame(false, 1.0), limits).has_value());
|
||||
auto const verdict{run(progress, limits, 60.0, frame(true, -1.0))};
|
||||
REQUIRE(verdict.has_value());
|
||||
CHECK(verdict->result == test_result::pass);
|
||||
}
|
||||
SUBCASE("1 with test.no_movement_before_pass no passes at once")
|
||||
{
|
||||
limits.no_movement_before_pass = false;
|
||||
auto const verdict{simulation::evaluate_frame(progress, frame(false, 1.0, "reached_the_stop"), limits)};
|
||||
REQUIRE(verdict.has_value());
|
||||
CHECK(verdict->result == test_result::pass);
|
||||
CHECK(verdict->reason == "__test.result set to 1 (reached_the_stop)");
|
||||
}
|
||||
SUBCASE("standing still without a result ends after test.standstill")
|
||||
{
|
||||
limits.standstill = 120.0;
|
||||
auto const verdict{run(progress, limits, 1000.0, frame(true))};
|
||||
REQUIRE(verdict.has_value());
|
||||
CHECK(verdict->result == test_result::standstill);
|
||||
CHECK(progress.time == doctest::Approx(120.0).epsilon(0.001));
|
||||
}
|
||||
SUBCASE("a move starts the standstill over")
|
||||
{
|
||||
limits.standstill = 120.0;
|
||||
CHECK_FALSE(run(progress, limits, 100.0, frame(true)).has_value());
|
||||
CHECK_FALSE(simulation::evaluate_frame(progress, frame(false), limits).has_value());
|
||||
CHECK(progress.resttime == doctest::Approx(0.0));
|
||||
CHECK_FALSE(run(progress, limits, 200.0, frame(true)).has_value());
|
||||
}
|
||||
SUBCASE("test.pass_on_standstill makes the standstill a pass")
|
||||
{
|
||||
limits.standstill = 10.0;
|
||||
limits.pass_on_standstill = true;
|
||||
auto const verdict{run(progress, limits, 1000.0, frame(true))};
|
||||
REQUIRE(verdict.has_value());
|
||||
CHECK(verdict->result == test_result::pass);
|
||||
}
|
||||
SUBCASE("test.standstill 0 never ends the test")
|
||||
{
|
||||
limits.standstill = 0.0;
|
||||
limits.timeout = 0.0;
|
||||
CHECK_FALSE(run(progress, limits, 5000.0, frame(true)).has_value());
|
||||
}
|
||||
SUBCASE("no result by test.timeout")
|
||||
{
|
||||
limits.timeout = 240.0;
|
||||
auto const verdict{run(progress, limits, 1000.0, frame(false))};
|
||||
REQUIRE(verdict.has_value());
|
||||
CHECK(verdict->result == test_result::timeout);
|
||||
CHECK(verdict->reason == "no result after 240 s");
|
||||
CHECK(progress.time == doctest::Approx(240.0).epsilon(0.001));
|
||||
}
|
||||
SUBCASE("a result in the last frame wins over the timeout")
|
||||
{
|
||||
limits.timeout = 0.03;
|
||||
auto const verdict{simulation::evaluate_frame(progress, frame(false, 0.0), limits)};
|
||||
REQUIRE(verdict.has_value());
|
||||
CHECK(verdict->result == test_result::fail);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("test mode: what ended a run the simulator didn't end")
|
||||
{
|
||||
SUBCASE("an exception says what it was")
|
||||
{
|
||||
CHECK(simulation::describe_exception(std::make_exception_ptr(std::runtime_error("no scenery"))) == "critical error: no scenery");
|
||||
// e.g. std::stof on a token of a scenery which isn't a number
|
||||
CHECK(simulation::describe_exception(std::make_exception_ptr(std::invalid_argument("stof"))) == "critical error: stof");
|
||||
CHECK(simulation::describe_exception(std::make_exception_ptr(std::bad_alloc())).starts_with("critical error: "));
|
||||
}
|
||||
SUBCASE("something thrown which isn't a std::exception")
|
||||
{
|
||||
CHECK(simulation::describe_exception(std::make_exception_ptr(42)) == "critical error: an exception of unknown type");
|
||||
}
|
||||
SUBCASE("terminated without an exception")
|
||||
{
|
||||
CHECK(simulation::describe_exception(nullptr) == "the simulator was terminated");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user