diff --git a/CMakeLists.txt b/CMakeLists.txt index 1d5c0ad33..e563e587f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -545,6 +545,7 @@ set(CORE_SOURCES "McZapkie/Oerlikon_ESt.cpp" "utilities/parser.cpp" "utilities/utilities.cpp" +"utilities/crashsignals.cpp" "scripting/ladderlogic.cpp" "scripting/scriptscreenrules.cpp" "scripting/scriptscreenkernels.cpp" diff --git a/EU07.cpp b/EU07.cpp index 1c2c36aab..fa6bb15f2 100644 --- a/EU07.cpp +++ b/EU07.cpp @@ -22,9 +22,13 @@ Stele, firleju, szociu, hunter, ZiomalCl, OLI_EU and others #include "simulation/simulationtest.h" #include "utilities/Globals.h" #include "utilities/Logs.h" +#include "utilities/crashsignals.h" #include "utilities/physicslog.h" +#include "version_info.h" #include #include +#include +#include #include #include #include @@ -78,8 +82,12 @@ LONG WINAPI CrashHandler(EXCEPTION_POINTERS *ExceptionInfo) dumpInfo.ExceptionPointers = ExceptionInfo; dumpInfo.ClientPointers = FALSE; - // Wybrana kombinacja flag - MINIDUMP_TYPE dumpType = MINIDUMP_TYPE(MiniDumpWithFullMemory | MiniDumpWithHandleData | MiniDumpWithThreadInfo | MiniDumpWithUnloadedModules | MiniDumpWithIndirectlyReferencedMemory | + // a small dump by default: the stacks with the memory they point to, the threads with their process and thread + // blocks (the command line with the scenery and the vehicle), the modules, the handles and the modules' data + // (Global among them). all the memory of the simulator, as large as the memory it took, only with + // "crashdump.full yes" in eu07.ini + auto const dumpflags{Global.CrashDumpFull ? MiniDumpWithFullMemory : MiniDumpWithDataSegs}; + MINIDUMP_TYPE dumpType = MINIDUMP_TYPE(dumpflags | MiniDumpWithProcessThreadData | MiniDumpWithHandleData | MiniDumpWithThreadInfo | MiniDumpWithUnloadedModules | MiniDumpWithIndirectlyReferencedMemory | MiniDumpWithFullMemoryInfo | MiniDumpWithTokenInformation); // in test mode nobody is there to close the box, and the run would wait for it until killed @@ -172,9 +180,15 @@ int main(int argc, char *argv[]) SetUnhandledExceptionFilter(CrashHandler); #endif #endif + // on Linux, where no dump is written, the stack of a crash in the log and on the standard error + crash_signals::install(VERSION_INFO); // init start timestamp Global.startTimestamp = std::chrono::steady_clock::now(); + // a start that failed ends with an error code, so whoever started the simulator (the starter) can tell it + // from a closing; how the simulator ends after a start that worked stays as it was + int exitcode{0}; + // quick short-circuit for standalone e3d export if (argc == 6 && std::string(argv[1]) == "-e3d") { @@ -221,34 +235,54 @@ int main(int argc, char *argv[]) } else { + // below 0 when the start failed (see eu07_application::init()) + int initresult{0}; try { - auto result{Application.init(argc, argv)}; - if (result == 0) + initresult = Application.init(argc, argv); + if (initresult == 0) { - result = Application.run(); + Application.run(); Application.exit(); } } + // an exception that ends the simulator is a crash, whether in the start or later: it says what it was in the + // log and ends with an error code catch (std::bad_alloc const &Error) { ErrorLog("Critical error, memory allocation failure: " + std::string(Error.what())); + exitcode = 1; } #ifdef _WIN32 catch (std::runtime_error const &Error) { + ErrorLog("Critical error: " + std::string(Error.what())); std::string msg = "Simulator crash occured :(\n"; msg += Error.what(); MessageBoxA(nullptr, msg.c_str(), "Simulator crashed :(", MB_ICONERROR); + exitcode = 1; + } +#else + // elsewhere one not caught here ends the simulator through std::terminate, with no word in the log. on + // Windows it is left to CrashHandler, whose dump shows where it was thrown + catch (std::exception const &Error) + { + ErrorLog("Critical error: " + std::string(Error.what())); + exitcode = 1; } #endif // the last two seconds of the physics logs, also after an error caught above; Application.exit() stops them // otherwise, and a second stop only writes what came since PhysicsLogs.stop(); + // after a failed start or an error caught above the log thread still runs, and what it hasn't written yet + // (why the start failed) would go with the process. after Application.exit() it has stopped already + finish_log(); + if (initresult < 0) + exitcode = 1; } #ifndef _WIN32 fflush(stdout); fflush(stderr); #endif - std::_Exit(0); // skip destructors, there are ordering errors which causes segfaults + std::_Exit(exitcode); // skip destructors, there are ordering errors which causes segfaults } diff --git a/application/application.cpp b/application/application.cpp index 12bc9020a..d11ed8bbd 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -37,6 +37,9 @@ http://mozilla.org/MPL/2.0/. #include "utilities/physicslog.h" #include #include +#include +#include +#include #include #include "utilities/translation.h" @@ -171,6 +174,28 @@ void char_callback(GLFWwindow *window, unsigned int c) Application.on_char(c); } +namespace +{ + +// what glfw says went wrong during the start (init()), such as why the window or its context couldn't be made (no +// OpenGL driver, no pixel format that fits). each message once, so one repeated, for each window or worker's context, +// is told once. glfw may call this from any thread that uses it +void glfw_error_callback(int const Code, char const *Description) +{ + auto message{std::format("GLFW error 0x{:08X}: {}", static_cast(Code), Description != nullptr ? Description : "(no description)")}; + + static std::mutex reportedmutex; + static std::set reported; + { + std::scoped_lock lock(reportedmutex); + if (false == reported.insert(message).second) + return; + } + ErrorLog(message); +} + +} // namespace + // public: void eu07_application::queue_screenshot() @@ -253,7 +278,8 @@ int eu07_application::run_crashgui() return 0; } } - return -1; + // the player closed the window: the simulator closes with it, as it always has, but that is no failed start + return 1; } #if WITH_DISCORD_RPC void eu07_application::DiscordRPCService() @@ -436,7 +462,11 @@ int eu07_application::init(int Argc, char *Argv[]) { // run crashgui as early as possible if ((result = run_crashgui()) != 0) { - ErrorLog("Failed to run crash gui!"); + // above 0 the player closed the window, which is no error + if (result < 0) + ErrorLog("Failed to run crash gui!"); + else + WriteLog("Crash report window closed, closing"); return result; } } @@ -503,6 +533,11 @@ int eu07_application::init(int Argc, char *Argv[]) if (!init_network()) return -1; + // glfw's errors are told to explain a failed start. once the simulator runs they would only repeat what the + // platform can't do (no setting of the cursor position on Wayland) or what a scenery asks for in vain (old key + // codes of its launchers), every frame + glfwSetErrorCallback(nullptr); + return result; } @@ -1391,6 +1426,9 @@ int eu07_application::init_locale() int eu07_application::init_glfw() { + // set before glfwInit(), so a failure of the start itself is told too + glfwSetErrorCallback(glfw_error_callback); + { int glfw_major, glfw_minor, glfw_rev; glfwGetVersion(&glfw_major, &glfw_minor, &glfw_rev); diff --git a/application/application.h b/application/application.h index 850b66e1e..881d4f073 100644 --- a/application/application.h +++ b/application/application.h @@ -33,6 +33,7 @@ public: // constructors eu07_application() = default; // methods + // 0 when the simulator started, below 0 when its start failed, above 0 when the player closed it before it started int init( int Argc, char *Argv[] ); int diff --git a/tests/unit/test_crashsignals.cpp b/tests/unit/test_crashsignals.cpp new file mode 100644 index 000000000..4c4dd1e86 --- /dev/null +++ b/tests/unit/test_crashsignals.cpp @@ -0,0 +1,89 @@ +#include "stdafx.h" +#include "doctest.h" +#include "scratch.h" + +#include "utilities/crashsignals.h" + +// the stack of a crash on Linux (utilities/crashsignals.h): written to the log and to the standard error, after which +// the signal ends the process as it always did. the crash happens in a child process, which the test watches +#if defined(__linux__) + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace { + +std::string read_all(int const File) { + std::string text; + std::array buffer{}; + for (auto count{::read(File, buffer.data(), buffer.size())}; count > 0; count = ::read(File, buffer.data(), buffer.size())) + text.append(buffer.data(), static_cast(count)); + return text; +} + +std::string read_file(std::string const &Path) { + std::ifstream file{Path, std::ios::binary}; + return {std::istreambuf_iterator{file}, std::istreambuf_iterator{}}; +} + +// the child: the report goes to the pipe in place of the standard error, no core dump is left behind +[[noreturn]] void crash_in_child(std::array const &Pipe, std::string const &Log) { + // a child stuck before the crash (a lock some other thread held at the fork) ends anyway, and the test with it + alarm(10); + dup2(Pipe[1], STDERR_FILENO); + close(Pipe[0]); + close(Pipe[1]); + rlimit const nocore{0, 0}; + setrlimit(RLIMIT_CORE, &nocore); + + crash_signals::install("test version 1.2"); + crash_signals::set_log_path(Log); + raise(SIGSEGV); + _exit(0); // not reached: the signal ends the child +} + +} // namespace + +TEST_CASE("crash signals: a crash writes its stack to the log and to the standard error, and still ends by its signal") { + test::scratch_folder const folder{"crash_signals"}; + auto const log{folder.write("log.txt", "the last line of the log\n")}; + + std::array pipe_ends{}; + REQUIRE(pipe(pipe_ends.data()) == 0); + auto const child{fork()}; + REQUIRE(child >= 0); + if (child == 0) + crash_in_child(pipe_ends, log); + + close(pipe_ends[1]); + auto const errors{read_all(pipe_ends[0])}; + close(pipe_ends[0]); + int status{0}; + REQUIRE(waitpid(child, &status, 0) == child); + + CAPTURE(errors); + // the process ends as it would without the handler + CHECK(WIFSIGNALED(status)); + CHECK(WTERMSIG(status) == SIGSEGV); + // the report on the standard error, which the starter shows + CHECK(errors.find("Crash: signal 11 (SIGSEGV) at address 0x") != std::string::npos); + CHECK(errors.find("Version: test version 1.2") != std::string::npos); + CHECK(errors.find("Stack (addr2line") != std::string::npos); + // with the frames of the program that crashed + CHECK(errors.find("eu07_unit_tests") != std::string::npos); + // and the same at the end of the log, after what it held + auto const logtext{read_file(log)}; + CAPTURE(logtext); + CHECK(logtext.starts_with("the last line of the log\n")); + CHECK(logtext.find("Crash: signal 11 (SIGSEGV)") != std::string::npos); + CHECK(logtext.find("eu07_unit_tests") != std::string::npos); +} + +#endif diff --git a/utilities/Globals.cpp b/utilities/Globals.cpp index 864338442..abfa6d5e7 100644 --- a/utilities/Globals.cpp +++ b/utilities/Globals.cpp @@ -1294,6 +1294,12 @@ bool global_settings::ConfigParseDebug(cParser& Parser, const std::string& token return true; } + if (token == "crashdump.full") + { + ParseOne(Parser, CrashDumpFull); + return true; + } + if (token == "showsystemconsole") { ParseOne(Parser, ShowSystemConsole); @@ -1753,6 +1759,7 @@ global_settings::export_as_text( std::ostream &Output ) const { export_as_text( Output, "fullphysics", FullPhysics ); export_as_text( Output, "debuglog", iWriteLogEnabled ); export_as_text( Output, "multiplelogs", MultipleLogs ); + export_as_text( Output, "crashdump.full", CrashDumpFull ); export_as_text( Output, "showsystemconsole", ShowSystemConsole ); export_as_text( Output, "logs.filter", DisabledLogTypes ); Output diff --git a/utilities/Globals.h b/utilities/Globals.h index c4a9a7267..c20adc05e 100644 --- a/utilities/Globals.h +++ b/utilities/Globals.h @@ -113,6 +113,10 @@ struct global_settings { bool priorityLoadText3D{false}; // ladowanie T3D priorytetowo int iWriteLogEnabled{ 3 }; // maska bitowa: 1-zapis do pliku, 2-okienko, 4-nazwy torów bool MultipleLogs{ false }; + // the dump written when the simulator crashes (Windows): a small one by default, with the stacks, the threads, the + // modules, the memory the stacks point to and the modules' data. "crashdump.full yes" writes all the memory of the + // simulator instead, as large as the memory it took + bool CrashDumpFull{ false }; // the black box of the session (blackbox_doc.md): "blackbox yes" writes it whatever the physics logs, "blackbox no" // never, and with no entry it is written along with the physics logs std::optional BlackBox; diff --git a/utilities/Logs.cpp b/utilities/Logs.cpp index e0949aba6..f4032d808 100644 --- a/utilities/Logs.cpp +++ b/utilities/Logs.cpp @@ -9,6 +9,7 @@ http://mozilla.org/MPL/2.0/. #include "stdafx.h" #include "utilities/Logs.h" +#include "utilities/crashsignals.h" #include @@ -115,6 +116,8 @@ void LogService() { std::string filename = Global.MultipleLogs ? "logs/log (" + filename_scenery() + ") " + filename_date() + ".txt" : "log.txt"; output.open(filename, std::ios::trunc); + // the stack of a crash on Linux goes to the end of this file + crash_signals::set_log_path(filename); } output << msg << "\n"; output.flush(); diff --git a/utilities/crashsignals.cpp b/utilities/crashsignals.cpp new file mode 100644 index 000000000..fa62e8955 --- /dev/null +++ b/utilities/crashsignals.cpp @@ -0,0 +1,234 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#include "stdafx.h" +#include "utilities/crashsignals.h" + +#if defined(__linux__) + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +// the handler may only call what is safe in a signal: no allocation, no locks, no streams. so what it needs is +// kept in fixed buffers, filled in beforehand, and written with write() + +constexpr std::array crash_signal_list{SIGSEGV, SIGBUS, SIGFPE, SIGILL, SIGABRT}; +// a handler that hangs (a crash inside the allocator or the loader, which backtrace() may need) ends this long after +constexpr unsigned int crash_report_timeout_seconds{5}; + +// a text of fixed size, filled outside the handler and read in it +template struct fixed_text { + std::array text{}; + std::atomic ready{false}; + + void set(std::string_view const Text) { + ready = false; + auto const length{std::min(Text.size(), text.size() - 1)}; + std::copy_n(Text.data(), length, text.data()); + text[length] = '\0'; + ready = true; + } + char const *get() const { return ready ? text.data() : nullptr; } +}; + +fixed_text<1024> &log_path() { + static fixed_text<1024> path; + return path; +} + +fixed_text<256> &version_text() { + static fixed_text<256> version; + return version; +} + +// the stack the handler runs on, so an overflow of the main thread's own stack is reported too +std::array &alternate_stack() { + static std::array stack{}; + return stack; +} + +void write_all(int const File, std::string_view Text) { + while (false == Text.empty()) { + auto const written{::write(File, Text.data(), Text.size())}; + if (written <= 0) + return; + Text.remove_prefix(static_cast(written)); + } +} + +void write_hex(int const File, std::uintptr_t const Value) { + constexpr std::size_t digits{sizeof(Value) * 2}; + constexpr std::string_view hexdigits{"0123456789abcdef"}; + std::array text{'0', 'x'}; + for (std::size_t digit = 0; digit < digits; ++digit) + text[2 + digit] = hexdigits[(Value >> (4 * (digits - 1 - digit))) & 0xF]; + write_all(File, {text.data(), text.size()}); +} + +void write_decimal(int const File, int const Value) { + std::array text{}; + auto position{text.size()}; + auto rest{static_cast(Value < 0 ? -Value : Value)}; + do { + text[--position] = static_cast('0' + rest % 10); + rest /= 10; + } while (rest != 0 && position > 1); + if (Value < 0) + text[--position] = '-'; + write_all(File, {text.data() + position, text.size() - position}); +} + +std::string_view signal_name(int const Signal) { + switch (Signal) { + case SIGSEGV: + return "SIGSEGV"; + case SIGBUS: + return "SIGBUS"; + case SIGFPE: + return "SIGFPE"; + case SIGILL: + return "SIGILL"; + case SIGABRT: + return "SIGABRT"; + default: + return "?"; + } +} + +// the signal being reported, for a report that hangs +std::atomic &crash_signal() { + static std::atomic signal{0}; + return signal; +} + +// a report that hangs is given up: the process ends by the signal of the crash, as it would have without the handler +void handle_report_timeout(int) { + auto const crashsignal{crash_signal().load()}; + raise(crashsignal); + // it is blocked while its own handler runs, which this one interrupted + sigset_t unblocked; + sigemptyset(&unblocked); + sigaddset(&unblocked, crashsignal); + sigprocmask(SIG_UNBLOCK, &unblocked, nullptr); +} + +struct crash_stack { + std::array frames{}; + int count{0}; +}; + +void write_report(int const File, int const Signal, siginfo_t const *Info, crash_stack const &Stack) { + write_all(File, "\nCrash: signal "); + write_decimal(File, Signal); + write_all(File, " ("); + write_all(File, signal_name(Signal)); + write_all(File, ")"); + // the address of the fault; an abort has none + if (Signal != SIGABRT && Info != nullptr) { + write_all(File, " at address "); + write_hex(File, reinterpret_cast(Info->si_addr)); + } + write_all(File, "\n"); + if (auto const *version{version_text().get()}) { + write_all(File, "Version: "); + write_all(File, version); + write_all(File, "\n"); + } + write_all(File, "Stack (addr2line -f -C -e ):\n"); + backtrace_symbols_fd(Stack.frames.data(), Stack.count, File); +} + +void handle_crash_signal(int const Signal, siginfo_t *Info, void *) { + crash_signal() = Signal; + struct sigaction timeout{}; + timeout.sa_handler = handle_report_timeout; + sigemptyset(&timeout.sa_mask); + sigaction(SIGALRM, &timeout, nullptr); + alarm(crash_report_timeout_seconds); + + crash_stack stack; + stack.count = backtrace(stack.frames.data(), static_cast(stack.frames.size())); + + if (auto const *path{log_path().get()}) { + if (auto const log{::open(path, O_WRONLY | O_APPEND | O_CLOEXEC)}; log >= 0) { + write_report(log, Signal, Info, stack); + ::close(log); + } + } + write_report(STDERR_FILENO, Signal, Info, stack); + + // the handler was taken off as it was called (SA_RESETHAND): the signal does what it always did + raise(Signal); +} + +} // namespace + +namespace crash_signals { + +void install(std::string_view const Version) { + static std::atomic installed{false}; + if (installed.exchange(true)) + return; + + version_text().set(Version); + + // backtrace() loads what it needs on its first call, which allocates: done here, not in the handler + std::array warmup{}; + backtrace(warmup.data(), static_cast(warmup.size())); + + stack_t alternate{}; + alternate.ss_sp = alternate_stack().data(); + alternate.ss_size = alternate_stack().size(); + sigaltstack(&alternate, nullptr); + + struct sigaction action{}; + action.sa_sigaction = handle_crash_signal; + action.sa_flags = SA_SIGINFO | SA_ONSTACK | SA_RESETHAND; + // another crash signal of the same thread waits until the report is written, rather than starting a second one in + // its middle (a fault that comes while it is blocked still ends the process). a crash of another thread at the + // same moment may still write its report alongside + sigemptyset(&action.sa_mask); + for (auto const crashsignal : crash_signal_list) + sigaddset(&action.sa_mask, crashsignal); + for (auto const crashsignal : crash_signal_list) + sigaction(crashsignal, &action, nullptr); +} + +void set_log_path(std::string_view const Path) { + // whole, so the report finds the file wherever the simulator stands then + std::error_code error; + auto const path{std::filesystem::absolute(std::filesystem::path{Path}, error)}; + log_path().set(error ? std::string{Path} : path.string()); +} + +} // namespace crash_signals + +#else + +namespace crash_signals { + +void install(std::string_view) {} +void set_log_path(std::string_view) {} + +} // namespace crash_signals + +#endif diff --git a/utilities/crashsignals.h b/utilities/crashsignals.h new file mode 100644 index 000000000..feb526869 --- /dev/null +++ b/utilities/crashsignals.h @@ -0,0 +1,25 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +#include + +// the stack of a crash on Linux, where no dump is written: a signal of a fault (SIGSEGV, SIGBUS, SIGFPE, SIGILL) or +// SIGABRT writes the stack of the thread that took it to the log and to the standard error, which the starter shows, +// and then ends the simulator as the signal always did (a core dump where the system makes one). the addresses are +// read with addr2line on the eu07 file of the same version, which carries its symbols. elsewhere it does nothing +namespace crash_signals { + +// sets the handlers up, once. Version goes into the report, to tell which eu07 file the addresses belong to +void install(std::string_view Version); +// the log file the report is added to, once the log has opened it +void set_log_path(std::string_view Path); + +} // namespace crash_signals