Report a simulator that crashes or closes right after the start

The exit code, read with its signal or NTSTATUS, and the last lines of the
error stream - or of log.txt when it stayed silent. The stream is captured
only while the starter stays to read it.

StarterNG.Tests/SimulatorExitTests.cs

@@ -0,0 +1,65 @@
+using StarterNG.Application;
+using StarterNG.Infrastructure.Adapters;
+using StarterNG.Tests.Fakes;
+
+namespace StarterNG.Tests;
+
+public class SimulatorExitTests
+{
+    private static readonly TimeSpan Session = TimeSpan.FromMinutes(20);
+
+    [Fact]
+    public void A_session_ended_cleanly_is_no_failure()
+    {
+        Assert.False(new SimulatorExit(0, Session, []).Failed);
+    }
+
+    [Fact]
+    public void An_error_code_is_a_failure()
+    {
+        Assert.True(new SimulatorExit(1, Session, []).Failed);
+    }
+
+    [Fact]
+    public void A_window_that_only_flashed_is_a_failure_even_with_a_clean_code()
+    {
+        var exit = new SimulatorExit(0, TimeSpan.FromSeconds(0.2), []);
+
+        Assert.True(exit.Failed);
+        Assert.True(exit.Quick);
+    }
+
+    [Theory]
+    [InlineData(139, "139 (SIGSEGV)")]
+    [InlineData(134, "134 (SIGABRT)")]
+    [InlineData(-1073741819, "0xC0000005 (access violation)")]
+    [InlineData(3, "3")]
+    public void The_code_says_what_killed_it(int code, string described)
+    {
+        Assert.Equal(described, new SimulatorExit(code, Session, []).DescribeCode());
+    }
+
+    [Fact]
+    public void Colour_codes_and_blank_lines_are_dropped_from_the_errors()
+    {
+        var exit = new SimulatorExit(1, Session, ["\u001b[31mBad file: texture\u001b[0m", "", "   "]);
+
+        Assert.Equal(["Bad file: texture"], exit.CleanErrors);
+    }
+
+    [Fact]
+    public async Task A_real_process_leaves_its_exit_code_and_error_stream()
+    {
+        var launcher = new SystemProcessLauncher(new TestInstallation().Log);
+        var (shell, script) = OperatingSystem.IsWindows()
+            ? ("cmd", new[] { "/c", "echo boom 1>&2 & exit 3" })
+            : ("/bin/sh", new[] { "-c", "echo boom >&2; exit 3" });
+
+        var process = launcher.Start(shell, script, null, captureErrors: true, out string? error);
+        Assert.Null(error);
+        await process!.WaitForExitAsync();
+
+        Assert.Equal(3, process.ExitCode);
+        Assert.Equal("boom", Assert.Single(process.ErrorTail).Trim());
+    }
+}

StarterNG.Tests/StartSimulationTests.cs

@@ -54,6 +54,30 @@ public class StartSimulationTests
         Assert.Equal(installation.Paths.Root, launcher.WorkingDirectory);
     }
 
+    [Fact]
+    public void The_error_stream_is_kept_only_while_the_starter_stays_to_read_it()
+    {
+        var (start, launcher, installation) = Rig(files => files.WithExecutable(Build));
+
+        start.Execute(freeFly: true, saveSettings: false);
+        Assert.True(launcher.CapturedErrors);
+
+        // Closing right after the launch would leave the pipe with no reader.
+        installation.SettingsStore.Settings.AutoCloseStarter = true;
+        start.Execute(freeFly: true, saveSettings: false);
+        Assert.False(launcher.CapturedErrors);
+    }
+
+    [Fact]
+    public void The_end_of_the_simulator_log_is_read_for_a_silent_crash()
+    {
+        var (start, _, _) = Rig(files => files
+            .WithExecutable(Build)
+            .WithFile(TestInstallation.At("log.txt"), "first\r\nsecond\r\n\r\nthird\r\n"));
+
+        Assert.Equal(new[] { "second", "third" }, start.SimulatorLogTail(2));
+    }
+
     [Fact]
     public void A_file_that_cannot_be_run_is_not()
     {
@@ -102,10 +126,13 @@ public class StartSimulationTests
 
         public IReadOnlyList<string>? Arguments { get; private set; }
 
+        public bool? CapturedErrors { get; private set; }
+
         public IProcessHandle? Start(string executablePath, IReadOnlyList<string> arguments, string? workingDirectory,
-                                     out string? error)
+                                     bool captureErrors, out string? error)
         {
             Executable = executablePath;
+            CapturedErrors = captureErrors;
             Arguments = arguments;
             WorkingDirectory = workingDirectory;
             error = null;
@@ -121,6 +148,10 @@ public class StartSimulationTests
     {
         public bool HasExited => false;
 
+        public int? ExitCode => null;
+
+        public IReadOnlyList<string> ErrorTail => Array.Empty<string>();
+
         public Task WaitForExitAsync(CancellationToken cancellationToken = default) => Task.CompletedTask;
     }
 }

StarterNG/Application/Abstractions/IProcessLauncher.cs

@@ -8,13 +8,24 @@ public interface IProcessHandle
 {
     bool HasExited { get; }
 
+    /// <summary>Once it has ended; null while running, or for a process the starter did not start.</summary>
+    int? ExitCode { get; }
+
+    /// <summary>The last lines of the error stream, when it was captured.</summary>
+    IReadOnlyList<string> ErrorTail { get; }
+
     Task WaitForExitAsync(CancellationToken cancellationToken = default);
 }
 
 public interface IProcessLauncher
 {
+    /// <summary>
+    /// Starts a process. With captureErrors its error stream is kept for the
+    /// handle - only for a process the starter outlives: a write to a pipe nobody
+    /// reads any more can kill the writer.
+    /// </summary>
     IProcessHandle? Start(string executablePath, IReadOnlyList<string> arguments, string? workingDirectory,
-                          out string? error);
+                          bool captureErrors, out string? error);
 
     bool OpenInShell(string pathOrUrl);
 

StarterNG/Application/SimulatorExit.cs

@@ -0,0 +1,58 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Text.RegularExpressions;
+
+namespace StarterNG.Application;
+
+/// <summary>How a simulator the starter launched has ended.</summary>
+public sealed record SimulatorExit(int ExitCode, TimeSpan RunTime, IReadOnlyList<string> Errors)
+{
+    /// <summary>Gone before anyone could have played - however clean the exit code.</summary>
+    public static readonly TimeSpan QuickExit = TimeSpan.FromSeconds(10);
+
+    public bool Quick => RunTime < QuickExit;
+
+    public bool Failed => ExitCode != 0 || Quick;
+
+    /// <summary>The error stream without its colour codes and blank lines.</summary>
+    public IReadOnlyList<string> CleanErrors =>
+        Errors.Select(line => AnsiCode.Replace(line, "").TrimEnd())
+              .Where(line => line.Length > 0)
+              .ToList();
+
+    /// <summary>The exit code, with what it means where it is a known way to die.</summary>
+    public string DescribeCode()
+    {
+        // Linux reports a death by signal as 128 plus the signal.
+        string? meaning = ExitCode switch
+        {
+            132 => "SIGILL",
+            134 => "SIGABRT",
+            135 => "SIGBUS",
+            136 => "SIGFPE",
+            137 => "SIGKILL",
+            139 => "SIGSEGV",
+            143 => "SIGTERM",
+            _ => null
+        };
+
+        // Windows reports an unhandled exception as its NTSTATUS.
+        meaning ??= unchecked((uint)ExitCode) switch
+        {
+            0xC0000005 => "access violation",
+            0xC00000FD => "stack overflow",
+            0xC0000374 => "heap corruption",
+            0xC0000409 => "stack buffer overrun",
+            _ => null
+        };
+
+        string code = ExitCode < 0
+            ? "0x" + unchecked((uint)ExitCode).ToString("X8", CultureInfo.InvariantCulture)
+            : ExitCode.ToString(CultureInfo.InvariantCulture);
+        return meaning is null ? code : $"{code} ({meaning})";
+    }
+
+    private static readonly Regex AnsiCode = new(@"\x1B\[[0-9;]*m");
+}

StarterNG/Application/StartSimulation.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text;
@@ -62,6 +63,29 @@ public sealed class StartSimulation
         _log = log;
     }
 
+    /// <summary>The end of the simulator's own log, for when it died without a word on its error stream.</summary>
+    public IReadOnlyList<string> SimulatorLogTail(int count)
+    {
+        string path = _paths.FromRoot("log.txt");
+        try
+        {
+            if (!_files.FileExists(path))
+                return Array.Empty<string>();
+
+            var lines = LegacyText.Decode(_files.ReadAllBytes(path))
+                .Split('\n')
+                .Select(line => line.TrimEnd('\r'))
+                .Where(line => line.Trim().Length > 0)
+                .ToList();
+            return lines.Skip(Math.Max(0, lines.Count - count)).ToList();
+        }
+        catch (Exception ex)
+        {
+            _log.Log($"reading {path}", ex);
+            return Array.Empty<string>();
+        }
+    }
+
     public static string? StartableVehicle(Trainset? trainset, string? preferred)
     {
         if (trainset is null || trainset.Vehicles.Count == 0)
@@ -139,7 +163,10 @@ public sealed class StartSimulation
         // The installation, not the folder of the binary: a simulator picked from a
         // build tree lives beside the data, and the data is what it opens.
         var (program, programArguments) = _executables.LaunchCommand(executable, arguments);
-        var process = _processes.Start(program, programArguments, _paths.Root, out string? error);
+        // Its error stream is kept for a report on a crash - unless the starter closes
+        // now, leaving nobody to read it or to report.
+        bool watched = !_settings.Settings.AutoCloseStarter;
+        var process = _processes.Start(program, programArguments, _paths.Root, watched, out string? error);
         if (process is null)
             return new SimulationStartResult(SimulationStartOutcome.LaunchFailed, ExecutablePath: executable,
                                              Detail: error);

StarterNG/Infrastructure/Adapters/SystemProcessLauncher.cs

@@ -18,29 +18,38 @@ public sealed class SystemProcessLauncher : IProcessLauncher
     }
 
     public IProcessHandle? Start(string executablePath, IReadOnlyList<string> arguments, string? workingDirectory,
-                                 out string? error)
+                                 bool captureErrors, out string? error)
     {
         var info = new ProcessStartInfo
         {
             FileName = executablePath,
             WorkingDirectory = workingDirectory ?? string.Empty,
-            UseShellExecute = false
+            UseShellExecute = false,
+            RedirectStandardError = captureErrors
         };
         foreach (string argument in arguments)
             info.ArgumentList.Add(argument);
 
         try
         {
-            var process = Process.Start(info);
-            if (process is null)
+            var process = new Process { StartInfo = info };
+            var handle = new ProcessHandle(process, started: true);
+            if (captureErrors)
+                process.ErrorDataReceived += (_, e) => handle.AddError(e.Data);
+
+            if (!process.Start())
             {
                 error = executablePath;
                 _log.Log($"start {executablePath}: process not created");
                 return null;
             }
 
+            // Drained as it comes: a full pipe would stall the simulator.
+            if (captureErrors)
+                process.BeginErrorReadLine();
+
             error = null;
-            return new ProcessHandle(process);
+            return handle;
         }
         catch (Exception ex)
         {
@@ -69,7 +78,7 @@ public sealed class SystemProcessLauncher : IProcessLauncher
         try
         {
             var process = Process.GetProcessesByName(processName).FirstOrDefault();
-            return process is null ? null : new ProcessHandle(process);
+            return process is null ? null : new ProcessHandle(process, started: false);
         }
         catch (Exception ex)
         {
@@ -80,11 +89,16 @@ public sealed class SystemProcessLauncher : IProcessLauncher
 
     private sealed class ProcessHandle : IProcessHandle
     {
-        private readonly Process _process;
+        private const int KeptErrorLines = 20;
 
-        public ProcessHandle(Process process)
+        private readonly Process _process;
+        private readonly bool _started;
+        private readonly Queue<string> _errors = new();
+
+        public ProcessHandle(Process process, bool started)
         {
             _process = process;
+            _started = started;
         }
 
         public bool HasExited
@@ -96,6 +110,39 @@ public sealed class SystemProcessLauncher : IProcessLauncher
             }
         }
 
+        // Only a process started here has an exit code to read; one merely found
+        // running throws on it.
+        public int? ExitCode
+        {
+            get
+            {
+                try { return _started && _process.HasExited ? _process.ExitCode : null; }
+                catch { return null; }
+            }
+        }
+
+        public IReadOnlyList<string> ErrorTail
+        {
+            get
+            {
+                lock (_errors)
+                    return _errors.ToList();
+            }
+        }
+
+        public void AddError(string? line)
+        {
+            if (line is null)
+                return;
+
+            lock (_errors)
+            {
+                _errors.Enqueue(line);
+                if (_errors.Count > KeptErrorLines)
+                    _errors.Dequeue();
+            }
+        }
+
         public Task WaitForExitAsync(CancellationToken cancellationToken = default) =>
             _process.WaitForExitAsync(cancellationToken);
     }

StarterNG/MainWindow.axaml.cs

@@ -531,6 +531,7 @@ public partial class MainWindow : Window
     /// </summary>
     private void WatchSimulator(IProcessHandle? simulator)
     {
+        var started = DateTime.UtcNow;
         simulator ??= AppServices.Current.Processes.FindRunning(
             Path.GetFileNameWithoutExtension(AppServices.Current.SettingsStore.ResolveExecutable()));
 
@@ -540,8 +541,46 @@ public partial class MainWindow : Window
             return;
         }
 
-        _ = simulator.WaitForExitAsync().ContinueWith(
-            _ => Dispatcher.UIThread.Post(RestoreFromSimulator), TaskScheduler.Default);
+        _ = simulator.WaitForExitAsync().ContinueWith(_ => Dispatcher.UIThread.Post(() =>
+        {
+            RestoreFromSimulator();
+            ReportIfCrashed(simulator, DateTime.UtcNow - started);
+        }), TaskScheduler.Default);
+    }
+
+    /// <summary>
+    /// Says why the simulator is gone when it did not end the way a session does:
+    /// an error code, or a close so soon after the start that nobody played - a
+    /// window that only flashed. Its own words come from the error stream, or from
+    /// its log when that stayed silent.
+    /// </summary>
+    private async void ReportIfCrashed(IProcessHandle simulator, TimeSpan runTime)
+    {
+        if (simulator.ExitCode is not { } code)
+            return;
+
+        var exit = new SimulatorExit(code, runTime, simulator.ErrorTail);
+        if (!exit.Failed)
+            return;
+
+        var words = exit.CleanErrors.Count > 0
+            ? exit.CleanErrors
+            : AppServices.Current.StartSimulation.SimulatorLogTail(12);
+
+        var message = new StringBuilder();
+        message.AppendLine(string.Format(App.Loc["SimExitCode"], exit.DescribeCode()));
+        if (exit.Quick)
+            message.AppendLine(string.Format(App.Loc["SimExitQuick"], runTime.TotalSeconds));
+        if (words.Count > 0)
+        {
+            message.AppendLine().AppendLine(App.Loc["SimExitMessages"]);
+            foreach (string line in words)
+                message.AppendLine(line);
+        }
+        message.AppendLine().Append(App.Loc["SimExitSeeLog"]);
+
+        Diagnostics.Log(message.ToString().Replace(Environment.NewLine, " | "));
+        await MessageBox.Show(this, message.ToString(), App.Loc["SimExitTitle"], MessageBoxButtons.Ok);
     }
 
     private async void CheckExternalSettings()

StarterNG/startercfg/lang/cz.xml

@@ -253,6 +253,11 @@
     <String key="ExeNotFound">Spustitelný soubor simulátoru nebyl nalezen.</String>
     <String key="ExeNotExecutable">Soubor není označen jako spustitelný (chmod +x).</String>
     <String key="ExeWrongPlatform">Tento binární soubor je pro jiný operační systém. Na Linuxu může verze pro Windows běžet přes Wine nebo Proton - pokud je používáte, nejde o chybu.</String>
+    <String key="SimExitTitle">Simulátor se neočekávaně ukončil</String>
+    <String key="SimExitCode">Návratový kód: {0}</String>
+    <String key="SimExitQuick">Ukončil se {0:0.0} s po spuštění.</String>
+    <String key="SimExitMessages">Poslední zprávy:</String>
+    <String key="SimExitSeeLog">Celý průběh je v souboru log.txt ve složce simulátoru.</String>
     <String key="TabInfo">Informace</String>
     <String key="SettingsFileChangedTitle">Nastavení</String>
     <String key="SettingsFileChanged">Soubor eu07.ini byl změněn mimo starter. Načíst jej znovu?</String>

StarterNG/startercfg/lang/en.xml

@@ -253,6 +253,11 @@
     <String key="ExeNotFound">The simulator executable was not found.</String>
     <String key="ExeNotExecutable">The file is not marked as executable (chmod +x).</String>
     <String key="ExeWrongPlatform">This binary is built for a different operating system. On Linux a Windows build can still run through Wine or Proton - if you use them, this is not an error.</String>
+    <String key="SimExitTitle">The simulator ended unexpectedly</String>
+    <String key="SimExitCode">Exit code: {0}</String>
+    <String key="SimExitQuick">It closed {0:0.0} s after starting.</String>
+    <String key="SimExitMessages">Last messages:</String>
+    <String key="SimExitSeeLog">The full run is in log.txt in the simulator folder.</String>
     <String key="TabInfo">Info</String>
     <String key="SettingsFileChangedTitle">Settings</String>
     <String key="SettingsFileChanged">eu07.ini was changed outside the starter. Reload it?</String>

StarterNG/startercfg/lang/hu.xml

@@ -253,6 +253,11 @@
     <String key="ExeNotFound">A szimulátor futtatható fájlja nem található.</String>
     <String key="ExeNotExecutable">A fájl nincs futtathatóként megjelölve (chmod +x).</String>
     <String key="ExeWrongPlatform">Ez a bináris másik operációs rendszerhez készült. Linuxon a Windows-verzió Wine vagy Proton segítségével futtatható - ha ezeket használod, ez nem hiba.</String>
+    <String key="SimExitTitle">A szimulátor váratlanul leállt</String>
+    <String key="SimExitCode">Kilépési kód: {0}</String>
+    <String key="SimExitQuick">{0:0.0} másodperccel az indítás után bezárult.</String>
+    <String key="SimExitMessages">Utolsó üzenetek:</String>
+    <String key="SimExitSeeLog">A teljes futás a szimulátor mappájában lévő log.txt fájlban található.</String>
     <String key="TabInfo">Információ</String>
     <String key="SettingsFileChangedTitle">Beállítások</String>
     <String key="SettingsFileChanged">Az eu07.ini fájl a starteren kívül módosult. Újratölti?</String>

StarterNG/startercfg/lang/pl.xml

@@ -253,6 +253,11 @@
     <String key="ExeNotFound">Nie znaleziono pliku wykonywalnego symulatora.</String>
     <String key="ExeNotExecutable">Plik nie ma prawa wykonywania (chmod +x).</String>
     <String key="ExeWrongPlatform">To jest plik wykonywalny dla innego systemu niż uruchomiony. Na Linuksie wersja dla Windows może działać przez Wine lub Proton - jeśli ich używasz, to nie jest błąd.</String>
+    <String key="SimExitTitle">Symulator zakończył pracę nieoczekiwanie</String>
+    <String key="SimExitCode">Kod wyjścia: {0}</String>
+    <String key="SimExitQuick">Zamknął się {0:0.0} s po uruchomieniu.</String>
+    <String key="SimExitMessages">Ostatnie komunikaty:</String>
+    <String key="SimExitSeeLog">Pełny przebieg jest w pliku log.txt w katalogu symulatora.</String>
     <String key="TabInfo">Informacje</String>
     <String key="SettingsFileChangedTitle">Ustawienia</String>
     <String key="SettingsFileChanged">Plik eu07.ini został zmieniony poza starterem. Wczytać ponownie?</String>

StarterNG/startercfg/lang/ru.xml

@@ -253,6 +253,11 @@
     <String key="ExeNotFound">Исполняемый файл симулятора не найден.</String>
     <String key="ExeNotExecutable">Файл не помечен как исполняемый (chmod +x).</String>
     <String key="ExeWrongPlatform">Этот исполняемый файл собран для другой операционной системы. В Linux версия для Windows может работать через Wine или Proton - если вы их используете, это не ошибка.</String>
+    <String key="SimExitTitle">Симулятор неожиданно завершил работу</String>
+    <String key="SimExitCode">Код выхода: {0}</String>
+    <String key="SimExitQuick">Он закрылся через {0:0.0} с после запуска.</String>
+    <String key="SimExitMessages">Последние сообщения:</String>
+    <String key="SimExitSeeLog">Полный журнал находится в файле log.txt в папке симулятора.</String>
     <String key="TabInfo">Информация</String>
     <String key="SettingsFileChangedTitle">Настройки</String>
     <String key="SettingsFileChanged">Файл eu07.ini изменён вне стартера. Загрузить заново?</String>