diff --git a/CMakeLists.txt b/CMakeLists.txt index d128ed2f..552bc40d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -195,6 +195,7 @@ set(SOURCES "network/manager.cpp" "network/entities.cpp" "network/session.cpp" +"network/statehash.cpp" "network/backend/asio.cpp" "widgets/vehiclelist.cpp" diff --git a/application/application.cpp b/application/application.cpp index b2434b50..05babab6 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -30,6 +30,7 @@ http://mozilla.org/MPL/2.0/. #include "utilities/Timer.h" #include "utilities/dictionary.h" #include "version_info.h" +#include "network/statehash.h" #include #include "utilities/translation.h" @@ -422,20 +423,6 @@ int eu07_application::init(int Argc, char *Argv[]) return result; } -double eu07_application::generate_sync() -{ - if (Timer::GetDeltaTime() == 0.0) - return 0.0; - double sync = 0.0; - for (const TDynamicObject *vehicle : simulation::Vehicles.sequence()) - { - auto const pos{vehicle->GetPosition()}; - sync += pos.x + pos.y + pos.z; - } - sync += Random(1.0, 100.0); - return sync; -} - void eu07_application::queue_quit(bool direct) { // a network client leaving is its own business; replicating the request would shut @@ -507,7 +494,7 @@ int eu07_application::run() { command_queue::commands_map commands_to_exec; command_queue::commands_map local_commands = simulation::Commands.pop_intercept_queue(); - double slave_sync; + network::frame_delta authoritative; // if we're the server if (m_network && m_network->servers) @@ -527,18 +514,20 @@ int eu07_application::run() if (m_network && m_network->client) { // fetch frame info from network layer, - auto frame_info = m_network->client->get_next_delta(MAX_NETWORK_PER_FRAME - loop_remaining); + authoritative = m_network->client->get_next_delta(MAX_NETWORK_PER_FRAME - loop_remaining); // use delta and commands received from master - double delta = std::get<0>(frame_info); - Timer::set_delta_override(delta); - slave_sync = std::get<1>(frame_info); - add_to_dequemap(commands_to_exec, std::get<2>(frame_info)); + Timer::set_delta_override(authoritative.dt); + add_to_dequemap(commands_to_exec, authoritative.commands); + + // the authority owns the timeline, so we take its step number as ours + if (authoritative.valid) + Global.simulation_tick = authoritative.tick; // and send our local commands to master m_network->client->send_commands(local_commands); - if (delta == 0.0) + if (!authoritative.valid) loop_remaining = -1; } // if we're master @@ -547,6 +536,9 @@ int eu07_application::run() // just push local commands to execution add_to_dequemap(commands_to_exec, local_commands); + // we are the one counting the steps of this world + ++Global.simulation_tick; + loop_remaining = -1; } @@ -560,26 +552,51 @@ int eu07_application::run() // update continuous commands simulation::Commands.update(); - double sync = generate_sync(); + auto const statehash = (m_network ? network::state_hash() : 0); // if we're the server if (m_network && m_network->servers) { - // send delta, sync, and commands we just executed to clients + // send delta, state digest, and commands we just executed to clients double delta = Timer::GetDeltaTime(); double render = Timer::GetDeltaRenderTime(); - m_network->servers->push_delta(render, delta, sync, commands_to_exec); + m_network->servers->push_delta(render, delta, Global.simulation_tick, statehash, commands_to_exec); } // if we're slave if (m_network && m_network->client) { - // verify sync - if (sync != slave_sync) + if (authoritative.valid) { - WriteLog("net: desync! calculated: " + std::to_string(sync) + ", received: " + std::to_string(slave_sync), logtype::net); - - Global.desync = slave_sync - sync; + if (authoritative.state_hash_version != network::STATE_HASH_VERSION) + { + // nothing sensible to compare; say so once and stop pretending + if (m_statemismatches == 0) + { + ErrorLog("net: state digest version mismatch, desync detection is off", logtype::net); + m_statemismatches = 1; + } + } + else if (statehash != authoritative.state_hash) + { + ++m_statemismatches; + // a drift usually persists, so this is reported when it starts + // and then only now and then, instead of every single step + if (m_statemismatches == 1 || (m_statemismatches % 300) == 0) + { + WriteLog("net: state mismatch at tick " + std::to_string(authoritative.tick) + " (" + std::to_string(m_statemismatches) + + " steps): local " + std::to_string(statehash) + ", authoritative " + std::to_string(authoritative.state_hash), + logtype::net); + } + Global.desync = (float)m_statemismatches; + } + else + { + if (m_statemismatches != 0) + WriteLog("net: state back in step with the authority at tick " + std::to_string(authoritative.tick), logtype::net); + m_statemismatches = 0; + Global.desync = 0.0f; + } } // set total delta for rendering code diff --git a/application/application.h b/application/application.h index 99cc6565..cedf5753 100644 --- a/application/application.h +++ b/application/application.h @@ -84,9 +84,6 @@ public: window( int const Windowindex = 0, bool visible = false, int width = 1, int height = 1, GLFWmonitor *monitor = nullptr, bool keep_ownership = true, bool share_ctx = true ); GLFWmonitor * find_monitor( const std::string &str ) const; std::string describe_monitor( GLFWmonitor *monitor ) const; - // generate network sync verification number - double - generate_sync(); void queue_quit(bool direct); bool @@ -123,6 +120,8 @@ private: // members bool m_screenshot_queued = false; + // consecutive authoritative steps whose state digest did not match ours + std::uint64_t m_statemismatches { 0 }; modeptr_array m_modes { nullptr }; // collection of available application behaviour modes mode_stack m_modestack; // current behaviour mode diff --git a/application/driveruilayer.cpp b/application/driveruilayer.cpp index 6999be42..48d7d871 100644 --- a/application/driveruilayer.cpp +++ b/application/driveruilayer.cpp @@ -282,7 +282,7 @@ void driver_ui::render_() { ImGui::SetNextWindowSize(ImVec2(-1, -1)); if (ImGui::Begin("network", nullptr, ImGuiWindowFlags_NoCollapse)) - ImGui::Text("desync: %0.2f", Global.desync); + ImGui::Text("out of step with the server for %d ticks (at %llu)", (int)Global.desync, (unsigned long long)Global.simulation_tick); ImGui::End(); } } diff --git a/network/manager.cpp b/network/manager.cpp index 6a64f97b..07b0a610 100644 --- a/network/manager.cpp +++ b/network/manager.cpp @@ -12,6 +12,7 @@ http://mozilla.org/MPL/2.0/. #include "simulation/simulation.h" #include "utilities/Logs.h" #include "utilities/Globals.h" +#include "network/statehash.h" network::server_manager::server_manager() { @@ -28,7 +29,7 @@ command_queue::commands_map network::server_manager::pop_commands() return map; } -void network::server_manager::push_delta(double render_dt, double dt, double sync, const command_queue::commands_map &commands) +void network::server_manager::push_delta(double render_dt, double dt, uint64_t tick, uint64_t state_hash, const command_queue::commands_map &commands) { if (dt == 0.0 && commands.empty()) return; @@ -36,7 +37,9 @@ void network::server_manager::push_delta(double render_dt, double dt, double syn frame_info msg; msg.render_dt = render_dt; msg.dt = dt; - msg.sync = sync; + msg.tick = tick; + msg.state_hash = state_hash; + msg.state_hash_version = STATE_HASH_VERSION; msg.commands = commands; for (auto srv : servers) diff --git a/network/manager.h b/network/manager.h index a530fa64..551c24c7 100644 --- a/network/manager.h +++ b/network/manager.h @@ -14,7 +14,7 @@ namespace network public: server_manager(); - void push_delta(double render_dt, double dt, double sync, const command_queue::commands_map &commands); + void push_delta(double render_dt, double dt, uint64_t tick, uint64_t state_hash, const command_queue::commands_map &commands); command_queue::commands_map pop_commands(); void create_server(const std::string &backend, const std::string &conf); // refreshes the authoritative vehicle roster and publishes it when it changed diff --git a/network/message.cpp b/network/message.cpp index 29935348..2b00f314 100644 --- a/network/message.cpp +++ b/network/message.cpp @@ -143,6 +143,7 @@ void network::crew_update::deserialize(std::istream &stream) void ::network::request_command::serialize(std::ostream &stream) const { + sn_utils::ls_uint64(stream, tick); sn_utils::ls_uint32(stream, commands.size()); for (auto const &kv : commands) { @@ -167,6 +168,8 @@ void ::network::request_command::serialize(std::ostream &stream) const void network::request_command::deserialize(std::istream &stream) { + tick = sn_utils::ld_uint64(stream); + uint32_t commands_size = sn_utils::ld_uint32(stream); for (uint32_t i = 0; i < commands_size; i++) { @@ -200,7 +203,8 @@ void network::frame_info::serialize(std::ostream &stream) const { sn_utils::ls_float64(stream, render_dt); sn_utils::ls_float64(stream, dt); - sn_utils::ls_float64(stream, sync); + sn_utils::ls_uint64(stream, state_hash); + sn_utils::ls_uint32(stream, state_hash_version); request_command::serialize(stream); } @@ -209,7 +213,8 @@ void network::frame_info::deserialize(std::istream &stream) { render_dt = sn_utils::ld_float64(stream); dt = sn_utils::ld_float64(stream); - sync = sn_utils::ld_float64(stream); + state_hash = sn_utils::ld_uint64(stream); + state_hash_version = sn_utils::ld_uint32(stream); request_command::deserialize(stream); } diff --git a/network/message.h b/network/message.h index 61602162..fc304b07 100644 --- a/network/message.h +++ b/network/message.h @@ -148,6 +148,9 @@ struct request_command : public message request_command() : message(REQUEST_COMMAND) {} command_queue::commands_map commands; + // logical step the sender was on when it produced these. the protocol is anchored to + // this counter rather than to whatever frame the renderer happened to be drawing + uint64_t tick{ 0 }; virtual void serialize(std::ostream &stream) const override; virtual void deserialize(std::istream &stream) override; @@ -159,7 +162,9 @@ struct frame_info : public request_command double render_dt; double dt; - double sync; + // digest of the authoritative world after this step; a client compares its own + uint64_t state_hash{ 0 }; + uint32_t state_hash_version{ 0 }; virtual void serialize(std::ostream &stream) const override; virtual void deserialize(std::istream &stream) override; diff --git a/network/network.cpp b/network/network.cpp index 945dd3c8..e7ee6182 100644 --- a/network/network.cpp +++ b/network/network.cpp @@ -12,7 +12,9 @@ // 2 - legacy lockstep protocol // 3 - handshake carries build identification and an explicit rejection message -std::uint32_t const EU07_NETWORK_VERSION = 3; +// 4 - peer identity, vehicle roster and crew handshake, command source, +// logical simulation tick and a versioned state digest instead of the position sum +std::uint32_t const EU07_NETWORK_VERSION = 4; namespace network { @@ -279,7 +281,7 @@ void network::client::update() } // client -std::tuple network::client::get_next_delta(int counter) +network::frame_delta network::client::get_next_delta(int counter) { auto now = std::chrono::high_resolution_clock::now(); if (counter == 1) { @@ -289,8 +291,7 @@ std::tuple network::client::get_nex if (delta_queue.empty()) { // buffer underflow - return std::tuple(0.0, 0.0, command_queue::commands_map()); + return frame_delta(); } @@ -316,13 +317,20 @@ std::tuple network::client::get_nex consume_counter = std::clamp(consume_counter - mult, -MAX_BUFFER_SIZE, MAX_BUFFER_SIZE); } + frame_delta delta; + delta.dt = entry.dt; + delta.tick = entry.tick; + delta.state_hash = entry.state_hash; + delta.state_hash_version = entry.state_hash_version; + delta.commands = entry.commands; + delta.valid = true; + delta_queue.pop(); - return std::make_tuple(entry.dt, entry.sync, entry.commands); + return delta; } else { // nothing to push - return std::tuple(0.0, 0.0, command_queue::commands_map()); + return frame_delta(); } } @@ -333,6 +341,7 @@ void network::client::send_commands(command_queue::commands_map commands) // eh, maybe queue lost messages request_command msg; + msg.tick = Global.simulation_tick; msg.commands = commands; conn->send_message(msg); diff --git a/network/network.h b/network/network.h index c681e6fd..28479cd0 100644 --- a/network/network.h +++ b/network/network.h @@ -79,6 +79,18 @@ namespace network command_queue::commands_map pop_commands(); }; + // one authoritative simulation step handed to the client + struct frame_delta + { + double dt { 0.0 }; + uint64_t tick { 0 }; + uint64_t state_hash { 0 }; + uint32_t state_hash_version { 0 }; + command_queue::commands_map commands; + // false when there was nothing ready to consume in this pass + bool valid { false }; + }; + class client { protected: @@ -108,7 +120,7 @@ namespace network public: void update(); - std::tuple get_next_delta(int counter); + frame_delta get_next_delta(int counter); void send_commands(command_queue::commands_map commands); // lobby requests; the server is the one that decides void send_claim(NetworkEntityId entity_id); diff --git a/network/statehash.cpp b/network/statehash.cpp new file mode 100644 index 00000000..f5a32a6c --- /dev/null +++ b/network/statehash.cpp @@ -0,0 +1,89 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#include "stdafx.h" +#include "network/statehash.h" + +#include "McZapkie/MOVER.h" +#include "simulation/simulation.h" +#include "simulation/simulationtime.h" +#include "vehicle/DynObj.h" + +namespace +{ + +uint64_t const FNV_OFFSET{14695981039346656037ull}; +uint64_t const FNV_PRIME{1099511628211ull}; + +uint64_t fold(uint64_t Hash, uint64_t Value) +{ + for (int i = 0; i < 8; ++i) + { + Hash ^= (Value >> (i * 8)) & 0xff; + Hash *= FNV_PRIME; + } + return Hash; +} + +uint64_t fold(uint64_t Hash, std::string const &Text) +{ + for (char const character : Text) + { + Hash ^= (uint64_t)(unsigned char)character; + Hash *= FNV_PRIME; + } + return Hash; +} + +// rounding to a fixed step keeps the digest from reacting to the last bits of a float +int64_t quantize(double Value, double Step) +{ + if (!std::isfinite(Value)) + return std::numeric_limits::min(); + + return (int64_t)std::llround(Value / Step); +} + +} // namespace + +uint64_t network::state_hash() +{ + uint64_t digest{0}; + uint64_t counted{0}; + + for (TDynamicObject const *vehicle : simulation::Vehicles.sequence()) + { + if (vehicle == nullptr || vehicle->MoverParameters == nullptr) + continue; + + auto const &mover = *vehicle->MoverParameters; + auto const position = vehicle->GetPosition(); + + uint64_t entry{FNV_OFFSET}; + entry = fold(entry, vehicle->name()); + entry = fold(entry, (uint64_t)quantize(position.x, 0.01)); + entry = fold(entry, (uint64_t)quantize(position.y, 0.01)); + entry = fold(entry, (uint64_t)quantize(position.z, 0.01)); + entry = fold(entry, (uint64_t)quantize(mover.V, 0.001)); + entry = fold(entry, (uint64_t)(int64_t)mover.DirActive); + entry = fold(entry, (uint64_t)quantize(mover.PipePress, 0.001)); + entry = fold(entry, (uint64_t)quantize(mover.BrakePress, 0.001)); + + // summing keeps the whole thing independent of iteration order + digest += entry; + ++counted; + } + + digest = fold(digest, counted); + + auto const &time = simulation::Time.data(); + digest = fold(digest, (uint64_t)(time.wHour * 3600 + time.wMinute * 60 + time.wSecond)); + + return digest; +} diff --git a/network/statehash.h b/network/statehash.h new file mode 100644 index 00000000..6b78e3be --- /dev/null +++ b/network/statehash.h @@ -0,0 +1,32 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +#include + +namespace network +{ + +// what goes into the digest. bump it whenever the layout changes, so that two builds +// cannot silently compare digests that mean different things +constexpr uint32_t STATE_HASH_VERSION = 1; + +// digest of the gameplay critical state of the world, used to notice that two peers have +// drifted apart. deliberately order independent: every vehicle is folded in under its own +// identity and the results are summed, so the order the scenario happened to create them +// in does not matter. values are quantized, so that harmless float noise does not read as +// a desync +// +// version 1 covers vehicle position, speed, direction and the two brake pressures, plus +// the simulation clock. positions of switches, memcells and the event queue are the +// obvious next candidates - they go in when the snapshot work of stage 7 defines them +uint64_t state_hash(); + +} // namespace network diff --git a/utilities/Globals.h b/utilities/Globals.h index 23f02b47..96b627f4 100644 --- a/utilities/Globals.h +++ b/utilities/Globals.h @@ -362,6 +362,10 @@ struct global_settings { bool network_leave_pending = false; // last answer of the server to a lobby request, shown in the lobby std::string network_lobby_message; + // logical simulation step. the authority counts it up, a client takes it from the + // authoritative frame it is executing; the network timeline hangs off this, not off + // the number of frames the renderer happened to draw + uint64_t simulation_tick = 0; std::unordered_map trainset_overrides;