multiplayer stage 6: a logical simulation tick and a real state digest

The protocol hung off whatever frame the renderer happened to be drawing, and
the only thing standing in for synchronisation was the sum of every vehicle's
coordinates with a random number added on top - a value that cannot be equal on
two machines and therefore reported a desync on every single step.

- Global.simulation_tick: the authority counts the steps, a client takes the
  number from the authoritative frame it is executing. FRAME_INFO and the
  command requests carry it, so the network timeline has one unambiguous order
  independent of frame rates
- network/statehash: a versioned FNV digest of the gameplay critical state -
  vehicle position, speed, direction, brake pipe and cylinder pressure, plus the
  clock. Values are quantized so that float noise is not mistaken for drift, and
  vehicles are folded in under their own identity and summed, so the digest does
  not depend on the order the scenario created them in. Switches, memcells and
  the event queue are the next candidates; they go in when stage 7 pins down
  what a snapshot contains
- a client compares its own digest against the authoritative one and reports the
  drift when it starts and then only now and then, rather than every step; the
  overlay says how long it has been out of step and at which tick
- refusing to compare at all when the two builds disagree on the digest layout

EU07_NETWORK_VERSION is now 4.

CMakeLists.txt

@@ -195,6 +195,7 @@ set(SOURCES
 "network/manager.cpp"
 "network/entities.cpp"
 "network/session.cpp"
+"network/statehash.cpp"
 "network/backend/asio.cpp"
 
 "widgets/vehiclelist.cpp"

application/application.cpp

@@ -30,6 +30,7 @@ http://mozilla.org/MPL/2.0/.
 #include "utilities/Timer.h"
 #include "utilities/dictionary.h"
 #include "version_info.h"
+#include "network/statehash.h"
 #include <chrono>
 #include "utilities/translation.h"
 
@@ -422,20 +423,6 @@ int eu07_application::init(int Argc, char *Argv[])
 	return result;
 }
 
-double eu07_application::generate_sync()
-{
-	if (Timer::GetDeltaTime() == 0.0)
-		return 0.0;
-	double sync = 0.0;
-	for (const TDynamicObject *vehicle : simulation::Vehicles.sequence())
-	{
-		auto const pos{vehicle->GetPosition()};
-		sync += pos.x + pos.y + pos.z;
-	}
-	sync += Random(1.0, 100.0);
-	return sync;
-}
-
 void eu07_application::queue_quit(bool direct)
 {
 	// a network client leaving is its own business; replicating the request would shut
@@ -507,7 +494,7 @@ int eu07_application::run()
 			{
 				command_queue::commands_map commands_to_exec;
 				command_queue::commands_map local_commands = simulation::Commands.pop_intercept_queue();
-				double slave_sync;
+				network::frame_delta authoritative;
 
 				// if we're the server
 				if (m_network && m_network->servers)
@@ -527,18 +514,20 @@ int eu07_application::run()
 				if (m_network && m_network->client)
 				{
 					// fetch frame info from network layer,
-					auto frame_info = m_network->client->get_next_delta(MAX_NETWORK_PER_FRAME - loop_remaining);
+					authoritative = m_network->client->get_next_delta(MAX_NETWORK_PER_FRAME - loop_remaining);
 
 					// use delta and commands received from master
-					double delta = std::get<0>(frame_info);
-					Timer::set_delta_override(delta);
-					slave_sync = std::get<1>(frame_info);
-					add_to_dequemap(commands_to_exec, std::get<2>(frame_info));
+					Timer::set_delta_override(authoritative.dt);
+					add_to_dequemap(commands_to_exec, authoritative.commands);
+
+					// the authority owns the timeline, so we take its step number as ours
+					if (authoritative.valid)
+						Global.simulation_tick = authoritative.tick;
 
 					// and send our local commands to master
 					m_network->client->send_commands(local_commands);
 
-					if (delta == 0.0)
+					if (!authoritative.valid)
 						loop_remaining = -1;
 				}
 				// if we're master
@@ -547,6 +536,9 @@ int eu07_application::run()
 					// just push local commands to execution
 					add_to_dequemap(commands_to_exec, local_commands);
 
+					// we are the one counting the steps of this world
+					++Global.simulation_tick;
+
 					loop_remaining = -1;
 				}
 
@@ -560,26 +552,51 @@ int eu07_application::run()
 				// update continuous commands
 				simulation::Commands.update();
 
-				double sync = generate_sync();
+				auto const statehash = (m_network ? network::state_hash() : 0);
 
 				// if we're the server
 				if (m_network && m_network->servers)
 				{
-					// send delta, sync, and commands we just executed to clients
+					// send delta, state digest, and commands we just executed to clients
 					double delta = Timer::GetDeltaTime();
 					double render = Timer::GetDeltaRenderTime();
-					m_network->servers->push_delta(render, delta, sync, commands_to_exec);
+					m_network->servers->push_delta(render, delta, Global.simulation_tick, statehash, commands_to_exec);
 				}
 
 				// if we're slave
 				if (m_network && m_network->client)
 				{
-					// verify sync
-					if (sync != slave_sync)
+					if (authoritative.valid)
 					{
-						WriteLog("net: desync! calculated: " + std::to_string(sync) + ", received: " + std::to_string(slave_sync), logtype::net);
-
-						Global.desync = slave_sync - sync;
+						if (authoritative.state_hash_version != network::STATE_HASH_VERSION)
+						{
+							// nothing sensible to compare; say so once and stop pretending
+							if (m_statemismatches == 0)
+							{
+								ErrorLog("net: state digest version mismatch, desync detection is off", logtype::net);
+								m_statemismatches = 1;
+							}
+						}
+						else if (statehash != authoritative.state_hash)
+						{
+							++m_statemismatches;
+							// a drift usually persists, so this is reported when it starts
+							// and then only now and then, instead of every single step
+							if (m_statemismatches == 1 || (m_statemismatches % 300) == 0)
+							{
+								WriteLog("net: state mismatch at tick " + std::to_string(authoritative.tick) + " (" + std::to_string(m_statemismatches) +
+								             " steps): local " + std::to_string(statehash) + ", authoritative " + std::to_string(authoritative.state_hash),
+								         logtype::net);
+							}
+							Global.desync = (float)m_statemismatches;
+						}
+						else
+						{
+							if (m_statemismatches != 0)
+								WriteLog("net: state back in step with the authority at tick " + std::to_string(authoritative.tick), logtype::net);
+							m_statemismatches = 0;
+							Global.desync = 0.0f;
+						}
 					}
 
 					// set total delta for rendering code

application/application.h

@@ -84,9 +84,6 @@ public:
         window( int const Windowindex = 0, bool visible = false, int width = 1, int height = 1, GLFWmonitor *monitor = nullptr, bool keep_ownership = true, bool share_ctx = true );
     GLFWmonitor * find_monitor( const std::string &str ) const;
     std::string describe_monitor( GLFWmonitor *monitor ) const;
-	// generate network sync verification number
-	double
-	    generate_sync();
 	void
         queue_quit(bool direct);
     bool
@@ -123,6 +120,8 @@ private:
 // members
 
     bool m_screenshot_queued = false;
+    // consecutive authoritative steps whose state digest did not match ours
+    std::uint64_t m_statemismatches { 0 };
 
     modeptr_array m_modes { nullptr }; // collection of available application behaviour modes
     mode_stack m_modestack; // current behaviour mode

application/driveruilayer.cpp

@@ -282,7 +282,7 @@ void driver_ui::render_()
 	{
 		ImGui::SetNextWindowSize(ImVec2(-1, -1));
 		if (ImGui::Begin("network", nullptr, ImGuiWindowFlags_NoCollapse))
-			ImGui::Text("desync: %0.2f", Global.desync);
+			ImGui::Text("out of step with the server for %d ticks (at %llu)", (int)Global.desync, (unsigned long long)Global.simulation_tick);
 		ImGui::End();
 	}
 }

network/manager.cpp

@@ -12,6 +12,7 @@ http://mozilla.org/MPL/2.0/.
 #include "simulation/simulation.h"
 #include "utilities/Logs.h"
 #include "utilities/Globals.h"
+#include "network/statehash.h"
 
 network::server_manager::server_manager()
 {
@@ -28,7 +29,7 @@ command_queue::commands_map network::server_manager::pop_commands()
 	return map;
 }
 
-void network::server_manager::push_delta(double render_dt, double dt, double sync, const command_queue::commands_map &commands)
+void network::server_manager::push_delta(double render_dt, double dt, uint64_t tick, uint64_t state_hash, const command_queue::commands_map &commands)
 {
 	if (dt == 0.0 && commands.empty())
 		return;
@@ -36,7 +37,9 @@ void network::server_manager::push_delta(double render_dt, double dt, double syn
 	frame_info msg;
 	msg.render_dt = render_dt;
 	msg.dt = dt;
-	msg.sync = sync;
+	msg.tick = tick;
+	msg.state_hash = state_hash;
+	msg.state_hash_version = STATE_HASH_VERSION;
 	msg.commands = commands;
 
 	for (auto srv : servers)

network/manager.h

@@ -14,7 +14,7 @@ namespace network
 	public:
 		server_manager();
 
-		void push_delta(double render_dt, double dt, double sync, const command_queue::commands_map &commands);
+		void push_delta(double render_dt, double dt, uint64_t tick, uint64_t state_hash, const command_queue::commands_map &commands);
 		command_queue::commands_map pop_commands();
 		void create_server(const std::string &backend, const std::string &conf);
 		// refreshes the authoritative vehicle roster and publishes it when it changed

network/message.cpp

@@ -143,6 +143,7 @@ void network::crew_update::deserialize(std::istream &stream)
 
 void ::network::request_command::serialize(std::ostream &stream) const
 {
+	sn_utils::ls_uint64(stream, tick);
 	sn_utils::ls_uint32(stream, commands.size());
 	for (auto const &kv : commands)
 	{
@@ -167,6 +168,8 @@ void ::network::request_command::serialize(std::ostream &stream) const
 
 void network::request_command::deserialize(std::istream &stream)
 {
+	tick = sn_utils::ld_uint64(stream);
+
 	uint32_t commands_size = sn_utils::ld_uint32(stream);
 	for (uint32_t i = 0; i < commands_size; i++)
 	{
@@ -200,7 +203,8 @@ void network::frame_info::serialize(std::ostream &stream) const
 {
 	sn_utils::ls_float64(stream, render_dt);
 	sn_utils::ls_float64(stream, dt);
-	sn_utils::ls_float64(stream, sync);
+	sn_utils::ls_uint64(stream, state_hash);
+	sn_utils::ls_uint32(stream, state_hash_version);
 
 	request_command::serialize(stream);
 }
@@ -209,7 +213,8 @@ void network::frame_info::deserialize(std::istream &stream)
 {
 	render_dt = sn_utils::ld_float64(stream);
 	dt = sn_utils::ld_float64(stream);
-	sync = sn_utils::ld_float64(stream);
+	state_hash = sn_utils::ld_uint64(stream);
+	state_hash_version = sn_utils::ld_uint32(stream);
 
 	request_command::deserialize(stream);
 }

network/message.h

@@ -148,6 +148,9 @@ struct request_command : public message
 	request_command() : message(REQUEST_COMMAND) {}
 
 	command_queue::commands_map commands;
+	// logical step the sender was on when it produced these. the protocol is anchored to
+	// this counter rather than to whatever frame the renderer happened to be drawing
+	uint64_t tick{ 0 };
 
 	virtual void serialize(std::ostream &stream) const override;
 	virtual void deserialize(std::istream &stream) override;
@@ -159,7 +162,9 @@ struct frame_info : public request_command
 
 	double render_dt;
 	double dt;
-	double sync;
+	// digest of the authoritative world after this step; a client compares its own
+	uint64_t state_hash{ 0 };
+	uint32_t state_hash_version{ 0 };
 
 	virtual void serialize(std::ostream &stream) const override;
 	virtual void deserialize(std::istream &stream) override;

network/network.cpp

@@ -12,7 +12,9 @@
 
 // 2 - legacy lockstep protocol
 // 3 - handshake carries build identification and an explicit rejection message
-std::uint32_t const EU07_NETWORK_VERSION = 3;
+// 4 - peer identity, vehicle roster and crew handshake, command source,
+//     logical simulation tick and a versioned state digest instead of the position sum
+std::uint32_t const EU07_NETWORK_VERSION = 4;
 
 namespace network {
 
@@ -279,7 +281,7 @@ void network::client::update()
 }
 
 // client
-std::tuple<double, double, command_queue::commands_map> network::client::get_next_delta(int counter)
+network::frame_delta network::client::get_next_delta(int counter)
 {
 	auto now = std::chrono::high_resolution_clock::now();
 	if (counter == 1) {
@@ -289,8 +291,7 @@ std::tuple<double, double, command_queue::commands_map> network::client::get_nex
 
 	if (delta_queue.empty()) {
 		// buffer underflow
-		return std::tuple<double, double,
-		        command_queue::commands_map>(0.0, 0.0, command_queue::commands_map());
+		return frame_delta();
 	}
 
 
@@ -316,13 +317,20 @@ std::tuple<double, double, command_queue::commands_map> network::client::get_nex
 			consume_counter = std::clamp(consume_counter - mult, -MAX_BUFFER_SIZE, MAX_BUFFER_SIZE);
 		}
 
+		frame_delta delta;
+		delta.dt = entry.dt;
+		delta.tick = entry.tick;
+		delta.state_hash = entry.state_hash;
+		delta.state_hash_version = entry.state_hash_version;
+		delta.commands = entry.commands;
+		delta.valid = true;
+
 		delta_queue.pop();
 
-		return std::make_tuple(entry.dt, entry.sync, entry.commands);
+		return delta;
 	} else {
 		// nothing to push
-		return std::tuple<double, double,
-		        command_queue::commands_map>(0.0, 0.0, command_queue::commands_map());
+		return frame_delta();
 	}
 }
 
@@ -333,6 +341,7 @@ void network::client::send_commands(command_queue::commands_map commands)
 	// eh, maybe queue lost messages
 
 	request_command msg;
+	msg.tick = Global.simulation_tick;
 	msg.commands = commands;
 
 	conn->send_message(msg);

network/network.h

@@ -79,6 +79,18 @@ namespace network
 		command_queue::commands_map pop_commands();
 	};
 
+	// one authoritative simulation step handed to the client
+	struct frame_delta
+	{
+		double dt { 0.0 };
+		uint64_t tick { 0 };
+		uint64_t state_hash { 0 };
+		uint32_t state_hash_version { 0 };
+		command_queue::commands_map commands;
+		// false when there was nothing ready to consume in this pass
+		bool valid { false };
+	};
+
 	class client
 	{
 	protected:
@@ -108,7 +120,7 @@ namespace network
 
 	public:
 		void update();
-		std::tuple<double, double, command_queue::commands_map> get_next_delta(int counter);
+		frame_delta get_next_delta(int counter);
 		void send_commands(command_queue::commands_map commands);
 		// lobby requests; the server is the one that decides
 		void send_claim(NetworkEntityId entity_id);

network/statehash.cpp

@@ -0,0 +1,89 @@
+/*
+This Source Code Form is subject to the
+terms of the Mozilla Public License, v.
+2.0. If a copy of the MPL was not
+distributed with this file, You can
+obtain one at
+http://mozilla.org/MPL/2.0/.
+*/
+
+#include "stdafx.h"
+#include "network/statehash.h"
+
+#include "McZapkie/MOVER.h"
+#include "simulation/simulation.h"
+#include "simulation/simulationtime.h"
+#include "vehicle/DynObj.h"
+
+namespace
+{
+
+uint64_t const FNV_OFFSET{14695981039346656037ull};
+uint64_t const FNV_PRIME{1099511628211ull};
+
+uint64_t fold(uint64_t Hash, uint64_t Value)
+{
+	for (int i = 0; i < 8; ++i)
+	{
+		Hash ^= (Value >> (i * 8)) & 0xff;
+		Hash *= FNV_PRIME;
+	}
+	return Hash;
+}
+
+uint64_t fold(uint64_t Hash, std::string const &Text)
+{
+	for (char const character : Text)
+	{
+		Hash ^= (uint64_t)(unsigned char)character;
+		Hash *= FNV_PRIME;
+	}
+	return Hash;
+}
+
+// rounding to a fixed step keeps the digest from reacting to the last bits of a float
+int64_t quantize(double Value, double Step)
+{
+	if (!std::isfinite(Value))
+		return std::numeric_limits<int64_t>::min();
+
+	return (int64_t)std::llround(Value / Step);
+}
+
+} // namespace
+
+uint64_t network::state_hash()
+{
+	uint64_t digest{0};
+	uint64_t counted{0};
+
+	for (TDynamicObject const *vehicle : simulation::Vehicles.sequence())
+	{
+		if (vehicle == nullptr || vehicle->MoverParameters == nullptr)
+			continue;
+
+		auto const &mover = *vehicle->MoverParameters;
+		auto const position = vehicle->GetPosition();
+
+		uint64_t entry{FNV_OFFSET};
+		entry = fold(entry, vehicle->name());
+		entry = fold(entry, (uint64_t)quantize(position.x, 0.01));
+		entry = fold(entry, (uint64_t)quantize(position.y, 0.01));
+		entry = fold(entry, (uint64_t)quantize(position.z, 0.01));
+		entry = fold(entry, (uint64_t)quantize(mover.V, 0.001));
+		entry = fold(entry, (uint64_t)(int64_t)mover.DirActive);
+		entry = fold(entry, (uint64_t)quantize(mover.PipePress, 0.001));
+		entry = fold(entry, (uint64_t)quantize(mover.BrakePress, 0.001));
+
+		// summing keeps the whole thing independent of iteration order
+		digest += entry;
+		++counted;
+	}
+
+	digest = fold(digest, counted);
+
+	auto const &time = simulation::Time.data();
+	digest = fold(digest, (uint64_t)(time.wHour * 3600 + time.wMinute * 60 + time.wSecond));
+
+	return digest;
+}

network/statehash.h

@@ -0,0 +1,32 @@
+/*
+This Source Code Form is subject to the
+terms of the Mozilla Public License, v.
+2.0. If a copy of the MPL was not
+distributed with this file, You can
+obtain one at
+http://mozilla.org/MPL/2.0/.
+*/
+
+#pragma once
+
+#include <cstdint>
+
+namespace network
+{
+
+// what goes into the digest. bump it whenever the layout changes, so that two builds
+// cannot silently compare digests that mean different things
+constexpr uint32_t STATE_HASH_VERSION = 1;
+
+// digest of the gameplay critical state of the world, used to notice that two peers have
+// drifted apart. deliberately order independent: every vehicle is folded in under its own
+// identity and the results are summed, so the order the scenario happened to create them
+// in does not matter. values are quantized, so that harmless float noise does not read as
+// a desync
+//
+// version 1 covers vehicle position, speed, direction and the two brake pressures, plus
+// the simulation clock. positions of switches, memcells and the event queue are the
+// obvious next candidates - they go in when the snapshot work of stage 7 defines them
+uint64_t state_hash();
+
+} // namespace network

utilities/Globals.h

@@ -362,6 +362,10 @@ struct global_settings {
 	bool network_leave_pending = false;
 	// last answer of the server to a lobby request, shown in the lobby
 	std::string network_lobby_message;
+	// logical simulation step. the authority counts it up, a client takes it from the
+	// authoritative frame it is executing; the network timeline hangs off this, not off
+	// the number of frames the renderer happened to draw
+	uint64_t simulation_tick = 0;
 
 	std::unordered_map<int, std::string> trainset_overrides;