multiplayer stage 6: a logical simulation tick and a real state digest

The protocol hung off whatever frame the renderer happened to be drawing, and
the only thing standing in for synchronisation was the sum of every vehicle's
coordinates with a random number added on top - a value that cannot be equal on
two machines and therefore reported a desync on every single step.

- Global.simulation_tick: the authority counts the steps, a client takes the
  number from the authoritative frame it is executing. FRAME_INFO and the
  command requests carry it, so the network timeline has one unambiguous order
  independent of frame rates
- network/statehash: a versioned FNV digest of the gameplay critical state -
  vehicle position, speed, direction, brake pipe and cylinder pressure, plus the
  clock. Values are quantized so that float noise is not mistaken for drift, and
  vehicles are folded in under their own identity and summed, so the digest does
  not depend on the order the scenario created them in. Switches, memcells and
  the event queue are the next candidates; they go in when stage 7 pins down
  what a snapshot contains
- a client compares its own digest against the authoritative one and reports the
  drift when it starts and then only now and then, rather than every step; the
  overlay says how long it has been out of step and at which tick
- refusing to compare at all when the two builds disagree on the digest layout

EU07_NETWORK_VERSION is now 4.

network/manager.cpp

@@ -12,6 +12,7 @@ http://mozilla.org/MPL/2.0/.
 #include "simulation/simulation.h"
 #include "utilities/Logs.h"
 #include "utilities/Globals.h"
+#include "network/statehash.h"
 
 network::server_manager::server_manager()
 {
@@ -28,7 +29,7 @@ command_queue::commands_map network::server_manager::pop_commands()
 	return map;
 }
 
-void network::server_manager::push_delta(double render_dt, double dt, double sync, const command_queue::commands_map &commands)
+void network::server_manager::push_delta(double render_dt, double dt, uint64_t tick, uint64_t state_hash, const command_queue::commands_map &commands)
 {
 	if (dt == 0.0 && commands.empty())
 		return;
@@ -36,7 +37,9 @@ void network::server_manager::push_delta(double render_dt, double dt, double syn
 	frame_info msg;
 	msg.render_dt = render_dt;
 	msg.dt = dt;
-	msg.sync = sync;
+	msg.tick = tick;
+	msg.state_hash = state_hash;
+	msg.state_hash_version = STATE_HASH_VERSION;
 	msg.commands = commands;
 
 	for (auto srv : servers)

network/manager.h

@@ -14,7 +14,7 @@ namespace network
 	public:
 		server_manager();
 
-		void push_delta(double render_dt, double dt, double sync, const command_queue::commands_map &commands);
+		void push_delta(double render_dt, double dt, uint64_t tick, uint64_t state_hash, const command_queue::commands_map &commands);
 		command_queue::commands_map pop_commands();
 		void create_server(const std::string &backend, const std::string &conf);
 		// refreshes the authoritative vehicle roster and publishes it when it changed

network/message.cpp

@@ -143,6 +143,7 @@ void network::crew_update::deserialize(std::istream &stream)
 
 void ::network::request_command::serialize(std::ostream &stream) const
 {
+	sn_utils::ls_uint64(stream, tick);
 	sn_utils::ls_uint32(stream, commands.size());
 	for (auto const &kv : commands)
 	{
@@ -167,6 +168,8 @@ void ::network::request_command::serialize(std::ostream &stream) const
 
 void network::request_command::deserialize(std::istream &stream)
 {
+	tick = sn_utils::ld_uint64(stream);
+
 	uint32_t commands_size = sn_utils::ld_uint32(stream);
 	for (uint32_t i = 0; i < commands_size; i++)
 	{
@@ -200,7 +203,8 @@ void network::frame_info::serialize(std::ostream &stream) const
 {
 	sn_utils::ls_float64(stream, render_dt);
 	sn_utils::ls_float64(stream, dt);
-	sn_utils::ls_float64(stream, sync);
+	sn_utils::ls_uint64(stream, state_hash);
+	sn_utils::ls_uint32(stream, state_hash_version);
 
 	request_command::serialize(stream);
 }
@@ -209,7 +213,8 @@ void network::frame_info::deserialize(std::istream &stream)
 {
 	render_dt = sn_utils::ld_float64(stream);
 	dt = sn_utils::ld_float64(stream);
-	sync = sn_utils::ld_float64(stream);
+	state_hash = sn_utils::ld_uint64(stream);
+	state_hash_version = sn_utils::ld_uint32(stream);
 
 	request_command::deserialize(stream);
 }

network/message.h

@@ -148,6 +148,9 @@ struct request_command : public message
 	request_command() : message(REQUEST_COMMAND) {}
 
 	command_queue::commands_map commands;
+	// logical step the sender was on when it produced these. the protocol is anchored to
+	// this counter rather than to whatever frame the renderer happened to be drawing
+	uint64_t tick{ 0 };
 
 	virtual void serialize(std::ostream &stream) const override;
 	virtual void deserialize(std::istream &stream) override;
@@ -159,7 +162,9 @@ struct frame_info : public request_command
 
 	double render_dt;
 	double dt;
-	double sync;
+	// digest of the authoritative world after this step; a client compares its own
+	uint64_t state_hash{ 0 };
+	uint32_t state_hash_version{ 0 };
 
 	virtual void serialize(std::ostream &stream) const override;
 	virtual void deserialize(std::istream &stream) override;

network/network.cpp

@@ -12,7 +12,9 @@
 
 // 2 - legacy lockstep protocol
 // 3 - handshake carries build identification and an explicit rejection message
-std::uint32_t const EU07_NETWORK_VERSION = 3;
+// 4 - peer identity, vehicle roster and crew handshake, command source,
+//     logical simulation tick and a versioned state digest instead of the position sum
+std::uint32_t const EU07_NETWORK_VERSION = 4;
 
 namespace network {
 
@@ -279,7 +281,7 @@ void network::client::update()
 }
 
 // client
-std::tuple<double, double, command_queue::commands_map> network::client::get_next_delta(int counter)
+network::frame_delta network::client::get_next_delta(int counter)
 {
 	auto now = std::chrono::high_resolution_clock::now();
 	if (counter == 1) {
@@ -289,8 +291,7 @@ std::tuple<double, double, command_queue::commands_map> network::client::get_nex
 
 	if (delta_queue.empty()) {
 		// buffer underflow
-		return std::tuple<double, double,
-		        command_queue::commands_map>(0.0, 0.0, command_queue::commands_map());
+		return frame_delta();
 	}
 
 
@@ -316,13 +317,20 @@ std::tuple<double, double, command_queue::commands_map> network::client::get_nex
 			consume_counter = std::clamp(consume_counter - mult, -MAX_BUFFER_SIZE, MAX_BUFFER_SIZE);
 		}
 
+		frame_delta delta;
+		delta.dt = entry.dt;
+		delta.tick = entry.tick;
+		delta.state_hash = entry.state_hash;
+		delta.state_hash_version = entry.state_hash_version;
+		delta.commands = entry.commands;
+		delta.valid = true;
+
 		delta_queue.pop();
 
-		return std::make_tuple(entry.dt, entry.sync, entry.commands);
+		return delta;
 	} else {
 		// nothing to push
-		return std::tuple<double, double,
-		        command_queue::commands_map>(0.0, 0.0, command_queue::commands_map());
+		return frame_delta();
 	}
 }
 
@@ -333,6 +341,7 @@ void network::client::send_commands(command_queue::commands_map commands)
 	// eh, maybe queue lost messages
 
 	request_command msg;
+	msg.tick = Global.simulation_tick;
 	msg.commands = commands;
 
 	conn->send_message(msg);

network/network.h

@@ -79,6 +79,18 @@ namespace network
 		command_queue::commands_map pop_commands();
 	};
 
+	// one authoritative simulation step handed to the client
+	struct frame_delta
+	{
+		double dt { 0.0 };
+		uint64_t tick { 0 };
+		uint64_t state_hash { 0 };
+		uint32_t state_hash_version { 0 };
+		command_queue::commands_map commands;
+		// false when there was nothing ready to consume in this pass
+		bool valid { false };
+	};
+
 	class client
 	{
 	protected:
@@ -108,7 +120,7 @@ namespace network
 
 	public:
 		void update();
-		std::tuple<double, double, command_queue::commands_map> get_next_delta(int counter);
+		frame_delta get_next_delta(int counter);
 		void send_commands(command_queue::commands_map commands);
 		// lobby requests; the server is the one that decides
 		void send_claim(NetworkEntityId entity_id);

network/statehash.cpp

@@ -0,0 +1,89 @@
+/*
+This Source Code Form is subject to the
+terms of the Mozilla Public License, v.
+2.0. If a copy of the MPL was not
+distributed with this file, You can
+obtain one at
+http://mozilla.org/MPL/2.0/.
+*/
+
+#include "stdafx.h"
+#include "network/statehash.h"
+
+#include "McZapkie/MOVER.h"
+#include "simulation/simulation.h"
+#include "simulation/simulationtime.h"
+#include "vehicle/DynObj.h"
+
+namespace
+{
+
+uint64_t const FNV_OFFSET{14695981039346656037ull};
+uint64_t const FNV_PRIME{1099511628211ull};
+
+uint64_t fold(uint64_t Hash, uint64_t Value)
+{
+	for (int i = 0; i < 8; ++i)
+	{
+		Hash ^= (Value >> (i * 8)) & 0xff;
+		Hash *= FNV_PRIME;
+	}
+	return Hash;
+}
+
+uint64_t fold(uint64_t Hash, std::string const &Text)
+{
+	for (char const character : Text)
+	{
+		Hash ^= (uint64_t)(unsigned char)character;
+		Hash *= FNV_PRIME;
+	}
+	return Hash;
+}
+
+// rounding to a fixed step keeps the digest from reacting to the last bits of a float
+int64_t quantize(double Value, double Step)
+{
+	if (!std::isfinite(Value))
+		return std::numeric_limits<int64_t>::min();
+
+	return (int64_t)std::llround(Value / Step);
+}
+
+} // namespace
+
+uint64_t network::state_hash()
+{
+	uint64_t digest{0};
+	uint64_t counted{0};
+
+	for (TDynamicObject const *vehicle : simulation::Vehicles.sequence())
+	{
+		if (vehicle == nullptr || vehicle->MoverParameters == nullptr)
+			continue;
+
+		auto const &mover = *vehicle->MoverParameters;
+		auto const position = vehicle->GetPosition();
+
+		uint64_t entry{FNV_OFFSET};
+		entry = fold(entry, vehicle->name());
+		entry = fold(entry, (uint64_t)quantize(position.x, 0.01));
+		entry = fold(entry, (uint64_t)quantize(position.y, 0.01));
+		entry = fold(entry, (uint64_t)quantize(position.z, 0.01));
+		entry = fold(entry, (uint64_t)quantize(mover.V, 0.001));
+		entry = fold(entry, (uint64_t)(int64_t)mover.DirActive);
+		entry = fold(entry, (uint64_t)quantize(mover.PipePress, 0.001));
+		entry = fold(entry, (uint64_t)quantize(mover.BrakePress, 0.001));
+
+		// summing keeps the whole thing independent of iteration order
+		digest += entry;
+		++counted;
+	}
+
+	digest = fold(digest, counted);
+
+	auto const &time = simulation::Time.data();
+	digest = fold(digest, (uint64_t)(time.wHour * 3600 + time.wMinute * 60 + time.wSecond));
+
+	return digest;
+}

network/statehash.h

@@ -0,0 +1,32 @@
+/*
+This Source Code Form is subject to the
+terms of the Mozilla Public License, v.
+2.0. If a copy of the MPL was not
+distributed with this file, You can
+obtain one at
+http://mozilla.org/MPL/2.0/.
+*/
+
+#pragma once
+
+#include <cstdint>
+
+namespace network
+{
+
+// what goes into the digest. bump it whenever the layout changes, so that two builds
+// cannot silently compare digests that mean different things
+constexpr uint32_t STATE_HASH_VERSION = 1;
+
+// digest of the gameplay critical state of the world, used to notice that two peers have
+// drifted apart. deliberately order independent: every vehicle is folded in under its own
+// identity and the results are summed, so the order the scenario happened to create them
+// in does not matter. values are quantized, so that harmless float noise does not read as
+// a desync
+//
+// version 1 covers vehicle position, speed, direction and the two brake pressures, plus
+// the simulation clock. positions of switches, memcells and the event queue are the
+// obvious next candidates - they go in when the snapshot work of stage 7 defines them
+uint64_t state_hash();
+
+} // namespace network