multiplayer stage 4: server-authoritative input, permissions, per-source holds

Three things stood in the way of two people sharing a cab and of a client not
being able to reach past its own vehicle.

Vehicle commands were addressed by TTrain::id(), which every peer hands out
locally from its own counter, so the same number means a different vehicle on
a different peer. Commands now travel with the session wide NetworkEntityId
and are translated back to the local cab on arrival; outside of multiplayer
the registry is empty and both directions are a no-op, so singleplayer is
untouched.

A client could send any command it liked, including setweather, queueevent or
spawntrainset. The server now validates every incoming command: a player may
work the controls of the vehicle whose crew they are on and nothing else, and
world wide commands stay with the host until dispatcher roles exist. The
source is stamped by the server, never taken from what the peer sent, and
rejections are logged once per peer/command instead of every frame. Host input
runs through the very same validation, just without the transport.

Continuous commands were tracked per command and recipient only, so with two
people in one cab the release sent by one of them cancelled the other one's
hold. command_data now carries the source peer and the active set is keyed by
{source, recipient, command}.

Also: a client asking to quit no longer replicates quitsimulation, which would
have shut down everybody else's session.

application/application.cpp

@@ -438,7 +438,9 @@ double eu07_application::generate_sync()
 
 void eu07_application::queue_quit(bool direct)
 {
-	if (direct || !m_modes[m_modestack.top()]->is_command_processor())
+	// a network client leaving is its own business; replicating the request would shut
+	// down everybody else's session as well
+	if (direct || is_client() || !m_modes[m_modestack.top()]->is_command_processor())
 	{
 		glfwSetWindowShouldClose(m_windows[0], GLFW_TRUE);
 		return;
@@ -510,6 +512,10 @@ int eu07_application::run()
 				// if we're the server
 				if (m_network && m_network->servers)
 				{
+					// our own input goes through the same authority layer as everybody
+					// else's, only without the transport in between
+					network::filter_commands(network::PEER_HOST, local_commands);
+
 					// fetch from network layer command requests received from clients
 					command_queue::commands_map remote_commands = m_network->servers->pop_commands();
 

input/command.cpp

@@ -15,6 +15,8 @@ http://mozilla.org/MPL/2.0/.
 #include "utilities/Timer.h"
 #include "simulation/simulation.h"
 #include "vehicle/Train.h"
+#include "vehicle/DynObj.h"
+#include "network/entities.h"
 
 namespace simulation {
 
@@ -793,13 +795,61 @@ std::unordered_map<std::string, user_command> commandMap = {
 
 } // namespace simulation
 
+namespace {
+
+// vehicle commands are addressed with the session wide vehicle id while they are in the
+// hands of the network layer, because TTrain ids are handed out locally and two peers
+// numbering the same vehicles differently would send each other's commands astray.
+// outside of multiplayer the registry is empty and both directions are a no-op.
+
+bool is_vehicle_recipient(uint32_t const Recipient) {
+	return (command_target)(Recipient & ~0xffff) == command_target::vehicle;
+}
+
+uint32_t to_network_recipient(uint32_t const Recipient) {
+	if (!is_vehicle_recipient(Recipient) || network::Entities.empty())
+		return Recipient;
+
+	TTrain *train = simulation::Trains.find_id((uint16_t)(Recipient & 0xffff));
+	if (train == nullptr || train->Dynamic() == nullptr)
+		return Recipient;
+
+	auto const id = network::Entities.id_of(train->Dynamic()->name());
+	if (id == network::ENTITY_NONE || id > 0xffff)
+		return Recipient;
+
+	return (uint32_t)command_target::vehicle | id;
+}
+
+// returns false when the addressed vehicle has no cab on this peer yet
+bool from_network_recipient(uint32_t const Recipient, uint32_t &Local) {
+	Local = Recipient;
+
+	if (!is_vehicle_recipient(Recipient) || network::Entities.empty())
+		return true;
+
+	auto const name = network::Entities.name_of(Recipient & 0xffff);
+	if (name.empty())
+		return true;
+
+	TTrain *train = simulation::Trains.find(name);
+	if (train == nullptr)
+		return false;
+
+	Local = (uint32_t)command_target::vehicle | train->id();
+	return true;
+}
+
+} // namespace
+
 void command_queue::update()
 {
 	double delta = Timer::GetDeltaTime();
-	for (auto c : m_active_continuous)
+	for (auto const &c : m_active_continuous)
 	{
-		command_data data({c.first, GLFW_REPEAT, 0.0, 0.0, delta, false, glm::vec3()}); // todo: improve
-		auto lookup = m_commands.emplace( c.second, commanddata_sequence() );
+		command_data data({c.command, GLFW_REPEAT, 0.0, 0.0, delta, false, glm::vec3()}); // todo: improve
+		data.source = c.source;
+		auto lookup = m_commands.emplace( c.recipient, commanddata_sequence() );
 		// recipient stack was either located or created, so we can add to it quite safely
 		lookup.first->second.emplace_back( data );
 	}
@@ -809,7 +859,7 @@ void command_queue::update()
 void
 command_queue::push( command_data const &Command, uint32_t const Recipient ) {
 	if (is_network_target(Recipient)) {
-		auto lookup = m_intercept_queue.emplace(Recipient, commanddata_sequence());
+		auto lookup = m_intercept_queue.emplace(to_network_recipient(Recipient), commanddata_sequence());
 		lookup.first->second.emplace_back(Command);
 	} else {
 		push_direct(Command, Recipient);
@@ -820,10 +870,12 @@ void command_queue::push_direct(const command_data &Command, const uint32_t Reci
 	auto const &desc = simulation::Commands_descriptions[ static_cast<std::size_t>( Command.command ) ];
 	if (desc.mode == command_mode::continuous)
 	{
+		continuous_key const key { Command.source, Recipient, Command.command };
+
 		if (Command.action == GLFW_PRESS)
-			m_active_continuous.emplace(std::make_pair(Command.command, Recipient));
+			m_active_continuous.emplace(key);
 		else if (Command.action == GLFW_RELEASE)
-			m_active_continuous.erase(std::make_pair(Command.command, Recipient));
+			m_active_continuous.erase(key);
 		else if (Command.action == GLFW_REPEAT)
 			return;
 	}
@@ -870,9 +922,16 @@ command_queue::commands_map command_queue::pop_intercept_queue() {
 }
 
 void command_queue::push_commands(const commands_map &commands) {
-	for (auto const &kv : commands)
+	for (auto const &kv : commands) {
+		uint32_t recipient;
+		if (!from_network_recipient(kv.first, recipient)) {
+			// the vehicle has no cab on this peer yet; the command has nowhere to go
+			continue;
+		}
+
 		for (command_data const &data : kv.second)
-			push_direct(data, kv.first);
+			push_direct(data, recipient);
+	}
 }
 
 void
@@ -900,6 +959,7 @@ command_relay::post(user_command const Command, double const Param1, double cons
 
 	uint32_t combined_recipient = static_cast<uint32_t>( command.target ) | Recipient;
 	command_data commanddata({Command, Action, Param1, Param2, Timer::GetDeltaTime(), FreeFlyModeFlag, Position });
+	commanddata.source = Global.network_peer_id;
 	if (Payload)
 		commanddata.payload = *Payload;
 

input/command.h

@@ -445,6 +445,12 @@ struct command_data {
 	glm::vec3 location;
 
     std::string payload;
+
+    // network::PeerId of the participant the command came from. kept as a plain integer
+    // so that this header stays independent of the network layer. it matters wherever two
+    // people work the same vehicle: a continuous command held by one of them must not be
+    // cancelled by the other one letting go of the same key
+    uint32_t source { 0 };
 };
 
 // command_queues: collects and holds commands from input sources, for processing by their intended recipients
@@ -487,15 +493,29 @@ private:
 
 	void push_direct( command_data const &Command, uint32_t const Recipient );
 
-	// hash operator for m_active_continuous
-	struct command_set_hash {
-		uint64_t operator() (const std::pair<user_command, uint32_t> &pair) const {
-			return (uint64_t)pair.first << 32 | (uint64_t)pair.second;
-		}
+	// a continuous command is held per source, not just per recipient: with two people in
+	// one cab, the release sent by one of them may only cancel their own hold
+	struct continuous_key {
+		uint32_t source;
+		uint32_t recipient;
+		user_command command;
+
+		bool operator==( continuous_key const &Other ) const {
+			return source == Other.source
+			    && recipient == Other.recipient
+			    && command == Other.command; }
+	};
+
+	struct continuous_key_hash {
+		std::size_t operator() ( continuous_key const &Key ) const {
+			std::size_t hash = (std::size_t)Key.source * 2654435761u;
+			hash ^= (std::size_t)Key.recipient + 0x9e3779b9 + ( hash << 6 ) + ( hash >> 2 );
+			hash ^= (std::size_t)Key.command + 0x9e3779b9 + ( hash << 6 ) + ( hash >> 2 );
+			return hash; }
 	};
 
 	// currently pressed continuous commands
-	std::unordered_set<std::pair<user_command, uint32_t>, command_set_hash> m_active_continuous;
+	std::unordered_set<continuous_key, continuous_key_hash> m_active_continuous;
 };
 
 template<typename A, typename B>

network/message.cpp

@@ -160,6 +160,7 @@ void ::network::request_command::serialize(std::ostream &stream) const
 			sn_utils::s_vec3(stream, data.location);
 
 			sn_utils::s_str(stream, data.payload);
+			sn_utils::ls_uint32(stream, data.source);
 		}
 	}
 }
@@ -186,6 +187,7 @@ void network::request_command::deserialize(std::istream &stream)
 			data.location = sn_utils::d_vec3(stream);
 
 			data.payload = sn_utils::d_str(stream);
+			data.source = sn_utils::ld_uint32(stream);
 
 			sequence.emplace_back(data);
 		}

network/network.cpp

@@ -6,6 +6,9 @@
 #include "utilities/Timer.h"
 #include "application/application.h"
 #include "utilities/Globals.h"
+#include "simulation/simulation.h"
+#include <set>
+#include <tuple>
 
 // 2 - legacy lockstep protocol
 // 3 - handshake carries build identification and an explicit rejection message
@@ -97,6 +100,23 @@ network::server::server(std::shared_ptr<std::istream> buf) : backbuffer(buf)
 
 }
 
+namespace {
+
+// keeps the log readable: a peer holding a forbidden key would otherwise produce a line
+// every single frame, so each peer/command/verdict combination is reported once
+void report_rejected_command(network::PeerId Peer, user_command Command, network::command_verdict Verdict)
+{
+	static std::set<std::tuple<network::PeerId, user_command, network::command_verdict>> reported;
+
+	if (!reported.emplace(Peer, Command, Verdict).second)
+		return;
+
+	auto const &description = simulation::Commands_descriptions[static_cast<std::size_t>(Command)];
+	ErrorLog("net: command \"" + description.name + "\" from peer " + std::to_string(Peer) + " rejected: " + network::describe(Verdict), logtype::net);
+}
+
+} // namespace
+
 void network::server::prune_clients()
 {
 	for (auto it = clients.begin(); it != clients.end(); ) {
@@ -217,8 +237,22 @@ void network::server::handle_message(std::shared_ptr<connection> conn, const mes
 	else if (msg.type == message::REQUEST_COMMAND) {
 		const auto& cmd = dynamic_cast<const request_command&>(msg);
 
-		for (auto const &kv : cmd.commands)
-			client_commands_queue.emplace(kv);
+		for (auto const &kv : cmd.commands) {
+			for (command_data const &data : kv.second) {
+				auto const verdict = validate_command(conn->peer_id, data.command, kv.first);
+				if (verdict != command_verdict::accepted) {
+					report_rejected_command(conn->peer_id, data.command, verdict);
+					continue;
+				}
+
+				command_data accepted = data;
+				// the source is decided here, never taken from what the peer sent
+				accepted.source = conn->peer_id;
+
+				auto lookup = client_commands_queue.emplace(kv.first, command_queue::commanddata_sequence());
+				lookup.first->second.emplace_back(accepted);
+			}
+		}
 	}
 }
 

network/session.cpp

@@ -42,6 +42,75 @@ std::string describe(claim_result Result)
 	return "unknown";
 }
 
+std::string describe(command_verdict Verdict)
+{
+	switch (Verdict)
+	{
+	case command_verdict::accepted:
+		return "accepted";
+	case command_verdict::not_in_crew:
+		return "not a member of the crew of that vehicle";
+	case command_verdict::privileged:
+		return "reserved for the session host";
+	case command_verdict::unsupported_target:
+		return "command target not allowed over the network";
+	}
+	return "unknown";
+}
+
+command_verdict validate_command(PeerId Peer, user_command Command, uint32_t Recipient)
+{
+	if (Peer == PEER_HOST)
+	{
+		// the host runs the session and holds every right in it
+		return command_verdict::accepted;
+	}
+
+	auto const target = (command_target)(Recipient & ~0xffff);
+
+	if (target == command_target::vehicle)
+	{
+		// a player works the controls of the vehicle they are on, and of no other
+		return Crews.is_member(Peer, (NetworkEntityId)(Recipient & 0xffff)) ? command_verdict::accepted : command_verdict::not_in_crew;
+	}
+
+	if (target == command_target::simulation)
+	{
+		// setweather, setdatetime, spawntrainset, destroytrainset, queueevent, the pause
+		// and the debug commands all reach the whole world, so they stay with the authority.
+		// TODO: open a subset of them once dispatcher and admin roles exist
+		(void)Command;
+		return command_verdict::privileged;
+	}
+
+	return command_verdict::unsupported_target;
+}
+
+void filter_commands(PeerId Peer, command_queue::commands_map &Commands)
+{
+	for (auto it = Commands.begin(); it != Commands.end();)
+	{
+		auto &sequence = it->second;
+
+		for (auto command = sequence.begin(); command != sequence.end();)
+		{
+			if (validate_command(Peer, command->command, it->first) != command_verdict::accepted)
+			{
+				command = sequence.erase(command);
+				continue;
+			}
+
+			command->source = Peer;
+			++command;
+		}
+
+		if (sequence.empty())
+			it = Commands.erase(it);
+		else
+			++it;
+	}
+}
+
 vehicle_crew &crew_registry::entry(NetworkEntityId Id)
 {
 	auto lookup = m_vehicles.find(Id);

network/session.h

@@ -10,6 +10,7 @@ http://mozilla.org/MPL/2.0/.
 #pragma once
 
 #include "network/entities.h"
+#include "input/command.h"
 
 #include <string>
 #include <unordered_map>
@@ -87,4 +88,24 @@ private:
 
 extern crew_registry Crews;
 
+// what a peer is allowed to ask the simulation for
+enum class command_verdict
+{
+	accepted,
+	not_in_crew,
+	privileged,
+	unsupported_target
+};
+
+std::string describe(command_verdict Verdict);
+
+// the whole of the authority check for an incoming command. the host passes it too, it
+// simply holds every right - there is no separate path for the local participant
+command_verdict validate_command(PeerId Peer, user_command Command, uint32_t Recipient);
+
+// drops everything the peer is not allowed to ask for and stamps the rest with its
+// identity. used for the local participant as well, so that host input goes through the
+// very same authority layer as input arriving over the wire, only without the transport
+void filter_commands(PeerId Peer, command_queue::commands_map &Commands);
+
 } // namespace network