feat(network): Repair datagram loss with adaptive redundancy

Core/GameEngine/CMakeLists.txt

@@ -505,6 +505,7 @@ set(GAMEENGINE_SRC
 #    Include/GameLogic/WeaponStatus.h
     Include/GameNetwork/Connection.h
     Include/GameNetwork/ConnectionManager.h
+    Include/GameNetwork/ConnectionStats.h
     Include/GameNetwork/DisconnectManager.h
     Include/GameNetwork/DownloadManager.h
     Include/GameNetwork/FileTransfer.h
@@ -1099,6 +1100,7 @@ set(GAMEENGINE_SRC
     Source/GameLogic/System/RankInfo.cpp
     Source/GameNetwork/Connection.cpp
     Source/GameNetwork/ConnectionManager.cpp
+    Source/GameNetwork/ConnectionStats.cpp
     Source/GameNetwork/DisconnectManager.cpp
     Source/GameNetwork/DownloadManager.cpp
     Source/GameNetwork/FileTransfer.cpp

Core/GameEngine/Include/GameNetwork/Connection.h

@@ -39,11 +39,9 @@
 #include "GameNetwork/User.h"
 #include "GameNetwork/Transport.h"
 #include "GameNetwork/NetPacket.h"
-
+#include "GameNetwork/ConnectionStats.h"
 #if defined(GENERALS_ONLINE)
-#define CONNECTION_LATENCY_HISTORY_LENGTH 10
-#else
-#define CONNECTION_LATENCY_HISTORY_LENGTH 200
+#include "GameNetwork/NetworkSettings.h"
 #endif
 
 class Connection : public MemoryPoolObject
@@ -66,6 +64,12 @@ class Connection : public MemoryPoolObject
 	void setUser(User *user);
 	User *getUser();
 	void setFrameGrouping(time_t frameGrouping);
+#if defined(GENERALS_ONLINE)
+	void setNetworkSettings(const NetworkSettings &settings, Bool unreliableTransport) { m_networkSettings = settings; m_unreliableTransport = unreliableTransport; }
+#endif
+	void setSlot(Int slot) { m_slot = slot; }
+	Int getRedundancyDepth() const { return m_redundancyDepth; }
+	time_t getFrameGrouping() const { return m_frameGrouping; }	///< The send interval in use, shorter than the configured one while redundancy is on.
 
 	void sendNetCommandMsg(NetCommandMsg *msg, UnsignedByte relay);
 
@@ -80,12 +84,27 @@ class Connection : public MemoryPoolObject
 	void setQuitting();
 	Bool isQuitting() { return m_isQuitting; }
 
+	const ConnectionStats& getStats() const { return m_stats; }
+
 #if defined(RTS_DEBUG)
 	void debugPrintCommands();
 #endif
 
 protected:
-	void doRetryMetrics();
+#if defined(GENERALS_ONLINE)
+	NetworkSettings m_networkSettings;
+	Bool m_unreliableTransport;
+#endif
+
+	void resetRedundancy();
+	void updateRedundancy(time_t curtime, Bool &active, time_t &window, Int &depth);
+	Bool hasLossEvidence();	///< A packet was lost in either direction lately, or nothing says otherwise
+	Bool getOutgoingLoss(float &lossPercent, Int &longestRun);	///< FALSE when the peer sent no packet feedback
+	void noteLossEvent(time_t now, Bool burst);
+	void setRedundancyDepth(Int depth, Int cap, time_t now);
+	Bool isQueueBuilding(time_t now);	///< Round trip at 2x its low while packets are lost: a full queue
+	void applyFrameGrouping();
+	void addRedundantCommands(NetPacket *packet, time_t curtime, time_t window, Int depth);
 
 	Bool m_isQuitting;
 	UnsignedInt m_quitTime;
@@ -95,11 +114,16 @@ class Connection : public MemoryPoolObject
 
 	NetCommandList *m_netCommandList;
 	time_t m_retryTime;						///< The time between sending retry packets for this connection.  Time is in milliseconds.
-	Real m_averageLatency;			///< The average time between sending a command and receiving an ACK.
-	Real m_latencies[CONNECTION_LATENCY_HISTORY_LENGTH];	///< List of the last 100 latencies.
+	ConnectionStats m_stats;			///< Latency, jitter and loss of this connection.
+
+	Int m_redundancyDepth;					///< How many later packets each un-acked command is repeated in, 0 on a clean connection.
+	Int m_redundancyCap;						///< The deepest depth, from the longest burst to repair and the send interval.
+	time_t m_lastLossEventTime;			///< What the decay counts from.
+	time_t m_lastDepthChangeTime;		///< A burst needs a round trip to show whether the new depth helped.
+	Bool m_queueSuspected;				///< A full queue on the path, redundancy steps down instead of up.
+	Int m_slot;
 
+	time_t m_configuredFrameGrouping;
 	time_t m_frameGrouping;				///< The minimum time between packet sends.
 	time_t m_lastTimeSent;				///< The time of the last packet send.
-	Int m_numRetries;							///< The number of retries for the last second.
-	time_t m_retryMetricsTime;		///< The start time of the current retry metrics thing.
 };

Core/GameEngine/Include/GameNetwork/ConnectionManager.h

@@ -37,6 +37,9 @@
 #include "GameNetwork/FrameMetrics.h"
 #include "GameNetwork/NetworkDefs.h"
 #include "GameNetwork/DisconnectManager.h"
+#if defined(GENERALS_ONLINE)
+#include "GameNetwork/NetworkSettings.h"
+#endif
 
 class GameInfo;
 class NetCommandWrapperList;
@@ -56,6 +59,7 @@ class ConnectionManager
 	virtual void update(Bool isInGame);			///< Service the Connections being managed by this instance.
 
 #if defined(GENERALS_ONLINE)
+	const NetworkSettings &getNetworkSettings() const { return m_networkSettings; }
 	void SeedLatencyData(int highestLatency)
 	{
 		for (int i = 0; i < MAX_SLOTS; ++i) {
@@ -154,6 +158,10 @@ class ConnectionManager
 
 	Int getAverageFPS();
 	Int getSlotAverageFPS(Int slot);
+#if defined(GENERALS_ONLINE)
+	void noteStalling(Bool stalling, UnsignedInt frame);	///< Called every network update with whether the game is waiting for commands.
+#endif
+	const ConnectionStats* getConnectionStats(Int slot) const;	///< Returns nullptr when there is no connection for the slot.
 
 #if defined(RTS_DEBUG)
 	void debugPrintConnectionCommands();
@@ -165,6 +173,10 @@ class ConnectionManager
 	Int getPingsReceived();
 
 private:
+#if defined(GENERALS_ONLINE)
+	NetworkSettings m_networkSettings;
+#endif
+
 	void doRelay();
 	void doKeepAlive();
 	void sendRemoteCommand(NetCommandRef *msg);
@@ -226,6 +238,32 @@ class ConnectionManager
 	UnsignedInt m_smallestPacketArrivalCushion;
 	Bool m_didSelfSlug;
 
+#if defined(GENERALS_ONLINE)
+	// the last command ids seen per sender, so a redundant copy of a command that is not synchronized is dropped
+	enum { RECENT_ID_COUNT = 256 };
+	struct RecentCommandIds
+	{
+		Bool valid;
+		UnsignedShort highest;
+		Bool seen[RECENT_ID_COUNT];
+	};
+	RecentCommandIds m_recentIds[MAX_SLOTS];
+
+	Bool isDuplicateCommand(NetCommandMsg *msg);
+	void resetRecentCommandIds();
+
+	ArrivalCushionStats m_cushionStats[MAX_SLOTS];
+	StallStats m_stallStats;
+	Bool m_stallActive;
+	time_t m_stallStartTime;
+	UnsignedInt m_stallMissingMask;
+	Int m_lastKnownRunAhead;
+	Int m_frameGroupingMs;
+public:
+	Bool getOverlayInfo(Int slot, NetworkOverlayInfo &info);	///< Fills what the overlay and the connection rating need about one slot.
+protected:
+#endif
+
 	// -----------------------------------------------------------------------------
 	FileCommandMap s_fileCommandMap;
 	FileMaskMap s_fileRecipientMaskMap;

Core/GameEngine/Include/GameNetwork/ConnectionStats.h

@@ -0,0 +1,190 @@
+/*
+**	Command & Conquer Generals Zero Hour(tm)
+**	Copyright 2025 Electronic Arts Inc.
+**
+**	This program is free software: you can redistribute it and/or modify
+**	it under the terms of the GNU General Public License as published by
+**	the Free Software Foundation, either version 3 of the License, or
+**	(at your option) any later version.
+**
+**	This program is distributed in the hope that it will be useful,
+**	but WITHOUT ANY WARRANTY; without even the implied warranty of
+**	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+**	GNU General Public License for more details.
+**
+**	You should have received a copy of the GNU General Public License
+**	along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <time.h>
+
+// Fixed size ring of items with a time member, oldest first, that drops the items older than a window.
+template <typename T, Int N>
+class TimedRing
+{
+public:
+	TimedRing() { clear(); }
+
+	void clear() { m_head = 0; m_count = 0; }
+	Int size() const { return m_count; }
+	const T& operator[](Int i) const { return m_items[(m_head + i) % N]; }
+
+	void push(const T& item)
+	{
+		if (m_count == N)
+		{
+			m_head = (m_head + 1) % N;
+			--m_count;
+		}
+		m_items[(m_head + m_count) % N] = item;
+		++m_count;
+	}
+
+	void prune(time_t now, time_t windowMs)
+	{
+		while (m_count > 0 && now - m_items[m_head].time > windowMs)
+		{
+			m_head = (m_head + 1) % N;
+			--m_count;
+		}
+	}
+
+private:
+	T m_items[N];
+	Int m_head;
+	Int m_count;
+};
+
+// Tracks latency, jitter and loss of one connection over a sliding time window.
+class ConnectionStats
+{
+public:
+	ConnectionStats();
+
+	void reset();
+	void noteAcked(Real rttMs, time_t now);	///< A command that was acked on its first send.
+	void noteRetry(time_t now);							///< A command that was not acked in time and had to be sent again.
+	void noteLateAck(time_t now);						///< A retried command was acked by its earlier send after all, the retry was not a loss.
+	void setRedundancyDepth(Int depth, Int cap);
+	void update(time_t now);								///< Drop the samples that have left the window.
+
+	Real getLatency() const { return m_srtt; }
+	Real getMinLatency() const;
+	Real getLatencyPercentile(Int percent, time_t since = 0) const;	///< Round trip that this share of the samples in the window, from since on, stays within
+	Int getSampleCountSince(time_t since) const;
+	Real getJitter() const { return m_jitter; }
+	Real getLossPercent() const;						///< Share of retries the earlier send was not acked for
+	Real getLateAckPercent() const;					///< Share of retries whose earlier send was acked after the retry
+	Real getRetryPercent() const;						///< Share of retries, lost and late acked together
+	Real getLatePercent() const;						///< Share of acks that took more than twice the smoothed round trip time.
+	Int getSampleCount() const { return m_samples.size(); }
+	enum { MAX_REDUNDANCY_DEPTH = 16 };
+	Int getRedundancyDepth() const { return m_depth; }
+	Int getRedundancyCap() const { return m_depthCap; }
+
+private:
+	enum
+	{
+		MAX_SAMPLES = 1024,	///< Covers the window at 60 acked frames per second.
+		MAX_RETRIES = 1024	///< As many as acks, under heavy loss a smaller ring undercounts the retries of the window.
+	};
+
+	struct Sample
+	{
+		time_t time;
+		Real rtt;
+		Bool late;
+	};
+
+	struct Stamp
+	{
+		time_t time;
+	};
+
+	TimedRing<Sample, MAX_SAMPLES> m_samples;
+	TimedRing<Stamp, MAX_RETRIES> m_retries;
+	TimedRing<Stamp, MAX_RETRIES> m_lateAcks;
+
+	Real m_srtt;
+	Real m_jitter;
+	Real m_previousRtt;
+	Bool m_hasRtt;
+
+	Int m_depth;
+	Int m_depthCap;
+};
+
+// Tracks how many frames before its execution frame the frame commands of one sender arrive,
+// over the same sliding window as ConnectionStats. Zero or less means the frame had to wait.
+class ArrivalCushionStats
+{
+public:
+	ArrivalCushionStats();
+
+	void reset();
+	void note(Int cushion, time_t now);
+	void update(time_t now);
+
+	Int getSampleCount() const { return m_samples.size(); }
+	Int getPercentile(Int percent) const;			///< Cushion that percent of the samples are at or above the lowest value of.
+
+private:
+	enum { MAX_SAMPLES = 1024 };
+
+	struct Sample
+	{
+		time_t time;
+		Int cushion;
+	};
+
+	TimedRing<Sample, MAX_SAMPLES> m_samples;
+};
+
+// Frames the game wanted to advance but had to wait for commands, kept for a minute and counted over a window.
+class StallStats
+{
+public:
+	StallStats();
+
+	void reset();
+	void note(Int durationMs, UnsignedInt missingSlotMask, time_t now);	///< One finished stall and the slots whose commands were missing when it began.
+	void update(time_t now);
+
+	Int getCount(time_t now, Int windowMs) const;
+	Int getCountForSlot(Int slot, time_t now, Int windowMs) const;
+
+private:
+	enum { MAX_EVENTS = 512, HISTORY_MS = 60000 };
+
+	struct Event
+	{
+		time_t time;
+		Int ms;
+		UnsignedInt slotMask;
+	};
+
+	TimedRing<Event, MAX_EVENTS> m_events;
+};
+
+// What the network overlay and the connection rating need to know about one slot beyond its ConnectionStats.
+struct NetworkOverlayInfo
+{
+	NetworkOverlayInfo()
+		: cushionSamples(0), cushionP5(0), cushionP50(0)
+		, stallsWindow(0), stallsMinute(0), stalledNow(FALSE), waitingOnSlot(FALSE)
+		, slotInBandMs(-1), runAhead(0), frameGroupingMs(0) {}
+
+	Int cushionSamples;		///< Frames before execution that the frame commands of this slot arrived, over the stats window.
+	Int cushionP5;
+	Int cushionP50;
+	Int stallsWindow;			///< Stalls the local game had to wait on this slot for, over the stats window.
+	Int stallsMinute;			///< The same over the last minute.
+	Bool stalledNow;			///< The local game is waiting for commands right now.
+	Bool waitingOnSlot;		///< ... and this slot is one it is waiting on.
+	Int slotInBandMs;			///< In-band round trip this slot contributes to the run ahead, -1 if unknown.
+
+	Int runAhead;				///< Frames.
+	Int frameGroupingMs;		///< The send interval.
+};

Core/GameEngine/Include/GameNetwork/FrameData.h

@@ -48,7 +48,7 @@ class FrameData {
 	NetCommandList * getCommandList();
 	void setFrameCommandCount(UnsignedInt totalCommandCount);
 	UnsignedInt getFrameCommandCount();
-	void addCommand(NetCommandMsg *msg);
+	Bool addCommand(NetCommandMsg *msg);	///< Returns FALSE when the command was already in the frame.
 	UnsignedInt getCommandCount();
 	void zeroFrame();
 	void destroyGameMessages();

Core/GameEngine/Include/GameNetwork/FrameDataManager.h

@@ -38,7 +38,7 @@ class FrameDataManager : public MemoryPoolObject
 	void reset();
 	void update();
 
-	void addNetCommandMsg(NetCommandMsg *msg);
+	Bool addNetCommandMsg(NetCommandMsg *msg);	///< Returns FALSE when the command was already in its frame.
 	void setIsLocal(Bool isLocal);
 	FrameDataReturnType allCommandsReady(UnsignedInt frame, Bool debugSpewage);
 	NetCommandList * getFrameCommandList(UnsignedInt frame);

Core/GameEngine/Include/GameNetwork/FrameMetrics.h

@@ -59,6 +59,9 @@ class FrameMetrics {
 	Real *m_fpsList;								///< A record of how many game logic frames per second there were for the last 60 seconds.
 	time_t m_lastFpsTimeThing;																///< The time when the last fps entry started being recorded.
 	Int m_fpsListIndex;																				///< Index into the array of the current fps list entry being measured.
+#if defined(GENERALS_ONLINE)
+	UnsignedInt m_fpsListLength;																						///< Entries m_fpsList holds, the history length can change after construction.
+#endif
 	Real m_averageFps;																				///< The current average logic fps, computed just like m_averageLatency below but with the fps numbers.
 
 	// round trip time to packet router variables.