feat(lobby): Retry failed peer connections and block the start until everyone is connected

Core/GameEngine/Include/Common/UserPreferences.h

@@ -60,11 +60,13 @@ class UserPreferences : public PreferenceMap
 	Real getReal(AsciiString key, Real defaultValue) const;
 	Int getInt(AsciiString key, Int defaultValue) const;
 	AsciiString getAsciiString(AsciiString key, AsciiString defaultValue) const;
+	UnicodeString getUnicodeString(AsciiString key, UnicodeString defaultValue) const;
 
 	void setBool(AsciiString key, Bool val);
 	void setReal(AsciiString key, Real val);
 	void setInt(AsciiString key, Int val);
 	void setAsciiString(AsciiString key, AsciiString val);
+	void setUnicodeString(AsciiString key, UnicodeString val);
 
 protected:
 	AsciiString m_filename;

Core/GameEngine/Include/GameNetwork/ConnectionManager.h

@@ -37,6 +37,9 @@
 #include "GameNetwork/FrameMetrics.h"
 #include "GameNetwork/NetworkDefs.h"
 #include "GameNetwork/DisconnectManager.h"
+#if defined(GENERALS_ONLINE)
+#include "GameNetwork/NetworkSettings.h"
+#endif
 
 class GameInfo;
 class NetCommandWrapperList;
@@ -56,6 +59,7 @@ class ConnectionManager
 	virtual void update(Bool isInGame);			///< Service the Connections being managed by this instance.
 
 #if defined(GENERALS_ONLINE)
+	const NetworkSettings &getNetworkSettings() const { return m_networkSettings; }
 	void SeedLatencyData(int highestLatency)
 	{
 		for (int i = 0; i < MAX_SLOTS; ++i) {
@@ -165,6 +169,10 @@ class ConnectionManager
 	Int getPingsReceived();
 
 private:
+#if defined(GENERALS_ONLINE)
+	NetworkSettings m_networkSettings;
+#endif
+
 	void doRelay();
 	void doKeepAlive();
 	void sendRemoteCommand(NetCommandRef *msg);

Core/GameEngine/Include/GameNetwork/FrameMetrics.h

@@ -59,6 +59,9 @@ class FrameMetrics {
 	Real *m_fpsList;								///< A record of how many game logic frames per second there were for the last 60 seconds.
 	time_t m_lastFpsTimeThing;																///< The time when the last fps entry started being recorded.
 	Int m_fpsListIndex;																				///< Index into the array of the current fps list entry being measured.
+#if defined(GENERALS_ONLINE)
+	UnsignedInt m_fpsListLength;																						///< Entries m_fpsList holds, the history length can change after construction.
+#endif
 	Real m_averageFps;																				///< The current average logic fps, computed just like m_averageLatency below but with the fps numbers.
 
 	// round trip time to packet router variables.

Core/GameEngine/Include/GameNetwork/NetworkDefs.h

@@ -38,7 +38,7 @@ static constexpr const Int MIN_LOGIC_FRAMES = 5;
 static constexpr const Int MAX_FRAMES_AHEAD = 128;
 
 #if defined(GENERALS_ONLINE)
-static Int MIN_RUNAHEAD = 6;
+extern Int MIN_RUNAHEAD;
 #else
 static constexpr const Int MIN_RUNAHEAD = 4;
 #endif

Core/GameEngine/Include/GameNetwork/NetworkSettings.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#if defined(GENERALS_ONLINE)
+
+#include <string>
+
+// Per-match game networking policy, independent of online-service lifetime.
+struct NetworkSettings
+{
+	struct Send
+	{
+		int frames_per_packet = 2;      // send interval in frames
+		bool flush_every_frame = true;  // flush GNS after each frame
+		int flags = -1;                 // GNS send flags tried after a failed reliable send, -1 = none
+	} send;
+	struct InputDelay
+	{
+		int min_frames = 4;
+		int decision_interval_ms = 5000;
+		int fps_history_samples = 30;   // per-second FPS samples averaged
+		int slack_percent = 30;         // percent added to the measured round trip, also sets the start run ahead and the rating
+
+		struct Formula
+		{
+			int latency_history_samples = 200; // in-band ping samples averaged
+		} formula;
+	} input_delay;
+
+	static NetworkSettings LocalDefaults()
+	{
+		return NetworkSettings();
+	}
+};
+
+#endif

Core/GameEngine/Source/Common/UserPreferences.cpp

@@ -260,6 +260,19 @@ AsciiString UserPreferences::getAsciiString(AsciiString key, AsciiString default
 	return it->second;
 }
 
+UnicodeString UserPreferences::getUnicodeString(AsciiString key, UnicodeString defaultValue) const
+{
+	UserPreferences::const_iterator it = find(key);
+	if (it == end())
+	{
+		return defaultValue;
+	}
+
+	UnicodeString ret;
+	ret.translate(it->second);
+	return ret;
+}
+
 void UserPreferences::setBool(AsciiString key, Bool val)
 {
 	(*this)[key] = boolAsStr(val);
@@ -280,6 +293,13 @@ void UserPreferences::setAsciiString(AsciiString key, AsciiString val)
 	(*this)[key] = val;
 }
 
+void UserPreferences::setUnicodeString(AsciiString key, UnicodeString val)
+{
+	AsciiString str;
+	str.translate(val);
+	(*this)[key] = str;
+}
+
 //-----------------------------------------------------------------------------
 // QuickMatchPreferences base class
 //-----------------------------------------------------------------------------
@@ -515,22 +535,16 @@ void CustomMatchPreferences::setLastLadder(const AsciiString& addr, UnsignedShor
 	(*this)["LastLadderPort"] = strVal;
 }
 
-AsciiString CustomMatchPreferences::getLastLobbyName() const
+UnicodeString CustomMatchPreferences::getLastLobbyName() const
 {
-	CustomMatchPreferences::const_iterator it = find("LastLobbyName");
-	if (it == end())
-	{
-		return AsciiString::TheEmptyString;
-	}
-
-	AsciiString ret = it->second;
+	UnicodeString ret = getUnicodeString("LastLobbyName", UnicodeString::TheEmptyString);
 	ret.trim();
 	return ret;
 }
 
-void CustomMatchPreferences::setLastLobbyName(const AsciiString& name)
+void CustomMatchPreferences::setLastLobbyName(const UnicodeString& name)
 {
-	(*this)["LastLobbyName"] = name;
+	setUnicodeString("LastLobbyName", name);
 }
 
 AsciiString CustomMatchPreferences::getLastLadderAddr( void )

Core/GameEngine/Source/GameNetwork/Connection.cpp

@@ -271,7 +271,7 @@ UnsignedInt Connection::doSend() {
 
 	if ((curtime - m_lastTimeSent) < m_frameGrouping) {
 //		DEBUG_LOG(("not sending packet, time = %d, m_lastFrameSent = %d, m_frameGrouping = %d", curtime, m_lastTimeSent, m_frameGrouping));
-		NetworkLog(ELogVerbosity::LOG_DEBUG, "not sending packet, time = %d, m_lastFrameSent = %d, m_frameGrouping = %d", curtime, m_lastTimeSent, m_frameGrouping);
+		NetworkLog(ELogVerbosity::LOG_DEBUG, "not sending packet, time = %I64d, m_lastFrameSent = %I64d, m_frameGrouping = %I64d", (Int64)curtime, (Int64)m_lastTimeSent, (Int64)m_frameGrouping);
 		return 0;
 	}
 

Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp

@@ -59,6 +59,10 @@
 #include "../NetworkMesh.h"
 #include "../ngmp_interfaces.h"
 
+#if defined(GENERALS_ONLINE)
+Int MIN_RUNAHEAD = 4;
+#endif
+
 static Bool hasValidTransferFileExtension(const AsciiString& filePath)
 {
 	static const char* const validExtensions[] = {
@@ -272,6 +276,23 @@ ConnectionManager::ConnectionManager()
  */
 void ConnectionManager::init()
 {
+#if defined(GENERALS_ONLINE)
+	m_networkSettings = NetworkSettings::LocalDefaults();
+	NGMP_OnlineServicesManager *services = NGMP_OnlineServicesManager::GetInstance();
+	if (TheLAN == nullptr && TheNGMPGame != nullptr && services != nullptr) {
+		const ServiceConfig &overrides = services->GetServiceConfig();
+		m_networkSettings.send = overrides.send;
+		m_networkSettings.input_delay = overrides.input_delay;
+	}
+	const NetworkSettings::InputDelay &runAhead = m_networkSettings.input_delay;
+	TheWritableGlobalData->m_networkFPSHistoryLength = (std::max)(1, runAhead.fps_history_samples);
+	TheWritableGlobalData->m_networkLatencyHistoryLength = (std::max)(1, runAhead.formula.latency_history_samples);
+	TheWritableGlobalData->m_networkRunAheadSlack = runAhead.slack_percent;
+	TheWritableGlobalData->m_networkRunAheadMetricsTime = runAhead.decision_interval_ms;
+	MIN_RUNAHEAD = runAhead.min_frames;
+	FRAME_GROUPING_CAP = (int)(m_networkSettings.send.frames_per_packet * (float)(1000.f / GENERALS_ONLINE_HIGH_FPS_LIMIT));
+#endif
+
 //	if (m_transport == nullptr) {
 //		m_transport = new Transport;
 //	}
@@ -389,10 +410,10 @@ void ConnectionManager::reset()
 #endif
 	m_packetRouterSlot = -1;
 
-	for (i = 0; i < TheGlobalData->m_networkFPSHistoryLength; ++i) {
+	for (i = 0; i < (UnsignedInt)MAX_SLOTS; ++i) {
 		m_fpsAverages[i] = -1;
 	}
-	for (i = 0; i < TheGlobalData->m_networkLatencyHistoryLength; ++i) {
+	for (i = 0; i < (UnsignedInt)MAX_SLOTS; ++i) {
 		m_latencyAverages[i] = 0.0;
 	}
 
@@ -1995,7 +2016,7 @@ void ConnectionManager::setFrameGrouping(time_t frameGrouping) {
 	// This is probably ok since the packet router should have the fastest connection of all
 	// the players in the game.
 
-	NetworkLog(ELogVerbosity::LOG_DEBUG, "ConnectionManager::setFrameGrouping Frame grouping is %d, is packet router %d", frameGrouping, m_localSlot == m_packetRouterSlot);
+	NetworkLog(ELogVerbosity::LOG_DEBUG, "ConnectionManager::setFrameGrouping Frame grouping is %I64d, is packet router %d", (Int64)frameGrouping, m_localSlot == m_packetRouterSlot);
 
 	if (m_localSlot == m_packetRouterSlot) {
 		frameGrouping = frameGrouping / 2;
@@ -2785,7 +2806,7 @@ void ConnectionManager::sendFrameDataToPlayer(UnsignedInt playerID, UnsignedInt
 }
 
 void ConnectionManager::sendSingleFrameToPlayer(UnsignedInt playerID, UnsignedInt frame) {
-	if ((TheGameLogic->getFrame() - FRAMES_TO_KEEP) > frame) {
+	if (frame + FRAMES_TO_KEEP < TheGameLogic->getFrame()) {
 		DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("ConnectionManager::sendSingleFrameToPlayer - player %d requested frame %d when we are on frame %d, this is too far in the past.", playerID, frame, TheGameLogic->getFrame()));
 		return;
 	}

Core/GameEngine/Source/GameNetwork/DisconnectManager.cpp

@@ -214,7 +214,7 @@ void DisconnectManager::updateDisconnectStatus(ConnectionManager *conMgr) {
 						m_haveNotifiedOtherPlayersOfCurrentFrame = TRUE;
 					}
 
-					DEBUG_LOG(("DisconnectManager::updateDisconnectStatus - curTime = %d, m_timeOfDisconnectScreenOn = %d, curTime - m_timeOfDisconnectScreenOn = %d", curTime, m_timeOfDisconnectScreenOn, curTime - m_timeOfDisconnectScreenOn));
+					DEBUG_LOG(("DisconnectManager::updateDisconnectStatus - curTime = %I64d, m_timeOfDisconnectScreenOn = %I64d, curTime - m_timeOfDisconnectScreenOn = %I64d", (Int64)curTime, (Int64)m_timeOfDisconnectScreenOn, (Int64)(curTime - m_timeOfDisconnectScreenOn)));
 
 					if (m_timeOfDisconnectScreenOn != 0) {
 						if ((curTime - m_timeOfDisconnectScreenOn) > TheGlobalData->m_networkDisconnectScreenNotifyTime) {

Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp

@@ -61,14 +61,17 @@ FrameMetrics::FrameMetrics()
 
 
 	m_fpsList = NEW Real[TheGlobalData->m_networkFPSHistoryLength];
+#if defined(GENERALS_ONLINE)
+	m_fpsListLength = TheGlobalData->m_networkFPSHistoryLength;
+#endif
 }
 
 FrameMetrics::~FrameMetrics() {
-	delete m_fpsList;
+	delete[] m_fpsList;
 	m_fpsList = NULL;
 
 #if !defined(USE_NEW_FRAMEMETRIC_LOGIC)
-    delete m_latencyList;
+    delete[] m_latencyList;
     m_latencyList = NULL;
 
     delete[] m_pendingLatencies;
@@ -115,6 +118,15 @@ void FrameMetrics::init() {
 #endif
 	m_minimumCushion = -1;
 
+#if defined(GENERALS_ONLINE)
+	// the game network settings can change the history length after construction
+	if (m_fpsListLength != TheGlobalData->m_networkFPSHistoryLength) {
+		delete[] m_fpsList;
+		m_fpsList = NEW Real[TheGlobalData->m_networkFPSHistoryLength];
+		m_fpsListLength = TheGlobalData->m_networkFPSHistoryLength;
+	}
+#endif
+
 	UnsignedInt i = 0;
 	for (; i < TheGlobalData->m_networkFPSHistoryLength; ++i) {
 #if defined(GENERALS_ONLINE_HIGH_FPS_SERVER)
@@ -228,8 +240,8 @@ void FrameMetrics::processLatencyResponse(UnsignedInt frame)
     {
         m_averageLatency += kvPaAir.second;
     }
-	NetworkLog(ELogVerbosity::LOG_DEBUG, "Avg latency is: %f / %d = %f", m_averageLatency, m_mapLatenciesLookup.size(), m_averageLatency /= m_mapLatenciesLookup.size());
     m_averageLatency /= m_mapLatenciesLookup.size();
+	NetworkLog(ELogVerbosity::LOG_DEBUG, "Avg latency is: %f over %d samples", m_averageLatency, (int)m_mapLatenciesLookup.size());
 
 
     if (timeDiff > 1000)
@@ -280,4 +292,16 @@ void FrameMetrics::SeedLatencyData(int latency)
 {
 	m_averageFps = GENERALS_ONLINE_HIGH_FPS_LIMIT;
 	m_averageLatency = latency / 1000.f;
+
+#if defined(USE_NEW_FRAMEMETRIC_LOGIC)
+	// the next response recomputes the average from these maps, so they must carry the seed too
+	for (std::unordered_map<UnsignedInt, Real>::iterator it = m_mapLatenciesLookup.begin(); it != m_mapLatenciesLookup.end(); ++it)
+	{
+		it->second = m_averageLatency;
+	}
+	for (std::map<UnsignedInt, Real>::iterator it = m_mapLatenciesSorted.begin(); it != m_mapLatenciesSorted.end(); ++it)
+	{
+		it->second = m_averageLatency;
+	}
+#endif
 }

GeneralsMD/Code/GameEngine/Include/Common/CustomMatchPreferences.h

@@ -44,8 +44,8 @@ class CustomMatchPreferences : public UserPreferences
 	CustomMatchPreferences();
 	virtual ~CustomMatchPreferences() override;
 
-	AsciiString getLastLobbyName() const;
-	void setLastLobbyName(const AsciiString& name);
+	UnicodeString getLastLobbyName() const;
+	void setLastLobbyName(const UnicodeString& name);
 
 	void setLastLadder(const AsciiString& addr, UnsignedShort port);
 	AsciiString getLastLadderAddr();

GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h

@@ -54,7 +54,7 @@ class HTTPManager
 	// NOTE: set bDisableServiceAuth for endpoints that must be called with something other than the session token (e.g. the refresh endpoint, which needs the refresh token), otherwise the session token overwrites any Authorization header passed in inHeaders
 	void SendPOSTRequest(const char* szURI, EIPProtocolVersion protover, std::map<std::string, std::string>& inHeaders, const char* szPostData, std::function<void(bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq)> completionCallback, std::function<void(size_t bytesReceived)> progressCallback = nullptr, int timeoutMS = -1, bool bDisableServiceAuth = false);
 	void SendPUTRequest(const char* szURI, EIPProtocolVersion protover, std::map<std::string, std::string>& inHeaders, const char* szData, std::function<void(bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq)> completionCallback, std::function<void(size_t bytesReceived)> progressCallback = nullptr, int timeoutMS = -1);
-	void SendS3PUTRequest(const char* szURI, EIPProtocolVersion protover, std::map<std::string, std::string>& inHeaders, std::vector<uint8_t> vecBuffer, std::function<void(bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq)> completionCallback, std::function<void(size_t bytesReceived)> progressCallback = nullptr, int timeoutMS = -1);
+	void SendS3PUTRequest(const char* szURI, EIPProtocolVersion protover, std::map<std::string, std::string>& inHeaders, std::vector<uint8_t> vecBuffer, std::function<void(bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq)> completionCallback, std::function<void(size_t bytesReceived)> progressCallback = nullptr, int timeoutMS = -1, int maxSendBytesPerSec = 0);
 	void SendDELETERequest(const char* szURI, EIPProtocolVersion protover, std::map<std::string, std::string>& inHeaders, const char* szData, std::function<void(bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq)> completionCallback, std::function<void(size_t bytesReceived)> progressCallback = nullptr, int timeoutMS = -1);
 
 	void Shutdown();

GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h

@@ -36,6 +36,11 @@ class HTTPRequest
 		m_bAppendAuthIfPresent = false;
 	}
 
+	void SetMaxSendSpeed(int bytesPerSec)
+	{
+		m_maxSendBytesPerSec = bytesPerSec;
+	}
+
 	void OnResponsePartialWrite(std::uint8_t* pBuffer, size_t numBytes);
 
 	bool HasStarted() const { return m_bIsStarted; }
@@ -77,6 +82,8 @@ class HTTPRequest
 
 	bool m_bAppendAuthIfPresent = true;
 
+	int m_maxSendBytesPerSec = 0; // 0 = unlimited
+
 	EIPProtocolVersion m_protover;
 
 	int m_timeoutMS = 5000;

GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/JsonHelpers.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "GameNetwork/GeneralsOnline/json.hpp"
+
+// Optional field: dest keeps its default when the key is missing or null
+template <typename T>
+inline void JsonGetOptional(const nlohmann::json& j, const char* szKey, T& dest)
+{
+	auto it = j.find(szKey);
+	if (it != j.end() && !it->is_null())
+	{
+		it->get_to(dest);
+	}
+}
+
+// Required field: throws when the key is missing or the wrong type
+template <typename T>
+inline void JsonGetRequired(const nlohmann::json& j, const char* szKey, T& dest)
+{
+	j.at(szKey).get_to(dest);
+}

GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/LobbyConnectionPolicy.h

@@ -0,0 +1,17 @@
+#pragma once
+#include <cstdint>
+#include <vector>
+
+namespace LobbyConnectionPolicy
+{
+inline bool IsCurrentAttempt(int64_t expectedLobby, uint64_t expectedGeneration, const std::vector<int64_t>& expectedMembers,
+    int64_t lobby, uint64_t generation, const std::vector<int64_t>& members)
+{
+    return expectedLobby >= 0 && expectedLobby == lobby && expectedGeneration == generation && expectedMembers == members;
+}
+
+inline bool CanRetry(bool manualLobby, bool inMatch, bool member, bool local, bool connected, bool connecting, int64_t sinceLastAttemptMs)
+{
+    return manualLobby && !inMatch && member && !local && !connected && !connecting && sinceLastAttemptMs >= 1000;
+}
+}