From 3c448f2290e8c1865fd7d5eaa59bf6c2f4ad85ff Mon Sep 17 00:00:00 2001 From: Professor Fartsalot Date: Thu, 4 Sep 2025 17:18:26 -0400 Subject: [PATCH 1/3] Fix health check, reconnect automatically on error + Made health check actually do something instead of just logging connection health issues. + Requires server code change to enable the health check so it's not just returning false all the time. + Forcibly reconnects the client when they've had connection issues. --- .../WebAPI/SignalR/ApiController.cs | 57 ++++++++++++++++++- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/MareSynchronos/WebAPI/SignalR/ApiController.cs b/MareSynchronos/WebAPI/SignalR/ApiController.cs index 6046198..bc6da4e 100644 --- a/MareSynchronos/WebAPI/SignalR/ApiController.cs +++ b/MareSynchronos/WebAPI/SignalR/ApiController.cs @@ -308,14 +308,25 @@ public sealed partial class ApiController : DisposableMediatorSubscriberBase, IM _ = Task.Run(async () => await StopConnection(ServerState.Disconnected).ConfigureAwait(false)); _connectionCancellationTokenSource?.Cancel(); } - + private int _unhealthy = 0; private async Task ClientHealthCheck(CancellationToken ct) { while (!ct.IsCancellationRequested && _mareHub != null) { await Task.Delay(TimeSpan.FromSeconds(30), ct).ConfigureAwait(false); - Logger.LogDebug("Checking Client Health State"); - _ = await CheckClientHealth().ConfigureAwait(false); + var healthy = await CheckClientHealth().ConfigureAwait(false); + Logger.LogDebug("Checking Client Health State returned {0}", healthy); + if (!healthy) + { + _unhealthy++; + if (_unhealthy > 1) + { + Logger.LogWarning("Health check failed more than once, forcing reconnect."); + await ForceResetConnection().ConfigureAwait(false); + _unhealthy = 0; + } + } + else _unhealthy = 0; } } @@ -478,5 +489,45 @@ public sealed partial class ApiController : DisposableMediatorSubscriberBase, IM ServerState = state; } + //Because this plugin really likes to bug out with connections, lets "fix" it.... + public async Task ForceResetConnection() + { + if (_mareHub == null || !_initialized) return; + Logger.LogInformation("ForceReconnect called"); + ServerState = ServerState.Reconnecting; + + try + { + // Cancel previous health checks to avoid overlaps + _healthCheckTokenSource?.Cancel(); + _healthCheckTokenSource?.Dispose(); + _healthCheckTokenSource = null; + + // Re-initialize hooks and refresh connection state + InitializeApiHooks(); + _connectionDto = await GetConnectionDto(publishConnected: false).ConfigureAwait(false); + + if (_connectionDto.ServerVersion != IMareHub.ApiVersion) + { + await StopConnection(ServerState.VersionMisMatch).ConfigureAwait(false); + return; + } + + ServerState = ServerState.Connected; + + // Reload all pairs + await LoadIninitialPairs().ConfigureAwait(false); + await LoadOnlinePairs().ConfigureAwait(false); + + Mediator.Publish(new ConnectedMessage(_connectionDto)); + Logger.LogInformation("ForceReconnect completed successfully"); + } + catch (Exception ex) + { + Logger.LogCritical(ex, "Failure during ForceReconnect, disconnecting"); + await StopConnection(ServerState.Disconnected).ConfigureAwait(false); + } + } + } #pragma warning restore MA0040 \ No newline at end of file From 18da07763c637e131611a4c2558376479f934f03 Mon Sep 17 00:00:00 2001 From: Professor Fartsalot Date: Thu, 4 Sep 2025 18:41:44 -0400 Subject: [PATCH 2/3] Add in extra check to see if hub server DC'd --- MareSynchronos/WebAPI/SignalR/ApiController.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MareSynchronos/WebAPI/SignalR/ApiController.cs b/MareSynchronos/WebAPI/SignalR/ApiController.cs index bc6da4e..57cd23b 100644 --- a/MareSynchronos/WebAPI/SignalR/ApiController.cs +++ b/MareSynchronos/WebAPI/SignalR/ApiController.cs @@ -315,8 +315,8 @@ public sealed partial class ApiController : DisposableMediatorSubscriberBase, IM { await Task.Delay(TimeSpan.FromSeconds(30), ct).ConfigureAwait(false); var healthy = await CheckClientHealth().ConfigureAwait(false); - Logger.LogDebug("Checking Client Health State returned {0}", healthy); - if (!healthy) + Logger.LogDebug("Checking Client Health State returned {0} and hub connection {1}", healthy, _mareHub.State == HubConnectionState.Connected); + if (!healthy || _mareHub.State != HubConnectionState.Connected) { _unhealthy++; if (_unhealthy > 1) From d8160effd0f7da45defb45dcff383866b58c0c21 Mon Sep 17 00:00:00 2001 From: Professor Fartsalot Date: Thu, 4 Sep 2025 18:53:57 -0400 Subject: [PATCH 3/3] Fix up client reconnect code Simplified the reconnect code so that redis is made aware that we have a new connection. Made the reconnect fire immediately on error rather than waiting for subsequent errors. --- .../WebAPI/SignalR/ApiController.cs | 36 ++++++------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/MareSynchronos/WebAPI/SignalR/ApiController.cs b/MareSynchronos/WebAPI/SignalR/ApiController.cs index 57cd23b..1351a3d 100644 --- a/MareSynchronos/WebAPI/SignalR/ApiController.cs +++ b/MareSynchronos/WebAPI/SignalR/ApiController.cs @@ -183,6 +183,8 @@ public sealed partial class ApiController : DisposableMediatorSubscriberBase, IM _connectionDto = await GetConnectionDto().ConfigureAwait(false); + await CheckClientHealth().ConfigureAwait(false); + ServerState = ServerState.Connected; var currentClientVer = Assembly.GetExecutingAssembly().GetName().Version!; @@ -315,13 +317,12 @@ public sealed partial class ApiController : DisposableMediatorSubscriberBase, IM { await Task.Delay(TimeSpan.FromSeconds(30), ct).ConfigureAwait(false); var healthy = await CheckClientHealth().ConfigureAwait(false); - Logger.LogDebug("Checking Client Health State returned {0} and hub connection {1}", healthy, _mareHub.State == HubConnectionState.Connected); if (!healthy || _mareHub.State != HubConnectionState.Connected) { _unhealthy++; - if (_unhealthy > 1) + if (_unhealthy > 0) { - Logger.LogWarning("Health check failed more than once, forcing reconnect."); + Logger.LogWarning("Health check failed, forcing reconnect. ClientHealth: {0} HubConnected: {1}", healthy, _mareHub.State != HubConnectionState.Connected); await ForceResetConnection().ConfigureAwait(false); _unhealthy = 0; } @@ -402,7 +403,7 @@ public sealed partial class ApiController : DisposableMediatorSubscriberBase, IM { var users = await GroupsGetUsersInGroup(group).ConfigureAwait(false); foreach (var user in users) - { + { Logger.LogDebug("Group Pair: {user}", user); _pairManager.AddGroupPair(user); } @@ -492,42 +493,27 @@ public sealed partial class ApiController : DisposableMediatorSubscriberBase, IM //Because this plugin really likes to bug out with connections, lets "fix" it.... public async Task ForceResetConnection() { - if (_mareHub == null || !_initialized) return; + if (!_initialized) return; Logger.LogInformation("ForceReconnect called"); - ServerState = ServerState.Reconnecting; try { - // Cancel previous health checks to avoid overlaps + await StopConnection(ServerState.Disconnected).ConfigureAwait(false); + + // Cancel any ongoing health checks to prevent conflicts _healthCheckTokenSource?.Cancel(); _healthCheckTokenSource?.Dispose(); _healthCheckTokenSource = null; - // Re-initialize hooks and refresh connection state - InitializeApiHooks(); - _connectionDto = await GetConnectionDto(publishConnected: false).ConfigureAwait(false); + await CreateConnections().ConfigureAwait(false); - if (_connectionDto.ServerVersion != IMareHub.ApiVersion) - { - await StopConnection(ServerState.VersionMisMatch).ConfigureAwait(false); - return; - } - - ServerState = ServerState.Connected; - - // Reload all pairs - await LoadIninitialPairs().ConfigureAwait(false); - await LoadOnlinePairs().ConfigureAwait(false); - - Mediator.Publish(new ConnectedMessage(_connectionDto)); Logger.LogInformation("ForceReconnect completed successfully"); } catch (Exception ex) { - Logger.LogCritical(ex, "Failure during ForceReconnect, disconnecting"); + Logger.LogError(ex, "Failure during ForceReconnect, disconnecting"); await StopConnection(ServerState.Disconnected).ConfigureAwait(false); } } - } #pragma warning restore MA0040 \ No newline at end of file