Add API liveness probes

osu.Game.Tests/Visual/UserInterface/TestSceneNotificationOverlay.cs

@@ -64,6 +64,8 @@ public void TestBasicFlow()
             AddStep(@"simple #2", sendAmazingNotification);
             AddStep(@"progress #1", sendUploadProgress);
             AddStep(@"progress #2", sendDownloadProgress);
+            AddStep("outage", () => notificationOverlay.Post(new OutageNotification("Things are on fire. Investigating...")));
+            AddStep("failed submission", () => notificationOverlay.Post(new ScoreSubmissionFailureNotification("Score will not be submitted", "This beatmap does not match the online version. Please update or redownload it.")));
 
             checkProgressingCount(2);
 

osu.Game/Online/API/APIAccess.cs

@@ -11,6 +11,7 @@
 using System.Net.Sockets;
 using System.Threading;
 using System.Threading.Tasks;
+using JetBrains.Annotations;
 using Newtonsoft.Json.Linq;
 using osu.Framework.Bindables;
 using osu.Framework.Development;
@@ -25,6 +26,7 @@
 using osu.Game.Online.API.Requests.Responses;
 using osu.Game.Online.Chat;
 using osu.Game.Online.Notifications.WebSocket;
+using osu.Game.Overlays.Notifications;
 
 namespace osu.Game.Online.API
 {
@@ -71,6 +73,9 @@ public partial class APIAccess : CompositeComponent, IAPIProvider
         private readonly CancellationTokenSource cancellationToken = new CancellationTokenSource();
         private readonly Logger log;
 
+        [CanBeNull]
+        public Action<Notification> PostNotification { get; set; }
+
         public APIAccess(OsuGameBase game, OsuConfigManager config, EndpointConfiguration endpoints, string versionHash)
         {
             this.game = game;
@@ -156,6 +161,8 @@ private WebSocketNotificationsClientConnector setUpNotificationsClient()
         /// </summary>
         private int failureCount;
 
+        private readonly Stopwatch livenessStopwatch = new Stopwatch();
+
         /// <summary>
         /// The main API thread loop, which will continue to run until the game is shut down.
         /// </summary>
@@ -165,10 +172,14 @@ private void run()
             {
                 if (state.Value == APIState.Failing)
                 {
-                    // To recover from a failing state, falling through and running the full reconnection process seems safest for now.
-                    // This could probably be replaced with a ping-style request if we want to avoid the reconnection overheads.
                     log.Add($@"{nameof(APIAccess)} is in a failing state, waiting a bit before we try again...");
                     Thread.Sleep(5000);
+
+                    // if the liveness probe actively returns a failure state, there's no need to be retrying anything
+                    if (!probeLiveness(out _))
+                        continue;
+
+                    // In any other circumstance, let's attempt the full reconnection flow.
                 }
 
                 // Ensure that we have valid credentials.
@@ -201,11 +212,77 @@ private void run()
                     continue;
                 }
 
+                if (livenessStopwatch.Elapsed.TotalMinutes >= 1)
+                {
+                    bool alive = probeLiveness(out string reason);
+
+                    if (!alive)
+                    {
+                        triggerOutage(reason);
+                        livenessStopwatch.Stop();
+                        continue;
+                    }
+
+                    livenessStopwatch.Restart();
+                }
+
                 processQueuedRequests();
                 Thread.Sleep(50);
             }
         }
 
+        /// <summary>
+        /// Query the liveness probe to check whether to transition to / remain in <see cref="APIState.Failing"/> state.
+        /// </summary>
+        /// <returns>
+        /// <see langword="true"/> if the liveness probe is disabled, returns that online functions are available, or cannot be reached.
+        /// <see langword="false"/> if the liveness probe explicitly returns that online functions are not available. A user-facing message may be returned via <paramref name="reason"/>.
+        /// </returns>
+        private bool probeLiveness([CanBeNull] out string reason)
+        {
+            if (Endpoints.LivenessProbeUrl == null)
+            {
+                reason = null;
+                return true;
+            }
+
+            var req = new OsuJsonWebRequest<LivenessProbeResponse>(Endpoints.LivenessProbeUrl);
+
+            try
+            {
+                req.Perform();
+            }
+            catch (Exception ex)
+            {
+                Logger.Log($"Liveness probe failed: {ex}", LoggingTarget.Network);
+                reason = null;
+                return true;
+            }
+
+            if (req.Aborted || req.ResponseObject == null)
+            {
+                reason = null;
+                return true;
+            }
+
+            reason = req.ResponseObject.Reason;
+            return req.ResponseObject.Status == LivenessProbeResponse.LivenessStatus.Up;
+        }
+
+        private void triggerOutage(string reason)
+        {
+            state.Value = APIState.Failing;
+            string userFacingMessage = reason ?? "Online functionality is not available due to an outage. Sorry for the inconvenience.";
+
+            Schedule(() =>
+            {
+                if (PostNotification != null)
+                    PostNotification?.Invoke(new OutageNotification(userFacingMessage));
+                else
+                    log.Add(userFacingMessage, LogLevel.Important);
+            });
+        }
+
         /// <summary>
         /// Dequeue from the queue and run each request synchronously until the queue is empty.
         /// </summary>
@@ -245,7 +322,13 @@ private void attemptConnect()
             // save the username at this point, if the user requested for it to be.
             config.SetValue(OsuSetting.Username, config.Get<bool>(OsuSetting.SaveUsername) ? ProvidedUsername : string.Empty);
 
-            if (!authentication.HasValidAccessToken && HasLogin)
+            if (!probeLiveness(out string reason))
+            {
+                triggerOutage(reason);
+                return;
+            }
+
+            if (!authentication.HasValidAccessToken)
             {
                 state.Value = APIState.Connecting;
                 LastLoginError = null;
@@ -341,6 +424,7 @@ private void attemptConnect()
                         localUserState.SetLocalUser(me);
                         SessionVerificationMethod = me.SessionVerificationMethod;
                         state.Value = SessionVerificationMethod == null ? APIState.Online : APIState.RequiresSecondFactorAuth;
+                        livenessStopwatch.Restart();
                         failureCount = 0;
                     };
 

osu.Game/Online/API/Requests/Responses/LivenessProbeResponse.cs

@@ -0,0 +1,26 @@
+// Copyright (c) ppy Pty Ltd <[email protected]>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System.Runtime.Serialization;
+using Newtonsoft.Json;
+
+namespace osu.Game.Online.API.Requests.Responses
+{
+    public class LivenessProbeResponse
+    {
+        [JsonProperty("status")]
+        public LivenessStatus Status { get; set; }
+
+        [JsonProperty("reason")]
+        public string? Reason { get; set; }
+
+        public enum LivenessStatus
+        {
+            [EnumMember(Value = "up")]
+            Up,
+
+            [EnumMember(Value = "down")]
+            Down,
+        }
+    }
+}

osu.Game/Online/EndpointConfiguration.cs

@@ -47,5 +47,19 @@ public class EndpointConfiguration
         /// The endpoint for the SignalR metadata server.
         /// </summary>
         public string MetadataUrl { get; set; } = string.Empty;
+
+        /// <summary>
+        /// The URL to a separate endpoint that serves as a "liveness probe" for online services, indicating any potential active outages.
+        /// </summary>
+        /// <remarks>
+        /// <list type="bullet">
+        /// <item>The liveness probe's presence is optional. If this is <see langword="null"/>, the entire mechanism predicated on it will be turned off.</item>
+        /// <item>
+        /// The liveness probe only has any effect if it is reachable and actively returns a response that indicates an ongoing outage.
+        /// Failing to reach the liveness probe has no effect as it is indistinguishable from a problem that is local to the machine the client is running on.
+        /// </item>
+        /// </list>
+        /// </remarks>
+        public string? LivenessProbeUrl { get; set; }
     }
 }

osu.Game/OsuGame.cs

@@ -45,6 +45,7 @@
 using osu.Game.IO;
 using osu.Game.Localisation;
 using osu.Game.Online;
+using osu.Game.Online.API;
 using osu.Game.Online.API.Requests;
 using osu.Game.Online.Chat;
 using osu.Game.Online.Leaderboards;
@@ -1098,6 +1099,9 @@ protected override void LoadComplete()
             MultiplayerClient.PostNotification = n => Notifications.Post(n);
             MultiplayerClient.PresentMatch = PresentMultiplayerMatch;
 
+            if (API is APIAccess api)
+                api.PostNotification = n => Notifications.Post(n);
+
             ScreenFooter.BackReceptor backReceptor;
 
             dependencies.CacheAs(idleTracker = new GameIdleTracker(6000));

osu.Game/Overlays/Notifications/OutageNotification.cs

@@ -0,0 +1,38 @@
+// Copyright (c) ppy Pty Ltd <[email protected]>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using osu.Framework.Allocation;
+using osu.Framework.Graphics;
+using osu.Framework.Graphics.Colour;
+using osu.Framework.Graphics.Sprites;
+using osu.Game.Graphics;
+
+namespace osu.Game.Overlays.Notifications
+{
+    public partial class OutageNotification : SimpleNotification
+    {
+        private readonly string message;
+
+        public OutageNotification(string message)
+        {
+            Text = this.message = message;
+
+            IsCritical = true;
+        }
+
+        [BackgroundDependencyLoader]
+        private void load()
+        {
+            Icon = FontAwesome.Solid.FireExtinguisher;
+            IconContent.Colour = ColourInfo.GradientVertical(Colour4.Orange, Colour4.OrangeRed);
+
+            TextFlow.Clear();
+            TextFlow.AddText("Online server outage in progress".ToUpperInvariant(), s =>
+            {
+                s.Font = OsuFont.Style.Caption2.With(weight: FontWeight.Bold);
+                s.Colour = Colour4.Orange;
+            });
+            TextFlow.AddParagraph(message, s => s.Font = OsuFont.Style.Caption1);
+        }
+    }
+}

osu.Game/Overlays/Notifications/ScoreSubmissionFailureNotification.cs

@@ -0,0 +1,40 @@
+// Copyright (c) ppy Pty Ltd <[email protected]>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using osu.Framework.Allocation;
+using osu.Framework.Graphics.Sprites;
+using osu.Game.Graphics;
+
+namespace osu.Game.Overlays.Notifications
+{
+    public partial class ScoreSubmissionFailureNotification : SimpleNotification
+    {
+        private readonly string heading;
+        private readonly string reason;
+
+        public ScoreSubmissionFailureNotification(string heading, string reason)
+        {
+            this.heading = heading;
+            this.reason = reason;
+
+            IsCritical = true;
+
+            Text = $"{heading}: {reason}";
+        }
+
+        [BackgroundDependencyLoader]
+        private void load(OsuColour colours)
+        {
+            Icon = FontAwesome.Solid.Unlink;
+            IconContent.Colour = colours.RedDark;
+
+            TextFlow.Clear();
+            TextFlow.AddText(heading.ToUpperInvariant(), s =>
+            {
+                s.Font = OsuFont.Style.Caption2.With(weight: FontWeight.Bold);
+                s.Colour = colours.Red0;
+            });
+            TextFlow.AddParagraph(reason, s => s.Font = OsuFont.Style.Caption1);
+        }
+    }
+}

osu.Game/Screens/Play/SubmittingPlayer.cs

@@ -19,6 +19,8 @@
 using osu.Game.Online.Multiplayer;
 using osu.Game.Online.Rooms;
 using osu.Game.Online.Spectator;
+using osu.Game.Overlays;
+using osu.Game.Overlays.Notifications;
 using osu.Game.Rulesets.Scoring;
 using osu.Game.Scoring;
 using osu.Game.Screens.Ranking;
@@ -48,6 +50,10 @@ public abstract partial class SubmittingPlayer : Player
         [CanBeNull]
         private UserStatisticsWatcher userStatisticsWatcher { get; set; }
 
+        [Resolved(canBeNull: true)]
+        [CanBeNull]
+        private INotificationOverlay notifications { get; set; }
+
         private readonly object scoreSubmissionLock = new object();
         private TaskCompletionSource<bool> scoreSubmissionSource;
 
@@ -99,9 +105,9 @@ private bool handleTokenRetrieval()
                 return false;
             }
 
-            if (!api.IsLoggedIn)
+            if (!api.IsLoggedIn || api.State.Value == APIState.Failing)
             {
-                handleTokenFailure(new InvalidOperationException("API is not online."));
+                handleTokenFailure(new InvalidOperationException("Online functionality is not available."), displayNotification: api.State.Value == APIState.Failing);
                 return false;
             }
 
@@ -138,31 +144,31 @@ void handleTokenFailure(Exception exception, bool displayNotification = false)
                 if (displayNotification || shouldExit)
                 {
                     string whatWillHappen = shouldExit
-                        ? "Play in this state is not permitted."
-                        : "Your score will not be submitted.";
+                        ? "Cannot start play"
+                        : "Score will not be submitted";
 
                     if (string.IsNullOrEmpty(exception.Message))
-                        Logger.Error(exception, $"Failed to retrieve a score submission token.\n\n{whatWillHappen}");
+                        notifications?.Post(new ScoreSubmissionFailureNotification(whatWillHappen, "Failed to retrieve a score submission token."));
                     else
                     {
                         switch (exception.Message)
                         {
                             case @"missing token header":
                             case @"invalid client hash":
                             case @"invalid verification hash":
-                                Logger.Log($"Please ensure that you are using the latest version of the official game releases.\n\n{whatWillHappen}", level: LogLevel.Important);
+                                notifications?.Post(new ScoreSubmissionFailureNotification(whatWillHappen, "Please ensure that you are using the latest version of the official game releases."));
                                 break;
 
                             case @"invalid or missing beatmap_hash":
-                                Logger.Log($"This beatmap does not match the online version. Please update or redownload it.\n\n{whatWillHappen}", level: LogLevel.Important);
+                                notifications?.Post(new ScoreSubmissionFailureNotification(whatWillHappen, "This beatmap does not match the online version. Please update or redownload it."));
                                 break;
 
                             case @"expired token":
-                                Logger.Log($"Your system clock is set incorrectly. Please check your system time, date and timezone.\n\n{whatWillHappen}", level: LogLevel.Important);
+                                notifications?.Post(new ScoreSubmissionFailureNotification(whatWillHappen, "Your system clock is set incorrectly. Please check your system time, date and timezone."));
                                 break;
 
                             default:
-                                Logger.Log($"{whatWillHappen} {exception.Message}", level: LogLevel.Important);
+                                notifications?.Post(new ScoreSubmissionFailureNotification(whatWillHappen, exception.Message));
                                 break;
                         }
                     }