diff --git a/osu.Game.Tests/Visual/UserInterface/TestSceneNotificationOverlay.cs b/osu.Game.Tests/Visual/UserInterface/TestSceneNotificationOverlay.cs index 9d23b2130a1a..eadb713f688a 100644 --- a/osu.Game.Tests/Visual/UserInterface/TestSceneNotificationOverlay.cs +++ b/osu.Game.Tests/Visual/UserInterface/TestSceneNotificationOverlay.cs @@ -64,6 +64,8 @@ public void TestBasicFlow() AddStep(@"simple #2", sendAmazingNotification); AddStep(@"progress #1", sendUploadProgress); AddStep(@"progress #2", sendDownloadProgress); + AddStep("outage", () => notificationOverlay.Post(new OutageNotification("Things are on fire. Investigating..."))); + AddStep("failed submission", () => notificationOverlay.Post(new ScoreSubmissionFailureNotification("Score will not be submitted", "This beatmap does not match the online version. Please update or redownload it."))); checkProgressingCount(2); diff --git a/osu.Game/Online/API/APIAccess.cs b/osu.Game/Online/API/APIAccess.cs index 6694003b31d9..def58f60da61 100644 --- a/osu.Game/Online/API/APIAccess.cs +++ b/osu.Game/Online/API/APIAccess.cs @@ -11,6 +11,7 @@ using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; +using JetBrains.Annotations; using Newtonsoft.Json.Linq; using osu.Framework.Bindables; using osu.Framework.Development; @@ -25,6 +26,7 @@ using osu.Game.Online.API.Requests.Responses; using osu.Game.Online.Chat; using osu.Game.Online.Notifications.WebSocket; +using osu.Game.Overlays.Notifications; namespace osu.Game.Online.API { @@ -71,6 +73,9 @@ public partial class APIAccess : CompositeComponent, IAPIProvider private readonly CancellationTokenSource cancellationToken = new CancellationTokenSource(); private readonly Logger log; + [CanBeNull] + public Action PostNotification { get; set; } + public APIAccess(OsuGameBase game, OsuConfigManager config, EndpointConfiguration endpoints, string versionHash) { this.game = game; @@ -156,6 +161,8 @@ private WebSocketNotificationsClientConnector setUpNotificationsClient() /// private int failureCount; + private readonly Stopwatch livenessStopwatch = new Stopwatch(); + /// /// The main API thread loop, which will continue to run until the game is shut down. /// @@ -165,10 +172,14 @@ private void run() { if (state.Value == APIState.Failing) { - // To recover from a failing state, falling through and running the full reconnection process seems safest for now. - // This could probably be replaced with a ping-style request if we want to avoid the reconnection overheads. log.Add($@"{nameof(APIAccess)} is in a failing state, waiting a bit before we try again..."); Thread.Sleep(5000); + + // if the liveness probe actively returns a failure state, there's no need to be retrying anything + if (!probeLiveness(out _)) + continue; + + // In any other circumstance, let's attempt the full reconnection flow. } // Ensure that we have valid credentials. @@ -201,11 +212,77 @@ private void run() continue; } + if (livenessStopwatch.Elapsed.TotalMinutes >= 1) + { + bool alive = probeLiveness(out string reason); + + if (!alive) + { + triggerOutage(reason); + livenessStopwatch.Stop(); + continue; + } + + livenessStopwatch.Restart(); + } + processQueuedRequests(); Thread.Sleep(50); } } + /// + /// Query the liveness probe to check whether to transition to / remain in state. + /// + /// + /// if the liveness probe is disabled, returns that online functions are available, or cannot be reached. + /// if the liveness probe explicitly returns that online functions are not available. A user-facing message may be returned via . + /// + private bool probeLiveness([CanBeNull] out string reason) + { + if (Endpoints.LivenessProbeUrl == null) + { + reason = null; + return true; + } + + var req = new OsuJsonWebRequest(Endpoints.LivenessProbeUrl); + + try + { + req.Perform(); + } + catch (Exception ex) + { + Logger.Log($"Liveness probe failed: {ex}", LoggingTarget.Network); + reason = null; + return true; + } + + if (req.Aborted || req.ResponseObject == null) + { + reason = null; + return true; + } + + reason = req.ResponseObject.Reason; + return req.ResponseObject.Status == LivenessProbeResponse.LivenessStatus.Up; + } + + private void triggerOutage(string reason) + { + state.Value = APIState.Failing; + string userFacingMessage = reason ?? "Online functionality is not available due to an outage. Sorry for the inconvenience."; + + Schedule(() => + { + if (PostNotification != null) + PostNotification?.Invoke(new OutageNotification(userFacingMessage)); + else + log.Add(userFacingMessage, LogLevel.Important); + }); + } + /// /// Dequeue from the queue and run each request synchronously until the queue is empty. /// @@ -245,7 +322,13 @@ private void attemptConnect() // save the username at this point, if the user requested for it to be. config.SetValue(OsuSetting.Username, config.Get(OsuSetting.SaveUsername) ? ProvidedUsername : string.Empty); - if (!authentication.HasValidAccessToken && HasLogin) + if (!probeLiveness(out string reason)) + { + triggerOutage(reason); + return; + } + + if (!authentication.HasValidAccessToken) { state.Value = APIState.Connecting; LastLoginError = null; @@ -341,6 +424,7 @@ private void attemptConnect() localUserState.SetLocalUser(me); SessionVerificationMethod = me.SessionVerificationMethod; state.Value = SessionVerificationMethod == null ? APIState.Online : APIState.RequiresSecondFactorAuth; + livenessStopwatch.Restart(); failureCount = 0; }; diff --git a/osu.Game/Online/API/Requests/Responses/LivenessProbeResponse.cs b/osu.Game/Online/API/Requests/Responses/LivenessProbeResponse.cs new file mode 100644 index 000000000000..ca47c9a4be69 --- /dev/null +++ b/osu.Game/Online/API/Requests/Responses/LivenessProbeResponse.cs @@ -0,0 +1,26 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System.Runtime.Serialization; +using Newtonsoft.Json; + +namespace osu.Game.Online.API.Requests.Responses +{ + public class LivenessProbeResponse + { + [JsonProperty("status")] + public LivenessStatus Status { get; set; } + + [JsonProperty("reason")] + public string? Reason { get; set; } + + public enum LivenessStatus + { + [EnumMember(Value = "up")] + Up, + + [EnumMember(Value = "down")] + Down, + } + } +} diff --git a/osu.Game/Online/EndpointConfiguration.cs b/osu.Game/Online/EndpointConfiguration.cs index 2d5ea3234523..2ce14320ec37 100644 --- a/osu.Game/Online/EndpointConfiguration.cs +++ b/osu.Game/Online/EndpointConfiguration.cs @@ -47,5 +47,19 @@ public class EndpointConfiguration /// The endpoint for the SignalR metadata server. /// public string MetadataUrl { get; set; } = string.Empty; + + /// + /// The URL to a separate endpoint that serves as a "liveness probe" for online services, indicating any potential active outages. + /// + /// + /// + /// The liveness probe's presence is optional. If this is , the entire mechanism predicated on it will be turned off. + /// + /// The liveness probe only has any effect if it is reachable and actively returns a response that indicates an ongoing outage. + /// Failing to reach the liveness probe has no effect as it is indistinguishable from a problem that is local to the machine the client is running on. + /// + /// + /// + public string? LivenessProbeUrl { get; set; } } } diff --git a/osu.Game/OsuGame.cs b/osu.Game/OsuGame.cs index 4ea9fae1838b..6aab4535ae82 100644 --- a/osu.Game/OsuGame.cs +++ b/osu.Game/OsuGame.cs @@ -45,6 +45,7 @@ using osu.Game.IO; using osu.Game.Localisation; using osu.Game.Online; +using osu.Game.Online.API; using osu.Game.Online.API.Requests; using osu.Game.Online.Chat; using osu.Game.Online.Leaderboards; @@ -1098,6 +1099,9 @@ protected override void LoadComplete() MultiplayerClient.PostNotification = n => Notifications.Post(n); MultiplayerClient.PresentMatch = PresentMultiplayerMatch; + if (API is APIAccess api) + api.PostNotification = n => Notifications.Post(n); + ScreenFooter.BackReceptor backReceptor; dependencies.CacheAs(idleTracker = new GameIdleTracker(6000)); diff --git a/osu.Game/Overlays/Notifications/OutageNotification.cs b/osu.Game/Overlays/Notifications/OutageNotification.cs new file mode 100644 index 000000000000..1785e8498bef --- /dev/null +++ b/osu.Game/Overlays/Notifications/OutageNotification.cs @@ -0,0 +1,38 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using osu.Framework.Allocation; +using osu.Framework.Graphics; +using osu.Framework.Graphics.Colour; +using osu.Framework.Graphics.Sprites; +using osu.Game.Graphics; + +namespace osu.Game.Overlays.Notifications +{ + public partial class OutageNotification : SimpleNotification + { + private readonly string message; + + public OutageNotification(string message) + { + Text = this.message = message; + + IsCritical = true; + } + + [BackgroundDependencyLoader] + private void load() + { + Icon = FontAwesome.Solid.FireExtinguisher; + IconContent.Colour = ColourInfo.GradientVertical(Colour4.Orange, Colour4.OrangeRed); + + TextFlow.Clear(); + TextFlow.AddText("Online server outage in progress".ToUpperInvariant(), s => + { + s.Font = OsuFont.Style.Caption2.With(weight: FontWeight.Bold); + s.Colour = Colour4.Orange; + }); + TextFlow.AddParagraph(message, s => s.Font = OsuFont.Style.Caption1); + } + } +} diff --git a/osu.Game/Overlays/Notifications/ScoreSubmissionFailureNotification.cs b/osu.Game/Overlays/Notifications/ScoreSubmissionFailureNotification.cs new file mode 100644 index 000000000000..581be7dc31f0 --- /dev/null +++ b/osu.Game/Overlays/Notifications/ScoreSubmissionFailureNotification.cs @@ -0,0 +1,40 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using osu.Framework.Allocation; +using osu.Framework.Graphics.Sprites; +using osu.Game.Graphics; + +namespace osu.Game.Overlays.Notifications +{ + public partial class ScoreSubmissionFailureNotification : SimpleNotification + { + private readonly string heading; + private readonly string reason; + + public ScoreSubmissionFailureNotification(string heading, string reason) + { + this.heading = heading; + this.reason = reason; + + IsCritical = true; + + Text = $"{heading}: {reason}"; + } + + [BackgroundDependencyLoader] + private void load(OsuColour colours) + { + Icon = FontAwesome.Solid.Unlink; + IconContent.Colour = colours.RedDark; + + TextFlow.Clear(); + TextFlow.AddText(heading.ToUpperInvariant(), s => + { + s.Font = OsuFont.Style.Caption2.With(weight: FontWeight.Bold); + s.Colour = colours.Red0; + }); + TextFlow.AddParagraph(reason, s => s.Font = OsuFont.Style.Caption1); + } + } +} diff --git a/osu.Game/Screens/Play/SubmittingPlayer.cs b/osu.Game/Screens/Play/SubmittingPlayer.cs index 06f1a9c53073..2de8cf2ec5e8 100644 --- a/osu.Game/Screens/Play/SubmittingPlayer.cs +++ b/osu.Game/Screens/Play/SubmittingPlayer.cs @@ -19,6 +19,8 @@ using osu.Game.Online.Multiplayer; using osu.Game.Online.Rooms; using osu.Game.Online.Spectator; +using osu.Game.Overlays; +using osu.Game.Overlays.Notifications; using osu.Game.Rulesets.Scoring; using osu.Game.Scoring; using osu.Game.Screens.Ranking; @@ -48,6 +50,10 @@ public abstract partial class SubmittingPlayer : Player [CanBeNull] private UserStatisticsWatcher userStatisticsWatcher { get; set; } + [Resolved(canBeNull: true)] + [CanBeNull] + private INotificationOverlay notifications { get; set; } + private readonly object scoreSubmissionLock = new object(); private TaskCompletionSource scoreSubmissionSource; @@ -99,9 +105,9 @@ private bool handleTokenRetrieval() return false; } - if (!api.IsLoggedIn) + if (!api.IsLoggedIn || api.State.Value == APIState.Failing) { - handleTokenFailure(new InvalidOperationException("API is not online.")); + handleTokenFailure(new InvalidOperationException("Online functionality is not available."), displayNotification: api.State.Value == APIState.Failing); return false; } @@ -138,11 +144,11 @@ void handleTokenFailure(Exception exception, bool displayNotification = false) if (displayNotification || shouldExit) { string whatWillHappen = shouldExit - ? "Play in this state is not permitted." - : "Your score will not be submitted."; + ? "Cannot start play" + : "Score will not be submitted"; if (string.IsNullOrEmpty(exception.Message)) - Logger.Error(exception, $"Failed to retrieve a score submission token.\n\n{whatWillHappen}"); + notifications?.Post(new ScoreSubmissionFailureNotification(whatWillHappen, "Failed to retrieve a score submission token.")); else { switch (exception.Message) @@ -150,19 +156,19 @@ void handleTokenFailure(Exception exception, bool displayNotification = false) case @"missing token header": case @"invalid client hash": case @"invalid verification hash": - Logger.Log($"Please ensure that you are using the latest version of the official game releases.\n\n{whatWillHappen}", level: LogLevel.Important); + notifications?.Post(new ScoreSubmissionFailureNotification(whatWillHappen, "Please ensure that you are using the latest version of the official game releases.")); break; case @"invalid or missing beatmap_hash": - Logger.Log($"This beatmap does not match the online version. Please update or redownload it.\n\n{whatWillHappen}", level: LogLevel.Important); + notifications?.Post(new ScoreSubmissionFailureNotification(whatWillHappen, "This beatmap does not match the online version. Please update or redownload it.")); break; case @"expired token": - Logger.Log($"Your system clock is set incorrectly. Please check your system time, date and timezone.\n\n{whatWillHappen}", level: LogLevel.Important); + notifications?.Post(new ScoreSubmissionFailureNotification(whatWillHappen, "Your system clock is set incorrectly. Please check your system time, date and timezone.")); break; default: - Logger.Log($"{whatWillHappen} {exception.Message}", level: LogLevel.Important); + notifications?.Post(new ScoreSubmissionFailureNotification(whatWillHappen, exception.Message)); break; } }