diff --git a/rubberduckvba.Server/GitHubSettings.cs b/rubberduckvba.Server/GitHubSettings.cs index fc941b6..4ca4143 100644 --- a/rubberduckvba.Server/GitHubSettings.cs +++ b/rubberduckvba.Server/GitHubSettings.cs @@ -47,6 +47,9 @@ public record class GitHubSettings public record class HangfireSettings { + public int MaxInitializationAttempts { get; set; } = 5; + public int InitializationRetryDelaySeconds { get; set; } = 10; + public int ServerCheckIntervalMinutes { get; set; } = 15; public int QueuePollIntervalSeconds { get; set; } = 30; public int SchedulePollIntervalSeconds { get; set; } = 30; diff --git a/rubberduckvba.Server/Program.cs b/rubberduckvba.Server/Program.cs index 7c628c5..2e204e6 100644 --- a/rubberduckvba.Server/Program.cs +++ b/rubberduckvba.Server/Program.cs @@ -7,6 +7,8 @@ using NLog.Config; using NLog.Extensions.Logging; using NLog.Targets; +using Polly; +using Polly.Retry; using Rubberduck.SmartIndenter; using RubberduckServices; using rubberduckvba.Server.Api.Admin; @@ -110,7 +112,25 @@ public static void Main(string[] args) app.MapControllers(); app.MapFallbackToFile("/index.html"); - StartHangfire(app); + var logger = app.Services.GetRequiredService>(); + logger.LogInformation("App configuration completed. Starting hangfire..."); + + var hangfireOptions = app.Services.GetService>()?.Value ?? new(); + new ResiliencePipelineBuilder().AddRetry(new RetryStrategyOptions + { + Delay = TimeSpan.FromSeconds(10), + MaxRetryAttempts = hangfireOptions.MaxInitializationAttempts, + OnRetry = (context) => + { + var retryCount = context.AttemptNumber; + var delay = context.RetryDelay; + + logger.LogError(context.Outcome.Exception, $"Hangfire failed to start | Retrying storage connection in {delay.TotalSeconds} seconds. Attempt {retryCount} of {hangfireOptions.MaxInitializationAttempts}"); + return ValueTask.CompletedTask; + } + }).Build().Execute(() => StartHangfire(app)); + + logger.LogInformation("Hangfire initialization completed. Starting application..."); app.Run(); } diff --git a/rubberduckvba.Server/rubberduckvba.Server.csproj b/rubberduckvba.Server/rubberduckvba.Server.csproj index 7af4d85..58c4d89 100644 --- a/rubberduckvba.Server/rubberduckvba.Server.csproj +++ b/rubberduckvba.Server/rubberduckvba.Server.csproj @@ -30,6 +30,7 @@ +