| | | 1 | | using AsiBackbone.Core.Outbox; |
| | | 2 | | using Microsoft.Extensions.DependencyInjection; |
| | | 3 | | using Microsoft.Extensions.Hosting; |
| | | 4 | | using Microsoft.Extensions.Logging; |
| | | 5 | | using Microsoft.Extensions.Options; |
| | | 6 | | |
| | | 7 | | namespace AsiBackbone.AspNetCore.Outbox; |
| | | 8 | | |
| | | 9 | | /// <summary> |
| | | 10 | | /// Runs the provider-neutral governance outbox drain from an ASP.NET Core or generic-host background worker. |
| | | 11 | | /// </summary> |
| | | 12 | | /// <remarks> |
| | | 13 | | /// Hosting remains outside Core. The worker resolves the drain through a scoped service provider so durable providers t |
| | | 14 | | /// </remarks> |
| | 20 | 15 | | public sealed class AsiBackboneGovernanceOutboxDrainHostedService( |
| | 20 | 16 | | IServiceScopeFactory scopeFactory, |
| | 20 | 17 | | IOptionsMonitor<AsiBackboneGovernanceOutboxDrainWorkerOptions> optionsMonitor, |
| | 20 | 18 | | ILogger<AsiBackboneGovernanceOutboxDrainHostedService> logger) : BackgroundService |
| | | 19 | | { |
| | 1 | 20 | | private static readonly Action<ILogger, Exception?> LogShutdownDrainCanceled = LoggerMessage.Define( |
| | 1 | 21 | | LogLevel.Debug, |
| | 1 | 22 | | new EventId(19801, nameof(LogShutdownDrainCanceled)), |
| | 1 | 23 | | "Governance outbox shutdown drain was canceled."); |
| | | 24 | | |
| | 1 | 25 | | private static readonly Action<ILogger, Exception?> LogShutdownDrainFailed = LoggerMessage.Define( |
| | 1 | 26 | | LogLevel.Warning, |
| | 1 | 27 | | new EventId(19802, nameof(LogShutdownDrainFailed)), |
| | 1 | 28 | | "Governance outbox shutdown drain failed."); |
| | | 29 | | |
| | 1 | 30 | | private static readonly Action<ILogger, Exception?> LogWorkerDisabled = LoggerMessage.Define( |
| | 1 | 31 | | LogLevel.Debug, |
| | 1 | 32 | | new EventId(19803, nameof(LogWorkerDisabled)), |
| | 1 | 33 | | "Governance outbox drain worker is disabled."); |
| | | 34 | | |
| | 1 | 35 | | private static readonly Action<ILogger, int, Exception?> LogDrainAttempted = LoggerMessage.Define<int>( |
| | 1 | 36 | | LogLevel.Debug, |
| | 1 | 37 | | new EventId(19804, nameof(LogDrainAttempted)), |
| | 1 | 38 | | "Governance outbox drain attempted {DrainedCount} entries."); |
| | | 39 | | |
| | 1 | 40 | | private static readonly Action<ILogger, Exception?> LogWorkerFailed = LoggerMessage.Define( |
| | 1 | 41 | | LogLevel.Warning, |
| | 1 | 42 | | new EventId(19805, nameof(LogWorkerFailed)), |
| | 1 | 43 | | "Governance outbox drain worker failed before the next polling interval."); |
| | | 44 | | |
| | 20 | 45 | | private readonly IServiceScopeFactory scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFac |
| | 20 | 46 | | private readonly IOptionsMonitor<AsiBackboneGovernanceOutboxDrainWorkerOptions> optionsMonitor = optionsMonitor ?? t |
| | 20 | 47 | | private readonly ILogger<AsiBackboneGovernanceOutboxDrainHostedService> logger = logger ?? throw new ArgumentNullExc |
| | 20 | 48 | | private readonly Lock optionsChangedSync = new(); |
| | 20 | 49 | | private TaskCompletionSource optionsChanged = CreateOptionsChangedSource(); |
| | | 50 | | private long optionsVersion; |
| | | 51 | | |
| | | 52 | | /// <inheritdoc /> |
| | | 53 | | public override async Task StopAsync(CancellationToken cancellationToken) |
| | | 54 | | { |
| | 19 | 55 | | await base.StopAsync(cancellationToken).ConfigureAwait(false); |
| | | 56 | | |
| | 19 | 57 | | AsiBackboneGovernanceOutboxDrainWorkerOptions options = optionsMonitor.CurrentValue; |
| | | 58 | | |
| | 19 | 59 | | if (options.Enabled && options.DrainOnShutdown && !cancellationToken.IsCancellationRequested) |
| | | 60 | | { |
| | 5 | 61 | | using var shutdownDrainCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); |
| | 5 | 62 | | shutdownDrainCancellation.CancelAfter(options.ShutdownDrainTimeout); |
| | | 63 | | |
| | | 64 | | try |
| | | 65 | | { |
| | 5 | 66 | | _ = await DrainOnceAsync(options, shutdownDrainCancellation.Token).ConfigureAwait(false); |
| | 2 | 67 | | } |
| | 2 | 68 | | catch (OperationCanceledException) when (shutdownDrainCancellation.IsCancellationRequested) |
| | | 69 | | { |
| | 2 | 70 | | LogShutdownDrainCanceled(logger, null); |
| | 2 | 71 | | } |
| | 1 | 72 | | catch (Exception ex) |
| | | 73 | | { |
| | 1 | 74 | | LogShutdownDrainFailed(logger, ex); |
| | 1 | 75 | | } |
| | 5 | 76 | | } |
| | 19 | 77 | | } |
| | | 78 | | |
| | | 79 | | /// <inheritdoc /> |
| | | 80 | | protected override async Task ExecuteAsync(CancellationToken stoppingToken) |
| | | 81 | | { |
| | 14 | 82 | | using IDisposable? optionsChangeRegistration = optionsMonitor.OnChange((_, _) => SignalOptionsChanged()); |
| | 11 | 83 | | bool disabledLogged = false; |
| | | 84 | | |
| | 21 | 85 | | while (!stoppingToken.IsCancellationRequested) |
| | | 86 | | { |
| | 21 | 87 | | (long observedVersion, Task optionsChangedTask) = CaptureOptionsChangeState(); |
| | 21 | 88 | | AsiBackboneGovernanceOutboxDrainWorkerOptions options = optionsMonitor.CurrentValue; |
| | | 89 | | |
| | 21 | 90 | | if (!options.Enabled) |
| | | 91 | | { |
| | 9 | 92 | | if (!disabledLogged) |
| | | 93 | | { |
| | 5 | 94 | | LogWorkerDisabled(logger, null); |
| | 5 | 95 | | disabledLogged = true; |
| | | 96 | | } |
| | | 97 | | |
| | 9 | 98 | | await WaitForDelayOrOptionsChangeAsync( |
| | 9 | 99 | | options.PollingInterval, |
| | 9 | 100 | | observedVersion, |
| | 9 | 101 | | optionsChangedTask, |
| | 9 | 102 | | stoppingToken).ConfigureAwait(false); |
| | 7 | 103 | | continue; |
| | | 104 | | } |
| | | 105 | | |
| | 12 | 106 | | disabledLogged = false; |
| | | 107 | | |
| | | 108 | | try |
| | | 109 | | { |
| | 12 | 110 | | int drainedCount = await DrainOnceAsync(options, stoppingToken).ConfigureAwait(false); |
| | 9 | 111 | | LogDrainAttempted(logger, drainedCount, null); |
| | 9 | 112 | | await WaitForDelayOrOptionsChangeAsync( |
| | 9 | 113 | | options.PollingInterval, |
| | 9 | 114 | | observedVersion, |
| | 9 | 115 | | optionsChangedTask, |
| | 9 | 116 | | stoppingToken).ConfigureAwait(false); |
| | 2 | 117 | | } |
| | 7 | 118 | | catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) |
| | | 119 | | { |
| | 7 | 120 | | break; |
| | | 121 | | } |
| | 3 | 122 | | catch (Exception ex) |
| | | 123 | | { |
| | 3 | 124 | | LogWorkerFailed(logger, ex); |
| | 3 | 125 | | await WaitForDelayOrOptionsChangeAsync( |
| | 3 | 126 | | options.FailureDelay, |
| | 3 | 127 | | observedVersion, |
| | 3 | 128 | | optionsChangedTask, |
| | 3 | 129 | | stoppingToken).ConfigureAwait(false); |
| | | 130 | | } |
| | 3 | 131 | | } |
| | 7 | 132 | | } |
| | | 133 | | |
| | | 134 | | private async ValueTask<int> DrainOnceAsync( |
| | | 135 | | AsiBackboneGovernanceOutboxDrainWorkerOptions options, |
| | | 136 | | CancellationToken cancellationToken) |
| | | 137 | | { |
| | 17 | 138 | | options.Validate(); |
| | 17 | 139 | | cancellationToken.ThrowIfCancellationRequested(); |
| | | 140 | | |
| | 17 | 141 | | using IServiceScope scope = scopeFactory.CreateScope(); |
| | 17 | 142 | | AsiBackboneGovernanceOutboxDrain drain = scope.ServiceProvider.GetRequiredService<AsiBackboneGovernanceOutboxDra |
| | 14 | 143 | | DateTimeOffset retryUtc = options.RetryClock().ToUniversalTime(); |
| | 14 | 144 | | IReadOnlyList<GovernanceOutboxEntry> drainedEntries = await drain.DrainAsync( |
| | 14 | 145 | | retryUtc, |
| | 14 | 146 | | options.BatchSize, |
| | 14 | 147 | | cancellationToken) |
| | 14 | 148 | | .ConfigureAwait(false); |
| | | 149 | | |
| | 11 | 150 | | return drainedEntries.Count; |
| | 11 | 151 | | } |
| | | 152 | | |
| | | 153 | | private async ValueTask WaitForDelayOrOptionsChangeAsync( |
| | | 154 | | TimeSpan delay, |
| | | 155 | | long observedVersion, |
| | | 156 | | Task optionsChangedTask, |
| | | 157 | | CancellationToken cancellationToken) |
| | 21 | 158 | | { |
| | | 159 | | lock (optionsChangedSync) |
| | | 160 | | { |
| | 21 | 161 | | if (optionsVersion != observedVersion) |
| | | 162 | | { |
| | 1 | 163 | | return; |
| | | 164 | | } |
| | 20 | 165 | | } |
| | | 166 | | |
| | 20 | 167 | | using var delayCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); |
| | 20 | 168 | | var delayTask = Task.Delay(delay, delayCancellation.Token); |
| | 20 | 169 | | Task completedTask = await Task.WhenAny(delayTask, optionsChangedTask).ConfigureAwait(false); |
| | | 170 | | |
| | 20 | 171 | | if (completedTask == optionsChangedTask) |
| | | 172 | | { |
| | 2 | 173 | | await delayCancellation.CancelAsync().ConfigureAwait(false); |
| | 2 | 174 | | return; |
| | | 175 | | } |
| | | 176 | | |
| | 18 | 177 | | await delayTask.ConfigureAwait(false); |
| | 10 | 178 | | } |
| | | 179 | | |
| | | 180 | | private (long Version, Task ChangedTask) CaptureOptionsChangeState() |
| | 21 | 181 | | { |
| | | 182 | | lock (optionsChangedSync) |
| | | 183 | | { |
| | 21 | 184 | | return (optionsVersion, optionsChanged.Task); |
| | | 185 | | } |
| | 21 | 186 | | } |
| | | 187 | | |
| | | 188 | | private void SignalOptionsChanged() |
| | 3 | 189 | | { |
| | | 190 | | TaskCompletionSource completedSource; |
| | | 191 | | lock (optionsChangedSync) |
| | | 192 | | { |
| | 3 | 193 | | completedSource = optionsChanged; |
| | 3 | 194 | | optionsChanged = CreateOptionsChangedSource(); |
| | 3 | 195 | | optionsVersion++; |
| | 3 | 196 | | } |
| | | 197 | | |
| | 3 | 198 | | _ = completedSource.TrySetResult(); |
| | 3 | 199 | | } |
| | | 200 | | |
| | | 201 | | private static TaskCompletionSource CreateOptionsChangedSource() |
| | | 202 | | { |
| | 23 | 203 | | return new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); |
| | | 204 | | } |
| | | 205 | | } |