| | | 1 | | using Microsoft.Extensions.Diagnostics.HealthChecks; |
| | | 2 | | using Microsoft.Extensions.Options; |
| | | 3 | | using ProjectTemplate.Infrastructure.Data.Auditing; |
| | | 4 | | |
| | | 5 | | namespace ProjectTemplate.Web.HealthChecks; |
| | | 6 | | |
| | | 7 | | /// <summary> |
| | | 8 | | /// Reports audit reconciliation findings, audit delivery state, and whether reconciliation is still running. |
| | | 9 | | /// </summary> |
| | | 10 | | /// <remarks> |
| | | 11 | | /// The check is registered with the <see cref="ApplicationHealthCheckTags.Audit"/> tag and exposed through |
| | | 12 | | /// <c>/health/audit-integrity</c>. It is intentionally excluded from readiness: an integrity finding requires operator |
| | | 13 | | /// and a readiness failure would remove every replica from load balancing at the same moment. |
| | | 14 | | /// </remarks> |
| | 7 | 15 | | public sealed class ApplicationAuditIntegrityHealthCheck( |
| | 7 | 16 | | IServiceScopeFactory scopeFactory, |
| | 7 | 17 | | IOptions<ApplicationAuditReconciliationOptions> options, |
| | 7 | 18 | | TimeProvider timeProvider) |
| | | 19 | | : IHealthCheck |
| | | 20 | | { |
| | | 21 | | internal const string NeverRunDescription = |
| | | 22 | | "Audit reconciliation has not completed a run in this process."; |
| | | 23 | | |
| | | 24 | | internal const string StaleRunDescription = |
| | | 25 | | "The last successful audit reconciliation run is older than the configured stale-run threshold."; |
| | | 26 | | |
| | | 27 | | private const int _defaultStaleRunIntervalMultiplier = 3; |
| | | 28 | | |
| | 7 | 29 | | private readonly IServiceScopeFactory _scopeFactory = |
| | 7 | 30 | | scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); |
| | 7 | 31 | | private readonly ApplicationAuditReconciliationOptions _options = |
| | 7 | 32 | | options?.Value ?? throw new ArgumentNullException(nameof(options)); |
| | 7 | 33 | | private readonly TimeProvider _timeProvider = |
| | 7 | 34 | | timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); |
| | | 35 | | |
| | | 36 | | /// <summary> |
| | | 37 | | /// Evaluates audit reconciliation findings, delivery health, and reconciliation freshness. |
| | | 38 | | /// </summary> |
| | | 39 | | /// <param name="context">The health check context.</param> |
| | | 40 | | /// <param name="cancellationToken">A token that cancels the check.</param> |
| | | 41 | | /// <returns>The audit integrity health result.</returns> |
| | | 42 | | public async Task<HealthCheckResult> CheckHealthAsync( |
| | | 43 | | HealthCheckContext context, |
| | | 44 | | CancellationToken cancellationToken = default) |
| | | 45 | | { |
| | 7 | 46 | | cancellationToken.ThrowIfCancellationRequested(); |
| | 7 | 47 | | if (!_options.Enabled) |
| | | 48 | | { |
| | 1 | 49 | | return HealthCheckResult.Healthy("Audit reconciliation is disabled."); |
| | | 50 | | } |
| | | 51 | | |
| | 6 | 52 | | using IServiceScope scope = _scopeFactory.CreateScope(); |
| | 6 | 53 | | IApplicationAuditReconciler reconciler = scope.ServiceProvider |
| | 6 | 54 | | .GetRequiredService<IApplicationAuditReconciler>(); |
| | 6 | 55 | | ApplicationAuditReconciliationSummary summary = await reconciler |
| | 6 | 56 | | .GetSummaryAsync(cancellationToken) |
| | 6 | 57 | | .ConfigureAwait(false); |
| | | 58 | | |
| | 6 | 59 | | ApplicationAuditCompletionOutboxHealth? deliveryHealth = null; |
| | 6 | 60 | | IApplicationAuditCompletionOutboxQuery? outboxQuery = scope.ServiceProvider |
| | 6 | 61 | | .GetService<IApplicationAuditCompletionOutboxQuery>(); |
| | 6 | 62 | | if (outboxQuery is not null) |
| | | 63 | | { |
| | 0 | 64 | | deliveryHealth = await outboxQuery.GetHealthAsync(cancellationToken).ConfigureAwait(false); |
| | 0 | 65 | | scope.ServiceProvider |
| | 0 | 66 | | .GetRequiredService<ApplicationAuditReconciliationMetrics>() |
| | 0 | 67 | | .UpdateDelivery(deliveryHealth); |
| | | 68 | | } |
| | | 69 | | |
| | 6 | 70 | | var data = new Dictionary<string, object> |
| | 6 | 71 | | { |
| | 6 | 72 | | ["openFindings"] = summary.OpenFindingCount, |
| | 6 | 73 | | ["errorFindings"] = summary.ErrorFindingCount, |
| | 6 | 74 | | ["criticalFindings"] = summary.CriticalFindingCount, |
| | 6 | 75 | | ["manifestVerificationFailures"] = summary.ManifestVerificationFailureCount, |
| | 6 | 76 | | ["missingCompletions"] = summary.MissingCompletionCount, |
| | 6 | 77 | | ["staleDeliveryFindings"] = summary.StaleDeliveryCount, |
| | 6 | 78 | | ["deadLetterFindings"] = summary.DeadLetterCount |
| | 6 | 79 | | }; |
| | | 80 | | |
| | 6 | 81 | | if (summary.LastRunUtc.HasValue) |
| | | 82 | | { |
| | 4 | 83 | | data["lastReconciliationUtc"] = summary.LastRunUtc.Value; |
| | | 84 | | } |
| | | 85 | | |
| | 6 | 86 | | if (deliveryHealth is not null) |
| | | 87 | | { |
| | 0 | 88 | | data["outboxBacklog"] = deliveryHealth.BacklogCount; |
| | 0 | 89 | | data["outboxRetryCount"] = deliveryHealth.TotalRetryCount; |
| | 0 | 90 | | data["outboxDeadLetters"] = deliveryHealth.DeadLetterCount; |
| | 0 | 91 | | if (deliveryHealth.OldestPendingAge.HasValue) |
| | | 92 | | { |
| | 0 | 93 | | data["oldestPendingAgeSeconds"] = deliveryHealth.OldestPendingAge.Value.TotalSeconds; |
| | | 94 | | } |
| | | 95 | | } |
| | | 96 | | |
| | 6 | 97 | | string? freshnessProblem = EvaluateRunFreshness(summary.LastRunUtc, data); |
| | | 98 | | |
| | 6 | 99 | | bool unhealthy = summary.CriticalFindingCount > 0 || |
| | 6 | 100 | | summary.ManifestVerificationFailureCount > 0 || |
| | 6 | 101 | | summary.OpenFindingCount >= _options.HealthUnhealthyFindingCount; |
| | 6 | 102 | | if (unhealthy) |
| | | 103 | | { |
| | 1 | 104 | | return HealthCheckResult.Unhealthy( |
| | 1 | 105 | | "Critical audit-integrity findings require operator review.", |
| | 1 | 106 | | data: data); |
| | | 107 | | } |
| | | 108 | | |
| | | 109 | | // Zero findings only means something when reconciliation is actually running. A worker that never completed a |
| | | 110 | | // run, stopped, or keeps failing would otherwise read as healthy indefinitely. |
| | 5 | 111 | | if (freshnessProblem is not null) |
| | | 112 | | { |
| | 2 | 113 | | return HealthCheckResult.Degraded(freshnessProblem, data: data); |
| | | 114 | | } |
| | | 115 | | |
| | 3 | 116 | | bool degraded = summary.OpenFindingCount >= _options.HealthWarningFindingCount || |
| | 3 | 117 | | summary.StaleDeliveryCount > 0 || |
| | 3 | 118 | | summary.DeadLetterCount > 0 || |
| | 3 | 119 | | deliveryHealth?.DeadLetterCount > 0; |
| | 3 | 120 | | return degraded |
| | 3 | 121 | | ? HealthCheckResult.Degraded( |
| | 3 | 122 | | "Audit reconciliation or delivery findings require attention.", |
| | 3 | 123 | | data: data) |
| | 3 | 124 | | : HealthCheckResult.Healthy("Audit integrity and delivery state are within configured thresholds.", data); |
| | 7 | 125 | | } |
| | | 126 | | |
| | | 127 | | /// <summary> |
| | | 128 | | /// Returns the age after which a successful reconciliation run is considered stale. |
| | | 129 | | /// </summary> |
| | | 130 | | /// <param name="options">The reconciliation options.</param> |
| | | 131 | | /// <returns>The configured threshold, or three reconciliation intervals when none is configured.</returns> |
| | | 132 | | internal static TimeSpan GetStaleRunThreshold(ApplicationAuditReconciliationOptions options) |
| | | 133 | | { |
| | 6 | 134 | | ArgumentNullException.ThrowIfNull(options); |
| | | 135 | | |
| | 6 | 136 | | return options.HealthStaleRunThreshold ?? (options.Interval * _defaultStaleRunIntervalMultiplier); |
| | | 137 | | } |
| | | 138 | | |
| | | 139 | | private string? EvaluateRunFreshness(DateTime? lastRunUtc, Dictionary<string, object> data) |
| | | 140 | | { |
| | 6 | 141 | | if (!_options.RunWorker) |
| | | 142 | | { |
| | | 143 | | // The scheduled loop runs in another process, so this process cannot observe when reconciliation last ran. |
| | 1 | 144 | | data["reconciliationFreshnessTracked"] = false; |
| | 1 | 145 | | return null; |
| | | 146 | | } |
| | | 147 | | |
| | 5 | 148 | | TimeSpan staleRunThreshold = GetStaleRunThreshold(_options); |
| | 5 | 149 | | data["reconciliationFreshnessTracked"] = true; |
| | 5 | 150 | | data["reconciliationStaleAfterSeconds"] = staleRunThreshold.TotalSeconds; |
| | | 151 | | |
| | 5 | 152 | | if (!lastRunUtc.HasValue) |
| | | 153 | | { |
| | 1 | 154 | | return NeverRunDescription; |
| | | 155 | | } |
| | | 156 | | |
| | 4 | 157 | | TimeSpan sinceLastRun = _timeProvider.GetUtcNow().UtcDateTime - lastRunUtc.Value; |
| | 4 | 158 | | data["secondsSinceLastReconciliation"] = Math.Max(sinceLastRun.TotalSeconds, 0); |
| | | 159 | | |
| | 4 | 160 | | return sinceLastRun > staleRunThreshold ? StaleRunDescription : null; |
| | | 161 | | } |
| | | 162 | | } |