fix: trace background jobs in Sentry

Api.SeaHavenIndustries/Helper/VendorDocumentScanWorker.cs

@@ -1,5 +1,6 @@
 using Data.SeaHavenIndustries;
 using Microsoft.EntityFrameworkCore;
+using Sentry;
 
 namespace Api.SeaHavenIndustries.Helper
 {
@@ -8,15 +9,18 @@ namespace Api.SeaHavenIndustries.Helper
         private readonly IServiceScopeFactory _scopeFactory;
         private readonly IWebHostEnvironment _environment;
         private readonly ILogger<VendorDocumentScanWorker> _logger;
+        private readonly IHub _sentryHub;
 
         public VendorDocumentScanWorker(
             IServiceScopeFactory scopeFactory,
             IWebHostEnvironment environment,
-            ILogger<VendorDocumentScanWorker> logger)
+            ILogger<VendorDocumentScanWorker> logger,
+            IHub sentryHub)
         {
             _scopeFactory = scopeFactory;
             _environment = environment;
             _logger = logger;
+            _sentryHub = sentryHub;
         }
 
         protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@@ -30,36 +34,54 @@ namespace Api.SeaHavenIndustries.Helper
 
         private async Task ScanPendingAsync(CancellationToken cancellationToken)
         {
-            using var scope = _scopeFactory.CreateScope();
+            using var sentryScope = _sentryHub.PushScope();
-            var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
+            var transaction = _sentryHub.StartTransaction("vendor-documents.scan-pending", "task");
-            var scanner = scope.ServiceProvider.GetRequiredService<IDocumentScanner>();
+            _sentryHub.ConfigureScope(scope => scope.Transaction = transaction);
-            var pending = await db.VendorCompletionDocuments
+            try
-                .Where(document => document.ScanStatus == "Pending")
-                .OrderBy(document => document.CreatedDate)
-                .Take(20)
-                .ToListAsync(cancellationToken);
-
-            foreach (var document in pending)
             {
-                var path = VendorDocumentStorage.ResolvePath(_environment.ContentRootPath, document);
+                using var scope = _scopeFactory.CreateScope();
-                var result = await scanner.ScanAsync(path, cancellationToken);
+                var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
-                if (result == DocumentScanResult.Unavailable)
+                var scanner = scope.ServiceProvider.GetRequiredService<IDocumentScanner>();
+                var pending = await db.VendorCompletionDocuments
+                    .Where(document => document.ScanStatus == "Pending")
+                    .OrderBy(document => document.CreatedDate)
+                    .Take(20)
+                    .ToListAsync(cancellationToken);
+
+                foreach (var document in pending)
                 {
-                    _logger.LogWarning("Vendor document scan service unavailable; document {DocumentId} remains quarantined", document.Id);
+                    var path = VendorDocumentStorage.ResolvePath(_environment.ContentRootPath, document);
-                    continue;
+                    var result = await scanner.ScanAsync(path, cancellationToken);
+                    if (result == DocumentScanResult.Unavailable)
+                    {
+                        _logger.LogWarning("Vendor document scan service unavailable; document {DocumentId} remains quarantined", document.Id);
+                        continue;
+                    }
+
+                    document.ScanStatus = result == DocumentScanResult.Passed ? "Passed" : "Rejected";
+                    document.ReviewStatus = result == DocumentScanResult.Passed ? "Processing" : "Rejected";
+                    document.RejectionReason = result == DocumentScanResult.Infected
+                        ? "The upload failed malware scanning."
+                        : document.RejectionReason;
+                    document.ScannedAt = DateTime.UtcNow;
+                    document.LastModificationTime = DateTime.UtcNow;
+                    await db.SaveChangesAsync(cancellationToken);
+
+                    if (result == DocumentScanResult.Infected && File.Exists(path))
+                        File.Delete(path);
                 }
 
-                document.ScanStatus = result == DocumentScanResult.Passed ? "Passed" : "Rejected";
+                transaction.Finish(SpanStatus.Ok);
-                document.ReviewStatus = result == DocumentScanResult.Passed ? "Processing" : "Rejected";
+            }
-                document.RejectionReason = result == DocumentScanResult.Infected
+            catch (OperationCanceledException)
-                    ? "The upload failed malware scanning."
+            {
-                    : document.RejectionReason;
+                transaction.Finish(SpanStatus.Cancelled);
-                document.ScannedAt = DateTime.UtcNow;
+                throw;
-                document.LastModificationTime = DateTime.UtcNow;
+            }
-                await db.SaveChangesAsync(cancellationToken);
+            catch (Exception ex)
-
+            {
-                if (result == DocumentScanResult.Infected && File.Exists(path))
+                transaction.Finish(ex, SpanStatus.InternalError);
-                    File.Delete(path);
+                throw;
             }
         }
     }

Api.SeaHavenIndustries/HostedServices/PastDueCacheHostedService.cs

@@ -2,6 +2,7 @@ using Microsoft.Extensions.Options;
 using SeaHaven.Services.Implementation;
 using SeaHaven.Services.Interfaces;
 using Api.SeaHavenIndustries.Options;
+using Sentry;
 
 namespace Api.SeaHavenIndustries.HostedServices
 {
@@ -11,18 +12,21 @@ namespace Api.SeaHavenIndustries.HostedServices
         private readonly ILogger<PastDueCacheHostedService> _logger;
         private readonly PastDueCacheJobOptions _options;
         private readonly WorkOrderJobRunState _runState;
+        private readonly IHub _sentryHub;
         private DateOnly? _lastRunDate;
 
         public PastDueCacheHostedService(
             IServiceScopeFactory scopeFactory,
             IOptions<WorkOrderJobsOptions> options,
             WorkOrderJobRunState runState,
-            ILogger<PastDueCacheHostedService> logger)
+            ILogger<PastDueCacheHostedService> logger,
+            IHub sentryHub)
         {
             _scopeFactory = scopeFactory;
             _logger = logger;
             _options = options.Value.PastDueCache;
             _runState = runState;
+            _sentryHub = sentryHub;
         }
 
         protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@@ -45,6 +49,9 @@ namespace Api.SeaHavenIndustries.HostedServices
 
         private async Task<bool> RunJobAsync(CancellationToken stoppingToken)
         {
+            using var sentryScope = _sentryHub.PushScope();
+            var transaction = _sentryHub.StartTransaction("workorders.past-due-cache-job", "task");
+            _sentryHub.ConfigureScope(scope => scope.Transaction = transaction);
             try
             {
                 using var scope = _scopeFactory.CreateScope();
@@ -53,12 +60,19 @@ namespace Api.SeaHavenIndustries.HostedServices
 
                 _runState.LastPastDueCacheRunUtc = DateTime.UtcNow;
                 _runState.LastPastDueCacheError = null;
+                transaction.Finish(SpanStatus.Ok);
                 return true;
             }
+            catch (OperationCanceledException)
+            {
+                transaction.Finish(SpanStatus.Cancelled);
+                throw;
+            }
             catch (Exception ex) when (ex is not OperationCanceledException)
             {
                 _runState.LastPastDueCacheError = ex.Message;
                 _logger.LogError(ex, "PastDue cache hosted job failed.");
+                transaction.Finish(ex, SpanStatus.InternalError);
                 return false;
             }
         }

Api.SeaHavenIndustries/HostedServices/UpliftLifecycleHostedService.cs

@@ -1,6 +1,7 @@
 using Microsoft.Extensions.Options;
 using SeaHaven.Services.Configuration;
 using SeaHaven.Services.Interfaces;
+using Sentry;
 
 namespace Api.SeaHavenIndustries.HostedServices
 {
@@ -9,32 +10,43 @@ namespace Api.SeaHavenIndustries.HostedServices
         private readonly IServiceScopeFactory _scopeFactory;
         private readonly ILogger<UpliftLifecycleHostedService> _logger;
         private readonly ApprovalsOptions _options;
+        private readonly IHub _sentryHub;
 
         public UpliftLifecycleHostedService(
             IServiceScopeFactory scopeFactory,
             IOptions<ApprovalsOptions> options,
-            ILogger<UpliftLifecycleHostedService> logger)
+            ILogger<UpliftLifecycleHostedService> logger,
+            IHub sentryHub)
         {
             _scopeFactory = scopeFactory;
             _logger = logger;
             _options = options.Value;
+            _sentryHub = sentryHub;
         }
 
         protected override async Task ExecuteAsync(CancellationToken stoppingToken)
         {
             while (!stoppingToken.IsCancellationRequested)
             {
-                try
                 {
-                    await SweepAsync(stoppingToken);
+                    using var sentryScope = _sentryHub.PushScope();
-                }
+                    var transaction = _sentryHub.StartTransaction("uplifts.lifecycle-sweep", "task");
-                catch (OperationCanceledException)
+                    _sentryHub.ConfigureScope(scope => scope.Transaction = transaction);
-                {
+                    try
-                    throw;
+                    {
-                }
+                        await SweepAsync(stoppingToken);
-                catch (Exception ex)
+                        transaction.Finish(SpanStatus.Ok);
-                {
+                    }
-                    _logger.LogError(ex, "Uplift lifecycle sweep failed; will retry on next interval");
+                    catch (OperationCanceledException)
+                    {
+                        transaction.Finish(SpanStatus.Cancelled);
+                        throw;
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.LogError(ex, "Uplift lifecycle sweep failed; will retry on next interval");
+                        transaction.Finish(ex, SpanStatus.InternalError);
+                    }
                 }
 
                 await Task.Delay(_options.EffectiveSweepInterval, stoppingToken);

Api.SeaHavenIndustries/HostedServices/WorkOrderReconciliationHostedService.cs

@@ -1,6 +1,7 @@
 using Microsoft.Extensions.Options;
 using SeaHaven.Services.Configuration;
 using SeaHaven.Services.Interfaces;
+using Sentry;
 
 namespace Api.SeaHavenIndustries.HostedServices
 {
@@ -10,17 +11,20 @@ namespace Api.SeaHavenIndustries.HostedServices
         private readonly IOptionsMonitor<WorkOrderReconciliationOptions> _options;
         private readonly TimeProvider _timeProvider;
         private readonly ILogger<WorkOrderReconciliationHostedService> _logger;
+        private readonly IHub _sentryHub;
 
         public WorkOrderReconciliationHostedService(
             IServiceScopeFactory scopeFactory,
             IOptionsMonitor<WorkOrderReconciliationOptions> options,
             TimeProvider timeProvider,
-            ILogger<WorkOrderReconciliationHostedService> logger)
+            ILogger<WorkOrderReconciliationHostedService> logger,
+            IHub sentryHub)
         {
             _scopeFactory = scopeFactory;
             _options = options;
             _timeProvider = timeProvider;
             _logger = logger;
+            _sentryHub = sentryHub;
         }
 
         protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@@ -56,29 +60,50 @@ namespace Api.SeaHavenIndustries.HostedServices
 
         private async Task TriggerAsync(string reason, CancellationToken cancellationToken)
         {
+            using var sentryScope = _sentryHub.PushScope();
+            var transaction = _sentryHub.StartTransaction("workorders.reconciliation-trigger", "task");
+            transaction.Description = reason;
+            _sentryHub.ConfigureScope(scope => scope.Transaction = transaction);
             try
             {
                 await using var scope = _scopeFactory.CreateAsyncScope();
                 var runner = scope.ServiceProvider.GetRequiredService<IWorkOrderReconciliationRunner>();
                 await runner.TriggerAsync(reason, cancellationToken);
+                transaction.Finish(SpanStatus.Ok);
+            }
+            catch (OperationCanceledException)
+            {
+                transaction.Finish(SpanStatus.Cancelled);
+                throw;
             }
             catch (Exception ex) when (ex is not OperationCanceledException)
             {
                 _logger.LogError(ex, "Could not queue procurement reconciliation.");
+                transaction.Finish(ex, SpanStatus.InternalError);
             }
         }
 
         private async Task RunPendingAsync(CancellationToken cancellationToken)
         {
+            using var sentryScope = _sentryHub.PushScope();
+            var transaction = _sentryHub.StartTransaction("workorders.reconciliation-run-pending", "task");
+            _sentryHub.ConfigureScope(scope => scope.Transaction = transaction);
             try
             {
                 await using var scope = _scopeFactory.CreateAsyncScope();
                 var runner = scope.ServiceProvider.GetRequiredService<IWorkOrderReconciliationRunner>();
                 await runner.RunPendingAsync(cancellationToken);
+                transaction.Finish(SpanStatus.Ok);
+            }
+            catch (OperationCanceledException)
+            {
+                transaction.Finish(SpanStatus.Cancelled);
+                throw;
             }
             catch (Exception ex) when (ex is not OperationCanceledException)
             {
                 _logger.LogError(ex, "Could not execute procurement reconciliation.");
+                transaction.Finish(ex, SpanStatus.InternalError);
             }
         }
     }

Api.SeaHavenIndustries/HostedServices/WorkOrderWeekRolledHostedService.cs

@@ -3,6 +3,7 @@ using SeaHaven.Services.Helpers;
 using SeaHaven.Services.Implementation;
 using SeaHaven.Services.Interfaces;
 using Api.SeaHavenIndustries.Options;
+using Sentry;
 
 namespace Api.SeaHavenIndustries.HostedServices
 {
@@ -12,18 +13,21 @@ namespace Api.SeaHavenIndustries.HostedServices
         private readonly ILogger<WorkOrderWeekRolledHostedService> _logger;
         private readonly WeekRolledJobOptions _options;
         private readonly WorkOrderJobRunState _runState;
+        private readonly IHub _sentryHub;
         private DateOnly? _lastRunDate;
 
         public WorkOrderWeekRolledHostedService(
             IServiceScopeFactory scopeFactory,
             IOptions<WorkOrderJobsOptions> options,
             WorkOrderJobRunState runState,
-            ILogger<WorkOrderWeekRolledHostedService> logger)
+            ILogger<WorkOrderWeekRolledHostedService> logger,
+            IHub sentryHub)
         {
             _scopeFactory = scopeFactory;
             _logger = logger;
             _options = options.Value.WeekRolled;
             _runState = runState;
+            _sentryHub = sentryHub;
         }
 
         protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@@ -46,6 +50,9 @@ namespace Api.SeaHavenIndustries.HostedServices
 
         private async Task<bool> RunJobAsync(CancellationToken stoppingToken)
         {
+            using var sentryScope = _sentryHub.PushScope();
+            var transaction = _sentryHub.StartTransaction("workorders.week-rolled-job", "task");
+            _sentryHub.ConfigureScope(scope => scope.Transaction = transaction);
             try
             {
                 var sourceWeekStart = WorkOrderOperationalWeek.GetPreviousOperationalWeekStart(DateTime.UtcNow);
@@ -56,12 +63,19 @@ namespace Api.SeaHavenIndustries.HostedServices
 
                 _runState.LastWeekRolledRunUtc = DateTime.UtcNow;
                 _runState.LastWeekRolledError = null;
+                transaction.Finish(SpanStatus.Ok);
                 return true;
             }
+            catch (OperationCanceledException)
+            {
+                transaction.Finish(SpanStatus.Cancelled);
+                throw;
+            }
             catch (Exception ex) when (ex is not OperationCanceledException)
             {
                 _runState.LastWeekRolledError = ex.Message;
                 _logger.LogError(ex, "WeekRolled hosted job failed.");
+                transaction.Finish(ex, SpanStatus.InternalError);
                 return false;
             }
         }