feat: add API Sentry tracing (SH-298) (#105)
Some checks failed
Validate and deploy / Validate deployable source bundle (push) Has been cancelled
Validate and deploy / Deploy shoc-backend-dev through Terraform (push) Has been cancelled
Validate and deploy / Deploy shoc-backend-staging to Elastic Beanstalk (push) Has been cancelled

* feat: add API Sentry tracing

* feat: activate Sentry deployment environments

* fix: allow Sentry-free design-time tooling

* fix: trace background jobs in Sentry

* feat(observability): identify and scrub Sentry transactions

* fix(observability): finish abandoned transactions
This commit is contained in:
Alexandre Brandizzi 2026-09-04 14:11:32 -03:00 • committed by GitHub
parent 6d66737211
commit 6ceb274bfb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 959 additions and 44 deletions

View file

@ -16,6 +16,15 @@
# uses the AWS SDK default credential chain (env AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY, # uses the AWS SDK default credential chain (env AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY,
# the shared profile/SSO, or the EC2/ECS instance role). Prefer an instance role in prod. # the shared profile/SSO, or the EC2/ECS instance role). Prefer an instance role in prod.
# --- Sentry error and transaction monitoring ---
# The DSN is public ingestion configuration, not a secret. Leave it blank locally
# to keep telemetry inactive. AWS deployments receive SENTRY_DSN and
# SENTRY_ENVIRONMENT as Elastic Beanstalk environment settings managed by
# Terraform: dev is labeled "development", staging "staging". Future production
# wiring will pass the production DSN through the same sentry_dsn module variable.
SENTRY_DSN=
SENTRY_ENVIRONMENT=development
# --- SQL Server connection string (Api.SeaHavenIndustries + SeaHavenIndustries) --- # --- SQL Server connection string (Api.SeaHavenIndustries + SeaHavenIndustries) ---
ConnectionStrings__DefaultConnection=Server=<host>;Initial Catalog=<db>;User Id=<user>;Password=<password>;MultipleActiveResultSets=true ConnectionStrings__DefaultConnection=Server=<host>;Initial Catalog=<db>;User Id=<user>;Password=<password>;MultipleActiveResultSets=true
@ -44,4 +53,3 @@ Sync__Enabled=true
# --- Legacy endpoint deprecation headers --- # --- Legacy endpoint deprecation headers ---
LegacyEndpoints__DeprecationEnabled=false LegacyEndpoints__DeprecationEnabled=false
LegacyEndpoints__SunsetDate=2026-12-31 LegacyEndpoints__SunsetDate=2026-12-31

View file

@ -0,0 +1,293 @@
using System.Security.Claims;
using Api.SeaHavenIndustries.Observability;
using FluentAssertions;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc.Controllers;
using Microsoft.AspNetCore.Routing;
using Microsoft.AspNetCore.Routing.Patterns;
using Moq;
using Sentry;
using Xunit;
namespace Api.SeaHavenIndustries.Tests;
public class SentryObservabilityTests
{
private static readonly string ValidSha = new string('a', 40);
[Fact]
public void ResolveRelease_Accepts_ServicePrefixed40HexCommit()
{
SentryObservability.ResolveRelease("shoc-backend@" + ValidSha)
.Should().Be("shoc-backend@" + ValidSha);
}
[Theory]
[InlineData("1.0.0")]
[InlineData("shoc-backend@abc")]
[InlineData("shoc-backend@not-a-sha-at-all")]
[InlineData("other-service@" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")]
[InlineData("")]
public void ResolveRelease_FallsBackToLocalDevelopment_ForNonReleaseVersion(string version)
{
SentryObservability.ResolveRelease(version)
.Should().Be(SentryObservability.LocalDevelopmentRelease);
}
[Fact]
public void ResolveCommitSha_ReturnsSha_WhenReleaseShaped()
{
var release = SentryObservability.ResolveRelease("shoc-backend@" + ValidSha);
release[(SentryObservability.ServiceName.Length + 1)..].Should().Be(ValidSha);
}
[Fact]
public void ConfigureRequest_SetsUser_FromNameIdentifierOnly_AndUsesRouteTemplateNotValues()
{
var (hub, scope) = CreateHub();
var context = CreateHttpContext(
routePattern: "/api/work-orders/{workOrderId}",
claims: new[]
{
new Claim(ClaimTypes.NameIdentifier, "user-123"),
new Claim(ClaimTypes.Email, "secret.user@example.com"),
new Claim(ClaimTypes.Name, "Secret User"),
});
context.Request.QueryString = new QueryString("?ticket=secret-query-value");
context.Request.Method = "GET";
SentryObservability.ConfigureRequest(hub.Object, context);
scope.User.Should().NotBeNull();
scope.User!.Id.Should().Be("user-123");
scope.User.Email.Should().BeNull();
scope.User.Username.Should().BeNull();
scope.Tags["actor.type"].Should().Be("authenticated");
scope.Tags["operation.type"].Should().Be("http.server");
scope.Tags["code.function"].Should().Be("WorkOrders.GetWorkOrder");
scope.Tags["http.method"].Should().Be("GET");
scope.Tags["http.route"].Should().Be("/api/work-orders/{workOrderId}");
scope.Tags["trace_id"].Should().NotBeNullOrWhiteSpace();
scope.Tags["transaction_id"].Should().NotBeNullOrWhiteSpace();
scope.Tags.Values.Should().NotContain(value => value != null && value.Contains("secret", StringComparison.OrdinalIgnoreCase),
"no query, header, body, cookie, name, or email material may reach Sentry tags");
}
[Fact]
public void ConfigureRequest_AnonymousRequest_TagsAnonymous_AndSetsNoUser()
{
var (hub, scope) = CreateHub();
var context = CreateHttpContext("/api/vendors", Array.Empty<Claim>());
SentryObservability.ConfigureRequest(hub.Object, context);
scope.User.Id.Should().BeNull();
scope.Tags["actor.type"].Should().Be("anonymous");
scope.Tags["code.function"].Should().Be("Vendors.List");
scope.Tags["http.route"].Should().Be("/api/vendors");
}
[Fact]
public void ConfigureRequest_NonControllerEndpoint_OmitsCodeFunction()
{
var (hub, scope) = CreateHub();
var context = CreateHttpContext("/health", Array.Empty<Claim>(), includeActionDescriptor: false);
SentryObservability.ConfigureRequest(hub.Object, context);
scope.Tags.Should().NotContainKey("code.function");
}
[Fact]
public void BeginBackgroundTransaction_SetsScopeTransaction_AndSystemTags()
{
var (hub, scope) = CreateHub();
var transaction = new Mock<ITransactionTracer>();
hub.Setup(h => h.StartTransaction(
It.IsAny<ITransactionContext>(),
It.IsAny<IReadOnlyDictionary<string, object?>>()))
.Returns(transaction.Object);
using var handle = SentryObservability.BeginBackgroundTransaction(
hub.Object,
"workorders.job",
"MyWorker.Run",
"the-reason");
hub.Verify(h => h.StartTransaction(
It.Is<ITransactionContext>(context => context.Name == "workorders.job" && context.Operation == "task"),
It.IsAny<IReadOnlyDictionary<string, object?>>()), Times.Once);
scope.Transaction.Should().BeSameAs(transaction.Object);
transaction.VerifySet(t => t.Description = "the-reason", Times.Once);
transaction.Verify(t => t.SetTag("actor.type", "system"), Times.Once);
transaction.Verify(t => t.SetTag("operation.type", "background_job"), Times.Once);
transaction.Verify(t => t.SetTag("code.function", "MyWorker.Run"), Times.Once);
transaction.Verify(t => t.SetTag("trace_id", It.IsAny<string>()), Times.Once);
transaction.Verify(t => t.SetTag("transaction_id", It.IsAny<string>()), Times.Once);
scope.Tags["operation.type"].Should().Be("background_job");
}
[Fact]
public void BeginBackgroundTransaction_Handle_CannotDoubleFinish()
{
var (hub, _) = CreateHub();
var transaction = new Mock<ITransactionTracer>();
hub.Setup(h => h.StartTransaction(
It.IsAny<ITransactionContext>(),
It.IsAny<IReadOnlyDictionary<string, object?>>()))
.Returns(transaction.Object);
using var handle = SentryObservability.BeginBackgroundTransaction(hub.Object, "job", "Fn");
handle.FinishOk();
handle.FinishOk();
handle.FinishError(new InvalidOperationException("late"));
handle.FinishCancelled();
transaction.Verify(t => t.Finish(SpanStatus.Ok), Times.Once);
transaction.Verify(t => t.Finish(It.IsAny<Exception>(), It.IsAny<SpanStatus>()), Times.Never);
transaction.Verify(t => t.Finish(SpanStatus.Cancelled), Times.Never);
}
[Fact]
public void Dispose_UnfinishedTransaction_FinishesOnceWithUnknownErrorStatus()
{
var (hub, _) = CreateHub();
var transaction = new Mock<ITransactionTracer>();
hub.Setup(h => h.StartTransaction(
It.IsAny<ITransactionContext>(),
It.IsAny<IReadOnlyDictionary<string, object?>>()))
.Returns(transaction.Object);
var handle = SentryObservability.BeginBackgroundTransaction(hub.Object, "job", "Fn");
handle.Dispose();
handle.Dispose();
transaction.Verify(t => t.Finish(SpanStatus.UnknownError), Times.Once);
transaction.Verify(t => t.Finish(It.IsAny<SpanStatus>()), Times.Once);
}
[Fact]
public void Dispose_AfterFinish_DoesNotFinishAgain()
{
var (hub, _) = CreateHub();
var transaction = new Mock<ITransactionTracer>();
hub.Setup(h => h.StartTransaction(
It.IsAny<ITransactionContext>(),
It.IsAny<IReadOnlyDictionary<string, object?>>()))
.Returns(transaction.Object);
var handle = SentryObservability.BeginBackgroundTransaction(hub.Object, "job", "Fn");
handle.FinishOk();
handle.Dispose();
transaction.Verify(t => t.Finish(SpanStatus.Ok), Times.Once);
transaction.Verify(t => t.Finish(SpanStatus.UnknownError), Times.Never);
transaction.Verify(t => t.Finish(It.IsAny<SpanStatus>()), Times.Once);
}
[Fact]
public void TelemetryScrubber_RemovesRequestSecrets_AndKeepsOnlyOpaqueUserId()
{
var @event = new SentryEvent
{
User = new SentryUser
{
Id = "opaque-user-123",
Email = "secret.user@example.com",
Username = "Secret User",
IpAddress = "203.0.113.10",
},
Request = new SentryRequest
{
Data = "secret-body",
QueryString = "token=secret-query",
Cookies = "session=secret-cookie",
},
};
@event.Request.Headers["Authorization"] = "Bearer secret-token";
@event.Request.Env["private"] = "secret-env";
@event.Request.Other["private"] = "secret-other";
var scrubbed = SentryTelemetryScrubber.Scrub(@event);
scrubbed.Request!.Data.Should().BeNull();
scrubbed.Request.QueryString.Should().BeNull();
scrubbed.Request.Cookies.Should().BeNull();
scrubbed.Request.Headers.Should().BeEmpty();
scrubbed.Request.Env.Should().BeEmpty();
scrubbed.Request.Other.Should().BeEmpty();
scrubbed.User.Should().NotBeNull();
scrubbed.User!.Id.Should().Be("opaque-user-123");
scrubbed.User.Email.Should().BeNull();
scrubbed.User.Username.Should().BeNull();
scrubbed.User.IpAddress.Should().BeNull();
}
[Fact]
public void TelemetryScrubber_RemovesSensitiveTransactionData()
{
var transaction = new SentryTransaction("GET /api/work-orders/{id}", "http.server");
transaction.SetData("http.route", "/api/work-orders/{id}");
transaction.SetData("http.query", "token=secret-query");
#pragma warning disable CS0618 // Exercise the scrubber's legacy Extra cleanup contract.
transaction.SetExtra("secret", "secret-extra");
#pragma warning restore CS0618
var scrubbed = SentryTelemetryScrubber.Scrub(transaction);
scrubbed.Data.Should().ContainKey("http.route");
scrubbed.Data.Should().NotContainKey("http.query");
#pragma warning disable CS0618 // Exercise the scrubber's legacy Extra cleanup contract.
scrubbed.Extra.Should().NotContainKey("secret");
#pragma warning restore CS0618
scrubbed.Tags["trace_id"].Should().Be(scrubbed.TraceId.ToString());
scrubbed.Tags["transaction_id"].Should().Be(scrubbed.SpanId.ToString());
}
private static (Mock<IHub> Hub, Scope Scope) CreateHub()
{
var hub = new Mock<IHub>();
var scope = new Scope(new SentryOptions());
hub.Setup(h => h.ConfigureScope(It.IsAny<Action<Scope>>()))
.Callback<Action<Scope>>(action => action(scope));
hub.Setup(h => h.PushScope()).Returns(Mock.Of<IDisposable>());
var activeSpan = new Mock<ISpan>();
activeSpan.SetupGet(span => span.TraceId).Returns(SentryId.Create());
activeSpan.SetupGet(span => span.SpanId).Returns(SpanId.Create());
hub.Setup(h => h.GetSpan()).Returns(activeSpan.Object);
return (hub, scope);
}
private static HttpContext CreateHttpContext(
string routePattern,
Claim[] claims,
bool includeActionDescriptor = true)
{
var metadata = includeActionDescriptor
? new object[]
{
new ControllerActionDescriptor
{
ControllerName = routePattern.StartsWith("/api/work-orders") ? "WorkOrders" : "Vendors",
ActionName = routePattern.StartsWith("/api/work-orders") ? "GetWorkOrder" : "List",
},
}
: Array.Empty<object>();
var endpoint = new RouteEndpoint(
_ => Task.CompletedTask,
RoutePatternFactory.Parse(routePattern),
0,
new EndpointMetadataCollection(metadata),
"test-endpoint");
var context = new DefaultHttpContext();
context.SetEndpoint(endpoint);
context.User = new ClaimsPrincipal(new ClaimsIdentity(claims, "TestAuth"));
return context;
}
}

View file

@ -0,0 +1,72 @@
using FluentAssertions;
using Xunit;
namespace Api.SeaHavenIndustries.Tests;
public class SentryPipelineContractTests
{
private static string RepoRoot()
{
var directory = new DirectoryInfo(AppContext.BaseDirectory);
while (directory is not null
&& !File.Exists(Path.Combine(directory.FullName, "Api.SeaHavenIndustries", "Program.cs")))
{
directory = directory.Parent;
}
if (directory is null)
throw new InvalidOperationException("Could not locate the repository root from " + AppContext.BaseDirectory);
return directory.FullName;
}
[Fact]
public void Program_MiddlewareOrder_IsRouting_Authentication_SentryMetadata_Authorization()
{
var program = File.ReadAllText(Path.Combine(RepoRoot(), "Api.SeaHavenIndustries", "Program.cs"));
var routing = program.IndexOf("app.UseRouting()", StringComparison.Ordinal);
var authentication = program.IndexOf("app.UseAuthentication()", StringComparison.Ordinal);
var sentry = program.IndexOf("app.UseMiddleware<SentryRequestMetadataMiddleware>()", StringComparison.Ordinal);
var authorization = program.IndexOf("app.UseAuthorization()", StringComparison.Ordinal);
routing.Should().BeGreaterThan(-1, "UseRouting must be explicit");
authentication.Should().BeGreaterThan(-1);
sentry.Should().BeGreaterThan(-1);
authorization.Should().BeGreaterThan(-1);
routing.Should().BeLessThan(authentication, "routing must precede authentication");
authentication.Should().BeLessThan(sentry, "Sentry request metadata must be applied after authentication");
sentry.Should().BeLessThan(authorization, "Sentry request metadata must be applied before authorization");
}
[Fact]
public void Program_ConfiguresReleaseAndDefaultTags()
{
var program = File.ReadAllText(Path.Combine(RepoRoot(), "Api.SeaHavenIndustries", "Program.cs"));
program.Should().Contain("SentryObservability.ResolveRelease");
program.Should().Contain("options.Release = release");
program.Should().Contain("\"service\", SentryObservability.ServiceName");
program.Should().Contain("\"app.commit\"");
program.Should().Contain("MaxRequestBodySize = RequestSize.None");
program.Should().Contain("SetBeforeSend(SentryTelemetryScrubber.Scrub)");
program.Should().Contain("SetBeforeSendTransaction(SentryTelemetryScrubber.Scrub)");
}
[Theory]
[InlineData("Api.SeaHavenIndustries/Helper/VendorDocumentScanWorker.cs", "vendor-documents.scan-pending")]
[InlineData("Api.SeaHavenIndustries/HostedServices/WorkOrderReconciliationHostedService.cs", "workorders.reconciliation-trigger")]
[InlineData("Api.SeaHavenIndustries/HostedServices/WorkOrderReconciliationHostedService.cs", "workorders.reconciliation-run-pending")]
[InlineData("Api.SeaHavenIndustries/HostedServices/WorkOrderWeekRolledHostedService.cs", "workorders.week-rolled-job")]
[InlineData("Api.SeaHavenIndustries/HostedServices/PastDueCacheHostedService.cs", "workorders.past-due-cache-job")]
[InlineData("Api.SeaHavenIndustries/HostedServices/UpliftLifecycleHostedService.cs", "uplifts.lifecycle-sweep")]
public void Workers_UseSharedBackgroundTransactionHelper_WithStableNames(string relativePath, string transactionName)
{
var source = File.ReadAllText(Path.Combine(RepoRoot(), relativePath));
source.Should().Contain($"BeginBackgroundTransaction", "worker transactions must go through the shared helper");
source.Should().Contain($"\"{transactionName}\"");
source.Should().NotContain("PushScope", "scope pushing is owned by the shared helper");
}
}

View file

@ -15,6 +15,7 @@
<PrivateAssets>all</PrivateAssets> <PrivateAssets>all</PrivateAssets>
</PackageReference> </PackageReference>
<PackageReference Include="SendGrid" Version="9.29.3" /> <PackageReference Include="SendGrid" Version="9.29.3" />
<PackageReference Include="Sentry.AspNetCore" Version="6.10.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" /> <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
<PackageReference Include="PasswordGenerator" Version="2.1.0" /> <PackageReference Include="PasswordGenerator" Version="2.1.0" />
</ItemGroup> </ItemGroup>

View file

@ -1,5 +1,7 @@
using Api.SeaHavenIndustries.Observability;
using Data.SeaHavenIndustries; using Data.SeaHavenIndustries;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Sentry;
namespace Api.SeaHavenIndustries.Helper namespace Api.SeaHavenIndustries.Helper
{ {
@ -8,15 +10,18 @@ namespace Api.SeaHavenIndustries.Helper
private readonly IServiceScopeFactory _scopeFactory; private readonly IServiceScopeFactory _scopeFactory;
private readonly IWebHostEnvironment _environment; private readonly IWebHostEnvironment _environment;
private readonly ILogger<VendorDocumentScanWorker> _logger; private readonly ILogger<VendorDocumentScanWorker> _logger;
private readonly IHub _sentryHub;
public VendorDocumentScanWorker( public VendorDocumentScanWorker(
IServiceScopeFactory scopeFactory, IServiceScopeFactory scopeFactory,
IWebHostEnvironment environment, IWebHostEnvironment environment,
ILogger<VendorDocumentScanWorker> logger) ILogger<VendorDocumentScanWorker> logger,
IHub sentryHub)
{ {
_scopeFactory = scopeFactory; _scopeFactory = scopeFactory;
_environment = environment; _environment = environment;
_logger = logger; _logger = logger;
_sentryHub = sentryHub;
} }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@ -30,36 +35,55 @@ namespace Api.SeaHavenIndustries.Helper
private async Task ScanPendingAsync(CancellationToken cancellationToken) private async Task ScanPendingAsync(CancellationToken cancellationToken)
{ {
using var scope = _scopeFactory.CreateScope(); using var transaction = SentryObservability.BeginBackgroundTransaction(
var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>(); _sentryHub,
var scanner = scope.ServiceProvider.GetRequiredService<IDocumentScanner>(); "vendor-documents.scan-pending",
var pending = await db.VendorCompletionDocuments $"{nameof(VendorDocumentScanWorker)}.{nameof(ScanPendingAsync)}");
.Where(document => document.ScanStatus == "Pending") try
.OrderBy(document => document.CreatedDate)
.Take(20)
.ToListAsync(cancellationToken);
foreach (var document in pending)
{ {
var path = VendorDocumentStorage.ResolvePath(_environment.ContentRootPath, document); using var scope = _scopeFactory.CreateScope();
var result = await scanner.ScanAsync(path, cancellationToken); var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
if (result == DocumentScanResult.Unavailable) var scanner = scope.ServiceProvider.GetRequiredService<IDocumentScanner>();
var pending = await db.VendorCompletionDocuments
.Where(document => document.ScanStatus == "Pending")
.OrderBy(document => document.CreatedDate)
.Take(20)
.ToListAsync(cancellationToken);
foreach (var document in pending)
{ {
_logger.LogWarning("Vendor document scan service unavailable; document {DocumentId} remains quarantined", document.Id); var path = VendorDocumentStorage.ResolvePath(_environment.ContentRootPath, document);
continue; var result = await scanner.ScanAsync(path, cancellationToken);
if (result == DocumentScanResult.Unavailable)
{
_logger.LogWarning("Vendor document scan service unavailable; document {DocumentId} remains quarantined", document.Id);
continue;
}
document.ScanStatus = result == DocumentScanResult.Passed ? "Passed" : "Rejected";
document.ReviewStatus = result == DocumentScanResult.Passed ? "Processing" : "Rejected";
document.RejectionReason = result == DocumentScanResult.Infected
? "The upload failed malware scanning."
: document.RejectionReason;
document.ScannedAt = DateTime.UtcNow;
document.LastModificationTime = DateTime.UtcNow;
await db.SaveChangesAsync(cancellationToken);
if (result == DocumentScanResult.Infected && File.Exists(path))
File.Delete(path);
} }
document.ScanStatus = result == DocumentScanResult.Passed ? "Passed" : "Rejected"; transaction.FinishOk();
document.ReviewStatus = result == DocumentScanResult.Passed ? "Processing" : "Rejected"; }
document.RejectionReason = result == DocumentScanResult.Infected catch (OperationCanceledException)
? "The upload failed malware scanning." {
: document.RejectionReason; transaction.FinishCancelled();
document.ScannedAt = DateTime.UtcNow; throw;
document.LastModificationTime = DateTime.UtcNow; }
await db.SaveChangesAsync(cancellationToken); catch (Exception ex)
{
if (result == DocumentScanResult.Infected && File.Exists(path)) transaction.FinishError(ex);
File.Delete(path); throw;
} }
} }
} }

View file

@ -2,6 +2,8 @@ using Microsoft.Extensions.Options;
using SeaHaven.Services.Implementation; using SeaHaven.Services.Implementation;
using SeaHaven.Services.Interfaces; using SeaHaven.Services.Interfaces;
using Api.SeaHavenIndustries.Options; using Api.SeaHavenIndustries.Options;
using Api.SeaHavenIndustries.Observability;
using Sentry;
namespace Api.SeaHavenIndustries.HostedServices namespace Api.SeaHavenIndustries.HostedServices
{ {
@ -11,18 +13,21 @@ namespace Api.SeaHavenIndustries.HostedServices
private readonly ILogger<PastDueCacheHostedService> _logger; private readonly ILogger<PastDueCacheHostedService> _logger;
private readonly PastDueCacheJobOptions _options; private readonly PastDueCacheJobOptions _options;
private readonly WorkOrderJobRunState _runState; private readonly WorkOrderJobRunState _runState;
private readonly IHub _sentryHub;
private DateOnly? _lastRunDate; private DateOnly? _lastRunDate;
public PastDueCacheHostedService( public PastDueCacheHostedService(
IServiceScopeFactory scopeFactory, IServiceScopeFactory scopeFactory,
IOptions<WorkOrderJobsOptions> options, IOptions<WorkOrderJobsOptions> options,
WorkOrderJobRunState runState, WorkOrderJobRunState runState,
ILogger<PastDueCacheHostedService> logger) ILogger<PastDueCacheHostedService> logger,
IHub sentryHub)
{ {
_scopeFactory = scopeFactory; _scopeFactory = scopeFactory;
_logger = logger; _logger = logger;
_options = options.Value.PastDueCache; _options = options.Value.PastDueCache;
_runState = runState; _runState = runState;
_sentryHub = sentryHub;
} }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@ -45,6 +50,10 @@ namespace Api.SeaHavenIndustries.HostedServices
private async Task<bool> RunJobAsync(CancellationToken stoppingToken) private async Task<bool> RunJobAsync(CancellationToken stoppingToken)
{ {
using var transaction = SentryObservability.BeginBackgroundTransaction(
_sentryHub,
"workorders.past-due-cache-job",
$"{nameof(PastDueCacheHostedService)}.{nameof(RunJobAsync)}");
try try
{ {
using var scope = _scopeFactory.CreateScope(); using var scope = _scopeFactory.CreateScope();
@ -53,12 +62,19 @@ namespace Api.SeaHavenIndustries.HostedServices
_runState.LastPastDueCacheRunUtc = DateTime.UtcNow; _runState.LastPastDueCacheRunUtc = DateTime.UtcNow;
_runState.LastPastDueCacheError = null; _runState.LastPastDueCacheError = null;
transaction.FinishOk();
return true; return true;
} }
catch (OperationCanceledException)
{
transaction.FinishCancelled();
throw;
}
catch (Exception ex) when (ex is not OperationCanceledException) catch (Exception ex) when (ex is not OperationCanceledException)
{ {
_runState.LastPastDueCacheError = ex.Message; _runState.LastPastDueCacheError = ex.Message;
_logger.LogError(ex, "PastDue cache hosted job failed."); _logger.LogError(ex, "PastDue cache hosted job failed.");
transaction.FinishError(ex);
return false; return false;
} }
} }

View file

@ -1,6 +1,8 @@
using Api.SeaHavenIndustries.Observability;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using SeaHaven.Services.Configuration; using SeaHaven.Services.Configuration;
using SeaHaven.Services.Interfaces; using SeaHaven.Services.Interfaces;
using Sentry;
namespace Api.SeaHavenIndustries.HostedServices namespace Api.SeaHavenIndustries.HostedServices
{ {
@ -9,32 +11,44 @@ namespace Api.SeaHavenIndustries.HostedServices
private readonly IServiceScopeFactory _scopeFactory; private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<UpliftLifecycleHostedService> _logger; private readonly ILogger<UpliftLifecycleHostedService> _logger;
private readonly ApprovalsOptions _options; private readonly ApprovalsOptions _options;
private readonly IHub _sentryHub;
public UpliftLifecycleHostedService( public UpliftLifecycleHostedService(
IServiceScopeFactory scopeFactory, IServiceScopeFactory scopeFactory,
IOptions<ApprovalsOptions> options, IOptions<ApprovalsOptions> options,
ILogger<UpliftLifecycleHostedService> logger) ILogger<UpliftLifecycleHostedService> logger,
IHub sentryHub)
{ {
_scopeFactory = scopeFactory; _scopeFactory = scopeFactory;
_logger = logger; _logger = logger;
_options = options.Value; _options = options.Value;
_sentryHub = sentryHub;
} }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{ {
while (!stoppingToken.IsCancellationRequested) while (!stoppingToken.IsCancellationRequested)
{ {
try
{ {
await SweepAsync(stoppingToken); using var transaction = SentryObservability.BeginBackgroundTransaction(
} _sentryHub,
catch (OperationCanceledException) "uplifts.lifecycle-sweep",
{ $"{nameof(UpliftLifecycleHostedService)}.{nameof(SweepAsync)}");
throw; try
} {
catch (Exception ex) await SweepAsync(stoppingToken);
{ transaction.FinishOk();
_logger.LogError(ex, "Uplift lifecycle sweep failed; will retry on next interval"); }
catch (OperationCanceledException)
{
transaction.FinishCancelled();
throw;
}
catch (Exception ex)
{
_logger.LogError(ex, "Uplift lifecycle sweep failed; will retry on next interval");
transaction.FinishError(ex);
}
} }
await Task.Delay(_options.EffectiveSweepInterval, stoppingToken); await Task.Delay(_options.EffectiveSweepInterval, stoppingToken);

View file

@ -1,6 +1,8 @@
using Api.SeaHavenIndustries.Observability;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using SeaHaven.Services.Configuration; using SeaHaven.Services.Configuration;
using SeaHaven.Services.Interfaces; using SeaHaven.Services.Interfaces;
using Sentry;
namespace Api.SeaHavenIndustries.HostedServices namespace Api.SeaHavenIndustries.HostedServices
{ {
@ -10,17 +12,20 @@ namespace Api.SeaHavenIndustries.HostedServices
private readonly IOptionsMonitor<WorkOrderReconciliationOptions> _options; private readonly IOptionsMonitor<WorkOrderReconciliationOptions> _options;
private readonly TimeProvider _timeProvider; private readonly TimeProvider _timeProvider;
private readonly ILogger<WorkOrderReconciliationHostedService> _logger; private readonly ILogger<WorkOrderReconciliationHostedService> _logger;
private readonly IHub _sentryHub;
public WorkOrderReconciliationHostedService( public WorkOrderReconciliationHostedService(
IServiceScopeFactory scopeFactory, IServiceScopeFactory scopeFactory,
IOptionsMonitor<WorkOrderReconciliationOptions> options, IOptionsMonitor<WorkOrderReconciliationOptions> options,
TimeProvider timeProvider, TimeProvider timeProvider,
ILogger<WorkOrderReconciliationHostedService> logger) ILogger<WorkOrderReconciliationHostedService> logger,
IHub sentryHub)
{ {
_scopeFactory = scopeFactory; _scopeFactory = scopeFactory;
_options = options; _options = options;
_timeProvider = timeProvider; _timeProvider = timeProvider;
_logger = logger; _logger = logger;
_sentryHub = sentryHub;
} }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@ -56,29 +61,52 @@ namespace Api.SeaHavenIndustries.HostedServices
private async Task TriggerAsync(string reason, CancellationToken cancellationToken) private async Task TriggerAsync(string reason, CancellationToken cancellationToken)
{ {
using var transaction = SentryObservability.BeginBackgroundTransaction(
_sentryHub,
"workorders.reconciliation-trigger",
$"{nameof(WorkOrderReconciliationHostedService)}.{nameof(TriggerAsync)}",
description: reason);
try try
{ {
await using var scope = _scopeFactory.CreateAsyncScope(); await using var scope = _scopeFactory.CreateAsyncScope();
var runner = scope.ServiceProvider.GetRequiredService<IWorkOrderReconciliationRunner>(); var runner = scope.ServiceProvider.GetRequiredService<IWorkOrderReconciliationRunner>();
await runner.TriggerAsync(reason, cancellationToken); await runner.TriggerAsync(reason, cancellationToken);
transaction.FinishOk();
}
catch (OperationCanceledException)
{
transaction.FinishCancelled();
throw;
} }
catch (Exception ex) when (ex is not OperationCanceledException) catch (Exception ex) when (ex is not OperationCanceledException)
{ {
_logger.LogError(ex, "Could not queue procurement reconciliation."); _logger.LogError(ex, "Could not queue procurement reconciliation.");
transaction.FinishError(ex);
} }
} }
private async Task RunPendingAsync(CancellationToken cancellationToken) private async Task RunPendingAsync(CancellationToken cancellationToken)
{ {
using var transaction = SentryObservability.BeginBackgroundTransaction(
_sentryHub,
"workorders.reconciliation-run-pending",
$"{nameof(WorkOrderReconciliationHostedService)}.{nameof(RunPendingAsync)}");
try try
{ {
await using var scope = _scopeFactory.CreateAsyncScope(); await using var scope = _scopeFactory.CreateAsyncScope();
var runner = scope.ServiceProvider.GetRequiredService<IWorkOrderReconciliationRunner>(); var runner = scope.ServiceProvider.GetRequiredService<IWorkOrderReconciliationRunner>();
await runner.RunPendingAsync(cancellationToken); await runner.RunPendingAsync(cancellationToken);
transaction.FinishOk();
}
catch (OperationCanceledException)
{
transaction.FinishCancelled();
throw;
} }
catch (Exception ex) when (ex is not OperationCanceledException) catch (Exception ex) when (ex is not OperationCanceledException)
{ {
_logger.LogError(ex, "Could not execute procurement reconciliation."); _logger.LogError(ex, "Could not execute procurement reconciliation.");
transaction.FinishError(ex);
} }
} }
} }

View file

@ -3,6 +3,8 @@ using SeaHaven.Services.Helpers;
using SeaHaven.Services.Implementation; using SeaHaven.Services.Implementation;
using SeaHaven.Services.Interfaces; using SeaHaven.Services.Interfaces;
using Api.SeaHavenIndustries.Options; using Api.SeaHavenIndustries.Options;
using Api.SeaHavenIndustries.Observability;
using Sentry;
namespace Api.SeaHavenIndustries.HostedServices namespace Api.SeaHavenIndustries.HostedServices
{ {
@ -12,18 +14,21 @@ namespace Api.SeaHavenIndustries.HostedServices
private readonly ILogger<WorkOrderWeekRolledHostedService> _logger; private readonly ILogger<WorkOrderWeekRolledHostedService> _logger;
private readonly WeekRolledJobOptions _options; private readonly WeekRolledJobOptions _options;
private readonly WorkOrderJobRunState _runState; private readonly WorkOrderJobRunState _runState;
private readonly IHub _sentryHub;
private DateOnly? _lastRunDate; private DateOnly? _lastRunDate;
public WorkOrderWeekRolledHostedService( public WorkOrderWeekRolledHostedService(
IServiceScopeFactory scopeFactory, IServiceScopeFactory scopeFactory,
IOptions<WorkOrderJobsOptions> options, IOptions<WorkOrderJobsOptions> options,
WorkOrderJobRunState runState, WorkOrderJobRunState runState,
ILogger<WorkOrderWeekRolledHostedService> logger) ILogger<WorkOrderWeekRolledHostedService> logger,
IHub sentryHub)
{ {
_scopeFactory = scopeFactory; _scopeFactory = scopeFactory;
_logger = logger; _logger = logger;
_options = options.Value.WeekRolled; _options = options.Value.WeekRolled;
_runState = runState; _runState = runState;
_sentryHub = sentryHub;
} }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@ -46,6 +51,10 @@ namespace Api.SeaHavenIndustries.HostedServices
private async Task<bool> RunJobAsync(CancellationToken stoppingToken) private async Task<bool> RunJobAsync(CancellationToken stoppingToken)
{ {
using var transaction = SentryObservability.BeginBackgroundTransaction(
_sentryHub,
"workorders.week-rolled-job",
$"{nameof(WorkOrderWeekRolledHostedService)}.{nameof(RunJobAsync)}");
try try
{ {
var sourceWeekStart = WorkOrderOperationalWeek.GetPreviousOperationalWeekStart(DateTime.UtcNow); var sourceWeekStart = WorkOrderOperationalWeek.GetPreviousOperationalWeekStart(DateTime.UtcNow);
@ -56,12 +65,19 @@ namespace Api.SeaHavenIndustries.HostedServices
_runState.LastWeekRolledRunUtc = DateTime.UtcNow; _runState.LastWeekRolledRunUtc = DateTime.UtcNow;
_runState.LastWeekRolledError = null; _runState.LastWeekRolledError = null;
transaction.FinishOk();
return true; return true;
} }
catch (OperationCanceledException)
{
transaction.FinishCancelled();
throw;
}
catch (Exception ex) when (ex is not OperationCanceledException) catch (Exception ex) when (ex is not OperationCanceledException)
{ {
_runState.LastWeekRolledError = ex.Message; _runState.LastWeekRolledError = ex.Message;
_logger.LogError(ex, "WeekRolled hosted job failed."); _logger.LogError(ex, "WeekRolled hosted job failed.");
transaction.FinishError(ex);
return false; return false;
} }
} }

View file

@ -0,0 +1,20 @@
using Api.SeaHavenIndustries.Observability;
using Sentry;
namespace Api.SeaHavenIndustries.Middleware;
public sealed class SentryRequestMetadataMiddleware
{
private readonly RequestDelegate _next;
public SentryRequestMetadataMiddleware(RequestDelegate next)
{
_next = next;
}
public async Task InvokeAsync(HttpContext context, IHub sentryHub)
{
SentryObservability.ConfigureRequest(sentryHub, context);
await _next(context);
}
}

View file

@ -0,0 +1,272 @@
using System.Reflection;
using System.Security.Claims;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc.Controllers;
using Microsoft.AspNetCore.Routing;
using Sentry;
namespace Api.SeaHavenIndustries.Observability;
public static class SentryObservability
{
public const string ServiceName = "shoc-backend";
public const string LocalDevelopmentRelease = "local-development";
private const string ReleasePrefix = ServiceName + "@";
private const int CommitShaLength = 40;
public static string ResolveRelease(Assembly? assembly)
{
var informationalVersion = assembly?
.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?
.InformationalVersion;
return ResolveRelease(informationalVersion);
}
public static string? ResolveCommitSha(Assembly? assembly)
{
var informationalVersion = assembly?
.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?
.InformationalVersion;
return IsReleaseInformationalVersion(informationalVersion)
? informationalVersion![ReleasePrefix.Length..]
: null;
}
public static string ResolveRelease(string? informationalVersion) =>
IsReleaseInformationalVersion(informationalVersion)
? informationalVersion!
: LocalDevelopmentRelease;
public static bool IsReleaseInformationalVersion(string? value) =>
value is not null
&& value.StartsWith(ReleasePrefix, StringComparison.Ordinal)
&& value.Length == ReleasePrefix.Length + CommitShaLength
&& value[ReleasePrefix.Length..].All(c => c is >= '0' and <= '9' or >= 'a' and <= 'f' or >= 'A' and <= 'F');
public static void ConfigureRequest(IHub hub, HttpContext context)
{
var endpoint = context.GetEndpoint();
var routeEndpoint = endpoint as RouteEndpoint;
var routeTemplate = routeEndpoint?.RoutePattern.RawText;
var actionDescriptor = endpoint?.Metadata.GetMetadata<ControllerActionDescriptor>();
var userId = context.User.FindFirst(ClaimTypes.NameIdentifier)?.Value;
var authenticated = !string.IsNullOrEmpty(userId);
hub.ConfigureScope(scope =>
{
if (authenticated)
scope.User = new SentryUser { Id = userId };
scope.SetTag("actor.type", authenticated ? "authenticated" : "anonymous");
scope.SetTag("operation.type", "http.server");
if (actionDescriptor is not null)
scope.SetTag("code.function", $"{actionDescriptor.ControllerName}.{actionDescriptor.ActionName}");
scope.SetTag("http.method", context.Request.Method);
if (!string.IsNullOrEmpty(routeTemplate))
scope.SetTag("http.route", routeTemplate);
var activeSpan = hub.GetSpan();
if (activeSpan is not null)
{
scope.SetTag("trace_id", activeSpan.TraceId.ToString());
scope.SetTag("transaction_id", activeSpan.SpanId.ToString());
}
});
}
public static SentryBackgroundTransactionHandle BeginBackgroundTransaction(
IHub hub,
string name,
string function,
string? description = null)
{
var scope = hub.PushScope();
var transaction = hub.StartTransaction(name, "task");
if (!string.IsNullOrEmpty(description))
transaction.Description = description;
hub.ConfigureScope(s => s.Transaction = transaction);
foreach (var (key, value) in new[]
{
("actor.type", "system"),
("operation.type", "background_job"),
("code.function", function),
("trace_id", transaction.TraceId.ToString()),
("transaction_id", transaction.SpanId.ToString()),
})
{
transaction.SetTag(key, value);
hub.ConfigureScope(s => s.SetTag(key, value));
}
return new SentryBackgroundTransactionHandle(transaction, scope);
}
}
public static class SentryTelemetryScrubber
{
private static readonly HashSet<string> SafeSpanDataKeys = new(StringComparer.Ordinal)
{
"actor.type",
"code.function",
"db.operation.name",
"db.system",
"http.method",
"http.request.method",
"http.response.status_code",
"http.route",
"http.status_code",
"network.protocol.version",
"operation.type",
"server.address",
"url.scheme",
};
private static readonly HashSet<string> SafeSpanTagKeys = new(StringComparer.Ordinal)
{
"actor.type",
"code.function",
"http.method",
"http.route",
"operation.type",
};
public static SentryEvent Scrub(SentryEvent @event)
{
ScrubRequest(@event.Request);
@event.User = KeepOpaqueIdOnly(@event.User);
return @event;
}
public static SentryTransaction Scrub(SentryTransaction transaction)
{
ScrubRequest(transaction.Request);
transaction.User = KeepOpaqueIdOnly(transaction.User);
transaction.SetTag("trace_id", transaction.TraceId.ToString());
transaction.SetTag("transaction_id", transaction.SpanId.ToString());
ScrubDictionary(transaction.Data, SafeSpanDataKeys);
foreach (var span in transaction.Spans)
{
ScrubDictionary(span.Data, SafeSpanDataKeys);
foreach (var tag in span.Tags.Keys.Where(key => !SafeSpanTagKeys.Contains(key)).ToArray())
span.UnsetTag(tag);
span.Description = span.Operation?.Contains("http", StringComparison.OrdinalIgnoreCase) == true
? NormalizeHttpDescription(span.Description)
: null;
}
return transaction;
}
private static SentryUser KeepOpaqueIdOnly(SentryUser? user) =>
string.IsNullOrWhiteSpace(user?.Id) ? new SentryUser() : new SentryUser { Id = user.Id };
private static void ScrubRequest(SentryRequest? request)
{
if (request is null)
return;
request.Data = null;
request.QueryString = null;
request.Cookies = null;
request.Headers.Clear();
request.Env.Clear();
request.Other.Clear();
}
private static void ScrubDictionary<TValue>(
IReadOnlyDictionary<string, TValue> values,
IReadOnlySet<string> allowedKeys)
{
if (values is not IDictionary<string, TValue> mutable)
return;
foreach (var key in mutable.Keys.Where(key => !allowedKeys.Contains(key)).ToArray())
mutable.Remove(key);
}
private static string? NormalizeHttpDescription(string? description)
{
if (string.IsNullOrWhiteSpace(description))
return null;
var separator = description.IndexOf(' ');
if (separator <= 0 || separator == description.Length - 1)
return null;
var method = description[..separator].ToUpperInvariant();
if (method is not ("GET" or "POST" or "PUT" or "PATCH" or "DELETE" or "HEAD" or "OPTIONS"))
return null;
var rawUrl = description[(separator + 1)..];
var path = Uri.TryCreate(rawUrl, UriKind.Absolute, out var absolute)
? absolute.GetLeftPart(UriPartial.Authority) + absolute.AbsolutePath
: rawUrl.Split('?', '#')[0];
return $"{method} {NormalizePathIdentifiers(path)}";
}
private static string NormalizePathIdentifiers(string path) =>
string.Join('/', path.Split('/').Select(segment => IsIdentifierSegment(segment) ? ":id" : segment));
private static bool IsIdentifierSegment(string segment)
{
if (string.IsNullOrEmpty(segment))
return false;
if (long.TryParse(segment, out _) || Guid.TryParse(segment, out _))
return true;
var allHex = segment.All(Uri.IsHexDigit);
if (segment.Length >= 8 && allHex)
return true;
return segment.Length >= 16
&& segment.All(c => char.IsAsciiLetterOrDigit(c) || c is '.' or '_' or '~' or '-')
&& segment.Any(char.IsAsciiLetter)
&& segment.Any(char.IsAsciiDigit);
}
}
public sealed class SentryBackgroundTransactionHandle : IDisposable
{
private readonly ITransactionTracer _transaction;
private readonly IDisposable _scope;
private int _finished;
internal SentryBackgroundTransactionHandle(ITransactionTracer transaction, IDisposable scope)
{
_transaction = transaction;
_scope = scope;
}
public void FinishOk() => Finish(() => _transaction.Finish(SpanStatus.Ok));
public void FinishCancelled() => Finish(() => _transaction.Finish(SpanStatus.Cancelled));
public void FinishError(Exception exception) =>
Finish(() => _transaction.Finish(exception, SpanStatus.InternalError));
private void Finish(Action finish)
{
if (Interlocked.Exchange(ref _finished, 1) == 0)
finish();
}
public void Dispose()
{
Finish(() => _transaction.Finish(SpanStatus.UnknownError));
_scope.Dispose();
}
}

View file

@ -1,6 +1,7 @@
using Api.SeaHavenIndustries.Helper; using Api.SeaHavenIndustries.Helper;
using Api.SeaHavenIndustries.HostedServices; using Api.SeaHavenIndustries.HostedServices;
using Api.SeaHavenIndustries.Middleware; using Api.SeaHavenIndustries.Middleware;
using Api.SeaHavenIndustries.Observability;
using Api.SeaHavenIndustries.Options; using Api.SeaHavenIndustries.Options;
using Data.SeaHavenIndustries; using Data.SeaHavenIndustries;
using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authentication.JwtBearer;
@ -9,6 +10,8 @@ using Microsoft.AspNetCore.ResponseCompression;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.IdentityModel.Tokens; using Microsoft.IdentityModel.Tokens;
using Microsoft.OpenApi.Models; using Microsoft.OpenApi.Models;
using Sentry.AspNetCore;
using Sentry.Extensibility;
using System.Text; using System.Text;
using SeaHaven.DataServices.DependencyInjection; using SeaHaven.DataServices.DependencyInjection;
using SeaHaven.Services.DependencyInjection; using SeaHaven.Services.DependencyInjection;
@ -16,6 +19,26 @@ using SeaHaven.Services.Implementation;
using SeaHaven.Services.Interfaces; using SeaHaven.Services.Interfaces;
var builder = WebApplication.CreateBuilder(args); var builder = WebApplication.CreateBuilder(args);
var entryAssembly = System.Reflection.Assembly.GetEntryAssembly();
var release = SentryObservability.ResolveRelease(entryAssembly);
var commitSha = SentryObservability.ResolveCommitSha(entryAssembly);
builder.WebHost.UseSentry((SentryAspNetCoreOptions options) =>
{
options.Dsn = builder.Configuration["SENTRY_DSN"] ?? string.Empty;
options.Environment = builder.Configuration["SENTRY_ENVIRONMENT"]
?? builder.Environment.EnvironmentName.ToLowerInvariant();
options.Release = release;
options.DefaultTags.Add("service", SentryObservability.ServiceName);
options.DefaultTags.Add("app.commit", commitSha ?? SentryObservability.LocalDevelopmentRelease);
options.TracesSampleRate = 1.0;
options.SendDefaultPii = false;
options.MaxRequestBodySize = RequestSize.None;
options.SetBeforeSend(SentryTelemetryScrubber.Scrub);
options.SetBeforeSendTransaction(SentryTelemetryScrubber.Scrub);
});
ConfigurationManager configuration = builder.Configuration; ConfigurationManager configuration = builder.Configuration;
builder.Services.AddDbContext<ApplicationDbContext>(options => options.UseSqlServer(configuration.GetConnectionString("DefaultConnection"))); builder.Services.AddDbContext<ApplicationDbContext>(options => options.UseSqlServer(configuration.GetConnectionString("DefaultConnection")));
@ -192,7 +215,9 @@ if (!app.Environment.IsDevelopment())
app.UseStaticFiles(); app.UseStaticFiles();
app.UseCors(); app.UseCors();
app.UseMiddleware<LegacyDeprecationMiddleware>(); app.UseMiddleware<LegacyDeprecationMiddleware>();
app.UseRouting();
app.UseAuthentication(); app.UseAuthentication();
app.UseMiddleware<SentryRequestMetadataMiddleware>();
app.UseAuthorization(); app.UseAuthorization();
app.MapControllers(); app.MapControllers();

View file

@ -0,0 +1,76 @@
# ADR 0002 — Sentry observability: release identity and redaction-safe telemetry fields
- Status: Accepted
- Date: 2026-09-03
- Scope: `Api.SeaHavenIndustries` (observability layer only; no behavior changes)
## Context
The backend reports to Sentry (SH-298). Events arrived without a stable release
identity, without service/commit attribution, and background jobs each hand-rolled
their own transaction setup. HTTP request metadata risked leaking PII through
route values, query strings, or identity claims.
## Decision
1. **Release identity.** `SentryObservability.ResolveRelease(Assembly)` accepts an
informational version shaped `shoc-backend@<40 hex commit sha>` and otherwise
resolves to `local-development`. `scripts/package-elastic-beanstalk.sh` resolves
the commit from `APP_COMMIT_SHA` → `GITHUB_SHA` → `git rev-parse HEAD`, fails
when `CI=true` cannot produce a valid 40-hex sha, and publishes with
`-p:SourceRevisionId`, `-p:InformationalVersion=shoc-backend@<sha>`,
`-p:DebugType=portable`, `-p:DebugSymbols=true`.
2. **Default tags.** `service=shoc-backend` and `app.commit=<sha | local-development>`
on `SentryOptions.DefaultTags`. All events use the same commit-addressed
`shoc-backend@<sha>` release.
3. **HTTP requests.** `SentryRequestMetadataMiddleware` runs **after**
routing/authentication and **before** authorization. It sets on the current
scope only the fields below. Non-controller endpoints omit `code.function`.
4. **Background jobs.** All six worker transaction blocks (five files) use
`SentryObservability.BeginBackgroundTransaction`, which pushes a scope, starts
the transaction, sets the scope transaction, and returns a handle with
`FinishOk` / `FinishCancelled` / `FinishError(Exception)` that cannot
double-finish. Existing status/error behavior is preserved.
5. **Last-mile privacy.** Request-body extraction is disabled. Global before-send
processors clear request data, query strings, cookies, headers, environment
values, and miscellaneous request values from both error events and
transactions, retain only an opaque user ID, and attach native trace and
transaction IDs as searchable transaction tags. Root and child span data use
an allowlist for function, operation, actor, route template, method/status,
protocol, database operation, and service host; raw URLs, query values,
network addresses, and arbitrary extras are removed.
## Field / redaction contract
Allowed request fields (template values only — never resolved route values):
| Field | Source | Rule |
| --- | --- | --- |
| `actor.type` | auth state | `authenticated` / `anonymous` / `system` |
| `operation.type` | static | `http.server` / `background_job` |
| `code.function` | `ControllerActionDescriptor` | `ControllerName.ActionName` only |
| `http.method` | request | HTTP verb |
| `http.route` | `RouteEndpoint.RoutePattern.RawText` | route template; omitted when unresolved |
| `trace_id` / `transaction_id` | active Sentry span/transaction | native Sentry ids |
| scope `User.Id` | `ClaimTypes.NameIdentifier` only | set only when authenticated |
Never intentionally sent: query strings, headers, bodies, cookies, usernames,
emails, IP addresses, route values, credentials, or token values. The application
retains stack traces, exception types/messages, safe tags, operation status, and
route templates because those are required to identify the failing function and
operation.
## Consequences
- Release health maps 1:1 to a commit sha; local builds are clearly labeled.
- One redaction contract to audit, enforced by
`SentryObservabilityTests`/`SentryPipelineContractTests`.
- ASP.NET Core request transactions and the SDK's registered outgoing HTTP
instrumentation cover inbound API calls and the platform's typed procurement
client; explicit root transactions cover every hosted worker loop.
## Alternatives considered
- Per-worker transaction setup (rejected: duplication drifted across five files).
- Sentry's default request payload with PII scrubbing (rejected: default-deny is
safer than default-capture-then-scrub).

View file

@ -67,6 +67,23 @@ RUNTIME="${RUNTIME:-linux-x64}"
[[ -f "$API_PROJECT" ]] || die "missing API project: $API_PROJECT" [[ -f "$API_PROJECT" ]] || die "missing API project: $API_PROJECT"
[[ -f "$MIGRATIONS_PROJECT" ]] || die "missing migrations project: $MIGRATIONS_PROJECT" [[ -f "$MIGRATIONS_PROJECT" ]] || die "missing migrations project: $MIGRATIONS_PROJECT"
log "resolve commit SHA for release metadata"
COMMIT_SHA="${APP_COMMIT_SHA:-${GITHUB_SHA:-}}"
if [[ ! "$COMMIT_SHA" =~ ^[0-9a-fA-F]{40}$ ]] && command -v git >/dev/null 2>&1; then
GIT_SHA="$(git rev-parse HEAD 2>/dev/null || true)"
if [[ "$GIT_SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then
COMMIT_SHA="$GIT_SHA"
fi
fi
if [[ ! "$COMMIT_SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then
if [[ "${CI:-false}" == "true" ]]; then
die "CI build could not resolve a valid 40-hex commit SHA (APP_COMMIT_SHA/GITHUB_SHA/git)."
fi
die "could not resolve a valid 40-hex commit SHA (APP_COMMIT_SHA/GITHUB_SHA/git rev-parse HEAD)."
fi
COMMIT_SHA="$(printf '%s' "$COMMIT_SHA" | tr '[:upper:]' '[:lower:]')"
printf ' commit: %s\n' "$COMMIT_SHA"
if command -v zip >/dev/null 2>&1; then if command -v zip >/dev/null 2>&1; then
ARCHIVER="zip" ARCHIVER="zip"
elif command -v python >/dev/null 2>&1; then elif command -v python >/dev/null 2>&1; then
@ -92,7 +109,11 @@ log "publish $API_PROJECT (Release, self-contained, $RUNTIME)"
--runtime "$RUNTIME" \ --runtime "$RUNTIME" \
-o "$STAGING_DIR" \ -o "$STAGING_DIR" \
-p:ContinuousIntegrationBuild=true \ -p:ContinuousIntegrationBuild=true \
-p:UseAppHost=true -p:UseAppHost=true \
-p:SourceRevisionId="$COMMIT_SHA" \
-p:InformationalVersion="shoc-backend@$COMMIT_SHA" \
-p:DebugType=portable \
-p:DebugSymbols=true
log "install dotnet-ef $EF_VERSION (local tool path)" log "install dotnet-ef $EF_VERSION (local tool path)"
if ! "$DOTNET" tool install dotnet-ef --version "$EF_VERSION" --tool-path "$TOOLS_DIR" 2>/dev/null; then if ! "$DOTNET" tool install dotnet-ef --version "$EF_VERSION" --tool-path "$TOOLS_DIR" 2>/dev/null; then

View file

@ -14,6 +14,13 @@ Secret metadata is managed, but secret values are never authored in Terraform
configuration. Elastic Beanstalk receives secret values through configuration. Elastic Beanstalk receives secret values through
`environmentsecrets` ARN/key references. `environmentsecrets` ARN/key references.
The Sentry DSN is public ingestion configuration, not a secret: each root passes
it through the required `sentry_dsn` module variable as the plain
`SENTRY_DSN` environment setting. `SENTRY_ENVIRONMENT` is `development` for the
dev root and the root environment name otherwise. Production has no Terraform
root yet; production Sentry wiring will reuse the same variable when one is
added.
## HCP credentials ## HCP credentials
Org-baseline CloudFormation owns the HCP Terraform plan/apply roles and their Org-baseline CloudFormation owns the HCP Terraform plan/apply roles and their

View file

@ -21,7 +21,12 @@ Secret values are not Terraform resources, variables, outputs, or managed EB
settings. Terraform manages the app-config secret shell and maps approved JSON settings. Terraform manages the app-config secret shell and maps approved JSON
keys through `aws:elasticbeanstalk:application:environmentsecrets` using keys through `aws:elasticbeanstalk:application:environmentsecrets` using
`secret-arn:json-key` references. Ordinary application environment settings are `secret-arn:json-key` references. Ordinary application environment settings are
limited to non-secret ASP.NET and webhook configuration. The pinned .NET 8 limited to non-secret ASP.NET, webhook, and Sentry configuration. The Sentry
DSN is public ingestion configuration, delivered as the plain `SENTRY_DSN`
setting from the required `sentry_dsn` variable. `SENTRY_ENVIRONMENT` is
`development` for the dev root and the environment name (`staging`) otherwise.
No production root exists yet; production Sentry wiring will follow the same
variable when one is added. The pinned .NET 8
AL2023 platform 3.11.3 supports Secrets Manager JSON-key extraction. AL2023 platform 3.11.3 supports Secrets Manager JSON-key extraction.
## Mandatory live secret migration ## Mandatory live secret migration

View file

@ -61,6 +61,7 @@ module "environment" {
webhook_decrypt_policy_sid = "DecryptWebhookSecretViaSecretsManager" webhook_decrypt_policy_sid = "DecryptWebhookSecretViaSecretsManager"
dynamo_reader_role_arn = "arn:aws:iam::328440206208:role/shoc-dynamo-reader" dynamo_reader_role_arn = "arn:aws:iam::328440206208:role/shoc-dynamo-reader"
dynamo_policy_sid = "AssumeDynamoReaderInMain" dynamo_policy_sid = "AssumeDynamoReaderInMain"
sentry_dsn = "https://15aef90a0e14b5b7c12ec5824b978979@o4511989453029376.ingest.de.sentry.io/4511990377152592"
github_repo = "Sea-Haven-Industries/shoc-backend" github_repo = "Sea-Haven-Industries/shoc-backend"
github_environment = "dev" github_environment = "dev"
github_deploy_role_name = "githubdeploy-shoc-backend-dev" github_deploy_role_name = "githubdeploy-shoc-backend-dev"

View file

@ -408,6 +408,16 @@ locals {
name = "WorkOrderWebhook__Region" name = "WorkOrderWebhook__Region"
value = var.aws_region value = var.aws_region
}, },
{
namespace = "aws:elasticbeanstalk:application:environment"
name = "SENTRY_DSN"
value = var.sentry_dsn
},
{
namespace = "aws:elasticbeanstalk:application:environment"
name = "SENTRY_ENVIRONMENT"
value = var.environment == "dev" ? "development" : var.environment
},
], ],
var.instance_security_group_id == null ? [] : [ var.instance_security_group_id == null ? [] : [
{ {

View file

@ -71,6 +71,11 @@ variable "shared_certificate_arn" {
description = "Existing shared certificate for dev/staging" description = "Existing shared certificate for dev/staging"
} }
variable "sentry_dsn" {
type = string
description = "Sentry DSN for error and transaction monitoring. The DSN is public ingestion configuration, not a secret: it is safe to expose and is delivered as a plain Elastic Beanstalk environment setting instead of Secrets Manager."
}
variable "runtime_role_name" { variable "runtime_role_name" {
type = string type = string
} }

View file

@ -51,6 +51,7 @@ module "environment" {
"SendGrid__ApiKey", "SendGrid__ApiKey",
] ]
webhook_secret_arn = "arn:aws:secretsmanager:us-east-1:011934824531:secret:workorder-ingest/shoc-webhook-hmac-puYTcB" webhook_secret_arn = "arn:aws:secretsmanager:us-east-1:011934824531:secret:workorder-ingest/shoc-webhook-hmac-puYTcB"
sentry_dsn = "https://15aef90a0e14b5b7c12ec5824b978979@o4511989453029376.ingest.de.sentry.io/4511990377152592"
github_repo = "Sea-Haven-Industries/shoc-backend" github_repo = "Sea-Haven-Industries/shoc-backend"
github_environment = "staging" github_environment = "staging"
github_deploy_role_name = "githubdeploy-shoc-backend-staging" github_deploy_role_name = "githubdeploy-shoc-backend-staging"