Расследование сбоя .NET-сервиса

log.LogError(ex,
    "Payment failed for order {OrderId} provider {Provider} trace {TraceId}",
    orderId, provider, Activity.Current?.TraceId);

2026-07-14T10:00:00Z deploy version=1231
2026-07-14T10:02:10Z pool_wait_ms=4200
2026-07-14T10:02:12Z checkout status=500 trace=abc

try
{
    await ImportAsync(fileId, token);
}
catch (Exception ex)
{
    log.LogError(ex.Message);
}

requests_total{route="/checkout"} 1200
request_errors_total{route="/checkout"} 96
request_duration_p99_ms{route="/checkout"} 4800

requests.Add(1,
    new KeyValuePair<string, object?>("user_id", userId));

// Пример идеи: алерт, если расход бюджета одновременно высок
// на окне 5m и подтверждается окном 1h.

dotnet-dump collect --process-id 1234 --output hang.dmp
# Снимать во время симптома до перезапуска.

dotnet-dump collect --process-id 1234 \
  --output incident-20260714.dmp

// Incident storage policy:
// encrypted bucket + least-privilege access + retention + audit trail.

// Новая версия уже записала поле status="paused".
// Старая версия знает только "new" и "done".
await deployment.RollbackAsync(previousVersion);

// Expand: add nullable new_column, deploy dual-read/write.
// Migrate: backfill.
// Contract: remove old_column в отдельном выпуске.

timeline.Add(new IncidentAction(
    AtUtc: clock.UtcNow,
    Action: "disabled new-pricing",
    Expected: "reduce checkout 500s",
    Owner: onCall));

var plan = new GameDayPlan(
    failure: Inject.HttpStatus("payments", 503),
    duration: TimeSpan.FromMinutes(10),
    successCriteria:
    [
        "alert within 2m",
        "checkout degrades without data loss",
        "on-call follows the runbook",
        "service recovers after injection stops"
    ]);

await exercise.RunAsync(plan, token);

handler.Enqueue(
    Delay(TimeSpan.FromSeconds(3)),
    Status(HttpStatusCode.ServiceUnavailable),
    Status(HttpStatusCode.OK));

await bothWorkersReachedRead.Task;
releaseWrites.SetResult();
await Task.WhenAll(workerA, workerB);
AssertInvariant(state);

// Один сбой payment provider затронул:
// checkout, refunds, reports и admin panel.
// Это большой blast radius общей зависимости.

if (!recommendationsBreaker.PermitCall())
    return page with { Recommendations = Array.Empty<Item>(), IsDegraded = true };

actions = [
    new("limit report concurrency", owner: "platform", due: date),
    new("add pool-wait SLO alert", owner: "observability", due: date),
    new("exercise rollback runbook", owner: "oncall", due: date)
];