feat: track scoped storage incidents

test/coverage.toml

@@ -321,3 +321,27 @@ id = "CRASH-STORE-04"
 layer = "integration"
 status = "implemented"
 tests = ["test/integration/store/store_integration_test.go:TestRetentionCrashStagesRollForward_CRASH_STORE_04"]
+
+[[requirements]]
+id = "HP-STORE-10"
+layer = "unit"
+status = "implemented"
+tests = ["internal/server/store/health_test.go:TestScopeHealthBestEffortReadiness_HP_STORE_10"]
+
+[[requirements]]
+id = "RACE-STORE-03"
+layer = "unit"
+status = "implemented"
+tests = ["internal/server/store/health_test.go:TestScopeHealthConcurrentUpdates_RACE_STORE_03"]
+
+[[requirements]]
+id = "HP-STORE-11"
+layer = "integration"
+status = "implemented"
+tests = ["test/integration/store/store_integration_test.go:TestIncidentDirtyResolutionIdempotencyAndRecurrence_HP_STORE_11"]
+
+[[requirements]]
+id = "BH-STORE-09"
+layer = "integration"
+status = "implemented"
+tests = ["test/integration/store/store_integration_test.go:TestIrreparableIncidentRequiresExplicitAcknowledgement_BH_STORE_09"]

test/integration/store/store_integration_test.go

@@ -779,6 +779,110 @@ func TestRetentionCrashStagesRollForward_CRASH_STORE_04(t *testing.T) {
 	}
 }
 
+func TestIncidentDirtyResolutionIdempotencyAndRecurrence_HP_STORE_11(t *testing.T) {
+	t.Parallel()
+
+	opened := openStore(t, filepath.Join(t.TempDir(), "state"))
+	now := time.Unix(5_000_000, 0).UTC()
+	firstID := uuidBytes(10)
+	input := store.IncidentInput{
+		IncidentUUID: firstID, DetectedAt: now, Kind: store.IncidentCounterMismatch,
+		Scope: store.IncidentScopeClient, ScopeKey: "client-a", ClientID: "client-a",
+		Summary: "charged totals disagree", Evidence: []byte("counter-version=1"), AutomaticallyRepairable: true,
+	}
+	created, err := opened.RecordIncident(context.Background(), input)
+	if err != nil || !created.Created || created.IncidentUUID != firstID {
+		t.Fatalf("RecordIncident = (%+v, %v)", created, err)
+	}
+	duplicateInput := input
+	duplicateInput.IncidentUUID = uuidBytes(30)
+	duplicate, err := opened.RecordIncident(context.Background(), duplicateInput)
+	if err != nil || duplicate.Created || duplicate.IncidentUUID != firstID {
+		t.Fatalf("duplicate RecordIncident = (%+v, %v)", duplicate, err)
+	}
+	dirty, err := opened.HasDirtyIncidents(context.Background())
+	if err != nil || !dirty {
+		t.Fatalf("dirty = %v, err = %v", dirty, err)
+	}
+	requestID := uuidBytes(50)
+	resolution := store.IncidentResolution{
+		RequestUUID: requestID, IncidentUUID: firstID, State: store.IncidentRepaired,
+		Note: "recomputed counters from command rows", ResolvedAt: now.Add(time.Minute),
+	}
+	resolved, err := opened.ResolveIncident(context.Background(), resolution)
+	if err != nil || resolved.State != store.IncidentRepaired {
+		t.Fatalf("ResolveIncident = (%+v, %v)", resolved, err)
+	}
+	replayed, err := opened.ResolveIncident(context.Background(), resolution)
+	if err != nil || replayed != resolved {
+		t.Fatalf("resolution replay = (%+v, %v)", replayed, err)
+	}
+	conflict := resolution
+	conflict.Note = "different repair claim"
+	if _, err := opened.ResolveIncident(context.Background(), conflict); !errors.Is(err, store.ErrMutationConflict) {
+		t.Fatalf("resolution conflict error = %v", err)
+	}
+	dirty, err = opened.HasDirtyIncidents(context.Background())
+	if err != nil || dirty {
+		t.Fatalf("resolved dirty = %v, err = %v", dirty, err)
+	}
+	var auditCount int
+	if err := opened.DB().QueryRow(`SELECT count(*) FROM audit_events WHERE action = 'repair_storage_incident'`).Scan(&auditCount); err != nil || auditCount != 1 {
+		t.Fatalf("repair audit count = %d, err = %v", auditCount, err)
+	}
+	recurrence := input
+	recurrence.IncidentUUID = uuidBytes(70)
+	recurrence.DetectedAt = now.Add(2 * time.Minute)
+	recorded, err := opened.RecordIncident(context.Background(), recurrence)
+	if err != nil || !recorded.Created || recorded.IncidentUUID != recurrence.IncidentUUID {
+		t.Fatalf("incident recurrence = (%+v, %v)", recorded, err)
+	}
+	if err := opened.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestIrreparableIncidentRequiresExplicitAcknowledgement_BH_STORE_09(t *testing.T) {
+	t.Parallel()
+
+	opened := openStore(t, filepath.Join(t.TempDir(), "state"))
+	now := time.Unix(6_000_000, 0).UTC()
+	incidentID := uuidBytes(90)
+	if _, err := opened.RecordIncident(context.Background(), store.IncidentInput{
+		IncidentUUID: incidentID, DetectedAt: now, Kind: store.IncidentMissingCommittedBytes,
+		Scope: store.IncidentScopeSegment, ScopeKey: "segment-redacted", Summary: "committed bytes are missing",
+		Evidence: []byte("offset=128"), DataLoss: true,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := opened.ResolveIncident(context.Background(), store.IncidentResolution{
+		RequestUUID: uuidBytes(110), IncidentUUID: incidentID, State: store.IncidentRepaired,
+		Note: "cannot really repair", ResolvedAt: now.Add(time.Minute),
+	}); !errors.Is(err, store.ErrIncidentResolution) {
+		t.Fatalf("unsafe repair error = %v", err)
+	}
+	if _, err := opened.ResolveIncident(context.Background(), store.IncidentResolution{
+		RequestUUID: uuidBytes(130), IncidentUUID: incidentID, State: store.IncidentAcknowledged,
+		ResolvedAt: now.Add(2 * time.Minute),
+	}); !errors.Is(err, store.ErrIncidentResolution) {
+		t.Fatalf("empty acknowledgement error = %v", err)
+	}
+	acknowledged, err := opened.ResolveIncident(context.Background(), store.IncidentResolution{
+		RequestUUID: uuidBytes(150), IncidentUUID: incidentID, State: store.IncidentAcknowledged,
+		Note: "operator accepts loss after external verification", ResolvedAt: now.Add(3 * time.Minute),
+	})
+	if err != nil || acknowledged.State != store.IncidentAcknowledged {
+		t.Fatalf("acknowledgement = (%+v, %v)", acknowledged, err)
+	}
+	dirty, err := opened.HasDirtyIncidents(context.Background())
+	if err != nil || dirty {
+		t.Fatalf("acknowledged dirty = %v, err = %v", dirty, err)
+	}
+	if err := opened.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func openStore(t *testing.T, dataDir string) *store.Store {
 	t.Helper()
 	return openStoreWithOptions(t, store.Options{DataDir: dataDir, BusyTimeout: busyTimeout})