feat: track scoped storage incidents

internal/server/store/health_test.go

@@ -0,0 +1,52 @@
+package store
+
+import (
+	"sync"
+	"testing"
+)
+
+func TestScopeHealthBestEffortReadiness_HP_STORE_10(t *testing.T) {
+	t.Parallel()
+
+	global := ScopeKey{Kind: "global", ID: "store"}
+	client := ScopeKey{Kind: "client", ID: "client-a"}
+	health := NewScopeHealth([]ScopeKey{global, client})
+	if health.GloballyReady() || health.ReadAllowed(global) || health.MutationAllowed(global) {
+		t.Fatal("recovering scope reported ready")
+	}
+	if !health.Set(global, ScopeReady) || !health.Set(client, ScopeDirtyReadable) {
+		t.Fatal("valid state update failed")
+	}
+	if health.GloballyReady() || !health.ReadAllowed(client) || health.MutationAllowed(client) {
+		t.Fatal("dirty-readable semantics are wrong")
+	}
+	if !health.Set(client, ScopeReady) || !health.GloballyReady() {
+		t.Fatal("resolved required scopes did not become globally ready")
+	}
+	if health.Set(ScopeKey{}, ScopeReady) || health.Set(global, 0) {
+		t.Fatal("invalid scope state accepted")
+	}
+}
+
+func TestScopeHealthConcurrentUpdates_RACE_STORE_03(t *testing.T) {
+	t.Parallel()
+
+	scope := ScopeKey{Kind: "client", ID: "client-a"}
+	health := NewScopeHealth([]ScopeKey{scope})
+	var wait sync.WaitGroup
+	for index := range 32 {
+		wait.Add(1)
+		go func(index int) {
+			defer wait.Done()
+			states := []ScopeState{ScopeRecovering, ScopeReady, ScopeDirtyReadable, ScopeUnavailable}
+			health.Set(scope, states[index%len(states)])
+			_ = health.ReadAllowed(scope)
+			_ = health.MutationAllowed(scope)
+			_ = health.GloballyReady()
+		}(index)
+	}
+	wait.Wait()
+	if state := health.State(scope); state < ScopeRecovering || state > ScopeUnavailable {
+		t.Fatalf("final state = %d", state)
+	}
+}