feat: enforce tiered storage reservations

2026-08-31 09:48:04 +00:00
parent 583b539f86
commit 611100f67c
12 changed files with 780 additions and 6 deletions
@@ -66,7 +66,7 @@ func TestRealSQLiteInitializationAndRestart_HP_STORE_01(t *testing.T) {
 	if err := rows.Close(); err != nil {
 		t.Fatal(err)
 	}
-	want := []string{"audit_events", "clients", "command_events", "command_payloads", "command_tombstones", "commands", "control_mutations", "output_segments", "output_truncations", "schema_migrations", "sessions", "stdin_writes", "storage_incidents", "takeover_authorizations"}
+	want := []string{"audit_events", "clients", "command_events", "command_payloads", "command_tombstones", "commands", "control_mutations", "output_segments", "output_truncations", "schema_migrations", "sessions", "stdin_writes", "storage_counters", "storage_incidents", "takeover_authorizations"}
 	sort.Strings(want)
 	if strings.Join(names, ",") != strings.Join(want, ",") {
 		t.Fatalf("tables = %v, want %v", names, want)
@@ -531,6 +531,112 @@ func TestEventGapRejectedBeforeFileWriteAndRotation_HP_STORE_03(t *testing.T) {
 	}
 }

+func TestEventQuotaCountersAtomicAndCapacityRejectsBeforeWrite_HP_STORE_07(t *testing.T) {
+	t.Parallel()
+
+	dataDir := filepath.Join(t.TempDir(), "state")
+	limits := store.QuotaLimits{
+		HardAllocationBytes: 1000, CommandOutputBytes: 1000, CommandTotalBytes: 2000,
+		ClientTotalBytes: 3000, ServerTotalBytes: 4000, CloseoutReserveBytes: 200, FilesystemFloorBytes: 50,
+	}
+	opened := openStoreWithOptions(t, store.Options{
+		DataDir: dataDir, BusyTimeout: busyTimeout, QuotaLimits: limits,
+		FreeSpaceProbe: store.FreeSpaceProbeFunc(func(string) (uint64, error) { return 10_000, nil }),
+	})
+	issue := uuidBytes(90)
+	seedCommand(t, opened.DB(), issue)
+	if _, err := opened.DB().Exec(`UPDATE commands SET closeout_remaining_bytes = ? WHERE issue_uuid = ?`, limits.CloseoutReserveBytes, issue[:]); err != nil {
+		t.Fatal(err)
+	}
+	first := appendEvent(issue, 1, "first")
+	if _, err := opened.AppendCommandEvent(context.Background(), first); err != nil {
+		t.Fatal(err)
+	}
+	var commandOutput, commandTotal, closeout, clientTotal, serverTotal uint64
+	if err := opened.DB().QueryRow(`SELECT output_charged_bytes, charged_bytes, closeout_remaining_bytes FROM commands WHERE issue_uuid = ?`, issue[:]).Scan(&commandOutput, &commandTotal, &closeout); err != nil {
+		t.Fatal(err)
+	}
+	if err := opened.DB().QueryRow(`SELECT charged_bytes FROM clients WHERE client_id = 'client-a'`).Scan(&clientTotal); err != nil {
+		t.Fatal(err)
+	}
+	if err := opened.DB().QueryRow(`SELECT command_charged_bytes FROM storage_counters WHERE singleton = 1`).Scan(&serverTotal); err != nil {
+		t.Fatal(err)
+	}
+	if commandOutput == 0 || commandOutput != commandTotal || commandTotal != clientTotal || clientTotal != serverTotal || closeout != limits.CloseoutReserveBytes {
+		t.Fatalf("quota counters = output:%d command:%d client:%d server:%d closeout:%d", commandOutput, commandTotal, clientTotal, serverTotal, closeout)
+	}
+	entries, err := os.ReadDir(filepath.Join(dataDir, "segments"))
+	if err != nil || len(entries) != 1 {
+		t.Fatalf("segment entries = %v, err = %v", entries, err)
+	}
+	segmentPath := filepath.Join(dataDir, "segments", entries[0].Name())
+	before, err := os.Stat(segmentPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result, err := opened.AppendCommandEvent(context.Background(), first); err != nil || !result.Duplicate {
+		t.Fatalf("duplicate = (%+v, %v)", result, err)
+	}
+	second := appendEvent(issue, 2, "second")
+	if _, err := opened.AppendCommandEvent(context.Background(), second); err == nil {
+		t.Fatal("over-quota event was accepted")
+	} else {
+		var capacity *store.CapacityError
+		if !errors.As(err, &capacity) || capacity.Tier != store.CapacityTierCommandOut {
+			t.Fatalf("capacity error = %v", err)
+		}
+	}
+	after, err := os.Stat(segmentPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if after.Size() != before.Size() {
+		t.Fatalf("rejected event changed segment size from %d to %d", before.Size(), after.Size())
+	}
+	assertCommandEventState(t, opened.DB(), issue, 1, 1)
+	if err := opened.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestFilesystemFloorAndCounterMismatch_BH_STORE_08(t *testing.T) {
+	t.Parallel()
+
+	dataDir := filepath.Join(t.TempDir(), "state")
+	opened := openStoreWithOptions(t, store.Options{
+		DataDir: dataDir, BusyTimeout: busyTimeout,
+		FreeSpaceProbe: store.FreeSpaceProbeFunc(func(string) (uint64, error) { return 0, nil }),
+	})
+	issue := uuidBytes(110)
+	seedCommand(t, opened.DB(), issue)
+	if _, err := opened.AppendCommandEvent(context.Background(), appendEvent(issue, 1, "floor")); err == nil {
+		t.Fatal("filesystem-floor event was accepted")
+	} else {
+		var capacity *store.CapacityError
+		if !errors.As(err, &capacity) || capacity.Tier != store.CapacityTierFilesystem {
+			t.Fatalf("floor error = %v", err)
+		}
+	}
+	entries, err := os.ReadDir(filepath.Join(dataDir, "segments"))
+	if err != nil || len(entries) != 0 {
+		t.Fatalf("segments after floor rejection = %v, err = %v", entries, err)
+	}
+	if err := opened.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	reopened := openStore(t, dataDir)
+	if _, err := reopened.DB().Exec(`UPDATE clients SET charged_bytes = 1 WHERE client_id = 'client-a'`); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := reopened.RecoverCommandSegments(context.Background()); !errors.Is(err, store.ErrQuotaCounterMismatch) {
+		t.Fatalf("counter mismatch recovery error = %v", err)
+	}
+	if err := reopened.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func openStore(t *testing.T, dataDir string) *store.Store {
 	t.Helper()
 	return openStoreWithOptions(t, store.Options{DataDir: dataDir, BusyTimeout: busyTimeout})