feat: persist events across commit boundaries

2026-08-31 09:35:20 +00:00
parent 43dcd56313
commit 583b539f86
7 changed files with 722 additions and 14 deletions
@@ -237,3 +237,27 @@ id = "RACE-STORE-02"
 layer = "integration"
 status = "implemented"
 tests = ["test/integration/store/store_integration_test.go:TestSegmentConcurrentAppendSerializesRecords_RACE_STORE_02"]
+
+[[requirements]]
+id = "CRASH-STORE-01"
+layer = "integration"
+status = "implemented"
+tests = ["test/integration/store/store_integration_test.go:TestEventCrashBeforeMetadataDropsUncommittedTail_CRASH_STORE_01"]
+
+[[requirements]]
+id = "CRASH-STORE-02"
+layer = "integration"
+status = "implemented"
+tests = ["test/integration/store/store_integration_test.go:TestEventCrashAfterCommitReplaysWithoutDuplication_CRASH_STORE_02"]
+
+[[requirements]]
+id = "HP-STORE-03"
+layer = "integration"
+status = "implemented"
+tests = ["test/integration/store/store_integration_test.go:TestEventGapRejectedBeforeFileWriteAndRotation_HP_STORE_03"]
+
+[[requirements]]
+id = "CRASH-STORE-03"
+layer = "integration"
+status = "implemented"
+tests = ["test/integration/store/store_integration_test.go:TestEventCrashOnExistingSegmentTruncatesOnlyTail_CRASH_STORE_03"]
@@ -4,6 +4,7 @@ package store_test

 import (
 	"context"
+	"crypto/sha256"
 	"database/sql"
 	"errors"
 	"os"
@@ -378,11 +379,168 @@ func TestSegmentConcurrentAppendSerializesRecords_RACE_STORE_02(t *testing.T) {
 	}
 }

+func TestEventCrashBeforeMetadataDropsUncommittedTail_CRASH_STORE_01(t *testing.T) {
+	t.Parallel()
+
+	dataDir := filepath.Join(t.TempDir(), "state")
+	injected := errors.New("injected crash before metadata")
+	opened := openStoreWithOptions(t, store.Options{
+		DataDir: dataDir, BusyTimeout: busyTimeout,
+		FaultInjector: &failOnceInjector{target: store.FaultAfterSegmentSyncBeforeMetadata, failure: injected},
+	})
+	issue := uuidBytes(40)
+	seedCommand(t, opened.DB(), issue)
+	event := appendEvent(issue, 1, "not-acknowledged")
+	if result, err := opened.AppendCommandEvent(context.Background(), event); !errors.Is(err, injected) || result.ThroughEventSeq != 0 {
+		t.Fatalf("AppendCommandEvent = (%+v, %v)", result, err)
+	}
+	assertCommandEventState(t, opened.DB(), issue, 0, 0)
+	if err := opened.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	reopened := openStore(t, dataDir)
+	recovery, err := reopened.RecoverCommandSegments(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if recovery.OrphansRemoved != 1 || recovery.RecordsChecked != 0 {
+		t.Fatalf("recovery = %+v", recovery)
+	}
+	result, err := reopened.AppendCommandEvent(context.Background(), event)
+	if err != nil || result.ThroughEventSeq != 1 || result.Duplicate {
+		t.Fatalf("retry = (%+v, %v)", result, err)
+	}
+	assertCommandEventState(t, reopened.DB(), issue, 1, 1)
+	if err := reopened.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestEventCrashAfterCommitReplaysWithoutDuplication_CRASH_STORE_02(t *testing.T) {
+	t.Parallel()
+
+	dataDir := filepath.Join(t.TempDir(), "state")
+	injected := errors.New("injected lost acknowledgement")
+	opened := openStoreWithOptions(t, store.Options{
+		DataDir: dataDir, BusyTimeout: busyTimeout,
+		FaultInjector: &failOnceInjector{target: store.FaultAfterMetadataCommit, failure: injected},
+	})
+	issue := uuidBytes(60)
+	seedCommand(t, opened.DB(), issue)
+	event := appendEvent(issue, 1, "committed")
+	if result, err := opened.AppendCommandEvent(context.Background(), event); !errors.Is(err, injected) || result.ThroughEventSeq != 1 {
+		t.Fatalf("AppendCommandEvent = (%+v, %v)", result, err)
+	}
+	assertCommandEventState(t, opened.DB(), issue, 1, 1)
+	if err := opened.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	reopened := openStore(t, dataDir)
+	recovery, err := reopened.RecoverCommandSegments(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if recovery.SegmentsChecked != 1 || recovery.RecordsChecked != 1 || recovery.OrphansRemoved != 0 {
+		t.Fatalf("recovery = %+v", recovery)
+	}
+	result, err := reopened.AppendCommandEvent(context.Background(), event)
+	if err != nil || !result.Duplicate || result.ThroughEventSeq != 1 {
+		t.Fatalf("duplicate retry = (%+v, %v)", result, err)
+	}
+	conflict := event
+	conflict.ImmutableSHA256 = sha256.Sum256([]byte("different immutable event"))
+	if _, err := reopened.AppendCommandEvent(context.Background(), conflict); !errors.Is(err, store.ErrEventConflict) {
+		t.Fatalf("conflicting duplicate error = %v", err)
+	}
+	assertCommandEventState(t, reopened.DB(), issue, 1, 1)
+	if err := reopened.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestEventCrashOnExistingSegmentTruncatesOnlyTail_CRASH_STORE_03(t *testing.T) {
+	t.Parallel()
+
+	dataDir := filepath.Join(t.TempDir(), "state")
+	injected := errors.New("injected second append crash")
+	opened := openStoreWithOptions(t, store.Options{
+		DataDir: dataDir, BusyTimeout: busyTimeout,
+		FaultInjector: &failOnceInjector{target: store.FaultAfterSegmentSyncBeforeMetadata, failure: injected, skip: 1},
+	})
+	issue := uuidBytes(70)
+	seedCommand(t, opened.DB(), issue)
+	if _, err := opened.AppendCommandEvent(context.Background(), appendEvent(issue, 1, "committed-first")); err != nil {
+		t.Fatal(err)
+	}
+	second := appendEvent(issue, 2, "uncommitted-second")
+	if _, err := opened.AppendCommandEvent(context.Background(), second); !errors.Is(err, injected) {
+		t.Fatalf("second append error = %v", err)
+	}
+	assertCommandEventState(t, opened.DB(), issue, 1, 1)
+	if err := opened.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	reopened := openStore(t, dataDir)
+	recovery, err := reopened.RecoverCommandSegments(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if recovery.SegmentsChecked != 1 || recovery.RecordsChecked != 1 || recovery.TailBytesTruncated == 0 || recovery.OrphansRemoved != 0 {
+		t.Fatalf("recovery = %+v", recovery)
+	}
+	if result, err := reopened.AppendCommandEvent(context.Background(), second); err != nil || result.ThroughEventSeq != 2 {
+		t.Fatalf("second retry = (%+v, %v)", result, err)
+	}
+	assertCommandEventState(t, reopened.DB(), issue, 2, 2)
+	if err := reopened.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestEventGapRejectedBeforeFileWriteAndRotation_HP_STORE_03(t *testing.T) {
+	t.Parallel()
+
+	dataDir := filepath.Join(t.TempDir(), "state")
+	opened := openStoreWithOptions(t, store.Options{DataDir: dataDir, BusyTimeout: busyTimeout, SegmentTargetSize: 200})
+	issue := uuidBytes(80)
+	seedCommand(t, opened.DB(), issue)
+	if _, err := opened.AppendCommandEvent(context.Background(), appendEvent(issue, 2, "gap")); !errors.Is(err, store.ErrEventSequenceGap) {
+		t.Fatalf("gap error = %v", err)
+	}
+	entries, err := os.ReadDir(filepath.Join(dataDir, "segments"))
+	if err != nil || len(entries) != 0 {
+		t.Fatalf("segments after gap = %v, err = %v", entries, err)
+	}
+	for sequence, payload := range []string{"one", "two"} {
+		if _, err := opened.AppendCommandEvent(context.Background(), appendEvent(issue, uint64(sequence+1), payload)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	var segmentCount int
+	if err := opened.DB().QueryRow(`SELECT count(*) FROM output_segments WHERE issue_uuid = ?`, issue[:]).Scan(&segmentCount); err != nil || segmentCount != 2 {
+		t.Fatalf("segment count = %d, err = %v", segmentCount, err)
+	}
+	if err := opened.QuickCheck(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if err := opened.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func openStore(t *testing.T, dataDir string) *store.Store {
 	t.Helper()
-	opened, err := store.Open(context.Background(), store.Options{DataDir: dataDir, BusyTimeout: busyTimeout})
+	return openStoreWithOptions(t, store.Options{DataDir: dataDir, BusyTimeout: busyTimeout})
+}
+
+func openStoreWithOptions(t *testing.T, options store.Options) *store.Store {
+	t.Helper()
+	opened, err := store.Open(context.Background(), options)
 	if err != nil {
-		t.Fatalf("Open(%s): %v", dataDir, err)
+		t.Fatalf("Open(%s): %v", options.DataDir, err)
 	}
 	return opened
 }
@@ -420,3 +578,65 @@ func integrationRecord(owner [16]byte, sequence uint64, payload string) store.Se
 		Kind: store.PayloadKindCommandEvent, Stream: 1, Compression: 1, RawLength: uint64(len(payload)), Payload: []byte(payload),
 	}
 }
+
+type failOnceInjector struct {
+	mu      sync.Mutex
+	target  string
+	failure error
+	skip    int
+	fired   bool
+}
+
+func (injector *failOnceInjector) Checkpoint(name string) error {
+	injector.mu.Lock()
+	defer injector.mu.Unlock()
+	if name == injector.target && !injector.fired {
+		if injector.skip > 0 {
+			injector.skip--
+			return nil
+		}
+		injector.fired = true
+		return injector.failure
+	}
+	return nil
+}
+
+func seedCommand(t *testing.T, database *sql.DB, issue [16]byte) {
+	t.Helper()
+	clientInstance := uuidBytes(100)
+	if _, err := database.Exec(`INSERT OR IGNORE INTO clients (
+client_id, platform, architecture, daemon_version, daemon_cwd, supported_shells, capabilities, client_instance_id
+) VALUES ('client-a', 3, 'amd64', 'test', 'C:\\work', x'01', x'', ?)`, clientInstance[:]); err != nil {
+		t.Fatal(err)
+	}
+	requestHash := sha256.Sum256([]byte("request"))
+	if _, err := database.Exec(`INSERT INTO commands (
+issue_uuid, client_id, issue_time, server_receipt_time, lifecycle, revision, immutable_request_sha256,
+execution_spec, execution_spec_raw_bytes, execution_spec_stored_bytes, execution_spec_compression
+) VALUES (?, 'client-a', 1, 1, 1, 1, ?, x'01', 1, 1, 1)`, issue[:], requestHash[:]); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func appendEvent(issue [16]byte, sequence uint64, payload string) store.EventAppend {
+	immutableHash := sha256.Sum256(append([]byte{byte(sequence)}, []byte(payload)...))
+	return store.EventAppend{
+		IssueUUID: issue, EventSeq: sequence, ObservedUnixNano: int64(sequence), ReceiptUnixNano: int64(sequence + 10),
+		EventType: 1, Stream: 1, Compression: 1, RawLength: uint64(len(payload)), Payload: []byte(payload),
+		ImmutableSHA256: immutableHash,
+	}
+}
+
+func assertCommandEventState(t *testing.T, database *sql.DB, issue [16]byte, wantLast, wantEvents int) {
+	t.Helper()
+	var last, events int
+	if err := database.QueryRow(`SELECT last_event_seq FROM commands WHERE issue_uuid = ?`, issue[:]).Scan(&last); err != nil {
+		t.Fatal(err)
+	}
+	if err := database.QueryRow(`SELECT count(*) FROM command_events WHERE issue_uuid = ?`, issue[:]).Scan(&events); err != nil {
+		t.Fatal(err)
+	}
+	if last != wantLast || events != wantEvents {
+		t.Fatalf("event state = (last=%d, rows=%d), want (%d, %d)", last, events, wantLast, wantEvents)
+	}
+}