feat: persist events across commit boundaries
This commit is contained in:
@@ -4,6 +4,7 @@ package store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"os"
|
||||
@@ -378,11 +379,168 @@ func TestSegmentConcurrentAppendSerializesRecords_RACE_STORE_02(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventCrashBeforeMetadataDropsUncommittedTail_CRASH_STORE_01(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dataDir := filepath.Join(t.TempDir(), "state")
|
||||
injected := errors.New("injected crash before metadata")
|
||||
opened := openStoreWithOptions(t, store.Options{
|
||||
DataDir: dataDir, BusyTimeout: busyTimeout,
|
||||
FaultInjector: &failOnceInjector{target: store.FaultAfterSegmentSyncBeforeMetadata, failure: injected},
|
||||
})
|
||||
issue := uuidBytes(40)
|
||||
seedCommand(t, opened.DB(), issue)
|
||||
event := appendEvent(issue, 1, "not-acknowledged")
|
||||
if result, err := opened.AppendCommandEvent(context.Background(), event); !errors.Is(err, injected) || result.ThroughEventSeq != 0 {
|
||||
t.Fatalf("AppendCommandEvent = (%+v, %v)", result, err)
|
||||
}
|
||||
assertCommandEventState(t, opened.DB(), issue, 0, 0)
|
||||
if err := opened.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
reopened := openStore(t, dataDir)
|
||||
recovery, err := reopened.RecoverCommandSegments(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if recovery.OrphansRemoved != 1 || recovery.RecordsChecked != 0 {
|
||||
t.Fatalf("recovery = %+v", recovery)
|
||||
}
|
||||
result, err := reopened.AppendCommandEvent(context.Background(), event)
|
||||
if err != nil || result.ThroughEventSeq != 1 || result.Duplicate {
|
||||
t.Fatalf("retry = (%+v, %v)", result, err)
|
||||
}
|
||||
assertCommandEventState(t, reopened.DB(), issue, 1, 1)
|
||||
if err := reopened.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventCrashAfterCommitReplaysWithoutDuplication_CRASH_STORE_02(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dataDir := filepath.Join(t.TempDir(), "state")
|
||||
injected := errors.New("injected lost acknowledgement")
|
||||
opened := openStoreWithOptions(t, store.Options{
|
||||
DataDir: dataDir, BusyTimeout: busyTimeout,
|
||||
FaultInjector: &failOnceInjector{target: store.FaultAfterMetadataCommit, failure: injected},
|
||||
})
|
||||
issue := uuidBytes(60)
|
||||
seedCommand(t, opened.DB(), issue)
|
||||
event := appendEvent(issue, 1, "committed")
|
||||
if result, err := opened.AppendCommandEvent(context.Background(), event); !errors.Is(err, injected) || result.ThroughEventSeq != 1 {
|
||||
t.Fatalf("AppendCommandEvent = (%+v, %v)", result, err)
|
||||
}
|
||||
assertCommandEventState(t, opened.DB(), issue, 1, 1)
|
||||
if err := opened.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
reopened := openStore(t, dataDir)
|
||||
recovery, err := reopened.RecoverCommandSegments(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if recovery.SegmentsChecked != 1 || recovery.RecordsChecked != 1 || recovery.OrphansRemoved != 0 {
|
||||
t.Fatalf("recovery = %+v", recovery)
|
||||
}
|
||||
result, err := reopened.AppendCommandEvent(context.Background(), event)
|
||||
if err != nil || !result.Duplicate || result.ThroughEventSeq != 1 {
|
||||
t.Fatalf("duplicate retry = (%+v, %v)", result, err)
|
||||
}
|
||||
conflict := event
|
||||
conflict.ImmutableSHA256 = sha256.Sum256([]byte("different immutable event"))
|
||||
if _, err := reopened.AppendCommandEvent(context.Background(), conflict); !errors.Is(err, store.ErrEventConflict) {
|
||||
t.Fatalf("conflicting duplicate error = %v", err)
|
||||
}
|
||||
assertCommandEventState(t, reopened.DB(), issue, 1, 1)
|
||||
if err := reopened.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventCrashOnExistingSegmentTruncatesOnlyTail_CRASH_STORE_03(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dataDir := filepath.Join(t.TempDir(), "state")
|
||||
injected := errors.New("injected second append crash")
|
||||
opened := openStoreWithOptions(t, store.Options{
|
||||
DataDir: dataDir, BusyTimeout: busyTimeout,
|
||||
FaultInjector: &failOnceInjector{target: store.FaultAfterSegmentSyncBeforeMetadata, failure: injected, skip: 1},
|
||||
})
|
||||
issue := uuidBytes(70)
|
||||
seedCommand(t, opened.DB(), issue)
|
||||
if _, err := opened.AppendCommandEvent(context.Background(), appendEvent(issue, 1, "committed-first")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second := appendEvent(issue, 2, "uncommitted-second")
|
||||
if _, err := opened.AppendCommandEvent(context.Background(), second); !errors.Is(err, injected) {
|
||||
t.Fatalf("second append error = %v", err)
|
||||
}
|
||||
assertCommandEventState(t, opened.DB(), issue, 1, 1)
|
||||
if err := opened.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
reopened := openStore(t, dataDir)
|
||||
recovery, err := reopened.RecoverCommandSegments(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if recovery.SegmentsChecked != 1 || recovery.RecordsChecked != 1 || recovery.TailBytesTruncated == 0 || recovery.OrphansRemoved != 0 {
|
||||
t.Fatalf("recovery = %+v", recovery)
|
||||
}
|
||||
if result, err := reopened.AppendCommandEvent(context.Background(), second); err != nil || result.ThroughEventSeq != 2 {
|
||||
t.Fatalf("second retry = (%+v, %v)", result, err)
|
||||
}
|
||||
assertCommandEventState(t, reopened.DB(), issue, 2, 2)
|
||||
if err := reopened.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventGapRejectedBeforeFileWriteAndRotation_HP_STORE_03(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dataDir := filepath.Join(t.TempDir(), "state")
|
||||
opened := openStoreWithOptions(t, store.Options{DataDir: dataDir, BusyTimeout: busyTimeout, SegmentTargetSize: 200})
|
||||
issue := uuidBytes(80)
|
||||
seedCommand(t, opened.DB(), issue)
|
||||
if _, err := opened.AppendCommandEvent(context.Background(), appendEvent(issue, 2, "gap")); !errors.Is(err, store.ErrEventSequenceGap) {
|
||||
t.Fatalf("gap error = %v", err)
|
||||
}
|
||||
entries, err := os.ReadDir(filepath.Join(dataDir, "segments"))
|
||||
if err != nil || len(entries) != 0 {
|
||||
t.Fatalf("segments after gap = %v, err = %v", entries, err)
|
||||
}
|
||||
for sequence, payload := range []string{"one", "two"} {
|
||||
if _, err := opened.AppendCommandEvent(context.Background(), appendEvent(issue, uint64(sequence+1), payload)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var segmentCount int
|
||||
if err := opened.DB().QueryRow(`SELECT count(*) FROM output_segments WHERE issue_uuid = ?`, issue[:]).Scan(&segmentCount); err != nil || segmentCount != 2 {
|
||||
t.Fatalf("segment count = %d, err = %v", segmentCount, err)
|
||||
}
|
||||
if err := opened.QuickCheck(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := opened.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func openStore(t *testing.T, dataDir string) *store.Store {
|
||||
t.Helper()
|
||||
opened, err := store.Open(context.Background(), store.Options{DataDir: dataDir, BusyTimeout: busyTimeout})
|
||||
return openStoreWithOptions(t, store.Options{DataDir: dataDir, BusyTimeout: busyTimeout})
|
||||
}
|
||||
|
||||
func openStoreWithOptions(t *testing.T, options store.Options) *store.Store {
|
||||
t.Helper()
|
||||
opened, err := store.Open(context.Background(), options)
|
||||
if err != nil {
|
||||
t.Fatalf("Open(%s): %v", dataDir, err)
|
||||
t.Fatalf("Open(%s): %v", options.DataDir, err)
|
||||
}
|
||||
return opened
|
||||
}
|
||||
@@ -420,3 +578,65 @@ func integrationRecord(owner [16]byte, sequence uint64, payload string) store.Se
|
||||
Kind: store.PayloadKindCommandEvent, Stream: 1, Compression: 1, RawLength: uint64(len(payload)), Payload: []byte(payload),
|
||||
}
|
||||
}
|
||||
|
||||
type failOnceInjector struct {
|
||||
mu sync.Mutex
|
||||
target string
|
||||
failure error
|
||||
skip int
|
||||
fired bool
|
||||
}
|
||||
|
||||
func (injector *failOnceInjector) Checkpoint(name string) error {
|
||||
injector.mu.Lock()
|
||||
defer injector.mu.Unlock()
|
||||
if name == injector.target && !injector.fired {
|
||||
if injector.skip > 0 {
|
||||
injector.skip--
|
||||
return nil
|
||||
}
|
||||
injector.fired = true
|
||||
return injector.failure
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func seedCommand(t *testing.T, database *sql.DB, issue [16]byte) {
|
||||
t.Helper()
|
||||
clientInstance := uuidBytes(100)
|
||||
if _, err := database.Exec(`INSERT OR IGNORE INTO clients (
|
||||
client_id, platform, architecture, daemon_version, daemon_cwd, supported_shells, capabilities, client_instance_id
|
||||
) VALUES ('client-a', 3, 'amd64', 'test', 'C:\\work', x'01', x'', ?)`, clientInstance[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requestHash := sha256.Sum256([]byte("request"))
|
||||
if _, err := database.Exec(`INSERT INTO commands (
|
||||
issue_uuid, client_id, issue_time, server_receipt_time, lifecycle, revision, immutable_request_sha256,
|
||||
execution_spec, execution_spec_raw_bytes, execution_spec_stored_bytes, execution_spec_compression
|
||||
) VALUES (?, 'client-a', 1, 1, 1, 1, ?, x'01', 1, 1, 1)`, issue[:], requestHash[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func appendEvent(issue [16]byte, sequence uint64, payload string) store.EventAppend {
|
||||
immutableHash := sha256.Sum256(append([]byte{byte(sequence)}, []byte(payload)...))
|
||||
return store.EventAppend{
|
||||
IssueUUID: issue, EventSeq: sequence, ObservedUnixNano: int64(sequence), ReceiptUnixNano: int64(sequence + 10),
|
||||
EventType: 1, Stream: 1, Compression: 1, RawLength: uint64(len(payload)), Payload: []byte(payload),
|
||||
ImmutableSHA256: immutableHash,
|
||||
}
|
||||
}
|
||||
|
||||
func assertCommandEventState(t *testing.T, database *sql.DB, issue [16]byte, wantLast, wantEvents int) {
|
||||
t.Helper()
|
||||
var last, events int
|
||||
if err := database.QueryRow(`SELECT last_event_seq FROM commands WHERE issue_uuid = ?`, issue[:]).Scan(&last); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := database.QueryRow(`SELECT count(*) FROM command_events WHERE issue_uuid = ?`, issue[:]).Scan(&events); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if last != wantLast || events != wantEvents {
|
||||
t.Fatalf("event state = (last=%d, rows=%d), want (%d, %d)", last, events, wantLast, wantEvents)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user