//go:build integration package store_test import ( "context" "crypto/sha256" "database/sql" "errors" "os" "path/filepath" "runtime" "sort" "strings" "sync" "testing" "time" "github.com/rvbox/rvbox/internal/server/store" ) const busyTimeout = 2 * time.Second func TestRealSQLiteInitializationAndRestart_HP_STORE_01(t *testing.T) { t.Parallel() dataDir := filepath.Join(t.TempDir(), "state") opened := openStore(t, dataDir) for _, directory := range []string{dataDir, filepath.Join(dataDir, "segments"), filepath.Join(dataDir, "audit")} { info, err := os.Stat(directory) if err != nil { t.Fatal(err) } if runtime.GOOS != "windows" && info.Mode().Perm() != 0o700 { t.Fatalf("%s mode = %04o, want 0700", directory, info.Mode().Perm()) } } for _, file := range []string{"rvbox.db", "server.lock"} { info, err := os.Stat(filepath.Join(dataDir, file)) if err != nil { t.Fatal(err) } if runtime.GOOS != "windows" && info.Mode().Perm() != 0o600 { t.Fatalf("%s mode = %04o, want 0600", file, info.Mode().Perm()) } } assertPragma(t, opened.DB(), "journal_mode", "wal") assertPragma(t, opened.DB(), "foreign_keys", "1") assertPragma(t, opened.DB(), "synchronous", "2") assertPragma(t, opened.DB(), "busy_timeout", "2000") rows, err := opened.DB().Query(`SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name`) if err != nil { t.Fatal(err) } var names []string for rows.Next() { var name string if err := rows.Scan(&name); err != nil { t.Fatal(err) } names = append(names, name) } if err := rows.Close(); err != nil { t.Fatal(err) } want := []string{"audit_events", "clients", "command_events", "command_payloads", "command_tombstones", "commands", "control_mutations", "output_segments", "output_truncations", "schema_migrations", "sessions", "stdin_writes", "storage_counters", "storage_incidents", "takeover_authorizations"} sort.Strings(want) if strings.Join(names, ",") != strings.Join(want, ",") { t.Fatalf("tables = %v, want %v", names, want) } var migrationCount int if err := opened.DB().QueryRow(`SELECT count(*) FROM schema_migrations`).Scan(&migrationCount); err != nil || migrationCount != 1 { t.Fatalf("migration count = %d, err = %v", migrationCount, err) } if err := opened.Close(); err != nil { t.Fatal(err) } reopened := openStore(t, dataDir) if err := reopened.DB().QueryRow(`SELECT count(*) FROM schema_migrations`).Scan(&migrationCount); err != nil || migrationCount != 1 { t.Fatalf("reopened migration count = %d, err = %v", migrationCount, err) } if err := reopened.Close(); err != nil { t.Fatal(err) } } func TestSingleInstanceLockAndConcurrentClose_RACE_STORE_01(t *testing.T) { t.Parallel() dataDir := filepath.Join(t.TempDir(), "state") first := openStore(t, dataDir) if _, err := store.Open(context.Background(), store.Options{DataDir: dataDir, BusyTimeout: busyTimeout}); !errors.Is(err, store.ErrAlreadyOpen) { t.Fatalf("second Open error = %v, want ErrAlreadyOpen", err) } var wait sync.WaitGroup for range 8 { wait.Add(1) go func() { defer wait.Done() if err := first.Close(); err != nil && !errors.Is(err, sql.ErrConnDone) { t.Errorf("Close: %v", err) } }() } wait.Wait() reopened := openStore(t, dataDir) if err := reopened.Close(); err != nil { t.Fatal(err) } } func TestSchemaRejectsInvalidIDsAndForeignKeys_BH_STORE_01(t *testing.T) { t.Parallel() opened := openStore(t, filepath.Join(t.TempDir(), "state")) defer opened.Close() _, err := opened.DB().Exec(`INSERT INTO clients ( client_id, platform, architecture, daemon_version, daemon_cwd, supported_shells, capabilities, client_instance_id ) VALUES ('client-a', 3, 'amd64', 'test', 'C:\\work', x'', x'', x'01')`) if err == nil { t.Fatal("invalid client instance UUID was accepted") } _, err = opened.DB().Exec(`INSERT INTO sessions ( session_id, client_id, client_instance_id, generation, opened_at ) VALUES (?, 'missing-client', ?, 1, 1)`, bytesOf(16, 1), bytesOf(16, 2)) if err == nil || !strings.Contains(strings.ToLower(err.Error()), "foreign key") { t.Fatalf("foreign-key insert error = %v", err) } } func TestUnsafePathsAndCancelledOpen_BH_STORE_02(t *testing.T) { t.Parallel() if _, err := store.Open(context.Background(), store.Options{DataDir: "relative", BusyTimeout: busyTimeout}); !errors.Is(err, store.ErrUnsafeDataDirectory) { t.Fatalf("relative path error = %v", err) } if _, err := store.Open(context.Background(), store.Options{DataDir: string(filepath.Separator), BusyTimeout: busyTimeout}); !errors.Is(err, store.ErrUnsafeDataDirectory) { t.Fatalf("root path error = %v", err) } permissive := filepath.Join(t.TempDir(), "permissive") if err := os.Mkdir(permissive, 0o755); err != nil { t.Fatal(err) } if _, err := store.Open(context.Background(), store.Options{DataDir: permissive, BusyTimeout: busyTimeout}); !errors.Is(err, store.ErrUnsafeDataDirectory) { t.Fatalf("permissive path error = %v", err) } if runtime.GOOS != "windows" { target := filepath.Join(t.TempDir(), "target") if err := os.Mkdir(target, 0o700); err != nil { t.Fatal(err) } link := filepath.Join(t.TempDir(), "linked") if err := os.Symlink(target, link); err != nil { t.Fatal(err) } if _, err := store.Open(context.Background(), store.Options{DataDir: link, BusyTimeout: busyTimeout}); !errors.Is(err, store.ErrUnsafeDataDirectory) { t.Fatalf("symlink path error = %v", err) } } cancelled, cancel := context.WithCancel(context.Background()) cancel() dataDir := filepath.Join(t.TempDir(), "cancelled") if _, err := store.Open(cancelled, store.Options{DataDir: dataDir, BusyTimeout: busyTimeout}); !errors.Is(err, context.Canceled) { t.Fatalf("cancelled Open error = %v", err) } opened := openStore(t, dataDir) if err := opened.Close(); err != nil { t.Fatal(err) } } func TestMigrationChecksumMismatchPreventsOpen_REC_STORE_01(t *testing.T) { t.Parallel() dataDir := filepath.Join(t.TempDir(), "state") opened := openStore(t, dataDir) if err := opened.Close(); err != nil { t.Fatal(err) } raw, err := sql.Open("sqlite", filepath.Join(dataDir, "rvbox.db")) if err != nil { t.Fatal(err) } if _, err := raw.Exec(`UPDATE schema_migrations SET checksum = 'tampered' WHERE version = 1`); err != nil { t.Fatal(err) } if err := raw.Close(); err != nil { t.Fatal(err) } if _, err := store.Open(context.Background(), store.Options{DataDir: dataDir, BusyTimeout: busyTimeout}); err == nil || !strings.Contains(err.Error(), "checksum mismatch") { t.Fatalf("Open error = %v, want checksum mismatch", err) } // A failed open must release the process lock for offline inspection/repair. raw, err = sql.Open("sqlite", filepath.Join(dataDir, "rvbox.db")) if err != nil { t.Fatal(err) } if err := raw.Ping(); err != nil { t.Fatal(err) } if err := raw.Close(); err != nil { t.Fatal(err) } } func TestSegmentAppendTailRecoveryAndCommittedCorruption_REC_STORE_02(t *testing.T) { t.Parallel() directory := filepath.Join(t.TempDir(), "segments") if err := os.Mkdir(directory, 0o700); err != nil { t.Fatal(err) } owner := uuidBytes(10) segment, err := store.CreateSegment(directory, owner, 1, store.DefaultSegmentLimits()) if err != nil { t.Fatal(err) } name := segment.Name() first := integrationRecord(owner, 1, "first") second := integrationRecord(owner, 2, "second") if _, err := segment.Append(first); err != nil { t.Fatal(err) } committed, err := segment.Append(second) if err != nil { t.Fatal(err) } if err := segment.Close(); err != nil { t.Fatal(err) } path := filepath.Join(directory, name) file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0) if err != nil { t.Fatal(err) } if _, err := file.Write([]byte("unacknowledged-tail")); err != nil { t.Fatal(err) } if err := file.Sync(); err != nil { t.Fatal(err) } if err := file.Close(); err != nil { t.Fatal(err) } recovery, err := store.RecoverSegment(directory, name, committed, store.DefaultSegmentLimits()) if err != nil { t.Fatal(err) } if recovery.TruncatedTail != uint64(len("unacknowledged-tail")) || len(recovery.Records) != 2 { t.Fatalf("recovery = %+v", recovery) } info, err := os.Stat(path) if err != nil { t.Fatal(err) } if uint64(info.Size()) != committed { t.Fatalf("size = %d, want %d", info.Size(), committed) } corrupt, err := os.OpenFile(path, os.O_RDWR, 0) if err != nil { t.Fatal(err) } if _, err := corrupt.WriteAt([]byte{0xff}, 128); err != nil { t.Fatal(err) } if err := corrupt.Close(); err != nil { t.Fatal(err) } if _, err := store.RecoverSegment(directory, name, committed, store.DefaultSegmentLimits()); !errors.Is(err, store.ErrSegmentChecksum) { t.Fatalf("corrupt committed range error = %v", err) } info, err = os.Stat(path) if err != nil { t.Fatal(err) } if uint64(info.Size()) != committed { t.Fatalf("corruption changed committed file: size=%d", info.Size()) } } func TestSegmentShortRangeUnsafeReferenceAndExclusiveCreate_BH_STORE_05(t *testing.T) { t.Parallel() directory := filepath.Join(t.TempDir(), "segments") if err := os.Mkdir(directory, 0o700); err != nil { t.Fatal(err) } owner := uuidBytes(20) segment, err := store.CreateSegment(directory, owner, 9, store.DefaultSegmentLimits()) if err != nil { t.Fatal(err) } name := segment.Name() committed, err := segment.Append(integrationRecord(owner, 1, "payload")) if err != nil { t.Fatal(err) } if _, err := store.CreateSegment(directory, owner, 9, store.DefaultSegmentLimits()); err == nil { t.Fatal("exclusive segment name collision succeeded") } if err := segment.Close(); err != nil { t.Fatal(err) } path := filepath.Join(directory, name) if err := os.Truncate(path, int64(committed-1)); err != nil { t.Fatal(err) } if _, err := store.RecoverSegment(directory, name, committed, store.DefaultSegmentLimits()); !errors.Is(err, store.ErrCommittedRangeMissing) { t.Fatalf("short committed range error = %v", err) } for _, unsafe := range []string{"../" + name, "not-a-segment", filepath.Join("nested", name)} { if _, err := store.RecoverSegment(directory, unsafe, 0, store.DefaultSegmentLimits()); !errors.Is(err, store.ErrUnsafeSegmentReference) { t.Errorf("unsafe reference %q error = %v", unsafe, err) } } } func TestSegmentConcurrentAppendSerializesRecords_RACE_STORE_02(t *testing.T) { t.Parallel() directory := filepath.Join(t.TempDir(), "segments") if err := os.Mkdir(directory, 0o700); err != nil { t.Fatal(err) } owner := uuidBytes(30) segment, err := store.CreateSegment(directory, owner, 1, store.DefaultSegmentLimits()) if err != nil { t.Fatal(err) } const count = 16 ends := make(chan uint64, count) errorsFound := make(chan error, count) var wait sync.WaitGroup for sequence := 1; sequence <= count; sequence++ { wait.Add(1) go func(sequence uint64) { defer wait.Done() end, err := segment.Append(integrationRecord(owner, sequence, "parallel")) if err != nil { errorsFound <- err return } ends <- end }(uint64(sequence)) } wait.Wait() close(errorsFound) for err := range errorsFound { t.Fatal(err) } close(ends) var committed uint64 seenEnds := map[uint64]bool{} for end := range ends { if seenEnds[end] { t.Fatalf("duplicate committed end %d", end) } seenEnds[end] = true if end > committed { committed = end } } name := segment.Name() if err := segment.Close(); err != nil { t.Fatal(err) } recovery, err := store.RecoverSegment(directory, name, committed, store.DefaultSegmentLimits()) if err != nil { t.Fatal(err) } if len(recovery.Records) != count { t.Fatalf("record count = %d, want %d", len(recovery.Records), count) } } func TestEventCrashBeforeMetadataDropsUncommittedTail_CRASH_STORE_01(t *testing.T) { t.Parallel() dataDir := filepath.Join(t.TempDir(), "state") injected := errors.New("injected crash before metadata") opened := openStoreWithOptions(t, store.Options{ DataDir: dataDir, BusyTimeout: busyTimeout, FaultInjector: &failOnceInjector{target: store.FaultAfterSegmentSyncBeforeMetadata, failure: injected}, }) issue := uuidBytes(40) seedCommand(t, opened.DB(), issue) event := appendEvent(issue, 1, "not-acknowledged") if result, err := opened.AppendCommandEvent(context.Background(), event); !errors.Is(err, injected) || result.ThroughEventSeq != 0 { t.Fatalf("AppendCommandEvent = (%+v, %v)", result, err) } assertCommandEventState(t, opened.DB(), issue, 0, 0) if err := opened.Close(); err != nil { t.Fatal(err) } reopened := openStore(t, dataDir) recovery, err := reopened.RecoverCommandSegments(context.Background()) if err != nil { t.Fatal(err) } if recovery.OrphansRemoved != 1 || recovery.RecordsChecked != 0 { t.Fatalf("recovery = %+v", recovery) } result, err := reopened.AppendCommandEvent(context.Background(), event) if err != nil || result.ThroughEventSeq != 1 || result.Duplicate { t.Fatalf("retry = (%+v, %v)", result, err) } assertCommandEventState(t, reopened.DB(), issue, 1, 1) if err := reopened.Close(); err != nil { t.Fatal(err) } } func TestEventCrashAfterCommitReplaysWithoutDuplication_CRASH_STORE_02(t *testing.T) { t.Parallel() dataDir := filepath.Join(t.TempDir(), "state") injected := errors.New("injected lost acknowledgement") opened := openStoreWithOptions(t, store.Options{ DataDir: dataDir, BusyTimeout: busyTimeout, FaultInjector: &failOnceInjector{target: store.FaultAfterMetadataCommit, failure: injected}, }) issue := uuidBytes(60) seedCommand(t, opened.DB(), issue) event := appendEvent(issue, 1, "committed") if result, err := opened.AppendCommandEvent(context.Background(), event); !errors.Is(err, injected) || result.ThroughEventSeq != 1 { t.Fatalf("AppendCommandEvent = (%+v, %v)", result, err) } assertCommandEventState(t, opened.DB(), issue, 1, 1) if err := opened.Close(); err != nil { t.Fatal(err) } reopened := openStore(t, dataDir) recovery, err := reopened.RecoverCommandSegments(context.Background()) if err != nil { t.Fatal(err) } if recovery.SegmentsChecked != 1 || recovery.RecordsChecked != 1 || recovery.OrphansRemoved != 0 { t.Fatalf("recovery = %+v", recovery) } result, err := reopened.AppendCommandEvent(context.Background(), event) if err != nil || !result.Duplicate || result.ThroughEventSeq != 1 { t.Fatalf("duplicate retry = (%+v, %v)", result, err) } conflict := event conflict.ImmutableSHA256 = sha256.Sum256([]byte("different immutable event")) if _, err := reopened.AppendCommandEvent(context.Background(), conflict); !errors.Is(err, store.ErrEventConflict) { t.Fatalf("conflicting duplicate error = %v", err) } assertCommandEventState(t, reopened.DB(), issue, 1, 1) if err := reopened.Close(); err != nil { t.Fatal(err) } } func TestEventCrashOnExistingSegmentTruncatesOnlyTail_CRASH_STORE_03(t *testing.T) { t.Parallel() dataDir := filepath.Join(t.TempDir(), "state") injected := errors.New("injected second append crash") opened := openStoreWithOptions(t, store.Options{ DataDir: dataDir, BusyTimeout: busyTimeout, FaultInjector: &failOnceInjector{target: store.FaultAfterSegmentSyncBeforeMetadata, failure: injected, skip: 1}, }) issue := uuidBytes(70) seedCommand(t, opened.DB(), issue) if _, err := opened.AppendCommandEvent(context.Background(), appendEvent(issue, 1, "committed-first")); err != nil { t.Fatal(err) } second := appendEvent(issue, 2, "uncommitted-second") if _, err := opened.AppendCommandEvent(context.Background(), second); !errors.Is(err, injected) { t.Fatalf("second append error = %v", err) } assertCommandEventState(t, opened.DB(), issue, 1, 1) if err := opened.Close(); err != nil { t.Fatal(err) } reopened := openStore(t, dataDir) recovery, err := reopened.RecoverCommandSegments(context.Background()) if err != nil { t.Fatal(err) } if recovery.SegmentsChecked != 1 || recovery.RecordsChecked != 1 || recovery.TailBytesTruncated == 0 || recovery.OrphansRemoved != 0 { t.Fatalf("recovery = %+v", recovery) } if result, err := reopened.AppendCommandEvent(context.Background(), second); err != nil || result.ThroughEventSeq != 2 { t.Fatalf("second retry = (%+v, %v)", result, err) } assertCommandEventState(t, reopened.DB(), issue, 2, 2) if err := reopened.Close(); err != nil { t.Fatal(err) } } func TestEventGapRejectedBeforeFileWriteAndRotation_HP_STORE_03(t *testing.T) { t.Parallel() dataDir := filepath.Join(t.TempDir(), "state") opened := openStoreWithOptions(t, store.Options{DataDir: dataDir, BusyTimeout: busyTimeout, SegmentTargetSize: 200}) issue := uuidBytes(80) seedCommand(t, opened.DB(), issue) if _, err := opened.AppendCommandEvent(context.Background(), appendEvent(issue, 2, "gap")); !errors.Is(err, store.ErrEventSequenceGap) { t.Fatalf("gap error = %v", err) } entries, err := os.ReadDir(filepath.Join(dataDir, "segments")) if err != nil || len(entries) != 0 { t.Fatalf("segments after gap = %v, err = %v", entries, err) } for sequence, payload := range []string{"one", "two"} { if _, err := opened.AppendCommandEvent(context.Background(), appendEvent(issue, uint64(sequence+1), payload)); err != nil { t.Fatal(err) } } var segmentCount int if err := opened.DB().QueryRow(`SELECT count(*) FROM output_segments WHERE issue_uuid = ?`, issue[:]).Scan(&segmentCount); err != nil || segmentCount != 2 { t.Fatalf("segment count = %d, err = %v", segmentCount, err) } if err := opened.QuickCheck(context.Background()); err != nil { t.Fatal(err) } if err := opened.Close(); err != nil { t.Fatal(err) } } func TestEventQuotaCountersAtomicAndCapacityRejectsBeforeWrite_HP_STORE_07(t *testing.T) { t.Parallel() dataDir := filepath.Join(t.TempDir(), "state") limits := store.QuotaLimits{ HardAllocationBytes: 1000, CommandOutputBytes: 1000, CommandTotalBytes: 2000, ClientTotalBytes: 3000, ServerTotalBytes: 4000, CloseoutReserveBytes: 200, FilesystemFloorBytes: 50, } opened := openStoreWithOptions(t, store.Options{ DataDir: dataDir, BusyTimeout: busyTimeout, QuotaLimits: limits, FreeSpaceProbe: store.FreeSpaceProbeFunc(func(string) (uint64, error) { return 10_000, nil }), }) issue := uuidBytes(90) seedCommand(t, opened.DB(), issue) if _, err := opened.DB().Exec(`UPDATE commands SET closeout_remaining_bytes = ? WHERE issue_uuid = ?`, limits.CloseoutReserveBytes, issue[:]); err != nil { t.Fatal(err) } first := appendEvent(issue, 1, "first") if _, err := opened.AppendCommandEvent(context.Background(), first); err != nil { t.Fatal(err) } var commandOutput, commandTotal, closeout, clientTotal, serverTotal uint64 if err := opened.DB().QueryRow(`SELECT output_charged_bytes, charged_bytes, closeout_remaining_bytes FROM commands WHERE issue_uuid = ?`, issue[:]).Scan(&commandOutput, &commandTotal, &closeout); err != nil { t.Fatal(err) } if err := opened.DB().QueryRow(`SELECT charged_bytes FROM clients WHERE client_id = 'client-a'`).Scan(&clientTotal); err != nil { t.Fatal(err) } if err := opened.DB().QueryRow(`SELECT command_charged_bytes FROM storage_counters WHERE singleton = 1`).Scan(&serverTotal); err != nil { t.Fatal(err) } if commandOutput == 0 || commandOutput != commandTotal || commandTotal != clientTotal || clientTotal != serverTotal || closeout != limits.CloseoutReserveBytes { t.Fatalf("quota counters = output:%d command:%d client:%d server:%d closeout:%d", commandOutput, commandTotal, clientTotal, serverTotal, closeout) } entries, err := os.ReadDir(filepath.Join(dataDir, "segments")) if err != nil || len(entries) != 1 { t.Fatalf("segment entries = %v, err = %v", entries, err) } segmentPath := filepath.Join(dataDir, "segments", entries[0].Name()) before, err := os.Stat(segmentPath) if err != nil { t.Fatal(err) } if result, err := opened.AppendCommandEvent(context.Background(), first); err != nil || !result.Duplicate { t.Fatalf("duplicate = (%+v, %v)", result, err) } second := appendEvent(issue, 2, "second") if _, err := opened.AppendCommandEvent(context.Background(), second); err == nil { t.Fatal("over-quota event was accepted") } else { var capacity *store.CapacityError if !errors.As(err, &capacity) || capacity.Tier != store.CapacityTierCommandOut { t.Fatalf("capacity error = %v", err) } } after, err := os.Stat(segmentPath) if err != nil { t.Fatal(err) } if after.Size() != before.Size() { t.Fatalf("rejected event changed segment size from %d to %d", before.Size(), after.Size()) } assertCommandEventState(t, opened.DB(), issue, 1, 1) if err := opened.Close(); err != nil { t.Fatal(err) } } func TestFilesystemFloorAndCounterMismatch_BH_STORE_08(t *testing.T) { t.Parallel() dataDir := filepath.Join(t.TempDir(), "state") opened := openStoreWithOptions(t, store.Options{ DataDir: dataDir, BusyTimeout: busyTimeout, FreeSpaceProbe: store.FreeSpaceProbeFunc(func(string) (uint64, error) { return 0, nil }), }) issue := uuidBytes(110) seedCommand(t, opened.DB(), issue) if _, err := opened.AppendCommandEvent(context.Background(), appendEvent(issue, 1, "floor")); err == nil { t.Fatal("filesystem-floor event was accepted") } else { var capacity *store.CapacityError if !errors.As(err, &capacity) || capacity.Tier != store.CapacityTierFilesystem { t.Fatalf("floor error = %v", err) } } entries, err := os.ReadDir(filepath.Join(dataDir, "segments")) if err != nil || len(entries) != 0 { t.Fatalf("segments after floor rejection = %v, err = %v", entries, err) } if err := opened.Close(); err != nil { t.Fatal(err) } reopened := openStore(t, dataDir) if _, err := reopened.DB().Exec(`UPDATE clients SET charged_bytes = 1 WHERE client_id = 'client-a'`); err != nil { t.Fatal(err) } if _, err := reopened.RecoverCommandSegments(context.Background()); !errors.Is(err, store.ErrQuotaCounterMismatch) { t.Fatalf("counter mismatch recovery error = %v", err) } if err := reopened.Close(); err != nil { t.Fatal(err) } } func openStore(t *testing.T, dataDir string) *store.Store { t.Helper() return openStoreWithOptions(t, store.Options{DataDir: dataDir, BusyTimeout: busyTimeout}) } func openStoreWithOptions(t *testing.T, options store.Options) *store.Store { t.Helper() opened, err := store.Open(context.Background(), options) if err != nil { t.Fatalf("Open(%s): %v", options.DataDir, err) } return opened } func assertPragma(t *testing.T, database *sql.DB, name, want string) { t.Helper() var got string if err := database.QueryRow(`PRAGMA ` + name).Scan(&got); err != nil { t.Fatalf("PRAGMA %s: %v", name, err) } if strings.ToLower(got) != want { t.Fatalf("PRAGMA %s = %q, want %q", name, got, want) } } func bytesOf(length int, value byte) []byte { result := make([]byte, length) for index := range result { result[index] = value } return result } func uuidBytes(start byte) [16]byte { var result [16]byte for index := range result { result[index] = start + byte(index) } return result } func integrationRecord(owner [16]byte, sequence uint64, payload string) store.SegmentRecord { return store.SegmentRecord{ OwnerUUID: owner, Sequence: sequence, ObservedUnixNano: int64(sequence), ReceiptUnixNano: int64(sequence + 1), Kind: store.PayloadKindCommandEvent, Stream: 1, Compression: 1, RawLength: uint64(len(payload)), Payload: []byte(payload), } } type failOnceInjector struct { mu sync.Mutex target string failure error skip int fired bool } func (injector *failOnceInjector) Checkpoint(name string) error { injector.mu.Lock() defer injector.mu.Unlock() if name == injector.target && !injector.fired { if injector.skip > 0 { injector.skip-- return nil } injector.fired = true return injector.failure } return nil } func seedCommand(t *testing.T, database *sql.DB, issue [16]byte) { t.Helper() clientInstance := uuidBytes(100) if _, err := database.Exec(`INSERT OR IGNORE INTO clients ( client_id, platform, architecture, daemon_version, daemon_cwd, supported_shells, capabilities, client_instance_id ) VALUES ('client-a', 3, 'amd64', 'test', 'C:\\work', x'01', x'', ?)`, clientInstance[:]); err != nil { t.Fatal(err) } requestHash := sha256.Sum256([]byte("request")) if _, err := database.Exec(`INSERT INTO commands ( issue_uuid, client_id, issue_time, server_receipt_time, lifecycle, revision, immutable_request_sha256, execution_spec, execution_spec_raw_bytes, execution_spec_stored_bytes, execution_spec_compression ) VALUES (?, 'client-a', 1, 1, 1, 1, ?, x'01', 1, 1, 1)`, issue[:], requestHash[:]); err != nil { t.Fatal(err) } } func appendEvent(issue [16]byte, sequence uint64, payload string) store.EventAppend { immutableHash := sha256.Sum256(append([]byte{byte(sequence)}, []byte(payload)...)) return store.EventAppend{ IssueUUID: issue, EventSeq: sequence, ObservedUnixNano: int64(sequence), ReceiptUnixNano: int64(sequence + 10), EventType: 1, Stream: 1, Compression: 1, RawLength: uint64(len(payload)), Payload: []byte(payload), ImmutableSHA256: immutableHash, } } func assertCommandEventState(t *testing.T, database *sql.DB, issue [16]byte, wantLast, wantEvents int) { t.Helper() var last, events int if err := database.QueryRow(`SELECT last_event_seq FROM commands WHERE issue_uuid = ?`, issue[:]).Scan(&last); err != nil { t.Fatal(err) } if err := database.QueryRow(`SELECT count(*) FROM command_events WHERE issue_uuid = ?`, issue[:]).Scan(&events); err != nil { t.Fatal(err) } if last != wantLast || events != wantEvents { t.Fatalf("event state = (last=%d, rows=%d), want (%d, %d)", last, events, wantLast, wantEvents) } }