423 lines
13 KiB
Go
423 lines
13 KiB
Go
//go:build integration
|
|
|
|
package store_test
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rvbox/rvbox/internal/server/store"
|
|
)
|
|
|
|
const busyTimeout = 2 * time.Second
|
|
|
|
func TestRealSQLiteInitializationAndRestart_HP_STORE_01(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
dataDir := filepath.Join(t.TempDir(), "state")
|
|
opened := openStore(t, dataDir)
|
|
|
|
for _, directory := range []string{dataDir, filepath.Join(dataDir, "segments"), filepath.Join(dataDir, "audit")} {
|
|
info, err := os.Stat(directory)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if runtime.GOOS != "windows" && info.Mode().Perm() != 0o700 {
|
|
t.Fatalf("%s mode = %04o, want 0700", directory, info.Mode().Perm())
|
|
}
|
|
}
|
|
for _, file := range []string{"rvbox.db", "server.lock"} {
|
|
info, err := os.Stat(filepath.Join(dataDir, file))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if runtime.GOOS != "windows" && info.Mode().Perm() != 0o600 {
|
|
t.Fatalf("%s mode = %04o, want 0600", file, info.Mode().Perm())
|
|
}
|
|
}
|
|
|
|
assertPragma(t, opened.DB(), "journal_mode", "wal")
|
|
assertPragma(t, opened.DB(), "foreign_keys", "1")
|
|
assertPragma(t, opened.DB(), "synchronous", "2")
|
|
assertPragma(t, opened.DB(), "busy_timeout", "2000")
|
|
|
|
rows, err := opened.DB().Query(`SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var names []string
|
|
for rows.Next() {
|
|
var name string
|
|
if err := rows.Scan(&name); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
names = append(names, name)
|
|
}
|
|
if err := rows.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := []string{"audit_events", "clients", "command_events", "command_payloads", "command_tombstones", "commands", "control_mutations", "output_segments", "output_truncations", "schema_migrations", "sessions", "stdin_writes", "storage_incidents", "takeover_authorizations"}
|
|
sort.Strings(want)
|
|
if strings.Join(names, ",") != strings.Join(want, ",") {
|
|
t.Fatalf("tables = %v, want %v", names, want)
|
|
}
|
|
var migrationCount int
|
|
if err := opened.DB().QueryRow(`SELECT count(*) FROM schema_migrations`).Scan(&migrationCount); err != nil || migrationCount != 1 {
|
|
t.Fatalf("migration count = %d, err = %v", migrationCount, err)
|
|
}
|
|
if err := opened.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
reopened := openStore(t, dataDir)
|
|
if err := reopened.DB().QueryRow(`SELECT count(*) FROM schema_migrations`).Scan(&migrationCount); err != nil || migrationCount != 1 {
|
|
t.Fatalf("reopened migration count = %d, err = %v", migrationCount, err)
|
|
}
|
|
if err := reopened.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestSingleInstanceLockAndConcurrentClose_RACE_STORE_01(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
dataDir := filepath.Join(t.TempDir(), "state")
|
|
first := openStore(t, dataDir)
|
|
if _, err := store.Open(context.Background(), store.Options{DataDir: dataDir, BusyTimeout: busyTimeout}); !errors.Is(err, store.ErrAlreadyOpen) {
|
|
t.Fatalf("second Open error = %v, want ErrAlreadyOpen", err)
|
|
}
|
|
var wait sync.WaitGroup
|
|
for range 8 {
|
|
wait.Add(1)
|
|
go func() {
|
|
defer wait.Done()
|
|
if err := first.Close(); err != nil && !errors.Is(err, sql.ErrConnDone) {
|
|
t.Errorf("Close: %v", err)
|
|
}
|
|
}()
|
|
}
|
|
wait.Wait()
|
|
reopened := openStore(t, dataDir)
|
|
if err := reopened.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestSchemaRejectsInvalidIDsAndForeignKeys_BH_STORE_01(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
opened := openStore(t, filepath.Join(t.TempDir(), "state"))
|
|
defer opened.Close()
|
|
_, err := opened.DB().Exec(`INSERT INTO clients (
|
|
client_id, platform, architecture, daemon_version, daemon_cwd, supported_shells, capabilities, client_instance_id
|
|
) VALUES ('client-a', 3, 'amd64', 'test', 'C:\\work', x'', x'', x'01')`)
|
|
if err == nil {
|
|
t.Fatal("invalid client instance UUID was accepted")
|
|
}
|
|
_, err = opened.DB().Exec(`INSERT INTO sessions (
|
|
session_id, client_id, client_instance_id, generation, opened_at
|
|
) VALUES (?, 'missing-client', ?, 1, 1)`, bytesOf(16, 1), bytesOf(16, 2))
|
|
if err == nil || !strings.Contains(strings.ToLower(err.Error()), "foreign key") {
|
|
t.Fatalf("foreign-key insert error = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestUnsafePathsAndCancelledOpen_BH_STORE_02(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
if _, err := store.Open(context.Background(), store.Options{DataDir: "relative", BusyTimeout: busyTimeout}); !errors.Is(err, store.ErrUnsafeDataDirectory) {
|
|
t.Fatalf("relative path error = %v", err)
|
|
}
|
|
if _, err := store.Open(context.Background(), store.Options{DataDir: string(filepath.Separator), BusyTimeout: busyTimeout}); !errors.Is(err, store.ErrUnsafeDataDirectory) {
|
|
t.Fatalf("root path error = %v", err)
|
|
}
|
|
permissive := filepath.Join(t.TempDir(), "permissive")
|
|
if err := os.Mkdir(permissive, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Open(context.Background(), store.Options{DataDir: permissive, BusyTimeout: busyTimeout}); !errors.Is(err, store.ErrUnsafeDataDirectory) {
|
|
t.Fatalf("permissive path error = %v", err)
|
|
}
|
|
if runtime.GOOS != "windows" {
|
|
target := filepath.Join(t.TempDir(), "target")
|
|
if err := os.Mkdir(target, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
link := filepath.Join(t.TempDir(), "linked")
|
|
if err := os.Symlink(target, link); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Open(context.Background(), store.Options{DataDir: link, BusyTimeout: busyTimeout}); !errors.Is(err, store.ErrUnsafeDataDirectory) {
|
|
t.Fatalf("symlink path error = %v", err)
|
|
}
|
|
}
|
|
|
|
cancelled, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
dataDir := filepath.Join(t.TempDir(), "cancelled")
|
|
if _, err := store.Open(cancelled, store.Options{DataDir: dataDir, BusyTimeout: busyTimeout}); !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("cancelled Open error = %v", err)
|
|
}
|
|
opened := openStore(t, dataDir)
|
|
if err := opened.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestMigrationChecksumMismatchPreventsOpen_REC_STORE_01(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
dataDir := filepath.Join(t.TempDir(), "state")
|
|
opened := openStore(t, dataDir)
|
|
if err := opened.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raw, err := sql.Open("sqlite", filepath.Join(dataDir, "rvbox.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := raw.Exec(`UPDATE schema_migrations SET checksum = 'tampered' WHERE version = 1`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := raw.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Open(context.Background(), store.Options{DataDir: dataDir, BusyTimeout: busyTimeout}); err == nil || !strings.Contains(err.Error(), "checksum mismatch") {
|
|
t.Fatalf("Open error = %v, want checksum mismatch", err)
|
|
}
|
|
// A failed open must release the process lock for offline inspection/repair.
|
|
raw, err = sql.Open("sqlite", filepath.Join(dataDir, "rvbox.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := raw.Ping(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := raw.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestSegmentAppendTailRecoveryAndCommittedCorruption_REC_STORE_02(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
directory := filepath.Join(t.TempDir(), "segments")
|
|
if err := os.Mkdir(directory, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
owner := uuidBytes(10)
|
|
segment, err := store.CreateSegment(directory, owner, 1, store.DefaultSegmentLimits())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
name := segment.Name()
|
|
first := integrationRecord(owner, 1, "first")
|
|
second := integrationRecord(owner, 2, "second")
|
|
if _, err := segment.Append(first); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
committed, err := segment.Append(second)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := segment.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
path := filepath.Join(directory, name)
|
|
file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := file.Write([]byte("unacknowledged-tail")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := file.Sync(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := file.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
recovery, err := store.RecoverSegment(directory, name, committed, store.DefaultSegmentLimits())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if recovery.TruncatedTail != uint64(len("unacknowledged-tail")) || len(recovery.Records) != 2 {
|
|
t.Fatalf("recovery = %+v", recovery)
|
|
}
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if uint64(info.Size()) != committed {
|
|
t.Fatalf("size = %d, want %d", info.Size(), committed)
|
|
}
|
|
|
|
corrupt, err := os.OpenFile(path, os.O_RDWR, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := corrupt.WriteAt([]byte{0xff}, 128); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := corrupt.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.RecoverSegment(directory, name, committed, store.DefaultSegmentLimits()); !errors.Is(err, store.ErrSegmentChecksum) {
|
|
t.Fatalf("corrupt committed range error = %v", err)
|
|
}
|
|
info, err = os.Stat(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if uint64(info.Size()) != committed {
|
|
t.Fatalf("corruption changed committed file: size=%d", info.Size())
|
|
}
|
|
}
|
|
|
|
func TestSegmentShortRangeUnsafeReferenceAndExclusiveCreate_BH_STORE_05(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
directory := filepath.Join(t.TempDir(), "segments")
|
|
if err := os.Mkdir(directory, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
owner := uuidBytes(20)
|
|
segment, err := store.CreateSegment(directory, owner, 9, store.DefaultSegmentLimits())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
name := segment.Name()
|
|
committed, err := segment.Append(integrationRecord(owner, 1, "payload"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.CreateSegment(directory, owner, 9, store.DefaultSegmentLimits()); err == nil {
|
|
t.Fatal("exclusive segment name collision succeeded")
|
|
}
|
|
if err := segment.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
path := filepath.Join(directory, name)
|
|
if err := os.Truncate(path, int64(committed-1)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.RecoverSegment(directory, name, committed, store.DefaultSegmentLimits()); !errors.Is(err, store.ErrCommittedRangeMissing) {
|
|
t.Fatalf("short committed range error = %v", err)
|
|
}
|
|
for _, unsafe := range []string{"../" + name, "not-a-segment", filepath.Join("nested", name)} {
|
|
if _, err := store.RecoverSegment(directory, unsafe, 0, store.DefaultSegmentLimits()); !errors.Is(err, store.ErrUnsafeSegmentReference) {
|
|
t.Errorf("unsafe reference %q error = %v", unsafe, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSegmentConcurrentAppendSerializesRecords_RACE_STORE_02(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
directory := filepath.Join(t.TempDir(), "segments")
|
|
if err := os.Mkdir(directory, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
owner := uuidBytes(30)
|
|
segment, err := store.CreateSegment(directory, owner, 1, store.DefaultSegmentLimits())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
const count = 16
|
|
ends := make(chan uint64, count)
|
|
errorsFound := make(chan error, count)
|
|
var wait sync.WaitGroup
|
|
for sequence := 1; sequence <= count; sequence++ {
|
|
wait.Add(1)
|
|
go func(sequence uint64) {
|
|
defer wait.Done()
|
|
end, err := segment.Append(integrationRecord(owner, sequence, "parallel"))
|
|
if err != nil {
|
|
errorsFound <- err
|
|
return
|
|
}
|
|
ends <- end
|
|
}(uint64(sequence))
|
|
}
|
|
wait.Wait()
|
|
close(errorsFound)
|
|
for err := range errorsFound {
|
|
t.Fatal(err)
|
|
}
|
|
close(ends)
|
|
var committed uint64
|
|
seenEnds := map[uint64]bool{}
|
|
for end := range ends {
|
|
if seenEnds[end] {
|
|
t.Fatalf("duplicate committed end %d", end)
|
|
}
|
|
seenEnds[end] = true
|
|
if end > committed {
|
|
committed = end
|
|
}
|
|
}
|
|
name := segment.Name()
|
|
if err := segment.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
recovery, err := store.RecoverSegment(directory, name, committed, store.DefaultSegmentLimits())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(recovery.Records) != count {
|
|
t.Fatalf("record count = %d, want %d", len(recovery.Records), count)
|
|
}
|
|
}
|
|
|
|
func openStore(t *testing.T, dataDir string) *store.Store {
|
|
t.Helper()
|
|
opened, err := store.Open(context.Background(), store.Options{DataDir: dataDir, BusyTimeout: busyTimeout})
|
|
if err != nil {
|
|
t.Fatalf("Open(%s): %v", dataDir, err)
|
|
}
|
|
return opened
|
|
}
|
|
|
|
func assertPragma(t *testing.T, database *sql.DB, name, want string) {
|
|
t.Helper()
|
|
var got string
|
|
if err := database.QueryRow(`PRAGMA ` + name).Scan(&got); err != nil {
|
|
t.Fatalf("PRAGMA %s: %v", name, err)
|
|
}
|
|
if strings.ToLower(got) != want {
|
|
t.Fatalf("PRAGMA %s = %q, want %q", name, got, want)
|
|
}
|
|
}
|
|
|
|
func bytesOf(length int, value byte) []byte {
|
|
result := make([]byte, length)
|
|
for index := range result {
|
|
result[index] = value
|
|
}
|
|
return result
|
|
}
|
|
|
|
func uuidBytes(start byte) [16]byte {
|
|
var result [16]byte
|
|
for index := range result {
|
|
result[index] = start + byte(index)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func integrationRecord(owner [16]byte, sequence uint64, payload string) store.SegmentRecord {
|
|
return store.SegmentRecord{
|
|
OwnerUUID: owner, Sequence: sequence, ObservedUnixNano: int64(sequence), ReceiptUnixNano: int64(sequence + 1),
|
|
Kind: store.PayloadKindCommandEvent, Stream: 1, Compression: 1, RawLength: uint64(len(payload)), Payload: []byte(payload),
|
|
}
|
|
}
|