feat: persist events across commit boundaries

2026-08-31 09:35:20 +00:00
parent 43dcd56313
commit 583b539f86
7 changed files with 722 additions and 14 deletions
@@ -0,0 +1,187 @@
+package store
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+var ErrIntegrityCheck = errors.New("SQLite integrity check failed")
+
+type RecoveryReport struct {
+	SegmentsChecked    uint64
+	RecordsChecked     uint64
+	TailBytesTruncated uint64
+	OrphansRemoved     uint64
+}
+
+func (store *Store) QuickCheck(ctx context.Context) error {
+	store.writeMu.Lock()
+	defer store.writeMu.Unlock()
+	database, err := store.openDatabase()
+	if err != nil {
+		return err
+	}
+	return store.quickCheckDatabase(ctx, database)
+}
+
+func (store *Store) RecoverCommandSegments(ctx context.Context) (RecoveryReport, error) {
+	var report RecoveryReport
+	store.writeMu.Lock()
+	defer store.writeMu.Unlock()
+	database, err := store.openDatabase()
+	if err != nil {
+		return report, err
+	}
+	if err := store.quickCheckDatabase(ctx, database); err != nil {
+		return report, err
+	}
+	rows, err := database.QueryContext(ctx, `SELECT issue_uuid, ordinal, path, committed_end_offset,
+min_event_seq, max_event_seq, stream_mix, compressed_bytes, raw_bytes
+FROM output_segments ORDER BY issue_uuid, ordinal`)
+	if err != nil {
+		return report, err
+	}
+	type segmentMetadata struct {
+		owner                                                                 [16]byte
+		ordinal                                                               uint32
+		name                                                                  string
+		committed, minSequence, maxSequence, streamMix, storedBytes, rawBytes uint64
+	}
+	var metadata []segmentMetadata
+	for rows.Next() {
+		var current segmentMetadata
+		var ownerBytes []byte
+		if err := rows.Scan(&ownerBytes, &current.ordinal, &current.name, &current.committed, &current.minSequence, &current.maxSequence, &current.streamMix, &current.storedBytes, &current.rawBytes); err != nil {
+			_ = rows.Close()
+			return report, err
+		}
+		if len(ownerBytes) != 16 {
+			_ = rows.Close()
+			return report, fmt.Errorf("%w: invalid segment owner", ErrCommittedRangeMissing)
+		}
+		copy(current.owner[:], ownerBytes)
+		metadata = append(metadata, current)
+	}
+	if err := rows.Close(); err != nil {
+		return report, err
+	}
+	referenced := map[string]bool{}
+	for _, current := range metadata {
+		nameOwner, nameOrdinal, err := parseSegmentName(current.name)
+		if err != nil || nameOwner != current.owner || nameOrdinal != current.ordinal {
+			return report, ErrUnsafeSegmentReference
+		}
+		recovery, err := RecoverSegment(store.segmentDirectory(), current.name, current.committed, DefaultSegmentLimits())
+		if err != nil {
+			return report, err
+		}
+		if err := validateRecoveredMetadata(ctx, database, current.owner, current.ordinal, recovery, current.minSequence, current.maxSequence, current.streamMix, current.storedBytes, current.rawBytes); err != nil {
+			return report, err
+		}
+		referenced[current.name] = true
+		report.SegmentsChecked++
+		report.RecordsChecked += uint64(len(recovery.Records))
+		report.TailBytesTruncated += recovery.TruncatedTail
+	}
+	entries, err := os.ReadDir(store.segmentDirectory())
+	if err != nil {
+		return report, err
+	}
+	directoryChanged := false
+	for _, entry := range entries {
+		name := entry.Name()
+		if referenced[name] {
+			continue
+		}
+		if _, _, err := parseSegmentName(name); err != nil {
+			return report, ErrUnsafeSegmentReference
+		}
+		path := filepath.Join(store.segmentDirectory(), name)
+		info, err := os.Lstat(path)
+		if err != nil {
+			return report, err
+		}
+		if !info.Mode().IsRegular() || info.Mode().Perm()&0o077 != 0 {
+			return report, ErrUnsafeSegmentReference
+		}
+		if err := os.Remove(path); err != nil {
+			return report, err
+		}
+		directoryChanged = true
+		report.OrphansRemoved++
+	}
+	if directoryChanged {
+		if err := syncDirectory(store.segmentDirectory()); err != nil {
+			return report, err
+		}
+	}
+	if _, err := database.ExecContext(ctx, `UPDATE output_segments SET sealed = 1 WHERE sealed = 0`); err != nil {
+		return report, err
+	}
+	return report, nil
+}
+
+func (store *Store) quickCheckDatabase(ctx context.Context, database *sql.DB) error {
+	rows, err := database.QueryContext(ctx, `PRAGMA quick_check(100)`)
+	if err != nil {
+		return fmt.Errorf("%w: query failed", ErrIntegrityCheck)
+	}
+	defer rows.Close()
+	count := 0
+	for rows.Next() {
+		var result string
+		if err := rows.Scan(&result); err != nil {
+			return fmt.Errorf("%w: result unreadable", ErrIntegrityCheck)
+		}
+		count++
+		if result != "ok" {
+			return fmt.Errorf("%w: database reported an inconsistency", ErrIntegrityCheck)
+		}
+	}
+	if err := rows.Err(); err != nil || count != 1 {
+		return fmt.Errorf("%w: incomplete result", ErrIntegrityCheck)
+	}
+	return nil
+}
+
+func validateRecoveredMetadata(ctx context.Context, database *sql.DB, owner [16]byte, ordinal uint32, recovery SegmentRecovery, minSequence, maxSequence, streamMix, storedBytes, rawBytes uint64) error {
+	rows, err := database.QueryContext(ctx, `SELECT event_seq, segment_record_offset, segment_record_length,
+compression, raw_bytes, stored_bytes FROM command_events
+WHERE issue_uuid = ? AND segment_ordinal = ? ORDER BY segment_record_offset`, owner[:], ordinal)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+	var index int
+	var computedStored, computedRaw, computedStreams uint64
+	for rows.Next() {
+		if index >= len(recovery.Records) {
+			return ErrCommittedRangeMissing
+		}
+		var sequence, offset, length, compression, raw, stored uint64
+		if err := rows.Scan(&sequence, &offset, &length, &compression, &raw, &stored); err != nil {
+			return err
+		}
+		recovered := recovery.Records[index]
+		if recovered.Offset != offset || recovered.Length != length || recovered.Record.OwnerUUID != owner || recovered.Record.Sequence != sequence || uint64(recovered.Record.Compression) != compression || recovered.Record.RawLength != raw || uint64(len(recovered.Record.Payload)) != stored {
+			return ErrCommittedRangeMissing
+		}
+		if recovered.Record.Stream > 0 {
+			computedStreams |= uint64(1) << (recovered.Record.Stream - 1)
+		}
+		computedStored += stored
+		computedRaw += raw
+		index++
+	}
+	if err := rows.Err(); err != nil {
+		return err
+	}
+	if index != len(recovery.Records) || index == 0 || recovery.Records[0].Record.Sequence != minSequence || recovery.Records[index-1].Record.Sequence != maxSequence || computedStreams != streamMix || computedStored != storedBytes || computedRaw != rawBytes {
+		return ErrCommittedRangeMissing
+	}
+	return nil
+}