fix: fence reconciliation admission race
This commit is contained in:
@@ -16,11 +16,26 @@ import (
|
||||
// client. It is a read-only snapshot used to tell a reconnecting agent which
|
||||
// UUIDs and durable cursors must be compared before fresh dispatch is enabled.
|
||||
func (store *Store) ReconcileTargets(ctx context.Context, clientID string) ([]*rvboxv1.ReconcileTarget, error) {
|
||||
return store.ReconcileTargetsAt(ctx, clientID, time.Time{})
|
||||
}
|
||||
|
||||
// ReconcileTargetsAt returns the durable non-terminal view at the supplied
|
||||
// receipt-time boundary. A connecting session uses one boundary for both the
|
||||
// request and its reply: commands admitted after it are fresh work, not absent
|
||||
// client evidence to be reconciled.
|
||||
func (store *Store) ReconcileTargetsAt(ctx context.Context, clientID string, receiptBoundary time.Time) ([]*rvboxv1.ReconcileTarget, error) {
|
||||
if clientID == "" {
|
||||
return nil, errors.New("client ID is required")
|
||||
}
|
||||
rows, err := store.db.QueryContext(ctx, `SELECT issue_uuid, last_event_seq, revision, immutable_request_sha256
|
||||
FROM commands WHERE client_id = ? AND lifecycle BETWEEN 1 AND 4 ORDER BY issue_time, issue_uuid`, clientID)
|
||||
query := `SELECT issue_uuid, last_event_seq, revision, immutable_request_sha256
|
||||
FROM commands WHERE client_id = ? AND lifecycle BETWEEN 1 AND 4`
|
||||
args := []any{clientID}
|
||||
if !receiptBoundary.IsZero() {
|
||||
query += ` AND server_receipt_time <= ?`
|
||||
args = append(args, receiptBoundary.UTC().UnixNano())
|
||||
}
|
||||
query += ` ORDER BY issue_time, issue_uuid`
|
||||
rows, err := store.db.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -62,10 +77,19 @@ func (store *Store) ReconcileClientSnapshot(ctx context.Context, clientID string
|
||||
// incidented. Retained non-terminal rows are retargeted to this generation so
|
||||
// late events from the previous connection cannot advance the command.
|
||||
func (store *Store) ReconcileClientSnapshotForSession(ctx context.Context, clientID string, generation uint64, snapshot *rvboxv1.ReconcileSnapshot) (*rvboxv1.ReconcileResult, error) {
|
||||
return store.ReconcileClientSnapshotForSessionAt(ctx, clientID, generation, snapshot, time.Time{})
|
||||
}
|
||||
|
||||
// ReconcileClientSnapshotForSessionAt reconciles exactly the server state that
|
||||
// was included in the matching ReconcileRequest. Work admitted after the
|
||||
// boundary is intentionally left for the dispatch loop once reconciliation
|
||||
// completes; treating it as absent client evidence would lose a valid command
|
||||
// during the Hello/reconcile race.
|
||||
func (store *Store) ReconcileClientSnapshotForSessionAt(ctx context.Context, clientID string, generation uint64, snapshot *rvboxv1.ReconcileSnapshot, receiptBoundary time.Time) (*rvboxv1.ReconcileResult, error) {
|
||||
if generation == 0 {
|
||||
return nil, errors.New("session generation is required")
|
||||
}
|
||||
return store.reconcileClientSnapshot(ctx, clientID, generation, snapshot)
|
||||
return store.reconcileClientSnapshotAt(ctx, clientID, generation, snapshot, receiptBoundary)
|
||||
}
|
||||
|
||||
type reconcileServerRow struct {
|
||||
@@ -78,6 +102,10 @@ type reconcileServerRow struct {
|
||||
}
|
||||
|
||||
func (store *Store) reconcileClientSnapshot(ctx context.Context, clientID string, generation uint64, snapshot *rvboxv1.ReconcileSnapshot) (*rvboxv1.ReconcileResult, error) {
|
||||
return store.reconcileClientSnapshotAt(ctx, clientID, generation, snapshot, time.Time{})
|
||||
}
|
||||
|
||||
func (store *Store) reconcileClientSnapshotAt(ctx context.Context, clientID string, generation uint64, snapshot *rvboxv1.ReconcileSnapshot, receiptBoundary time.Time) (*rvboxv1.ReconcileResult, error) {
|
||||
if clientID == "" {
|
||||
return nil, errors.New("client ID is required")
|
||||
}
|
||||
@@ -112,7 +140,7 @@ func (store *Store) reconcileClientSnapshot(ctx context.Context, clientID string
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
serverRows, tombstones, err := loadReconcileRows(ctx, tx, clientID)
|
||||
serverRows, tombstones, err := loadReconcileRows(ctx, tx, clientID, receiptBoundary)
|
||||
if err != nil {
|
||||
store.writeMu.Unlock()
|
||||
return nil, err
|
||||
@@ -215,8 +243,14 @@ type reconcileIncident struct {
|
||||
dataLoss bool
|
||||
}
|
||||
|
||||
func loadReconcileRows(ctx context.Context, tx *sql.Tx, clientID string) (map[string]reconcileServerRow, map[string][]byte, error) {
|
||||
rows, err := tx.QueryContext(ctx, `SELECT issue_uuid, lifecycle, revision, last_event_seq, immutable_request_sha256, target_session_generation FROM commands WHERE client_id = ?`, clientID)
|
||||
func loadReconcileRows(ctx context.Context, tx *sql.Tx, clientID string, receiptBoundary time.Time) (map[string]reconcileServerRow, map[string][]byte, error) {
|
||||
query := `SELECT issue_uuid, lifecycle, revision, last_event_seq, immutable_request_sha256, target_session_generation FROM commands WHERE client_id = ?`
|
||||
args := []any{clientID}
|
||||
if !receiptBoundary.IsZero() {
|
||||
query += ` AND server_receipt_time <= ?`
|
||||
args = append(args, receiptBoundary.UTC().UnixNano())
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
@@ -71,6 +71,31 @@ func TestReconcileSnapshotMutatesMissingAndRetargets_HP_SES_13(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileSessionBoundaryLeavesNewlyQueuedCommandForDispatch_HP_RECONCILE_08(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
opened, err := Open(ctx, Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = opened.Close() })
|
||||
boundary := time.Date(2026, time.September, 11, 12, 0, 0, 0, time.UTC)
|
||||
if _, err := opened.RegisterClientSession(ctx, ClientRegistration{ClientID: "boundary-client", Platform: 3, Architecture: "amd64", DaemonVersion: "test", DaemonCWD: `C:\\`, SupportedShells: []byte{1}, ClientInstanceID: [16]byte{31}, SessionID: [16]byte{32}, ConnectedAt: boundary}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
issue := mustReconcileIssue(t, "019c46f1-1d02-7000-8000-0000000000d1")
|
||||
if _, err := opened.QueueCommand(ctx, QueueCommandInput{IssueUUID: issue, ClientID: "boundary-client", IssueTime: boundary.Add(time.Nanosecond), ReceiptTime: boundary.Add(time.Nanosecond), ImmutableSHA256: sha256.Sum256([]byte("new-after-reconcile-boundary")), ExecutionSpec: []byte("echo queued")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := opened.ReconcileClientSnapshotForSessionAt(ctx, "boundary-client", 1, &rvboxv1.ReconcileSnapshot{}, boundary); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
view, err := opened.GetCommandView(ctx, "boundary-client", issue)
|
||||
if err != nil || view.Lifecycle != uint32(rvboxv1.CommandLifecycle_COMMAND_QUEUED) {
|
||||
t.Fatalf("post-boundary command = %#v, %v", view, err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustReconcileIssue(t *testing.T, value string) domain.UUID {
|
||||
t.Helper()
|
||||
issue, err := domain.ParseUUIDv7(value)
|
||||
|
||||
Reference in New Issue
Block a user