cabbage/rvbox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: fence reconciliation admission race

Browse Source
This commit is contained in:
cabbage
2026-09-11 07:36:11 +00:00
parent d753e8b698
commit e79f882993
10 changed files with 193 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/server/session/agent_server.go
+6 -2
View File
@@ -102,6 +102,10 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
return
}
hello := helloEnvelope.GetClientHello()
// This cutoff precedes registration, which is when control callers can
// first observe the live client. Reconciliation must compare only the
// state included in its request; later control commands are fresh work.
reconcileBoundary := server.now()
instanceID, err := domain.ParseUUIDv7(hello.GetClientInstanceId())
if err != nil {
server.close(connection, websocket.StatusPolicyViolation, "invalid client instance ID")
@@ -192,7 +196,7 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
server.close(connection, websocket.StatusInternalError, "could not queue welcome")
return
}
targets, err := server.Store.ReconcileTargets(parent, hello.GetClientId())
targets, err := server.Store.ReconcileTargetsAt(parent, hello.GetClientId(), reconcileBoundary)
if err != nil {
server.close(connection, websocket.StatusInternalError, "could not build reconciliation request")
return
@@ -237,7 +241,7 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
return
}
if snapshot := envelope.GetReconcileSnapshot(); snapshot != nil {
result, reconcileErr := server.Store.ReconcileClientSnapshotForSession(sessionContext, hello.GetClientId(), registration.Generation, snapshot)
result, reconcileErr := server.Store.ReconcileClientSnapshotForSessionAt(sessionContext, hello.GetClientId(), registration.Generation, snapshot, reconcileBoundary)
if reconcileErr != nil {
server.close(connection, websocket.StatusPolicyViolation, "reconciliation failed")
return
internal/server/session/agent_server_test.go
+57 -1
View File
@@ -98,6 +98,57 @@ func TestAgentServerRegistrationAndReplacement_HP_SES_05(t *testing.T) {
}
}
func TestAgentServerDispatchesCommandAdmittedDuringReconcile_HP_SES_11(t *testing.T) {
server, agent, cleanup := newTestAgentServerWithStore(t)
defer cleanup()
instance, err := domain.NewUUIDv7()
if err != nil {
t.Fatal(err)
}
connection, welcome := dialAndHello(t, server, "client-race", instance.String())
defer connection.CloseNow()
issue, err := domain.NewUUIDv7()
if err != nil {
t.Fatal(err)
}
spec, err := proto.Marshal(&rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "echo dispatch"}})
if err != nil {
t.Fatal(err)
}
now := time.Now().UTC()
if _, err := agent.Store.QueueCommand(context.Background(), store.QueueCommandInput{IssueUUID: issue, ClientID: "client-race", IssueTime: now, ReceiptTime: now, ImmutableSHA256: sha256.Sum256([]byte("during-reconcile")), ExecutionSpec: spec}); err != nil {
t.Fatal(err)
}
snapshot, err := proto.Marshal(&rvboxv1.AgentEnvelope{
SessionId: welcome.GetSessionId(), SessionGeneration: welcome.GetSessionGeneration(),
Payload: &rvboxv1.AgentEnvelope_ReconcileSnapshot{ReconcileSnapshot: &rvboxv1.ReconcileSnapshot{}},
})
if err != nil {
t.Fatal(err)
}
if err := connection.Write(context.Background(), websocket.MessageBinary, snapshot); err != nil {
t.Fatal(err)
}
readContext, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
for index := 0; index < 2; index++ {
_, payload, err := connection.Read(readContext)
if err != nil {
t.Fatal(err)
}
var envelope rvboxv1.AgentEnvelope
if err := proto.Unmarshal(payload, &envelope); err != nil {
t.Fatal(err)
}
if index == 0 && envelope.GetReconcileResult() == nil {
t.Fatalf("first post-snapshot envelope = %T, want reconcile result", envelope.Payload)
}
if index == 1 && (envelope.GetCommandDispatch() == nil || envelope.GetCommandDispatch().GetIssueUuid() != issue.String()) {
t.Fatalf("second post-snapshot envelope = %+v, want dispatch %s", envelope.Payload, issue)
}
}
}
func TestAgentServerRejectsHostileWireInputs_BH_SES_04(t *testing.T) {
server, cleanup := newTestAgentServer(t)
defer cleanup()
@@ -174,6 +225,11 @@ func TestWireEventAppendCarriesClientBinding_HP_EVENT_01(t *testing.T) {
}
func newTestAgentServer(t *testing.T) (*httptest.Server, func()) {
server, _, cleanup := newTestAgentServerWithStore(t)
return server, cleanup
}
func newTestAgentServerWithStore(t *testing.T) (*httptest.Server, *AgentServer, func()) {
t.Helper()
dataDirectory := filepath.Join(t.TempDir(), "store")
if err := os.Mkdir(dataDirectory, 0o700); err != nil {
@@ -190,7 +246,7 @@ func newTestAgentServer(t *testing.T) (*httptest.Server, func()) {
HeartbeatIdle: time.Second, LivenessTimeout: 2 * time.Second,
}
httpServer := httptest.NewServer(agent)
return httpServer, func() {
return httpServer, agent, func() {
httpServer.Close()
if err := persistence.Close(); err != nil {
t.Errorf("close persistence: %v", err)
internal/server/store/reconcile.go
+40 -6
View File
@@ -16,11 +16,26 @@ import (
// client. It is a read-only snapshot used to tell a reconnecting agent which
// UUIDs and durable cursors must be compared before fresh dispatch is enabled.
func (store *Store) ReconcileTargets(ctx context.Context, clientID string) ([]*rvboxv1.ReconcileTarget, error) {
return store.ReconcileTargetsAt(ctx, clientID, time.Time{})
}
// ReconcileTargetsAt returns the durable non-terminal view at the supplied
// receipt-time boundary. A connecting session uses one boundary for both the
// request and its reply: commands admitted after it are fresh work, not absent
// client evidence to be reconciled.
func (store *Store) ReconcileTargetsAt(ctx context.Context, clientID string, receiptBoundary time.Time) ([]*rvboxv1.ReconcileTarget, error) {
if clientID == "" {
return nil, errors.New("client ID is required")
}
rows, err := store.db.QueryContext(ctx, `SELECT issue_uuid, last_event_seq, revision, immutable_request_sha256
FROM commands WHERE client_id = ? AND lifecycle BETWEEN 1 AND 4 ORDER BY issue_time, issue_uuid`, clientID)
query := `SELECT issue_uuid, last_event_seq, revision, immutable_request_sha256
FROM commands WHERE client_id = ? AND lifecycle BETWEEN 1 AND 4`
args := []any{clientID}
if !receiptBoundary.IsZero() {
query += ` AND server_receipt_time <= ?`
args = append(args, receiptBoundary.UTC().UnixNano())
}
query += ` ORDER BY issue_time, issue_uuid`
rows, err := store.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
@@ -62,10 +77,19 @@ func (store *Store) ReconcileClientSnapshot(ctx context.Context, clientID string
// incidented. Retained non-terminal rows are retargeted to this generation so
// late events from the previous connection cannot advance the command.
func (store *Store) ReconcileClientSnapshotForSession(ctx context.Context, clientID string, generation uint64, snapshot *rvboxv1.ReconcileSnapshot) (*rvboxv1.ReconcileResult, error) {
return store.ReconcileClientSnapshotForSessionAt(ctx, clientID, generation, snapshot, time.Time{})
}
// ReconcileClientSnapshotForSessionAt reconciles exactly the server state that
// was included in the matching ReconcileRequest. Work admitted after the
// boundary is intentionally left for the dispatch loop once reconciliation
// completes; treating it as absent client evidence would lose a valid command
// during the Hello/reconcile race.
func (store *Store) ReconcileClientSnapshotForSessionAt(ctx context.Context, clientID string, generation uint64, snapshot *rvboxv1.ReconcileSnapshot, receiptBoundary time.Time) (*rvboxv1.ReconcileResult, error) {
if generation == 0 {
return nil, errors.New("session generation is required")
}
return store.reconcileClientSnapshot(ctx, clientID, generation, snapshot)
return store.reconcileClientSnapshotAt(ctx, clientID, generation, snapshot, receiptBoundary)
}
type reconcileServerRow struct {
@@ -78,6 +102,10 @@ type reconcileServerRow struct {
}
func (store *Store) reconcileClientSnapshot(ctx context.Context, clientID string, generation uint64, snapshot *rvboxv1.ReconcileSnapshot) (*rvboxv1.ReconcileResult, error) {
return store.reconcileClientSnapshotAt(ctx, clientID, generation, snapshot, time.Time{})
}
func (store *Store) reconcileClientSnapshotAt(ctx context.Context, clientID string, generation uint64, snapshot *rvboxv1.ReconcileSnapshot, receiptBoundary time.Time) (*rvboxv1.ReconcileResult, error) {
if clientID == "" {
return nil, errors.New("client ID is required")
}
@@ -112,7 +140,7 @@ func (store *Store) reconcileClientSnapshot(ctx context.Context, clientID string
return nil, err
}
defer tx.Rollback()
serverRows, tombstones, err := loadReconcileRows(ctx, tx, clientID)
serverRows, tombstones, err := loadReconcileRows(ctx, tx, clientID, receiptBoundary)
if err != nil {
store.writeMu.Unlock()
return nil, err
@@ -215,8 +243,14 @@ type reconcileIncident struct {
dataLoss bool
}
func loadReconcileRows(ctx context.Context, tx *sql.Tx, clientID string) (map[string]reconcileServerRow, map[string][]byte, error) {
rows, err := tx.QueryContext(ctx, `SELECT issue_uuid, lifecycle, revision, last_event_seq, immutable_request_sha256, target_session_generation FROM commands WHERE client_id = ?`, clientID)
func loadReconcileRows(ctx context.Context, tx *sql.Tx, clientID string, receiptBoundary time.Time) (map[string]reconcileServerRow, map[string][]byte, error) {
query := `SELECT issue_uuid, lifecycle, revision, last_event_seq, immutable_request_sha256, target_session_generation FROM commands WHERE client_id = ?`
args := []any{clientID}
if !receiptBoundary.IsZero() {
query += ` AND server_receipt_time <= ?`
args = append(args, receiptBoundary.UTC().UnixNano())
}
rows, err := tx.QueryContext(ctx, query, args...)
if err != nil {
return nil, nil, err
}
internal/server/store/reconcile_test.go
+25
View File
@@ -71,6 +71,31 @@ func TestReconcileSnapshotMutatesMissingAndRetargets_HP_SES_13(t *testing.T) {
}
}
func TestReconcileSessionBoundaryLeavesNewlyQueuedCommandForDispatch_HP_RECONCILE_08(t *testing.T) {
t.Parallel()
ctx := context.Background()
opened, err := Open(ctx, Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = opened.Close() })
boundary := time.Date(2026, time.September, 11, 12, 0, 0, 0, time.UTC)
if _, err := opened.RegisterClientSession(ctx, ClientRegistration{ClientID: "boundary-client", Platform: 3, Architecture: "amd64", DaemonVersion: "test", DaemonCWD: `C:\\`, SupportedShells: []byte{1}, ClientInstanceID: [16]byte{31}, SessionID: [16]byte{32}, ConnectedAt: boundary}); err != nil {
t.Fatal(err)
}
issue := mustReconcileIssue(t, "019c46f1-1d02-7000-8000-0000000000d1")
if _, err := opened.QueueCommand(ctx, QueueCommandInput{IssueUUID: issue, ClientID: "boundary-client", IssueTime: boundary.Add(time.Nanosecond), ReceiptTime: boundary.Add(time.Nanosecond), ImmutableSHA256: sha256.Sum256([]byte("new-after-reconcile-boundary")), ExecutionSpec: []byte("echo queued")}); err != nil {
t.Fatal(err)
}
if _, err := opened.ReconcileClientSnapshotForSessionAt(ctx, "boundary-client", 1, &rvboxv1.ReconcileSnapshot{}, boundary); err != nil {
t.Fatal(err)
}
view, err := opened.GetCommandView(ctx, "boundary-client", issue)
if err != nil || view.Lifecycle != uint32(rvboxv1.CommandLifecycle_COMMAND_QUEUED) {
t.Fatalf("post-boundary command = %#v, %v", view, err)
}
}
func mustReconcileIssue(t *testing.T, value string) domain.UUID {
t.Helper()
issue, err := domain.ParseUUIDv7(value)