fix: fence reconciliation admission race
This commit is contained in:
@@ -102,6 +102,10 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
|
||||
return
|
||||
}
|
||||
hello := helloEnvelope.GetClientHello()
|
||||
// This cutoff precedes registration, which is when control callers can
|
||||
// first observe the live client. Reconciliation must compare only the
|
||||
// state included in its request; later control commands are fresh work.
|
||||
reconcileBoundary := server.now()
|
||||
instanceID, err := domain.ParseUUIDv7(hello.GetClientInstanceId())
|
||||
if err != nil {
|
||||
server.close(connection, websocket.StatusPolicyViolation, "invalid client instance ID")
|
||||
@@ -192,7 +196,7 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
|
||||
server.close(connection, websocket.StatusInternalError, "could not queue welcome")
|
||||
return
|
||||
}
|
||||
targets, err := server.Store.ReconcileTargets(parent, hello.GetClientId())
|
||||
targets, err := server.Store.ReconcileTargetsAt(parent, hello.GetClientId(), reconcileBoundary)
|
||||
if err != nil {
|
||||
server.close(connection, websocket.StatusInternalError, "could not build reconciliation request")
|
||||
return
|
||||
@@ -237,7 +241,7 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
|
||||
return
|
||||
}
|
||||
if snapshot := envelope.GetReconcileSnapshot(); snapshot != nil {
|
||||
result, reconcileErr := server.Store.ReconcileClientSnapshotForSession(sessionContext, hello.GetClientId(), registration.Generation, snapshot)
|
||||
result, reconcileErr := server.Store.ReconcileClientSnapshotForSessionAt(sessionContext, hello.GetClientId(), registration.Generation, snapshot, reconcileBoundary)
|
||||
if reconcileErr != nil {
|
||||
server.close(connection, websocket.StatusPolicyViolation, "reconciliation failed")
|
||||
return
|
||||
|
||||
@@ -98,6 +98,57 @@ func TestAgentServerRegistrationAndReplacement_HP_SES_05(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentServerDispatchesCommandAdmittedDuringReconcile_HP_SES_11(t *testing.T) {
|
||||
server, agent, cleanup := newTestAgentServerWithStore(t)
|
||||
defer cleanup()
|
||||
instance, err := domain.NewUUIDv7()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
connection, welcome := dialAndHello(t, server, "client-race", instance.String())
|
||||
defer connection.CloseNow()
|
||||
issue, err := domain.NewUUIDv7()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
spec, err := proto.Marshal(&rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "echo dispatch"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if _, err := agent.Store.QueueCommand(context.Background(), store.QueueCommandInput{IssueUUID: issue, ClientID: "client-race", IssueTime: now, ReceiptTime: now, ImmutableSHA256: sha256.Sum256([]byte("during-reconcile")), ExecutionSpec: spec}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snapshot, err := proto.Marshal(&rvboxv1.AgentEnvelope{
|
||||
SessionId: welcome.GetSessionId(), SessionGeneration: welcome.GetSessionGeneration(),
|
||||
Payload: &rvboxv1.AgentEnvelope_ReconcileSnapshot{ReconcileSnapshot: &rvboxv1.ReconcileSnapshot{}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := connection.Write(context.Background(), websocket.MessageBinary, snapshot); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
readContext, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
for index := 0; index < 2; index++ {
|
||||
_, payload, err := connection.Read(readContext)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var envelope rvboxv1.AgentEnvelope
|
||||
if err := proto.Unmarshal(payload, &envelope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if index == 0 && envelope.GetReconcileResult() == nil {
|
||||
t.Fatalf("first post-snapshot envelope = %T, want reconcile result", envelope.Payload)
|
||||
}
|
||||
if index == 1 && (envelope.GetCommandDispatch() == nil || envelope.GetCommandDispatch().GetIssueUuid() != issue.String()) {
|
||||
t.Fatalf("second post-snapshot envelope = %+v, want dispatch %s", envelope.Payload, issue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentServerRejectsHostileWireInputs_BH_SES_04(t *testing.T) {
|
||||
server, cleanup := newTestAgentServer(t)
|
||||
defer cleanup()
|
||||
@@ -174,6 +225,11 @@ func TestWireEventAppendCarriesClientBinding_HP_EVENT_01(t *testing.T) {
|
||||
}
|
||||
|
||||
func newTestAgentServer(t *testing.T) (*httptest.Server, func()) {
|
||||
server, _, cleanup := newTestAgentServerWithStore(t)
|
||||
return server, cleanup
|
||||
}
|
||||
|
||||
func newTestAgentServerWithStore(t *testing.T) (*httptest.Server, *AgentServer, func()) {
|
||||
t.Helper()
|
||||
dataDirectory := filepath.Join(t.TempDir(), "store")
|
||||
if err := os.Mkdir(dataDirectory, 0o700); err != nil {
|
||||
@@ -190,7 +246,7 @@ func newTestAgentServer(t *testing.T) (*httptest.Server, func()) {
|
||||
HeartbeatIdle: time.Second, LivenessTimeout: 2 * time.Second,
|
||||
}
|
||||
httpServer := httptest.NewServer(agent)
|
||||
return httpServer, func() {
|
||||
return httpServer, agent, func() {
|
||||
httpServer.Close()
|
||||
if err := persistence.Close(); err != nil {
|
||||
t.Errorf("close persistence: %v", err)
|
||||
|
||||
Reference in New Issue
Block a user