fix(client): acknowledge signals after process exit

This commit is contained in:
cabbage committed 2026-10-27 16:18:32 +00:00
1 parent 789342ab4f
commit 30e7bb8545
7 files changed
+465 -8

No files matched your search

@@ -0,0 +1,214 @@
package clientagent_test
import (
"context"
"net/http/httptest"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/client/agent"
"github.com/rvbox/rvbox/internal/client/spool"
"github.com/rvbox/rvbox/internal/domain"
"github.com/rvbox/rvbox/internal/server/control"
"github.com/rvbox/rvbox/internal/server/session"
"github.com/rvbox/rvbox/internal/server/store"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
func TestPendingSignalAfterExitSurvivesReconnect_REC_RUNTIME_05(t *testing.T) {
ctx, cancelAll := context.WithTimeout(context.Background(), 20*time.Second)
defer cancelAll()
persistence, err := store.Open(ctx, store.Options{DataDir: filepath.Join(t.TempDir(), "server"), BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
defer persistence.Close()
registry := session.NewRegistry()
server := httptest.NewServer(&session.AgentServer{Store: persistence, Registry: registry, Path: "/v1/agent", Limits: agentproto.DefaultLimits()})
defer server.Close()
controlService, err := control.NewService(control.Options{Store: persistence, WakeClient: registry.Wake})
if err != nil {
t.Fatal(err)
}
directory := filepath.Join(t.TempDir(), "client")
client, err := spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
defer func() { _ = client.Close() }()
first, err := domain.NewUUIDv7()
if err != nil {
t.Fatal(err)
}
hello := &rvboxv1.ClientHello{ClientId: "signal-recovery-client", ClientInstanceId: client.ClientInstanceID().String(), SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1}, DaemonVersion: "signal-recovery-test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS, Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_CMD}, MaxRunningCommands: 16, MaxQueuedCommands: 100, SentAt: timestamppb.Now()}
var signalCalls, originalDispatches atomic.Int32
start := func() (context.CancelFunc, <-chan error) {
runCtx, cancel := context.WithCancel(ctx)
active, done := make(chan struct{}, 1), make(chan error, 1)
executor := &agent.Executor{Store: client, Now: time.Now} // Restarted: no native process remains.
go func() {
done <- agent.RunOnce(runCtx, agent.RunnerOptions{
Store: client, Hello: hello, Limits: agentproto.DefaultLimits(), Now: time.Now,
Backoff: agent.BackoffOptions{Initial: time.Millisecond, Maximum: time.Millisecond, StableReset: time.Second}, Jitter: func(time.Duration) time.Duration { return 0 },
Dial: func(ctx context.Context) (agent.Transport, error) {
return agent.DialWebSocket(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/v1/agent", nil)
},
OnSessionActive: func(value bool) {
if value {
active <- struct{}{}
}
},
OnDispatch: func(ctx context.Context, _ agent.Session, dispatch *rvboxv1.CommandDispatch) error {
issue, err := domain.ParseUUIDv7(dispatch.GetIssueUuid())
if err != nil {
return err
}
if _, err := client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), 1, "fixture running", time.Now()); err != nil {
return err
}
if issue == first {
originalDispatches.Add(1)
return nil
}
_, err = client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "fixture finished", time.Now())
return err
},
OnSignal: func(ctx context.Context, accepted agent.Session, signal *rvboxv1.SignalCommand) error {
signalCalls.Add(1)
// The old intent was persisted before reconnect. The process
// finishes locally just as that intent arrives, before its
// terminal event can reach the server.
if _, err := client.AppendLifecycle(ctx, first, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "exit won signal race", time.Now()); err != nil {
return err
}
return executor.Signal(ctx, accepted, signal)
},
})
close(done)
}()
select {
case <-active:
case err := <-done:
cancel()
t.Fatalf("client failed before active: %v", err)
case <-ctx.Done():
cancel()
t.Fatal("client did not become active")
}
return cancel, done
}
await := func(done <-chan error, description string, condition func() bool) {
t.Helper()
deadline := time.NewTimer(5 * time.Second)
defer deadline.Stop()
for {
if condition() {
return
}
select {
case err := <-done:
t.Fatalf("%s: session exited: %v", description, err)
case <-deadline.C:
t.Fatalf("timed out: %s", description)
case <-ctx.Done():
t.Fatal(ctx.Err())
case <-time.After(5 * time.Millisecond):
}
}
}
stop := func(cancel context.CancelFunc, done <-chan error) {
t.Helper()
cancel()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("client did not stop")
}
await(nil, "server session closed", func() bool {
view, err := persistence.GetClientView(ctx, hello.ClientId)
return err == nil && !view.Connected
})
}
queue := func(issue domain.UUID) {
t.Helper()
if _, err := controlService.RunCommand(ctx, &rvboxv1.RunCommandRequest{TargetClientId: hello.ClientId, RequestId: issue.String(), Spec: &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "echo fixture"}}}); err != nil {
t.Fatal(err)
}
}
phase := func(issue domain.UUID, want rvboxv1.CommandLifecycle) bool {
response, err := controlService.GetCommand(ctx, &rvboxv1.GetCommandRequest{ClientId: hello.ClientId, IssueUuid: issue.String()})
return err == nil && response.GetCommand().GetLifecycle() == want
}
cancel, done := start()
defer cancel()
queue(first)
await(done, "original command running", func() bool { return phase(first, rvboxv1.CommandLifecycle_COMMAND_RUNNING) })
stop(cancel, done)
requestID, _ := domain.NewUUIDv7()
if _, err := controlService.SignalCommand(ctx, &rvboxv1.ControlSignalCommandRequest{ClientId: hello.ClientId, IssueUuid: first.String(), RequestId: requestID.String(), Signal: rvboxv1.SignalKind_SIGNAL_KILL}); err != nil {
t.Fatal(err)
}
if err := client.Close(); err != nil {
t.Fatal(err)
}
client, err = spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
cancel, done = start()
defer cancel()
await(done, "negative signal result acknowledged", func() bool {
var acknowledged int
err := persistence.DB().QueryRowContext(ctx, `SELECT acknowledged FROM signal_intents WHERE request_uuid = ?`, requestID[:]).Scan(&acknowledged)
return err == nil && acknowledged == 1
})
if !phase(first, rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED) || signalCalls.Load() != 1 || originalDispatches.Load() != 1 {
t.Fatalf("original command replayed or terminal result lost: signals=%d dispatches=%d", signalCalls.Load(), originalDispatches.Load())
}
events, err := persistence.ReadCommandEvents(ctx, first, 0, 100)
if err != nil || len(events) != 3 {
t.Fatalf("events=%d err=%v", len(events), err)
}
payload, err := store.DecodeEventPayload(events[2])
if err != nil {
t.Fatal(err)
}
var event rvboxv1.CommandEvent
if err := proto.Unmarshal(payload, &event); err != nil {
t.Fatal(err)
}
result := event.GetSignalResult()
if result == nil || result.GetAccepted() || result.GetForcedTerminationUsed() || result.GetGracefulDeliveryAttempted() || result.GetCommandRevision() != 1 || !strings.Contains(result.GetDetail(), "not running") {
t.Fatalf("lost negative signal evidence: %v", result)
}
for attempt := 0; attempt < 2; attempt++ {
next, _ := domain.NewUUIDv7()
queue(next)
await(done, "fresh command succeeds without reconnect", func() bool { return phase(next, rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED) })
if attempt == 0 {
stop(cancel, done)
if err := client.Close(); err != nil {
t.Fatal(err)
}
client, err = spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
cancel, done = start()
defer cancel()
}
}
if signalCalls.Load() != 1 || originalDispatches.Load() != 1 {
t.Fatal("acknowledged signal or original command replayed after another restart")
}
stop(cancel, done)
if _, err := client.Check(ctx); err != nil {
t.Fatal(err)
}
}