fix(client): acknowledge signals after process exit
This commit is contained in:
1 parent
789342ab4f
commit
30e7bb8545
7 files changed
+465
-8
No files matched your search
@@ -0,0 +1,214 @@
|
||||
package clientagent_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||
"github.com/rvbox/rvbox/internal/agentproto"
|
||||
"github.com/rvbox/rvbox/internal/client/agent"
|
||||
"github.com/rvbox/rvbox/internal/client/spool"
|
||||
"github.com/rvbox/rvbox/internal/domain"
|
||||
"github.com/rvbox/rvbox/internal/server/control"
|
||||
"github.com/rvbox/rvbox/internal/server/session"
|
||||
"github.com/rvbox/rvbox/internal/server/store"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
func TestPendingSignalAfterExitSurvivesReconnect_REC_RUNTIME_05(t *testing.T) {
|
||||
ctx, cancelAll := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancelAll()
|
||||
persistence, err := store.Open(ctx, store.Options{DataDir: filepath.Join(t.TempDir(), "server"), BusyTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer persistence.Close()
|
||||
registry := session.NewRegistry()
|
||||
server := httptest.NewServer(&session.AgentServer{Store: persistence, Registry: registry, Path: "/v1/agent", Limits: agentproto.DefaultLimits()})
|
||||
defer server.Close()
|
||||
controlService, err := control.NewService(control.Options{Store: persistence, WakeClient: registry.Wake})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
directory := filepath.Join(t.TempDir(), "client")
|
||||
client, err := spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = client.Close() }()
|
||||
first, err := domain.NewUUIDv7()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hello := &rvboxv1.ClientHello{ClientId: "signal-recovery-client", ClientInstanceId: client.ClientInstanceID().String(), SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1}, DaemonVersion: "signal-recovery-test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS, Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_CMD}, MaxRunningCommands: 16, MaxQueuedCommands: 100, SentAt: timestamppb.Now()}
|
||||
var signalCalls, originalDispatches atomic.Int32
|
||||
start := func() (context.CancelFunc, <-chan error) {
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
active, done := make(chan struct{}, 1), make(chan error, 1)
|
||||
executor := &agent.Executor{Store: client, Now: time.Now} // Restarted: no native process remains.
|
||||
go func() {
|
||||
done <- agent.RunOnce(runCtx, agent.RunnerOptions{
|
||||
Store: client, Hello: hello, Limits: agentproto.DefaultLimits(), Now: time.Now,
|
||||
Backoff: agent.BackoffOptions{Initial: time.Millisecond, Maximum: time.Millisecond, StableReset: time.Second}, Jitter: func(time.Duration) time.Duration { return 0 },
|
||||
Dial: func(ctx context.Context) (agent.Transport, error) {
|
||||
return agent.DialWebSocket(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/v1/agent", nil)
|
||||
},
|
||||
OnSessionActive: func(value bool) {
|
||||
if value {
|
||||
active <- struct{}{}
|
||||
}
|
||||
},
|
||||
OnDispatch: func(ctx context.Context, _ agent.Session, dispatch *rvboxv1.CommandDispatch) error {
|
||||
issue, err := domain.ParseUUIDv7(dispatch.GetIssueUuid())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), 1, "fixture running", time.Now()); err != nil {
|
||||
return err
|
||||
}
|
||||
if issue == first {
|
||||
originalDispatches.Add(1)
|
||||
return nil
|
||||
}
|
||||
_, err = client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "fixture finished", time.Now())
|
||||
return err
|
||||
},
|
||||
OnSignal: func(ctx context.Context, accepted agent.Session, signal *rvboxv1.SignalCommand) error {
|
||||
signalCalls.Add(1)
|
||||
// The old intent was persisted before reconnect. The process
|
||||
// finishes locally just as that intent arrives, before its
|
||||
// terminal event can reach the server.
|
||||
if _, err := client.AppendLifecycle(ctx, first, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "exit won signal race", time.Now()); err != nil {
|
||||
return err
|
||||
}
|
||||
return executor.Signal(ctx, accepted, signal)
|
||||
},
|
||||
})
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-active:
|
||||
case err := <-done:
|
||||
cancel()
|
||||
t.Fatalf("client failed before active: %v", err)
|
||||
case <-ctx.Done():
|
||||
cancel()
|
||||
t.Fatal("client did not become active")
|
||||
}
|
||||
return cancel, done
|
||||
}
|
||||
await := func(done <-chan error, description string, condition func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.NewTimer(5 * time.Second)
|
||||
defer deadline.Stop()
|
||||
for {
|
||||
if condition() {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("%s: session exited: %v", description, err)
|
||||
case <-deadline.C:
|
||||
t.Fatalf("timed out: %s", description)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
case <-time.After(5 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
stop := func(cancel context.CancelFunc, done <-chan error) {
|
||||
t.Helper()
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("client did not stop")
|
||||
}
|
||||
await(nil, "server session closed", func() bool {
|
||||
view, err := persistence.GetClientView(ctx, hello.ClientId)
|
||||
return err == nil && !view.Connected
|
||||
})
|
||||
}
|
||||
queue := func(issue domain.UUID) {
|
||||
t.Helper()
|
||||
if _, err := controlService.RunCommand(ctx, &rvboxv1.RunCommandRequest{TargetClientId: hello.ClientId, RequestId: issue.String(), Spec: &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "echo fixture"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
phase := func(issue domain.UUID, want rvboxv1.CommandLifecycle) bool {
|
||||
response, err := controlService.GetCommand(ctx, &rvboxv1.GetCommandRequest{ClientId: hello.ClientId, IssueUuid: issue.String()})
|
||||
return err == nil && response.GetCommand().GetLifecycle() == want
|
||||
}
|
||||
cancel, done := start()
|
||||
defer cancel()
|
||||
queue(first)
|
||||
await(done, "original command running", func() bool { return phase(first, rvboxv1.CommandLifecycle_COMMAND_RUNNING) })
|
||||
stop(cancel, done)
|
||||
requestID, _ := domain.NewUUIDv7()
|
||||
if _, err := controlService.SignalCommand(ctx, &rvboxv1.ControlSignalCommandRequest{ClientId: hello.ClientId, IssueUuid: first.String(), RequestId: requestID.String(), Signal: rvboxv1.SignalKind_SIGNAL_KILL}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := client.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err = spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel, done = start()
|
||||
defer cancel()
|
||||
await(done, "negative signal result acknowledged", func() bool {
|
||||
var acknowledged int
|
||||
err := persistence.DB().QueryRowContext(ctx, `SELECT acknowledged FROM signal_intents WHERE request_uuid = ?`, requestID[:]).Scan(&acknowledged)
|
||||
return err == nil && acknowledged == 1
|
||||
})
|
||||
if !phase(first, rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED) || signalCalls.Load() != 1 || originalDispatches.Load() != 1 {
|
||||
t.Fatalf("original command replayed or terminal result lost: signals=%d dispatches=%d", signalCalls.Load(), originalDispatches.Load())
|
||||
}
|
||||
events, err := persistence.ReadCommandEvents(ctx, first, 0, 100)
|
||||
if err != nil || len(events) != 3 {
|
||||
t.Fatalf("events=%d err=%v", len(events), err)
|
||||
}
|
||||
payload, err := store.DecodeEventPayload(events[2])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var event rvboxv1.CommandEvent
|
||||
if err := proto.Unmarshal(payload, &event); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := event.GetSignalResult()
|
||||
if result == nil || result.GetAccepted() || result.GetForcedTerminationUsed() || result.GetGracefulDeliveryAttempted() || result.GetCommandRevision() != 1 || !strings.Contains(result.GetDetail(), "not running") {
|
||||
t.Fatalf("lost negative signal evidence: %v", result)
|
||||
}
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
next, _ := domain.NewUUIDv7()
|
||||
queue(next)
|
||||
await(done, "fresh command succeeds without reconnect", func() bool { return phase(next, rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED) })
|
||||
if attempt == 0 {
|
||||
stop(cancel, done)
|
||||
if err := client.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err = spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel, done = start()
|
||||
defer cancel()
|
||||
}
|
||||
}
|
||||
if signalCalls.Load() != 1 || originalDispatches.Load() != 1 {
|
||||
t.Fatal("acknowledged signal or original command replayed after another restart")
|
||||
}
|
||||
stop(cancel, done)
|
||||
if _, err := client.Check(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user