fix(client): acknowledge signals after process exit
This commit is contained in:
1 parent
789342ab4f
commit
30e7bb8545
7 files changed
+465
-8
No files matched your search
@@ -1838,6 +1838,19 @@ was not authorized; otherwise retain the revisioned signal result and actual
|
|||||||
terminal outcome. The server maps control `request_id` to this revision/result
|
terminal outcome. The server maps control `request_id` to this revision/result
|
||||||
and never sends that request ID across the agent protocol.
|
and never sends that request ID across the agent protocol.
|
||||||
|
|
||||||
|
A signal may arrive after local exit or before a native process exists. Treat
|
||||||
|
that as a command-level negative `SignalResult`, not a failed session: retain
|
||||||
|
the requested revision/signal, set `accepted=false`, state `command is not
|
||||||
|
running`, and leave delivery-attempt/forced-termination flags false. Persist
|
||||||
|
and notify/flush this event through the normal command event sequence, including
|
||||||
|
after a terminal lifecycle, so the server can retire its durable signal intent.
|
||||||
|
Reject mismatched revisions and unsupported Windows signals the same way
|
||||||
|
without touching a process. Preserve the real terminal outcome. Malformed
|
||||||
|
messages, unknown durable owners, and storage failures remain explicit errors.
|
||||||
|
Test offline intent replay, process-exit races, acknowledgement after terminal,
|
||||||
|
continued fresh dispatch on the same connection, and another spool restart
|
||||||
|
without repeating the acknowledged intent or original execution.
|
||||||
|
|
||||||
### 6.5 Server session unit and integration tests
|
### 6.5 Server session unit and integration tests
|
||||||
|
|
||||||
Unit-test version selection, registry/generation transitions, takeover grant
|
Unit-test version selection, registry/generation transitions, takeover grant
|
||||||
|
|||||||
@@ -133,6 +133,13 @@ exact ignored `dist/` directory after preserving any evidence.
|
|||||||
protection. Inspect command retention and free space, allow terminal-age or
|
protection. Inspect command retention and free space, allow terminal-age or
|
||||||
quota rotation to reclaim eligible data, or enlarge the owned filesystem.
|
quota rotation to reclaim eligible data, or enlarge the owned filesystem.
|
||||||
Never manually remove live SQLite/WAL/segment files.
|
Never manually remove live SQLite/WAL/segment files.
|
||||||
|
- **Repeated `signal command ...: command is not running`:** early v1 clients
|
||||||
|
closed their session instead of returning a negative signal result when an
|
||||||
|
old queued/replayed signal reached an already-exited process. Upgrade the
|
||||||
|
Windows client with the corrected installer, preserving TOML/spool/history.
|
||||||
|
The client records `accepted=false` for that signal; the existing server
|
||||||
|
acknowledges it and stops replaying it. Do not wipe state or resend the
|
||||||
|
original command to clear this loop.
|
||||||
- **Output truncated:** query command status and output history for its explicit
|
- **Output truncated:** query command status and output history for its explicit
|
||||||
loss/truncation markers. The command may still have completed correctly.
|
loss/truncation markers. The command may still have completed correctly.
|
||||||
- **Server restart / dirty health:** wait for readiness and inspect the
|
- **Server restart / dirty health:** wait for readiness and inspect the
|
||||||
|
|||||||
@@ -188,6 +188,24 @@ loss marker plus terminal result, restart the same client spool, discard the
|
|||||||
server-confirmed terminal row, and complete a fresh command. It also requires
|
server-confirmed terminal row, and complete a fresh command. It also requires
|
||||||
prompt server-session cleanup rather than waiting for a heartbeat tick.
|
prompt server-session cleanup rather than waiting for a heartbeat tick.
|
||||||
|
|
||||||
|
`BH-RUNTIME-09..11` and `HP-RUNTIME-05` exercise signal requests after process
|
||||||
|
exit, accepted work with no process, terminal state before active-map removal,
|
||||||
|
revision mismatches, unsupported Windows signals, actual TERM/KILL outcomes,
|
||||||
|
and supervisor failure. Expected command-level rejection must persist a typed
|
||||||
|
`SignalResult` and notify the sender without returning a session-fatal error.
|
||||||
|
Do not claim graceful/forced delivery when no process signal was attempted.
|
||||||
|
Malformed requests, unknown command ownership, and store failures still fail
|
||||||
|
explicitly; the fix is not a blanket suppression of callback errors.
|
||||||
|
|
||||||
|
`REC-RUNTIME-05` uses actual WebSockets and durable stores. Persist an active
|
||||||
|
command, disconnect, queue a signal while offline, reopen the same client spool,
|
||||||
|
and finish the process locally just as the pending signal arrives. Require the
|
||||||
|
actual executor's negative result to acknowledge the server intent after the
|
||||||
|
terminal event, then complete fresh work on that same session. Reopen/reconnect
|
||||||
|
again and require more fresh work with no replay of the acknowledged signal or
|
||||||
|
original execution. This reproduces the prior endless `signal command ...:
|
||||||
|
command is not running` reconnect loop without needing a Windows OS fixture.
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker compose -f deploy/compose.yaml run --rm toolchain \
|
docker compose -f deploy/compose.yaml run --rm toolchain \
|
||||||
go test -race ./internal/client/agent ./test/integration/clientagent
|
go test -race ./internal/client/agent ./test/integration/clientagent
|
||||||
|
|||||||
@@ -324,24 +324,37 @@ func (executor *Executor) appendStdinAck(ctx context.Context, issue domain.UUID,
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (executor *Executor) Signal(ctx context.Context, _ Session, input *rvboxv1.SignalCommand) error {
|
func (executor *Executor) Signal(ctx context.Context, _ Session, input *rvboxv1.SignalCommand) error {
|
||||||
if input == nil {
|
if input == nil || input.GetCommandRevision() == 0 || input.GetSignal() < rvboxv1.SignalKind_SIGNAL_HUP || input.GetSignal() > rvboxv1.SignalKind_SIGNAL_USR2 {
|
||||||
return errors.New("missing signal request")
|
return errors.New("invalid signal request")
|
||||||
}
|
}
|
||||||
issue, err := domain.ParseUUIDv7(input.GetIssueUuid())
|
issue, err := domain.ParseUUIDv7(input.GetIssueUuid())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
command, err := executor.Store.GetCommand(ctx, issue)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
reject := func(detail string) error {
|
||||||
|
return executor.appendSignalResult(ctx, issue, input.GetCommandRevision(), input.GetSignal(), false, false, supervisor.SignalOutcome{}, errors.New(detail))
|
||||||
|
}
|
||||||
|
if command.Revision != input.GetCommandRevision() {
|
||||||
|
return reject("signal revision does not match command")
|
||||||
|
}
|
||||||
executor.mu.Lock()
|
executor.mu.Lock()
|
||||||
process := executor.active[issue]
|
process := executor.active[issue]
|
||||||
executor.mu.Unlock()
|
executor.mu.Unlock()
|
||||||
if process == nil {
|
if process == nil || command.Terminal {
|
||||||
return errors.New("command is not running")
|
// A process can exit while its signal is queued or replayed. Persist a
|
||||||
|
// command-local negative result so the server can retire the intent;
|
||||||
|
// returning an error here would close/reconnect and replay it forever.
|
||||||
|
return reject("command is not running")
|
||||||
}
|
}
|
||||||
kind := supervisor.SignalTerm
|
kind := supervisor.SignalTerm
|
||||||
if input.GetSignal() == rvboxv1.SignalKind_SIGNAL_KILL {
|
if input.GetSignal() == rvboxv1.SignalKind_SIGNAL_KILL {
|
||||||
kind = supervisor.SignalKill
|
kind = supervisor.SignalKill
|
||||||
} else if input.GetSignal() != rvboxv1.SignalKind_SIGNAL_TERM {
|
} else if input.GetSignal() != rvboxv1.SignalKind_SIGNAL_TERM {
|
||||||
return errors.New("unsupported Windows signal")
|
return reject("unsupported Windows signal")
|
||||||
}
|
}
|
||||||
outcome, err := executor.Supervisor.Signal(ctx, process, kind)
|
outcome, err := executor.Supervisor.Signal(ctx, process, kind)
|
||||||
if kind == supervisor.SignalKill || outcome.Escalated {
|
if kind == supervisor.SignalKill || outcome.Escalated {
|
||||||
@@ -349,15 +362,15 @@ func (executor *Executor) Signal(ctx context.Context, _ Session, input *rvboxv1.
|
|||||||
executor.forced[issue] = true
|
executor.forced[issue] = true
|
||||||
executor.mu.Unlock()
|
executor.mu.Unlock()
|
||||||
}
|
}
|
||||||
return executor.appendSignalResult(ctx, issue, input.GetCommandRevision(), input.GetSignal(), err == nil && outcome.Delivered, outcome, err)
|
return executor.appendSignalResult(ctx, issue, input.GetCommandRevision(), input.GetSignal(), err == nil && outcome.Delivered, true, outcome, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (executor *Executor) appendSignalResult(ctx context.Context, issue domain.UUID, revision uint64, signal rvboxv1.SignalKind, accepted bool, outcome supervisor.SignalOutcome, cause error) error {
|
func (executor *Executor) appendSignalResult(ctx context.Context, issue domain.UUID, revision uint64, signal rvboxv1.SignalKind, accepted, attempted bool, outcome supervisor.SignalOutcome, cause error) error {
|
||||||
detail := outcome.Detail
|
detail := outcome.Detail
|
||||||
if cause != nil {
|
if cause != nil {
|
||||||
detail = boundedError(cause)
|
detail = boundedError(cause)
|
||||||
}
|
}
|
||||||
payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(&rvboxv1.CommandEvent{IssueUuid: issue.String(), ObservedAt: timestamppb.New(executor.Now()), Payload: &rvboxv1.CommandEvent_SignalResult{SignalResult: &rvboxv1.SignalResult{Signal: signal, Accepted: accepted, GracefulDeliveryAttempted: signal == rvboxv1.SignalKind_SIGNAL_TERM, ForcedTerminationUsed: outcome.Escalated || signal == rvboxv1.SignalKind_SIGNAL_KILL, Detail: detail, CommandRevision: revision}}})
|
payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(&rvboxv1.CommandEvent{IssueUuid: issue.String(), ObservedAt: timestamppb.New(executor.Now()), Payload: &rvboxv1.CommandEvent_SignalResult{SignalResult: &rvboxv1.SignalResult{Signal: signal, Accepted: accepted, GracefulDeliveryAttempted: attempted && signal == rvboxv1.SignalKind_SIGNAL_TERM, ForcedTerminationUsed: attempted && (outcome.Escalated || signal == rvboxv1.SignalKind_SIGNAL_KILL), Detail: detail, CommandRevision: revision}}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,162 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||||
|
"github.com/rvbox/rvbox/internal/agentproto"
|
||||||
|
"github.com/rvbox/rvbox/internal/client/spool"
|
||||||
|
"github.com/rvbox/rvbox/internal/client/supervisor"
|
||||||
|
"github.com/rvbox/rvbox/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
type signalProbeSupervisor struct {
|
||||||
|
supervisor.Supervisor
|
||||||
|
calls int
|
||||||
|
outcome supervisor.SignalOutcome
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (probe *signalProbeSupervisor) Signal(context.Context, supervisor.Process, supervisor.SignalKind) (supervisor.SignalOutcome, error) {
|
||||||
|
probe.calls++
|
||||||
|
return probe.outcome, probe.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStoppedSignalIsDurablyRejected_BH_RUNTIME_09(t *testing.T) {
|
||||||
|
for _, phase := range []rvboxv1.CommandLifecycle{
|
||||||
|
rvboxv1.CommandLifecycle_COMMAND_ACCEPTED,
|
||||||
|
rvboxv1.CommandLifecycle_COMMAND_RUNNING,
|
||||||
|
rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
|
||||||
|
} {
|
||||||
|
for _, signal := range []rvboxv1.SignalKind{rvboxv1.SignalKind_SIGNAL_TERM, rvboxv1.SignalKind_SIGNAL_KILL} {
|
||||||
|
t.Run(phase.String()+"/"+signal.String(), func(t *testing.T) {
|
||||||
|
store, issue := recoveryStore(t)
|
||||||
|
if phase != rvboxv1.CommandLifecycle_COMMAND_RUNNING {
|
||||||
|
if phase == rvboxv1.CommandLifecycle_COMMAND_ACCEPTED {
|
||||||
|
// Use a separate accepted row rather than reversing its lifecycle.
|
||||||
|
issue, _ = domain.NewUUIDv7()
|
||||||
|
if _, err := store.AcceptCommand(context.Background(), spool.Command{IssueUUID: issue, Revision: 1, Phase: uint32(phase), ImmutableSHA256: [32]byte{1}}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
} else if _, err := store.AppendLifecycle(context.Background(), issue, uint32(phase), 1, "finished", time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
probe := &signalProbeSupervisor{}
|
||||||
|
notified := false
|
||||||
|
executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, Notify: func(domain.UUID) { notified = true }}
|
||||||
|
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: signal}); err != nil {
|
||||||
|
t.Fatalf("ordinary signal rejection broke session: %v", err)
|
||||||
|
}
|
||||||
|
result := lastSignalResult(t, store, issue)
|
||||||
|
if result.GetAccepted() || result.GetCommandRevision() != 1 || result.GetSignal() != signal || result.GetGracefulDeliveryAttempted() || result.GetForcedTerminationUsed() || !strings.Contains(result.GetDetail(), "not running") || probe.calls != 0 || !notified {
|
||||||
|
t.Fatalf("result=%v supervisor calls=%d notified=%v", result, probe.calls, notified)
|
||||||
|
}
|
||||||
|
command, err := store.GetCommand(context.Background(), issue)
|
||||||
|
if err != nil || command.Phase != uint32(phase) {
|
||||||
|
t.Fatalf("signal rejection rewrote lifecycle: %v %v", command, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func lastSignalResult(t *testing.T, store *spool.Store, issue domain.UUID) *rvboxv1.SignalResult {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := store.AssignSendWindow(context.Background(), issue, 128, 1<<20); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
events, err := store.PendingEvents(context.Background(), issue)
|
||||||
|
if err != nil || len(events) == 0 {
|
||||||
|
t.Fatalf("missing durable signal result: %v", err)
|
||||||
|
}
|
||||||
|
transport := &runnerTransport{}
|
||||||
|
if err := SendStoredEvent(context.Background(), transport, Session{ID: "session", Generation: 1}, events[len(events)-1], agentproto.DefaultLimits()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
frame, err := agentproto.DecodeEnvelope(transport.writes[0], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
|
||||||
|
if err != nil || frame.GetCommandEvent().GetSignalResult() == nil {
|
||||||
|
t.Fatalf("invalid signal result frame: %v %v", frame, err)
|
||||||
|
}
|
||||||
|
return frame.GetCommandEvent().GetSignalResult()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSignalRejectionsDoNotTouchProcess_BH_RUNTIME_10(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
revision uint64
|
||||||
|
signal rvboxv1.SignalKind
|
||||||
|
terminal bool
|
||||||
|
detail string
|
||||||
|
}{
|
||||||
|
{"stale revision", 2, rvboxv1.SignalKind_SIGNAL_KILL, false, "revision"},
|
||||||
|
{"unsupported Windows signal", 1, rvboxv1.SignalKind_SIGNAL_HUP, false, "unsupported"},
|
||||||
|
{"process exit before active-map removal", 1, rvboxv1.SignalKind_SIGNAL_KILL, true, "not running"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
store, issue := recoveryStore(t)
|
||||||
|
if tc.terminal {
|
||||||
|
if _, err := store.AppendLifecycle(context.Background(), issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "finished", time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
probe := &signalProbeSupervisor{}
|
||||||
|
executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, active: map[domain.UUID]supervisor.Process{issue: &failedCapture{}}, forced: make(map[domain.UUID]bool)}
|
||||||
|
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: tc.revision, Signal: tc.signal}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
result := lastSignalResult(t, store, issue)
|
||||||
|
if result.GetAccepted() || result.GetCommandRevision() != tc.revision || !strings.Contains(result.GetDetail(), tc.detail) || result.GetGracefulDeliveryAttempted() || result.GetForcedTerminationUsed() || probe.calls != 0 {
|
||||||
|
t.Fatalf("result=%v supervisor calls=%d", result, probe.calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSignalDeliveryAndFailureStayCommandLocal_HP_RUNTIME_05(t *testing.T) {
|
||||||
|
for _, signal := range []rvboxv1.SignalKind{rvboxv1.SignalKind_SIGNAL_TERM, rvboxv1.SignalKind_SIGNAL_KILL} {
|
||||||
|
for _, delivered := range []bool{true, false} {
|
||||||
|
store, issue := recoveryStore(t)
|
||||||
|
probe := &signalProbeSupervisor{outcome: supervisor.SignalOutcome{Delivered: delivered}}
|
||||||
|
if !delivered {
|
||||||
|
probe.err = errors.New("process exited during signal")
|
||||||
|
}
|
||||||
|
executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, active: map[domain.UUID]supervisor.Process{issue: &failedCapture{}}, forced: make(map[domain.UUID]bool)}
|
||||||
|
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: signal}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
result := lastSignalResult(t, store, issue)
|
||||||
|
if result.GetAccepted() != delivered || result.GetGracefulDeliveryAttempted() != (signal == rvboxv1.SignalKind_SIGNAL_TERM) || probe.calls != 1 {
|
||||||
|
t.Fatalf("result=%v supervisor calls=%d", result, probe.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSignalProtocolAndStoreFailuresRemainErrors_BH_RUNTIME_11(t *testing.T) {
|
||||||
|
store, issue := recoveryStore(t)
|
||||||
|
executor := &Executor{Store: store, Now: time.Now}
|
||||||
|
for _, input := range []*rvboxv1.SignalCommand{
|
||||||
|
nil,
|
||||||
|
{IssueUuid: issue.String(), Signal: rvboxv1.SignalKind_SIGNAL_KILL},
|
||||||
|
{IssueUuid: "invalid", CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL},
|
||||||
|
{IssueUuid: issue.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KIND_UNSPECIFIED},
|
||||||
|
} {
|
||||||
|
if err := executor.Signal(context.Background(), Session{}, input); err == nil {
|
||||||
|
t.Fatalf("malformed signal was silently accepted: %v", input)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
unknown, _ := domain.NewUUIDv7()
|
||||||
|
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: unknown.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL}); !errors.Is(err, spool.ErrUnknownCommand) {
|
||||||
|
t.Fatalf("unknown owner error lost: %v", err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if err := executor.Signal(ctx, Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL}); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("cancelled durable query was reported as successful signal rejection: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,35 @@
|
|||||||
version = 1
|
version = 1
|
||||||
|
|
||||||
|
[[requirements]]
|
||||||
|
id = "BH-RUNTIME-09"
|
||||||
|
layer = "unit"
|
||||||
|
status = "implemented"
|
||||||
|
tests = ["internal/client/agent/signal_recovery_test.go:TestStoppedSignalIsDurablyRejected_BH_RUNTIME_09"]
|
||||||
|
|
||||||
|
[[requirements]]
|
||||||
|
id = "BH-RUNTIME-10"
|
||||||
|
layer = "unit"
|
||||||
|
status = "implemented"
|
||||||
|
tests = ["internal/client/agent/signal_recovery_test.go:TestSignalRejectionsDoNotTouchProcess_BH_RUNTIME_10"]
|
||||||
|
|
||||||
|
[[requirements]]
|
||||||
|
id = "BH-RUNTIME-11"
|
||||||
|
layer = "unit"
|
||||||
|
status = "implemented"
|
||||||
|
tests = ["internal/client/agent/signal_recovery_test.go:TestSignalProtocolAndStoreFailuresRemainErrors_BH_RUNTIME_11"]
|
||||||
|
|
||||||
|
[[requirements]]
|
||||||
|
id = "HP-RUNTIME-05"
|
||||||
|
layer = "unit"
|
||||||
|
status = "implemented"
|
||||||
|
tests = ["internal/client/agent/signal_recovery_test.go:TestSignalDeliveryAndFailureStayCommandLocal_HP_RUNTIME_05"]
|
||||||
|
|
||||||
|
[[requirements]]
|
||||||
|
id = "REC-RUNTIME-05"
|
||||||
|
layer = "integration"
|
||||||
|
status = "implemented"
|
||||||
|
tests = ["test/integration/clientagent/signal_recovery_integration_test.go:TestPendingSignalAfterExitSurvivesReconnect_REC_RUNTIME_05"]
|
||||||
|
|
||||||
[[requirements]]
|
[[requirements]]
|
||||||
id = "HP-WINCLI-10"
|
id = "HP-WINCLI-10"
|
||||||
layer = "unit"
|
layer = "unit"
|
||||||
|
|||||||
@@ -0,0 +1,214 @@
|
|||||||
|
package clientagent_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http/httptest"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||||
|
"github.com/rvbox/rvbox/internal/agentproto"
|
||||||
|
"github.com/rvbox/rvbox/internal/client/agent"
|
||||||
|
"github.com/rvbox/rvbox/internal/client/spool"
|
||||||
|
"github.com/rvbox/rvbox/internal/domain"
|
||||||
|
"github.com/rvbox/rvbox/internal/server/control"
|
||||||
|
"github.com/rvbox/rvbox/internal/server/session"
|
||||||
|
"github.com/rvbox/rvbox/internal/server/store"
|
||||||
|
"google.golang.org/protobuf/proto"
|
||||||
|
"google.golang.org/protobuf/types/known/timestamppb"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPendingSignalAfterExitSurvivesReconnect_REC_RUNTIME_05(t *testing.T) {
|
||||||
|
ctx, cancelAll := context.WithTimeout(context.Background(), 20*time.Second)
|
||||||
|
defer cancelAll()
|
||||||
|
persistence, err := store.Open(ctx, store.Options{DataDir: filepath.Join(t.TempDir(), "server"), BusyTimeout: time.Second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer persistence.Close()
|
||||||
|
registry := session.NewRegistry()
|
||||||
|
server := httptest.NewServer(&session.AgentServer{Store: persistence, Registry: registry, Path: "/v1/agent", Limits: agentproto.DefaultLimits()})
|
||||||
|
defer server.Close()
|
||||||
|
controlService, err := control.NewService(control.Options{Store: persistence, WakeClient: registry.Wake})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
directory := filepath.Join(t.TempDir(), "client")
|
||||||
|
client, err := spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = client.Close() }()
|
||||||
|
first, err := domain.NewUUIDv7()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hello := &rvboxv1.ClientHello{ClientId: "signal-recovery-client", ClientInstanceId: client.ClientInstanceID().String(), SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1}, DaemonVersion: "signal-recovery-test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS, Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_CMD}, MaxRunningCommands: 16, MaxQueuedCommands: 100, SentAt: timestamppb.Now()}
|
||||||
|
var signalCalls, originalDispatches atomic.Int32
|
||||||
|
start := func() (context.CancelFunc, <-chan error) {
|
||||||
|
runCtx, cancel := context.WithCancel(ctx)
|
||||||
|
active, done := make(chan struct{}, 1), make(chan error, 1)
|
||||||
|
executor := &agent.Executor{Store: client, Now: time.Now} // Restarted: no native process remains.
|
||||||
|
go func() {
|
||||||
|
done <- agent.RunOnce(runCtx, agent.RunnerOptions{
|
||||||
|
Store: client, Hello: hello, Limits: agentproto.DefaultLimits(), Now: time.Now,
|
||||||
|
Backoff: agent.BackoffOptions{Initial: time.Millisecond, Maximum: time.Millisecond, StableReset: time.Second}, Jitter: func(time.Duration) time.Duration { return 0 },
|
||||||
|
Dial: func(ctx context.Context) (agent.Transport, error) {
|
||||||
|
return agent.DialWebSocket(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/v1/agent", nil)
|
||||||
|
},
|
||||||
|
OnSessionActive: func(value bool) {
|
||||||
|
if value {
|
||||||
|
active <- struct{}{}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
OnDispatch: func(ctx context.Context, _ agent.Session, dispatch *rvboxv1.CommandDispatch) error {
|
||||||
|
issue, err := domain.ParseUUIDv7(dispatch.GetIssueUuid())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), 1, "fixture running", time.Now()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if issue == first {
|
||||||
|
originalDispatches.Add(1)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
_, err = client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "fixture finished", time.Now())
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
OnSignal: func(ctx context.Context, accepted agent.Session, signal *rvboxv1.SignalCommand) error {
|
||||||
|
signalCalls.Add(1)
|
||||||
|
// The old intent was persisted before reconnect. The process
|
||||||
|
// finishes locally just as that intent arrives, before its
|
||||||
|
// terminal event can reach the server.
|
||||||
|
if _, err := client.AppendLifecycle(ctx, first, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "exit won signal race", time.Now()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return executor.Signal(ctx, accepted, signal)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-active:
|
||||||
|
case err := <-done:
|
||||||
|
cancel()
|
||||||
|
t.Fatalf("client failed before active: %v", err)
|
||||||
|
case <-ctx.Done():
|
||||||
|
cancel()
|
||||||
|
t.Fatal("client did not become active")
|
||||||
|
}
|
||||||
|
return cancel, done
|
||||||
|
}
|
||||||
|
await := func(done <-chan error, description string, condition func() bool) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.NewTimer(5 * time.Second)
|
||||||
|
defer deadline.Stop()
|
||||||
|
for {
|
||||||
|
if condition() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("%s: session exited: %v", description, err)
|
||||||
|
case <-deadline.C:
|
||||||
|
t.Fatalf("timed out: %s", description)
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal(ctx.Err())
|
||||||
|
case <-time.After(5 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stop := func(cancel context.CancelFunc, done <-chan error) {
|
||||||
|
t.Helper()
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("client did not stop")
|
||||||
|
}
|
||||||
|
await(nil, "server session closed", func() bool {
|
||||||
|
view, err := persistence.GetClientView(ctx, hello.ClientId)
|
||||||
|
return err == nil && !view.Connected
|
||||||
|
})
|
||||||
|
}
|
||||||
|
queue := func(issue domain.UUID) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := controlService.RunCommand(ctx, &rvboxv1.RunCommandRequest{TargetClientId: hello.ClientId, RequestId: issue.String(), Spec: &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "echo fixture"}}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
phase := func(issue domain.UUID, want rvboxv1.CommandLifecycle) bool {
|
||||||
|
response, err := controlService.GetCommand(ctx, &rvboxv1.GetCommandRequest{ClientId: hello.ClientId, IssueUuid: issue.String()})
|
||||||
|
return err == nil && response.GetCommand().GetLifecycle() == want
|
||||||
|
}
|
||||||
|
cancel, done := start()
|
||||||
|
defer cancel()
|
||||||
|
queue(first)
|
||||||
|
await(done, "original command running", func() bool { return phase(first, rvboxv1.CommandLifecycle_COMMAND_RUNNING) })
|
||||||
|
stop(cancel, done)
|
||||||
|
requestID, _ := domain.NewUUIDv7()
|
||||||
|
if _, err := controlService.SignalCommand(ctx, &rvboxv1.ControlSignalCommandRequest{ClientId: hello.ClientId, IssueUuid: first.String(), RequestId: requestID.String(), Signal: rvboxv1.SignalKind_SIGNAL_KILL}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := client.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client, err = spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cancel, done = start()
|
||||||
|
defer cancel()
|
||||||
|
await(done, "negative signal result acknowledged", func() bool {
|
||||||
|
var acknowledged int
|
||||||
|
err := persistence.DB().QueryRowContext(ctx, `SELECT acknowledged FROM signal_intents WHERE request_uuid = ?`, requestID[:]).Scan(&acknowledged)
|
||||||
|
return err == nil && acknowledged == 1
|
||||||
|
})
|
||||||
|
if !phase(first, rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED) || signalCalls.Load() != 1 || originalDispatches.Load() != 1 {
|
||||||
|
t.Fatalf("original command replayed or terminal result lost: signals=%d dispatches=%d", signalCalls.Load(), originalDispatches.Load())
|
||||||
|
}
|
||||||
|
events, err := persistence.ReadCommandEvents(ctx, first, 0, 100)
|
||||||
|
if err != nil || len(events) != 3 {
|
||||||
|
t.Fatalf("events=%d err=%v", len(events), err)
|
||||||
|
}
|
||||||
|
payload, err := store.DecodeEventPayload(events[2])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var event rvboxv1.CommandEvent
|
||||||
|
if err := proto.Unmarshal(payload, &event); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
result := event.GetSignalResult()
|
||||||
|
if result == nil || result.GetAccepted() || result.GetForcedTerminationUsed() || result.GetGracefulDeliveryAttempted() || result.GetCommandRevision() != 1 || !strings.Contains(result.GetDetail(), "not running") {
|
||||||
|
t.Fatalf("lost negative signal evidence: %v", result)
|
||||||
|
}
|
||||||
|
for attempt := 0; attempt < 2; attempt++ {
|
||||||
|
next, _ := domain.NewUUIDv7()
|
||||||
|
queue(next)
|
||||||
|
await(done, "fresh command succeeds without reconnect", func() bool { return phase(next, rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED) })
|
||||||
|
if attempt == 0 {
|
||||||
|
stop(cancel, done)
|
||||||
|
if err := client.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client, err = spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cancel, done = start()
|
||||||
|
defer cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if signalCalls.Load() != 1 || originalDispatches.Load() != 1 {
|
||||||
|
t.Fatal("acknowledged signal or original command replayed after another restart")
|
||||||
|
}
|
||||||
|
stop(cancel, done)
|
||||||
|
if _, err := client.Check(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user