fix(client): acknowledge signals after process exit

This commit is contained in:
cabbage committed 2026-10-27 16:18:32 +00:00
1 parent 789342ab4f
commit 30e7bb8545
7 files changed
+465 -8

No files matched your search

+13
View File
@@ -1838,6 +1838,19 @@ was not authorized; otherwise retain the revisioned signal result and actual
terminal outcome. The server maps control `request_id` to this revision/result
and never sends that request ID across the agent protocol.
A signal may arrive after local exit or before a native process exists. Treat
that as a command-level negative `SignalResult`, not a failed session: retain
the requested revision/signal, set `accepted=false`, state `command is not
running`, and leave delivery-attempt/forced-termination flags false. Persist
and notify/flush this event through the normal command event sequence, including
after a terminal lifecycle, so the server can retire its durable signal intent.
Reject mismatched revisions and unsupported Windows signals the same way
without touching a process. Preserve the real terminal outcome. Malformed
messages, unknown durable owners, and storage failures remain explicit errors.
Test offline intent replay, process-exit races, acknowledgement after terminal,
continued fresh dispatch on the same connection, and another spool restart
without repeating the acknowledged intent or original execution.
### 6.5 Server session unit and integration tests
Unit-test version selection, registry/generation transitions, takeover grant
+7
View File
@@ -133,6 +133,13 @@ exact ignored `dist/` directory after preserving any evidence.
protection. Inspect command retention and free space, allow terminal-age or
quota rotation to reclaim eligible data, or enlarge the owned filesystem.
Never manually remove live SQLite/WAL/segment files.
- **Repeated `signal command ...: command is not running`:** early v1 clients
closed their session instead of returning a negative signal result when an
old queued/replayed signal reached an already-exited process. Upgrade the
Windows client with the corrected installer, preserving TOML/spool/history.
The client records `accepted=false` for that signal; the existing server
acknowledges it and stops replaying it. Do not wipe state or resend the
original command to clear this loop.
- **Output truncated:** query command status and output history for its explicit
loss/truncation markers. The command may still have completed correctly.
- **Server restart / dirty health:** wait for readiness and inspect the
+18
View File
@@ -188,6 +188,24 @@ loss marker plus terminal result, restart the same client spool, discard the
server-confirmed terminal row, and complete a fresh command. It also requires
prompt server-session cleanup rather than waiting for a heartbeat tick.
`BH-RUNTIME-09..11` and `HP-RUNTIME-05` exercise signal requests after process
exit, accepted work with no process, terminal state before active-map removal,
revision mismatches, unsupported Windows signals, actual TERM/KILL outcomes,
and supervisor failure. Expected command-level rejection must persist a typed
`SignalResult` and notify the sender without returning a session-fatal error.
Do not claim graceful/forced delivery when no process signal was attempted.
Malformed requests, unknown command ownership, and store failures still fail
explicitly; the fix is not a blanket suppression of callback errors.
`REC-RUNTIME-05` uses actual WebSockets and durable stores. Persist an active
command, disconnect, queue a signal while offline, reopen the same client spool,
and finish the process locally just as the pending signal arrives. Require the
actual executor's negative result to acknowledge the server intent after the
terminal event, then complete fresh work on that same session. Reopen/reconnect
again and require more fresh work with no replay of the acknowledged signal or
original execution. This reproduces the prior endless `signal command ...:
command is not running` reconnect loop without needing a Windows OS fixture.
```sh
docker compose -f deploy/compose.yaml run --rm toolchain \
go test -race ./internal/client/agent ./test/integration/clientagent
+21 -8
View File
@@ -324,24 +324,37 @@ func (executor *Executor) appendStdinAck(ctx context.Context, issue domain.UUID,
}
func (executor *Executor) Signal(ctx context.Context, _ Session, input *rvboxv1.SignalCommand) error {
if input == nil {
return errors.New("missing signal request")
if input == nil || input.GetCommandRevision() == 0 || input.GetSignal() < rvboxv1.SignalKind_SIGNAL_HUP || input.GetSignal() > rvboxv1.SignalKind_SIGNAL_USR2 {
return errors.New("invalid signal request")
}
issue, err := domain.ParseUUIDv7(input.GetIssueUuid())
if err != nil {
return err
}
command, err := executor.Store.GetCommand(ctx, issue)
if err != nil {
return err
}
reject := func(detail string) error {
return executor.appendSignalResult(ctx, issue, input.GetCommandRevision(), input.GetSignal(), false, false, supervisor.SignalOutcome{}, errors.New(detail))
}
if command.Revision != input.GetCommandRevision() {
return reject("signal revision does not match command")
}
executor.mu.Lock()
process := executor.active[issue]
executor.mu.Unlock()
if process == nil {
return errors.New("command is not running")
if process == nil || command.Terminal {
// A process can exit while its signal is queued or replayed. Persist a
// command-local negative result so the server can retire the intent;
// returning an error here would close/reconnect and replay it forever.
return reject("command is not running")
}
kind := supervisor.SignalTerm
if input.GetSignal() == rvboxv1.SignalKind_SIGNAL_KILL {
kind = supervisor.SignalKill
} else if input.GetSignal() != rvboxv1.SignalKind_SIGNAL_TERM {
return errors.New("unsupported Windows signal")
return reject("unsupported Windows signal")
}
outcome, err := executor.Supervisor.Signal(ctx, process, kind)
if kind == supervisor.SignalKill || outcome.Escalated {
@@ -349,15 +362,15 @@ func (executor *Executor) Signal(ctx context.Context, _ Session, input *rvboxv1.
executor.forced[issue] = true
executor.mu.Unlock()
}
return executor.appendSignalResult(ctx, issue, input.GetCommandRevision(), input.GetSignal(), err == nil && outcome.Delivered, outcome, err)
return executor.appendSignalResult(ctx, issue, input.GetCommandRevision(), input.GetSignal(), err == nil && outcome.Delivered, true, outcome, err)
}
func (executor *Executor) appendSignalResult(ctx context.Context, issue domain.UUID, revision uint64, signal rvboxv1.SignalKind, accepted bool, outcome supervisor.SignalOutcome, cause error) error {
func (executor *Executor) appendSignalResult(ctx context.Context, issue domain.UUID, revision uint64, signal rvboxv1.SignalKind, accepted, attempted bool, outcome supervisor.SignalOutcome, cause error) error {
detail := outcome.Detail
if cause != nil {
detail = boundedError(cause)
}
payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(&rvboxv1.CommandEvent{IssueUuid: issue.String(), ObservedAt: timestamppb.New(executor.Now()), Payload: &rvboxv1.CommandEvent_SignalResult{SignalResult: &rvboxv1.SignalResult{Signal: signal, Accepted: accepted, GracefulDeliveryAttempted: signal == rvboxv1.SignalKind_SIGNAL_TERM, ForcedTerminationUsed: outcome.Escalated || signal == rvboxv1.SignalKind_SIGNAL_KILL, Detail: detail, CommandRevision: revision}}})
payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(&rvboxv1.CommandEvent{IssueUuid: issue.String(), ObservedAt: timestamppb.New(executor.Now()), Payload: &rvboxv1.CommandEvent_SignalResult{SignalResult: &rvboxv1.SignalResult{Signal: signal, Accepted: accepted, GracefulDeliveryAttempted: attempted && signal == rvboxv1.SignalKind_SIGNAL_TERM, ForcedTerminationUsed: attempted && (outcome.Escalated || signal == rvboxv1.SignalKind_SIGNAL_KILL), Detail: detail, CommandRevision: revision}}})
if err != nil {
return err
}
@@ -0,0 +1,162 @@
package agent
import (
"context"
"errors"
"strings"
"testing"
"time"
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/client/spool"
"github.com/rvbox/rvbox/internal/client/supervisor"
"github.com/rvbox/rvbox/internal/domain"
)
type signalProbeSupervisor struct {
supervisor.Supervisor
calls int
outcome supervisor.SignalOutcome
err error
}
func (probe *signalProbeSupervisor) Signal(context.Context, supervisor.Process, supervisor.SignalKind) (supervisor.SignalOutcome, error) {
probe.calls++
return probe.outcome, probe.err
}
func TestStoppedSignalIsDurablyRejected_BH_RUNTIME_09(t *testing.T) {
for _, phase := range []rvboxv1.CommandLifecycle{
rvboxv1.CommandLifecycle_COMMAND_ACCEPTED,
rvboxv1.CommandLifecycle_COMMAND_RUNNING,
rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
} {
for _, signal := range []rvboxv1.SignalKind{rvboxv1.SignalKind_SIGNAL_TERM, rvboxv1.SignalKind_SIGNAL_KILL} {
t.Run(phase.String()+"/"+signal.String(), func(t *testing.T) {
store, issue := recoveryStore(t)
if phase != rvboxv1.CommandLifecycle_COMMAND_RUNNING {
if phase == rvboxv1.CommandLifecycle_COMMAND_ACCEPTED {
// Use a separate accepted row rather than reversing its lifecycle.
issue, _ = domain.NewUUIDv7()
if _, err := store.AcceptCommand(context.Background(), spool.Command{IssueUUID: issue, Revision: 1, Phase: uint32(phase), ImmutableSHA256: [32]byte{1}}, time.Now()); err != nil {
t.Fatal(err)
}
} else if _, err := store.AppendLifecycle(context.Background(), issue, uint32(phase), 1, "finished", time.Now()); err != nil {
t.Fatal(err)
}
}
probe := &signalProbeSupervisor{}
notified := false
executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, Notify: func(domain.UUID) { notified = true }}
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: signal}); err != nil {
t.Fatalf("ordinary signal rejection broke session: %v", err)
}
result := lastSignalResult(t, store, issue)
if result.GetAccepted() || result.GetCommandRevision() != 1 || result.GetSignal() != signal || result.GetGracefulDeliveryAttempted() || result.GetForcedTerminationUsed() || !strings.Contains(result.GetDetail(), "not running") || probe.calls != 0 || !notified {
t.Fatalf("result=%v supervisor calls=%d notified=%v", result, probe.calls, notified)
}
command, err := store.GetCommand(context.Background(), issue)
if err != nil || command.Phase != uint32(phase) {
t.Fatalf("signal rejection rewrote lifecycle: %v %v", command, err)
}
})
}
}
}
func lastSignalResult(t *testing.T, store *spool.Store, issue domain.UUID) *rvboxv1.SignalResult {
t.Helper()
if _, err := store.AssignSendWindow(context.Background(), issue, 128, 1<<20); err != nil {
t.Fatal(err)
}
events, err := store.PendingEvents(context.Background(), issue)
if err != nil || len(events) == 0 {
t.Fatalf("missing durable signal result: %v", err)
}
transport := &runnerTransport{}
if err := SendStoredEvent(context.Background(), transport, Session{ID: "session", Generation: 1}, events[len(events)-1], agentproto.DefaultLimits()); err != nil {
t.Fatal(err)
}
frame, err := agentproto.DecodeEnvelope(transport.writes[0], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
if err != nil || frame.GetCommandEvent().GetSignalResult() == nil {
t.Fatalf("invalid signal result frame: %v %v", frame, err)
}
return frame.GetCommandEvent().GetSignalResult()
}
func TestSignalRejectionsDoNotTouchProcess_BH_RUNTIME_10(t *testing.T) {
for _, tc := range []struct {
name string
revision uint64
signal rvboxv1.SignalKind
terminal bool
detail string
}{
{"stale revision", 2, rvboxv1.SignalKind_SIGNAL_KILL, false, "revision"},
{"unsupported Windows signal", 1, rvboxv1.SignalKind_SIGNAL_HUP, false, "unsupported"},
{"process exit before active-map removal", 1, rvboxv1.SignalKind_SIGNAL_KILL, true, "not running"},
} {
t.Run(tc.name, func(t *testing.T) {
store, issue := recoveryStore(t)
if tc.terminal {
if _, err := store.AppendLifecycle(context.Background(), issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "finished", time.Now()); err != nil {
t.Fatal(err)
}
}
probe := &signalProbeSupervisor{}
executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, active: map[domain.UUID]supervisor.Process{issue: &failedCapture{}}, forced: make(map[domain.UUID]bool)}
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: tc.revision, Signal: tc.signal}); err != nil {
t.Fatal(err)
}
result := lastSignalResult(t, store, issue)
if result.GetAccepted() || result.GetCommandRevision() != tc.revision || !strings.Contains(result.GetDetail(), tc.detail) || result.GetGracefulDeliveryAttempted() || result.GetForcedTerminationUsed() || probe.calls != 0 {
t.Fatalf("result=%v supervisor calls=%d", result, probe.calls)
}
})
}
}
func TestSignalDeliveryAndFailureStayCommandLocal_HP_RUNTIME_05(t *testing.T) {
for _, signal := range []rvboxv1.SignalKind{rvboxv1.SignalKind_SIGNAL_TERM, rvboxv1.SignalKind_SIGNAL_KILL} {
for _, delivered := range []bool{true, false} {
store, issue := recoveryStore(t)
probe := &signalProbeSupervisor{outcome: supervisor.SignalOutcome{Delivered: delivered}}
if !delivered {
probe.err = errors.New("process exited during signal")
}
executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, active: map[domain.UUID]supervisor.Process{issue: &failedCapture{}}, forced: make(map[domain.UUID]bool)}
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: signal}); err != nil {
t.Fatal(err)
}
result := lastSignalResult(t, store, issue)
if result.GetAccepted() != delivered || result.GetGracefulDeliveryAttempted() != (signal == rvboxv1.SignalKind_SIGNAL_TERM) || probe.calls != 1 {
t.Fatalf("result=%v supervisor calls=%d", result, probe.calls)
}
}
}
}
func TestSignalProtocolAndStoreFailuresRemainErrors_BH_RUNTIME_11(t *testing.T) {
store, issue := recoveryStore(t)
executor := &Executor{Store: store, Now: time.Now}
for _, input := range []*rvboxv1.SignalCommand{
nil,
{IssueUuid: issue.String(), Signal: rvboxv1.SignalKind_SIGNAL_KILL},
{IssueUuid: "invalid", CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL},
{IssueUuid: issue.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KIND_UNSPECIFIED},
} {
if err := executor.Signal(context.Background(), Session{}, input); err == nil {
t.Fatalf("malformed signal was silently accepted: %v", input)
}
}
unknown, _ := domain.NewUUIDv7()
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: unknown.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL}); !errors.Is(err, spool.ErrUnknownCommand) {
t.Fatalf("unknown owner error lost: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if err := executor.Signal(ctx, Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL}); !errors.Is(err, context.Canceled) {
t.Fatalf("cancelled durable query was reported as successful signal rejection: %v", err)
}
}
+30
View File
@@ -1,5 +1,35 @@
version = 1
[[requirements]]
id = "BH-RUNTIME-09"
layer = "unit"
status = "implemented"
tests = ["internal/client/agent/signal_recovery_test.go:TestStoppedSignalIsDurablyRejected_BH_RUNTIME_09"]
[[requirements]]
id = "BH-RUNTIME-10"
layer = "unit"
status = "implemented"
tests = ["internal/client/agent/signal_recovery_test.go:TestSignalRejectionsDoNotTouchProcess_BH_RUNTIME_10"]
[[requirements]]
id = "BH-RUNTIME-11"
layer = "unit"
status = "implemented"
tests = ["internal/client/agent/signal_recovery_test.go:TestSignalProtocolAndStoreFailuresRemainErrors_BH_RUNTIME_11"]
[[requirements]]
id = "HP-RUNTIME-05"
layer = "unit"
status = "implemented"
tests = ["internal/client/agent/signal_recovery_test.go:TestSignalDeliveryAndFailureStayCommandLocal_HP_RUNTIME_05"]
[[requirements]]
id = "REC-RUNTIME-05"
layer = "integration"
status = "implemented"
tests = ["test/integration/clientagent/signal_recovery_integration_test.go:TestPendingSignalAfterExitSurvivesReconnect_REC_RUNTIME_05"]
[[requirements]]
id = "HP-WINCLI-10"
layer = "unit"
@@ -0,0 +1,214 @@
package clientagent_test
import (
"context"
"net/http/httptest"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/client/agent"
"github.com/rvbox/rvbox/internal/client/spool"
"github.com/rvbox/rvbox/internal/domain"
"github.com/rvbox/rvbox/internal/server/control"
"github.com/rvbox/rvbox/internal/server/session"
"github.com/rvbox/rvbox/internal/server/store"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
func TestPendingSignalAfterExitSurvivesReconnect_REC_RUNTIME_05(t *testing.T) {
ctx, cancelAll := context.WithTimeout(context.Background(), 20*time.Second)
defer cancelAll()
persistence, err := store.Open(ctx, store.Options{DataDir: filepath.Join(t.TempDir(), "server"), BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
defer persistence.Close()
registry := session.NewRegistry()
server := httptest.NewServer(&session.AgentServer{Store: persistence, Registry: registry, Path: "/v1/agent", Limits: agentproto.DefaultLimits()})
defer server.Close()
controlService, err := control.NewService(control.Options{Store: persistence, WakeClient: registry.Wake})
if err != nil {
t.Fatal(err)
}
directory := filepath.Join(t.TempDir(), "client")
client, err := spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
defer func() { _ = client.Close() }()
first, err := domain.NewUUIDv7()
if err != nil {
t.Fatal(err)
}
hello := &rvboxv1.ClientHello{ClientId: "signal-recovery-client", ClientInstanceId: client.ClientInstanceID().String(), SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1}, DaemonVersion: "signal-recovery-test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS, Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_CMD}, MaxRunningCommands: 16, MaxQueuedCommands: 100, SentAt: timestamppb.Now()}
var signalCalls, originalDispatches atomic.Int32
start := func() (context.CancelFunc, <-chan error) {
runCtx, cancel := context.WithCancel(ctx)
active, done := make(chan struct{}, 1), make(chan error, 1)
executor := &agent.Executor{Store: client, Now: time.Now} // Restarted: no native process remains.
go func() {
done <- agent.RunOnce(runCtx, agent.RunnerOptions{
Store: client, Hello: hello, Limits: agentproto.DefaultLimits(), Now: time.Now,
Backoff: agent.BackoffOptions{Initial: time.Millisecond, Maximum: time.Millisecond, StableReset: time.Second}, Jitter: func(time.Duration) time.Duration { return 0 },
Dial: func(ctx context.Context) (agent.Transport, error) {
return agent.DialWebSocket(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/v1/agent", nil)
},
OnSessionActive: func(value bool) {
if value {
active <- struct{}{}
}
},
OnDispatch: func(ctx context.Context, _ agent.Session, dispatch *rvboxv1.CommandDispatch) error {
issue, err := domain.ParseUUIDv7(dispatch.GetIssueUuid())
if err != nil {
return err
}
if _, err := client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), 1, "fixture running", time.Now()); err != nil {
return err
}
if issue == first {
originalDispatches.Add(1)
return nil
}
_, err = client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "fixture finished", time.Now())
return err
},
OnSignal: func(ctx context.Context, accepted agent.Session, signal *rvboxv1.SignalCommand) error {
signalCalls.Add(1)
// The old intent was persisted before reconnect. The process
// finishes locally just as that intent arrives, before its
// terminal event can reach the server.
if _, err := client.AppendLifecycle(ctx, first, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "exit won signal race", time.Now()); err != nil {
return err
}
return executor.Signal(ctx, accepted, signal)
},
})
close(done)
}()
select {
case <-active:
case err := <-done:
cancel()
t.Fatalf("client failed before active: %v", err)
case <-ctx.Done():
cancel()
t.Fatal("client did not become active")
}
return cancel, done
}
await := func(done <-chan error, description string, condition func() bool) {
t.Helper()
deadline := time.NewTimer(5 * time.Second)
defer deadline.Stop()
for {
if condition() {
return
}
select {
case err := <-done:
t.Fatalf("%s: session exited: %v", description, err)
case <-deadline.C:
t.Fatalf("timed out: %s", description)
case <-ctx.Done():
t.Fatal(ctx.Err())
case <-time.After(5 * time.Millisecond):
}
}
}
stop := func(cancel context.CancelFunc, done <-chan error) {
t.Helper()
cancel()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("client did not stop")
}
await(nil, "server session closed", func() bool {
view, err := persistence.GetClientView(ctx, hello.ClientId)
return err == nil && !view.Connected
})
}
queue := func(issue domain.UUID) {
t.Helper()
if _, err := controlService.RunCommand(ctx, &rvboxv1.RunCommandRequest{TargetClientId: hello.ClientId, RequestId: issue.String(), Spec: &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "echo fixture"}}}); err != nil {
t.Fatal(err)
}
}
phase := func(issue domain.UUID, want rvboxv1.CommandLifecycle) bool {
response, err := controlService.GetCommand(ctx, &rvboxv1.GetCommandRequest{ClientId: hello.ClientId, IssueUuid: issue.String()})
return err == nil && response.GetCommand().GetLifecycle() == want
}
cancel, done := start()
defer cancel()
queue(first)
await(done, "original command running", func() bool { return phase(first, rvboxv1.CommandLifecycle_COMMAND_RUNNING) })
stop(cancel, done)
requestID, _ := domain.NewUUIDv7()
if _, err := controlService.SignalCommand(ctx, &rvboxv1.ControlSignalCommandRequest{ClientId: hello.ClientId, IssueUuid: first.String(), RequestId: requestID.String(), Signal: rvboxv1.SignalKind_SIGNAL_KILL}); err != nil {
t.Fatal(err)
}
if err := client.Close(); err != nil {
t.Fatal(err)
}
client, err = spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
cancel, done = start()
defer cancel()
await(done, "negative signal result acknowledged", func() bool {
var acknowledged int
err := persistence.DB().QueryRowContext(ctx, `SELECT acknowledged FROM signal_intents WHERE request_uuid = ?`, requestID[:]).Scan(&acknowledged)
return err == nil && acknowledged == 1
})
if !phase(first, rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED) || signalCalls.Load() != 1 || originalDispatches.Load() != 1 {
t.Fatalf("original command replayed or terminal result lost: signals=%d dispatches=%d", signalCalls.Load(), originalDispatches.Load())
}
events, err := persistence.ReadCommandEvents(ctx, first, 0, 100)
if err != nil || len(events) != 3 {
t.Fatalf("events=%d err=%v", len(events), err)
}
payload, err := store.DecodeEventPayload(events[2])
if err != nil {
t.Fatal(err)
}
var event rvboxv1.CommandEvent
if err := proto.Unmarshal(payload, &event); err != nil {
t.Fatal(err)
}
result := event.GetSignalResult()
if result == nil || result.GetAccepted() || result.GetForcedTerminationUsed() || result.GetGracefulDeliveryAttempted() || result.GetCommandRevision() != 1 || !strings.Contains(result.GetDetail(), "not running") {
t.Fatalf("lost negative signal evidence: %v", result)
}
for attempt := 0; attempt < 2; attempt++ {
next, _ := domain.NewUUIDv7()
queue(next)
await(done, "fresh command succeeds without reconnect", func() bool { return phase(next, rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED) })
if attempt == 0 {
stop(cancel, done)
if err := client.Close(); err != nil {
t.Fatal(err)
}
client, err = spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
cancel, done = start()
defer cancel()
}
}
if signalCalls.Load() != 1 || originalDispatches.Load() != 1 {
t.Fatal("acknowledged signal or original command replayed after another restart")
}
stop(cancel, done)
if _, err := client.Check(ctx); err != nil {
t.Fatal(err)
}
}