diff --git a/docs/implementation-plan.v1.md b/docs/implementation-plan.v1.md index 05fd0ec..a2833f5 100644 --- a/docs/implementation-plan.v1.md +++ b/docs/implementation-plan.v1.md @@ -1838,6 +1838,19 @@ was not authorized; otherwise retain the revisioned signal result and actual terminal outcome. The server maps control `request_id` to this revision/result and never sends that request ID across the agent protocol. +A signal may arrive after local exit or before a native process exists. Treat +that as a command-level negative `SignalResult`, not a failed session: retain +the requested revision/signal, set `accepted=false`, state `command is not +running`, and leave delivery-attempt/forced-termination flags false. Persist +and notify/flush this event through the normal command event sequence, including +after a terminal lifecycle, so the server can retire its durable signal intent. +Reject mismatched revisions and unsupported Windows signals the same way +without touching a process. Preserve the real terminal outcome. Malformed +messages, unknown durable owners, and storage failures remain explicit errors. +Test offline intent replay, process-exit races, acknowledgement after terminal, +continued fresh dispatch on the same connection, and another spool restart +without repeating the acknowledged intent or original execution. + ### 6.5 Server session unit and integration tests Unit-test version selection, registry/generation transitions, takeover grant diff --git a/docs/operations-runbook.md b/docs/operations-runbook.md index be5e2d9..eb8b63d 100644 --- a/docs/operations-runbook.md +++ b/docs/operations-runbook.md @@ -133,6 +133,13 @@ exact ignored `dist/` directory after preserving any evidence. protection. Inspect command retention and free space, allow terminal-age or quota rotation to reclaim eligible data, or enlarge the owned filesystem. Never manually remove live SQLite/WAL/segment files. +- **Repeated `signal command ...: command is not running`:** early v1 clients + closed their session instead of returning a negative signal result when an + old queued/replayed signal reached an already-exited process. Upgrade the + Windows client with the corrected installer, preserving TOML/spool/history. + The client records `accepted=false` for that signal; the existing server + acknowledges it and stops replaying it. Do not wipe state or resend the + original command to clear this loop. - **Output truncated:** query command status and output history for its explicit loss/truncation markers. The command may still have completed correctly. - **Server restart / dirty health:** wait for readiness and inspect the diff --git a/docs/testing.md b/docs/testing.md index 9fabba1..5ce585d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -188,6 +188,24 @@ loss marker plus terminal result, restart the same client spool, discard the server-confirmed terminal row, and complete a fresh command. It also requires prompt server-session cleanup rather than waiting for a heartbeat tick. +`BH-RUNTIME-09..11` and `HP-RUNTIME-05` exercise signal requests after process +exit, accepted work with no process, terminal state before active-map removal, +revision mismatches, unsupported Windows signals, actual TERM/KILL outcomes, +and supervisor failure. Expected command-level rejection must persist a typed +`SignalResult` and notify the sender without returning a session-fatal error. +Do not claim graceful/forced delivery when no process signal was attempted. +Malformed requests, unknown command ownership, and store failures still fail +explicitly; the fix is not a blanket suppression of callback errors. + +`REC-RUNTIME-05` uses actual WebSockets and durable stores. Persist an active +command, disconnect, queue a signal while offline, reopen the same client spool, +and finish the process locally just as the pending signal arrives. Require the +actual executor's negative result to acknowledge the server intent after the +terminal event, then complete fresh work on that same session. Reopen/reconnect +again and require more fresh work with no replay of the acknowledged signal or +original execution. This reproduces the prior endless `signal command ...: +command is not running` reconnect loop without needing a Windows OS fixture. + ```sh docker compose -f deploy/compose.yaml run --rm toolchain \ go test -race ./internal/client/agent ./test/integration/clientagent diff --git a/internal/client/agent/executor.go b/internal/client/agent/executor.go index def088a..c522374 100644 --- a/internal/client/agent/executor.go +++ b/internal/client/agent/executor.go @@ -324,24 +324,37 @@ func (executor *Executor) appendStdinAck(ctx context.Context, issue domain.UUID, } func (executor *Executor) Signal(ctx context.Context, _ Session, input *rvboxv1.SignalCommand) error { - if input == nil { - return errors.New("missing signal request") + if input == nil || input.GetCommandRevision() == 0 || input.GetSignal() < rvboxv1.SignalKind_SIGNAL_HUP || input.GetSignal() > rvboxv1.SignalKind_SIGNAL_USR2 { + return errors.New("invalid signal request") } issue, err := domain.ParseUUIDv7(input.GetIssueUuid()) if err != nil { return err } + command, err := executor.Store.GetCommand(ctx, issue) + if err != nil { + return err + } + reject := func(detail string) error { + return executor.appendSignalResult(ctx, issue, input.GetCommandRevision(), input.GetSignal(), false, false, supervisor.SignalOutcome{}, errors.New(detail)) + } + if command.Revision != input.GetCommandRevision() { + return reject("signal revision does not match command") + } executor.mu.Lock() process := executor.active[issue] executor.mu.Unlock() - if process == nil { - return errors.New("command is not running") + if process == nil || command.Terminal { + // A process can exit while its signal is queued or replayed. Persist a + // command-local negative result so the server can retire the intent; + // returning an error here would close/reconnect and replay it forever. + return reject("command is not running") } kind := supervisor.SignalTerm if input.GetSignal() == rvboxv1.SignalKind_SIGNAL_KILL { kind = supervisor.SignalKill } else if input.GetSignal() != rvboxv1.SignalKind_SIGNAL_TERM { - return errors.New("unsupported Windows signal") + return reject("unsupported Windows signal") } outcome, err := executor.Supervisor.Signal(ctx, process, kind) if kind == supervisor.SignalKill || outcome.Escalated { @@ -349,15 +362,15 @@ func (executor *Executor) Signal(ctx context.Context, _ Session, input *rvboxv1. executor.forced[issue] = true executor.mu.Unlock() } - return executor.appendSignalResult(ctx, issue, input.GetCommandRevision(), input.GetSignal(), err == nil && outcome.Delivered, outcome, err) + return executor.appendSignalResult(ctx, issue, input.GetCommandRevision(), input.GetSignal(), err == nil && outcome.Delivered, true, outcome, err) } -func (executor *Executor) appendSignalResult(ctx context.Context, issue domain.UUID, revision uint64, signal rvboxv1.SignalKind, accepted bool, outcome supervisor.SignalOutcome, cause error) error { +func (executor *Executor) appendSignalResult(ctx context.Context, issue domain.UUID, revision uint64, signal rvboxv1.SignalKind, accepted, attempted bool, outcome supervisor.SignalOutcome, cause error) error { detail := outcome.Detail if cause != nil { detail = boundedError(cause) } - payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(&rvboxv1.CommandEvent{IssueUuid: issue.String(), ObservedAt: timestamppb.New(executor.Now()), Payload: &rvboxv1.CommandEvent_SignalResult{SignalResult: &rvboxv1.SignalResult{Signal: signal, Accepted: accepted, GracefulDeliveryAttempted: signal == rvboxv1.SignalKind_SIGNAL_TERM, ForcedTerminationUsed: outcome.Escalated || signal == rvboxv1.SignalKind_SIGNAL_KILL, Detail: detail, CommandRevision: revision}}}) + payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(&rvboxv1.CommandEvent{IssueUuid: issue.String(), ObservedAt: timestamppb.New(executor.Now()), Payload: &rvboxv1.CommandEvent_SignalResult{SignalResult: &rvboxv1.SignalResult{Signal: signal, Accepted: accepted, GracefulDeliveryAttempted: attempted && signal == rvboxv1.SignalKind_SIGNAL_TERM, ForcedTerminationUsed: attempted && (outcome.Escalated || signal == rvboxv1.SignalKind_SIGNAL_KILL), Detail: detail, CommandRevision: revision}}}) if err != nil { return err } diff --git a/internal/client/agent/signal_recovery_test.go b/internal/client/agent/signal_recovery_test.go new file mode 100644 index 0000000..3099b60 --- /dev/null +++ b/internal/client/agent/signal_recovery_test.go @@ -0,0 +1,162 @@ +package agent + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1" + "github.com/rvbox/rvbox/internal/agentproto" + "github.com/rvbox/rvbox/internal/client/spool" + "github.com/rvbox/rvbox/internal/client/supervisor" + "github.com/rvbox/rvbox/internal/domain" +) + +type signalProbeSupervisor struct { + supervisor.Supervisor + calls int + outcome supervisor.SignalOutcome + err error +} + +func (probe *signalProbeSupervisor) Signal(context.Context, supervisor.Process, supervisor.SignalKind) (supervisor.SignalOutcome, error) { + probe.calls++ + return probe.outcome, probe.err +} + +func TestStoppedSignalIsDurablyRejected_BH_RUNTIME_09(t *testing.T) { + for _, phase := range []rvboxv1.CommandLifecycle{ + rvboxv1.CommandLifecycle_COMMAND_ACCEPTED, + rvboxv1.CommandLifecycle_COMMAND_RUNNING, + rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED, + } { + for _, signal := range []rvboxv1.SignalKind{rvboxv1.SignalKind_SIGNAL_TERM, rvboxv1.SignalKind_SIGNAL_KILL} { + t.Run(phase.String()+"/"+signal.String(), func(t *testing.T) { + store, issue := recoveryStore(t) + if phase != rvboxv1.CommandLifecycle_COMMAND_RUNNING { + if phase == rvboxv1.CommandLifecycle_COMMAND_ACCEPTED { + // Use a separate accepted row rather than reversing its lifecycle. + issue, _ = domain.NewUUIDv7() + if _, err := store.AcceptCommand(context.Background(), spool.Command{IssueUUID: issue, Revision: 1, Phase: uint32(phase), ImmutableSHA256: [32]byte{1}}, time.Now()); err != nil { + t.Fatal(err) + } + } else if _, err := store.AppendLifecycle(context.Background(), issue, uint32(phase), 1, "finished", time.Now()); err != nil { + t.Fatal(err) + } + } + probe := &signalProbeSupervisor{} + notified := false + executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, Notify: func(domain.UUID) { notified = true }} + if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: signal}); err != nil { + t.Fatalf("ordinary signal rejection broke session: %v", err) + } + result := lastSignalResult(t, store, issue) + if result.GetAccepted() || result.GetCommandRevision() != 1 || result.GetSignal() != signal || result.GetGracefulDeliveryAttempted() || result.GetForcedTerminationUsed() || !strings.Contains(result.GetDetail(), "not running") || probe.calls != 0 || !notified { + t.Fatalf("result=%v supervisor calls=%d notified=%v", result, probe.calls, notified) + } + command, err := store.GetCommand(context.Background(), issue) + if err != nil || command.Phase != uint32(phase) { + t.Fatalf("signal rejection rewrote lifecycle: %v %v", command, err) + } + }) + } + } +} + +func lastSignalResult(t *testing.T, store *spool.Store, issue domain.UUID) *rvboxv1.SignalResult { + t.Helper() + if _, err := store.AssignSendWindow(context.Background(), issue, 128, 1<<20); err != nil { + t.Fatal(err) + } + events, err := store.PendingEvents(context.Background(), issue) + if err != nil || len(events) == 0 { + t.Fatalf("missing durable signal result: %v", err) + } + transport := &runnerTransport{} + if err := SendStoredEvent(context.Background(), transport, Session{ID: "session", Generation: 1}, events[len(events)-1], agentproto.DefaultLimits()); err != nil { + t.Fatal(err) + } + frame, err := agentproto.DecodeEnvelope(transport.writes[0], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS) + if err != nil || frame.GetCommandEvent().GetSignalResult() == nil { + t.Fatalf("invalid signal result frame: %v %v", frame, err) + } + return frame.GetCommandEvent().GetSignalResult() +} + +func TestSignalRejectionsDoNotTouchProcess_BH_RUNTIME_10(t *testing.T) { + for _, tc := range []struct { + name string + revision uint64 + signal rvboxv1.SignalKind + terminal bool + detail string + }{ + {"stale revision", 2, rvboxv1.SignalKind_SIGNAL_KILL, false, "revision"}, + {"unsupported Windows signal", 1, rvboxv1.SignalKind_SIGNAL_HUP, false, "unsupported"}, + {"process exit before active-map removal", 1, rvboxv1.SignalKind_SIGNAL_KILL, true, "not running"}, + } { + t.Run(tc.name, func(t *testing.T) { + store, issue := recoveryStore(t) + if tc.terminal { + if _, err := store.AppendLifecycle(context.Background(), issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "finished", time.Now()); err != nil { + t.Fatal(err) + } + } + probe := &signalProbeSupervisor{} + executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, active: map[domain.UUID]supervisor.Process{issue: &failedCapture{}}, forced: make(map[domain.UUID]bool)} + if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: tc.revision, Signal: tc.signal}); err != nil { + t.Fatal(err) + } + result := lastSignalResult(t, store, issue) + if result.GetAccepted() || result.GetCommandRevision() != tc.revision || !strings.Contains(result.GetDetail(), tc.detail) || result.GetGracefulDeliveryAttempted() || result.GetForcedTerminationUsed() || probe.calls != 0 { + t.Fatalf("result=%v supervisor calls=%d", result, probe.calls) + } + }) + } +} + +func TestSignalDeliveryAndFailureStayCommandLocal_HP_RUNTIME_05(t *testing.T) { + for _, signal := range []rvboxv1.SignalKind{rvboxv1.SignalKind_SIGNAL_TERM, rvboxv1.SignalKind_SIGNAL_KILL} { + for _, delivered := range []bool{true, false} { + store, issue := recoveryStore(t) + probe := &signalProbeSupervisor{outcome: supervisor.SignalOutcome{Delivered: delivered}} + if !delivered { + probe.err = errors.New("process exited during signal") + } + executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, active: map[domain.UUID]supervisor.Process{issue: &failedCapture{}}, forced: make(map[domain.UUID]bool)} + if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: signal}); err != nil { + t.Fatal(err) + } + result := lastSignalResult(t, store, issue) + if result.GetAccepted() != delivered || result.GetGracefulDeliveryAttempted() != (signal == rvboxv1.SignalKind_SIGNAL_TERM) || probe.calls != 1 { + t.Fatalf("result=%v supervisor calls=%d", result, probe.calls) + } + } + } +} + +func TestSignalProtocolAndStoreFailuresRemainErrors_BH_RUNTIME_11(t *testing.T) { + store, issue := recoveryStore(t) + executor := &Executor{Store: store, Now: time.Now} + for _, input := range []*rvboxv1.SignalCommand{ + nil, + {IssueUuid: issue.String(), Signal: rvboxv1.SignalKind_SIGNAL_KILL}, + {IssueUuid: "invalid", CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL}, + {IssueUuid: issue.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KIND_UNSPECIFIED}, + } { + if err := executor.Signal(context.Background(), Session{}, input); err == nil { + t.Fatalf("malformed signal was silently accepted: %v", input) + } + } + unknown, _ := domain.NewUUIDv7() + if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: unknown.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL}); !errors.Is(err, spool.ErrUnknownCommand) { + t.Fatalf("unknown owner error lost: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := executor.Signal(ctx, Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL}); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled durable query was reported as successful signal rejection: %v", err) + } +} diff --git a/test/coverage.toml b/test/coverage.toml index 7392caf..83aa30d 100644 --- a/test/coverage.toml +++ b/test/coverage.toml @@ -1,5 +1,35 @@ version = 1 +[[requirements]] +id = "BH-RUNTIME-09" +layer = "unit" +status = "implemented" +tests = ["internal/client/agent/signal_recovery_test.go:TestStoppedSignalIsDurablyRejected_BH_RUNTIME_09"] + +[[requirements]] +id = "BH-RUNTIME-10" +layer = "unit" +status = "implemented" +tests = ["internal/client/agent/signal_recovery_test.go:TestSignalRejectionsDoNotTouchProcess_BH_RUNTIME_10"] + +[[requirements]] +id = "BH-RUNTIME-11" +layer = "unit" +status = "implemented" +tests = ["internal/client/agent/signal_recovery_test.go:TestSignalProtocolAndStoreFailuresRemainErrors_BH_RUNTIME_11"] + +[[requirements]] +id = "HP-RUNTIME-05" +layer = "unit" +status = "implemented" +tests = ["internal/client/agent/signal_recovery_test.go:TestSignalDeliveryAndFailureStayCommandLocal_HP_RUNTIME_05"] + +[[requirements]] +id = "REC-RUNTIME-05" +layer = "integration" +status = "implemented" +tests = ["test/integration/clientagent/signal_recovery_integration_test.go:TestPendingSignalAfterExitSurvivesReconnect_REC_RUNTIME_05"] + [[requirements]] id = "HP-WINCLI-10" layer = "unit" diff --git a/test/integration/clientagent/signal_recovery_integration_test.go b/test/integration/clientagent/signal_recovery_integration_test.go new file mode 100644 index 0000000..e88ddc4 --- /dev/null +++ b/test/integration/clientagent/signal_recovery_integration_test.go @@ -0,0 +1,214 @@ +package clientagent_test + +import ( + "context" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1" + "github.com/rvbox/rvbox/internal/agentproto" + "github.com/rvbox/rvbox/internal/client/agent" + "github.com/rvbox/rvbox/internal/client/spool" + "github.com/rvbox/rvbox/internal/domain" + "github.com/rvbox/rvbox/internal/server/control" + "github.com/rvbox/rvbox/internal/server/session" + "github.com/rvbox/rvbox/internal/server/store" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestPendingSignalAfterExitSurvivesReconnect_REC_RUNTIME_05(t *testing.T) { + ctx, cancelAll := context.WithTimeout(context.Background(), 20*time.Second) + defer cancelAll() + persistence, err := store.Open(ctx, store.Options{DataDir: filepath.Join(t.TempDir(), "server"), BusyTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + defer persistence.Close() + registry := session.NewRegistry() + server := httptest.NewServer(&session.AgentServer{Store: persistence, Registry: registry, Path: "/v1/agent", Limits: agentproto.DefaultLimits()}) + defer server.Close() + controlService, err := control.NewService(control.Options{Store: persistence, WakeClient: registry.Wake}) + if err != nil { + t.Fatal(err) + } + directory := filepath.Join(t.TempDir(), "client") + client, err := spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = client.Close() }() + first, err := domain.NewUUIDv7() + if err != nil { + t.Fatal(err) + } + hello := &rvboxv1.ClientHello{ClientId: "signal-recovery-client", ClientInstanceId: client.ClientInstanceID().String(), SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1}, DaemonVersion: "signal-recovery-test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS, Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_CMD}, MaxRunningCommands: 16, MaxQueuedCommands: 100, SentAt: timestamppb.Now()} + var signalCalls, originalDispatches atomic.Int32 + start := func() (context.CancelFunc, <-chan error) { + runCtx, cancel := context.WithCancel(ctx) + active, done := make(chan struct{}, 1), make(chan error, 1) + executor := &agent.Executor{Store: client, Now: time.Now} // Restarted: no native process remains. + go func() { + done <- agent.RunOnce(runCtx, agent.RunnerOptions{ + Store: client, Hello: hello, Limits: agentproto.DefaultLimits(), Now: time.Now, + Backoff: agent.BackoffOptions{Initial: time.Millisecond, Maximum: time.Millisecond, StableReset: time.Second}, Jitter: func(time.Duration) time.Duration { return 0 }, + Dial: func(ctx context.Context) (agent.Transport, error) { + return agent.DialWebSocket(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/v1/agent", nil) + }, + OnSessionActive: func(value bool) { + if value { + active <- struct{}{} + } + }, + OnDispatch: func(ctx context.Context, _ agent.Session, dispatch *rvboxv1.CommandDispatch) error { + issue, err := domain.ParseUUIDv7(dispatch.GetIssueUuid()) + if err != nil { + return err + } + if _, err := client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), 1, "fixture running", time.Now()); err != nil { + return err + } + if issue == first { + originalDispatches.Add(1) + return nil + } + _, err = client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "fixture finished", time.Now()) + return err + }, + OnSignal: func(ctx context.Context, accepted agent.Session, signal *rvboxv1.SignalCommand) error { + signalCalls.Add(1) + // The old intent was persisted before reconnect. The process + // finishes locally just as that intent arrives, before its + // terminal event can reach the server. + if _, err := client.AppendLifecycle(ctx, first, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "exit won signal race", time.Now()); err != nil { + return err + } + return executor.Signal(ctx, accepted, signal) + }, + }) + close(done) + }() + select { + case <-active: + case err := <-done: + cancel() + t.Fatalf("client failed before active: %v", err) + case <-ctx.Done(): + cancel() + t.Fatal("client did not become active") + } + return cancel, done + } + await := func(done <-chan error, description string, condition func() bool) { + t.Helper() + deadline := time.NewTimer(5 * time.Second) + defer deadline.Stop() + for { + if condition() { + return + } + select { + case err := <-done: + t.Fatalf("%s: session exited: %v", description, err) + case <-deadline.C: + t.Fatalf("timed out: %s", description) + case <-ctx.Done(): + t.Fatal(ctx.Err()) + case <-time.After(5 * time.Millisecond): + } + } + } + stop := func(cancel context.CancelFunc, done <-chan error) { + t.Helper() + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("client did not stop") + } + await(nil, "server session closed", func() bool { + view, err := persistence.GetClientView(ctx, hello.ClientId) + return err == nil && !view.Connected + }) + } + queue := func(issue domain.UUID) { + t.Helper() + if _, err := controlService.RunCommand(ctx, &rvboxv1.RunCommandRequest{TargetClientId: hello.ClientId, RequestId: issue.String(), Spec: &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "echo fixture"}}}); err != nil { + t.Fatal(err) + } + } + phase := func(issue domain.UUID, want rvboxv1.CommandLifecycle) bool { + response, err := controlService.GetCommand(ctx, &rvboxv1.GetCommandRequest{ClientId: hello.ClientId, IssueUuid: issue.String()}) + return err == nil && response.GetCommand().GetLifecycle() == want + } + cancel, done := start() + defer cancel() + queue(first) + await(done, "original command running", func() bool { return phase(first, rvboxv1.CommandLifecycle_COMMAND_RUNNING) }) + stop(cancel, done) + requestID, _ := domain.NewUUIDv7() + if _, err := controlService.SignalCommand(ctx, &rvboxv1.ControlSignalCommandRequest{ClientId: hello.ClientId, IssueUuid: first.String(), RequestId: requestID.String(), Signal: rvboxv1.SignalKind_SIGNAL_KILL}); err != nil { + t.Fatal(err) + } + if err := client.Close(); err != nil { + t.Fatal(err) + } + client, err = spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + cancel, done = start() + defer cancel() + await(done, "negative signal result acknowledged", func() bool { + var acknowledged int + err := persistence.DB().QueryRowContext(ctx, `SELECT acknowledged FROM signal_intents WHERE request_uuid = ?`, requestID[:]).Scan(&acknowledged) + return err == nil && acknowledged == 1 + }) + if !phase(first, rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED) || signalCalls.Load() != 1 || originalDispatches.Load() != 1 { + t.Fatalf("original command replayed or terminal result lost: signals=%d dispatches=%d", signalCalls.Load(), originalDispatches.Load()) + } + events, err := persistence.ReadCommandEvents(ctx, first, 0, 100) + if err != nil || len(events) != 3 { + t.Fatalf("events=%d err=%v", len(events), err) + } + payload, err := store.DecodeEventPayload(events[2]) + if err != nil { + t.Fatal(err) + } + var event rvboxv1.CommandEvent + if err := proto.Unmarshal(payload, &event); err != nil { + t.Fatal(err) + } + result := event.GetSignalResult() + if result == nil || result.GetAccepted() || result.GetForcedTerminationUsed() || result.GetGracefulDeliveryAttempted() || result.GetCommandRevision() != 1 || !strings.Contains(result.GetDetail(), "not running") { + t.Fatalf("lost negative signal evidence: %v", result) + } + for attempt := 0; attempt < 2; attempt++ { + next, _ := domain.NewUUIDv7() + queue(next) + await(done, "fresh command succeeds without reconnect", func() bool { return phase(next, rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED) }) + if attempt == 0 { + stop(cancel, done) + if err := client.Close(); err != nil { + t.Fatal(err) + } + client, err = spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + cancel, done = start() + defer cancel() + } + } + if signalCalls.Load() != 1 || originalDispatches.Load() != 1 { + t.Fatal("acknowledged signal or original command replayed after another restart") + } + stop(cancel, done) + if _, err := client.Check(ctx); err != nil { + t.Fatal(err) + } +}