fix(client): acknowledge signals after process exit

This commit is contained in:
cabbage committed 2026-10-27 16:18:32 +00:00
1 parent 789342ab4f
commit 30e7bb8545
7 files changed
+465 -8

No files matched your search

+21 -8
View File
@@ -324,24 +324,37 @@ func (executor *Executor) appendStdinAck(ctx context.Context, issue domain.UUID,
}
func (executor *Executor) Signal(ctx context.Context, _ Session, input *rvboxv1.SignalCommand) error {
if input == nil {
return errors.New("missing signal request")
if input == nil || input.GetCommandRevision() == 0 || input.GetSignal() < rvboxv1.SignalKind_SIGNAL_HUP || input.GetSignal() > rvboxv1.SignalKind_SIGNAL_USR2 {
return errors.New("invalid signal request")
}
issue, err := domain.ParseUUIDv7(input.GetIssueUuid())
if err != nil {
return err
}
command, err := executor.Store.GetCommand(ctx, issue)
if err != nil {
return err
}
reject := func(detail string) error {
return executor.appendSignalResult(ctx, issue, input.GetCommandRevision(), input.GetSignal(), false, false, supervisor.SignalOutcome{}, errors.New(detail))
}
if command.Revision != input.GetCommandRevision() {
return reject("signal revision does not match command")
}
executor.mu.Lock()
process := executor.active[issue]
executor.mu.Unlock()
if process == nil {
return errors.New("command is not running")
if process == nil || command.Terminal {
// A process can exit while its signal is queued or replayed. Persist a
// command-local negative result so the server can retire the intent;
// returning an error here would close/reconnect and replay it forever.
return reject("command is not running")
}
kind := supervisor.SignalTerm
if input.GetSignal() == rvboxv1.SignalKind_SIGNAL_KILL {
kind = supervisor.SignalKill
} else if input.GetSignal() != rvboxv1.SignalKind_SIGNAL_TERM {
return errors.New("unsupported Windows signal")
return reject("unsupported Windows signal")
}
outcome, err := executor.Supervisor.Signal(ctx, process, kind)
if kind == supervisor.SignalKill || outcome.Escalated {
@@ -349,15 +362,15 @@ func (executor *Executor) Signal(ctx context.Context, _ Session, input *rvboxv1.
executor.forced[issue] = true
executor.mu.Unlock()
}
return executor.appendSignalResult(ctx, issue, input.GetCommandRevision(), input.GetSignal(), err == nil && outcome.Delivered, outcome, err)
return executor.appendSignalResult(ctx, issue, input.GetCommandRevision(), input.GetSignal(), err == nil && outcome.Delivered, true, outcome, err)
}
func (executor *Executor) appendSignalResult(ctx context.Context, issue domain.UUID, revision uint64, signal rvboxv1.SignalKind, accepted bool, outcome supervisor.SignalOutcome, cause error) error {
func (executor *Executor) appendSignalResult(ctx context.Context, issue domain.UUID, revision uint64, signal rvboxv1.SignalKind, accepted, attempted bool, outcome supervisor.SignalOutcome, cause error) error {
detail := outcome.Detail
if cause != nil {
detail = boundedError(cause)
}
payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(&rvboxv1.CommandEvent{IssueUuid: issue.String(), ObservedAt: timestamppb.New(executor.Now()), Payload: &rvboxv1.CommandEvent_SignalResult{SignalResult: &rvboxv1.SignalResult{Signal: signal, Accepted: accepted, GracefulDeliveryAttempted: signal == rvboxv1.SignalKind_SIGNAL_TERM, ForcedTerminationUsed: outcome.Escalated || signal == rvboxv1.SignalKind_SIGNAL_KILL, Detail: detail, CommandRevision: revision}}})
payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(&rvboxv1.CommandEvent{IssueUuid: issue.String(), ObservedAt: timestamppb.New(executor.Now()), Payload: &rvboxv1.CommandEvent_SignalResult{SignalResult: &rvboxv1.SignalResult{Signal: signal, Accepted: accepted, GracefulDeliveryAttempted: attempted && signal == rvboxv1.SignalKind_SIGNAL_TERM, ForcedTerminationUsed: attempted && (outcome.Escalated || signal == rvboxv1.SignalKind_SIGNAL_KILL), Detail: detail, CommandRevision: revision}}})
if err != nil {
return err
}
@@ -0,0 +1,162 @@
package agent
import (
"context"
"errors"
"strings"
"testing"
"time"
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/client/spool"
"github.com/rvbox/rvbox/internal/client/supervisor"
"github.com/rvbox/rvbox/internal/domain"
)
type signalProbeSupervisor struct {
supervisor.Supervisor
calls int
outcome supervisor.SignalOutcome
err error
}
func (probe *signalProbeSupervisor) Signal(context.Context, supervisor.Process, supervisor.SignalKind) (supervisor.SignalOutcome, error) {
probe.calls++
return probe.outcome, probe.err
}
func TestStoppedSignalIsDurablyRejected_BH_RUNTIME_09(t *testing.T) {
for _, phase := range []rvboxv1.CommandLifecycle{
rvboxv1.CommandLifecycle_COMMAND_ACCEPTED,
rvboxv1.CommandLifecycle_COMMAND_RUNNING,
rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
} {
for _, signal := range []rvboxv1.SignalKind{rvboxv1.SignalKind_SIGNAL_TERM, rvboxv1.SignalKind_SIGNAL_KILL} {
t.Run(phase.String()+"/"+signal.String(), func(t *testing.T) {
store, issue := recoveryStore(t)
if phase != rvboxv1.CommandLifecycle_COMMAND_RUNNING {
if phase == rvboxv1.CommandLifecycle_COMMAND_ACCEPTED {
// Use a separate accepted row rather than reversing its lifecycle.
issue, _ = domain.NewUUIDv7()
if _, err := store.AcceptCommand(context.Background(), spool.Command{IssueUUID: issue, Revision: 1, Phase: uint32(phase), ImmutableSHA256: [32]byte{1}}, time.Now()); err != nil {
t.Fatal(err)
}
} else if _, err := store.AppendLifecycle(context.Background(), issue, uint32(phase), 1, "finished", time.Now()); err != nil {
t.Fatal(err)
}
}
probe := &signalProbeSupervisor{}
notified := false
executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, Notify: func(domain.UUID) { notified = true }}
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: signal}); err != nil {
t.Fatalf("ordinary signal rejection broke session: %v", err)
}
result := lastSignalResult(t, store, issue)
if result.GetAccepted() || result.GetCommandRevision() != 1 || result.GetSignal() != signal || result.GetGracefulDeliveryAttempted() || result.GetForcedTerminationUsed() || !strings.Contains(result.GetDetail(), "not running") || probe.calls != 0 || !notified {
t.Fatalf("result=%v supervisor calls=%d notified=%v", result, probe.calls, notified)
}
command, err := store.GetCommand(context.Background(), issue)
if err != nil || command.Phase != uint32(phase) {
t.Fatalf("signal rejection rewrote lifecycle: %v %v", command, err)
}
})
}
}
}
func lastSignalResult(t *testing.T, store *spool.Store, issue domain.UUID) *rvboxv1.SignalResult {
t.Helper()
if _, err := store.AssignSendWindow(context.Background(), issue, 128, 1<<20); err != nil {
t.Fatal(err)
}
events, err := store.PendingEvents(context.Background(), issue)
if err != nil || len(events) == 0 {
t.Fatalf("missing durable signal result: %v", err)
}
transport := &runnerTransport{}
if err := SendStoredEvent(context.Background(), transport, Session{ID: "session", Generation: 1}, events[len(events)-1], agentproto.DefaultLimits()); err != nil {
t.Fatal(err)
}
frame, err := agentproto.DecodeEnvelope(transport.writes[0], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
if err != nil || frame.GetCommandEvent().GetSignalResult() == nil {
t.Fatalf("invalid signal result frame: %v %v", frame, err)
}
return frame.GetCommandEvent().GetSignalResult()
}
func TestSignalRejectionsDoNotTouchProcess_BH_RUNTIME_10(t *testing.T) {
for _, tc := range []struct {
name string
revision uint64
signal rvboxv1.SignalKind
terminal bool
detail string
}{
{"stale revision", 2, rvboxv1.SignalKind_SIGNAL_KILL, false, "revision"},
{"unsupported Windows signal", 1, rvboxv1.SignalKind_SIGNAL_HUP, false, "unsupported"},
{"process exit before active-map removal", 1, rvboxv1.SignalKind_SIGNAL_KILL, true, "not running"},
} {
t.Run(tc.name, func(t *testing.T) {
store, issue := recoveryStore(t)
if tc.terminal {
if _, err := store.AppendLifecycle(context.Background(), issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "finished", time.Now()); err != nil {
t.Fatal(err)
}
}
probe := &signalProbeSupervisor{}
executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, active: map[domain.UUID]supervisor.Process{issue: &failedCapture{}}, forced: make(map[domain.UUID]bool)}
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: tc.revision, Signal: tc.signal}); err != nil {
t.Fatal(err)
}
result := lastSignalResult(t, store, issue)
if result.GetAccepted() || result.GetCommandRevision() != tc.revision || !strings.Contains(result.GetDetail(), tc.detail) || result.GetGracefulDeliveryAttempted() || result.GetForcedTerminationUsed() || probe.calls != 0 {
t.Fatalf("result=%v supervisor calls=%d", result, probe.calls)
}
})
}
}
func TestSignalDeliveryAndFailureStayCommandLocal_HP_RUNTIME_05(t *testing.T) {
for _, signal := range []rvboxv1.SignalKind{rvboxv1.SignalKind_SIGNAL_TERM, rvboxv1.SignalKind_SIGNAL_KILL} {
for _, delivered := range []bool{true, false} {
store, issue := recoveryStore(t)
probe := &signalProbeSupervisor{outcome: supervisor.SignalOutcome{Delivered: delivered}}
if !delivered {
probe.err = errors.New("process exited during signal")
}
executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, active: map[domain.UUID]supervisor.Process{issue: &failedCapture{}}, forced: make(map[domain.UUID]bool)}
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: signal}); err != nil {
t.Fatal(err)
}
result := lastSignalResult(t, store, issue)
if result.GetAccepted() != delivered || result.GetGracefulDeliveryAttempted() != (signal == rvboxv1.SignalKind_SIGNAL_TERM) || probe.calls != 1 {
t.Fatalf("result=%v supervisor calls=%d", result, probe.calls)
}
}
}
}
func TestSignalProtocolAndStoreFailuresRemainErrors_BH_RUNTIME_11(t *testing.T) {
store, issue := recoveryStore(t)
executor := &Executor{Store: store, Now: time.Now}
for _, input := range []*rvboxv1.SignalCommand{
nil,
{IssueUuid: issue.String(), Signal: rvboxv1.SignalKind_SIGNAL_KILL},
{IssueUuid: "invalid", CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL},
{IssueUuid: issue.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KIND_UNSPECIFIED},
} {
if err := executor.Signal(context.Background(), Session{}, input); err == nil {
t.Fatalf("malformed signal was silently accepted: %v", input)
}
}
unknown, _ := domain.NewUUIDv7()
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: unknown.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL}); !errors.Is(err, spool.ErrUnknownCommand) {
t.Fatalf("unknown owner error lost: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if err := executor.Signal(ctx, Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL}); !errors.Is(err, context.Canceled) {
t.Fatalf("cancelled durable query was reported as successful signal rejection: %v", err)
}
}