fix(client): acknowledge signals after process exit
This commit is contained in:
1 parent
789342ab4f
commit
30e7bb8545
7 files changed
+465
-8
No files matched your search
@@ -0,0 +1,162 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||
"github.com/rvbox/rvbox/internal/agentproto"
|
||||
"github.com/rvbox/rvbox/internal/client/spool"
|
||||
"github.com/rvbox/rvbox/internal/client/supervisor"
|
||||
"github.com/rvbox/rvbox/internal/domain"
|
||||
)
|
||||
|
||||
type signalProbeSupervisor struct {
|
||||
supervisor.Supervisor
|
||||
calls int
|
||||
outcome supervisor.SignalOutcome
|
||||
err error
|
||||
}
|
||||
|
||||
func (probe *signalProbeSupervisor) Signal(context.Context, supervisor.Process, supervisor.SignalKind) (supervisor.SignalOutcome, error) {
|
||||
probe.calls++
|
||||
return probe.outcome, probe.err
|
||||
}
|
||||
|
||||
func TestStoppedSignalIsDurablyRejected_BH_RUNTIME_09(t *testing.T) {
|
||||
for _, phase := range []rvboxv1.CommandLifecycle{
|
||||
rvboxv1.CommandLifecycle_COMMAND_ACCEPTED,
|
||||
rvboxv1.CommandLifecycle_COMMAND_RUNNING,
|
||||
rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
|
||||
} {
|
||||
for _, signal := range []rvboxv1.SignalKind{rvboxv1.SignalKind_SIGNAL_TERM, rvboxv1.SignalKind_SIGNAL_KILL} {
|
||||
t.Run(phase.String()+"/"+signal.String(), func(t *testing.T) {
|
||||
store, issue := recoveryStore(t)
|
||||
if phase != rvboxv1.CommandLifecycle_COMMAND_RUNNING {
|
||||
if phase == rvboxv1.CommandLifecycle_COMMAND_ACCEPTED {
|
||||
// Use a separate accepted row rather than reversing its lifecycle.
|
||||
issue, _ = domain.NewUUIDv7()
|
||||
if _, err := store.AcceptCommand(context.Background(), spool.Command{IssueUUID: issue, Revision: 1, Phase: uint32(phase), ImmutableSHA256: [32]byte{1}}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else if _, err := store.AppendLifecycle(context.Background(), issue, uint32(phase), 1, "finished", time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
probe := &signalProbeSupervisor{}
|
||||
notified := false
|
||||
executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, Notify: func(domain.UUID) { notified = true }}
|
||||
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: signal}); err != nil {
|
||||
t.Fatalf("ordinary signal rejection broke session: %v", err)
|
||||
}
|
||||
result := lastSignalResult(t, store, issue)
|
||||
if result.GetAccepted() || result.GetCommandRevision() != 1 || result.GetSignal() != signal || result.GetGracefulDeliveryAttempted() || result.GetForcedTerminationUsed() || !strings.Contains(result.GetDetail(), "not running") || probe.calls != 0 || !notified {
|
||||
t.Fatalf("result=%v supervisor calls=%d notified=%v", result, probe.calls, notified)
|
||||
}
|
||||
command, err := store.GetCommand(context.Background(), issue)
|
||||
if err != nil || command.Phase != uint32(phase) {
|
||||
t.Fatalf("signal rejection rewrote lifecycle: %v %v", command, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func lastSignalResult(t *testing.T, store *spool.Store, issue domain.UUID) *rvboxv1.SignalResult {
|
||||
t.Helper()
|
||||
if _, err := store.AssignSendWindow(context.Background(), issue, 128, 1<<20); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
events, err := store.PendingEvents(context.Background(), issue)
|
||||
if err != nil || len(events) == 0 {
|
||||
t.Fatalf("missing durable signal result: %v", err)
|
||||
}
|
||||
transport := &runnerTransport{}
|
||||
if err := SendStoredEvent(context.Background(), transport, Session{ID: "session", Generation: 1}, events[len(events)-1], agentproto.DefaultLimits()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
frame, err := agentproto.DecodeEnvelope(transport.writes[0], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
|
||||
if err != nil || frame.GetCommandEvent().GetSignalResult() == nil {
|
||||
t.Fatalf("invalid signal result frame: %v %v", frame, err)
|
||||
}
|
||||
return frame.GetCommandEvent().GetSignalResult()
|
||||
}
|
||||
|
||||
func TestSignalRejectionsDoNotTouchProcess_BH_RUNTIME_10(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
revision uint64
|
||||
signal rvboxv1.SignalKind
|
||||
terminal bool
|
||||
detail string
|
||||
}{
|
||||
{"stale revision", 2, rvboxv1.SignalKind_SIGNAL_KILL, false, "revision"},
|
||||
{"unsupported Windows signal", 1, rvboxv1.SignalKind_SIGNAL_HUP, false, "unsupported"},
|
||||
{"process exit before active-map removal", 1, rvboxv1.SignalKind_SIGNAL_KILL, true, "not running"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
store, issue := recoveryStore(t)
|
||||
if tc.terminal {
|
||||
if _, err := store.AppendLifecycle(context.Background(), issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "finished", time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
probe := &signalProbeSupervisor{}
|
||||
executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, active: map[domain.UUID]supervisor.Process{issue: &failedCapture{}}, forced: make(map[domain.UUID]bool)}
|
||||
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: tc.revision, Signal: tc.signal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := lastSignalResult(t, store, issue)
|
||||
if result.GetAccepted() || result.GetCommandRevision() != tc.revision || !strings.Contains(result.GetDetail(), tc.detail) || result.GetGracefulDeliveryAttempted() || result.GetForcedTerminationUsed() || probe.calls != 0 {
|
||||
t.Fatalf("result=%v supervisor calls=%d", result, probe.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignalDeliveryAndFailureStayCommandLocal_HP_RUNTIME_05(t *testing.T) {
|
||||
for _, signal := range []rvboxv1.SignalKind{rvboxv1.SignalKind_SIGNAL_TERM, rvboxv1.SignalKind_SIGNAL_KILL} {
|
||||
for _, delivered := range []bool{true, false} {
|
||||
store, issue := recoveryStore(t)
|
||||
probe := &signalProbeSupervisor{outcome: supervisor.SignalOutcome{Delivered: delivered}}
|
||||
if !delivered {
|
||||
probe.err = errors.New("process exited during signal")
|
||||
}
|
||||
executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, active: map[domain.UUID]supervisor.Process{issue: &failedCapture{}}, forced: make(map[domain.UUID]bool)}
|
||||
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: signal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := lastSignalResult(t, store, issue)
|
||||
if result.GetAccepted() != delivered || result.GetGracefulDeliveryAttempted() != (signal == rvboxv1.SignalKind_SIGNAL_TERM) || probe.calls != 1 {
|
||||
t.Fatalf("result=%v supervisor calls=%d", result, probe.calls)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignalProtocolAndStoreFailuresRemainErrors_BH_RUNTIME_11(t *testing.T) {
|
||||
store, issue := recoveryStore(t)
|
||||
executor := &Executor{Store: store, Now: time.Now}
|
||||
for _, input := range []*rvboxv1.SignalCommand{
|
||||
nil,
|
||||
{IssueUuid: issue.String(), Signal: rvboxv1.SignalKind_SIGNAL_KILL},
|
||||
{IssueUuid: "invalid", CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL},
|
||||
{IssueUuid: issue.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KIND_UNSPECIFIED},
|
||||
} {
|
||||
if err := executor.Signal(context.Background(), Session{}, input); err == nil {
|
||||
t.Fatalf("malformed signal was silently accepted: %v", input)
|
||||
}
|
||||
}
|
||||
unknown, _ := domain.NewUUIDv7()
|
||||
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: unknown.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL}); !errors.Is(err, spool.ErrUnknownCommand) {
|
||||
t.Fatalf("unknown owner error lost: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if err := executor.Signal(ctx, Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL}); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancelled durable query was reported as successful signal rejection: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user