Files
rvbox/internal/client/agent/signal_recovery_test.go
T

163 lines
7.4 KiB
Go

package agent
import (
"context"
"errors"
"strings"
"testing"
"time"
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/client/spool"
"github.com/rvbox/rvbox/internal/client/supervisor"
"github.com/rvbox/rvbox/internal/domain"
)
type signalProbeSupervisor struct {
supervisor.Supervisor
calls int
outcome supervisor.SignalOutcome
err error
}
func (probe *signalProbeSupervisor) Signal(context.Context, supervisor.Process, supervisor.SignalKind) (supervisor.SignalOutcome, error) {
probe.calls++
return probe.outcome, probe.err
}
func TestStoppedSignalIsDurablyRejected_BH_RUNTIME_09(t *testing.T) {
for _, phase := range []rvboxv1.CommandLifecycle{
rvboxv1.CommandLifecycle_COMMAND_ACCEPTED,
rvboxv1.CommandLifecycle_COMMAND_RUNNING,
rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
} {
for _, signal := range []rvboxv1.SignalKind{rvboxv1.SignalKind_SIGNAL_TERM, rvboxv1.SignalKind_SIGNAL_KILL} {
t.Run(phase.String()+"/"+signal.String(), func(t *testing.T) {
store, issue := recoveryStore(t)
if phase != rvboxv1.CommandLifecycle_COMMAND_RUNNING {
if phase == rvboxv1.CommandLifecycle_COMMAND_ACCEPTED {
// Use a separate accepted row rather than reversing its lifecycle.
issue, _ = domain.NewUUIDv7()
if _, err := store.AcceptCommand(context.Background(), spool.Command{IssueUUID: issue, Revision: 1, Phase: uint32(phase), ImmutableSHA256: [32]byte{1}}, time.Now()); err != nil {
t.Fatal(err)
}
} else if _, err := store.AppendLifecycle(context.Background(), issue, uint32(phase), 1, "finished", time.Now()); err != nil {
t.Fatal(err)
}
}
probe := &signalProbeSupervisor{}
notified := false
executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, Notify: func(domain.UUID) { notified = true }}
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: signal}); err != nil {
t.Fatalf("ordinary signal rejection broke session: %v", err)
}
result := lastSignalResult(t, store, issue)
if result.GetAccepted() || result.GetCommandRevision() != 1 || result.GetSignal() != signal || result.GetGracefulDeliveryAttempted() || result.GetForcedTerminationUsed() || !strings.Contains(result.GetDetail(), "not running") || probe.calls != 0 || !notified {
t.Fatalf("result=%v supervisor calls=%d notified=%v", result, probe.calls, notified)
}
command, err := store.GetCommand(context.Background(), issue)
if err != nil || command.Phase != uint32(phase) {
t.Fatalf("signal rejection rewrote lifecycle: %v %v", command, err)
}
})
}
}
}
func lastSignalResult(t *testing.T, store *spool.Store, issue domain.UUID) *rvboxv1.SignalResult {
t.Helper()
if _, err := store.AssignSendWindow(context.Background(), issue, 128, 1<<20); err != nil {
t.Fatal(err)
}
events, err := store.PendingEvents(context.Background(), issue)
if err != nil || len(events) == 0 {
t.Fatalf("missing durable signal result: %v", err)
}
transport := &runnerTransport{}
if err := SendStoredEvent(context.Background(), transport, Session{ID: "session", Generation: 1}, events[len(events)-1], agentproto.DefaultLimits()); err != nil {
t.Fatal(err)
}
frame, err := agentproto.DecodeEnvelope(transport.writes[0], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
if err != nil || frame.GetCommandEvent().GetSignalResult() == nil {
t.Fatalf("invalid signal result frame: %v %v", frame, err)
}
return frame.GetCommandEvent().GetSignalResult()
}
func TestSignalRejectionsDoNotTouchProcess_BH_RUNTIME_10(t *testing.T) {
for _, tc := range []struct {
name string
revision uint64
signal rvboxv1.SignalKind
terminal bool
detail string
}{
{"stale revision", 2, rvboxv1.SignalKind_SIGNAL_KILL, false, "revision"},
{"unsupported Windows signal", 1, rvboxv1.SignalKind_SIGNAL_HUP, false, "unsupported"},
{"process exit before active-map removal", 1, rvboxv1.SignalKind_SIGNAL_KILL, true, "not running"},
} {
t.Run(tc.name, func(t *testing.T) {
store, issue := recoveryStore(t)
if tc.terminal {
if _, err := store.AppendLifecycle(context.Background(), issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "finished", time.Now()); err != nil {
t.Fatal(err)
}
}
probe := &signalProbeSupervisor{}
executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, active: map[domain.UUID]supervisor.Process{issue: &failedCapture{}}, forced: make(map[domain.UUID]bool)}
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: tc.revision, Signal: tc.signal}); err != nil {
t.Fatal(err)
}
result := lastSignalResult(t, store, issue)
if result.GetAccepted() || result.GetCommandRevision() != tc.revision || !strings.Contains(result.GetDetail(), tc.detail) || result.GetGracefulDeliveryAttempted() || result.GetForcedTerminationUsed() || probe.calls != 0 {
t.Fatalf("result=%v supervisor calls=%d", result, probe.calls)
}
})
}
}
func TestSignalDeliveryAndFailureStayCommandLocal_HP_RUNTIME_05(t *testing.T) {
for _, signal := range []rvboxv1.SignalKind{rvboxv1.SignalKind_SIGNAL_TERM, rvboxv1.SignalKind_SIGNAL_KILL} {
for _, delivered := range []bool{true, false} {
store, issue := recoveryStore(t)
probe := &signalProbeSupervisor{outcome: supervisor.SignalOutcome{Delivered: delivered}}
if !delivered {
probe.err = errors.New("process exited during signal")
}
executor := &Executor{Store: store, Now: time.Now, Supervisor: probe, active: map[domain.UUID]supervisor.Process{issue: &failedCapture{}}, forced: make(map[domain.UUID]bool)}
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: signal}); err != nil {
t.Fatal(err)
}
result := lastSignalResult(t, store, issue)
if result.GetAccepted() != delivered || result.GetGracefulDeliveryAttempted() != (signal == rvboxv1.SignalKind_SIGNAL_TERM) || probe.calls != 1 {
t.Fatalf("result=%v supervisor calls=%d", result, probe.calls)
}
}
}
}
func TestSignalProtocolAndStoreFailuresRemainErrors_BH_RUNTIME_11(t *testing.T) {
store, issue := recoveryStore(t)
executor := &Executor{Store: store, Now: time.Now}
for _, input := range []*rvboxv1.SignalCommand{
nil,
{IssueUuid: issue.String(), Signal: rvboxv1.SignalKind_SIGNAL_KILL},
{IssueUuid: "invalid", CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL},
{IssueUuid: issue.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KIND_UNSPECIFIED},
} {
if err := executor.Signal(context.Background(), Session{}, input); err == nil {
t.Fatalf("malformed signal was silently accepted: %v", input)
}
}
unknown, _ := domain.NewUUIDv7()
if err := executor.Signal(context.Background(), Session{}, &rvboxv1.SignalCommand{IssueUuid: unknown.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL}); !errors.Is(err, spool.ErrUnknownCommand) {
t.Fatalf("unknown owner error lost: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if err := executor.Signal(ctx, Session{}, &rvboxv1.SignalCommand{IssueUuid: issue.String(), CommandRevision: 1, Signal: rvboxv1.SignalKind_SIGNAL_KILL}); !errors.Is(err, context.Canceled) {
t.Fatalf("cancelled durable query was reported as successful signal rejection: %v", err)
}
}