fix: recover reconnect after incomplete output capture
This commit is contained in:
1 parent
b52ca5a7c2
commit
6906ed52f2
9 files changed
+692
-3
No files matched your search
@@ -1921,9 +1921,27 @@ reconciling -> active -> closing`. Only pipe/supervisor workers outlive the
|
|||||||
replaceable network session. After welcome, send the complete snapshot, apply
|
replaceable network session. After welcome, send the complete snapshot, apply
|
||||||
`ReconcileResult` durably, replay assigned rows first, then assign/send local
|
`ReconcileResult` durably, replay assigned rows first, then assign/send local
|
||||||
rows in order. Advertise capacity and accept new dispatch only in `active`.
|
rows in order. Advertise capacity and accept new dispatch only in `active`.
|
||||||
|
Re-read the durable snapshot after applying reconciliation: discarded terminal
|
||||||
|
rows are now tombstones and must not be replayed from the pre-apply snapshot.
|
||||||
|
Stale internal event-ready hints for removed rows are harmless, while unknown
|
||||||
|
wire acknowledgements remain errors. Every cumulative acknowledgement releases
|
||||||
|
send-window credit and must flush any durable tail without waiting for another
|
||||||
|
producer notification. Cancel the server writer before joining it on reader
|
||||||
|
exit, so session cleanup does not wait for a heartbeat tick.
|
||||||
Cancellation of one session context must join all its readers/writers before a
|
Cancellation of one session context must join all its readers/writers before a
|
||||||
new session can use their queues.
|
new session can use their queues.
|
||||||
|
|
||||||
|
Incomplete-output closeout events must name stdout/stderr scope, retain the
|
||||||
|
capture reason, consume reserved closeout space when normal allocation is full,
|
||||||
|
and precede terminal lifecycle. Early v1 clients persisted markers without any
|
||||||
|
stream scope and rejected them before transmission. On stored-event readback,
|
||||||
|
only that known legacy shape is normalized to both streams before deriving its
|
||||||
|
wire digest; preserve the persisted payload/hash, ordinal, sequence, observation
|
||||||
|
time, and reason. Explicit invalid/duplicate stream values still fail validation.
|
||||||
|
Verify restart replay, deterministic digests, strict rejection, quota closeout,
|
||||||
|
terminal-discard wakeup races, and fresh dispatch after real WebSocket reconnect
|
||||||
|
with `BH-RUNTIME-05..08` and `REC-RUNTIME-01..04`.
|
||||||
|
|
||||||
The current implementation checkpoint is intentionally split at this seam:
|
The current implementation checkpoint is intentionally split at this seam:
|
||||||
`internal/client/agent.RunOnce` owns the reconnect/session reader and remains
|
`internal/client/agent.RunOnce` owns the reconnect/session reader and remains
|
||||||
the sole live-session writer; `internal/client/spool` owns the SQLite source of
|
the sole live-session writer; `internal/client/spool` owns the SQLite source of
|
||||||
|
|||||||
@@ -170,6 +170,34 @@ The Windows artifact is linked with the GUI subsystem (`-H=windowsgui`) so
|
|||||||
service, launcher, and tray startup do not flash a console. Human-facing modes
|
service, launcher, and tray startup do not flash a console. Human-facing modes
|
||||||
still attach to a parent console explicitly when one exists.
|
still attach to a parent console explicitly when one exists.
|
||||||
|
|
||||||
|
### Persisted output-loss and reconnect regressions
|
||||||
|
|
||||||
|
`BH-RUNTIME-05` through `08` cover a capture-reader failure producing a valid,
|
||||||
|
normally ordered incomplete-output event before terminal state; replay of an
|
||||||
|
early v1 event with missing stream scope after reopening its original spool;
|
||||||
|
deterministic replay digests without rewriting stored evidence; continued
|
||||||
|
rejection of invalid/duplicate explicit streams; and reserved closeout under
|
||||||
|
normal command-quota exhaustion. The known legacy repair conservatively names
|
||||||
|
stdout and stderr. It does not weaken the protocol validator or delete history.
|
||||||
|
|
||||||
|
`REC-RUNTIME-01` through `03` cover replay from the post-reconciliation snapshot,
|
||||||
|
stale internal wakeups after terminal discard, and acknowledgement credit
|
||||||
|
draining a durable tail without a new producer notification. `REC-RUNTIME-04`
|
||||||
|
uses a real WebSocket server and durable client/server stores: deliver a legacy
|
||||||
|
loss marker plus terminal result, restart the same client spool, discard the
|
||||||
|
server-confirmed terminal row, and complete a fresh command. It also requires
|
||||||
|
prompt server-session cleanup rather than waiting for a heartbeat tick.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker compose -f deploy/compose.yaml run --rm toolchain \
|
||||||
|
go test -race ./internal/client/agent ./test/integration/clientagent
|
||||||
|
```
|
||||||
|
|
||||||
|
The runtime unit cases are platform-neutral and can also be compiled as a
|
||||||
|
Windows console test executable. Inspect the persisted typed loss event itself
|
||||||
|
when checking evidence: the existing server's GetOutput summary does not yet
|
||||||
|
derive its incomplete flag/stream details from every incoming peer loss marker.
|
||||||
|
|
||||||
### Focused native tray and installer regression checks
|
### Focused native tray and installer regression checks
|
||||||
|
|
||||||
`internal/client/windowstray/native_windows_test.go` contains real Win32 tests,
|
`internal/client/windowstray/native_windows_test.go` contains real Win32 tests,
|
||||||
|
|||||||
@@ -252,11 +252,15 @@ func (executor *Executor) rejectWithIdentity(ctx context.Context, issue domain.U
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (executor *Executor) appendIncomplete(ctx context.Context, issue domain.UUID, _ uint64, detail string) error {
|
func (executor *Executor) appendIncomplete(ctx context.Context, issue domain.UUID, _ uint64, detail string) error {
|
||||||
payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(&rvboxv1.CommandEvent{IssueUuid: issue.String(), ObservedAt: timestamppb.New(executor.Now()), Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{Reason: boundedError(errors.New(detail))}}})
|
payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(&rvboxv1.CommandEvent{IssueUuid: issue.String(), ObservedAt: timestamppb.New(executor.Now()), Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{
|
||||||
|
// A capture failure does not prove either stream was complete.
|
||||||
|
Streams: []rvboxv1.StreamKind{rvboxv1.StreamKind_STREAM_STDOUT, rvboxv1.StreamKind_STREAM_STDERR},
|
||||||
|
Reason: boundedError(errors.New(detail)),
|
||||||
|
}}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
_, err = executor.Store.AppendEvent(ctx, issue, spool.EventInput{Kind: 11, Compression: 1, RawBytes: uint64(len(payload)), Payload: payload, CreatedAt: executor.Now()})
|
_, err = executor.Store.AppendEvent(ctx, issue, spool.EventInput{Kind: spool.EventKindOutputIncomplete, Compression: 1, RawBytes: uint64(len(payload)), Payload: payload, UseCloseout: true, CreatedAt: executor.Now()})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
executor.notify(issue)
|
executor.notify(issue)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,383 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||||
|
"github.com/rvbox/rvbox/internal/agentproto"
|
||||||
|
"github.com/rvbox/rvbox/internal/client/spool"
|
||||||
|
"github.com/rvbox/rvbox/internal/client/supervisor"
|
||||||
|
"github.com/rvbox/rvbox/internal/domain"
|
||||||
|
"google.golang.org/protobuf/proto"
|
||||||
|
"google.golang.org/protobuf/types/known/timestamppb"
|
||||||
|
)
|
||||||
|
|
||||||
|
func recoveryStore(t *testing.T) (*spool.Store, domain.UUID) {
|
||||||
|
t.Helper()
|
||||||
|
return recoveryStoreAt(t, filepath.Join(t.TempDir(), "spool"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func recoveryStoreAt(t *testing.T, directory string) (*spool.Store, domain.UUID) {
|
||||||
|
t.Helper()
|
||||||
|
store, err := spool.Open(context.Background(), spool.Options{DataDir: directory, BusyTimeout: time.Second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = store.Close() })
|
||||||
|
issue, err := domain.NewUUIDv7()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := store.AcceptCommand(context.Background(), spool.Command{IssueUUID: issue, Revision: 1, Phase: uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), ImmutableSHA256: [32]byte{1}}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return store, issue
|
||||||
|
}
|
||||||
|
|
||||||
|
func recoveryEvent(t *testing.T, store *spool.Store, issue domain.UUID, event *rvboxv1.CommandEvent, kind uint32) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := proto.Marshal(event)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := store.AppendEvent(context.Background(), issue, spool.EventInput{Kind: kind, Compression: 1, RawBytes: uint64(len(data)), Payload: data, CreatedAt: time.Now()}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func recoveryOptions(store *spool.Store, transport Transport) RunnerOptions {
|
||||||
|
return RunnerOptions{Store: store, Dial: func(context.Context) (Transport, error) { return transport, nil }, Hello: &rvboxv1.ClientHello{
|
||||||
|
ClientId: "recovery-client", ClientInstanceId: store.ClientInstanceID().String(),
|
||||||
|
SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1}, DaemonVersion: "test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS,
|
||||||
|
Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_CMD},
|
||||||
|
MaxRunningCommands: 16, MaxQueuedCommands: 100, SentAt: timestamppb.Now(),
|
||||||
|
}, Limits: agentproto.DefaultLimits(), Backoff: BackoffOptions{Initial: time.Millisecond, Maximum: time.Millisecond, StableReset: time.Second}, Jitter: func(time.Duration) time.Duration { return 0 }, Now: time.Now}
|
||||||
|
}
|
||||||
|
|
||||||
|
func recoveryHandshake(t *testing.T, result *rvboxv1.ReconcileResult) *runnerTransport {
|
||||||
|
t.Helper()
|
||||||
|
transport := &runnerTransport{terminal: errors.New("fixture disconnected")}
|
||||||
|
for _, payload := range []*rvboxv1.AgentEnvelope{
|
||||||
|
{Payload: &rvboxv1.AgentEnvelope_ServerWelcome{ServerWelcome: &rvboxv1.ServerWelcome{SelectedProtocol: &rvboxv1.ProtocolVersion{Major: 1}, ServerTime: timestamppb.Now()}}},
|
||||||
|
{Payload: &rvboxv1.AgentEnvelope_ReconcileRequest{ReconcileRequest: &rvboxv1.ReconcileRequest{}}},
|
||||||
|
{Payload: &rvboxv1.AgentEnvelope_ReconcileResult{ReconcileResult: result}},
|
||||||
|
} {
|
||||||
|
payload.SessionId, payload.SessionGeneration = "recovery-session", 1
|
||||||
|
encoded, err := proto.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
transport.reads = append(transport.reads, encoded)
|
||||||
|
}
|
||||||
|
return transport
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCaptureFailurePublishesValidIncompleteBeforeTerminal_BH_RUNTIME_05(t *testing.T) {
|
||||||
|
store, issue := recoveryStore(t)
|
||||||
|
executor := &Executor{Store: store, Now: time.Now}
|
||||||
|
executor.watch(context.Background(), issue, 1, &failedCapture{}, func() {})
|
||||||
|
if _, err := store.AssignSendWindow(context.Background(), issue, 128, 1<<20); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
events, err := store.PendingEvents(context.Background(), issue)
|
||||||
|
if err != nil || len(events) != 2 {
|
||||||
|
t.Fatalf("capture closeout events=%d err=%v", len(events), err)
|
||||||
|
}
|
||||||
|
transport := &runnerTransport{}
|
||||||
|
for _, event := range events {
|
||||||
|
if err := SendStoredEvent(context.Background(), transport, Session{ID: "session", Generation: 1}, event, agentproto.DefaultLimits()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
first, err := agentproto.DecodeEnvelope(transport.writes[0], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
marker := first.GetCommandEvent().GetOutputIncomplete()
|
||||||
|
if marker == nil || len(marker.GetStreams()) != 2 || marker.GetReason() != "capture reader failed" {
|
||||||
|
t.Fatalf("incomplete marker=%v", marker)
|
||||||
|
}
|
||||||
|
last, err := agentproto.DecodeEnvelope(transport.writes[1], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
|
||||||
|
if err != nil || last.GetCommandEvent().GetLifecycle().GetLifecycle() != rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED {
|
||||||
|
t.Fatalf("terminal event=%v err=%v", last, err)
|
||||||
|
}
|
||||||
|
if _, err := store.Check(context.Background()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type failedCapture struct{}
|
||||||
|
|
||||||
|
func (*failedCapture) IssueUUID() domain.UUID { return domain.UUID{} }
|
||||||
|
func (*failedCapture) Identity() supervisor.EffectiveIdentity { return supervisor.EffectiveIdentity{} }
|
||||||
|
func (*failedCapture) PID() uint32 { return 0 }
|
||||||
|
func (*failedCapture) Release(context.Context) error { return nil }
|
||||||
|
func (*failedCapture) ReadOutput(context.Context) (supervisor.OutputChunk, error) {
|
||||||
|
return supervisor.OutputChunk{}, errors.New("capture reader failed")
|
||||||
|
}
|
||||||
|
func (*failedCapture) Wait(context.Context) (supervisor.ExitStatus, error) {
|
||||||
|
return supervisor.ExitStatus{Code: 0}, nil
|
||||||
|
}
|
||||||
|
func (*failedCapture) WriteStdin(context.Context, []byte, bool) error { return nil }
|
||||||
|
func (*failedCapture) CloseStdin(context.Context) error { return nil }
|
||||||
|
|
||||||
|
func TestLegacyIncompleteReplaySurvivesRestart_BH_RUNTIME_06(t *testing.T) {
|
||||||
|
directory := filepath.Join(t.TempDir(), "spool")
|
||||||
|
store, issue := recoveryStoreAt(t, directory)
|
||||||
|
recoveryEvent(t, store, issue, &rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{Reason: "legacy capture failure"}}}, spool.EventKindOutputIncomplete)
|
||||||
|
if _, err := store.AppendLifecycle(context.Background(), issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "exit code 0", time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := store.AssignSendWindow(context.Background(), issue, 128, 1<<20); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before, err := store.PendingEvents(context.Background(), issue)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := store.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
store, err = spool.Open(context.Background(), spool.Options{DataDir: directory, BusyTimeout: time.Second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer store.Close()
|
||||||
|
var digest []byte
|
||||||
|
for attempt := 0; attempt < 2; attempt++ {
|
||||||
|
transport := recoveryHandshake(t, &rvboxv1.ReconcileResult{})
|
||||||
|
active := false
|
||||||
|
options := recoveryOptions(store, transport)
|
||||||
|
options.OnSessionActive = func(value bool) { active = active || value }
|
||||||
|
if err := RunOnce(context.Background(), options); !errors.Is(err, transport.terminal) {
|
||||||
|
t.Fatalf("legacy replay error=%v", err)
|
||||||
|
}
|
||||||
|
if !active || len(transport.writes) != 5 {
|
||||||
|
t.Fatalf("active=%v writes=%d", active, len(transport.writes))
|
||||||
|
}
|
||||||
|
envelope, err := agentproto.DecodeEnvelope(transport.writes[2], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
event := envelope.GetCommandEvent()
|
||||||
|
if event.GetEventSeq() != 1 || event.GetOutputIncomplete().GetReason() != "legacy capture failure" || len(event.GetOutputIncomplete().GetStreams()) != 2 {
|
||||||
|
t.Fatalf("recovered event=%v", event)
|
||||||
|
}
|
||||||
|
if attempt == 0 {
|
||||||
|
digest = append([]byte(nil), event.GetImmutableEventSha256()...)
|
||||||
|
} else if !bytes.Equal(digest, event.GetImmutableEventSha256()) {
|
||||||
|
t.Fatal("legacy replay digest changed across reconnect")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
after, err := store.PendingEvents(context.Background(), issue)
|
||||||
|
if err != nil || len(after) != len(before) || !bytes.Equal(before[0].Payload, after[0].Payload) {
|
||||||
|
t.Fatalf("legacy persisted evidence changed: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Check(context.Background()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReconciliationDiscardUsesFreshReplaySnapshot_REC_RUNTIME_01(t *testing.T) {
|
||||||
|
store, issue := recoveryStore(t)
|
||||||
|
if _, err := store.AppendLifecycle(context.Background(), issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "done", time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
transport := recoveryHandshake(t, &rvboxv1.ReconcileResult{DiscardLocalTerminalIssueUuids: []string{issue.String()}})
|
||||||
|
options := recoveryOptions(store, transport)
|
||||||
|
if err := RunOnce(context.Background(), options); !errors.Is(err, transport.terminal) {
|
||||||
|
t.Fatalf("discarded command poisoned replay: %v", err)
|
||||||
|
}
|
||||||
|
if len(transport.writes) != 3 {
|
||||||
|
t.Fatalf("discarded rows replayed: writes=%d", len(transport.writes))
|
||||||
|
}
|
||||||
|
if _, err := store.GetCommand(context.Background(), issue); !errors.Is(err, spool.ErrUnknownCommand) {
|
||||||
|
t.Fatalf("discarded command remains: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Check(context.Background()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLegacyIncompleteCompatibilityKeepsStreamValidationStrict_BH_RUNTIME_07(t *testing.T) {
|
||||||
|
issue, _ := domain.NewUUIDv7()
|
||||||
|
for _, streams := range [][]rvboxv1.StreamKind{{rvboxv1.StreamKind_STREAM_KIND_UNSPECIFIED}, {99}, {rvboxv1.StreamKind_STREAM_STDOUT, rvboxv1.StreamKind_STREAM_STDOUT}} {
|
||||||
|
payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{Streams: streams, Reason: "bad stream"}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
transport := &runnerTransport{}
|
||||||
|
err = SendStoredEvent(context.Background(), transport, Session{ID: "session", Generation: 1}, spool.Event{IssueUUID: issue, EventSeq: 1, Kind: spool.EventKindOutputIncomplete, CreatedAt: time.Now(), Payload: payload}, agentproto.DefaultLimits())
|
||||||
|
if !errors.Is(err, agentproto.ErrInvalidEnvelope) || len(transport.writes) != 0 {
|
||||||
|
t.Fatalf("streams=%v err=%v writes=%d", streams, err, len(transport.writes))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIncompleteCloseoutSurvivesNormalQuotaExhaustion_BH_RUNTIME_08(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
limits := spool.DefaultQuotaLimits()
|
||||||
|
limits.CommandOutputBytes, limits.CommandTotalBytes = 4<<10, 12<<10
|
||||||
|
limits.ClientTotalBytes, limits.CloseoutReserveBytes = 32<<10, 4<<10
|
||||||
|
store, err := spool.Open(ctx, spool.Options{DataDir: filepath.Join(t.TempDir(), "spool"), BusyTimeout: time.Second, QuotaLimits: limits})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer store.Close()
|
||||||
|
issue, _ := domain.NewUUIDv7()
|
||||||
|
if _, err := store.AcceptCommand(ctx, spool.Command{IssueUUID: issue, Revision: 1, Phase: uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), ImmutableSHA256: [32]byte{1}}, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_Resource{Resource: &rvboxv1.ResourceSnapshot{}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
full := false
|
||||||
|
for i := 0; i < 100; i++ {
|
||||||
|
_, err := store.AppendEvent(ctx, issue, spool.EventInput{Kind: 6, Compression: 1, Payload: payload, RawBytes: uint64(len(payload)), CreatedAt: time.Now()})
|
||||||
|
if errors.Is(err, spool.ErrCapacityExhausted) {
|
||||||
|
full = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !full {
|
||||||
|
t.Fatal("normal allocation did not reach its quota")
|
||||||
|
}
|
||||||
|
executor := &Executor{Store: store, Now: time.Now}
|
||||||
|
if err := executor.appendIncomplete(ctx, issue, 1, "normal command quota exhausted"); err != nil {
|
||||||
|
t.Fatalf("reserved incomplete closeout rejected: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "done", time.Now()); err != nil {
|
||||||
|
t.Fatalf("reserved terminal closeout rejected: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Check(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type recoveryTransport struct {
|
||||||
|
reads chan []byte
|
||||||
|
writes chan []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRecoveryTransport() *recoveryTransport {
|
||||||
|
return &recoveryTransport{reads: make(chan []byte, 1), writes: make(chan []byte, 256)}
|
||||||
|
}
|
||||||
|
func (transport *recoveryTransport) Read(ctx context.Context) ([]byte, error) {
|
||||||
|
select {
|
||||||
|
case data := <-transport.reads:
|
||||||
|
return data, nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (transport *recoveryTransport) Write(ctx context.Context, data []byte) error {
|
||||||
|
select {
|
||||||
|
case transport.writes <- append([]byte(nil), data...):
|
||||||
|
return nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (*recoveryTransport) Close() error { return nil }
|
||||||
|
|
||||||
|
func recoveryFrame(t *testing.T, envelope *rvboxv1.AgentEnvelope) []byte {
|
||||||
|
t.Helper()
|
||||||
|
envelope.SessionId, envelope.SessionGeneration = "session", 1
|
||||||
|
data, err := proto.Marshal(envelope)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscardedCommandNotificationDoesNotDisconnect_REC_RUNTIME_02(t *testing.T) {
|
||||||
|
store, live := recoveryStore(t)
|
||||||
|
stale, _ := domain.NewUUIDv7()
|
||||||
|
notify := make(chan domain.UUID)
|
||||||
|
transport := newRecoveryTransport()
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
options := recoveryOptions(store, transport)
|
||||||
|
options.EventReady = notify
|
||||||
|
options.OnStdin = func(ctx context.Context, _ Session, write *rvboxv1.StdinWrite) error {
|
||||||
|
payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_Resource{Resource: &rvboxv1.ResourceSnapshot{DiagnosticDetail: "active after stale wakeup"}}})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = store.AppendEvent(ctx, live, spool.EventInput{Kind: 6, Compression: 1, RawBytes: uint64(len(payload)), Payload: payload, CreatedAt: time.Now()})
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
done <- serveActive(ctx, transport, options, Session{ID: "session", Generation: 1}, make(map[domain.UUID]uint64))
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case notify <- stale:
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("before wakeup: %v", err)
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("wakeup not consumed")
|
||||||
|
}
|
||||||
|
transport.reads <- recoveryFrame(t, &rvboxv1.AgentEnvelope{Payload: &rvboxv1.AgentEnvelope_StdinWrite{StdinWrite: &rvboxv1.StdinWrite{IssueUuid: live.String(), WriteSeq: 1, Data: []byte("probe")}}})
|
||||||
|
select {
|
||||||
|
case data := <-transport.writes:
|
||||||
|
envelope, err := agentproto.DecodeEnvelope(data, agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
|
||||||
|
if err != nil || envelope.GetCommandEvent().GetResource().GetDiagnosticDetail() != "active after stale wakeup" {
|
||||||
|
t.Fatalf("liveness response=%v err=%v", envelope, err)
|
||||||
|
}
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("stale wakeup disconnected active session: %v", err)
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("no progress after stale wakeup")
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
<-done
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEventAcknowledgementDrainsDurableTail_REC_RUNTIME_03(t *testing.T) {
|
||||||
|
store, issue := recoveryStore(t)
|
||||||
|
for i := 0; i < 130; i++ {
|
||||||
|
recoveryEvent(t, store, issue, &rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_Resource{Resource: &rvboxv1.ResourceSnapshot{}}}, 6)
|
||||||
|
}
|
||||||
|
transport := newRecoveryTransport()
|
||||||
|
sent := make(map[domain.UUID]uint64)
|
||||||
|
session := Session{ID: "session", Generation: 1}
|
||||||
|
if err := flushEvents(context.Background(), transport, store, session, issue, agentproto.DefaultLimits(), sent); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(transport.writes) != 128 {
|
||||||
|
t.Fatalf("first send window=%d", len(transport.writes))
|
||||||
|
}
|
||||||
|
for i := 0; i < 128; i++ {
|
||||||
|
<-transport.writes
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- serveActive(ctx, transport, recoveryOptions(store, transport), session, sent) }()
|
||||||
|
transport.reads <- recoveryFrame(t, &rvboxv1.AgentEnvelope{Payload: &rvboxv1.AgentEnvelope_EventAck{EventAck: &rvboxv1.EventAck{IssueUuid: issue.String(), ThroughEventSeq: 128}}})
|
||||||
|
for want := uint64(129); want <= 130; want++ {
|
||||||
|
select {
|
||||||
|
case data := <-transport.writes:
|
||||||
|
envelope, err := agentproto.DecodeEnvelope(data, agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
|
||||||
|
if err != nil || envelope.GetCommandEvent().GetEventSeq() != want {
|
||||||
|
t.Fatalf("tail sequence want=%d event=%v err=%v", want, envelope, err)
|
||||||
|
}
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("acknowledgement disconnected session: %v", err)
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("acknowledged credit did not release pending tail")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
<-done
|
||||||
|
}
|
||||||
@@ -209,6 +209,12 @@ func runOnce(ctx context.Context, options RunnerOptions) (resultErr error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Applying reconciliation can replace terminal rows with tombstones and
|
||||||
|
// terminate formerly active commands. Replay/capacity must use that new view.
|
||||||
|
snapshot, err = options.Store.ReconcileSnapshot(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("refresh reconciled client snapshot: %w", err)
|
||||||
|
}
|
||||||
sentEvents := make(map[domain.UUID]uint64)
|
sentEvents := make(map[domain.UUID]uint64)
|
||||||
if err := replayEvents(ctx, transport, options.Store, session, snapshot, limits, sentEvents); err != nil {
|
if err := replayEvents(ctx, transport, options.Store, session, snapshot, limits, sentEvents); err != nil {
|
||||||
return fmt.Errorf("replay client events: %w", err)
|
return fmt.Errorf("replay client events: %w", err)
|
||||||
@@ -294,6 +300,12 @@ func commandEventFromStored(event spool.Event) (*rvboxv1.CommandEvent, error) {
|
|||||||
if err := proto.Unmarshal(event.Payload, &stored); err != nil || stored.Payload == nil {
|
if err := proto.Unmarshal(event.Payload, &stored); err != nil || stored.Payload == nil {
|
||||||
return nil, fmt.Errorf("decode stored command event: %w", err)
|
return nil, fmt.Errorf("decode stored command event: %w", err)
|
||||||
}
|
}
|
||||||
|
if incomplete := stored.GetOutputIncomplete(); event.Kind == spool.EventKindOutputIncomplete && incomplete != nil && len(incomplete.Streams) == 0 {
|
||||||
|
// Early v1 writers omitted streams. Those events failed local wire
|
||||||
|
// validation before any server could accept their digest. Conservatively
|
||||||
|
// name both streams without changing persisted bytes, order, or reason.
|
||||||
|
incomplete.Streams = []rvboxv1.StreamKind{rvboxv1.StreamKind_STREAM_STDOUT, rvboxv1.StreamKind_STREAM_STDERR}
|
||||||
|
}
|
||||||
return &stored, nil
|
return &stored, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -362,6 +374,11 @@ func serveActive(ctx context.Context, transport Transport, options RunnerOptions
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if err := flushEvents(ctx, transport, options.Store, session, issue, limits, sent); err != nil {
|
if err := flushEvents(ctx, transport, options.Store, session, issue, limits, sent); err != nil {
|
||||||
|
if errors.Is(err, spool.ErrUnknownCommand) {
|
||||||
|
// This internal wakeup can outlive a server-confirmed discard.
|
||||||
|
// It is a hint, not a new dispatch or an event acknowledgement.
|
||||||
|
continue
|
||||||
|
}
|
||||||
return fmt.Errorf("flush ready command %s events: %w", issue, err)
|
return fmt.Errorf("flush ready command %s events: %w", issue, err)
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
@@ -391,6 +408,12 @@ func serveActive(ctx context.Context, transport Transport, options RunnerOptions
|
|||||||
if err := ApplyEventAck(ctx, options.Store, envelope.GetEventAck()); err != nil {
|
if err := ApplyEventAck(ctx, options.Store, envelope.GetEventAck()); err != nil {
|
||||||
return fmt.Errorf("apply event acknowledgement for %s: %w", envelope.GetEventAck().GetIssueUuid(), err)
|
return fmt.Errorf("apply event acknowledgement for %s: %w", envelope.GetEventAck().GetIssueUuid(), err)
|
||||||
}
|
}
|
||||||
|
issue, _ := domain.ParseUUIDv7(envelope.GetEventAck().GetIssueUuid())
|
||||||
|
// Released send-window credit must drain the durable tail even if no
|
||||||
|
// producer remains to emit another EventReady notification.
|
||||||
|
if err := flushEvents(ctx, transport, options.Store, session, issue, limits, sent); err != nil {
|
||||||
|
return fmt.Errorf("flush acknowledged command %s events: %w", issue, err)
|
||||||
|
}
|
||||||
case envelope.GetScriptChunk() != nil:
|
case envelope.GetScriptChunk() != nil:
|
||||||
if err := handleScriptChunk(ctx, transport, options.Store, session, envelope.GetScriptChunk(), limits, options.Now, sent); err != nil {
|
if err := handleScriptChunk(ctx, transport, options.Store, session, envelope.GetScriptChunk(), limits, options.Now, sent); err != nil {
|
||||||
return fmt.Errorf("apply script chunk for %s: %w", envelope.GetScriptChunk().GetIssueUuid(), err)
|
return fmt.Errorf("apply script chunk for %s: %w", envelope.GetScriptChunk().GetIssueUuid(), err)
|
||||||
|
|||||||
@@ -54,6 +54,7 @@ const (
|
|||||||
// reconstruct the envelope without a second event-kind translation table.
|
// reconstruct the envelope without a second event-kind translation table.
|
||||||
EventKindOutput uint32 = 5
|
EventKindOutput uint32 = 5
|
||||||
EventKindOutputTruncation uint32 = 10
|
EventKindOutputTruncation uint32 = 10
|
||||||
|
EventKindOutputIncomplete uint32 = 11
|
||||||
)
|
)
|
||||||
|
|
||||||
type Event struct {
|
type Event struct {
|
||||||
|
|||||||
@@ -232,7 +232,12 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
|
|||||||
server.writeLoop(sessionContext, connection, queue, heartbeat, started)
|
server.writeLoop(sessionContext, connection, queue, heartbeat, started)
|
||||||
cancel()
|
cancel()
|
||||||
}()
|
}()
|
||||||
defer func() { <-writerDone }()
|
defer func() {
|
||||||
|
// A reader can end before the writer observes the closed transport.
|
||||||
|
// Cancel first so cleanup does not wait for the next heartbeat write.
|
||||||
|
cancel()
|
||||||
|
<-writerDone
|
||||||
|
}()
|
||||||
|
|
||||||
for {
|
for {
|
||||||
messageType, payload, err = connection.Read(sessionContext)
|
messageType, payload, err = connection.Read(sessionContext)
|
||||||
|
|||||||
@@ -431,6 +431,54 @@ layer = "unit"
|
|||||||
status = "implemented"
|
status = "implemented"
|
||||||
tests = ["internal/client/agent/executor_test.go:TestExecutorRunsDurableCommandAndPublishesTerminalEvents_HP_EXECUTOR_01"]
|
tests = ["internal/client/agent/executor_test.go:TestExecutorRunsDurableCommandAndPublishesTerminalEvents_HP_EXECUTOR_01"]
|
||||||
|
|
||||||
|
[[requirements]]
|
||||||
|
id = "BH-RUNTIME-05"
|
||||||
|
layer = "unit"
|
||||||
|
status = "implemented"
|
||||||
|
tests = ["internal/client/agent/recovery_test.go:TestCaptureFailurePublishesValidIncompleteBeforeTerminal_BH_RUNTIME_05"]
|
||||||
|
|
||||||
|
[[requirements]]
|
||||||
|
id = "BH-RUNTIME-06"
|
||||||
|
layer = "unit"
|
||||||
|
status = "implemented"
|
||||||
|
tests = ["internal/client/agent/recovery_test.go:TestLegacyIncompleteReplaySurvivesRestart_BH_RUNTIME_06"]
|
||||||
|
|
||||||
|
[[requirements]]
|
||||||
|
id = "BH-RUNTIME-07"
|
||||||
|
layer = "unit"
|
||||||
|
status = "implemented"
|
||||||
|
tests = ["internal/client/agent/recovery_test.go:TestLegacyIncompleteCompatibilityKeepsStreamValidationStrict_BH_RUNTIME_07"]
|
||||||
|
|
||||||
|
[[requirements]]
|
||||||
|
id = "BH-RUNTIME-08"
|
||||||
|
layer = "unit"
|
||||||
|
status = "implemented"
|
||||||
|
tests = ["internal/client/agent/recovery_test.go:TestIncompleteCloseoutSurvivesNormalQuotaExhaustion_BH_RUNTIME_08"]
|
||||||
|
|
||||||
|
[[requirements]]
|
||||||
|
id = "REC-RUNTIME-01"
|
||||||
|
layer = "unit"
|
||||||
|
status = "implemented"
|
||||||
|
tests = ["internal/client/agent/recovery_test.go:TestReconciliationDiscardUsesFreshReplaySnapshot_REC_RUNTIME_01"]
|
||||||
|
|
||||||
|
[[requirements]]
|
||||||
|
id = "REC-RUNTIME-02"
|
||||||
|
layer = "unit"
|
||||||
|
status = "implemented"
|
||||||
|
tests = ["internal/client/agent/recovery_test.go:TestDiscardedCommandNotificationDoesNotDisconnect_REC_RUNTIME_02"]
|
||||||
|
|
||||||
|
[[requirements]]
|
||||||
|
id = "REC-RUNTIME-03"
|
||||||
|
layer = "unit"
|
||||||
|
status = "implemented"
|
||||||
|
tests = ["internal/client/agent/recovery_test.go:TestEventAcknowledgementDrainsDurableTail_REC_RUNTIME_03"]
|
||||||
|
|
||||||
|
[[requirements]]
|
||||||
|
id = "REC-RUNTIME-04"
|
||||||
|
layer = "integration"
|
||||||
|
status = "implemented"
|
||||||
|
tests = ["test/integration/clientagent/recovery_integration_test.go:TestLegacyOutputLossReconnectAndFreshDispatch_REC_RUNTIME_04"]
|
||||||
|
|
||||||
[[requirements]]
|
[[requirements]]
|
||||||
id = "HP-EXECUTOR-02"
|
id = "HP-EXECUTOR-02"
|
||||||
layer = "unit"
|
layer = "unit"
|
||||||
|
|||||||
@@ -0,0 +1,179 @@
|
|||||||
|
package clientagent_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http/httptest"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||||
|
"github.com/rvbox/rvbox/internal/agentproto"
|
||||||
|
"github.com/rvbox/rvbox/internal/client/agent"
|
||||||
|
"github.com/rvbox/rvbox/internal/client/spool"
|
||||||
|
"github.com/rvbox/rvbox/internal/domain"
|
||||||
|
"github.com/rvbox/rvbox/internal/server/control"
|
||||||
|
"github.com/rvbox/rvbox/internal/server/session"
|
||||||
|
"github.com/rvbox/rvbox/internal/server/store"
|
||||||
|
"google.golang.org/protobuf/proto"
|
||||||
|
"google.golang.org/protobuf/types/known/timestamppb"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLegacyOutputLossReconnectAndFreshDispatch_REC_RUNTIME_04(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
persistence, err := store.Open(ctx, store.Options{DataDir: filepath.Join(t.TempDir(), "server"), BusyTimeout: time.Second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer persistence.Close()
|
||||||
|
registry := session.NewRegistry()
|
||||||
|
server := httptest.NewServer(&session.AgentServer{Store: persistence, Registry: registry, Path: "/v1/agent", Limits: agentproto.DefaultLimits()})
|
||||||
|
defer server.Close()
|
||||||
|
controlService, err := control.NewService(control.Options{Store: persistence, WakeClient: registry.Wake})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
directory := filepath.Join(t.TempDir(), "client")
|
||||||
|
client, err := spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = client.Close() }()
|
||||||
|
hello := &rvboxv1.ClientHello{ClientId: "recovery-client", ClientInstanceId: client.ClientInstanceID().String(), SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1}, DaemonVersion: "legacy-recovery-test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS, Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_CMD}, MaxRunningCommands: 16, MaxQueuedCommands: 100, SentAt: timestamppb.Now()}
|
||||||
|
start := func() (context.CancelFunc, <-chan error) {
|
||||||
|
runCtx, cancel := context.WithCancel(ctx)
|
||||||
|
active := make(chan struct{}, 1)
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
done <- agent.RunOnce(runCtx, agent.RunnerOptions{
|
||||||
|
Store: client, Hello: hello, Limits: agentproto.DefaultLimits(), Now: time.Now,
|
||||||
|
Backoff: agent.BackoffOptions{Initial: time.Millisecond, Maximum: time.Millisecond, StableReset: time.Second}, Jitter: func(time.Duration) time.Duration { return 0 },
|
||||||
|
Dial: func(ctx context.Context) (agent.Transport, error) {
|
||||||
|
return agent.DialWebSocket(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/v1/agent", nil)
|
||||||
|
},
|
||||||
|
OnSessionActive: func(value bool) {
|
||||||
|
if value {
|
||||||
|
active <- struct{}{}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
OnDispatch: func(ctx context.Context, _ agent.Session, dispatch *rvboxv1.CommandDispatch) error {
|
||||||
|
issue, err := domain.ParseUUIDv7(dispatch.GetIssueUuid())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), 1, "fixture running", time.Now()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Reproduce the old writer's durable event exactly: no streams.
|
||||||
|
payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{Reason: "legacy reader failure"}}})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := client.AppendEvent(ctx, issue, spool.EventInput{Kind: spool.EventKindOutputIncomplete, Compression: 1, RawBytes: uint64(len(payload)), Payload: payload, CreatedAt: time.Now()}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "exit code 0", time.Now())
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-active:
|
||||||
|
case err := <-done:
|
||||||
|
cancel()
|
||||||
|
t.Fatalf("client failed to become active: %v", err)
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
cancel()
|
||||||
|
t.Fatal("client did not become active")
|
||||||
|
}
|
||||||
|
return cancel, done
|
||||||
|
}
|
||||||
|
queueAndAwait := func() string {
|
||||||
|
id, err := domain.NewUUIDv7()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
response, err := controlService.RunCommand(ctx, &rvboxv1.RunCommandRequest{TargetClientId: hello.ClientId, RequestId: id.String(), Spec: &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "echo fixture"}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
for {
|
||||||
|
command, err := controlService.GetCommand(ctx, &rvboxv1.GetCommandRequest{ClientId: hello.ClientId, IssueUuid: response.IssueUuid})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if command.GetCommand().GetLifecycle() == rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("legacy incomplete event prevented terminal delivery: %v", command.GetCommand().GetLifecycle())
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
events, err := persistence.ReadCommandEvents(ctx, id, 0, 100)
|
||||||
|
if err != nil || len(events) != 3 {
|
||||||
|
t.Fatalf("persisted events=%d err=%v", len(events), err)
|
||||||
|
}
|
||||||
|
payload, err := store.DecodeEventPayload(events[1])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var event rvboxv1.CommandEvent
|
||||||
|
if err := proto.Unmarshal(payload, &event); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(event.GetOutputIncomplete().GetStreams()) != 2 || event.GetOutputIncomplete().GetReason() != "legacy reader failure" {
|
||||||
|
t.Fatalf("loss evidence missing: %v", &event)
|
||||||
|
}
|
||||||
|
return id.String()
|
||||||
|
}
|
||||||
|
cancel, done := start()
|
||||||
|
defer cancel()
|
||||||
|
first := queueAndAwait()
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("client did not stop")
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
for {
|
||||||
|
view, err := persistence.GetClientView(ctx, hello.ClientId)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !view.Connected {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatal("old session did not close")
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
if err := client.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client, err = spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cancel, done = start()
|
||||||
|
defer cancel()
|
||||||
|
firstID, _ := domain.ParseUUIDv7(first)
|
||||||
|
if _, err := client.GetCommand(ctx, firstID); !errors.Is(err, spool.ErrUnknownCommand) {
|
||||||
|
t.Fatalf("server-confirmed terminal row not discarded: %v", err)
|
||||||
|
}
|
||||||
|
queueAndAwait()
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("reconnected client did not stop")
|
||||||
|
}
|
||||||
|
if _, err := client.Check(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user