diff --git a/docs/implementation-plan.v1.md b/docs/implementation-plan.v1.md index 0c0547a..8378ded 100644 --- a/docs/implementation-plan.v1.md +++ b/docs/implementation-plan.v1.md @@ -1921,9 +1921,27 @@ reconciling -> active -> closing`. Only pipe/supervisor workers outlive the replaceable network session. After welcome, send the complete snapshot, apply `ReconcileResult` durably, replay assigned rows first, then assign/send local rows in order. Advertise capacity and accept new dispatch only in `active`. +Re-read the durable snapshot after applying reconciliation: discarded terminal +rows are now tombstones and must not be replayed from the pre-apply snapshot. +Stale internal event-ready hints for removed rows are harmless, while unknown +wire acknowledgements remain errors. Every cumulative acknowledgement releases +send-window credit and must flush any durable tail without waiting for another +producer notification. Cancel the server writer before joining it on reader +exit, so session cleanup does not wait for a heartbeat tick. Cancellation of one session context must join all its readers/writers before a new session can use their queues. +Incomplete-output closeout events must name stdout/stderr scope, retain the +capture reason, consume reserved closeout space when normal allocation is full, +and precede terminal lifecycle. Early v1 clients persisted markers without any +stream scope and rejected them before transmission. On stored-event readback, +only that known legacy shape is normalized to both streams before deriving its +wire digest; preserve the persisted payload/hash, ordinal, sequence, observation +time, and reason. Explicit invalid/duplicate stream values still fail validation. +Verify restart replay, deterministic digests, strict rejection, quota closeout, +terminal-discard wakeup races, and fresh dispatch after real WebSocket reconnect +with `BH-RUNTIME-05..08` and `REC-RUNTIME-01..04`. + The current implementation checkpoint is intentionally split at this seam: `internal/client/agent.RunOnce` owns the reconnect/session reader and remains the sole live-session writer; `internal/client/spool` owns the SQLite source of diff --git a/docs/testing.md b/docs/testing.md index 7d717f0..e952864 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -170,6 +170,34 @@ The Windows artifact is linked with the GUI subsystem (`-H=windowsgui`) so service, launcher, and tray startup do not flash a console. Human-facing modes still attach to a parent console explicitly when one exists. +### Persisted output-loss and reconnect regressions + +`BH-RUNTIME-05` through `08` cover a capture-reader failure producing a valid, +normally ordered incomplete-output event before terminal state; replay of an +early v1 event with missing stream scope after reopening its original spool; +deterministic replay digests without rewriting stored evidence; continued +rejection of invalid/duplicate explicit streams; and reserved closeout under +normal command-quota exhaustion. The known legacy repair conservatively names +stdout and stderr. It does not weaken the protocol validator or delete history. + +`REC-RUNTIME-01` through `03` cover replay from the post-reconciliation snapshot, +stale internal wakeups after terminal discard, and acknowledgement credit +draining a durable tail without a new producer notification. `REC-RUNTIME-04` +uses a real WebSocket server and durable client/server stores: deliver a legacy +loss marker plus terminal result, restart the same client spool, discard the +server-confirmed terminal row, and complete a fresh command. It also requires +prompt server-session cleanup rather than waiting for a heartbeat tick. + +```sh +docker compose -f deploy/compose.yaml run --rm toolchain \ + go test -race ./internal/client/agent ./test/integration/clientagent +``` + +The runtime unit cases are platform-neutral and can also be compiled as a +Windows console test executable. Inspect the persisted typed loss event itself +when checking evidence: the existing server's GetOutput summary does not yet +derive its incomplete flag/stream details from every incoming peer loss marker. + ### Focused native tray and installer regression checks `internal/client/windowstray/native_windows_test.go` contains real Win32 tests, diff --git a/internal/client/agent/executor.go b/internal/client/agent/executor.go index b9664d7..def088a 100644 --- a/internal/client/agent/executor.go +++ b/internal/client/agent/executor.go @@ -252,11 +252,15 @@ func (executor *Executor) rejectWithIdentity(ctx context.Context, issue domain.U } func (executor *Executor) appendIncomplete(ctx context.Context, issue domain.UUID, _ uint64, detail string) error { - payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(&rvboxv1.CommandEvent{IssueUuid: issue.String(), ObservedAt: timestamppb.New(executor.Now()), Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{Reason: boundedError(errors.New(detail))}}}) + payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(&rvboxv1.CommandEvent{IssueUuid: issue.String(), ObservedAt: timestamppb.New(executor.Now()), Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{ + // A capture failure does not prove either stream was complete. + Streams: []rvboxv1.StreamKind{rvboxv1.StreamKind_STREAM_STDOUT, rvboxv1.StreamKind_STREAM_STDERR}, + Reason: boundedError(errors.New(detail)), + }}}) if err != nil { return err } - _, err = executor.Store.AppendEvent(ctx, issue, spool.EventInput{Kind: 11, Compression: 1, RawBytes: uint64(len(payload)), Payload: payload, CreatedAt: executor.Now()}) + _, err = executor.Store.AppendEvent(ctx, issue, spool.EventInput{Kind: spool.EventKindOutputIncomplete, Compression: 1, RawBytes: uint64(len(payload)), Payload: payload, UseCloseout: true, CreatedAt: executor.Now()}) if err == nil { executor.notify(issue) } diff --git a/internal/client/agent/recovery_test.go b/internal/client/agent/recovery_test.go new file mode 100644 index 0000000..6e53de8 --- /dev/null +++ b/internal/client/agent/recovery_test.go @@ -0,0 +1,383 @@ +package agent + +import ( + "bytes" + "context" + "errors" + "path/filepath" + "testing" + "time" + + rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1" + "github.com/rvbox/rvbox/internal/agentproto" + "github.com/rvbox/rvbox/internal/client/spool" + "github.com/rvbox/rvbox/internal/client/supervisor" + "github.com/rvbox/rvbox/internal/domain" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func recoveryStore(t *testing.T) (*spool.Store, domain.UUID) { + t.Helper() + return recoveryStoreAt(t, filepath.Join(t.TempDir(), "spool")) +} + +func recoveryStoreAt(t *testing.T, directory string) (*spool.Store, domain.UUID) { + t.Helper() + store, err := spool.Open(context.Background(), spool.Options{DataDir: directory, BusyTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + issue, err := domain.NewUUIDv7() + if err != nil { + t.Fatal(err) + } + if _, err := store.AcceptCommand(context.Background(), spool.Command{IssueUUID: issue, Revision: 1, Phase: uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), ImmutableSHA256: [32]byte{1}}, time.Now()); err != nil { + t.Fatal(err) + } + return store, issue +} + +func recoveryEvent(t *testing.T, store *spool.Store, issue domain.UUID, event *rvboxv1.CommandEvent, kind uint32) { + t.Helper() + data, err := proto.Marshal(event) + if err != nil { + t.Fatal(err) + } + if _, err := store.AppendEvent(context.Background(), issue, spool.EventInput{Kind: kind, Compression: 1, RawBytes: uint64(len(data)), Payload: data, CreatedAt: time.Now()}); err != nil { + t.Fatal(err) + } +} + +func recoveryOptions(store *spool.Store, transport Transport) RunnerOptions { + return RunnerOptions{Store: store, Dial: func(context.Context) (Transport, error) { return transport, nil }, Hello: &rvboxv1.ClientHello{ + ClientId: "recovery-client", ClientInstanceId: store.ClientInstanceID().String(), + SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1}, DaemonVersion: "test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS, + Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_CMD}, + MaxRunningCommands: 16, MaxQueuedCommands: 100, SentAt: timestamppb.Now(), + }, Limits: agentproto.DefaultLimits(), Backoff: BackoffOptions{Initial: time.Millisecond, Maximum: time.Millisecond, StableReset: time.Second}, Jitter: func(time.Duration) time.Duration { return 0 }, Now: time.Now} +} + +func recoveryHandshake(t *testing.T, result *rvboxv1.ReconcileResult) *runnerTransport { + t.Helper() + transport := &runnerTransport{terminal: errors.New("fixture disconnected")} + for _, payload := range []*rvboxv1.AgentEnvelope{ + {Payload: &rvboxv1.AgentEnvelope_ServerWelcome{ServerWelcome: &rvboxv1.ServerWelcome{SelectedProtocol: &rvboxv1.ProtocolVersion{Major: 1}, ServerTime: timestamppb.Now()}}}, + {Payload: &rvboxv1.AgentEnvelope_ReconcileRequest{ReconcileRequest: &rvboxv1.ReconcileRequest{}}}, + {Payload: &rvboxv1.AgentEnvelope_ReconcileResult{ReconcileResult: result}}, + } { + payload.SessionId, payload.SessionGeneration = "recovery-session", 1 + encoded, err := proto.Marshal(payload) + if err != nil { + t.Fatal(err) + } + transport.reads = append(transport.reads, encoded) + } + return transport +} + +func TestCaptureFailurePublishesValidIncompleteBeforeTerminal_BH_RUNTIME_05(t *testing.T) { + store, issue := recoveryStore(t) + executor := &Executor{Store: store, Now: time.Now} + executor.watch(context.Background(), issue, 1, &failedCapture{}, func() {}) + if _, err := store.AssignSendWindow(context.Background(), issue, 128, 1<<20); err != nil { + t.Fatal(err) + } + events, err := store.PendingEvents(context.Background(), issue) + if err != nil || len(events) != 2 { + t.Fatalf("capture closeout events=%d err=%v", len(events), err) + } + transport := &runnerTransport{} + for _, event := range events { + if err := SendStoredEvent(context.Background(), transport, Session{ID: "session", Generation: 1}, event, agentproto.DefaultLimits()); err != nil { + t.Fatal(err) + } + } + first, err := agentproto.DecodeEnvelope(transport.writes[0], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS) + if err != nil { + t.Fatal(err) + } + marker := first.GetCommandEvent().GetOutputIncomplete() + if marker == nil || len(marker.GetStreams()) != 2 || marker.GetReason() != "capture reader failed" { + t.Fatalf("incomplete marker=%v", marker) + } + last, err := agentproto.DecodeEnvelope(transport.writes[1], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS) + if err != nil || last.GetCommandEvent().GetLifecycle().GetLifecycle() != rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED { + t.Fatalf("terminal event=%v err=%v", last, err) + } + if _, err := store.Check(context.Background()); err != nil { + t.Fatal(err) + } +} + +type failedCapture struct{} + +func (*failedCapture) IssueUUID() domain.UUID { return domain.UUID{} } +func (*failedCapture) Identity() supervisor.EffectiveIdentity { return supervisor.EffectiveIdentity{} } +func (*failedCapture) PID() uint32 { return 0 } +func (*failedCapture) Release(context.Context) error { return nil } +func (*failedCapture) ReadOutput(context.Context) (supervisor.OutputChunk, error) { + return supervisor.OutputChunk{}, errors.New("capture reader failed") +} +func (*failedCapture) Wait(context.Context) (supervisor.ExitStatus, error) { + return supervisor.ExitStatus{Code: 0}, nil +} +func (*failedCapture) WriteStdin(context.Context, []byte, bool) error { return nil } +func (*failedCapture) CloseStdin(context.Context) error { return nil } + +func TestLegacyIncompleteReplaySurvivesRestart_BH_RUNTIME_06(t *testing.T) { + directory := filepath.Join(t.TempDir(), "spool") + store, issue := recoveryStoreAt(t, directory) + recoveryEvent(t, store, issue, &rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{Reason: "legacy capture failure"}}}, spool.EventKindOutputIncomplete) + if _, err := store.AppendLifecycle(context.Background(), issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "exit code 0", time.Now()); err != nil { + t.Fatal(err) + } + if _, err := store.AssignSendWindow(context.Background(), issue, 128, 1<<20); err != nil { + t.Fatal(err) + } + before, err := store.PendingEvents(context.Background(), issue) + if err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + store, err = spool.Open(context.Background(), spool.Options{DataDir: directory, BusyTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + defer store.Close() + var digest []byte + for attempt := 0; attempt < 2; attempt++ { + transport := recoveryHandshake(t, &rvboxv1.ReconcileResult{}) + active := false + options := recoveryOptions(store, transport) + options.OnSessionActive = func(value bool) { active = active || value } + if err := RunOnce(context.Background(), options); !errors.Is(err, transport.terminal) { + t.Fatalf("legacy replay error=%v", err) + } + if !active || len(transport.writes) != 5 { + t.Fatalf("active=%v writes=%d", active, len(transport.writes)) + } + envelope, err := agentproto.DecodeEnvelope(transport.writes[2], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS) + if err != nil { + t.Fatal(err) + } + event := envelope.GetCommandEvent() + if event.GetEventSeq() != 1 || event.GetOutputIncomplete().GetReason() != "legacy capture failure" || len(event.GetOutputIncomplete().GetStreams()) != 2 { + t.Fatalf("recovered event=%v", event) + } + if attempt == 0 { + digest = append([]byte(nil), event.GetImmutableEventSha256()...) + } else if !bytes.Equal(digest, event.GetImmutableEventSha256()) { + t.Fatal("legacy replay digest changed across reconnect") + } + } + after, err := store.PendingEvents(context.Background(), issue) + if err != nil || len(after) != len(before) || !bytes.Equal(before[0].Payload, after[0].Payload) { + t.Fatalf("legacy persisted evidence changed: %v", err) + } + if _, err := store.Check(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestReconciliationDiscardUsesFreshReplaySnapshot_REC_RUNTIME_01(t *testing.T) { + store, issue := recoveryStore(t) + if _, err := store.AppendLifecycle(context.Background(), issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "done", time.Now()); err != nil { + t.Fatal(err) + } + transport := recoveryHandshake(t, &rvboxv1.ReconcileResult{DiscardLocalTerminalIssueUuids: []string{issue.String()}}) + options := recoveryOptions(store, transport) + if err := RunOnce(context.Background(), options); !errors.Is(err, transport.terminal) { + t.Fatalf("discarded command poisoned replay: %v", err) + } + if len(transport.writes) != 3 { + t.Fatalf("discarded rows replayed: writes=%d", len(transport.writes)) + } + if _, err := store.GetCommand(context.Background(), issue); !errors.Is(err, spool.ErrUnknownCommand) { + t.Fatalf("discarded command remains: %v", err) + } + if _, err := store.Check(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestLegacyIncompleteCompatibilityKeepsStreamValidationStrict_BH_RUNTIME_07(t *testing.T) { + issue, _ := domain.NewUUIDv7() + for _, streams := range [][]rvboxv1.StreamKind{{rvboxv1.StreamKind_STREAM_KIND_UNSPECIFIED}, {99}, {rvboxv1.StreamKind_STREAM_STDOUT, rvboxv1.StreamKind_STREAM_STDOUT}} { + payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{Streams: streams, Reason: "bad stream"}}}) + if err != nil { + t.Fatal(err) + } + transport := &runnerTransport{} + err = SendStoredEvent(context.Background(), transport, Session{ID: "session", Generation: 1}, spool.Event{IssueUUID: issue, EventSeq: 1, Kind: spool.EventKindOutputIncomplete, CreatedAt: time.Now(), Payload: payload}, agentproto.DefaultLimits()) + if !errors.Is(err, agentproto.ErrInvalidEnvelope) || len(transport.writes) != 0 { + t.Fatalf("streams=%v err=%v writes=%d", streams, err, len(transport.writes)) + } + } +} + +func TestIncompleteCloseoutSurvivesNormalQuotaExhaustion_BH_RUNTIME_08(t *testing.T) { + ctx := context.Background() + limits := spool.DefaultQuotaLimits() + limits.CommandOutputBytes, limits.CommandTotalBytes = 4<<10, 12<<10 + limits.ClientTotalBytes, limits.CloseoutReserveBytes = 32<<10, 4<<10 + store, err := spool.Open(ctx, spool.Options{DataDir: filepath.Join(t.TempDir(), "spool"), BusyTimeout: time.Second, QuotaLimits: limits}) + if err != nil { + t.Fatal(err) + } + defer store.Close() + issue, _ := domain.NewUUIDv7() + if _, err := store.AcceptCommand(ctx, spool.Command{IssueUUID: issue, Revision: 1, Phase: uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), ImmutableSHA256: [32]byte{1}}, time.Now()); err != nil { + t.Fatal(err) + } + payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_Resource{Resource: &rvboxv1.ResourceSnapshot{}}}) + if err != nil { + t.Fatal(err) + } + full := false + for i := 0; i < 100; i++ { + _, err := store.AppendEvent(ctx, issue, spool.EventInput{Kind: 6, Compression: 1, Payload: payload, RawBytes: uint64(len(payload)), CreatedAt: time.Now()}) + if errors.Is(err, spool.ErrCapacityExhausted) { + full = true + break + } + if err != nil { + t.Fatal(err) + } + } + if !full { + t.Fatal("normal allocation did not reach its quota") + } + executor := &Executor{Store: store, Now: time.Now} + if err := executor.appendIncomplete(ctx, issue, 1, "normal command quota exhausted"); err != nil { + t.Fatalf("reserved incomplete closeout rejected: %v", err) + } + if _, err := store.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "done", time.Now()); err != nil { + t.Fatalf("reserved terminal closeout rejected: %v", err) + } + if _, err := store.Check(ctx); err != nil { + t.Fatal(err) + } +} + +type recoveryTransport struct { + reads chan []byte + writes chan []byte +} + +func newRecoveryTransport() *recoveryTransport { + return &recoveryTransport{reads: make(chan []byte, 1), writes: make(chan []byte, 256)} +} +func (transport *recoveryTransport) Read(ctx context.Context) ([]byte, error) { + select { + case data := <-transport.reads: + return data, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} +func (transport *recoveryTransport) Write(ctx context.Context, data []byte) error { + select { + case transport.writes <- append([]byte(nil), data...): + return nil + case <-ctx.Done(): + return ctx.Err() + } +} +func (*recoveryTransport) Close() error { return nil } + +func recoveryFrame(t *testing.T, envelope *rvboxv1.AgentEnvelope) []byte { + t.Helper() + envelope.SessionId, envelope.SessionGeneration = "session", 1 + data, err := proto.Marshal(envelope) + if err != nil { + t.Fatal(err) + } + return data +} + +func TestDiscardedCommandNotificationDoesNotDisconnect_REC_RUNTIME_02(t *testing.T) { + store, live := recoveryStore(t) + stale, _ := domain.NewUUIDv7() + notify := make(chan domain.UUID) + transport := newRecoveryTransport() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + options := recoveryOptions(store, transport) + options.EventReady = notify + options.OnStdin = func(ctx context.Context, _ Session, write *rvboxv1.StdinWrite) error { + payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_Resource{Resource: &rvboxv1.ResourceSnapshot{DiagnosticDetail: "active after stale wakeup"}}}) + if err != nil { + return err + } + _, err = store.AppendEvent(ctx, live, spool.EventInput{Kind: 6, Compression: 1, RawBytes: uint64(len(payload)), Payload: payload, CreatedAt: time.Now()}) + return err + } + done := make(chan error, 1) + go func() { + done <- serveActive(ctx, transport, options, Session{ID: "session", Generation: 1}, make(map[domain.UUID]uint64)) + }() + select { + case notify <- stale: + case err := <-done: + t.Fatalf("before wakeup: %v", err) + case <-ctx.Done(): + t.Fatal("wakeup not consumed") + } + transport.reads <- recoveryFrame(t, &rvboxv1.AgentEnvelope{Payload: &rvboxv1.AgentEnvelope_StdinWrite{StdinWrite: &rvboxv1.StdinWrite{IssueUuid: live.String(), WriteSeq: 1, Data: []byte("probe")}}}) + select { + case data := <-transport.writes: + envelope, err := agentproto.DecodeEnvelope(data, agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS) + if err != nil || envelope.GetCommandEvent().GetResource().GetDiagnosticDetail() != "active after stale wakeup" { + t.Fatalf("liveness response=%v err=%v", envelope, err) + } + case err := <-done: + t.Fatalf("stale wakeup disconnected active session: %v", err) + case <-ctx.Done(): + t.Fatal("no progress after stale wakeup") + } + cancel() + <-done +} + +func TestEventAcknowledgementDrainsDurableTail_REC_RUNTIME_03(t *testing.T) { + store, issue := recoveryStore(t) + for i := 0; i < 130; i++ { + recoveryEvent(t, store, issue, &rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_Resource{Resource: &rvboxv1.ResourceSnapshot{}}}, 6) + } + transport := newRecoveryTransport() + sent := make(map[domain.UUID]uint64) + session := Session{ID: "session", Generation: 1} + if err := flushEvents(context.Background(), transport, store, session, issue, agentproto.DefaultLimits(), sent); err != nil { + t.Fatal(err) + } + if len(transport.writes) != 128 { + t.Fatalf("first send window=%d", len(transport.writes)) + } + for i := 0; i < 128; i++ { + <-transport.writes + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- serveActive(ctx, transport, recoveryOptions(store, transport), session, sent) }() + transport.reads <- recoveryFrame(t, &rvboxv1.AgentEnvelope{Payload: &rvboxv1.AgentEnvelope_EventAck{EventAck: &rvboxv1.EventAck{IssueUuid: issue.String(), ThroughEventSeq: 128}}}) + for want := uint64(129); want <= 130; want++ { + select { + case data := <-transport.writes: + envelope, err := agentproto.DecodeEnvelope(data, agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS) + if err != nil || envelope.GetCommandEvent().GetEventSeq() != want { + t.Fatalf("tail sequence want=%d event=%v err=%v", want, envelope, err) + } + case err := <-done: + t.Fatalf("acknowledgement disconnected session: %v", err) + case <-ctx.Done(): + t.Fatal("acknowledged credit did not release pending tail") + } + } + cancel() + <-done +} diff --git a/internal/client/agent/runner.go b/internal/client/agent/runner.go index d39cc00..e8048bf 100644 --- a/internal/client/agent/runner.go +++ b/internal/client/agent/runner.go @@ -209,6 +209,12 @@ func runOnce(ctx context.Context, options RunnerOptions) (resultErr error) { } } } + // Applying reconciliation can replace terminal rows with tombstones and + // terminate formerly active commands. Replay/capacity must use that new view. + snapshot, err = options.Store.ReconcileSnapshot(ctx) + if err != nil { + return fmt.Errorf("refresh reconciled client snapshot: %w", err) + } sentEvents := make(map[domain.UUID]uint64) if err := replayEvents(ctx, transport, options.Store, session, snapshot, limits, sentEvents); err != nil { return fmt.Errorf("replay client events: %w", err) @@ -294,6 +300,12 @@ func commandEventFromStored(event spool.Event) (*rvboxv1.CommandEvent, error) { if err := proto.Unmarshal(event.Payload, &stored); err != nil || stored.Payload == nil { return nil, fmt.Errorf("decode stored command event: %w", err) } + if incomplete := stored.GetOutputIncomplete(); event.Kind == spool.EventKindOutputIncomplete && incomplete != nil && len(incomplete.Streams) == 0 { + // Early v1 writers omitted streams. Those events failed local wire + // validation before any server could accept their digest. Conservatively + // name both streams without changing persisted bytes, order, or reason. + incomplete.Streams = []rvboxv1.StreamKind{rvboxv1.StreamKind_STREAM_STDOUT, rvboxv1.StreamKind_STREAM_STDERR} + } return &stored, nil } @@ -362,6 +374,11 @@ func serveActive(ctx context.Context, transport Transport, options RunnerOptions continue } if err := flushEvents(ctx, transport, options.Store, session, issue, limits, sent); err != nil { + if errors.Is(err, spool.ErrUnknownCommand) { + // This internal wakeup can outlive a server-confirmed discard. + // It is a hint, not a new dispatch or an event acknowledgement. + continue + } return fmt.Errorf("flush ready command %s events: %w", issue, err) } continue @@ -391,6 +408,12 @@ func serveActive(ctx context.Context, transport Transport, options RunnerOptions if err := ApplyEventAck(ctx, options.Store, envelope.GetEventAck()); err != nil { return fmt.Errorf("apply event acknowledgement for %s: %w", envelope.GetEventAck().GetIssueUuid(), err) } + issue, _ := domain.ParseUUIDv7(envelope.GetEventAck().GetIssueUuid()) + // Released send-window credit must drain the durable tail even if no + // producer remains to emit another EventReady notification. + if err := flushEvents(ctx, transport, options.Store, session, issue, limits, sent); err != nil { + return fmt.Errorf("flush acknowledged command %s events: %w", issue, err) + } case envelope.GetScriptChunk() != nil: if err := handleScriptChunk(ctx, transport, options.Store, session, envelope.GetScriptChunk(), limits, options.Now, sent); err != nil { return fmt.Errorf("apply script chunk for %s: %w", envelope.GetScriptChunk().GetIssueUuid(), err) diff --git a/internal/client/spool/events.go b/internal/client/spool/events.go index 6767839..7574700 100644 --- a/internal/client/spool/events.go +++ b/internal/client/spool/events.go @@ -54,6 +54,7 @@ const ( // reconstruct the envelope without a second event-kind translation table. EventKindOutput uint32 = 5 EventKindOutputTruncation uint32 = 10 + EventKindOutputIncomplete uint32 = 11 ) type Event struct { diff --git a/internal/server/session/agent_server.go b/internal/server/session/agent_server.go index 496612e..b195ef3 100644 --- a/internal/server/session/agent_server.go +++ b/internal/server/session/agent_server.go @@ -232,7 +232,12 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w server.writeLoop(sessionContext, connection, queue, heartbeat, started) cancel() }() - defer func() { <-writerDone }() + defer func() { + // A reader can end before the writer observes the closed transport. + // Cancel first so cleanup does not wait for the next heartbeat write. + cancel() + <-writerDone + }() for { messageType, payload, err = connection.Read(sessionContext) diff --git a/test/coverage.toml b/test/coverage.toml index 03d45f5..39791c5 100644 --- a/test/coverage.toml +++ b/test/coverage.toml @@ -431,6 +431,54 @@ layer = "unit" status = "implemented" tests = ["internal/client/agent/executor_test.go:TestExecutorRunsDurableCommandAndPublishesTerminalEvents_HP_EXECUTOR_01"] +[[requirements]] +id = "BH-RUNTIME-05" +layer = "unit" +status = "implemented" +tests = ["internal/client/agent/recovery_test.go:TestCaptureFailurePublishesValidIncompleteBeforeTerminal_BH_RUNTIME_05"] + +[[requirements]] +id = "BH-RUNTIME-06" +layer = "unit" +status = "implemented" +tests = ["internal/client/agent/recovery_test.go:TestLegacyIncompleteReplaySurvivesRestart_BH_RUNTIME_06"] + +[[requirements]] +id = "BH-RUNTIME-07" +layer = "unit" +status = "implemented" +tests = ["internal/client/agent/recovery_test.go:TestLegacyIncompleteCompatibilityKeepsStreamValidationStrict_BH_RUNTIME_07"] + +[[requirements]] +id = "BH-RUNTIME-08" +layer = "unit" +status = "implemented" +tests = ["internal/client/agent/recovery_test.go:TestIncompleteCloseoutSurvivesNormalQuotaExhaustion_BH_RUNTIME_08"] + +[[requirements]] +id = "REC-RUNTIME-01" +layer = "unit" +status = "implemented" +tests = ["internal/client/agent/recovery_test.go:TestReconciliationDiscardUsesFreshReplaySnapshot_REC_RUNTIME_01"] + +[[requirements]] +id = "REC-RUNTIME-02" +layer = "unit" +status = "implemented" +tests = ["internal/client/agent/recovery_test.go:TestDiscardedCommandNotificationDoesNotDisconnect_REC_RUNTIME_02"] + +[[requirements]] +id = "REC-RUNTIME-03" +layer = "unit" +status = "implemented" +tests = ["internal/client/agent/recovery_test.go:TestEventAcknowledgementDrainsDurableTail_REC_RUNTIME_03"] + +[[requirements]] +id = "REC-RUNTIME-04" +layer = "integration" +status = "implemented" +tests = ["test/integration/clientagent/recovery_integration_test.go:TestLegacyOutputLossReconnectAndFreshDispatch_REC_RUNTIME_04"] + [[requirements]] id = "HP-EXECUTOR-02" layer = "unit" diff --git a/test/integration/clientagent/recovery_integration_test.go b/test/integration/clientagent/recovery_integration_test.go new file mode 100644 index 0000000..78dd25e --- /dev/null +++ b/test/integration/clientagent/recovery_integration_test.go @@ -0,0 +1,179 @@ +package clientagent_test + +import ( + "context" + "errors" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1" + "github.com/rvbox/rvbox/internal/agentproto" + "github.com/rvbox/rvbox/internal/client/agent" + "github.com/rvbox/rvbox/internal/client/spool" + "github.com/rvbox/rvbox/internal/domain" + "github.com/rvbox/rvbox/internal/server/control" + "github.com/rvbox/rvbox/internal/server/session" + "github.com/rvbox/rvbox/internal/server/store" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestLegacyOutputLossReconnectAndFreshDispatch_REC_RUNTIME_04(t *testing.T) { + ctx := context.Background() + persistence, err := store.Open(ctx, store.Options{DataDir: filepath.Join(t.TempDir(), "server"), BusyTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + defer persistence.Close() + registry := session.NewRegistry() + server := httptest.NewServer(&session.AgentServer{Store: persistence, Registry: registry, Path: "/v1/agent", Limits: agentproto.DefaultLimits()}) + defer server.Close() + controlService, err := control.NewService(control.Options{Store: persistence, WakeClient: registry.Wake}) + if err != nil { + t.Fatal(err) + } + directory := filepath.Join(t.TempDir(), "client") + client, err := spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = client.Close() }() + hello := &rvboxv1.ClientHello{ClientId: "recovery-client", ClientInstanceId: client.ClientInstanceID().String(), SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1}, DaemonVersion: "legacy-recovery-test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS, Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_CMD}, MaxRunningCommands: 16, MaxQueuedCommands: 100, SentAt: timestamppb.Now()} + start := func() (context.CancelFunc, <-chan error) { + runCtx, cancel := context.WithCancel(ctx) + active := make(chan struct{}, 1) + done := make(chan error, 1) + go func() { + done <- agent.RunOnce(runCtx, agent.RunnerOptions{ + Store: client, Hello: hello, Limits: agentproto.DefaultLimits(), Now: time.Now, + Backoff: agent.BackoffOptions{Initial: time.Millisecond, Maximum: time.Millisecond, StableReset: time.Second}, Jitter: func(time.Duration) time.Duration { return 0 }, + Dial: func(ctx context.Context) (agent.Transport, error) { + return agent.DialWebSocket(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/v1/agent", nil) + }, + OnSessionActive: func(value bool) { + if value { + active <- struct{}{} + } + }, + OnDispatch: func(ctx context.Context, _ agent.Session, dispatch *rvboxv1.CommandDispatch) error { + issue, err := domain.ParseUUIDv7(dispatch.GetIssueUuid()) + if err != nil { + return err + } + if _, err := client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), 1, "fixture running", time.Now()); err != nil { + return err + } + // Reproduce the old writer's durable event exactly: no streams. + payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{Reason: "legacy reader failure"}}}) + if err != nil { + return err + } + if _, err := client.AppendEvent(ctx, issue, spool.EventInput{Kind: spool.EventKindOutputIncomplete, Compression: 1, RawBytes: uint64(len(payload)), Payload: payload, CreatedAt: time.Now()}); err != nil { + return err + } + _, err = client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "exit code 0", time.Now()) + return err + }, + }) + }() + select { + case <-active: + case err := <-done: + cancel() + t.Fatalf("client failed to become active: %v", err) + case <-time.After(5 * time.Second): + cancel() + t.Fatal("client did not become active") + } + return cancel, done + } + queueAndAwait := func() string { + id, err := domain.NewUUIDv7() + if err != nil { + t.Fatal(err) + } + response, err := controlService.RunCommand(ctx, &rvboxv1.RunCommandRequest{TargetClientId: hello.ClientId, RequestId: id.String(), Spec: &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "echo fixture"}}}) + if err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for { + command, err := controlService.GetCommand(ctx, &rvboxv1.GetCommandRequest{ClientId: hello.ClientId, IssueUuid: response.IssueUuid}) + if err != nil { + t.Fatal(err) + } + if command.GetCommand().GetLifecycle() == rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED { + break + } + if time.Now().After(deadline) { + t.Fatalf("legacy incomplete event prevented terminal delivery: %v", command.GetCommand().GetLifecycle()) + } + time.Sleep(10 * time.Millisecond) + } + events, err := persistence.ReadCommandEvents(ctx, id, 0, 100) + if err != nil || len(events) != 3 { + t.Fatalf("persisted events=%d err=%v", len(events), err) + } + payload, err := store.DecodeEventPayload(events[1]) + if err != nil { + t.Fatal(err) + } + var event rvboxv1.CommandEvent + if err := proto.Unmarshal(payload, &event); err != nil { + t.Fatal(err) + } + if len(event.GetOutputIncomplete().GetStreams()) != 2 || event.GetOutputIncomplete().GetReason() != "legacy reader failure" { + t.Fatalf("loss evidence missing: %v", &event) + } + return id.String() + } + cancel, done := start() + defer cancel() + first := queueAndAwait() + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("client did not stop") + } + deadline := time.Now().Add(5 * time.Second) + for { + view, err := persistence.GetClientView(ctx, hello.ClientId) + if err != nil { + t.Fatal(err) + } + if !view.Connected { + break + } + if time.Now().After(deadline) { + t.Fatal("old session did not close") + } + time.Sleep(10 * time.Millisecond) + } + if err := client.Close(); err != nil { + t.Fatal(err) + } + client, err = spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + cancel, done = start() + defer cancel() + firstID, _ := domain.ParseUUIDv7(first) + if _, err := client.GetCommand(ctx, firstID); !errors.Is(err, spool.ErrUnknownCommand) { + t.Fatalf("server-confirmed terminal row not discarded: %v", err) + } + queueAndAwait() + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("reconnected client did not stop") + } + if _, err := client.Check(ctx); err != nil { + t.Fatal(err) + } +}