fix: recover reconnect after incomplete output capture

This commit is contained in:
cabbage committed 2026-10-27 14:36:06 +00:00
1 parent b52ca5a7c2
commit 6906ed52f2
9 files changed
+692 -3

No files matched your search

+18
View File
@@ -1921,9 +1921,27 @@ reconciling -> active -> closing`. Only pipe/supervisor workers outlive the
replaceable network session. After welcome, send the complete snapshot, apply
`ReconcileResult` durably, replay assigned rows first, then assign/send local
rows in order. Advertise capacity and accept new dispatch only in `active`.
Re-read the durable snapshot after applying reconciliation: discarded terminal
rows are now tombstones and must not be replayed from the pre-apply snapshot.
Stale internal event-ready hints for removed rows are harmless, while unknown
wire acknowledgements remain errors. Every cumulative acknowledgement releases
send-window credit and must flush any durable tail without waiting for another
producer notification. Cancel the server writer before joining it on reader
exit, so session cleanup does not wait for a heartbeat tick.
Cancellation of one session context must join all its readers/writers before a
new session can use their queues.
Incomplete-output closeout events must name stdout/stderr scope, retain the
capture reason, consume reserved closeout space when normal allocation is full,
and precede terminal lifecycle. Early v1 clients persisted markers without any
stream scope and rejected them before transmission. On stored-event readback,
only that known legacy shape is normalized to both streams before deriving its
wire digest; preserve the persisted payload/hash, ordinal, sequence, observation
time, and reason. Explicit invalid/duplicate stream values still fail validation.
Verify restart replay, deterministic digests, strict rejection, quota closeout,
terminal-discard wakeup races, and fresh dispatch after real WebSocket reconnect
with `BH-RUNTIME-05..08` and `REC-RUNTIME-01..04`.
The current implementation checkpoint is intentionally split at this seam:
`internal/client/agent.RunOnce` owns the reconnect/session reader and remains
the sole live-session writer; `internal/client/spool` owns the SQLite source of
+28
View File
@@ -170,6 +170,34 @@ The Windows artifact is linked with the GUI subsystem (`-H=windowsgui`) so
service, launcher, and tray startup do not flash a console. Human-facing modes
still attach to a parent console explicitly when one exists.
### Persisted output-loss and reconnect regressions
`BH-RUNTIME-05` through `08` cover a capture-reader failure producing a valid,
normally ordered incomplete-output event before terminal state; replay of an
early v1 event with missing stream scope after reopening its original spool;
deterministic replay digests without rewriting stored evidence; continued
rejection of invalid/duplicate explicit streams; and reserved closeout under
normal command-quota exhaustion. The known legacy repair conservatively names
stdout and stderr. It does not weaken the protocol validator or delete history.
`REC-RUNTIME-01` through `03` cover replay from the post-reconciliation snapshot,
stale internal wakeups after terminal discard, and acknowledgement credit
draining a durable tail without a new producer notification. `REC-RUNTIME-04`
uses a real WebSocket server and durable client/server stores: deliver a legacy
loss marker plus terminal result, restart the same client spool, discard the
server-confirmed terminal row, and complete a fresh command. It also requires
prompt server-session cleanup rather than waiting for a heartbeat tick.
```sh
docker compose -f deploy/compose.yaml run --rm toolchain \
go test -race ./internal/client/agent ./test/integration/clientagent
```
The runtime unit cases are platform-neutral and can also be compiled as a
Windows console test executable. Inspect the persisted typed loss event itself
when checking evidence: the existing server's GetOutput summary does not yet
derive its incomplete flag/stream details from every incoming peer loss marker.
### Focused native tray and installer regression checks
`internal/client/windowstray/native_windows_test.go` contains real Win32 tests,
+6 -2
View File
@@ -252,11 +252,15 @@ func (executor *Executor) rejectWithIdentity(ctx context.Context, issue domain.U
}
func (executor *Executor) appendIncomplete(ctx context.Context, issue domain.UUID, _ uint64, detail string) error {
payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(&rvboxv1.CommandEvent{IssueUuid: issue.String(), ObservedAt: timestamppb.New(executor.Now()), Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{Reason: boundedError(errors.New(detail))}}})
payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(&rvboxv1.CommandEvent{IssueUuid: issue.String(), ObservedAt: timestamppb.New(executor.Now()), Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{
// A capture failure does not prove either stream was complete.
Streams: []rvboxv1.StreamKind{rvboxv1.StreamKind_STREAM_STDOUT, rvboxv1.StreamKind_STREAM_STDERR},
Reason: boundedError(errors.New(detail)),
}}})
if err != nil {
return err
}
_, err = executor.Store.AppendEvent(ctx, issue, spool.EventInput{Kind: 11, Compression: 1, RawBytes: uint64(len(payload)), Payload: payload, CreatedAt: executor.Now()})
_, err = executor.Store.AppendEvent(ctx, issue, spool.EventInput{Kind: spool.EventKindOutputIncomplete, Compression: 1, RawBytes: uint64(len(payload)), Payload: payload, UseCloseout: true, CreatedAt: executor.Now()})
if err == nil {
executor.notify(issue)
}
+383
View File
@@ -0,0 +1,383 @@
package agent
import (
"bytes"
"context"
"errors"
"path/filepath"
"testing"
"time"
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/client/spool"
"github.com/rvbox/rvbox/internal/client/supervisor"
"github.com/rvbox/rvbox/internal/domain"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
func recoveryStore(t *testing.T) (*spool.Store, domain.UUID) {
t.Helper()
return recoveryStoreAt(t, filepath.Join(t.TempDir(), "spool"))
}
func recoveryStoreAt(t *testing.T, directory string) (*spool.Store, domain.UUID) {
t.Helper()
store, err := spool.Open(context.Background(), spool.Options{DataDir: directory, BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = store.Close() })
issue, err := domain.NewUUIDv7()
if err != nil {
t.Fatal(err)
}
if _, err := store.AcceptCommand(context.Background(), spool.Command{IssueUUID: issue, Revision: 1, Phase: uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), ImmutableSHA256: [32]byte{1}}, time.Now()); err != nil {
t.Fatal(err)
}
return store, issue
}
func recoveryEvent(t *testing.T, store *spool.Store, issue domain.UUID, event *rvboxv1.CommandEvent, kind uint32) {
t.Helper()
data, err := proto.Marshal(event)
if err != nil {
t.Fatal(err)
}
if _, err := store.AppendEvent(context.Background(), issue, spool.EventInput{Kind: kind, Compression: 1, RawBytes: uint64(len(data)), Payload: data, CreatedAt: time.Now()}); err != nil {
t.Fatal(err)
}
}
func recoveryOptions(store *spool.Store, transport Transport) RunnerOptions {
return RunnerOptions{Store: store, Dial: func(context.Context) (Transport, error) { return transport, nil }, Hello: &rvboxv1.ClientHello{
ClientId: "recovery-client", ClientInstanceId: store.ClientInstanceID().String(),
SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1}, DaemonVersion: "test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS,
Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_CMD},
MaxRunningCommands: 16, MaxQueuedCommands: 100, SentAt: timestamppb.Now(),
}, Limits: agentproto.DefaultLimits(), Backoff: BackoffOptions{Initial: time.Millisecond, Maximum: time.Millisecond, StableReset: time.Second}, Jitter: func(time.Duration) time.Duration { return 0 }, Now: time.Now}
}
func recoveryHandshake(t *testing.T, result *rvboxv1.ReconcileResult) *runnerTransport {
t.Helper()
transport := &runnerTransport{terminal: errors.New("fixture disconnected")}
for _, payload := range []*rvboxv1.AgentEnvelope{
{Payload: &rvboxv1.AgentEnvelope_ServerWelcome{ServerWelcome: &rvboxv1.ServerWelcome{SelectedProtocol: &rvboxv1.ProtocolVersion{Major: 1}, ServerTime: timestamppb.Now()}}},
{Payload: &rvboxv1.AgentEnvelope_ReconcileRequest{ReconcileRequest: &rvboxv1.ReconcileRequest{}}},
{Payload: &rvboxv1.AgentEnvelope_ReconcileResult{ReconcileResult: result}},
} {
payload.SessionId, payload.SessionGeneration = "recovery-session", 1
encoded, err := proto.Marshal(payload)
if err != nil {
t.Fatal(err)
}
transport.reads = append(transport.reads, encoded)
}
return transport
}
func TestCaptureFailurePublishesValidIncompleteBeforeTerminal_BH_RUNTIME_05(t *testing.T) {
store, issue := recoveryStore(t)
executor := &Executor{Store: store, Now: time.Now}
executor.watch(context.Background(), issue, 1, &failedCapture{}, func() {})
if _, err := store.AssignSendWindow(context.Background(), issue, 128, 1<<20); err != nil {
t.Fatal(err)
}
events, err := store.PendingEvents(context.Background(), issue)
if err != nil || len(events) != 2 {
t.Fatalf("capture closeout events=%d err=%v", len(events), err)
}
transport := &runnerTransport{}
for _, event := range events {
if err := SendStoredEvent(context.Background(), transport, Session{ID: "session", Generation: 1}, event, agentproto.DefaultLimits()); err != nil {
t.Fatal(err)
}
}
first, err := agentproto.DecodeEnvelope(transport.writes[0], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
if err != nil {
t.Fatal(err)
}
marker := first.GetCommandEvent().GetOutputIncomplete()
if marker == nil || len(marker.GetStreams()) != 2 || marker.GetReason() != "capture reader failed" {
t.Fatalf("incomplete marker=%v", marker)
}
last, err := agentproto.DecodeEnvelope(transport.writes[1], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
if err != nil || last.GetCommandEvent().GetLifecycle().GetLifecycle() != rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED {
t.Fatalf("terminal event=%v err=%v", last, err)
}
if _, err := store.Check(context.Background()); err != nil {
t.Fatal(err)
}
}
type failedCapture struct{}
func (*failedCapture) IssueUUID() domain.UUID { return domain.UUID{} }
func (*failedCapture) Identity() supervisor.EffectiveIdentity { return supervisor.EffectiveIdentity{} }
func (*failedCapture) PID() uint32 { return 0 }
func (*failedCapture) Release(context.Context) error { return nil }
func (*failedCapture) ReadOutput(context.Context) (supervisor.OutputChunk, error) {
return supervisor.OutputChunk{}, errors.New("capture reader failed")
}
func (*failedCapture) Wait(context.Context) (supervisor.ExitStatus, error) {
return supervisor.ExitStatus{Code: 0}, nil
}
func (*failedCapture) WriteStdin(context.Context, []byte, bool) error { return nil }
func (*failedCapture) CloseStdin(context.Context) error { return nil }
func TestLegacyIncompleteReplaySurvivesRestart_BH_RUNTIME_06(t *testing.T) {
directory := filepath.Join(t.TempDir(), "spool")
store, issue := recoveryStoreAt(t, directory)
recoveryEvent(t, store, issue, &rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{Reason: "legacy capture failure"}}}, spool.EventKindOutputIncomplete)
if _, err := store.AppendLifecycle(context.Background(), issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "exit code 0", time.Now()); err != nil {
t.Fatal(err)
}
if _, err := store.AssignSendWindow(context.Background(), issue, 128, 1<<20); err != nil {
t.Fatal(err)
}
before, err := store.PendingEvents(context.Background(), issue)
if err != nil {
t.Fatal(err)
}
if err := store.Close(); err != nil {
t.Fatal(err)
}
store, err = spool.Open(context.Background(), spool.Options{DataDir: directory, BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
defer store.Close()
var digest []byte
for attempt := 0; attempt < 2; attempt++ {
transport := recoveryHandshake(t, &rvboxv1.ReconcileResult{})
active := false
options := recoveryOptions(store, transport)
options.OnSessionActive = func(value bool) { active = active || value }
if err := RunOnce(context.Background(), options); !errors.Is(err, transport.terminal) {
t.Fatalf("legacy replay error=%v", err)
}
if !active || len(transport.writes) != 5 {
t.Fatalf("active=%v writes=%d", active, len(transport.writes))
}
envelope, err := agentproto.DecodeEnvelope(transport.writes[2], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
if err != nil {
t.Fatal(err)
}
event := envelope.GetCommandEvent()
if event.GetEventSeq() != 1 || event.GetOutputIncomplete().GetReason() != "legacy capture failure" || len(event.GetOutputIncomplete().GetStreams()) != 2 {
t.Fatalf("recovered event=%v", event)
}
if attempt == 0 {
digest = append([]byte(nil), event.GetImmutableEventSha256()...)
} else if !bytes.Equal(digest, event.GetImmutableEventSha256()) {
t.Fatal("legacy replay digest changed across reconnect")
}
}
after, err := store.PendingEvents(context.Background(), issue)
if err != nil || len(after) != len(before) || !bytes.Equal(before[0].Payload, after[0].Payload) {
t.Fatalf("legacy persisted evidence changed: %v", err)
}
if _, err := store.Check(context.Background()); err != nil {
t.Fatal(err)
}
}
func TestReconciliationDiscardUsesFreshReplaySnapshot_REC_RUNTIME_01(t *testing.T) {
store, issue := recoveryStore(t)
if _, err := store.AppendLifecycle(context.Background(), issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "done", time.Now()); err != nil {
t.Fatal(err)
}
transport := recoveryHandshake(t, &rvboxv1.ReconcileResult{DiscardLocalTerminalIssueUuids: []string{issue.String()}})
options := recoveryOptions(store, transport)
if err := RunOnce(context.Background(), options); !errors.Is(err, transport.terminal) {
t.Fatalf("discarded command poisoned replay: %v", err)
}
if len(transport.writes) != 3 {
t.Fatalf("discarded rows replayed: writes=%d", len(transport.writes))
}
if _, err := store.GetCommand(context.Background(), issue); !errors.Is(err, spool.ErrUnknownCommand) {
t.Fatalf("discarded command remains: %v", err)
}
if _, err := store.Check(context.Background()); err != nil {
t.Fatal(err)
}
}
func TestLegacyIncompleteCompatibilityKeepsStreamValidationStrict_BH_RUNTIME_07(t *testing.T) {
issue, _ := domain.NewUUIDv7()
for _, streams := range [][]rvboxv1.StreamKind{{rvboxv1.StreamKind_STREAM_KIND_UNSPECIFIED}, {99}, {rvboxv1.StreamKind_STREAM_STDOUT, rvboxv1.StreamKind_STREAM_STDOUT}} {
payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{Streams: streams, Reason: "bad stream"}}})
if err != nil {
t.Fatal(err)
}
transport := &runnerTransport{}
err = SendStoredEvent(context.Background(), transport, Session{ID: "session", Generation: 1}, spool.Event{IssueUUID: issue, EventSeq: 1, Kind: spool.EventKindOutputIncomplete, CreatedAt: time.Now(), Payload: payload}, agentproto.DefaultLimits())
if !errors.Is(err, agentproto.ErrInvalidEnvelope) || len(transport.writes) != 0 {
t.Fatalf("streams=%v err=%v writes=%d", streams, err, len(transport.writes))
}
}
}
func TestIncompleteCloseoutSurvivesNormalQuotaExhaustion_BH_RUNTIME_08(t *testing.T) {
ctx := context.Background()
limits := spool.DefaultQuotaLimits()
limits.CommandOutputBytes, limits.CommandTotalBytes = 4<<10, 12<<10
limits.ClientTotalBytes, limits.CloseoutReserveBytes = 32<<10, 4<<10
store, err := spool.Open(ctx, spool.Options{DataDir: filepath.Join(t.TempDir(), "spool"), BusyTimeout: time.Second, QuotaLimits: limits})
if err != nil {
t.Fatal(err)
}
defer store.Close()
issue, _ := domain.NewUUIDv7()
if _, err := store.AcceptCommand(ctx, spool.Command{IssueUUID: issue, Revision: 1, Phase: uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), ImmutableSHA256: [32]byte{1}}, time.Now()); err != nil {
t.Fatal(err)
}
payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_Resource{Resource: &rvboxv1.ResourceSnapshot{}}})
if err != nil {
t.Fatal(err)
}
full := false
for i := 0; i < 100; i++ {
_, err := store.AppendEvent(ctx, issue, spool.EventInput{Kind: 6, Compression: 1, Payload: payload, RawBytes: uint64(len(payload)), CreatedAt: time.Now()})
if errors.Is(err, spool.ErrCapacityExhausted) {
full = true
break
}
if err != nil {
t.Fatal(err)
}
}
if !full {
t.Fatal("normal allocation did not reach its quota")
}
executor := &Executor{Store: store, Now: time.Now}
if err := executor.appendIncomplete(ctx, issue, 1, "normal command quota exhausted"); err != nil {
t.Fatalf("reserved incomplete closeout rejected: %v", err)
}
if _, err := store.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "done", time.Now()); err != nil {
t.Fatalf("reserved terminal closeout rejected: %v", err)
}
if _, err := store.Check(ctx); err != nil {
t.Fatal(err)
}
}
type recoveryTransport struct {
reads chan []byte
writes chan []byte
}
func newRecoveryTransport() *recoveryTransport {
return &recoveryTransport{reads: make(chan []byte, 1), writes: make(chan []byte, 256)}
}
func (transport *recoveryTransport) Read(ctx context.Context) ([]byte, error) {
select {
case data := <-transport.reads:
return data, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (transport *recoveryTransport) Write(ctx context.Context, data []byte) error {
select {
case transport.writes <- append([]byte(nil), data...):
return nil
case <-ctx.Done():
return ctx.Err()
}
}
func (*recoveryTransport) Close() error { return nil }
func recoveryFrame(t *testing.T, envelope *rvboxv1.AgentEnvelope) []byte {
t.Helper()
envelope.SessionId, envelope.SessionGeneration = "session", 1
data, err := proto.Marshal(envelope)
if err != nil {
t.Fatal(err)
}
return data
}
func TestDiscardedCommandNotificationDoesNotDisconnect_REC_RUNTIME_02(t *testing.T) {
store, live := recoveryStore(t)
stale, _ := domain.NewUUIDv7()
notify := make(chan domain.UUID)
transport := newRecoveryTransport()
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
options := recoveryOptions(store, transport)
options.EventReady = notify
options.OnStdin = func(ctx context.Context, _ Session, write *rvboxv1.StdinWrite) error {
payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_Resource{Resource: &rvboxv1.ResourceSnapshot{DiagnosticDetail: "active after stale wakeup"}}})
if err != nil {
return err
}
_, err = store.AppendEvent(ctx, live, spool.EventInput{Kind: 6, Compression: 1, RawBytes: uint64(len(payload)), Payload: payload, CreatedAt: time.Now()})
return err
}
done := make(chan error, 1)
go func() {
done <- serveActive(ctx, transport, options, Session{ID: "session", Generation: 1}, make(map[domain.UUID]uint64))
}()
select {
case notify <- stale:
case err := <-done:
t.Fatalf("before wakeup: %v", err)
case <-ctx.Done():
t.Fatal("wakeup not consumed")
}
transport.reads <- recoveryFrame(t, &rvboxv1.AgentEnvelope{Payload: &rvboxv1.AgentEnvelope_StdinWrite{StdinWrite: &rvboxv1.StdinWrite{IssueUuid: live.String(), WriteSeq: 1, Data: []byte("probe")}}})
select {
case data := <-transport.writes:
envelope, err := agentproto.DecodeEnvelope(data, agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
if err != nil || envelope.GetCommandEvent().GetResource().GetDiagnosticDetail() != "active after stale wakeup" {
t.Fatalf("liveness response=%v err=%v", envelope, err)
}
case err := <-done:
t.Fatalf("stale wakeup disconnected active session: %v", err)
case <-ctx.Done():
t.Fatal("no progress after stale wakeup")
}
cancel()
<-done
}
func TestEventAcknowledgementDrainsDurableTail_REC_RUNTIME_03(t *testing.T) {
store, issue := recoveryStore(t)
for i := 0; i < 130; i++ {
recoveryEvent(t, store, issue, &rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_Resource{Resource: &rvboxv1.ResourceSnapshot{}}}, 6)
}
transport := newRecoveryTransport()
sent := make(map[domain.UUID]uint64)
session := Session{ID: "session", Generation: 1}
if err := flushEvents(context.Background(), transport, store, session, issue, agentproto.DefaultLimits(), sent); err != nil {
t.Fatal(err)
}
if len(transport.writes) != 128 {
t.Fatalf("first send window=%d", len(transport.writes))
}
for i := 0; i < 128; i++ {
<-transport.writes
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
done := make(chan error, 1)
go func() { done <- serveActive(ctx, transport, recoveryOptions(store, transport), session, sent) }()
transport.reads <- recoveryFrame(t, &rvboxv1.AgentEnvelope{Payload: &rvboxv1.AgentEnvelope_EventAck{EventAck: &rvboxv1.EventAck{IssueUuid: issue.String(), ThroughEventSeq: 128}}})
for want := uint64(129); want <= 130; want++ {
select {
case data := <-transport.writes:
envelope, err := agentproto.DecodeEnvelope(data, agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
if err != nil || envelope.GetCommandEvent().GetEventSeq() != want {
t.Fatalf("tail sequence want=%d event=%v err=%v", want, envelope, err)
}
case err := <-done:
t.Fatalf("acknowledgement disconnected session: %v", err)
case <-ctx.Done():
t.Fatal("acknowledged credit did not release pending tail")
}
}
cancel()
<-done
}
+23
View File
@@ -209,6 +209,12 @@ func runOnce(ctx context.Context, options RunnerOptions) (resultErr error) {
}
}
}
// Applying reconciliation can replace terminal rows with tombstones and
// terminate formerly active commands. Replay/capacity must use that new view.
snapshot, err = options.Store.ReconcileSnapshot(ctx)
if err != nil {
return fmt.Errorf("refresh reconciled client snapshot: %w", err)
}
sentEvents := make(map[domain.UUID]uint64)
if err := replayEvents(ctx, transport, options.Store, session, snapshot, limits, sentEvents); err != nil {
return fmt.Errorf("replay client events: %w", err)
@@ -294,6 +300,12 @@ func commandEventFromStored(event spool.Event) (*rvboxv1.CommandEvent, error) {
if err := proto.Unmarshal(event.Payload, &stored); err != nil || stored.Payload == nil {
return nil, fmt.Errorf("decode stored command event: %w", err)
}
if incomplete := stored.GetOutputIncomplete(); event.Kind == spool.EventKindOutputIncomplete && incomplete != nil && len(incomplete.Streams) == 0 {
// Early v1 writers omitted streams. Those events failed local wire
// validation before any server could accept their digest. Conservatively
// name both streams without changing persisted bytes, order, or reason.
incomplete.Streams = []rvboxv1.StreamKind{rvboxv1.StreamKind_STREAM_STDOUT, rvboxv1.StreamKind_STREAM_STDERR}
}
return &stored, nil
}
@@ -362,6 +374,11 @@ func serveActive(ctx context.Context, transport Transport, options RunnerOptions
continue
}
if err := flushEvents(ctx, transport, options.Store, session, issue, limits, sent); err != nil {
if errors.Is(err, spool.ErrUnknownCommand) {
// This internal wakeup can outlive a server-confirmed discard.
// It is a hint, not a new dispatch or an event acknowledgement.
continue
}
return fmt.Errorf("flush ready command %s events: %w", issue, err)
}
continue
@@ -391,6 +408,12 @@ func serveActive(ctx context.Context, transport Transport, options RunnerOptions
if err := ApplyEventAck(ctx, options.Store, envelope.GetEventAck()); err != nil {
return fmt.Errorf("apply event acknowledgement for %s: %w", envelope.GetEventAck().GetIssueUuid(), err)
}
issue, _ := domain.ParseUUIDv7(envelope.GetEventAck().GetIssueUuid())
// Released send-window credit must drain the durable tail even if no
// producer remains to emit another EventReady notification.
if err := flushEvents(ctx, transport, options.Store, session, issue, limits, sent); err != nil {
return fmt.Errorf("flush acknowledged command %s events: %w", issue, err)
}
case envelope.GetScriptChunk() != nil:
if err := handleScriptChunk(ctx, transport, options.Store, session, envelope.GetScriptChunk(), limits, options.Now, sent); err != nil {
return fmt.Errorf("apply script chunk for %s: %w", envelope.GetScriptChunk().GetIssueUuid(), err)
+1
View File
@@ -54,6 +54,7 @@ const (
// reconstruct the envelope without a second event-kind translation table.
EventKindOutput uint32 = 5
EventKindOutputTruncation uint32 = 10
EventKindOutputIncomplete uint32 = 11
)
type Event struct {
+6 -1
View File
@@ -232,7 +232,12 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
server.writeLoop(sessionContext, connection, queue, heartbeat, started)
cancel()
}()
defer func() { <-writerDone }()
defer func() {
// A reader can end before the writer observes the closed transport.
// Cancel first so cleanup does not wait for the next heartbeat write.
cancel()
<-writerDone
}()
for {
messageType, payload, err = connection.Read(sessionContext)
+48
View File
@@ -431,6 +431,54 @@ layer = "unit"
status = "implemented"
tests = ["internal/client/agent/executor_test.go:TestExecutorRunsDurableCommandAndPublishesTerminalEvents_HP_EXECUTOR_01"]
[[requirements]]
id = "BH-RUNTIME-05"
layer = "unit"
status = "implemented"
tests = ["internal/client/agent/recovery_test.go:TestCaptureFailurePublishesValidIncompleteBeforeTerminal_BH_RUNTIME_05"]
[[requirements]]
id = "BH-RUNTIME-06"
layer = "unit"
status = "implemented"
tests = ["internal/client/agent/recovery_test.go:TestLegacyIncompleteReplaySurvivesRestart_BH_RUNTIME_06"]
[[requirements]]
id = "BH-RUNTIME-07"
layer = "unit"
status = "implemented"
tests = ["internal/client/agent/recovery_test.go:TestLegacyIncompleteCompatibilityKeepsStreamValidationStrict_BH_RUNTIME_07"]
[[requirements]]
id = "BH-RUNTIME-08"
layer = "unit"
status = "implemented"
tests = ["internal/client/agent/recovery_test.go:TestIncompleteCloseoutSurvivesNormalQuotaExhaustion_BH_RUNTIME_08"]
[[requirements]]
id = "REC-RUNTIME-01"
layer = "unit"
status = "implemented"
tests = ["internal/client/agent/recovery_test.go:TestReconciliationDiscardUsesFreshReplaySnapshot_REC_RUNTIME_01"]
[[requirements]]
id = "REC-RUNTIME-02"
layer = "unit"
status = "implemented"
tests = ["internal/client/agent/recovery_test.go:TestDiscardedCommandNotificationDoesNotDisconnect_REC_RUNTIME_02"]
[[requirements]]
id = "REC-RUNTIME-03"
layer = "unit"
status = "implemented"
tests = ["internal/client/agent/recovery_test.go:TestEventAcknowledgementDrainsDurableTail_REC_RUNTIME_03"]
[[requirements]]
id = "REC-RUNTIME-04"
layer = "integration"
status = "implemented"
tests = ["test/integration/clientagent/recovery_integration_test.go:TestLegacyOutputLossReconnectAndFreshDispatch_REC_RUNTIME_04"]
[[requirements]]
id = "HP-EXECUTOR-02"
layer = "unit"
@@ -0,0 +1,179 @@
package clientagent_test
import (
"context"
"errors"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"time"
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/client/agent"
"github.com/rvbox/rvbox/internal/client/spool"
"github.com/rvbox/rvbox/internal/domain"
"github.com/rvbox/rvbox/internal/server/control"
"github.com/rvbox/rvbox/internal/server/session"
"github.com/rvbox/rvbox/internal/server/store"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
func TestLegacyOutputLossReconnectAndFreshDispatch_REC_RUNTIME_04(t *testing.T) {
ctx := context.Background()
persistence, err := store.Open(ctx, store.Options{DataDir: filepath.Join(t.TempDir(), "server"), BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
defer persistence.Close()
registry := session.NewRegistry()
server := httptest.NewServer(&session.AgentServer{Store: persistence, Registry: registry, Path: "/v1/agent", Limits: agentproto.DefaultLimits()})
defer server.Close()
controlService, err := control.NewService(control.Options{Store: persistence, WakeClient: registry.Wake})
if err != nil {
t.Fatal(err)
}
directory := filepath.Join(t.TempDir(), "client")
client, err := spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
defer func() { _ = client.Close() }()
hello := &rvboxv1.ClientHello{ClientId: "recovery-client", ClientInstanceId: client.ClientInstanceID().String(), SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1}, DaemonVersion: "legacy-recovery-test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS, Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_CMD}, MaxRunningCommands: 16, MaxQueuedCommands: 100, SentAt: timestamppb.Now()}
start := func() (context.CancelFunc, <-chan error) {
runCtx, cancel := context.WithCancel(ctx)
active := make(chan struct{}, 1)
done := make(chan error, 1)
go func() {
done <- agent.RunOnce(runCtx, agent.RunnerOptions{
Store: client, Hello: hello, Limits: agentproto.DefaultLimits(), Now: time.Now,
Backoff: agent.BackoffOptions{Initial: time.Millisecond, Maximum: time.Millisecond, StableReset: time.Second}, Jitter: func(time.Duration) time.Duration { return 0 },
Dial: func(ctx context.Context) (agent.Transport, error) {
return agent.DialWebSocket(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/v1/agent", nil)
},
OnSessionActive: func(value bool) {
if value {
active <- struct{}{}
}
},
OnDispatch: func(ctx context.Context, _ agent.Session, dispatch *rvboxv1.CommandDispatch) error {
issue, err := domain.ParseUUIDv7(dispatch.GetIssueUuid())
if err != nil {
return err
}
if _, err := client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), 1, "fixture running", time.Now()); err != nil {
return err
}
// Reproduce the old writer's durable event exactly: no streams.
payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{Reason: "legacy reader failure"}}})
if err != nil {
return err
}
if _, err := client.AppendEvent(ctx, issue, spool.EventInput{Kind: spool.EventKindOutputIncomplete, Compression: 1, RawBytes: uint64(len(payload)), Payload: payload, CreatedAt: time.Now()}); err != nil {
return err
}
_, err = client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "exit code 0", time.Now())
return err
},
})
}()
select {
case <-active:
case err := <-done:
cancel()
t.Fatalf("client failed to become active: %v", err)
case <-time.After(5 * time.Second):
cancel()
t.Fatal("client did not become active")
}
return cancel, done
}
queueAndAwait := func() string {
id, err := domain.NewUUIDv7()
if err != nil {
t.Fatal(err)
}
response, err := controlService.RunCommand(ctx, &rvboxv1.RunCommandRequest{TargetClientId: hello.ClientId, RequestId: id.String(), Spec: &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "echo fixture"}}})
if err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(5 * time.Second)
for {
command, err := controlService.GetCommand(ctx, &rvboxv1.GetCommandRequest{ClientId: hello.ClientId, IssueUuid: response.IssueUuid})
if err != nil {
t.Fatal(err)
}
if command.GetCommand().GetLifecycle() == rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED {
break
}
if time.Now().After(deadline) {
t.Fatalf("legacy incomplete event prevented terminal delivery: %v", command.GetCommand().GetLifecycle())
}
time.Sleep(10 * time.Millisecond)
}
events, err := persistence.ReadCommandEvents(ctx, id, 0, 100)
if err != nil || len(events) != 3 {
t.Fatalf("persisted events=%d err=%v", len(events), err)
}
payload, err := store.DecodeEventPayload(events[1])
if err != nil {
t.Fatal(err)
}
var event rvboxv1.CommandEvent
if err := proto.Unmarshal(payload, &event); err != nil {
t.Fatal(err)
}
if len(event.GetOutputIncomplete().GetStreams()) != 2 || event.GetOutputIncomplete().GetReason() != "legacy reader failure" {
t.Fatalf("loss evidence missing: %v", &event)
}
return id.String()
}
cancel, done := start()
defer cancel()
first := queueAndAwait()
cancel()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("client did not stop")
}
deadline := time.Now().Add(5 * time.Second)
for {
view, err := persistence.GetClientView(ctx, hello.ClientId)
if err != nil {
t.Fatal(err)
}
if !view.Connected {
break
}
if time.Now().After(deadline) {
t.Fatal("old session did not close")
}
time.Sleep(10 * time.Millisecond)
}
if err := client.Close(); err != nil {
t.Fatal(err)
}
client, err = spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
cancel, done = start()
defer cancel()
firstID, _ := domain.ParseUUIDv7(first)
if _, err := client.GetCommand(ctx, firstID); !errors.Is(err, spool.ErrUnknownCommand) {
t.Fatalf("server-confirmed terminal row not discarded: %v", err)
}
queueAndAwait()
cancel()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("reconnected client did not stop")
}
if _, err := client.Check(ctx); err != nil {
t.Fatal(err)
}
}