fix: recover reconnect after incomplete output capture

This commit is contained in:
cabbage committed 2026-10-27 14:36:06 +00:00
1 parent b52ca5a7c2
commit 6906ed52f2
9 files changed
+692 -3

No files matched your search

@@ -0,0 +1,179 @@
package clientagent_test
import (
"context"
"errors"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"time"
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/client/agent"
"github.com/rvbox/rvbox/internal/client/spool"
"github.com/rvbox/rvbox/internal/domain"
"github.com/rvbox/rvbox/internal/server/control"
"github.com/rvbox/rvbox/internal/server/session"
"github.com/rvbox/rvbox/internal/server/store"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
func TestLegacyOutputLossReconnectAndFreshDispatch_REC_RUNTIME_04(t *testing.T) {
ctx := context.Background()
persistence, err := store.Open(ctx, store.Options{DataDir: filepath.Join(t.TempDir(), "server"), BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
defer persistence.Close()
registry := session.NewRegistry()
server := httptest.NewServer(&session.AgentServer{Store: persistence, Registry: registry, Path: "/v1/agent", Limits: agentproto.DefaultLimits()})
defer server.Close()
controlService, err := control.NewService(control.Options{Store: persistence, WakeClient: registry.Wake})
if err != nil {
t.Fatal(err)
}
directory := filepath.Join(t.TempDir(), "client")
client, err := spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
defer func() { _ = client.Close() }()
hello := &rvboxv1.ClientHello{ClientId: "recovery-client", ClientInstanceId: client.ClientInstanceID().String(), SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1}, DaemonVersion: "legacy-recovery-test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS, Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_CMD}, MaxRunningCommands: 16, MaxQueuedCommands: 100, SentAt: timestamppb.Now()}
start := func() (context.CancelFunc, <-chan error) {
runCtx, cancel := context.WithCancel(ctx)
active := make(chan struct{}, 1)
done := make(chan error, 1)
go func() {
done <- agent.RunOnce(runCtx, agent.RunnerOptions{
Store: client, Hello: hello, Limits: agentproto.DefaultLimits(), Now: time.Now,
Backoff: agent.BackoffOptions{Initial: time.Millisecond, Maximum: time.Millisecond, StableReset: time.Second}, Jitter: func(time.Duration) time.Duration { return 0 },
Dial: func(ctx context.Context) (agent.Transport, error) {
return agent.DialWebSocket(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/v1/agent", nil)
},
OnSessionActive: func(value bool) {
if value {
active <- struct{}{}
}
},
OnDispatch: func(ctx context.Context, _ agent.Session, dispatch *rvboxv1.CommandDispatch) error {
issue, err := domain.ParseUUIDv7(dispatch.GetIssueUuid())
if err != nil {
return err
}
if _, err := client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), 1, "fixture running", time.Now()); err != nil {
return err
}
// Reproduce the old writer's durable event exactly: no streams.
payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{Reason: "legacy reader failure"}}})
if err != nil {
return err
}
if _, err := client.AppendEvent(ctx, issue, spool.EventInput{Kind: spool.EventKindOutputIncomplete, Compression: 1, RawBytes: uint64(len(payload)), Payload: payload, CreatedAt: time.Now()}); err != nil {
return err
}
_, err = client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "exit code 0", time.Now())
return err
},
})
}()
select {
case <-active:
case err := <-done:
cancel()
t.Fatalf("client failed to become active: %v", err)
case <-time.After(5 * time.Second):
cancel()
t.Fatal("client did not become active")
}
return cancel, done
}
queueAndAwait := func() string {
id, err := domain.NewUUIDv7()
if err != nil {
t.Fatal(err)
}
response, err := controlService.RunCommand(ctx, &rvboxv1.RunCommandRequest{TargetClientId: hello.ClientId, RequestId: id.String(), Spec: &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "echo fixture"}}})
if err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(5 * time.Second)
for {
command, err := controlService.GetCommand(ctx, &rvboxv1.GetCommandRequest{ClientId: hello.ClientId, IssueUuid: response.IssueUuid})
if err != nil {
t.Fatal(err)
}
if command.GetCommand().GetLifecycle() == rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED {
break
}
if time.Now().After(deadline) {
t.Fatalf("legacy incomplete event prevented terminal delivery: %v", command.GetCommand().GetLifecycle())
}
time.Sleep(10 * time.Millisecond)
}
events, err := persistence.ReadCommandEvents(ctx, id, 0, 100)
if err != nil || len(events) != 3 {
t.Fatalf("persisted events=%d err=%v", len(events), err)
}
payload, err := store.DecodeEventPayload(events[1])
if err != nil {
t.Fatal(err)
}
var event rvboxv1.CommandEvent
if err := proto.Unmarshal(payload, &event); err != nil {
t.Fatal(err)
}
if len(event.GetOutputIncomplete().GetStreams()) != 2 || event.GetOutputIncomplete().GetReason() != "legacy reader failure" {
t.Fatalf("loss evidence missing: %v", &event)
}
return id.String()
}
cancel, done := start()
defer cancel()
first := queueAndAwait()
cancel()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("client did not stop")
}
deadline := time.Now().Add(5 * time.Second)
for {
view, err := persistence.GetClientView(ctx, hello.ClientId)
if err != nil {
t.Fatal(err)
}
if !view.Connected {
break
}
if time.Now().After(deadline) {
t.Fatal("old session did not close")
}
time.Sleep(10 * time.Millisecond)
}
if err := client.Close(); err != nil {
t.Fatal(err)
}
client, err = spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
cancel, done = start()
defer cancel()
firstID, _ := domain.ParseUUIDv7(first)
if _, err := client.GetCommand(ctx, firstID); !errors.Is(err, spool.ErrUnknownCommand) {
t.Fatalf("server-confirmed terminal row not discarded: %v", err)
}
queueAndAwait()
cancel()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("reconnected client did not stop")
}
if _, err := client.Check(ctx); err != nil {
t.Fatal(err)
}
}