fix: recover reconnect after incomplete output capture
This commit is contained in:
1 parent
b52ca5a7c2
commit
6906ed52f2
9 files changed
+692
-3
No files matched your search
@@ -0,0 +1,179 @@
|
||||
package clientagent_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||
"github.com/rvbox/rvbox/internal/agentproto"
|
||||
"github.com/rvbox/rvbox/internal/client/agent"
|
||||
"github.com/rvbox/rvbox/internal/client/spool"
|
||||
"github.com/rvbox/rvbox/internal/domain"
|
||||
"github.com/rvbox/rvbox/internal/server/control"
|
||||
"github.com/rvbox/rvbox/internal/server/session"
|
||||
"github.com/rvbox/rvbox/internal/server/store"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
func TestLegacyOutputLossReconnectAndFreshDispatch_REC_RUNTIME_04(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
persistence, err := store.Open(ctx, store.Options{DataDir: filepath.Join(t.TempDir(), "server"), BusyTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer persistence.Close()
|
||||
registry := session.NewRegistry()
|
||||
server := httptest.NewServer(&session.AgentServer{Store: persistence, Registry: registry, Path: "/v1/agent", Limits: agentproto.DefaultLimits()})
|
||||
defer server.Close()
|
||||
controlService, err := control.NewService(control.Options{Store: persistence, WakeClient: registry.Wake})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
directory := filepath.Join(t.TempDir(), "client")
|
||||
client, err := spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = client.Close() }()
|
||||
hello := &rvboxv1.ClientHello{ClientId: "recovery-client", ClientInstanceId: client.ClientInstanceID().String(), SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1}, DaemonVersion: "legacy-recovery-test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS, Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_CMD}, MaxRunningCommands: 16, MaxQueuedCommands: 100, SentAt: timestamppb.Now()}
|
||||
start := func() (context.CancelFunc, <-chan error) {
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
active := make(chan struct{}, 1)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- agent.RunOnce(runCtx, agent.RunnerOptions{
|
||||
Store: client, Hello: hello, Limits: agentproto.DefaultLimits(), Now: time.Now,
|
||||
Backoff: agent.BackoffOptions{Initial: time.Millisecond, Maximum: time.Millisecond, StableReset: time.Second}, Jitter: func(time.Duration) time.Duration { return 0 },
|
||||
Dial: func(ctx context.Context) (agent.Transport, error) {
|
||||
return agent.DialWebSocket(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/v1/agent", nil)
|
||||
},
|
||||
OnSessionActive: func(value bool) {
|
||||
if value {
|
||||
active <- struct{}{}
|
||||
}
|
||||
},
|
||||
OnDispatch: func(ctx context.Context, _ agent.Session, dispatch *rvboxv1.CommandDispatch) error {
|
||||
issue, err := domain.ParseUUIDv7(dispatch.GetIssueUuid())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), 1, "fixture running", time.Now()); err != nil {
|
||||
return err
|
||||
}
|
||||
// Reproduce the old writer's durable event exactly: no streams.
|
||||
payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{Reason: "legacy reader failure"}}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := client.AppendEvent(ctx, issue, spool.EventInput{Kind: spool.EventKindOutputIncomplete, Compression: 1, RawBytes: uint64(len(payload)), Payload: payload, CreatedAt: time.Now()}); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = client.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "exit code 0", time.Now())
|
||||
return err
|
||||
},
|
||||
})
|
||||
}()
|
||||
select {
|
||||
case <-active:
|
||||
case err := <-done:
|
||||
cancel()
|
||||
t.Fatalf("client failed to become active: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
cancel()
|
||||
t.Fatal("client did not become active")
|
||||
}
|
||||
return cancel, done
|
||||
}
|
||||
queueAndAwait := func() string {
|
||||
id, err := domain.NewUUIDv7()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response, err := controlService.RunCommand(ctx, &rvboxv1.RunCommandRequest{TargetClientId: hello.ClientId, RequestId: id.String(), Spec: &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "echo fixture"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
command, err := controlService.GetCommand(ctx, &rvboxv1.GetCommandRequest{ClientId: hello.ClientId, IssueUuid: response.IssueUuid})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if command.GetCommand().GetLifecycle() == rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("legacy incomplete event prevented terminal delivery: %v", command.GetCommand().GetLifecycle())
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
events, err := persistence.ReadCommandEvents(ctx, id, 0, 100)
|
||||
if err != nil || len(events) != 3 {
|
||||
t.Fatalf("persisted events=%d err=%v", len(events), err)
|
||||
}
|
||||
payload, err := store.DecodeEventPayload(events[1])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var event rvboxv1.CommandEvent
|
||||
if err := proto.Unmarshal(payload, &event); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(event.GetOutputIncomplete().GetStreams()) != 2 || event.GetOutputIncomplete().GetReason() != "legacy reader failure" {
|
||||
t.Fatalf("loss evidence missing: %v", &event)
|
||||
}
|
||||
return id.String()
|
||||
}
|
||||
cancel, done := start()
|
||||
defer cancel()
|
||||
first := queueAndAwait()
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("client did not stop")
|
||||
}
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
view, err := persistence.GetClientView(ctx, hello.ClientId)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !view.Connected {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("old session did not close")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if err := client.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err = spool.Open(ctx, spool.Options{DataDir: directory, BusyTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel, done = start()
|
||||
defer cancel()
|
||||
firstID, _ := domain.ParseUUIDv7(first)
|
||||
if _, err := client.GetCommand(ctx, firstID); !errors.Is(err, spool.ErrUnknownCommand) {
|
||||
t.Fatalf("server-confirmed terminal row not discarded: %v", err)
|
||||
}
|
||||
queueAndAwait()
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("reconnected client did not stop")
|
||||
}
|
||||
if _, err := client.Check(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user