package agent import ( "bytes" "context" "errors" "path/filepath" "testing" "time" rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1" "github.com/rvbox/rvbox/internal/agentproto" "github.com/rvbox/rvbox/internal/client/spool" "github.com/rvbox/rvbox/internal/client/supervisor" "github.com/rvbox/rvbox/internal/domain" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" ) func recoveryStore(t *testing.T) (*spool.Store, domain.UUID) { t.Helper() return recoveryStoreAt(t, filepath.Join(t.TempDir(), "spool")) } func recoveryStoreAt(t *testing.T, directory string) (*spool.Store, domain.UUID) { t.Helper() store, err := spool.Open(context.Background(), spool.Options{DataDir: directory, BusyTimeout: time.Second}) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = store.Close() }) issue, err := domain.NewUUIDv7() if err != nil { t.Fatal(err) } if _, err := store.AcceptCommand(context.Background(), spool.Command{IssueUUID: issue, Revision: 1, Phase: uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), ImmutableSHA256: [32]byte{1}}, time.Now()); err != nil { t.Fatal(err) } return store, issue } func recoveryEvent(t *testing.T, store *spool.Store, issue domain.UUID, event *rvboxv1.CommandEvent, kind uint32) { t.Helper() data, err := proto.Marshal(event) if err != nil { t.Fatal(err) } if _, err := store.AppendEvent(context.Background(), issue, spool.EventInput{Kind: kind, Compression: 1, RawBytes: uint64(len(data)), Payload: data, CreatedAt: time.Now()}); err != nil { t.Fatal(err) } } func recoveryOptions(store *spool.Store, transport Transport) RunnerOptions { return RunnerOptions{Store: store, Dial: func(context.Context) (Transport, error) { return transport, nil }, Hello: &rvboxv1.ClientHello{ ClientId: "recovery-client", ClientInstanceId: store.ClientInstanceID().String(), SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1}, DaemonVersion: "test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS, Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_CMD}, MaxRunningCommands: 16, MaxQueuedCommands: 100, SentAt: timestamppb.Now(), }, Limits: agentproto.DefaultLimits(), Backoff: BackoffOptions{Initial: time.Millisecond, Maximum: time.Millisecond, StableReset: time.Second}, Jitter: func(time.Duration) time.Duration { return 0 }, Now: time.Now} } func recoveryHandshake(t *testing.T, result *rvboxv1.ReconcileResult) *runnerTransport { t.Helper() transport := &runnerTransport{terminal: errors.New("fixture disconnected")} for _, payload := range []*rvboxv1.AgentEnvelope{ {Payload: &rvboxv1.AgentEnvelope_ServerWelcome{ServerWelcome: &rvboxv1.ServerWelcome{SelectedProtocol: &rvboxv1.ProtocolVersion{Major: 1}, ServerTime: timestamppb.Now()}}}, {Payload: &rvboxv1.AgentEnvelope_ReconcileRequest{ReconcileRequest: &rvboxv1.ReconcileRequest{}}}, {Payload: &rvboxv1.AgentEnvelope_ReconcileResult{ReconcileResult: result}}, } { payload.SessionId, payload.SessionGeneration = "recovery-session", 1 encoded, err := proto.Marshal(payload) if err != nil { t.Fatal(err) } transport.reads = append(transport.reads, encoded) } return transport } func TestCaptureFailurePublishesValidIncompleteBeforeTerminal_BH_RUNTIME_05(t *testing.T) { store, issue := recoveryStore(t) executor := &Executor{Store: store, Now: time.Now} executor.watch(context.Background(), issue, 1, &failedCapture{}, func() {}) if _, err := store.AssignSendWindow(context.Background(), issue, 128, 1<<20); err != nil { t.Fatal(err) } events, err := store.PendingEvents(context.Background(), issue) if err != nil || len(events) != 2 { t.Fatalf("capture closeout events=%d err=%v", len(events), err) } transport := &runnerTransport{} for _, event := range events { if err := SendStoredEvent(context.Background(), transport, Session{ID: "session", Generation: 1}, event, agentproto.DefaultLimits()); err != nil { t.Fatal(err) } } first, err := agentproto.DecodeEnvelope(transport.writes[0], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS) if err != nil { t.Fatal(err) } marker := first.GetCommandEvent().GetOutputIncomplete() if marker == nil || len(marker.GetStreams()) != 2 || marker.GetReason() != "capture reader failed" { t.Fatalf("incomplete marker=%v", marker) } last, err := agentproto.DecodeEnvelope(transport.writes[1], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS) if err != nil || last.GetCommandEvent().GetLifecycle().GetLifecycle() != rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED { t.Fatalf("terminal event=%v err=%v", last, err) } if _, err := store.Check(context.Background()); err != nil { t.Fatal(err) } } type failedCapture struct{} func (*failedCapture) IssueUUID() domain.UUID { return domain.UUID{} } func (*failedCapture) Identity() supervisor.EffectiveIdentity { return supervisor.EffectiveIdentity{} } func (*failedCapture) PID() uint32 { return 0 } func (*failedCapture) Release(context.Context) error { return nil } func (*failedCapture) ReadOutput(context.Context) (supervisor.OutputChunk, error) { return supervisor.OutputChunk{}, errors.New("capture reader failed") } func (*failedCapture) Wait(context.Context) (supervisor.ExitStatus, error) { return supervisor.ExitStatus{Code: 0}, nil } func (*failedCapture) WriteStdin(context.Context, []byte, bool) error { return nil } func (*failedCapture) CloseStdin(context.Context) error { return nil } func TestLegacyIncompleteReplaySurvivesRestart_BH_RUNTIME_06(t *testing.T) { directory := filepath.Join(t.TempDir(), "spool") store, issue := recoveryStoreAt(t, directory) recoveryEvent(t, store, issue, &rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{Reason: "legacy capture failure"}}}, spool.EventKindOutputIncomplete) if _, err := store.AppendLifecycle(context.Background(), issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "exit code 0", time.Now()); err != nil { t.Fatal(err) } if _, err := store.AssignSendWindow(context.Background(), issue, 128, 1<<20); err != nil { t.Fatal(err) } before, err := store.PendingEvents(context.Background(), issue) if err != nil { t.Fatal(err) } if err := store.Close(); err != nil { t.Fatal(err) } store, err = spool.Open(context.Background(), spool.Options{DataDir: directory, BusyTimeout: time.Second}) if err != nil { t.Fatal(err) } defer store.Close() var digest []byte for attempt := 0; attempt < 2; attempt++ { transport := recoveryHandshake(t, &rvboxv1.ReconcileResult{}) active := false options := recoveryOptions(store, transport) options.OnSessionActive = func(value bool) { active = active || value } if err := RunOnce(context.Background(), options); !errors.Is(err, transport.terminal) { t.Fatalf("legacy replay error=%v", err) } if !active || len(transport.writes) != 5 { t.Fatalf("active=%v writes=%d", active, len(transport.writes)) } envelope, err := agentproto.DecodeEnvelope(transport.writes[2], agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS) if err != nil { t.Fatal(err) } event := envelope.GetCommandEvent() if event.GetEventSeq() != 1 || event.GetOutputIncomplete().GetReason() != "legacy capture failure" || len(event.GetOutputIncomplete().GetStreams()) != 2 { t.Fatalf("recovered event=%v", event) } if attempt == 0 { digest = append([]byte(nil), event.GetImmutableEventSha256()...) } else if !bytes.Equal(digest, event.GetImmutableEventSha256()) { t.Fatal("legacy replay digest changed across reconnect") } } after, err := store.PendingEvents(context.Background(), issue) if err != nil || len(after) != len(before) || !bytes.Equal(before[0].Payload, after[0].Payload) { t.Fatalf("legacy persisted evidence changed: %v", err) } if _, err := store.Check(context.Background()); err != nil { t.Fatal(err) } } func TestReconciliationDiscardUsesFreshReplaySnapshot_REC_RUNTIME_01(t *testing.T) { store, issue := recoveryStore(t) if _, err := store.AppendLifecycle(context.Background(), issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "done", time.Now()); err != nil { t.Fatal(err) } transport := recoveryHandshake(t, &rvboxv1.ReconcileResult{DiscardLocalTerminalIssueUuids: []string{issue.String()}}) options := recoveryOptions(store, transport) if err := RunOnce(context.Background(), options); !errors.Is(err, transport.terminal) { t.Fatalf("discarded command poisoned replay: %v", err) } if len(transport.writes) != 3 { t.Fatalf("discarded rows replayed: writes=%d", len(transport.writes)) } if _, err := store.GetCommand(context.Background(), issue); !errors.Is(err, spool.ErrUnknownCommand) { t.Fatalf("discarded command remains: %v", err) } if _, err := store.Check(context.Background()); err != nil { t.Fatal(err) } } func TestLegacyIncompleteCompatibilityKeepsStreamValidationStrict_BH_RUNTIME_07(t *testing.T) { issue, _ := domain.NewUUIDv7() for _, streams := range [][]rvboxv1.StreamKind{{rvboxv1.StreamKind_STREAM_KIND_UNSPECIFIED}, {99}, {rvboxv1.StreamKind_STREAM_STDOUT, rvboxv1.StreamKind_STREAM_STDOUT}} { payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_OutputIncomplete{OutputIncomplete: &rvboxv1.OutputIncomplete{Streams: streams, Reason: "bad stream"}}}) if err != nil { t.Fatal(err) } transport := &runnerTransport{} err = SendStoredEvent(context.Background(), transport, Session{ID: "session", Generation: 1}, spool.Event{IssueUUID: issue, EventSeq: 1, Kind: spool.EventKindOutputIncomplete, CreatedAt: time.Now(), Payload: payload}, agentproto.DefaultLimits()) if !errors.Is(err, agentproto.ErrInvalidEnvelope) || len(transport.writes) != 0 { t.Fatalf("streams=%v err=%v writes=%d", streams, err, len(transport.writes)) } } } func TestIncompleteCloseoutSurvivesNormalQuotaExhaustion_BH_RUNTIME_08(t *testing.T) { ctx := context.Background() limits := spool.DefaultQuotaLimits() limits.CommandOutputBytes, limits.CommandTotalBytes = 4<<10, 12<<10 limits.ClientTotalBytes, limits.CloseoutReserveBytes = 32<<10, 4<<10 store, err := spool.Open(ctx, spool.Options{DataDir: filepath.Join(t.TempDir(), "spool"), BusyTimeout: time.Second, QuotaLimits: limits}) if err != nil { t.Fatal(err) } defer store.Close() issue, _ := domain.NewUUIDv7() if _, err := store.AcceptCommand(ctx, spool.Command{IssueUUID: issue, Revision: 1, Phase: uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), ImmutableSHA256: [32]byte{1}}, time.Now()); err != nil { t.Fatal(err) } payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_Resource{Resource: &rvboxv1.ResourceSnapshot{}}}) if err != nil { t.Fatal(err) } full := false for i := 0; i < 100; i++ { _, err := store.AppendEvent(ctx, issue, spool.EventInput{Kind: 6, Compression: 1, Payload: payload, RawBytes: uint64(len(payload)), CreatedAt: time.Now()}) if errors.Is(err, spool.ErrCapacityExhausted) { full = true break } if err != nil { t.Fatal(err) } } if !full { t.Fatal("normal allocation did not reach its quota") } executor := &Executor{Store: store, Now: time.Now} if err := executor.appendIncomplete(ctx, issue, 1, "normal command quota exhausted"); err != nil { t.Fatalf("reserved incomplete closeout rejected: %v", err) } if _, err := store.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "done", time.Now()); err != nil { t.Fatalf("reserved terminal closeout rejected: %v", err) } if _, err := store.Check(ctx); err != nil { t.Fatal(err) } } type recoveryTransport struct { reads chan []byte writes chan []byte } func newRecoveryTransport() *recoveryTransport { return &recoveryTransport{reads: make(chan []byte, 1), writes: make(chan []byte, 256)} } func (transport *recoveryTransport) Read(ctx context.Context) ([]byte, error) { select { case data := <-transport.reads: return data, nil case <-ctx.Done(): return nil, ctx.Err() } } func (transport *recoveryTransport) Write(ctx context.Context, data []byte) error { select { case transport.writes <- append([]byte(nil), data...): return nil case <-ctx.Done(): return ctx.Err() } } func (*recoveryTransport) Close() error { return nil } func recoveryFrame(t *testing.T, envelope *rvboxv1.AgentEnvelope) []byte { t.Helper() envelope.SessionId, envelope.SessionGeneration = "session", 1 data, err := proto.Marshal(envelope) if err != nil { t.Fatal(err) } return data } func TestDiscardedCommandNotificationDoesNotDisconnect_REC_RUNTIME_02(t *testing.T) { store, live := recoveryStore(t) stale, _ := domain.NewUUIDv7() notify := make(chan domain.UUID) transport := newRecoveryTransport() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() options := recoveryOptions(store, transport) options.EventReady = notify options.OnStdin = func(ctx context.Context, _ Session, write *rvboxv1.StdinWrite) error { payload, err := proto.Marshal(&rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_Resource{Resource: &rvboxv1.ResourceSnapshot{DiagnosticDetail: "active after stale wakeup"}}}) if err != nil { return err } _, err = store.AppendEvent(ctx, live, spool.EventInput{Kind: 6, Compression: 1, RawBytes: uint64(len(payload)), Payload: payload, CreatedAt: time.Now()}) return err } done := make(chan error, 1) go func() { done <- serveActive(ctx, transport, options, Session{ID: "session", Generation: 1}, make(map[domain.UUID]uint64)) }() select { case notify <- stale: case err := <-done: t.Fatalf("before wakeup: %v", err) case <-ctx.Done(): t.Fatal("wakeup not consumed") } transport.reads <- recoveryFrame(t, &rvboxv1.AgentEnvelope{Payload: &rvboxv1.AgentEnvelope_StdinWrite{StdinWrite: &rvboxv1.StdinWrite{IssueUuid: live.String(), WriteSeq: 1, Data: []byte("probe")}}}) select { case data := <-transport.writes: envelope, err := agentproto.DecodeEnvelope(data, agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS) if err != nil || envelope.GetCommandEvent().GetResource().GetDiagnosticDetail() != "active after stale wakeup" { t.Fatalf("liveness response=%v err=%v", envelope, err) } case err := <-done: t.Fatalf("stale wakeup disconnected active session: %v", err) case <-ctx.Done(): t.Fatal("no progress after stale wakeup") } cancel() <-done } func TestEventAcknowledgementDrainsDurableTail_REC_RUNTIME_03(t *testing.T) { store, issue := recoveryStore(t) for i := 0; i < 130; i++ { recoveryEvent(t, store, issue, &rvboxv1.CommandEvent{Payload: &rvboxv1.CommandEvent_Resource{Resource: &rvboxv1.ResourceSnapshot{}}}, 6) } transport := newRecoveryTransport() sent := make(map[domain.UUID]uint64) session := Session{ID: "session", Generation: 1} if err := flushEvents(context.Background(), transport, store, session, issue, agentproto.DefaultLimits(), sent); err != nil { t.Fatal(err) } if len(transport.writes) != 128 { t.Fatalf("first send window=%d", len(transport.writes)) } for i := 0; i < 128; i++ { <-transport.writes } ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() done := make(chan error, 1) go func() { done <- serveActive(ctx, transport, recoveryOptions(store, transport), session, sent) }() transport.reads <- recoveryFrame(t, &rvboxv1.AgentEnvelope{Payload: &rvboxv1.AgentEnvelope_EventAck{EventAck: &rvboxv1.EventAck{IssueUuid: issue.String(), ThroughEventSeq: 128}}}) for want := uint64(129); want <= 130; want++ { select { case data := <-transport.writes: envelope, err := agentproto.DecodeEnvelope(data, agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS) if err != nil || envelope.GetCommandEvent().GetEventSeq() != want { t.Fatalf("tail sequence want=%d event=%v err=%v", want, envelope, err) } case err := <-done: t.Fatalf("acknowledgement disconnected session: %v", err) case <-ctx.Done(): t.Fatal("acknowledged credit did not release pending tail") } } cancel() <-done }