From 1b5c6dbf9844a9bd0b0b73ac81f46b3f0d3b55e6 Mon Sep 17 00:00:00 2001 From: cabbage Date: Mon, 9 Nov 2026 08:04:42 +0000 Subject: [PATCH] fix(client): tolerate script replay after command completion --- docs/operations-runbook.md | 7 ++ internal/client/agent/runner.go | 35 +++++- internal/client/agent/script_replay_test.go | 133 ++++++++++++++++++++ skills/rvbox/references/troubleshooting.md | 1 + test/coverage.toml | 6 + 5 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 internal/client/agent/script_replay_test.go diff --git a/docs/operations-runbook.md b/docs/operations-runbook.md index 3d5f40b..9607717 100644 --- a/docs/operations-runbook.md +++ b/docs/operations-runbook.md @@ -136,6 +136,13 @@ exact ignored `dist/` directory after preserving any evidence. Preserve spool/history: rebooting cannot remove a replayable oversized event. New server logs include bounded read/reconciliation errors and an `agent_message_too_big` counter; new clients advertise the actual build version. +- **`terminal command cannot accept script data` during reconnect:** the server + may replay an upload while the client is still delivering that command's + terminal events. Corrected clients ignore these frames only after verifying + the command is locally terminal; existing event replay reports the outcome. + They neither change script bytes nor invoke launch again. The same error for + a committed but still-live script, unknown owners, and malformed transfers + remains a failure. Upgrade the client while preserving its spool. - **No client / stale session:** verify nginx has WebSocket `101` entries for `/v1/agent`, server readiness is true, and the Windows service is running. Use `rvc stat CLIENT_ID`; do not restart the client merely to clear history. diff --git a/internal/client/agent/runner.go b/internal/client/agent/runner.go index e8048bf..d72e39a 100644 --- a/internal/client/agent/runner.go +++ b/internal/client/agent/runner.go @@ -416,11 +416,21 @@ func serveActive(ctx context.Context, transport Transport, options RunnerOptions } case envelope.GetScriptChunk() != nil: if err := handleScriptChunk(ctx, transport, options.Store, session, envelope.GetScriptChunk(), limits, options.Now, sent); err != nil { - return fmt.Errorf("apply script chunk for %s: %w", envelope.GetScriptChunk().GetIssueUuid(), err) + err = finishedScriptReplay(ctx, options.Store, envelope.GetScriptChunk().GetIssueUuid(), err) + if err != nil { + return fmt.Errorf("apply script chunk for %s: %w", envelope.GetScriptChunk().GetIssueUuid(), err) + } + options.incMetric("terminal_script_frame") + continue } case envelope.GetScriptCommit() != nil: if err := handleScriptCommit(ctx, transport, options.Store, session, envelope.GetScriptCommit(), limits, options.Now, sent); err != nil { - return fmt.Errorf("commit script for %s: %w", envelope.GetScriptCommit().GetIssueUuid(), err) + err = finishedScriptReplay(ctx, options.Store, envelope.GetScriptCommit().GetIssueUuid(), err) + if err != nil { + return fmt.Errorf("commit script for %s: %w", envelope.GetScriptCommit().GetIssueUuid(), err) + } + options.incMetric("terminal_script_frame") + continue } if options.OnScriptReady != nil { issue, err := domain.ParseUUIDv7(envelope.GetScriptCommit().GetIssueUuid()) @@ -563,6 +573,27 @@ func boundedError(err error) string { return message } +// A reconnect can race the server's upload replay with the client's terminal +// event replay. Leave the finished command unchanged and let its pending +// events drain; no script acknowledgement or launch should be fabricated. +func finishedScriptReplay(ctx context.Context, store *spool.Store, text string, transferErr error) error { + if !errors.Is(transferErr, spool.ErrScriptTerminal) { + return transferErr + } + issue, err := domain.ParseUUIDv7(text) + if err != nil { + return err + } + command, err := store.GetCommand(ctx, issue) + if err != nil { + return err + } + if !command.Terminal { + return transferErr + } + return nil +} + func handleScriptChunk(ctx context.Context, transport Transport, store *spool.Store, session Session, chunk *rvboxv1.ScriptChunk, limits agentproto.Limits, now func() time.Time, sent map[domain.UUID]uint64) error { status, err := ApplyScriptChunk(ctx, store, session, chunk, limits) if err != nil { diff --git a/internal/client/agent/script_replay_test.go b/internal/client/agent/script_replay_test.go new file mode 100644 index 0000000..a0b09cc --- /dev/null +++ b/internal/client/agent/script_replay_test.go @@ -0,0 +1,133 @@ +package agent + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "path/filepath" + "testing" + "time" + + rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1" + "github.com/rvbox/rvbox/internal/agentproto" + "github.com/rvbox/rvbox/internal/client/spool" + "github.com/rvbox/rvbox/internal/domain" + "github.com/rvbox/rvbox/internal/observability" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestFinishedScriptReplayKeepsSessionUsable_REC_RUNTIME_06(t *testing.T) { + t.Parallel() + ctx := context.Background() + store, err := spool.Open(ctx, spool.Options{DataDir: filepath.Join(t.TempDir(), "spool"), BusyTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + defer store.Close() + issue, _ := domain.ParseUUIDv7("019c46f1-1d02-7000-8000-0000000000e1") + body := []byte("echo finished script") + digest := sha256.Sum256(body) + descriptor := spool.ScriptDescriptor{SizeBytes: uint64(len(body)), SHA256: digest} + spec := &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_Script{ + Script: &rvboxv1.ScriptDescriptor{Filename: "done.cmd", SizeBytes: uint64(len(body)), Sha256: digest[:]}, + }} + encoded, _ := proto.Marshal(spec) + if _, err := store.AcceptCommand(ctx, spool.Command{IssueUUID: issue, ImmutableSHA256: digest, Revision: 1, + Phase: uint32(rvboxv1.CommandLifecycle_COMMAND_ACCEPTED), ExecutionSpec: encoded, Script: &descriptor}, time.Now()); err != nil { + t.Fatal(err) + } + if _, err := store.AppendScriptChunk(ctx, issue, 0, body, digest); err != nil { + t.Fatal(err) + } + if _, err := store.CommitScript(ctx, issue, descriptor); err != nil { + t.Fatal(err) + } + // A committed but live script must still reject new bytes. The shared + // ErrScriptTerminal value alone is not sufficient to ignore a frame. + if err := finishedScriptReplay(ctx, store, issue.String(), spool.ErrScriptTerminal); !errors.Is(err, spool.ErrScriptTerminal) { + t.Fatalf("live script error suppressed: %v", err) + } + for _, phase := range []rvboxv1.CommandLifecycle{rvboxv1.CommandLifecycle_COMMAND_RUNNING, rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED} { + if _, err := store.AppendLifecycle(ctx, issue, uint32(phase), 1, "finished offline", time.Now()); err != nil { + t.Fatal(err) + } + } + if err := finishedScriptReplay(ctx, store, issue.String(), spool.ErrScriptDigest); !errors.Is(err, spool.ErrScriptDigest) { + t.Fatalf("checksum failure suppressed: %v", err) + } + original, err := store.ScriptBody(ctx, issue) + if err != nil { + t.Fatal(err) + } + marshal := func(payload *rvboxv1.AgentEnvelope) []byte { + t.Helper() + payload.SessionId, payload.SessionGeneration = "session", 1 + encoded, err := proto.Marshal(payload) + if err != nil { + t.Fatal(err) + } + return encoded + } + fresh, _ := domain.ParseUUIDv7("019c46f1-1d02-7000-8000-0000000000e2") + transport := &scriptReplayTransport{runnerTransport: runnerTransport{reads: [][]byte{ + marshal(&rvboxv1.AgentEnvelope{Payload: &rvboxv1.AgentEnvelope_ScriptChunk{ScriptChunk: &rvboxv1.ScriptChunk{ + IssueUuid: issue.String(), Data: body, Sha256: digest[:], + }}}), + marshal(&rvboxv1.AgentEnvelope{Payload: &rvboxv1.AgentEnvelope_ScriptCommit{ScriptCommit: &rvboxv1.ScriptCommit{ + IssueUuid: issue.String(), SizeBytes: uint64(len(body)), Sha256: digest[:], + }}}), + marshal(&rvboxv1.AgentEnvelope{Payload: &rvboxv1.AgentEnvelope_CommandDispatch{CommandDispatch: &rvboxv1.CommandDispatch{ + IssueUuid: fresh.String(), CommandRevision: 1, TargetSessionGeneration: 1, IssueTime: timestamppb.Now(), + ImmutableRequestSha256: digest[:], Spec: &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, + Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "echo next"}}, + }}}), + }}} + runCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + dispatched, done := make(chan struct{}, 1), make(chan error, 1) + readyCalls := 0 + metrics := observability.New() + go func() { + done <- serveActive(runCtx, transport, RunnerOptions{Store: store, Limits: agentproto.DefaultLimits(), Now: time.Now, Metrics: metrics, + OnScriptReady: func(context.Context, Session, domain.UUID) error { readyCalls++; return nil }, + OnDispatch: func(context.Context, Session, *rvboxv1.CommandDispatch) error { dispatched <- struct{}{}; return nil }, + }, Session{ID: "session", Generation: 1}, map[domain.UUID]uint64{}) + }() + select { + case err := <-done: + t.Fatalf("stale upload closed session: %v", err) + case <-dispatched: + cancel() + <-done + case <-runCtx.Done(): + t.Fatal("fresh command could not pass stale upload") + } + if readyCalls != 0 { + t.Fatal("finished script was made launch-eligible again") + } + retained, err := store.ScriptBody(ctx, issue) + if err != nil || !bytes.Equal(original, retained) { + t.Fatalf("finished script bytes changed: %v", err) + } + _, _, counters := metrics.Snapshot() + if counters["terminal_script_frame"] != 2 { + t.Fatalf("stale frame count: %v", counters) + } +} + +type scriptReplayTransport struct{ runnerTransport } + +func (transport *scriptReplayTransport) Read(ctx context.Context) ([]byte, error) { + transport.mu.Lock() + if len(transport.reads) != 0 { + frame := transport.reads[0] + transport.reads = transport.reads[1:] + transport.mu.Unlock() + return frame, nil + } + transport.mu.Unlock() + <-ctx.Done() + return nil, ctx.Err() +} diff --git a/skills/rvbox/references/troubleshooting.md b/skills/rvbox/references/troubleshooting.md index 408a1f7..d2ff414 100644 --- a/skills/rvbox/references/troubleshooting.md +++ b/skills/rvbox/references/troubleshooting.md @@ -20,6 +20,7 @@ initial evidence; do not restart services or resubmit work merely to inspect it. | Client missing or NOT_FOUND | List all client pages; verify the exact registered ID and server instance. A hostname assumption or wrong Compose project can select different state. Retention can also remove command history. | | Disconnected / OFFLINE / UNAVAILABLE | Separate server recovery from client connectivity. Inspect SCM, WSS hostname/path, proxy WebSocket upgrade, certificate/time/CA, DNS, reconnect logs, and pending instance conflicts. Missing optional HTTP health is not proof of a stopped daemon. | | `StatusMessageTooBig` / `read limited at 32769 bytes` after output or during reconnect | Early v1 WebSocket readers kept the library's 32 KiB default despite allowing larger protocol envelopes. Upgrade the server first to accept pending output, then the client for large incoming dispatch/history. Preserve spool/history; rebooting replays the same pending message. Corrected readers use the configured envelope ceiling, normally 1 MiB. | +| Repeated `terminal command cannot accept script data` | Upload replay can race the finished command's pending event replay. Corrected clients ignore stale script frames for confirmed local terminal commands and let events drain, without modifying script bytes or relaunching. Upgrade the client and retain spool/history. A terminal-script error for a still-live command remains a protocol failure. | | Repeated `signal command ...: command is not running` | An older client disconnects on a late signal instead of persisting its rejection. Upgrade the client while retaining config/spool/history; corrected builds acknowledge the signal without disconnecting. Do not wipe state or reissue the original command. | | QUEUED persists | Check connection, supported shell, queue/running capacity, acceptance events, and queue expiry. Queuing does not promise execution, and queue TTL is not a runtime timeout. | | CLI deadline / broken stream | Work may still run. Query the captured issue and resume history via the API. For uncertain admission, retry only with the same request ID and payload. | diff --git a/test/coverage.toml b/test/coverage.toml index 4b167d6..9272f81 100644 --- a/test/coverage.toml +++ b/test/coverage.toml @@ -1163,6 +1163,12 @@ layer = "integration" status = "implemented" tests = ["test/integration/clientagent/websocket_limits_integration_test.go:TestLargeOutputReplaysAfterReconnect_REC_WS_01"] +[[requirements]] +id = "REC-RUNTIME-06" +layer = "unit" +status = "implemented" +tests = ["internal/client/agent/script_replay_test.go:TestFinishedScriptReplayKeepsSessionUsable_REC_RUNTIME_06"] + [[requirements]] id = "HP-WS-01" layer = "integration"