diff --git a/PENDING-ISSUES.md b/PENDING-ISSUES.md index 388a0c6..e294ef5 100644 --- a/PENDING-ISSUES.md +++ b/PENDING-ISSUES.md @@ -8,6 +8,37 @@ the repository and isolated fixtures. Do not restart/upgrade live services or replace the published installer for this feature without a later explicit instruction lifting that hold. See [the design](docs/transport-chunking.md). +## Legacy Windows client upgrade remains recommended + +On 2026-11-16 a legacy client entered a persistent reconnect loop because the +server echoed every already-compacted tombstone as a terminal-discard action. +The redundant UUID list exceeded that client's 32 KiB WebSocket receive limit. +The server correction omits those no-op actions only after the usual immutable +hash/revision checks and reconciliation decision. Retained terminal commands +still receive discard authorization, and contradictory non-terminal server +state is still interrupted and incidented. Tombstones and history are preserved. + +The isolated v16 hotfix excludes the held auto-chunking implementation. The +new regression reproduces the original error using TLS, no subprotocol, a real +32 KiB reader and 1,100 tombstones, then verifies repeat reconciliation after the +fix. Store tests cover missing/terminal server rows, hash mismatch, and a +tombstone contradicting running server state. Large genuine action lists, +inline commands and stdin can still exceed the legacy reader; upgrade old +Windows clients to the verified non-chunking installer. This optimization does +not change the remaining 1 MiB logical snapshot limit or add pagination. + +Recovery on 2026-11-16 was verified live: server version +`0.1.0-gui-manual.16-tombstone.1`, based on `474afb3` plus only the store +correction; no `internal/agentwire` package is present in that build source. +The active fixture's Compose override pins the binary under +`.test-runs/workstation-disconnect-20261116/`; treat it as a live deployment +artifact, not disposable diagnostics. That directory also holds build +provenance, the preserved pre-upgrade data/config and the guarded recovery script. +Do not delete it while mounted. The existing published v16 installer is +unchanged; its executable was checksum-verified on the user's desktop for +manual upgrade from v11. No Windows service restart or command reissue was +needed for server recovery. + ## Reconciliation must restore a lost command acceptance A reconnect can leave the server at `COMMAND_DISPATCHED` while the client has diff --git a/docs/operations-runbook.md b/docs/operations-runbook.md index bb73ab9..0e0de41 100644 --- a/docs/operations-runbook.md +++ b/docs/operations-runbook.md @@ -147,6 +147,16 @@ exact ignored `dist/` directory after preserving any evidence. the 1 MiB send window, so pending uploads can reach older clients. This does not lower the server's inbound limits. Large inline command text, stdin, or reconciliation responses still require the corrected client reader. +- **Reconnect fails after accumulating command history:** the server used to + echo the client's entire compact tombstone ledger as terminal-discard UUIDs. + Roughly 860 UUIDs plus envelope overhead exceed a legacy 32 KiB reader, even + when the commands produced no large output. The correction validates and + reconciles tombstones normally, then omits redundant discard instructions; + it retains history and still authorizes genuinely retained terminal bodies. + This can be backported independently of held auto-chunking. Keep client + spool/tombstones intact, back up the full server state before applying the + isolated fix, and verify both reconnect and a fresh command's terminal result. + Large genuine action lists still require an updated client reader. - **`terminal command cannot accept script data` during reconnect:** the server may replay an upload while the client is still delivering that command's terminal events. Corrected clients ignore these frames only after verifying diff --git a/internal/server/store/reconcile.go b/internal/server/store/reconcile.go index cddc0d9..6e6e18f 100644 --- a/internal/server/store/reconcile.go +++ b/internal/server/store/reconcile.go @@ -183,7 +183,13 @@ func (store *Store) reconcileClientSnapshotAt(ctx context.Context, clientID stri incidents = append(incidents, reconcileIncident{issue: parseIssueUnchecked(key), summary: "client and server command state disagree; local process must be terminated", dataLoss: true}) } case domain.ReconcileDiscardLocalTerminal: - result.DiscardLocalTerminalIssueUuids = append(result.DiscardLocalTerminalIssueUuids, key) + // Tombstones already have no command body or outbox to discard. + // Keep validating their hash/revision and reconciling contradictions + // above, but do not echo the growing compact ledger on every reconnect. + // The redundant UUID list alone can exceed a legacy peer's read limit. + if !client.GetTombstoned() { + result.DiscardLocalTerminalIssueUuids = append(result.DiscardLocalTerminalIssueUuids, key) + } case domain.ReconcileResumeDelivery: if generation != 0 && row.target.Valid && uint64(row.target.Int64) != generation || generation != 0 && !row.target.Valid { if _, err := tx.ExecContext(ctx, `UPDATE commands SET target_session_generation = ? WHERE issue_uuid = ? AND client_id = ? AND lifecycle BETWEEN 2 AND 4`, generation, row.issue[:], clientID); err != nil { diff --git a/internal/server/store/reconcile_tombstone_test.go b/internal/server/store/reconcile_tombstone_test.go new file mode 100644 index 0000000..5af13ef --- /dev/null +++ b/internal/server/store/reconcile_tombstone_test.go @@ -0,0 +1,73 @@ +package store + +import ( + "context" + "crypto/sha256" + "path/filepath" + "testing" + "time" + + rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1" +) + +func TestReconcileTombstonesStillValidateAndInterrupt_REC_RECONCILE_09(t *testing.T) { + for _, scenario := range []string{"missing", "terminal", "running", "hash-mismatch"} { + t.Run(scenario, func(t *testing.T) { + ctx := context.Background() + opened, err := Open(ctx, Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = opened.Close() }) + now := time.Now().UTC() + if _, err := opened.RegisterClientSession(ctx, ClientRegistration{ClientID: "tombstone-client", Platform: 3, Architecture: "amd64", DaemonVersion: "test", DaemonCWD: `C:\`, SupportedShells: []byte{1}, ClientInstanceID: [16]byte{1}, SessionID: [16]byte{2}, ConnectedAt: now}); err != nil { + t.Fatal(err) + } + issue := mustReconcileIssue(t, "019c46f1-1d02-7000-8000-0000000000d2") + hash := sha256.Sum256([]byte("terminal command")) + if scenario != "missing" { + if _, err := opened.QueueCommand(ctx, QueueCommandInput{IssueUUID: issue, ClientID: "tombstone-client", IssueTime: now, ReceiptTime: now, ImmutableSHA256: hash, ExecutionSpec: []byte("opaque")}); err != nil { + t.Fatal(err) + } + phase := rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED + if scenario == "running" { + phase = rvboxv1.CommandLifecycle_COMMAND_RUNNING + } + // Seed only the metadata being reconciled; no execution is involved. + if _, err := opened.DB().ExecContext(ctx, `UPDATE commands SET lifecycle = ?, terminal_time = CASE WHEN ? >= 5 THEN ? ELSE NULL END WHERE issue_uuid = ?`, phase, phase, now.UnixNano(), issue[:]); err != nil { + t.Fatal(err) + } + } + if scenario == "hash-mismatch" { + hash[0] ^= 0xff + } + result, err := opened.ReconcileClientSnapshotForSession(ctx, "tombstone-client", 1, &rvboxv1.ReconcileSnapshot{RetainedCommands: []*rvboxv1.ReconcileCommandState{{ + IssueUuid: issue.String(), CommandRevision: 1, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED, + Tombstoned: true, ImmutableRequestSha256: hash[:], + }}}) + if err != nil { + t.Fatal(err) + } + if len(result.GetDiscardLocalTerminalIssueUuids()) != 0 || len(result.GetTerminateLocalIssueUuids()) != 0 { + t.Fatalf("already compacted tombstone needs no local action: %v", result) + } + var incidents int + if err := opened.DB().QueryRowContext(ctx, `SELECT count(*) FROM storage_incidents`).Scan(&incidents); err != nil { + t.Fatal(err) + } + wantIncidents := 0 + if scenario == "running" || scenario == "hash-mismatch" { + wantIncidents = 1 + } + if incidents != wantIncidents { + t.Fatalf("incidents=%d, want %d", incidents, wantIncidents) + } + if scenario == "running" { + view, err := opened.GetCommandView(ctx, "tombstone-client", issue) + if err != nil || view.Lifecycle != uint32(rvboxv1.CommandLifecycle_COMMAND_INTERRUPTED) { + t.Fatalf("tombstone must still interrupt contradictory running state: %+v, %v", view, err) + } + } + }) + } +} diff --git a/skills/rvbox/references/troubleshooting.md b/skills/rvbox/references/troubleshooting.md index 476d300..243acfe 100644 --- a/skills/rvbox/references/troubleshooting.md +++ b/skills/rvbox/references/troubleshooting.md @@ -56,6 +56,15 @@ servers send scripts in 16 KiB chunks, allowing pending uploads to reach older clients without lowering inbound validation limits. Large inline commands, stdin, and reconciliation responses still need the updated client reader. +A compact tombstone is already discarded command data. Servers with the +tombstone-reconciliation correction validate that evidence but omit redundant +discard instructions for it. Without that correction, a growing tombstone +ledger alone can make the reconnect reply exceed an old client's 32 KiB reader +and cause a persistent reconnect loop. Preserve the ledger; upgrade the client +or apply the separately tested server correction. Genuine large action lists +and other large messages still need a current client reader. This correction +does not enable transport auto-chunking. + Development builds may negotiate `rvbox.agent.v1.chunked` in the WebSocket handshake. When both endpoints support it, messages automatically use bounded 16 KiB packets and are reassembled before protocol handling. Old peers retain diff --git a/test/coverage.toml b/test/coverage.toml index 4dd971d..24fcb13 100644 --- a/test/coverage.toml +++ b/test/coverage.toml @@ -1235,6 +1235,18 @@ layer = "integration" status = "implemented" tests = ["test/integration/clientagent/websocket_limits_integration_test.go:TestLargeScriptFitsLegacyReader_REC_WS_02"] +[[requirements]] +id = "REC-WS-03" +layer = "integration" +status = "implemented" +tests = ["test/integration/clientagent/tombstone_reconcile_integration_test.go:TestLegacyReconnectWithTombstoneHistory_REC_WS_03"] + +[[requirements]] +id = "REC-RECONCILE-09" +layer = "unit" +status = "implemented" +tests = ["internal/server/store/reconcile_tombstone_test.go:TestReconcileTombstonesStillValidateAndInterrupt_REC_RECONCILE_09"] + [[requirements]] id = "REC-RUNTIME-06" layer = "unit" diff --git a/test/integration/clientagent/clientagent_integration_test.go b/test/integration/clientagent/clientagent_integration_test.go index 30c06b3..130d4af 100644 --- a/test/integration/clientagent/clientagent_integration_test.go +++ b/test/integration/clientagent/clientagent_integration_test.go @@ -49,7 +49,7 @@ func TestWebSocketHelloWelcome_HP_SES_06(t *testing.T) { terminalIssue := "019c46f1-1d02-7000-8000-000000000073" reconciled, err := agent.Reconcile(ctx, transport, result, &rvboxv1.ReconcileSnapshot{RetainedCommands: []*rvboxv1.ReconcileCommandState{ {IssueUuid: activeIssue, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_RUNNING, CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32)}, - {IssueUuid: terminalIssue, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED, CommandRevision: 1, Tombstoned: true, ImmutableRequestSha256: make([]byte, 32)}, + {IssueUuid: terminalIssue, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED, CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32)}, }}, agentproto.DefaultLimits()) if err != nil { t.Fatal(err) diff --git a/test/integration/clientagent/tombstone_reconcile_integration_test.go b/test/integration/clientagent/tombstone_reconcile_integration_test.go new file mode 100644 index 0000000..d31d88d --- /dev/null +++ b/test/integration/clientagent/tombstone_reconcile_integration_test.go @@ -0,0 +1,84 @@ +package clientagent_test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/coder/websocket" + rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1" + "github.com/rvbox/rvbox/internal/agentproto" + "github.com/rvbox/rvbox/internal/client/agent" + "github.com/rvbox/rvbox/internal/domain" +) + +func TestLegacyReconnectWithTombstoneHistory_REC_WS_03(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + fixture := newFrameFixture(t) + snapshot := &rvboxv1.ReconcileSnapshot{} + for i := 0; i < 1100; i++ { + issue, err := domain.NewUUIDv7() + if err != nil { + t.Fatal(err) + } + snapshot.RetainedCommands = append(snapshot.RetainedCommands, &rvboxv1.ReconcileCommandState{ + IssueUuid: issue.String(), Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED, + CommandRevision: 1, Tombstoned: true, ImmutableRequestSha256: make([]byte, 32), + }) + } + // A genuinely retained terminal command still needs discard authorization. + terminal, err := domain.NewUUIDv7() + if err != nil { + t.Fatal(err) + } + snapshot.RetainedCommands = append(snapshot.RetainedCommands, &rvboxv1.ReconcileCommandState{ + IssueUuid: terminal.String(), Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED, + CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32), + }) + // Repeat the full snapshot to model an interrupted/lost reconciliation + // response. No subprotocol or updated read-limit wrapper is used. + for attempt := 0; attempt < 2; attempt++ { + raw, _, err := websocket.Dial(ctx, "wss"+strings.TrimPrefix(fixture.server.URL, "https")+"/v1/agent", &websocket.DialOptions{HTTPClient: fixture.server.Client()}) + if err != nil { + t.Fatal(err) + } + raw.SetReadLimit(32 << 10) + transport := tombstoneLegacyTransport{raw} + t.Cleanup(func() { _ = raw.CloseNow() }) + if raw.Subprotocol() != "" { + t.Fatal("legacy client unexpectedly negotiated a subprotocol") + } + accepted, err := agent.Handshake(ctx, transport, fixture.hello, agentproto.DefaultLimits()) + if err != nil { + t.Fatal(err) + } + result, err := agent.Reconcile(ctx, transport, accepted, snapshot, agentproto.DefaultLimits()) + if err != nil { + t.Fatalf("legacy reconnect %d: %v", attempt, err) + } + if got := result.GetDiscardLocalTerminalIssueUuids(); len(got) != 1 || got[0] != terminal.String() || len(result.GetTerminateLocalIssueUuids()) != 0 { + t.Fatalf("unexpected reconciliation instructions: %v", result) + } + if err := transport.Close(); err != nil { + t.Fatal(err) + } + } +} + +type tombstoneLegacyTransport struct{ connection *websocket.Conn } + +func (transport tombstoneLegacyTransport) Read(ctx context.Context) ([]byte, error) { + _, payload, err := transport.connection.Read(ctx) + return payload, err +} + +func (transport tombstoneLegacyTransport) Write(ctx context.Context, payload []byte) error { + return transport.connection.Write(ctx, websocket.MessageBinary, payload) +} + +func (transport tombstoneLegacyTransport) Close() error { + return transport.connection.Close(websocket.StatusNormalClosure, "") +} diff --git a/test/integration/clientagent/websocket_limits_integration_test.go b/test/integration/clientagent/websocket_limits_integration_test.go index 938353f..a9e31cf 100644 --- a/test/integration/clientagent/websocket_limits_integration_test.go +++ b/test/integration/clientagent/websocket_limits_integration_test.go @@ -83,7 +83,7 @@ func TestLargeDispatchAndReconciliation_HP_WS_01(t *testing.T) { } snapshot.RetainedCommands = append(snapshot.RetainedCommands, &rvboxv1.ReconcileCommandState{ IssueUuid: issue.String(), Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED, - CommandRevision: 1, Tombstoned: true, ImmutableRequestSha256: make([]byte, 32), + CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32), }) } transport, _ := fixture.connect(t, ctx, snapshot)