fix(server): omit redundant tombstone reconciliation discards

This commit is contained in:
cabbage committed 2026-11-16 15:18:07 +00:00
1 parent 9e7c409ba7
commit 86a9cfdd72
9 files changed
+227 -2

No files matched your search

+31
View File
@@ -8,6 +8,37 @@ the repository and isolated fixtures. Do not restart/upgrade live services or
replace the published installer for this feature without a later explicit replace the published installer for this feature without a later explicit
instruction lifting that hold. See [the design](docs/transport-chunking.md). instruction lifting that hold. See [the design](docs/transport-chunking.md).
## Legacy Windows client upgrade remains recommended
On 2026-11-16 a legacy client entered a persistent reconnect loop because the
server echoed every already-compacted tombstone as a terminal-discard action.
The redundant UUID list exceeded that client's 32 KiB WebSocket receive limit.
The server correction omits those no-op actions only after the usual immutable
hash/revision checks and reconciliation decision. Retained terminal commands
still receive discard authorization, and contradictory non-terminal server
state is still interrupted and incidented. Tombstones and history are preserved.
The isolated v16 hotfix excludes the held auto-chunking implementation. The
new regression reproduces the original error using TLS, no subprotocol, a real
32 KiB reader and 1,100 tombstones, then verifies repeat reconciliation after the
fix. Store tests cover missing/terminal server rows, hash mismatch, and a
tombstone contradicting running server state. Large genuine action lists,
inline commands and stdin can still exceed the legacy reader; upgrade old
Windows clients to the verified non-chunking installer. This optimization does
not change the remaining 1 MiB logical snapshot limit or add pagination.
Recovery on 2026-11-16 was verified live: server version
`0.1.0-gui-manual.16-tombstone.1`, based on `474afb3` plus only the store
correction; no `internal/agentwire` package is present in that build source.
The active fixture's Compose override pins the binary under
`.test-runs/workstation-disconnect-20261116/`; treat it as a live deployment
artifact, not disposable diagnostics. That directory also holds build
provenance, the preserved pre-upgrade data/config and the guarded recovery script.
Do not delete it while mounted. The existing published v16 installer is
unchanged; its executable was checksum-verified on the user's desktop for
manual upgrade from v11. No Windows service restart or command reissue was
needed for server recovery.
## Reconciliation must restore a lost command acceptance ## Reconciliation must restore a lost command acceptance
A reconnect can leave the server at `COMMAND_DISPATCHED` while the client has A reconnect can leave the server at `COMMAND_DISPATCHED` while the client has
+10
View File
@@ -147,6 +147,16 @@ exact ignored `dist/` directory after preserving any evidence.
the 1 MiB send window, so pending uploads can reach older clients. This does the 1 MiB send window, so pending uploads can reach older clients. This does
not lower the server's inbound limits. Large inline command text, stdin, or not lower the server's inbound limits. Large inline command text, stdin, or
reconciliation responses still require the corrected client reader. reconciliation responses still require the corrected client reader.
- **Reconnect fails after accumulating command history:** the server used to
echo the client's entire compact tombstone ledger as terminal-discard UUIDs.
Roughly 860 UUIDs plus envelope overhead exceed a legacy 32 KiB reader, even
when the commands produced no large output. The correction validates and
reconciles tombstones normally, then omits redundant discard instructions;
it retains history and still authorizes genuinely retained terminal bodies.
This can be backported independently of held auto-chunking. Keep client
spool/tombstones intact, back up the full server state before applying the
isolated fix, and verify both reconnect and a fresh command's terminal result.
Large genuine action lists still require an updated client reader.
- **`terminal command cannot accept script data` during reconnect:** the server - **`terminal command cannot accept script data` during reconnect:** the server
may replay an upload while the client is still delivering that command's may replay an upload while the client is still delivering that command's
terminal events. Corrected clients ignore these frames only after verifying terminal events. Corrected clients ignore these frames only after verifying
+6
View File
@@ -183,7 +183,13 @@ func (store *Store) reconcileClientSnapshotAt(ctx context.Context, clientID stri
incidents = append(incidents, reconcileIncident{issue: parseIssueUnchecked(key), summary: "client and server command state disagree; local process must be terminated", dataLoss: true}) incidents = append(incidents, reconcileIncident{issue: parseIssueUnchecked(key), summary: "client and server command state disagree; local process must be terminated", dataLoss: true})
} }
case domain.ReconcileDiscardLocalTerminal: case domain.ReconcileDiscardLocalTerminal:
// Tombstones already have no command body or outbox to discard.
// Keep validating their hash/revision and reconciling contradictions
// above, but do not echo the growing compact ledger on every reconnect.
// The redundant UUID list alone can exceed a legacy peer's read limit.
if !client.GetTombstoned() {
result.DiscardLocalTerminalIssueUuids = append(result.DiscardLocalTerminalIssueUuids, key) result.DiscardLocalTerminalIssueUuids = append(result.DiscardLocalTerminalIssueUuids, key)
}
case domain.ReconcileResumeDelivery: case domain.ReconcileResumeDelivery:
if generation != 0 && row.target.Valid && uint64(row.target.Int64) != generation || generation != 0 && !row.target.Valid { if generation != 0 && row.target.Valid && uint64(row.target.Int64) != generation || generation != 0 && !row.target.Valid {
if _, err := tx.ExecContext(ctx, `UPDATE commands SET target_session_generation = ? WHERE issue_uuid = ? AND client_id = ? AND lifecycle BETWEEN 2 AND 4`, generation, row.issue[:], clientID); err != nil { if _, err := tx.ExecContext(ctx, `UPDATE commands SET target_session_generation = ? WHERE issue_uuid = ? AND client_id = ? AND lifecycle BETWEEN 2 AND 4`, generation, row.issue[:], clientID); err != nil {
@@ -0,0 +1,73 @@
package store
import (
"context"
"crypto/sha256"
"path/filepath"
"testing"
"time"
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
)
func TestReconcileTombstonesStillValidateAndInterrupt_REC_RECONCILE_09(t *testing.T) {
for _, scenario := range []string{"missing", "terminal", "running", "hash-mismatch"} {
t.Run(scenario, func(t *testing.T) {
ctx := context.Background()
opened, err := Open(ctx, Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = opened.Close() })
now := time.Now().UTC()
if _, err := opened.RegisterClientSession(ctx, ClientRegistration{ClientID: "tombstone-client", Platform: 3, Architecture: "amd64", DaemonVersion: "test", DaemonCWD: `C:\`, SupportedShells: []byte{1}, ClientInstanceID: [16]byte{1}, SessionID: [16]byte{2}, ConnectedAt: now}); err != nil {
t.Fatal(err)
}
issue := mustReconcileIssue(t, "019c46f1-1d02-7000-8000-0000000000d2")
hash := sha256.Sum256([]byte("terminal command"))
if scenario != "missing" {
if _, err := opened.QueueCommand(ctx, QueueCommandInput{IssueUUID: issue, ClientID: "tombstone-client", IssueTime: now, ReceiptTime: now, ImmutableSHA256: hash, ExecutionSpec: []byte("opaque")}); err != nil {
t.Fatal(err)
}
phase := rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED
if scenario == "running" {
phase = rvboxv1.CommandLifecycle_COMMAND_RUNNING
}
// Seed only the metadata being reconciled; no execution is involved.
if _, err := opened.DB().ExecContext(ctx, `UPDATE commands SET lifecycle = ?, terminal_time = CASE WHEN ? >= 5 THEN ? ELSE NULL END WHERE issue_uuid = ?`, phase, phase, now.UnixNano(), issue[:]); err != nil {
t.Fatal(err)
}
}
if scenario == "hash-mismatch" {
hash[0] ^= 0xff
}
result, err := opened.ReconcileClientSnapshotForSession(ctx, "tombstone-client", 1, &rvboxv1.ReconcileSnapshot{RetainedCommands: []*rvboxv1.ReconcileCommandState{{
IssueUuid: issue.String(), CommandRevision: 1, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
Tombstoned: true, ImmutableRequestSha256: hash[:],
}}})
if err != nil {
t.Fatal(err)
}
if len(result.GetDiscardLocalTerminalIssueUuids()) != 0 || len(result.GetTerminateLocalIssueUuids()) != 0 {
t.Fatalf("already compacted tombstone needs no local action: %v", result)
}
var incidents int
if err := opened.DB().QueryRowContext(ctx, `SELECT count(*) FROM storage_incidents`).Scan(&incidents); err != nil {
t.Fatal(err)
}
wantIncidents := 0
if scenario == "running" || scenario == "hash-mismatch" {
wantIncidents = 1
}
if incidents != wantIncidents {
t.Fatalf("incidents=%d, want %d", incidents, wantIncidents)
}
if scenario == "running" {
view, err := opened.GetCommandView(ctx, "tombstone-client", issue)
if err != nil || view.Lifecycle != uint32(rvboxv1.CommandLifecycle_COMMAND_INTERRUPTED) {
t.Fatalf("tombstone must still interrupt contradictory running state: %+v, %v", view, err)
}
}
})
}
}
@@ -56,6 +56,15 @@ servers send scripts in 16 KiB chunks, allowing pending uploads to reach older
clients without lowering inbound validation limits. Large inline commands, clients without lowering inbound validation limits. Large inline commands,
stdin, and reconciliation responses still need the updated client reader. stdin, and reconciliation responses still need the updated client reader.
A compact tombstone is already discarded command data. Servers with the
tombstone-reconciliation correction validate that evidence but omit redundant
discard instructions for it. Without that correction, a growing tombstone
ledger alone can make the reconnect reply exceed an old client's 32 KiB reader
and cause a persistent reconnect loop. Preserve the ledger; upgrade the client
or apply the separately tested server correction. Genuine large action lists
and other large messages still need a current client reader. This correction
does not enable transport auto-chunking.
Development builds may negotiate `rvbox.agent.v1.chunked` in the WebSocket Development builds may negotiate `rvbox.agent.v1.chunked` in the WebSocket
handshake. When both endpoints support it, messages automatically use bounded handshake. When both endpoints support it, messages automatically use bounded
16 KiB packets and are reassembled before protocol handling. Old peers retain 16 KiB packets and are reassembled before protocol handling. Old peers retain
+12
View File
@@ -1235,6 +1235,18 @@ layer = "integration"
status = "implemented" status = "implemented"
tests = ["test/integration/clientagent/websocket_limits_integration_test.go:TestLargeScriptFitsLegacyReader_REC_WS_02"] tests = ["test/integration/clientagent/websocket_limits_integration_test.go:TestLargeScriptFitsLegacyReader_REC_WS_02"]
[[requirements]]
id = "REC-WS-03"
layer = "integration"
status = "implemented"
tests = ["test/integration/clientagent/tombstone_reconcile_integration_test.go:TestLegacyReconnectWithTombstoneHistory_REC_WS_03"]
[[requirements]]
id = "REC-RECONCILE-09"
layer = "unit"
status = "implemented"
tests = ["internal/server/store/reconcile_tombstone_test.go:TestReconcileTombstonesStillValidateAndInterrupt_REC_RECONCILE_09"]
[[requirements]] [[requirements]]
id = "REC-RUNTIME-06" id = "REC-RUNTIME-06"
layer = "unit" layer = "unit"
@@ -49,7 +49,7 @@ func TestWebSocketHelloWelcome_HP_SES_06(t *testing.T) {
terminalIssue := "019c46f1-1d02-7000-8000-000000000073" terminalIssue := "019c46f1-1d02-7000-8000-000000000073"
reconciled, err := agent.Reconcile(ctx, transport, result, &rvboxv1.ReconcileSnapshot{RetainedCommands: []*rvboxv1.ReconcileCommandState{ reconciled, err := agent.Reconcile(ctx, transport, result, &rvboxv1.ReconcileSnapshot{RetainedCommands: []*rvboxv1.ReconcileCommandState{
{IssueUuid: activeIssue, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_RUNNING, CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32)}, {IssueUuid: activeIssue, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_RUNNING, CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32)},
{IssueUuid: terminalIssue, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED, CommandRevision: 1, Tombstoned: true, ImmutableRequestSha256: make([]byte, 32)}, {IssueUuid: terminalIssue, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED, CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32)},
}}, agentproto.DefaultLimits()) }}, agentproto.DefaultLimits())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -0,0 +1,84 @@
package clientagent_test
import (
"context"
"strings"
"testing"
"time"
"github.com/coder/websocket"
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/client/agent"
"github.com/rvbox/rvbox/internal/domain"
)
func TestLegacyReconnectWithTombstoneHistory_REC_WS_03(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
fixture := newFrameFixture(t)
snapshot := &rvboxv1.ReconcileSnapshot{}
for i := 0; i < 1100; i++ {
issue, err := domain.NewUUIDv7()
if err != nil {
t.Fatal(err)
}
snapshot.RetainedCommands = append(snapshot.RetainedCommands, &rvboxv1.ReconcileCommandState{
IssueUuid: issue.String(), Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
CommandRevision: 1, Tombstoned: true, ImmutableRequestSha256: make([]byte, 32),
})
}
// A genuinely retained terminal command still needs discard authorization.
terminal, err := domain.NewUUIDv7()
if err != nil {
t.Fatal(err)
}
snapshot.RetainedCommands = append(snapshot.RetainedCommands, &rvboxv1.ReconcileCommandState{
IssueUuid: terminal.String(), Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32),
})
// Repeat the full snapshot to model an interrupted/lost reconciliation
// response. No subprotocol or updated read-limit wrapper is used.
for attempt := 0; attempt < 2; attempt++ {
raw, _, err := websocket.Dial(ctx, "wss"+strings.TrimPrefix(fixture.server.URL, "https")+"/v1/agent", &websocket.DialOptions{HTTPClient: fixture.server.Client()})
if err != nil {
t.Fatal(err)
}
raw.SetReadLimit(32 << 10)
transport := tombstoneLegacyTransport{raw}
t.Cleanup(func() { _ = raw.CloseNow() })
if raw.Subprotocol() != "" {
t.Fatal("legacy client unexpectedly negotiated a subprotocol")
}
accepted, err := agent.Handshake(ctx, transport, fixture.hello, agentproto.DefaultLimits())
if err != nil {
t.Fatal(err)
}
result, err := agent.Reconcile(ctx, transport, accepted, snapshot, agentproto.DefaultLimits())
if err != nil {
t.Fatalf("legacy reconnect %d: %v", attempt, err)
}
if got := result.GetDiscardLocalTerminalIssueUuids(); len(got) != 1 || got[0] != terminal.String() || len(result.GetTerminateLocalIssueUuids()) != 0 {
t.Fatalf("unexpected reconciliation instructions: %v", result)
}
if err := transport.Close(); err != nil {
t.Fatal(err)
}
}
}
type tombstoneLegacyTransport struct{ connection *websocket.Conn }
func (transport tombstoneLegacyTransport) Read(ctx context.Context) ([]byte, error) {
_, payload, err := transport.connection.Read(ctx)
return payload, err
}
func (transport tombstoneLegacyTransport) Write(ctx context.Context, payload []byte) error {
return transport.connection.Write(ctx, websocket.MessageBinary, payload)
}
func (transport tombstoneLegacyTransport) Close() error {
return transport.connection.Close(websocket.StatusNormalClosure, "")
}
@@ -83,7 +83,7 @@ func TestLargeDispatchAndReconciliation_HP_WS_01(t *testing.T) {
} }
snapshot.RetainedCommands = append(snapshot.RetainedCommands, &rvboxv1.ReconcileCommandState{ snapshot.RetainedCommands = append(snapshot.RetainedCommands, &rvboxv1.ReconcileCommandState{
IssueUuid: issue.String(), Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED, IssueUuid: issue.String(), Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
CommandRevision: 1, Tombstoned: true, ImmutableRequestSha256: make([]byte, 32), CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32),
}) })
} }
transport, _ := fixture.connect(t, ctx, snapshot) transport, _ := fixture.connect(t, ctx, snapshot)