fix(server): omit redundant tombstone reconciliation discards
This commit is contained in:
1 parent
9e7c409ba7
commit
86a9cfdd72
9 files changed
+228
-3
No files matched your search
@@ -8,6 +8,37 @@ the repository and isolated fixtures. Do not restart/upgrade live services or
|
||||
replace the published installer for this feature without a later explicit
|
||||
instruction lifting that hold. See [the design](docs/transport-chunking.md).
|
||||
|
||||
## Legacy Windows client upgrade remains recommended
|
||||
|
||||
On 2026-11-16 a legacy client entered a persistent reconnect loop because the
|
||||
server echoed every already-compacted tombstone as a terminal-discard action.
|
||||
The redundant UUID list exceeded that client's 32 KiB WebSocket receive limit.
|
||||
The server correction omits those no-op actions only after the usual immutable
|
||||
hash/revision checks and reconciliation decision. Retained terminal commands
|
||||
still receive discard authorization, and contradictory non-terminal server
|
||||
state is still interrupted and incidented. Tombstones and history are preserved.
|
||||
|
||||
The isolated v16 hotfix excludes the held auto-chunking implementation. The
|
||||
new regression reproduces the original error using TLS, no subprotocol, a real
|
||||
32 KiB reader and 1,100 tombstones, then verifies repeat reconciliation after the
|
||||
fix. Store tests cover missing/terminal server rows, hash mismatch, and a
|
||||
tombstone contradicting running server state. Large genuine action lists,
|
||||
inline commands and stdin can still exceed the legacy reader; upgrade old
|
||||
Windows clients to the verified non-chunking installer. This optimization does
|
||||
not change the remaining 1 MiB logical snapshot limit or add pagination.
|
||||
|
||||
Recovery on 2026-11-16 was verified live: server version
|
||||
`0.1.0-gui-manual.16-tombstone.1`, based on `474afb3` plus only the store
|
||||
correction; no `internal/agentwire` package is present in that build source.
|
||||
The active fixture's Compose override pins the binary under
|
||||
`.test-runs/workstation-disconnect-20261116/`; treat it as a live deployment
|
||||
artifact, not disposable diagnostics. That directory also holds build
|
||||
provenance, the preserved pre-upgrade data/config and the guarded recovery script.
|
||||
Do not delete it while mounted. The existing published v16 installer is
|
||||
unchanged; its executable was checksum-verified on the user's desktop for
|
||||
manual upgrade from v11. No Windows service restart or command reissue was
|
||||
needed for server recovery.
|
||||
|
||||
## Reconciliation must restore a lost command acceptance
|
||||
|
||||
A reconnect can leave the server at `COMMAND_DISPATCHED` while the client has
|
||||
|
||||
@@ -147,6 +147,16 @@ exact ignored `dist/` directory after preserving any evidence.
|
||||
the 1 MiB send window, so pending uploads can reach older clients. This does
|
||||
not lower the server's inbound limits. Large inline command text, stdin, or
|
||||
reconciliation responses still require the corrected client reader.
|
||||
- **Reconnect fails after accumulating command history:** the server used to
|
||||
echo the client's entire compact tombstone ledger as terminal-discard UUIDs.
|
||||
Roughly 860 UUIDs plus envelope overhead exceed a legacy 32 KiB reader, even
|
||||
when the commands produced no large output. The correction validates and
|
||||
reconciles tombstones normally, then omits redundant discard instructions;
|
||||
it retains history and still authorizes genuinely retained terminal bodies.
|
||||
This can be backported independently of held auto-chunking. Keep client
|
||||
spool/tombstones intact, back up the full server state before applying the
|
||||
isolated fix, and verify both reconnect and a fresh command's terminal result.
|
||||
Large genuine action lists still require an updated client reader.
|
||||
- **`terminal command cannot accept script data` during reconnect:** the server
|
||||
may replay an upload while the client is still delivering that command's
|
||||
terminal events. Corrected clients ignore these frames only after verifying
|
||||
|
||||
@@ -183,7 +183,13 @@ func (store *Store) reconcileClientSnapshotAt(ctx context.Context, clientID stri
|
||||
incidents = append(incidents, reconcileIncident{issue: parseIssueUnchecked(key), summary: "client and server command state disagree; local process must be terminated", dataLoss: true})
|
||||
}
|
||||
case domain.ReconcileDiscardLocalTerminal:
|
||||
result.DiscardLocalTerminalIssueUuids = append(result.DiscardLocalTerminalIssueUuids, key)
|
||||
// Tombstones already have no command body or outbox to discard.
|
||||
// Keep validating their hash/revision and reconciling contradictions
|
||||
// above, but do not echo the growing compact ledger on every reconnect.
|
||||
// The redundant UUID list alone can exceed a legacy peer's read limit.
|
||||
if !client.GetTombstoned() {
|
||||
result.DiscardLocalTerminalIssueUuids = append(result.DiscardLocalTerminalIssueUuids, key)
|
||||
}
|
||||
case domain.ReconcileResumeDelivery:
|
||||
if generation != 0 && row.target.Valid && uint64(row.target.Int64) != generation || generation != 0 && !row.target.Valid {
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE commands SET target_session_generation = ? WHERE issue_uuid = ? AND client_id = ? AND lifecycle BETWEEN 2 AND 4`, generation, row.issue[:], clientID); err != nil {
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||
)
|
||||
|
||||
func TestReconcileTombstonesStillValidateAndInterrupt_REC_RECONCILE_09(t *testing.T) {
|
||||
for _, scenario := range []string{"missing", "terminal", "running", "hash-mismatch"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
opened, err := Open(ctx, Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = opened.Close() })
|
||||
now := time.Now().UTC()
|
||||
if _, err := opened.RegisterClientSession(ctx, ClientRegistration{ClientID: "tombstone-client", Platform: 3, Architecture: "amd64", DaemonVersion: "test", DaemonCWD: `C:\`, SupportedShells: []byte{1}, ClientInstanceID: [16]byte{1}, SessionID: [16]byte{2}, ConnectedAt: now}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
issue := mustReconcileIssue(t, "019c46f1-1d02-7000-8000-0000000000d2")
|
||||
hash := sha256.Sum256([]byte("terminal command"))
|
||||
if scenario != "missing" {
|
||||
if _, err := opened.QueueCommand(ctx, QueueCommandInput{IssueUUID: issue, ClientID: "tombstone-client", IssueTime: now, ReceiptTime: now, ImmutableSHA256: hash, ExecutionSpec: []byte("opaque")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
phase := rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED
|
||||
if scenario == "running" {
|
||||
phase = rvboxv1.CommandLifecycle_COMMAND_RUNNING
|
||||
}
|
||||
// Seed only the metadata being reconciled; no execution is involved.
|
||||
if _, err := opened.DB().ExecContext(ctx, `UPDATE commands SET lifecycle = ?, terminal_time = CASE WHEN ? >= 5 THEN ? ELSE NULL END WHERE issue_uuid = ?`, phase, phase, now.UnixNano(), issue[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if scenario == "hash-mismatch" {
|
||||
hash[0] ^= 0xff
|
||||
}
|
||||
result, err := opened.ReconcileClientSnapshotForSession(ctx, "tombstone-client", 1, &rvboxv1.ReconcileSnapshot{RetainedCommands: []*rvboxv1.ReconcileCommandState{{
|
||||
IssueUuid: issue.String(), CommandRevision: 1, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
|
||||
Tombstoned: true, ImmutableRequestSha256: hash[:],
|
||||
}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.GetDiscardLocalTerminalIssueUuids()) != 0 || len(result.GetTerminateLocalIssueUuids()) != 0 {
|
||||
t.Fatalf("already compacted tombstone needs no local action: %v", result)
|
||||
}
|
||||
var incidents int
|
||||
if err := opened.DB().QueryRowContext(ctx, `SELECT count(*) FROM storage_incidents`).Scan(&incidents); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantIncidents := 0
|
||||
if scenario == "running" || scenario == "hash-mismatch" {
|
||||
wantIncidents = 1
|
||||
}
|
||||
if incidents != wantIncidents {
|
||||
t.Fatalf("incidents=%d, want %d", incidents, wantIncidents)
|
||||
}
|
||||
if scenario == "running" {
|
||||
view, err := opened.GetCommandView(ctx, "tombstone-client", issue)
|
||||
if err != nil || view.Lifecycle != uint32(rvboxv1.CommandLifecycle_COMMAND_INTERRUPTED) {
|
||||
t.Fatalf("tombstone must still interrupt contradictory running state: %+v, %v", view, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,15 @@ servers send scripts in 16 KiB chunks, allowing pending uploads to reach older
|
||||
clients without lowering inbound validation limits. Large inline commands,
|
||||
stdin, and reconciliation responses still need the updated client reader.
|
||||
|
||||
A compact tombstone is already discarded command data. Servers with the
|
||||
tombstone-reconciliation correction validate that evidence but omit redundant
|
||||
discard instructions for it. Without that correction, a growing tombstone
|
||||
ledger alone can make the reconnect reply exceed an old client's 32 KiB reader
|
||||
and cause a persistent reconnect loop. Preserve the ledger; upgrade the client
|
||||
or apply the separately tested server correction. Genuine large action lists
|
||||
and other large messages still need a current client reader. This correction
|
||||
does not enable transport auto-chunking.
|
||||
|
||||
Development builds may negotiate `rvbox.agent.v1.chunked` in the WebSocket
|
||||
handshake. When both endpoints support it, messages automatically use bounded
|
||||
16 KiB packets and are reassembled before protocol handling. Old peers retain
|
||||
|
||||
@@ -1235,6 +1235,18 @@ layer = "integration"
|
||||
status = "implemented"
|
||||
tests = ["test/integration/clientagent/websocket_limits_integration_test.go:TestLargeScriptFitsLegacyReader_REC_WS_02"]
|
||||
|
||||
[[requirements]]
|
||||
id = "REC-WS-03"
|
||||
layer = "integration"
|
||||
status = "implemented"
|
||||
tests = ["test/integration/clientagent/tombstone_reconcile_integration_test.go:TestLegacyReconnectWithTombstoneHistory_REC_WS_03"]
|
||||
|
||||
[[requirements]]
|
||||
id = "REC-RECONCILE-09"
|
||||
layer = "unit"
|
||||
status = "implemented"
|
||||
tests = ["internal/server/store/reconcile_tombstone_test.go:TestReconcileTombstonesStillValidateAndInterrupt_REC_RECONCILE_09"]
|
||||
|
||||
[[requirements]]
|
||||
id = "REC-RUNTIME-06"
|
||||
layer = "unit"
|
||||
|
||||
@@ -49,7 +49,7 @@ func TestWebSocketHelloWelcome_HP_SES_06(t *testing.T) {
|
||||
terminalIssue := "019c46f1-1d02-7000-8000-000000000073"
|
||||
reconciled, err := agent.Reconcile(ctx, transport, result, &rvboxv1.ReconcileSnapshot{RetainedCommands: []*rvboxv1.ReconcileCommandState{
|
||||
{IssueUuid: activeIssue, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_RUNNING, CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32)},
|
||||
{IssueUuid: terminalIssue, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED, CommandRevision: 1, Tombstoned: true, ImmutableRequestSha256: make([]byte, 32)},
|
||||
{IssueUuid: terminalIssue, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED, CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32)},
|
||||
}}, agentproto.DefaultLimits())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
package clientagent_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||
"github.com/rvbox/rvbox/internal/agentproto"
|
||||
"github.com/rvbox/rvbox/internal/client/agent"
|
||||
"github.com/rvbox/rvbox/internal/domain"
|
||||
)
|
||||
|
||||
func TestLegacyReconnectWithTombstoneHistory_REC_WS_03(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
fixture := newFrameFixture(t)
|
||||
snapshot := &rvboxv1.ReconcileSnapshot{}
|
||||
for i := 0; i < 1100; i++ {
|
||||
issue, err := domain.NewUUIDv7()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snapshot.RetainedCommands = append(snapshot.RetainedCommands, &rvboxv1.ReconcileCommandState{
|
||||
IssueUuid: issue.String(), Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
|
||||
CommandRevision: 1, Tombstoned: true, ImmutableRequestSha256: make([]byte, 32),
|
||||
})
|
||||
}
|
||||
// A genuinely retained terminal command still needs discard authorization.
|
||||
terminal, err := domain.NewUUIDv7()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snapshot.RetainedCommands = append(snapshot.RetainedCommands, &rvboxv1.ReconcileCommandState{
|
||||
IssueUuid: terminal.String(), Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
|
||||
CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32),
|
||||
})
|
||||
// Repeat the full snapshot to model an interrupted/lost reconciliation
|
||||
// response. No subprotocol or updated read-limit wrapper is used.
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
raw, _, err := websocket.Dial(ctx, "wss"+strings.TrimPrefix(fixture.server.URL, "https")+"/v1/agent", &websocket.DialOptions{HTTPClient: fixture.server.Client()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw.SetReadLimit(32 << 10)
|
||||
transport := tombstoneLegacyTransport{raw}
|
||||
t.Cleanup(func() { _ = raw.CloseNow() })
|
||||
if raw.Subprotocol() != "" {
|
||||
t.Fatal("legacy client unexpectedly negotiated a subprotocol")
|
||||
}
|
||||
accepted, err := agent.Handshake(ctx, transport, fixture.hello, agentproto.DefaultLimits())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := agent.Reconcile(ctx, transport, accepted, snapshot, agentproto.DefaultLimits())
|
||||
if err != nil {
|
||||
t.Fatalf("legacy reconnect %d: %v", attempt, err)
|
||||
}
|
||||
if got := result.GetDiscardLocalTerminalIssueUuids(); len(got) != 1 || got[0] != terminal.String() || len(result.GetTerminateLocalIssueUuids()) != 0 {
|
||||
t.Fatalf("unexpected reconciliation instructions: %v", result)
|
||||
}
|
||||
if err := transport.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type tombstoneLegacyTransport struct{ connection *websocket.Conn }
|
||||
|
||||
func (transport tombstoneLegacyTransport) Read(ctx context.Context) ([]byte, error) {
|
||||
_, payload, err := transport.connection.Read(ctx)
|
||||
return payload, err
|
||||
}
|
||||
|
||||
func (transport tombstoneLegacyTransport) Write(ctx context.Context, payload []byte) error {
|
||||
return transport.connection.Write(ctx, websocket.MessageBinary, payload)
|
||||
}
|
||||
|
||||
func (transport tombstoneLegacyTransport) Close() error {
|
||||
return transport.connection.Close(websocket.StatusNormalClosure, "")
|
||||
}
|
||||
@@ -83,7 +83,7 @@ func TestLargeDispatchAndReconciliation_HP_WS_01(t *testing.T) {
|
||||
}
|
||||
snapshot.RetainedCommands = append(snapshot.RetainedCommands, &rvboxv1.ReconcileCommandState{
|
||||
IssueUuid: issue.String(), Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
|
||||
CommandRevision: 1, Tombstoned: true, ImmutableRequestSha256: make([]byte, 32),
|
||||
CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32),
|
||||
})
|
||||
}
|
||||
transport, _ := fixture.connect(t, ctx, snapshot)
|
||||
|
||||
Reference in new issue
Block a user