fix(server): omit redundant tombstone reconciliation discards
This commit is contained in:
1 parent
9e7c409ba7
commit
86a9cfdd72
9 files changed
+227
-2
No files matched your search
@@ -8,6 +8,37 @@ the repository and isolated fixtures. Do not restart/upgrade live services or
|
|||||||
replace the published installer for this feature without a later explicit
|
replace the published installer for this feature without a later explicit
|
||||||
instruction lifting that hold. See [the design](docs/transport-chunking.md).
|
instruction lifting that hold. See [the design](docs/transport-chunking.md).
|
||||||
|
|
||||||
|
## Legacy Windows client upgrade remains recommended
|
||||||
|
|
||||||
|
On 2026-11-16 a legacy client entered a persistent reconnect loop because the
|
||||||
|
server echoed every already-compacted tombstone as a terminal-discard action.
|
||||||
|
The redundant UUID list exceeded that client's 32 KiB WebSocket receive limit.
|
||||||
|
The server correction omits those no-op actions only after the usual immutable
|
||||||
|
hash/revision checks and reconciliation decision. Retained terminal commands
|
||||||
|
still receive discard authorization, and contradictory non-terminal server
|
||||||
|
state is still interrupted and incidented. Tombstones and history are preserved.
|
||||||
|
|
||||||
|
The isolated v16 hotfix excludes the held auto-chunking implementation. The
|
||||||
|
new regression reproduces the original error using TLS, no subprotocol, a real
|
||||||
|
32 KiB reader and 1,100 tombstones, then verifies repeat reconciliation after the
|
||||||
|
fix. Store tests cover missing/terminal server rows, hash mismatch, and a
|
||||||
|
tombstone contradicting running server state. Large genuine action lists,
|
||||||
|
inline commands and stdin can still exceed the legacy reader; upgrade old
|
||||||
|
Windows clients to the verified non-chunking installer. This optimization does
|
||||||
|
not change the remaining 1 MiB logical snapshot limit or add pagination.
|
||||||
|
|
||||||
|
Recovery on 2026-11-16 was verified live: server version
|
||||||
|
`0.1.0-gui-manual.16-tombstone.1`, based on `474afb3` plus only the store
|
||||||
|
correction; no `internal/agentwire` package is present in that build source.
|
||||||
|
The active fixture's Compose override pins the binary under
|
||||||
|
`.test-runs/workstation-disconnect-20261116/`; treat it as a live deployment
|
||||||
|
artifact, not disposable diagnostics. That directory also holds build
|
||||||
|
provenance, the preserved pre-upgrade data/config and the guarded recovery script.
|
||||||
|
Do not delete it while mounted. The existing published v16 installer is
|
||||||
|
unchanged; its executable was checksum-verified on the user's desktop for
|
||||||
|
manual upgrade from v11. No Windows service restart or command reissue was
|
||||||
|
needed for server recovery.
|
||||||
|
|
||||||
## Reconciliation must restore a lost command acceptance
|
## Reconciliation must restore a lost command acceptance
|
||||||
|
|
||||||
A reconnect can leave the server at `COMMAND_DISPATCHED` while the client has
|
A reconnect can leave the server at `COMMAND_DISPATCHED` while the client has
|
||||||
|
|||||||
@@ -147,6 +147,16 @@ exact ignored `dist/` directory after preserving any evidence.
|
|||||||
the 1 MiB send window, so pending uploads can reach older clients. This does
|
the 1 MiB send window, so pending uploads can reach older clients. This does
|
||||||
not lower the server's inbound limits. Large inline command text, stdin, or
|
not lower the server's inbound limits. Large inline command text, stdin, or
|
||||||
reconciliation responses still require the corrected client reader.
|
reconciliation responses still require the corrected client reader.
|
||||||
|
- **Reconnect fails after accumulating command history:** the server used to
|
||||||
|
echo the client's entire compact tombstone ledger as terminal-discard UUIDs.
|
||||||
|
Roughly 860 UUIDs plus envelope overhead exceed a legacy 32 KiB reader, even
|
||||||
|
when the commands produced no large output. The correction validates and
|
||||||
|
reconciles tombstones normally, then omits redundant discard instructions;
|
||||||
|
it retains history and still authorizes genuinely retained terminal bodies.
|
||||||
|
This can be backported independently of held auto-chunking. Keep client
|
||||||
|
spool/tombstones intact, back up the full server state before applying the
|
||||||
|
isolated fix, and verify both reconnect and a fresh command's terminal result.
|
||||||
|
Large genuine action lists still require an updated client reader.
|
||||||
- **`terminal command cannot accept script data` during reconnect:** the server
|
- **`terminal command cannot accept script data` during reconnect:** the server
|
||||||
may replay an upload while the client is still delivering that command's
|
may replay an upload while the client is still delivering that command's
|
||||||
terminal events. Corrected clients ignore these frames only after verifying
|
terminal events. Corrected clients ignore these frames only after verifying
|
||||||
|
|||||||
@@ -183,7 +183,13 @@ func (store *Store) reconcileClientSnapshotAt(ctx context.Context, clientID stri
|
|||||||
incidents = append(incidents, reconcileIncident{issue: parseIssueUnchecked(key), summary: "client and server command state disagree; local process must be terminated", dataLoss: true})
|
incidents = append(incidents, reconcileIncident{issue: parseIssueUnchecked(key), summary: "client and server command state disagree; local process must be terminated", dataLoss: true})
|
||||||
}
|
}
|
||||||
case domain.ReconcileDiscardLocalTerminal:
|
case domain.ReconcileDiscardLocalTerminal:
|
||||||
|
// Tombstones already have no command body or outbox to discard.
|
||||||
|
// Keep validating their hash/revision and reconciling contradictions
|
||||||
|
// above, but do not echo the growing compact ledger on every reconnect.
|
||||||
|
// The redundant UUID list alone can exceed a legacy peer's read limit.
|
||||||
|
if !client.GetTombstoned() {
|
||||||
result.DiscardLocalTerminalIssueUuids = append(result.DiscardLocalTerminalIssueUuids, key)
|
result.DiscardLocalTerminalIssueUuids = append(result.DiscardLocalTerminalIssueUuids, key)
|
||||||
|
}
|
||||||
case domain.ReconcileResumeDelivery:
|
case domain.ReconcileResumeDelivery:
|
||||||
if generation != 0 && row.target.Valid && uint64(row.target.Int64) != generation || generation != 0 && !row.target.Valid {
|
if generation != 0 && row.target.Valid && uint64(row.target.Int64) != generation || generation != 0 && !row.target.Valid {
|
||||||
if _, err := tx.ExecContext(ctx, `UPDATE commands SET target_session_generation = ? WHERE issue_uuid = ? AND client_id = ? AND lifecycle BETWEEN 2 AND 4`, generation, row.issue[:], clientID); err != nil {
|
if _, err := tx.ExecContext(ctx, `UPDATE commands SET target_session_generation = ? WHERE issue_uuid = ? AND client_id = ? AND lifecycle BETWEEN 2 AND 4`, generation, row.issue[:], clientID); err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestReconcileTombstonesStillValidateAndInterrupt_REC_RECONCILE_09(t *testing.T) {
|
||||||
|
for _, scenario := range []string{"missing", "terminal", "running", "hash-mismatch"} {
|
||||||
|
t.Run(scenario, func(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
opened, err := Open(ctx, Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = opened.Close() })
|
||||||
|
now := time.Now().UTC()
|
||||||
|
if _, err := opened.RegisterClientSession(ctx, ClientRegistration{ClientID: "tombstone-client", Platform: 3, Architecture: "amd64", DaemonVersion: "test", DaemonCWD: `C:\`, SupportedShells: []byte{1}, ClientInstanceID: [16]byte{1}, SessionID: [16]byte{2}, ConnectedAt: now}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
issue := mustReconcileIssue(t, "019c46f1-1d02-7000-8000-0000000000d2")
|
||||||
|
hash := sha256.Sum256([]byte("terminal command"))
|
||||||
|
if scenario != "missing" {
|
||||||
|
if _, err := opened.QueueCommand(ctx, QueueCommandInput{IssueUUID: issue, ClientID: "tombstone-client", IssueTime: now, ReceiptTime: now, ImmutableSHA256: hash, ExecutionSpec: []byte("opaque")}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
phase := rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED
|
||||||
|
if scenario == "running" {
|
||||||
|
phase = rvboxv1.CommandLifecycle_COMMAND_RUNNING
|
||||||
|
}
|
||||||
|
// Seed only the metadata being reconciled; no execution is involved.
|
||||||
|
if _, err := opened.DB().ExecContext(ctx, `UPDATE commands SET lifecycle = ?, terminal_time = CASE WHEN ? >= 5 THEN ? ELSE NULL END WHERE issue_uuid = ?`, phase, phase, now.UnixNano(), issue[:]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if scenario == "hash-mismatch" {
|
||||||
|
hash[0] ^= 0xff
|
||||||
|
}
|
||||||
|
result, err := opened.ReconcileClientSnapshotForSession(ctx, "tombstone-client", 1, &rvboxv1.ReconcileSnapshot{RetainedCommands: []*rvboxv1.ReconcileCommandState{{
|
||||||
|
IssueUuid: issue.String(), CommandRevision: 1, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
|
||||||
|
Tombstoned: true, ImmutableRequestSha256: hash[:],
|
||||||
|
}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(result.GetDiscardLocalTerminalIssueUuids()) != 0 || len(result.GetTerminateLocalIssueUuids()) != 0 {
|
||||||
|
t.Fatalf("already compacted tombstone needs no local action: %v", result)
|
||||||
|
}
|
||||||
|
var incidents int
|
||||||
|
if err := opened.DB().QueryRowContext(ctx, `SELECT count(*) FROM storage_incidents`).Scan(&incidents); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantIncidents := 0
|
||||||
|
if scenario == "running" || scenario == "hash-mismatch" {
|
||||||
|
wantIncidents = 1
|
||||||
|
}
|
||||||
|
if incidents != wantIncidents {
|
||||||
|
t.Fatalf("incidents=%d, want %d", incidents, wantIncidents)
|
||||||
|
}
|
||||||
|
if scenario == "running" {
|
||||||
|
view, err := opened.GetCommandView(ctx, "tombstone-client", issue)
|
||||||
|
if err != nil || view.Lifecycle != uint32(rvboxv1.CommandLifecycle_COMMAND_INTERRUPTED) {
|
||||||
|
t.Fatalf("tombstone must still interrupt contradictory running state: %+v, %v", view, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -56,6 +56,15 @@ servers send scripts in 16 KiB chunks, allowing pending uploads to reach older
|
|||||||
clients without lowering inbound validation limits. Large inline commands,
|
clients without lowering inbound validation limits. Large inline commands,
|
||||||
stdin, and reconciliation responses still need the updated client reader.
|
stdin, and reconciliation responses still need the updated client reader.
|
||||||
|
|
||||||
|
A compact tombstone is already discarded command data. Servers with the
|
||||||
|
tombstone-reconciliation correction validate that evidence but omit redundant
|
||||||
|
discard instructions for it. Without that correction, a growing tombstone
|
||||||
|
ledger alone can make the reconnect reply exceed an old client's 32 KiB reader
|
||||||
|
and cause a persistent reconnect loop. Preserve the ledger; upgrade the client
|
||||||
|
or apply the separately tested server correction. Genuine large action lists
|
||||||
|
and other large messages still need a current client reader. This correction
|
||||||
|
does not enable transport auto-chunking.
|
||||||
|
|
||||||
Development builds may negotiate `rvbox.agent.v1.chunked` in the WebSocket
|
Development builds may negotiate `rvbox.agent.v1.chunked` in the WebSocket
|
||||||
handshake. When both endpoints support it, messages automatically use bounded
|
handshake. When both endpoints support it, messages automatically use bounded
|
||||||
16 KiB packets and are reassembled before protocol handling. Old peers retain
|
16 KiB packets and are reassembled before protocol handling. Old peers retain
|
||||||
|
|||||||
@@ -1235,6 +1235,18 @@ layer = "integration"
|
|||||||
status = "implemented"
|
status = "implemented"
|
||||||
tests = ["test/integration/clientagent/websocket_limits_integration_test.go:TestLargeScriptFitsLegacyReader_REC_WS_02"]
|
tests = ["test/integration/clientagent/websocket_limits_integration_test.go:TestLargeScriptFitsLegacyReader_REC_WS_02"]
|
||||||
|
|
||||||
|
[[requirements]]
|
||||||
|
id = "REC-WS-03"
|
||||||
|
layer = "integration"
|
||||||
|
status = "implemented"
|
||||||
|
tests = ["test/integration/clientagent/tombstone_reconcile_integration_test.go:TestLegacyReconnectWithTombstoneHistory_REC_WS_03"]
|
||||||
|
|
||||||
|
[[requirements]]
|
||||||
|
id = "REC-RECONCILE-09"
|
||||||
|
layer = "unit"
|
||||||
|
status = "implemented"
|
||||||
|
tests = ["internal/server/store/reconcile_tombstone_test.go:TestReconcileTombstonesStillValidateAndInterrupt_REC_RECONCILE_09"]
|
||||||
|
|
||||||
[[requirements]]
|
[[requirements]]
|
||||||
id = "REC-RUNTIME-06"
|
id = "REC-RUNTIME-06"
|
||||||
layer = "unit"
|
layer = "unit"
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ func TestWebSocketHelloWelcome_HP_SES_06(t *testing.T) {
|
|||||||
terminalIssue := "019c46f1-1d02-7000-8000-000000000073"
|
terminalIssue := "019c46f1-1d02-7000-8000-000000000073"
|
||||||
reconciled, err := agent.Reconcile(ctx, transport, result, &rvboxv1.ReconcileSnapshot{RetainedCommands: []*rvboxv1.ReconcileCommandState{
|
reconciled, err := agent.Reconcile(ctx, transport, result, &rvboxv1.ReconcileSnapshot{RetainedCommands: []*rvboxv1.ReconcileCommandState{
|
||||||
{IssueUuid: activeIssue, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_RUNNING, CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32)},
|
{IssueUuid: activeIssue, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_RUNNING, CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32)},
|
||||||
{IssueUuid: terminalIssue, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED, CommandRevision: 1, Tombstoned: true, ImmutableRequestSha256: make([]byte, 32)},
|
{IssueUuid: terminalIssue, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED, CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32)},
|
||||||
}}, agentproto.DefaultLimits())
|
}}, agentproto.DefaultLimits())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -0,0 +1,84 @@
|
|||||||
|
package clientagent_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/coder/websocket"
|
||||||
|
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||||
|
"github.com/rvbox/rvbox/internal/agentproto"
|
||||||
|
"github.com/rvbox/rvbox/internal/client/agent"
|
||||||
|
"github.com/rvbox/rvbox/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLegacyReconnectWithTombstoneHistory_REC_WS_03(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
fixture := newFrameFixture(t)
|
||||||
|
snapshot := &rvboxv1.ReconcileSnapshot{}
|
||||||
|
for i := 0; i < 1100; i++ {
|
||||||
|
issue, err := domain.NewUUIDv7()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
snapshot.RetainedCommands = append(snapshot.RetainedCommands, &rvboxv1.ReconcileCommandState{
|
||||||
|
IssueUuid: issue.String(), Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
|
||||||
|
CommandRevision: 1, Tombstoned: true, ImmutableRequestSha256: make([]byte, 32),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// A genuinely retained terminal command still needs discard authorization.
|
||||||
|
terminal, err := domain.NewUUIDv7()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
snapshot.RetainedCommands = append(snapshot.RetainedCommands, &rvboxv1.ReconcileCommandState{
|
||||||
|
IssueUuid: terminal.String(), Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
|
||||||
|
CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32),
|
||||||
|
})
|
||||||
|
// Repeat the full snapshot to model an interrupted/lost reconciliation
|
||||||
|
// response. No subprotocol or updated read-limit wrapper is used.
|
||||||
|
for attempt := 0; attempt < 2; attempt++ {
|
||||||
|
raw, _, err := websocket.Dial(ctx, "wss"+strings.TrimPrefix(fixture.server.URL, "https")+"/v1/agent", &websocket.DialOptions{HTTPClient: fixture.server.Client()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
raw.SetReadLimit(32 << 10)
|
||||||
|
transport := tombstoneLegacyTransport{raw}
|
||||||
|
t.Cleanup(func() { _ = raw.CloseNow() })
|
||||||
|
if raw.Subprotocol() != "" {
|
||||||
|
t.Fatal("legacy client unexpectedly negotiated a subprotocol")
|
||||||
|
}
|
||||||
|
accepted, err := agent.Handshake(ctx, transport, fixture.hello, agentproto.DefaultLimits())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
result, err := agent.Reconcile(ctx, transport, accepted, snapshot, agentproto.DefaultLimits())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("legacy reconnect %d: %v", attempt, err)
|
||||||
|
}
|
||||||
|
if got := result.GetDiscardLocalTerminalIssueUuids(); len(got) != 1 || got[0] != terminal.String() || len(result.GetTerminateLocalIssueUuids()) != 0 {
|
||||||
|
t.Fatalf("unexpected reconciliation instructions: %v", result)
|
||||||
|
}
|
||||||
|
if err := transport.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type tombstoneLegacyTransport struct{ connection *websocket.Conn }
|
||||||
|
|
||||||
|
func (transport tombstoneLegacyTransport) Read(ctx context.Context) ([]byte, error) {
|
||||||
|
_, payload, err := transport.connection.Read(ctx)
|
||||||
|
return payload, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (transport tombstoneLegacyTransport) Write(ctx context.Context, payload []byte) error {
|
||||||
|
return transport.connection.Write(ctx, websocket.MessageBinary, payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (transport tombstoneLegacyTransport) Close() error {
|
||||||
|
return transport.connection.Close(websocket.StatusNormalClosure, "")
|
||||||
|
}
|
||||||
@@ -83,7 +83,7 @@ func TestLargeDispatchAndReconciliation_HP_WS_01(t *testing.T) {
|
|||||||
}
|
}
|
||||||
snapshot.RetainedCommands = append(snapshot.RetainedCommands, &rvboxv1.ReconcileCommandState{
|
snapshot.RetainedCommands = append(snapshot.RetainedCommands, &rvboxv1.ReconcileCommandState{
|
||||||
IssueUuid: issue.String(), Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
|
IssueUuid: issue.String(), Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
|
||||||
CommandRevision: 1, Tombstoned: true, ImmutableRequestSha256: make([]byte, 32),
|
CommandRevision: 1, ImmutableRequestSha256: make([]byte, 32),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
transport, _ := fixture.connect(t, ctx, snapshot)
|
transport, _ := fixture.connect(t, ctx, snapshot)
|
||||||
|
|||||||
Reference in new issue
Block a user