From ef6e65e6f101aea07e9e91f449972c3dba4425d7 Mon Sep 17 00:00:00 2001 From: cabbage Date: Thu, 29 Oct 2026 08:34:39 +0000 Subject: [PATCH] fix(server): avoid deadlock during command payload recovery --- docs/operations-runbook.md | 6 + internal/server/store/payload_recovery.go | 61 +++++--- .../server/store/payload_recovery_test.go | 136 ++++++++++++++++++ test/coverage.toml | 6 + 4 files changed, 186 insertions(+), 23 deletions(-) create mode 100644 internal/server/store/payload_recovery_test.go diff --git a/docs/operations-runbook.md b/docs/operations-runbook.md index eb8b63d..2c9fa60 100644 --- a/docs/operations-runbook.md +++ b/docs/operations-runbook.md @@ -133,6 +133,12 @@ exact ignored `dist/` directory after preserving any evidence. protection. Inspect command retention and free space, allow terminal-age or quota rotation to reclaim eligible data, or enlarge the owned filesystem. Never manually remove live SQLite/WAL/segment files. +- **Startup remains unready and control requests time out:** a corrected server + release is required if payload recovery is waiting for SQLite's sole + connection while holding a command cursor. Recovery now joins script + descriptors and payloads in one query and decompresses stored execution + descriptions before validation. Preserve the existing data directory during + upgrade; deleting command history is unnecessary. - **Repeated `signal command ...: command is not running`:** early v1 clients closed their session instead of returning a negative signal result when an old queued/replayed signal reached an already-exited process. Upgrade the diff --git a/internal/server/store/payload_recovery.go b/internal/server/store/payload_recovery.go index a9e477d..968e5cb 100644 --- a/internal/server/store/payload_recovery.go +++ b/internal/server/store/payload_recovery.go @@ -45,50 +45,65 @@ func checkCommandPayloads(ctx context.Context, database *sql.DB) error { return err } - commands, err := database.QueryContext(ctx, `SELECT issue_uuid, execution_spec FROM commands`) + // Join the descriptor and payload in one cursor. The store deliberately has + // one SQLite connection; querying again while this cursor is open deadlocks. + commands, err := database.QueryContext(ctx, `SELECT c.issue_uuid, c.execution_spec, +c.execution_spec_raw_bytes, c.execution_spec_stored_bytes, c.execution_spec_compression, +p.raw_bytes, p.compression, p.sha256, p.inline_data +FROM commands c LEFT JOIN command_payloads p ON p.issue_uuid = c.issue_uuid AND p.kind = 'script'`) if err != nil { return err } defer commands.Close() for commands.Next() { var issue, encoded []byte - if err := commands.Scan(&issue, &encoded); err != nil { + var specRawBytes, specStoredBytes uint64 + var specCompression uint32 + var rawBytes, compression sql.NullInt64 + var digest, stored []byte + if err := commands.Scan(&issue, &encoded, &specRawBytes, &specStoredBytes, &specCompression, &rawBytes, &compression, &digest, &stored); err != nil { return err } - if len(issue) != 16 { + if len(issue) != 16 || specStoredBytes != uint64(len(encoded)) { + return ErrCommittedRangeMissing + } + switch specCompression { + case 1: + if specRawBytes != uint64(len(encoded)) || specRawBytes > maxStoredExecutionSpecBytes { + return ErrCommittedRangeMissing + } + case 2: + encoded, err = decompressCommandSpec(encoded, specRawBytes) + if err != nil { + return ErrCommittedRangeMissing + } + default: return ErrCommittedRangeMissing } var spec rvboxv1.ExecutionSpec if err := proto.Unmarshal(encoded, &spec); err != nil { // Some low-level store tests intentionally use opaque bytes. Such - // records cannot claim a script source, and dispatch validation will - // reject them before any user code runs. - continue - } - descriptor := spec.GetScript() - var payloadCount uint64 - if err := database.QueryRowContext(ctx, `SELECT count(*) FROM command_payloads WHERE issue_uuid = ? AND kind = 'script'`, issue).Scan(&payloadCount); err != nil { - return err - } - if descriptor == nil { - if payloadCount != 0 { + // records cannot own a script payload. Dispatch validation rejects + // their execution spec before any user code runs. + if rawBytes.Valid { return ErrCommittedRangeMissing } continue } - if payloadCount != 1 || descriptor.GetSizeBytes() > maxStoredScriptBytes || len(descriptor.GetSha256()) != sha256.Size { + descriptor := spec.GetScript() + if descriptor == nil { + if rawBytes.Valid { + return ErrCommittedRangeMissing + } + continue + } + if !rawBytes.Valid || !compression.Valid || descriptor.GetSizeBytes() > maxStoredScriptBytes || len(descriptor.GetSha256()) != sha256.Size { return ErrCommittedRangeMissing } - var rawBytes, storedBytes uint64 - var compression uint32 - var digest, stored []byte - if err := database.QueryRowContext(ctx, `SELECT raw_bytes, stored_bytes, compression, sha256, inline_data FROM command_payloads WHERE issue_uuid = ? AND kind = 'script'`, issue).Scan(&rawBytes, &storedBytes, &compression, &digest, &stored); err != nil { - return err - } - if rawBytes != descriptor.GetSizeBytes() || compression != 2 || len(digest) != sha256.Size || !bytes.Equal(digest, descriptor.GetSha256()) { + if uint64(rawBytes.Int64) != descriptor.GetSizeBytes() || compression.Int64 != 2 || len(digest) != sha256.Size || !bytes.Equal(digest, descriptor.GetSha256()) { return ErrCommittedRangeMissing } - decoded, err := decodeCommandPayload(stored, rawBytes, maxStoredScriptBytes) + decoded, err := decodeCommandPayload(stored, uint64(rawBytes.Int64), maxStoredScriptBytes) computed := sha256.Sum256(decoded) if err != nil || !bytes.Equal(computed[:], descriptor.GetSha256()) { return ErrCommittedRangeMissing diff --git a/internal/server/store/payload_recovery_test.go b/internal/server/store/payload_recovery_test.go new file mode 100644 index 0000000..7eca8db --- /dev/null +++ b/internal/server/store/payload_recovery_test.go @@ -0,0 +1,136 @@ +package store + +import ( + "context" + "crypto/sha256" + "errors" + "path/filepath" + "testing" + "time" + + rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1" + "google.golang.org/protobuf/proto" +) + +func TestPayloadRecoveryAfterRestart_REC_STORE_03(t *testing.T) { + t.Parallel() + for _, scenario := range []struct { + name string + script bool + empty bool + alter string + wantBroken bool + }{ + {name: "command"}, + {name: "script", script: true}, + {name: "empty script", script: true, empty: true}, + {name: "uncompressed command", alter: "uncompressed"}, + {name: "uncompressed script", script: true, alter: "uncompressed"}, + {name: "missing script", script: true, alter: "missing", wantBroken: true}, + {name: "orphan script", script: true, alter: "command", wantBroken: true}, + {name: "wrong descriptor", script: true, alter: "descriptor", wantBroken: true}, + {name: "damaged frame", alter: "frame", wantBroken: true}, + {name: "wrong stored size", alter: "size", wantBroken: true}, + {name: "opaque descriptor with script", script: true, alter: "opaque", wantBroken: true}, + } { + t.Run(scenario.name, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + options := Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second} + opened, err := Open(ctx, options) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = opened.Close() }) + if _, err := opened.RegisterClientSession(ctx, ClientRegistration{ + ClientID: "recovery-client", Platform: 2, Architecture: "amd64", DaemonVersion: "test", + DaemonCWD: `C:\`, SupportedShells: []byte{1}, ClientInstanceID: [16]byte{1}, + SessionID: [16]byte{2}, ConnectedAt: time.Now().UTC(), + }); err != nil { + t.Fatal(err) + } + body := []byte("Write-Output 'recovery'\r\n") + if scenario.empty { + body = []byte{} + } + digest := sha256.Sum256(body) + spec := &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_POWERSHELL} + if scenario.script { + spec.Source = &rvboxv1.ExecutionSpec_Script{Script: &rvboxv1.ScriptDescriptor{ + Filename: "recovery.ps1", SizeBytes: uint64(len(body)), Sha256: digest[:], + }} + } + encoded, err := proto.Marshal(spec) + if err != nil { + t.Fatal(err) + } + issue := fixedStoreIssue(0xd2) + now := time.Now().UTC() + input := QueueCommandInput{IssueUUID: issue, ClientID: "recovery-client", IssueTime: now, + ReceiptTime: now, ImmutableSHA256: sha256.Sum256(encoded), ExecutionSpec: encoded, + ScriptPresent: scenario.script} + if scenario.script { + input.ScriptContent = body + } + if _, err := opened.QueueCommand(ctx, input); err != nil { + t.Fatal(err) + } + exec := func(query string, args ...any) { + t.Helper() + if _, err := opened.DB().ExecContext(ctx, query, args...); err != nil { + t.Fatal(err) + } + } + switch scenario.alter { + case "uncompressed": + exec(`UPDATE commands SET execution_spec = ?, execution_spec_stored_bytes = ?, execution_spec_compression = 1`, encoded, len(encoded)) + case "missing": + exec(`DELETE FROM command_payloads`) + case "command", "descriptor", "opaque": + if scenario.alter == "command" { + spec.Source = nil + } else if scenario.alter == "descriptor" { + spec.GetScript().Sha256 = make([]byte, sha256.Size) + } + changed, err := proto.Marshal(spec) + if err != nil { + t.Fatal(err) + } + if scenario.alter == "opaque" { + changed = []byte{0xff} + } + stored, err := compressCommandSpec(changed) + if err != nil { + t.Fatal(err) + } + exec(`UPDATE commands SET execution_spec = ?, execution_spec_raw_bytes = ?, execution_spec_stored_bytes = ?`, stored, len(changed), len(stored)) + case "frame": + exec(`UPDATE commands SET execution_spec = zeroblob(execution_spec_stored_bytes)`) + case "size": + exec(`UPDATE commands SET execution_spec_stored_bytes = execution_spec_stored_bytes + 1`) + } + if err := opened.Close(); err != nil { + t.Fatal(err) + } + opened, err = Open(ctx, options) + if err != nil { + t.Fatal(err) + } + // The deadline makes a nested-query regression fail promptly instead + // of hanging the test or leaving startup permanently unready. + recoveryContext, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + _, err = opened.RecoverCommandSegments(recoveryContext) + if scenario.wantBroken { + if !errors.Is(err, ErrCommittedRangeMissing) { + t.Fatalf("recovery = %v, want integrity failure", err) + } + } else if err != nil { + t.Fatal(err) + } + if _, err := opened.Telemetry(ctx); err != nil { + t.Fatalf("recovery did not release SQLite connection: %v", err) + } + }) + } +} diff --git a/test/coverage.toml b/test/coverage.toml index ee55e7a..2d9148b 100644 --- a/test/coverage.toml +++ b/test/coverage.toml @@ -1157,6 +1157,12 @@ layer = "integration" status = "implemented" tests = ["test/integration/store/store_integration_test.go:TestIncidentDirtyResolutionIdempotencyAndRecurrence_HP_STORE_11"] +[[requirements]] +id = "REC-STORE-03" +layer = "unit" +status = "implemented" +tests = ["internal/server/store/payload_recovery_test.go:TestPayloadRecoveryAfterRestart_REC_STORE_03"] + [[requirements]] id = "BH-STORE-09" layer = "integration"