fix(server): avoid deadlock during command payload recovery

This commit is contained in:
cabbage committed 2026-10-29 08:34:39 +00:00
1 parent 8d28925e44
commit ef6e65e6f1
4 files changed
+182 -19

No files matched your search

+6
View File
@@ -133,6 +133,12 @@ exact ignored `dist/` directory after preserving any evidence.
protection. Inspect command retention and free space, allow terminal-age or protection. Inspect command retention and free space, allow terminal-age or
quota rotation to reclaim eligible data, or enlarge the owned filesystem. quota rotation to reclaim eligible data, or enlarge the owned filesystem.
Never manually remove live SQLite/WAL/segment files. Never manually remove live SQLite/WAL/segment files.
- **Startup remains unready and control requests time out:** a corrected server
release is required if payload recovery is waiting for SQLite's sole
connection while holding a command cursor. Recovery now joins script
descriptors and payloads in one query and decompresses stored execution
descriptions before validation. Preserve the existing data directory during
upgrade; deleting command history is unnecessary.
- **Repeated `signal command ...: command is not running`:** early v1 clients - **Repeated `signal command ...: command is not running`:** early v1 clients
closed their session instead of returning a negative signal result when an closed their session instead of returning a negative signal result when an
old queued/replayed signal reached an already-exited process. Upgrade the old queued/replayed signal reached an already-exited process. Upgrade the
+34 -19
View File
@@ -45,50 +45,65 @@ func checkCommandPayloads(ctx context.Context, database *sql.DB) error {
return err return err
} }
commands, err := database.QueryContext(ctx, `SELECT issue_uuid, execution_spec FROM commands`) // Join the descriptor and payload in one cursor. The store deliberately has
// one SQLite connection; querying again while this cursor is open deadlocks.
commands, err := database.QueryContext(ctx, `SELECT c.issue_uuid, c.execution_spec,
c.execution_spec_raw_bytes, c.execution_spec_stored_bytes, c.execution_spec_compression,
p.raw_bytes, p.compression, p.sha256, p.inline_data
FROM commands c LEFT JOIN command_payloads p ON p.issue_uuid = c.issue_uuid AND p.kind = 'script'`)
if err != nil { if err != nil {
return err return err
} }
defer commands.Close() defer commands.Close()
for commands.Next() { for commands.Next() {
var issue, encoded []byte var issue, encoded []byte
if err := commands.Scan(&issue, &encoded); err != nil { var specRawBytes, specStoredBytes uint64
var specCompression uint32
var rawBytes, compression sql.NullInt64
var digest, stored []byte
if err := commands.Scan(&issue, &encoded, &specRawBytes, &specStoredBytes, &specCompression, &rawBytes, &compression, &digest, &stored); err != nil {
return err return err
} }
if len(issue) != 16 { if len(issue) != 16 || specStoredBytes != uint64(len(encoded)) {
return ErrCommittedRangeMissing
}
switch specCompression {
case 1:
if specRawBytes != uint64(len(encoded)) || specRawBytes > maxStoredExecutionSpecBytes {
return ErrCommittedRangeMissing
}
case 2:
encoded, err = decompressCommandSpec(encoded, specRawBytes)
if err != nil {
return ErrCommittedRangeMissing
}
default:
return ErrCommittedRangeMissing return ErrCommittedRangeMissing
} }
var spec rvboxv1.ExecutionSpec var spec rvboxv1.ExecutionSpec
if err := proto.Unmarshal(encoded, &spec); err != nil { if err := proto.Unmarshal(encoded, &spec); err != nil {
// Some low-level store tests intentionally use opaque bytes. Such // Some low-level store tests intentionally use opaque bytes. Such
// records cannot claim a script source, and dispatch validation will // records cannot own a script payload. Dispatch validation rejects
// reject them before any user code runs. // their execution spec before any user code runs.
if rawBytes.Valid {
return ErrCommittedRangeMissing
}
continue continue
} }
descriptor := spec.GetScript() descriptor := spec.GetScript()
var payloadCount uint64
if err := database.QueryRowContext(ctx, `SELECT count(*) FROM command_payloads WHERE issue_uuid = ? AND kind = 'script'`, issue).Scan(&payloadCount); err != nil {
return err
}
if descriptor == nil { if descriptor == nil {
if payloadCount != 0 { if rawBytes.Valid {
return ErrCommittedRangeMissing return ErrCommittedRangeMissing
} }
continue continue
} }
if payloadCount != 1 || descriptor.GetSizeBytes() > maxStoredScriptBytes || len(descriptor.GetSha256()) != sha256.Size { if !rawBytes.Valid || !compression.Valid || descriptor.GetSizeBytes() > maxStoredScriptBytes || len(descriptor.GetSha256()) != sha256.Size {
return ErrCommittedRangeMissing return ErrCommittedRangeMissing
} }
var rawBytes, storedBytes uint64 if uint64(rawBytes.Int64) != descriptor.GetSizeBytes() || compression.Int64 != 2 || len(digest) != sha256.Size || !bytes.Equal(digest, descriptor.GetSha256()) {
var compression uint32
var digest, stored []byte
if err := database.QueryRowContext(ctx, `SELECT raw_bytes, stored_bytes, compression, sha256, inline_data FROM command_payloads WHERE issue_uuid = ? AND kind = 'script'`, issue).Scan(&rawBytes, &storedBytes, &compression, &digest, &stored); err != nil {
return err
}
if rawBytes != descriptor.GetSizeBytes() || compression != 2 || len(digest) != sha256.Size || !bytes.Equal(digest, descriptor.GetSha256()) {
return ErrCommittedRangeMissing return ErrCommittedRangeMissing
} }
decoded, err := decodeCommandPayload(stored, rawBytes, maxStoredScriptBytes) decoded, err := decodeCommandPayload(stored, uint64(rawBytes.Int64), maxStoredScriptBytes)
computed := sha256.Sum256(decoded) computed := sha256.Sum256(decoded)
if err != nil || !bytes.Equal(computed[:], descriptor.GetSha256()) { if err != nil || !bytes.Equal(computed[:], descriptor.GetSha256()) {
return ErrCommittedRangeMissing return ErrCommittedRangeMissing
@@ -0,0 +1,136 @@
package store
import (
"context"
"crypto/sha256"
"errors"
"path/filepath"
"testing"
"time"
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"google.golang.org/protobuf/proto"
)
func TestPayloadRecoveryAfterRestart_REC_STORE_03(t *testing.T) {
t.Parallel()
for _, scenario := range []struct {
name string
script bool
empty bool
alter string
wantBroken bool
}{
{name: "command"},
{name: "script", script: true},
{name: "empty script", script: true, empty: true},
{name: "uncompressed command", alter: "uncompressed"},
{name: "uncompressed script", script: true, alter: "uncompressed"},
{name: "missing script", script: true, alter: "missing", wantBroken: true},
{name: "orphan script", script: true, alter: "command", wantBroken: true},
{name: "wrong descriptor", script: true, alter: "descriptor", wantBroken: true},
{name: "damaged frame", alter: "frame", wantBroken: true},
{name: "wrong stored size", alter: "size", wantBroken: true},
{name: "opaque descriptor with script", script: true, alter: "opaque", wantBroken: true},
} {
t.Run(scenario.name, func(t *testing.T) {
t.Parallel()
ctx := context.Background()
options := Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second}
opened, err := Open(ctx, options)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = opened.Close() })
if _, err := opened.RegisterClientSession(ctx, ClientRegistration{
ClientID: "recovery-client", Platform: 2, Architecture: "amd64", DaemonVersion: "test",
DaemonCWD: `C:\`, SupportedShells: []byte{1}, ClientInstanceID: [16]byte{1},
SessionID: [16]byte{2}, ConnectedAt: time.Now().UTC(),
}); err != nil {
t.Fatal(err)
}
body := []byte("Write-Output 'recovery'\r\n")
if scenario.empty {
body = []byte{}
}
digest := sha256.Sum256(body)
spec := &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_POWERSHELL}
if scenario.script {
spec.Source = &rvboxv1.ExecutionSpec_Script{Script: &rvboxv1.ScriptDescriptor{
Filename: "recovery.ps1", SizeBytes: uint64(len(body)), Sha256: digest[:],
}}
}
encoded, err := proto.Marshal(spec)
if err != nil {
t.Fatal(err)
}
issue := fixedStoreIssue(0xd2)
now := time.Now().UTC()
input := QueueCommandInput{IssueUUID: issue, ClientID: "recovery-client", IssueTime: now,
ReceiptTime: now, ImmutableSHA256: sha256.Sum256(encoded), ExecutionSpec: encoded,
ScriptPresent: scenario.script}
if scenario.script {
input.ScriptContent = body
}
if _, err := opened.QueueCommand(ctx, input); err != nil {
t.Fatal(err)
}
exec := func(query string, args ...any) {
t.Helper()
if _, err := opened.DB().ExecContext(ctx, query, args...); err != nil {
t.Fatal(err)
}
}
switch scenario.alter {
case "uncompressed":
exec(`UPDATE commands SET execution_spec = ?, execution_spec_stored_bytes = ?, execution_spec_compression = 1`, encoded, len(encoded))
case "missing":
exec(`DELETE FROM command_payloads`)
case "command", "descriptor", "opaque":
if scenario.alter == "command" {
spec.Source = nil
} else if scenario.alter == "descriptor" {
spec.GetScript().Sha256 = make([]byte, sha256.Size)
}
changed, err := proto.Marshal(spec)
if err != nil {
t.Fatal(err)
}
if scenario.alter == "opaque" {
changed = []byte{0xff}
}
stored, err := compressCommandSpec(changed)
if err != nil {
t.Fatal(err)
}
exec(`UPDATE commands SET execution_spec = ?, execution_spec_raw_bytes = ?, execution_spec_stored_bytes = ?`, stored, len(changed), len(stored))
case "frame":
exec(`UPDATE commands SET execution_spec = zeroblob(execution_spec_stored_bytes)`)
case "size":
exec(`UPDATE commands SET execution_spec_stored_bytes = execution_spec_stored_bytes + 1`)
}
if err := opened.Close(); err != nil {
t.Fatal(err)
}
opened, err = Open(ctx, options)
if err != nil {
t.Fatal(err)
}
// The deadline makes a nested-query regression fail promptly instead
// of hanging the test or leaving startup permanently unready.
recoveryContext, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
_, err = opened.RecoverCommandSegments(recoveryContext)
if scenario.wantBroken {
if !errors.Is(err, ErrCommittedRangeMissing) {
t.Fatalf("recovery = %v, want integrity failure", err)
}
} else if err != nil {
t.Fatal(err)
}
if _, err := opened.Telemetry(ctx); err != nil {
t.Fatalf("recovery did not release SQLite connection: %v", err)
}
})
}
}
+6
View File
@@ -1157,6 +1157,12 @@ layer = "integration"
status = "implemented" status = "implemented"
tests = ["test/integration/store/store_integration_test.go:TestIncidentDirtyResolutionIdempotencyAndRecurrence_HP_STORE_11"] tests = ["test/integration/store/store_integration_test.go:TestIncidentDirtyResolutionIdempotencyAndRecurrence_HP_STORE_11"]
[[requirements]]
id = "REC-STORE-03"
layer = "unit"
status = "implemented"
tests = ["internal/server/store/payload_recovery_test.go:TestPayloadRecoveryAfterRestart_REC_STORE_03"]
[[requirements]] [[requirements]]
id = "BH-STORE-09" id = "BH-STORE-09"
layer = "integration" layer = "integration"