fix(server): avoid deadlock during command payload recovery
This commit is contained in:
1 parent
8d28925e44
commit
ef6e65e6f1
4 files changed
+186
-23
No files matched your search
@@ -133,6 +133,12 @@ exact ignored `dist/` directory after preserving any evidence.
|
||||
protection. Inspect command retention and free space, allow terminal-age or
|
||||
quota rotation to reclaim eligible data, or enlarge the owned filesystem.
|
||||
Never manually remove live SQLite/WAL/segment files.
|
||||
- **Startup remains unready and control requests time out:** a corrected server
|
||||
release is required if payload recovery is waiting for SQLite's sole
|
||||
connection while holding a command cursor. Recovery now joins script
|
||||
descriptors and payloads in one query and decompresses stored execution
|
||||
descriptions before validation. Preserve the existing data directory during
|
||||
upgrade; deleting command history is unnecessary.
|
||||
- **Repeated `signal command ...: command is not running`:** early v1 clients
|
||||
closed their session instead of returning a negative signal result when an
|
||||
old queued/replayed signal reached an already-exited process. Upgrade the
|
||||
|
||||
@@ -45,50 +45,65 @@ func checkCommandPayloads(ctx context.Context, database *sql.DB) error {
|
||||
return err
|
||||
}
|
||||
|
||||
commands, err := database.QueryContext(ctx, `SELECT issue_uuid, execution_spec FROM commands`)
|
||||
// Join the descriptor and payload in one cursor. The store deliberately has
|
||||
// one SQLite connection; querying again while this cursor is open deadlocks.
|
||||
commands, err := database.QueryContext(ctx, `SELECT c.issue_uuid, c.execution_spec,
|
||||
c.execution_spec_raw_bytes, c.execution_spec_stored_bytes, c.execution_spec_compression,
|
||||
p.raw_bytes, p.compression, p.sha256, p.inline_data
|
||||
FROM commands c LEFT JOIN command_payloads p ON p.issue_uuid = c.issue_uuid AND p.kind = 'script'`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer commands.Close()
|
||||
for commands.Next() {
|
||||
var issue, encoded []byte
|
||||
if err := commands.Scan(&issue, &encoded); err != nil {
|
||||
var specRawBytes, specStoredBytes uint64
|
||||
var specCompression uint32
|
||||
var rawBytes, compression sql.NullInt64
|
||||
var digest, stored []byte
|
||||
if err := commands.Scan(&issue, &encoded, &specRawBytes, &specStoredBytes, &specCompression, &rawBytes, &compression, &digest, &stored); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(issue) != 16 {
|
||||
if len(issue) != 16 || specStoredBytes != uint64(len(encoded)) {
|
||||
return ErrCommittedRangeMissing
|
||||
}
|
||||
switch specCompression {
|
||||
case 1:
|
||||
if specRawBytes != uint64(len(encoded)) || specRawBytes > maxStoredExecutionSpecBytes {
|
||||
return ErrCommittedRangeMissing
|
||||
}
|
||||
case 2:
|
||||
encoded, err = decompressCommandSpec(encoded, specRawBytes)
|
||||
if err != nil {
|
||||
return ErrCommittedRangeMissing
|
||||
}
|
||||
default:
|
||||
return ErrCommittedRangeMissing
|
||||
}
|
||||
var spec rvboxv1.ExecutionSpec
|
||||
if err := proto.Unmarshal(encoded, &spec); err != nil {
|
||||
// Some low-level store tests intentionally use opaque bytes. Such
|
||||
// records cannot claim a script source, and dispatch validation will
|
||||
// reject them before any user code runs.
|
||||
continue
|
||||
}
|
||||
descriptor := spec.GetScript()
|
||||
var payloadCount uint64
|
||||
if err := database.QueryRowContext(ctx, `SELECT count(*) FROM command_payloads WHERE issue_uuid = ? AND kind = 'script'`, issue).Scan(&payloadCount); err != nil {
|
||||
return err
|
||||
}
|
||||
if descriptor == nil {
|
||||
if payloadCount != 0 {
|
||||
// records cannot own a script payload. Dispatch validation rejects
|
||||
// their execution spec before any user code runs.
|
||||
if rawBytes.Valid {
|
||||
return ErrCommittedRangeMissing
|
||||
}
|
||||
continue
|
||||
}
|
||||
if payloadCount != 1 || descriptor.GetSizeBytes() > maxStoredScriptBytes || len(descriptor.GetSha256()) != sha256.Size {
|
||||
descriptor := spec.GetScript()
|
||||
if descriptor == nil {
|
||||
if rawBytes.Valid {
|
||||
return ErrCommittedRangeMissing
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !rawBytes.Valid || !compression.Valid || descriptor.GetSizeBytes() > maxStoredScriptBytes || len(descriptor.GetSha256()) != sha256.Size {
|
||||
return ErrCommittedRangeMissing
|
||||
}
|
||||
var rawBytes, storedBytes uint64
|
||||
var compression uint32
|
||||
var digest, stored []byte
|
||||
if err := database.QueryRowContext(ctx, `SELECT raw_bytes, stored_bytes, compression, sha256, inline_data FROM command_payloads WHERE issue_uuid = ? AND kind = 'script'`, issue).Scan(&rawBytes, &storedBytes, &compression, &digest, &stored); err != nil {
|
||||
return err
|
||||
}
|
||||
if rawBytes != descriptor.GetSizeBytes() || compression != 2 || len(digest) != sha256.Size || !bytes.Equal(digest, descriptor.GetSha256()) {
|
||||
if uint64(rawBytes.Int64) != descriptor.GetSizeBytes() || compression.Int64 != 2 || len(digest) != sha256.Size || !bytes.Equal(digest, descriptor.GetSha256()) {
|
||||
return ErrCommittedRangeMissing
|
||||
}
|
||||
decoded, err := decodeCommandPayload(stored, rawBytes, maxStoredScriptBytes)
|
||||
decoded, err := decodeCommandPayload(stored, uint64(rawBytes.Int64), maxStoredScriptBytes)
|
||||
computed := sha256.Sum256(decoded)
|
||||
if err != nil || !bytes.Equal(computed[:], descriptor.GetSha256()) {
|
||||
return ErrCommittedRangeMissing
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
func TestPayloadRecoveryAfterRestart_REC_STORE_03(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, scenario := range []struct {
|
||||
name string
|
||||
script bool
|
||||
empty bool
|
||||
alter string
|
||||
wantBroken bool
|
||||
}{
|
||||
{name: "command"},
|
||||
{name: "script", script: true},
|
||||
{name: "empty script", script: true, empty: true},
|
||||
{name: "uncompressed command", alter: "uncompressed"},
|
||||
{name: "uncompressed script", script: true, alter: "uncompressed"},
|
||||
{name: "missing script", script: true, alter: "missing", wantBroken: true},
|
||||
{name: "orphan script", script: true, alter: "command", wantBroken: true},
|
||||
{name: "wrong descriptor", script: true, alter: "descriptor", wantBroken: true},
|
||||
{name: "damaged frame", alter: "frame", wantBroken: true},
|
||||
{name: "wrong stored size", alter: "size", wantBroken: true},
|
||||
{name: "opaque descriptor with script", script: true, alter: "opaque", wantBroken: true},
|
||||
} {
|
||||
t.Run(scenario.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
options := Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second}
|
||||
opened, err := Open(ctx, options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = opened.Close() })
|
||||
if _, err := opened.RegisterClientSession(ctx, ClientRegistration{
|
||||
ClientID: "recovery-client", Platform: 2, Architecture: "amd64", DaemonVersion: "test",
|
||||
DaemonCWD: `C:\`, SupportedShells: []byte{1}, ClientInstanceID: [16]byte{1},
|
||||
SessionID: [16]byte{2}, ConnectedAt: time.Now().UTC(),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("Write-Output 'recovery'\r\n")
|
||||
if scenario.empty {
|
||||
body = []byte{}
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
spec := &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_POWERSHELL}
|
||||
if scenario.script {
|
||||
spec.Source = &rvboxv1.ExecutionSpec_Script{Script: &rvboxv1.ScriptDescriptor{
|
||||
Filename: "recovery.ps1", SizeBytes: uint64(len(body)), Sha256: digest[:],
|
||||
}}
|
||||
}
|
||||
encoded, err := proto.Marshal(spec)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
issue := fixedStoreIssue(0xd2)
|
||||
now := time.Now().UTC()
|
||||
input := QueueCommandInput{IssueUUID: issue, ClientID: "recovery-client", IssueTime: now,
|
||||
ReceiptTime: now, ImmutableSHA256: sha256.Sum256(encoded), ExecutionSpec: encoded,
|
||||
ScriptPresent: scenario.script}
|
||||
if scenario.script {
|
||||
input.ScriptContent = body
|
||||
}
|
||||
if _, err := opened.QueueCommand(ctx, input); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exec := func(query string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := opened.DB().ExecContext(ctx, query, args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
switch scenario.alter {
|
||||
case "uncompressed":
|
||||
exec(`UPDATE commands SET execution_spec = ?, execution_spec_stored_bytes = ?, execution_spec_compression = 1`, encoded, len(encoded))
|
||||
case "missing":
|
||||
exec(`DELETE FROM command_payloads`)
|
||||
case "command", "descriptor", "opaque":
|
||||
if scenario.alter == "command" {
|
||||
spec.Source = nil
|
||||
} else if scenario.alter == "descriptor" {
|
||||
spec.GetScript().Sha256 = make([]byte, sha256.Size)
|
||||
}
|
||||
changed, err := proto.Marshal(spec)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scenario.alter == "opaque" {
|
||||
changed = []byte{0xff}
|
||||
}
|
||||
stored, err := compressCommandSpec(changed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exec(`UPDATE commands SET execution_spec = ?, execution_spec_raw_bytes = ?, execution_spec_stored_bytes = ?`, stored, len(changed), len(stored))
|
||||
case "frame":
|
||||
exec(`UPDATE commands SET execution_spec = zeroblob(execution_spec_stored_bytes)`)
|
||||
case "size":
|
||||
exec(`UPDATE commands SET execution_spec_stored_bytes = execution_spec_stored_bytes + 1`)
|
||||
}
|
||||
if err := opened.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opened, err = Open(ctx, options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The deadline makes a nested-query regression fail promptly instead
|
||||
// of hanging the test or leaving startup permanently unready.
|
||||
recoveryContext, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
_, err = opened.RecoverCommandSegments(recoveryContext)
|
||||
if scenario.wantBroken {
|
||||
if !errors.Is(err, ErrCommittedRangeMissing) {
|
||||
t.Fatalf("recovery = %v, want integrity failure", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := opened.Telemetry(ctx); err != nil {
|
||||
t.Fatalf("recovery did not release SQLite connection: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1157,6 +1157,12 @@ layer = "integration"
|
||||
status = "implemented"
|
||||
tests = ["test/integration/store/store_integration_test.go:TestIncidentDirtyResolutionIdempotencyAndRecurrence_HP_STORE_11"]
|
||||
|
||||
[[requirements]]
|
||||
id = "REC-STORE-03"
|
||||
layer = "unit"
|
||||
status = "implemented"
|
||||
tests = ["internal/server/store/payload_recovery_test.go:TestPayloadRecoveryAfterRestart_REC_STORE_03"]
|
||||
|
||||
[[requirements]]
|
||||
id = "BH-STORE-09"
|
||||
layer = "integration"
|
||||
|
||||
Reference in new issue
Block a user