fix(server): avoid deadlock during command payload recovery
This commit is contained in:
1 parent
8d28925e44
commit
ef6e65e6f1
4 files changed
+186
-23
No files matched your search
@@ -133,6 +133,12 @@ exact ignored `dist/` directory after preserving any evidence.
|
|||||||
protection. Inspect command retention and free space, allow terminal-age or
|
protection. Inspect command retention and free space, allow terminal-age or
|
||||||
quota rotation to reclaim eligible data, or enlarge the owned filesystem.
|
quota rotation to reclaim eligible data, or enlarge the owned filesystem.
|
||||||
Never manually remove live SQLite/WAL/segment files.
|
Never manually remove live SQLite/WAL/segment files.
|
||||||
|
- **Startup remains unready and control requests time out:** a corrected server
|
||||||
|
release is required if payload recovery is waiting for SQLite's sole
|
||||||
|
connection while holding a command cursor. Recovery now joins script
|
||||||
|
descriptors and payloads in one query and decompresses stored execution
|
||||||
|
descriptions before validation. Preserve the existing data directory during
|
||||||
|
upgrade; deleting command history is unnecessary.
|
||||||
- **Repeated `signal command ...: command is not running`:** early v1 clients
|
- **Repeated `signal command ...: command is not running`:** early v1 clients
|
||||||
closed their session instead of returning a negative signal result when an
|
closed their session instead of returning a negative signal result when an
|
||||||
old queued/replayed signal reached an already-exited process. Upgrade the
|
old queued/replayed signal reached an already-exited process. Upgrade the
|
||||||
|
|||||||
@@ -45,50 +45,65 @@ func checkCommandPayloads(ctx context.Context, database *sql.DB) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
commands, err := database.QueryContext(ctx, `SELECT issue_uuid, execution_spec FROM commands`)
|
// Join the descriptor and payload in one cursor. The store deliberately has
|
||||||
|
// one SQLite connection; querying again while this cursor is open deadlocks.
|
||||||
|
commands, err := database.QueryContext(ctx, `SELECT c.issue_uuid, c.execution_spec,
|
||||||
|
c.execution_spec_raw_bytes, c.execution_spec_stored_bytes, c.execution_spec_compression,
|
||||||
|
p.raw_bytes, p.compression, p.sha256, p.inline_data
|
||||||
|
FROM commands c LEFT JOIN command_payloads p ON p.issue_uuid = c.issue_uuid AND p.kind = 'script'`)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer commands.Close()
|
defer commands.Close()
|
||||||
for commands.Next() {
|
for commands.Next() {
|
||||||
var issue, encoded []byte
|
var issue, encoded []byte
|
||||||
if err := commands.Scan(&issue, &encoded); err != nil {
|
var specRawBytes, specStoredBytes uint64
|
||||||
|
var specCompression uint32
|
||||||
|
var rawBytes, compression sql.NullInt64
|
||||||
|
var digest, stored []byte
|
||||||
|
if err := commands.Scan(&issue, &encoded, &specRawBytes, &specStoredBytes, &specCompression, &rawBytes, &compression, &digest, &stored); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if len(issue) != 16 {
|
if len(issue) != 16 || specStoredBytes != uint64(len(encoded)) {
|
||||||
|
return ErrCommittedRangeMissing
|
||||||
|
}
|
||||||
|
switch specCompression {
|
||||||
|
case 1:
|
||||||
|
if specRawBytes != uint64(len(encoded)) || specRawBytes > maxStoredExecutionSpecBytes {
|
||||||
|
return ErrCommittedRangeMissing
|
||||||
|
}
|
||||||
|
case 2:
|
||||||
|
encoded, err = decompressCommandSpec(encoded, specRawBytes)
|
||||||
|
if err != nil {
|
||||||
|
return ErrCommittedRangeMissing
|
||||||
|
}
|
||||||
|
default:
|
||||||
return ErrCommittedRangeMissing
|
return ErrCommittedRangeMissing
|
||||||
}
|
}
|
||||||
var spec rvboxv1.ExecutionSpec
|
var spec rvboxv1.ExecutionSpec
|
||||||
if err := proto.Unmarshal(encoded, &spec); err != nil {
|
if err := proto.Unmarshal(encoded, &spec); err != nil {
|
||||||
// Some low-level store tests intentionally use opaque bytes. Such
|
// Some low-level store tests intentionally use opaque bytes. Such
|
||||||
// records cannot claim a script source, and dispatch validation will
|
// records cannot own a script payload. Dispatch validation rejects
|
||||||
// reject them before any user code runs.
|
// their execution spec before any user code runs.
|
||||||
continue
|
if rawBytes.Valid {
|
||||||
}
|
|
||||||
descriptor := spec.GetScript()
|
|
||||||
var payloadCount uint64
|
|
||||||
if err := database.QueryRowContext(ctx, `SELECT count(*) FROM command_payloads WHERE issue_uuid = ? AND kind = 'script'`, issue).Scan(&payloadCount); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if descriptor == nil {
|
|
||||||
if payloadCount != 0 {
|
|
||||||
return ErrCommittedRangeMissing
|
return ErrCommittedRangeMissing
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if payloadCount != 1 || descriptor.GetSizeBytes() > maxStoredScriptBytes || len(descriptor.GetSha256()) != sha256.Size {
|
descriptor := spec.GetScript()
|
||||||
|
if descriptor == nil {
|
||||||
|
if rawBytes.Valid {
|
||||||
|
return ErrCommittedRangeMissing
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !rawBytes.Valid || !compression.Valid || descriptor.GetSizeBytes() > maxStoredScriptBytes || len(descriptor.GetSha256()) != sha256.Size {
|
||||||
return ErrCommittedRangeMissing
|
return ErrCommittedRangeMissing
|
||||||
}
|
}
|
||||||
var rawBytes, storedBytes uint64
|
if uint64(rawBytes.Int64) != descriptor.GetSizeBytes() || compression.Int64 != 2 || len(digest) != sha256.Size || !bytes.Equal(digest, descriptor.GetSha256()) {
|
||||||
var compression uint32
|
|
||||||
var digest, stored []byte
|
|
||||||
if err := database.QueryRowContext(ctx, `SELECT raw_bytes, stored_bytes, compression, sha256, inline_data FROM command_payloads WHERE issue_uuid = ? AND kind = 'script'`, issue).Scan(&rawBytes, &storedBytes, &compression, &digest, &stored); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if rawBytes != descriptor.GetSizeBytes() || compression != 2 || len(digest) != sha256.Size || !bytes.Equal(digest, descriptor.GetSha256()) {
|
|
||||||
return ErrCommittedRangeMissing
|
return ErrCommittedRangeMissing
|
||||||
}
|
}
|
||||||
decoded, err := decodeCommandPayload(stored, rawBytes, maxStoredScriptBytes)
|
decoded, err := decodeCommandPayload(stored, uint64(rawBytes.Int64), maxStoredScriptBytes)
|
||||||
computed := sha256.Sum256(decoded)
|
computed := sha256.Sum256(decoded)
|
||||||
if err != nil || !bytes.Equal(computed[:], descriptor.GetSha256()) {
|
if err != nil || !bytes.Equal(computed[:], descriptor.GetSha256()) {
|
||||||
return ErrCommittedRangeMissing
|
return ErrCommittedRangeMissing
|
||||||
|
|||||||
@@ -0,0 +1,136 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"errors"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||||
|
"google.golang.org/protobuf/proto"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPayloadRecoveryAfterRestart_REC_STORE_03(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
for _, scenario := range []struct {
|
||||||
|
name string
|
||||||
|
script bool
|
||||||
|
empty bool
|
||||||
|
alter string
|
||||||
|
wantBroken bool
|
||||||
|
}{
|
||||||
|
{name: "command"},
|
||||||
|
{name: "script", script: true},
|
||||||
|
{name: "empty script", script: true, empty: true},
|
||||||
|
{name: "uncompressed command", alter: "uncompressed"},
|
||||||
|
{name: "uncompressed script", script: true, alter: "uncompressed"},
|
||||||
|
{name: "missing script", script: true, alter: "missing", wantBroken: true},
|
||||||
|
{name: "orphan script", script: true, alter: "command", wantBroken: true},
|
||||||
|
{name: "wrong descriptor", script: true, alter: "descriptor", wantBroken: true},
|
||||||
|
{name: "damaged frame", alter: "frame", wantBroken: true},
|
||||||
|
{name: "wrong stored size", alter: "size", wantBroken: true},
|
||||||
|
{name: "opaque descriptor with script", script: true, alter: "opaque", wantBroken: true},
|
||||||
|
} {
|
||||||
|
t.Run(scenario.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
ctx := context.Background()
|
||||||
|
options := Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second}
|
||||||
|
opened, err := Open(ctx, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = opened.Close() })
|
||||||
|
if _, err := opened.RegisterClientSession(ctx, ClientRegistration{
|
||||||
|
ClientID: "recovery-client", Platform: 2, Architecture: "amd64", DaemonVersion: "test",
|
||||||
|
DaemonCWD: `C:\`, SupportedShells: []byte{1}, ClientInstanceID: [16]byte{1},
|
||||||
|
SessionID: [16]byte{2}, ConnectedAt: time.Now().UTC(),
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body := []byte("Write-Output 'recovery'\r\n")
|
||||||
|
if scenario.empty {
|
||||||
|
body = []byte{}
|
||||||
|
}
|
||||||
|
digest := sha256.Sum256(body)
|
||||||
|
spec := &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_POWERSHELL}
|
||||||
|
if scenario.script {
|
||||||
|
spec.Source = &rvboxv1.ExecutionSpec_Script{Script: &rvboxv1.ScriptDescriptor{
|
||||||
|
Filename: "recovery.ps1", SizeBytes: uint64(len(body)), Sha256: digest[:],
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
encoded, err := proto.Marshal(spec)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
issue := fixedStoreIssue(0xd2)
|
||||||
|
now := time.Now().UTC()
|
||||||
|
input := QueueCommandInput{IssueUUID: issue, ClientID: "recovery-client", IssueTime: now,
|
||||||
|
ReceiptTime: now, ImmutableSHA256: sha256.Sum256(encoded), ExecutionSpec: encoded,
|
||||||
|
ScriptPresent: scenario.script}
|
||||||
|
if scenario.script {
|
||||||
|
input.ScriptContent = body
|
||||||
|
}
|
||||||
|
if _, err := opened.QueueCommand(ctx, input); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
exec := func(query string, args ...any) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := opened.DB().ExecContext(ctx, query, args...); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch scenario.alter {
|
||||||
|
case "uncompressed":
|
||||||
|
exec(`UPDATE commands SET execution_spec = ?, execution_spec_stored_bytes = ?, execution_spec_compression = 1`, encoded, len(encoded))
|
||||||
|
case "missing":
|
||||||
|
exec(`DELETE FROM command_payloads`)
|
||||||
|
case "command", "descriptor", "opaque":
|
||||||
|
if scenario.alter == "command" {
|
||||||
|
spec.Source = nil
|
||||||
|
} else if scenario.alter == "descriptor" {
|
||||||
|
spec.GetScript().Sha256 = make([]byte, sha256.Size)
|
||||||
|
}
|
||||||
|
changed, err := proto.Marshal(spec)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if scenario.alter == "opaque" {
|
||||||
|
changed = []byte{0xff}
|
||||||
|
}
|
||||||
|
stored, err := compressCommandSpec(changed)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
exec(`UPDATE commands SET execution_spec = ?, execution_spec_raw_bytes = ?, execution_spec_stored_bytes = ?`, stored, len(changed), len(stored))
|
||||||
|
case "frame":
|
||||||
|
exec(`UPDATE commands SET execution_spec = zeroblob(execution_spec_stored_bytes)`)
|
||||||
|
case "size":
|
||||||
|
exec(`UPDATE commands SET execution_spec_stored_bytes = execution_spec_stored_bytes + 1`)
|
||||||
|
}
|
||||||
|
if err := opened.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
opened, err = Open(ctx, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The deadline makes a nested-query regression fail promptly instead
|
||||||
|
// of hanging the test or leaving startup permanently unready.
|
||||||
|
recoveryContext, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_, err = opened.RecoverCommandSegments(recoveryContext)
|
||||||
|
if scenario.wantBroken {
|
||||||
|
if !errors.Is(err, ErrCommittedRangeMissing) {
|
||||||
|
t.Fatalf("recovery = %v, want integrity failure", err)
|
||||||
|
}
|
||||||
|
} else if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := opened.Telemetry(ctx); err != nil {
|
||||||
|
t.Fatalf("recovery did not release SQLite connection: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1157,6 +1157,12 @@ layer = "integration"
|
|||||||
status = "implemented"
|
status = "implemented"
|
||||||
tests = ["test/integration/store/store_integration_test.go:TestIncidentDirtyResolutionIdempotencyAndRecurrence_HP_STORE_11"]
|
tests = ["test/integration/store/store_integration_test.go:TestIncidentDirtyResolutionIdempotencyAndRecurrence_HP_STORE_11"]
|
||||||
|
|
||||||
|
[[requirements]]
|
||||||
|
id = "REC-STORE-03"
|
||||||
|
layer = "unit"
|
||||||
|
status = "implemented"
|
||||||
|
tests = ["internal/server/store/payload_recovery_test.go:TestPayloadRecoveryAfterRestart_REC_STORE_03"]
|
||||||
|
|
||||||
[[requirements]]
|
[[requirements]]
|
||||||
id = "BH-STORE-09"
|
id = "BH-STORE-09"
|
||||||
layer = "integration"
|
layer = "integration"
|
||||||
|
|||||||
Reference in new issue
Block a user