fix(server): avoid deadlock during command payload recovery

This commit is contained in:
cabbage committed 2026-10-29 08:34:39 +00:00
1 parent 8d28925e44
commit ef6e65e6f1
4 files changed
+186 -23

No files matched your search

@@ -0,0 +1,136 @@
package store
import (
"context"
"crypto/sha256"
"errors"
"path/filepath"
"testing"
"time"
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"google.golang.org/protobuf/proto"
)
func TestPayloadRecoveryAfterRestart_REC_STORE_03(t *testing.T) {
t.Parallel()
for _, scenario := range []struct {
name string
script bool
empty bool
alter string
wantBroken bool
}{
{name: "command"},
{name: "script", script: true},
{name: "empty script", script: true, empty: true},
{name: "uncompressed command", alter: "uncompressed"},
{name: "uncompressed script", script: true, alter: "uncompressed"},
{name: "missing script", script: true, alter: "missing", wantBroken: true},
{name: "orphan script", script: true, alter: "command", wantBroken: true},
{name: "wrong descriptor", script: true, alter: "descriptor", wantBroken: true},
{name: "damaged frame", alter: "frame", wantBroken: true},
{name: "wrong stored size", alter: "size", wantBroken: true},
{name: "opaque descriptor with script", script: true, alter: "opaque", wantBroken: true},
} {
t.Run(scenario.name, func(t *testing.T) {
t.Parallel()
ctx := context.Background()
options := Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second}
opened, err := Open(ctx, options)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = opened.Close() })
if _, err := opened.RegisterClientSession(ctx, ClientRegistration{
ClientID: "recovery-client", Platform: 2, Architecture: "amd64", DaemonVersion: "test",
DaemonCWD: `C:\`, SupportedShells: []byte{1}, ClientInstanceID: [16]byte{1},
SessionID: [16]byte{2}, ConnectedAt: time.Now().UTC(),
}); err != nil {
t.Fatal(err)
}
body := []byte("Write-Output 'recovery'\r\n")
if scenario.empty {
body = []byte{}
}
digest := sha256.Sum256(body)
spec := &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_POWERSHELL}
if scenario.script {
spec.Source = &rvboxv1.ExecutionSpec_Script{Script: &rvboxv1.ScriptDescriptor{
Filename: "recovery.ps1", SizeBytes: uint64(len(body)), Sha256: digest[:],
}}
}
encoded, err := proto.Marshal(spec)
if err != nil {
t.Fatal(err)
}
issue := fixedStoreIssue(0xd2)
now := time.Now().UTC()
input := QueueCommandInput{IssueUUID: issue, ClientID: "recovery-client", IssueTime: now,
ReceiptTime: now, ImmutableSHA256: sha256.Sum256(encoded), ExecutionSpec: encoded,
ScriptPresent: scenario.script}
if scenario.script {
input.ScriptContent = body
}
if _, err := opened.QueueCommand(ctx, input); err != nil {
t.Fatal(err)
}
exec := func(query string, args ...any) {
t.Helper()
if _, err := opened.DB().ExecContext(ctx, query, args...); err != nil {
t.Fatal(err)
}
}
switch scenario.alter {
case "uncompressed":
exec(`UPDATE commands SET execution_spec = ?, execution_spec_stored_bytes = ?, execution_spec_compression = 1`, encoded, len(encoded))
case "missing":
exec(`DELETE FROM command_payloads`)
case "command", "descriptor", "opaque":
if scenario.alter == "command" {
spec.Source = nil
} else if scenario.alter == "descriptor" {
spec.GetScript().Sha256 = make([]byte, sha256.Size)
}
changed, err := proto.Marshal(spec)
if err != nil {
t.Fatal(err)
}
if scenario.alter == "opaque" {
changed = []byte{0xff}
}
stored, err := compressCommandSpec(changed)
if err != nil {
t.Fatal(err)
}
exec(`UPDATE commands SET execution_spec = ?, execution_spec_raw_bytes = ?, execution_spec_stored_bytes = ?`, stored, len(changed), len(stored))
case "frame":
exec(`UPDATE commands SET execution_spec = zeroblob(execution_spec_stored_bytes)`)
case "size":
exec(`UPDATE commands SET execution_spec_stored_bytes = execution_spec_stored_bytes + 1`)
}
if err := opened.Close(); err != nil {
t.Fatal(err)
}
opened, err = Open(ctx, options)
if err != nil {
t.Fatal(err)
}
// The deadline makes a nested-query regression fail promptly instead
// of hanging the test or leaving startup permanently unready.
recoveryContext, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
_, err = opened.RecoverCommandSegments(recoveryContext)
if scenario.wantBroken {
if !errors.Is(err, ErrCommittedRangeMissing) {
t.Fatalf("recovery = %v, want integrity failure", err)
}
} else if err != nil {
t.Fatal(err)
}
if _, err := opened.Telemetry(ctx); err != nil {
t.Fatalf("recovery did not release SQLite connection: %v", err)
}
})
}
}