fix(server): avoid deadlock during command payload recovery
This commit is contained in:
1 parent
8d28925e44
commit
ef6e65e6f1
4 files changed
+186
-23
No files matched your search
@@ -0,0 +1,136 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
func TestPayloadRecoveryAfterRestart_REC_STORE_03(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, scenario := range []struct {
|
||||
name string
|
||||
script bool
|
||||
empty bool
|
||||
alter string
|
||||
wantBroken bool
|
||||
}{
|
||||
{name: "command"},
|
||||
{name: "script", script: true},
|
||||
{name: "empty script", script: true, empty: true},
|
||||
{name: "uncompressed command", alter: "uncompressed"},
|
||||
{name: "uncompressed script", script: true, alter: "uncompressed"},
|
||||
{name: "missing script", script: true, alter: "missing", wantBroken: true},
|
||||
{name: "orphan script", script: true, alter: "command", wantBroken: true},
|
||||
{name: "wrong descriptor", script: true, alter: "descriptor", wantBroken: true},
|
||||
{name: "damaged frame", alter: "frame", wantBroken: true},
|
||||
{name: "wrong stored size", alter: "size", wantBroken: true},
|
||||
{name: "opaque descriptor with script", script: true, alter: "opaque", wantBroken: true},
|
||||
} {
|
||||
t.Run(scenario.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
options := Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second}
|
||||
opened, err := Open(ctx, options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = opened.Close() })
|
||||
if _, err := opened.RegisterClientSession(ctx, ClientRegistration{
|
||||
ClientID: "recovery-client", Platform: 2, Architecture: "amd64", DaemonVersion: "test",
|
||||
DaemonCWD: `C:\`, SupportedShells: []byte{1}, ClientInstanceID: [16]byte{1},
|
||||
SessionID: [16]byte{2}, ConnectedAt: time.Now().UTC(),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("Write-Output 'recovery'\r\n")
|
||||
if scenario.empty {
|
||||
body = []byte{}
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
spec := &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_POWERSHELL}
|
||||
if scenario.script {
|
||||
spec.Source = &rvboxv1.ExecutionSpec_Script{Script: &rvboxv1.ScriptDescriptor{
|
||||
Filename: "recovery.ps1", SizeBytes: uint64(len(body)), Sha256: digest[:],
|
||||
}}
|
||||
}
|
||||
encoded, err := proto.Marshal(spec)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
issue := fixedStoreIssue(0xd2)
|
||||
now := time.Now().UTC()
|
||||
input := QueueCommandInput{IssueUUID: issue, ClientID: "recovery-client", IssueTime: now,
|
||||
ReceiptTime: now, ImmutableSHA256: sha256.Sum256(encoded), ExecutionSpec: encoded,
|
||||
ScriptPresent: scenario.script}
|
||||
if scenario.script {
|
||||
input.ScriptContent = body
|
||||
}
|
||||
if _, err := opened.QueueCommand(ctx, input); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exec := func(query string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := opened.DB().ExecContext(ctx, query, args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
switch scenario.alter {
|
||||
case "uncompressed":
|
||||
exec(`UPDATE commands SET execution_spec = ?, execution_spec_stored_bytes = ?, execution_spec_compression = 1`, encoded, len(encoded))
|
||||
case "missing":
|
||||
exec(`DELETE FROM command_payloads`)
|
||||
case "command", "descriptor", "opaque":
|
||||
if scenario.alter == "command" {
|
||||
spec.Source = nil
|
||||
} else if scenario.alter == "descriptor" {
|
||||
spec.GetScript().Sha256 = make([]byte, sha256.Size)
|
||||
}
|
||||
changed, err := proto.Marshal(spec)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scenario.alter == "opaque" {
|
||||
changed = []byte{0xff}
|
||||
}
|
||||
stored, err := compressCommandSpec(changed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exec(`UPDATE commands SET execution_spec = ?, execution_spec_raw_bytes = ?, execution_spec_stored_bytes = ?`, stored, len(changed), len(stored))
|
||||
case "frame":
|
||||
exec(`UPDATE commands SET execution_spec = zeroblob(execution_spec_stored_bytes)`)
|
||||
case "size":
|
||||
exec(`UPDATE commands SET execution_spec_stored_bytes = execution_spec_stored_bytes + 1`)
|
||||
}
|
||||
if err := opened.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opened, err = Open(ctx, options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The deadline makes a nested-query regression fail promptly instead
|
||||
// of hanging the test or leaving startup permanently unready.
|
||||
recoveryContext, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
_, err = opened.RecoverCommandSegments(recoveryContext)
|
||||
if scenario.wantBroken {
|
||||
if !errors.Is(err, ErrCommittedRangeMissing) {
|
||||
t.Fatalf("recovery = %v, want integrity failure", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := opened.Telemetry(ctx); err != nil {
|
||||
t.Fatalf("recovery did not release SQLite connection: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user