feat: complete Windows client control and recovery paths

This commit is contained in:
2026-09-06 13:37:32 +00:00
parent 56b15c7f4f
commit 486894557d
38 changed files with 2188 additions and 106 deletions
+59 -1
View File
@@ -6,7 +6,9 @@ import (
"errors"
"time"
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/domain"
"google.golang.org/protobuf/proto"
)
var ErrDispatchMismatch = errors.New("command acceptance does not match dispatched generation or revision")
@@ -15,9 +17,31 @@ var ErrDispatchMismatch = errors.New("command acceptance does not match dispatch
// produced it. Repeated acknowledgements are harmless; a stale session cannot
// advance a newer dispatch or overwrite a terminal decision.
func (store *Store) RecordCommandAcceptance(ctx context.Context, issueUUID domain.UUID, clientID string, generation, revision uint64, accepted bool, now time.Time) (bool, error) {
return store.RecordCommandAcceptanceWithRejection(ctx, issueUUID, clientID, generation, revision, accepted, nil, now)
}
// RecordCommandAcceptanceWithRejection is the wire-facing variant that keeps
// a structured client admission rejection on the command row for status and
// history. The legacy method above remains convenient for reconciliation and
// tests that only have the accepted bit.
func (store *Store) RecordCommandAcceptanceWithRejection(ctx context.Context, issueUUID domain.UUID, clientID string, generation, revision uint64, accepted bool, rejection *rvboxv1.ControlError, now time.Time) (bool, error) {
if isZeroUUID([16]byte(issueUUID)) || clientID == "" || generation == 0 || revision == 0 || now.IsZero() {
return false, ErrDispatchMismatch
}
if accepted && rejection != nil {
return false, ErrDispatchMismatch
}
var storedRejection []byte
var err error
if rejection != nil {
if len(rejection.GetMessage()) > 4096 || len(rejection.GetIssueUuid()) > 128 {
return false, ErrDispatchMismatch
}
storedRejection, err = proto.MarshalOptions{Deterministic: true}.Marshal(rejection)
if err != nil || len(storedRejection) > 16<<10 {
return false, ErrDispatchMismatch
}
}
store.writeMu.Lock()
defer store.writeMu.Unlock()
database, err := store.openDatabase()
@@ -28,15 +52,37 @@ func (store *Store) RecordCommandAcceptance(ctx context.Context, issueUUID domai
if !accepted {
targetLifecycle = 11
}
tx, err := database.BeginTx(ctx, nil)
if err != nil {
return false, err
}
defer tx.Rollback()
var queueExpiry sql.NullInt64
var lateAfterExpiry int
if err := tx.QueryRowContext(ctx, `SELECT queue_expiry_time, late_after_expiry FROM commands WHERE issue_uuid = ? AND client_id = ? AND target_session_generation = ? AND revision = ?`, issueUUID[:], clientID, generation, revision).Scan(&queueExpiry, &lateAfterExpiry); errors.Is(err, sql.ErrNoRows) {
_ = tx.Rollback()
return false, ErrDispatchMismatch
} else if err != nil {
return false, err
}
query := `UPDATE commands SET lifecycle = ?`
args := []any{targetLifecycle}
if !accepted {
query += `, terminal_time = ?`
args = append(args, now.UTC().UnixNano())
}
if rejection != nil {
query += `, rejection = ?`
args = append(args, storedRejection)
}
// The command may have been dispatched just before its queue deadline and
// its acceptance can arrive after that deadline. Preserve the eventual
// client truth, but durably retain the contradiction for status/audit.
query += `, late_after_expiry = CASE WHEN queue_expiry_time IS NOT NULL AND queue_expiry_time <= ? THEN 1 ELSE late_after_expiry END`
args = append(args, now.UTC().UnixNano())
query += ` WHERE issue_uuid = ? AND client_id = ? AND lifecycle = 2 AND target_session_generation = ? AND revision = ?`
args = append(args, issueUUID[:], clientID, generation, revision)
result, err := database.ExecContext(ctx, query, args...)
result, err := tx.ExecContext(ctx, query, args...)
if err != nil {
return false, err
}
@@ -45,8 +91,20 @@ func (store *Store) RecordCommandAcceptance(ctx context.Context, issueUUID domai
return false, err
}
if changed == 1 {
late := queueExpiry.Valid && now.UTC().UnixNano() >= queueExpiry.Int64
if late && lateAfterExpiry == 0 {
if err := appendLateExpiryAudit(ctx, tx, now.UTC().UnixNano(), clientID, [16]byte(issueUUID), now.UTC().UnixNano()); err != nil {
return false, err
}
}
if err := tx.Commit(); err != nil {
return false, err
}
return true, nil
}
if err := tx.Commit(); err != nil {
return false, err
}
var lifecycle uint32
var storedGeneration, storedRevision uint64
err = database.QueryRowContext(ctx, `SELECT lifecycle, COALESCE(target_session_generation, 0), revision FROM commands WHERE issue_uuid = ? AND client_id = ?`, issueUUID[:], clientID).Scan(&lifecycle, &storedGeneration, &storedRevision)
+21
View File
@@ -0,0 +1,21 @@
package store
import (
"context"
"crypto/sha256"
"database/sql"
"fmt"
)
// appendLateExpiryAudit records the first observed queue-deadline
// contradiction without copying command text, script bodies, or output into
// the separate audit budget. The command row remains the fast status path;
// this row is the durable operator/audit history.
func appendLateExpiryAudit(ctx context.Context, tx *sql.Tx, occurredAt int64, clientID string, issue [16]byte, receiptUnixNano int64) error {
payload := []byte(fmt.Sprintf("receipt_unix_nano=%d", receiptUnixNano))
digest := sha256.Sum256(payload)
_, err := tx.ExecContext(ctx, `INSERT INTO audit_events (
occurred_at, source, client_id, issue_uuid, action, outcome, compression, payload, raw_bytes, stored_bytes, sha256
) VALUES (?, 'session', ?, ?, 'late_after_expiry', 'observed', 1, ?, ?, ?, ?)`, occurredAt, clientID, issue[:], payload, len(payload), len(payload), digest[:])
return err
}
+64
View File
@@ -126,6 +126,51 @@ func TestClaimDispatchExpiresAndFencesRequeue_HP_DISPATCH_02(t *testing.T) {
}
}
func TestLateDispatchAcceptanceAndEventRetainExpiryContradiction_BH_DISPATCH_09(t *testing.T) {
t.Parallel()
ctx := context.Background()
opened, err := Open(ctx, Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = opened.Close() })
base := time.Date(2026, time.September, 6, 13, 0, 0, 0, time.UTC)
if _, err := opened.RegisterClientSession(ctx, ClientRegistration{
ClientID: "late-client", Platform: 2, Architecture: "amd64", DaemonVersion: "test", DaemonCWD: `C:\`, SupportedShells: []byte{1},
ClientInstanceID: [16]byte{21}, SessionID: [16]byte{22}, ConnectedAt: base,
}); err != nil {
t.Fatal(err)
}
issue := fixedStoreIssue(0xd6)
expiry := base.Add(time.Second)
if _, err := opened.QueueCommand(ctx, QueueCommandInput{IssueUUID: issue, ClientID: "late-client", IssueTime: base, ReceiptTime: base, QueueExpiryTime: &expiry, ImmutableSHA256: sha256.Sum256([]byte("late")), ExecutionSpec: []byte("spec")}); err != nil {
t.Fatal(err)
}
if candidate, err := opened.ClaimNextDispatch(ctx, "late-client", 7, base.Add(500*time.Millisecond)); err != nil || candidate == nil {
t.Fatalf("dispatch claim = %#v, %v", candidate, err)
}
if changed, err := opened.RecordCommandAcceptance(ctx, issue, "late-client", 7, 1, true, base.Add(2*time.Second)); err != nil || !changed {
t.Fatalf("late acceptance = %t, %v", changed, err)
}
var late int
if err := opened.DB().QueryRow(`SELECT late_after_expiry FROM commands WHERE issue_uuid = ?`, issue[:]).Scan(&late); err != nil || late != 1 {
t.Fatalf("late acceptance flag = %d, %v", late, err)
}
var audits int
if err := opened.DB().QueryRow(`SELECT COUNT(*) FROM audit_events WHERE action = 'late_after_expiry' AND issue_uuid = ?`, issue[:]).Scan(&audits); err != nil || audits != 1 {
t.Fatalf("late acceptance audit rows = %d, %v", audits, err)
}
if _, err := opened.AppendCommandEvent(ctx, EventAppend{IssueUUID: [16]byte(issue), ClientID: "late-client", SessionGeneration: 7, EventSeq: 1, ObservedUnixNano: base.Add(2 * time.Second).UnixNano(), ReceiptUnixNano: base.Add(2 * time.Second).UnixNano(), EventType: 5, Stream: 1, Compression: 1, RawLength: 1, Payload: []byte("x"), ImmutableSHA256: sha256.Sum256([]byte("event")), Output: true}); err != nil {
t.Fatal(err)
}
if err := opened.DB().QueryRow(`SELECT late_after_expiry FROM commands WHERE issue_uuid = ?`, issue[:]).Scan(&late); err != nil || late != 1 {
t.Fatalf("late event flag = %d, %v", late, err)
}
if err := opened.DB().QueryRow(`SELECT COUNT(*) FROM audit_events WHERE action = 'late_after_expiry' AND issue_uuid = ?`, issue[:]).Scan(&audits); err != nil || audits != 1 {
t.Fatalf("late event duplicated audit rows = %d, %v", audits, err)
}
}
func TestQueueAndClaimScriptPayloadIsDurable_HP_SCRIPT_02(t *testing.T) {
t.Parallel()
ctx := context.Background()
@@ -195,6 +240,25 @@ func TestRecordCommandAcceptanceFencesGeneration_HP_DISPATCH_05(t *testing.T) {
if changed, err := opened.RecordCommandAcceptance(ctx, issue, "win-client", 3, 1, true, time.Now()); err != nil || changed {
t.Fatalf("duplicate acceptance = %t, %v", changed, err)
}
rejectedIssue := fixedStoreIssue(0x96)
if _, err := opened.QueueCommand(ctx, QueueCommandInput{IssueUUID: rejectedIssue, ClientID: "win-client", IssueTime: time.Now(), ReceiptTime: time.Now(), ImmutableSHA256: sha256.Sum256([]byte("reject")), ExecutionSpec: []byte("spec")}); err != nil {
t.Fatal(err)
}
if _, err := opened.ClaimNextDispatch(ctx, "win-client", 3, time.Now()); err != nil {
t.Fatal(err)
}
rejection := &rvboxv1.ControlError{Code: rvboxv1.ControlError_INVALID_ARGUMENT, Message: "invalid command"}
if changed, err := opened.RecordCommandAcceptanceWithRejection(ctx, rejectedIssue, "win-client", 3, 1, false, rejection, time.Now()); err != nil || !changed {
t.Fatalf("rejection acceptance = %t, %v", changed, err)
}
view, err := opened.GetCommandView(ctx, "win-client", rejectedIssue)
if err != nil {
t.Fatal(err)
}
stored := &rvboxv1.ControlError{}
if err := proto.Unmarshal(view.Rejection, stored); err != nil || stored.GetCode() != rejection.GetCode() || stored.GetMessage() != rejection.GetMessage() {
t.Fatalf("stored rejection = %v, %v", stored, err)
}
}
func timePtr(value time.Time) *time.Time { return &value }
+32 -4
View File
@@ -50,6 +50,7 @@ type EventAppend struct {
ImmutableSHA256 [32]byte
Lifecycle *rvboxv1.CommandLifecycle
LifecycleRevision uint64
WindowsIdentity *rvboxv1.WindowsExecutionIdentity
Output bool
UseCloseout bool
}
@@ -79,6 +80,16 @@ func (store *Store) AppendCommandEvent(ctx context.Context, event EventAppend) (
if err != nil {
return result, err
}
var windowsIdentity []byte
if event.WindowsIdentity != nil {
if len(event.WindowsIdentity.GetAttemptedContexts()) > 8 || len(event.WindowsIdentity.GetSelectionDetail()) > 4096 {
return result, ErrInvalidSegmentRecord
}
windowsIdentity, err = proto.MarshalOptions{Deterministic: true}.Marshal(event.WindowsIdentity)
if err != nil || len(windowsIdentity) > 16<<10 {
return result, ErrInvalidSegmentRecord
}
}
store.writeMu.Lock()
defer store.writeMu.Unlock()
@@ -89,10 +100,13 @@ func (store *Store) AppendCommandEvent(ctx context.Context, event EventAppend) (
var lastSequence, commandOutputCharged, commandCharged, clientCharged, serverCharged, closeoutRemaining uint64
var currentLifecycle uint32
var currentRevision uint64
var lateAfterExpiry int
var queueExpiry sql.NullInt64
var clientID string
query := `SELECT commands.last_event_seq, commands.output_charged_bytes,
commands.charged_bytes, commands.closeout_remaining_bytes, commands.client_id, clients.charged_bytes,
storage_counters.command_charged_bytes, commands.lifecycle, commands.revision
storage_counters.command_charged_bytes, commands.lifecycle, commands.revision,
commands.queue_expiry_time, commands.late_after_expiry
FROM commands JOIN clients ON clients.client_id = commands.client_id
JOIN storage_counters ON storage_counters.singleton = 1 WHERE commands.issue_uuid = ?`
arguments := []any{event.IssueUUID[:]}
@@ -105,7 +119,7 @@ JOIN storage_counters ON storage_counters.singleton = 1 WHERE commands.issue_uui
arguments = append(arguments, event.SessionGeneration)
}
err = database.QueryRowContext(ctx, query, arguments...).Scan(
&lastSequence, &commandOutputCharged, &commandCharged, &closeoutRemaining, &clientID, &clientCharged, &serverCharged, &currentLifecycle, &currentRevision)
&lastSequence, &commandOutputCharged, &commandCharged, &closeoutRemaining, &clientID, &clientCharged, &serverCharged, &currentLifecycle, &currentRevision, &queueExpiry, &lateAfterExpiry)
if errors.Is(err, sql.ErrNoRows) {
return result, ErrCommandNotFound
}
@@ -133,6 +147,11 @@ JOIN storage_counters ON storage_counters.singleton = 1 WHERE commands.issue_uui
return result, domain.ValidateTransition(rvboxv1.CommandLifecycle(currentLifecycle), *event.Lifecycle)
}
}
late := queueExpiry.Valid && event.ReceiptUnixNano >= queueExpiry.Int64
lateValue := 0
if late {
lateValue = 1
}
rowsCharged, indexesCharged := uint64(1), uint64(1)
if store.willCreateSegment(event.IssueUUID, uint64(len(encoded))) {
rowsCharged++
@@ -215,8 +234,12 @@ payload, segment_ordinal, segment_record_offset, segment_record_length, immutabl
var update sql.Result
commandUpdate := `UPDATE commands SET last_event_seq = ?,
retained_compressed_bytes = retained_compressed_bytes + ?, output_charged_bytes = ?, charged_bytes = ?,
closeout_remaining_bytes = ?`
commandArgs := []any{event.EventSeq, len(event.Payload), reservation.CommandOutputCharged, reservation.CommandTotalCharged, reservation.CloseoutRemaining}
closeout_remaining_bytes = ?, late_after_expiry = CASE WHEN ? = 1 THEN 1 ELSE late_after_expiry END`
commandArgs := []any{event.EventSeq, len(event.Payload), reservation.CommandOutputCharged, reservation.CommandTotalCharged, reservation.CloseoutRemaining, lateValue}
if event.WindowsIdentity != nil {
commandUpdate = `UPDATE commands SET windows_execution_identity = ?, ` + commandUpdate[len("UPDATE commands SET "):]
commandArgs = append([]any{windowsIdentity}, commandArgs...)
}
if event.Lifecycle != nil {
terminal := 0
if domain.IsTerminal(*event.Lifecycle) {
@@ -236,6 +259,11 @@ retained_compressed_bytes = retained_compressed_bytes + ?, output_charged_bytes
}
}
}
if err == nil && late && lateAfterExpiry == 0 {
if err = appendLateExpiryAudit(ctx, tx, event.ReceiptUnixNano, clientID, event.IssueUUID, event.ReceiptUnixNano); err == nil {
lateAfterExpiry = 1
}
}
if err == nil && event.EventType == 7 {
// Stdin acknowledgements are command events, but their durable delivery
// cursor lives in stdin_writes. Mark the cumulative prefix in the same
+13 -1
View File
@@ -34,10 +34,22 @@ func TestReadCommandEventsValidatesSegmentAndPreservesOrder_HP_EVENT_04(t *testi
}
for sequence, lifecycle := range []rvboxv1.CommandLifecycle{rvboxv1.CommandLifecycle_COMMAND_RUNNING, rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED} {
event := []byte{byte(sequence + 1), 'e', 'v', 't'}
if _, err := opened.AppendCommandEvent(ctx, EventAppend{IssueUUID: [16]byte(issue), ClientID: "win-client", SessionGeneration: 1, EventSeq: uint64(sequence + 1), ObservedUnixNano: now.UnixNano(), ReceiptUnixNano: now.Add(time.Duration(sequence) * time.Millisecond).UnixNano(), EventType: 4, Compression: 1, RawLength: uint64(len(event)), Payload: event, ImmutableSHA256: sha256.Sum256(event), Lifecycle: &lifecycle, LifecycleRevision: 1}); err != nil {
var identity *rvboxv1.WindowsExecutionIdentity
if sequence == 0 {
effective := rvboxv1.WindowsExecutionContext_WINDOWS_EXECUTION_CONTEXT_ACTIVE_USER
identity = &rvboxv1.WindowsExecutionIdentity{EffectiveContext: &effective, AttemptedContexts: []rvboxv1.WindowsExecutionContext{effective}, SelectionDetail: "selected"}
}
if _, err := opened.AppendCommandEvent(ctx, EventAppend{IssueUUID: [16]byte(issue), ClientID: "win-client", SessionGeneration: 1, EventSeq: uint64(sequence + 1), ObservedUnixNano: now.UnixNano(), ReceiptUnixNano: now.Add(time.Duration(sequence) * time.Millisecond).UnixNano(), EventType: 4, Compression: 1, RawLength: uint64(len(event)), Payload: event, ImmutableSHA256: sha256.Sum256(event), Lifecycle: &lifecycle, LifecycleRevision: 1, WindowsIdentity: identity}); err != nil {
t.Fatal(err)
}
}
view, err := opened.GetCommandView(ctx, "win-client", issue)
if err != nil {
t.Fatal(err)
}
if len(view.WindowsIdentity) == 0 {
t.Fatal("running lifecycle did not persist Windows identity")
}
events, err := opened.ReadCommandEvents(ctx, issue, 0, 10)
if err != nil || len(events) != 2 || events[0].EventSeq != 1 || events[1].EventSeq != 2 || string(events[1].Payload) != "\x02evt" {
t.Fatalf("read events = %#v, %v", events, err)
+16
View File
@@ -16,6 +16,8 @@ type migration struct {
var migrations = []migration{
{version: 1, sql: schemaV1},
{version: 2, sql: schemaV2},
{version: 3, sql: schemaV3},
{version: 4, sql: schemaV4},
}
func applyMigrations(ctx context.Context, db *sql.DB) error {
@@ -196,3 +198,17 @@ CREATE TABLE signal_intents (
) STRICT;
CREATE INDEX signal_intents_pending ON signal_intents(issue_uuid, command_revision, signal, acknowledged, created_at, request_uuid);
`
// schemaV3 records the accepted/executed-after-deadline contradiction on the
// command itself. It is separate from the public lifecycle: the eventual
// client-reported terminal result remains authoritative while operators can
// still see that acceptance or execution arrived after the queue deadline.
const schemaV3 = `
ALTER TABLE commands ADD COLUMN late_after_expiry INTEGER NOT NULL DEFAULT 0 CHECK(late_after_expiry IN (0,1));
`
// schemaV4 retains the structured client admission rejection so status and
// history do not have to reconstruct an error from a free-form event detail.
const schemaV4 = `
ALTER TABLE commands ADD COLUMN rejection BLOB;
`
+9 -3
View File
@@ -46,6 +46,8 @@ type CommandView struct {
OutputTruncated bool
OutputIncomplete bool
RetainedCompressedBytes uint64
LateAfterExpiry bool
Rejection []byte
ExecutionSpec []byte
WindowsIdentity []byte
}
@@ -182,7 +184,7 @@ func (store *Store) ListCommandViews(ctx context.Context, page CommandPage) ([]C
query := `SELECT issue_uuid, client_id, issue_time, server_receipt_time,
queue_expiry_time, terminal_time, lifecycle, revision, last_event_seq,
exit_code, output_truncated, output_incomplete, retained_compressed_bytes,
execution_spec, execution_spec_raw_bytes, windows_execution_identity
late_after_expiry, rejection, execution_spec, execution_spec_raw_bytes, windows_execution_identity
FROM commands WHERE client_id = ? AND issue_time <= ?`
args := []any{page.ClientID, page.SnapshotBoundary}
if !page.IncludeTerminal {
@@ -231,7 +233,7 @@ func (store *Store) GetCommandView(ctx context.Context, clientID string, issue d
query := `SELECT issue_uuid, client_id, issue_time, server_receipt_time,
queue_expiry_time, terminal_time, lifecycle, revision, last_event_seq,
exit_code, output_truncated, output_incomplete, retained_compressed_bytes,
execution_spec, execution_spec_raw_bytes, windows_execution_identity
late_after_expiry, rejection, execution_spec, execution_spec_raw_bytes, windows_execution_identity
FROM commands WHERE issue_uuid = ?`
args := []any{issue[:]}
if clientID != "" {
@@ -252,10 +254,12 @@ func scanCommandView(scanner interface{ Scan(...any) error }) (CommandView, erro
var expiry, terminal sql.NullInt64
var exit sql.NullInt64
var truncated, incomplete int
var lateAfterExpiry int
var rejection []byte
var rawBytes uint64
if err := scanner.Scan(&issue, &view.ClientID, &issueTime, &receipt, &expiry, &terminal,
&view.Lifecycle, &view.Revision, &view.LastEventSeq, &exit, &truncated, &incomplete,
&view.RetainedCompressedBytes, &stored, &rawBytes, &view.WindowsIdentity); err != nil {
&view.RetainedCompressedBytes, &lateAfterExpiry, &rejection, &stored, &rawBytes, &view.WindowsIdentity); err != nil {
return view, err
}
if len(issue) != 16 {
@@ -267,6 +271,8 @@ func scanCommandView(scanner interface{ Scan(...any) error }) (CommandView, erro
view.QueueExpiryTime = nullableTime(expiry)
view.TerminalTime = nullableTime(terminal)
view.OutputTruncated, view.OutputIncomplete = truncated == 1, incomplete == 1
view.LateAfterExpiry = lateAfterExpiry == 1
view.Rejection = append([]byte(nil), rejection...)
if exit.Valid {
if exit.Int64 < -1<<31 || exit.Int64 > 1<<31-1 {
return view, ErrInvalidSegmentRecord