feat: execute durable client commands through supervisor

internal/client/spool/events.go

@@ -8,9 +8,13 @@ import (
 	"errors"
 	"fmt"
 	"time"
+	"unicode/utf8"
 
 	"github.com/klauspost/compress/zstd"
+	rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
 	"github.com/rvbox/rvbox/internal/domain"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 type Command struct {
@@ -23,6 +27,11 @@ type Command struct {
 	// server. It is retained as raw protobuf bytes so the runtime can validate
 	// and execute exactly the admitted request after a restart.
 	ExecutionSpec []byte
+	// Script carries the immutable descriptor reservation for a script-backed
+	// command. Its bytes arrive later through AppendScriptChunk; creating the
+	// descriptor row in the same transaction as acceptance prevents an
+	// accepted command from being left without upload state after a crash.
+	Script *ScriptDescriptor
 }
 
 type Acceptance struct {
@@ -63,7 +72,7 @@ func (store *Store) AcceptCommand(ctx context.Context, command Command, accepted
 		return Acceptance{}, errors.New("invalid command acceptance")
 	}
 	var storedSpec []byte
-	var specCharge uint64
+	var specCharge, scriptCharge uint64
 	if len(command.ExecutionSpec) > 0 {
 		if uint64(len(command.ExecutionSpec)) > store.maxExecutionSpecBytes {
 			return Acceptance{}, errors.New("execution specification exceeds client limit")
@@ -78,6 +87,21 @@ func (store *Store) AcceptCommand(ctx context.Context, command Command, accepted
 			return Acceptance{}, err
 		}
 	}
+	var storedScript []byte
+	if command.Script != nil {
+		if err := validateAcceptedScript(command.ExecutionSpec, *command.Script, store.maxScriptBytes); err != nil {
+			return Acceptance{}, err
+		}
+		var err error
+		storedScript, err = compressScript(nil)
+		if err != nil {
+			return Acceptance{}, err
+		}
+		scriptCharge, err = EstimateCharge(ChargeInput{EncodedBytes: uint64(len(storedScript)), SQLiteRows: 1, IndexEntries: 1})
+		if err != nil {
+			return Acceptance{}, err
+		}
+	}
 	tx, err := store.db.BeginTx(ctx, nil)
 	if err != nil {
 		return Acceptance{}, err
@@ -116,6 +140,9 @@ func (store *Store) AcceptCommand(ctx context.Context, command Command, accepted
 		return Acceptance{}, err
 	}
 	charge, overflow := addChecked(baseCharge, specCharge)
+	if !overflow {
+		charge, overflow = addChecked(charge, scriptCharge)
+	}
 	if overflow {
 		return Acceptance{}, &CapacityError{Tier: CapacityTierHardMaximum, Requested: ^uint64(0), Available: store.quotaLimits.HardAllocationBytes}
 	}
@@ -136,12 +163,32 @@ func (store *Store) AcceptCommand(ctx context.Context, command Command, accepted
 			return Acceptance{}, err
 		}
 	}
+	if command.Script != nil {
+		if _, err := tx.ExecContext(ctx, `INSERT INTO scripts(issue_uuid, declared_raw_bytes, declared_sha256, stored_bytes, compression, stored_data, charged_bytes) VALUES (?, ?, ?, ?, 2, ?, ?)`, command.IssueUUID[:], command.Script.SizeBytes, command.Script.SHA256[:], len(storedScript), storedScript, scriptCharge); err != nil {
+			return Acceptance{}, err
+		}
+	}
 	if err := tx.Commit(); err != nil {
 		return Acceptance{}, err
 	}
 	return Acceptance{Command: command}, nil
 }
 
+func validateAcceptedScript(encodedSpec []byte, descriptor ScriptDescriptor, maximum uint64) error {
+	if descriptor.SizeBytes > maximum {
+		return ErrScriptBounds
+	}
+	var spec rvboxv1.ExecutionSpec
+	if err := proto.Unmarshal(encodedSpec, &spec); err != nil {
+		return ErrScriptConflict
+	}
+	declared := spec.GetScript()
+	if declared == nil || declared.GetSizeBytes() != descriptor.SizeBytes || len(declared.GetSha256()) != sha256.Size || !bytes.Equal(declared.GetSha256(), descriptor.SHA256[:]) {
+		return ErrScriptConflict
+	}
+	return nil
+}
+
 // GetCommand returns the durable command metadata and its immutable execution
 // specification. The returned protobuf bytes are a copy and can be decoded or
 // modified by the runtime without changing the spool's source of truth.
@@ -340,6 +387,156 @@ func (store *Store) MarkTerminal(ctx context.Context, issueUUID domain.UUID, pha
 	return nil
 }
 
+// LaunchEvidence is the durable pre/post-authorization record used to fence
+// uncertain OS launches across daemon restarts. PID is evidence only and is
+// never sufficient for recovery-time signalling without a native creation
+// identity check.
+type LaunchEvidence struct {
+	Phase   domain.LaunchPhase
+	Context string
+	PID     uint32
+}
+
+func (store *Store) SetLaunchPhase(ctx context.Context, issueUUID domain.UUID, phase domain.LaunchPhase, contextName string, pid uint32) error {
+	if !validUUID(issueUUID) || phase > domain.LaunchPhaseAuthorized || len(contextName) > 128 || !utf8.ValidString(contextName) {
+		return errors.New("invalid launch barrier")
+	}
+	if phase == domain.LaunchPhaseNone {
+		contextName, pid = "", 0
+	}
+	result, err := store.db.ExecContext(ctx, `UPDATE commands SET launch_phase = ?, launch_context = ?, launch_pid = ? WHERE issue_uuid = ? AND terminal = 0`, uint32(phase), contextName, pid, issueUUID[:])
+	if err != nil {
+		return err
+	}
+	count, err := result.RowsAffected()
+	if err != nil {
+		return err
+	}
+	if count == 0 {
+		return ErrUnknownCommand
+	}
+	return nil
+}
+
+// RecoverLaunchUncertainty converts every command whose authorization barrier
+// crossed before a restart into one durable interrupted terminal event. The
+// Windows Job kill-on-close guarantee makes this safe: a surviving process is
+// never redispatched, and recovery never signals a PID by itself.
+func (store *Store) RecoverLaunchUncertainty(ctx context.Context, now time.Time) ([]domain.UUID, error) {
+	if now.IsZero() {
+		return nil, errors.New("launch recovery time is required")
+	}
+	rows, err := store.db.QueryContext(ctx, `SELECT issue_uuid, command_revision FROM commands WHERE launch_phase = 2 AND terminal = 0 ORDER BY accepted_at, issue_uuid`)
+	if err != nil {
+		return nil, err
+	}
+	type pending struct {
+		issue    domain.UUID
+		revision uint64
+	}
+	var pendingRows []pending
+	for rows.Next() {
+		var encoded []byte
+		var revision uint64
+		if err := rows.Scan(&encoded, &revision); err != nil {
+			_ = rows.Close()
+			return nil, err
+		}
+		var issue domain.UUID
+		if len(encoded) != len(issue) {
+			_ = rows.Close()
+			return nil, ErrScriptState
+		}
+		copy(issue[:], encoded)
+		if !validUUID(issue) || revision == 0 {
+			_ = rows.Close()
+			return nil, ErrScriptState
+		}
+		pendingRows = append(pendingRows, pending{issue: issue, revision: revision})
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	interrupted := make([]domain.UUID, 0, len(pendingRows))
+	for _, item := range pendingRows {
+		if _, err := store.AppendLifecycle(ctx, item.issue, uint32(rvboxv1.CommandLifecycle_COMMAND_INTERRUPTED), item.revision, "uncertain launch recovered after daemon restart", now); err != nil {
+			return interrupted, err
+		}
+		interrupted = append(interrupted, item.issue)
+	}
+	return interrupted, nil
+}
+
+// AppendLifecycle atomically advances the durable command phase and appends
+// its lifecycle event. Keeping these writes in one transaction prevents a
+// crash between a terminal marker and its public event from creating a state
+// that can be replayed as a second execution.
+func (store *Store) AppendLifecycle(ctx context.Context, issueUUID domain.UUID, phase uint32, revision uint64, detail string, observedAt time.Time) (Event, error) {
+	if !validUUID(issueUUID) || phase == 0 || phase > 11 || observedAt.IsZero() || revision == 0 {
+		return Event{}, errors.New("invalid lifecycle event")
+	}
+	if len(detail) > 4096 || !utf8.ValidString(detail) {
+		return Event{}, errors.New("lifecycle detail is invalid or too large")
+	}
+	tx, err := store.db.BeginTx(ctx, nil)
+	if err != nil {
+		return Event{}, err
+	}
+	defer tx.Rollback()
+	var current uint32
+	var storedRevision uint64
+	var nextOrdinal, outputCharged, totalCharged, closeout uint64
+	if err := tx.QueryRowContext(ctx, `SELECT phase, command_revision, next_local_ordinal, output_charged_bytes, total_charged_bytes, closeout_remaining_bytes FROM commands WHERE issue_uuid = ?`, issueUUID[:]).Scan(&current, &storedRevision, &nextOrdinal, &outputCharged, &totalCharged, &closeout); err == sql.ErrNoRows {
+		return Event{}, ErrUnknownCommand
+	} else if err != nil {
+		return Event{}, err
+	}
+	if storedRevision != revision {
+		return Event{}, ErrCommandConflict
+	}
+	if current == phase {
+		return Event{}, nil
+	}
+	if !domain.CanTransition(rvboxv1.CommandLifecycle(current), rvboxv1.CommandLifecycle(phase)) {
+		return Event{}, fmt.Errorf("invalid lifecycle transition %s -> %s", rvboxv1.CommandLifecycle(current), rvboxv1.CommandLifecycle(phase))
+	}
+	lifecycle := &rvboxv1.LifecycleChange{Lifecycle: rvboxv1.CommandLifecycle(phase), CommandRevision: revision, Detail: detail}
+	payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(&rvboxv1.CommandEvent{IssueUuid: issueUUID.String(), ObservedAt: timestamppb.New(observedAt), Payload: &rvboxv1.CommandEvent_Lifecycle{Lifecycle: lifecycle}})
+	if err != nil {
+		return Event{}, err
+	}
+	charge, err := EstimateCharge(ChargeInput{EncodedBytes: uint64(len(payload)), SQLiteRows: 1, IndexEntries: 2})
+	if err != nil {
+		return Event{}, err
+	}
+	clientTotal, err := clientTotalCharge(ctx, tx)
+	if err != nil {
+		return Event{}, err
+	}
+	decision, err := CheckReservation(store.quotaLimits, ReservationState{CommandOutputCharged: outputCharged, CommandTotalCharged: totalCharged, ClientTotalCharged: clientTotal, CloseoutRemaining: closeout}, ReservationRequest{ChargedBytes: charge, UseCloseout: isTerminalPhase(phase)})
+	if err != nil {
+		return Event{}, err
+	}
+	digest := immutableDigest(payload)
+	if _, err := tx.ExecContext(ctx, `INSERT INTO events(issue_uuid, local_ordinal, event_kind, compression, raw_bytes, charged_bytes, output, payload, payload_sha256, created_at) VALUES (?, ?, 4, 1, ?, ?, 0, ?, ?, ?)`, issueUUID[:], nextOrdinal, len(payload), charge, payload, digest[:], observedAt.UnixNano()); err != nil {
+		return Event{}, err
+	}
+	terminal := isTerminalPhase(phase)
+	if _, err := tx.ExecContext(ctx, `UPDATE commands SET phase = ?, terminal = ?, launch_phase = CASE WHEN ? = 1 THEN 0 ELSE launch_phase END, launch_context = CASE WHEN ? = 1 THEN '' ELSE launch_context END, launch_pid = CASE WHEN ? = 1 THEN 0 ELSE launch_pid END, next_local_ordinal = ?, total_charged_bytes = ?, closeout_remaining_bytes = ? WHERE issue_uuid = ?`, phase, boolInt(terminal), boolInt(terminal), boolInt(terminal), boolInt(terminal), nextOrdinal+1, decision.CommandTotalCharged, decision.CloseoutRemaining, issueUUID[:]); err != nil {
+		return Event{}, err
+	}
+	if err := updateClientTotalCharge(ctx, tx, decision.ClientTotalCharged); err != nil {
+		return Event{}, err
+	}
+	if err := tx.Commit(); err != nil {
+		return Event{}, err
+	}
+	return Event{IssueUUID: issueUUID, LocalOrdinal: nextOrdinal, Kind: 4, Compression: 1, RawBytes: uint64(len(payload)), Payload: append([]byte(nil), payload...), CreatedAt: observedAt}, nil
+}
+
 // CleanupTerminal moves a fully acknowledged terminal command into the compact
 // tombstone ledger and removes all command-owned spool data in one transaction.
 func (store *Store) CleanupTerminal(ctx context.Context, issueUUID domain.UUID, acknowledgedAt time.Time) error {

internal/client/spool/migrations.go

@@ -16,6 +16,7 @@ type migration struct {
 var migrations = []migration{
 	{version: 1, sql: schemaV1},
 	{version: 2, sql: schemaV2},
+	{version: 3, sql: schemaV3},
 }
 
 func applyMigrations(ctx context.Context, db *sql.DB) error {
@@ -132,3 +133,15 @@ CREATE TABLE command_specs (
   charged_bytes INTEGER NOT NULL CHECK(charged_bytes > 0)
 ) STRICT, WITHOUT ROWID;
 `
+
+// schemaV3 adds a small durable launch barrier to every accepted command. A
+// value of 2 means launch authorization may have crossed the OS boundary; on
+// restart the client must interrupt that command instead of redispatching it.
+// Keeping these fields on commands makes the barrier part of the existing
+// command-owned quota/accounting row and lets terminal cleanup remove it with
+// the command.
+const schemaV3 = `
+ALTER TABLE commands ADD COLUMN launch_phase INTEGER NOT NULL DEFAULT 0 CHECK(launch_phase BETWEEN 0 AND 2);
+ALTER TABLE commands ADD COLUMN launch_context TEXT NOT NULL DEFAULT '';
+ALTER TABLE commands ADD COLUMN launch_pid INTEGER NOT NULL DEFAULT 0 CHECK(launch_pid >= 0);
+`

internal/client/spool/recovery_test.go

@@ -6,6 +6,9 @@ import (
 	"path/filepath"
 	"testing"
 	"time"
+
+	rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
+	"github.com/rvbox/rvbox/internal/domain"
 )
 
 func TestCheckSpoolPayloadsAndCounters_HP_CLIENT_09(t *testing.T) {
@@ -46,3 +49,33 @@ func TestCheckSpoolRejectsCounterDrift_BH_CLIENT_03(t *testing.T) {
 		t.Fatalf("counter-drift Check error = %v, want ErrQuotaCounterMismatch", err)
 	}
 }
+
+func TestRecoverLaunchUncertaintyFencesRedispatchAfterRestart_HP_CLIENT_12(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	store := openTestStore(t, ctx, filepath.Join(t.TempDir(), "spool"), DefaultTombstoneLimit)
+	issue := testUUID(t, "019c46f1-1d02-7000-8000-000000000043")
+	command := testCommand(issue, []byte("uncertain launch"))
+	command.Phase = uint32(rvboxv1.CommandLifecycle_COMMAND_ACCEPTED)
+	now := time.Date(2026, time.September, 6, 12, 0, 0, 0, time.UTC)
+	if _, err := store.AcceptCommand(ctx, command, now); err != nil {
+		t.Fatal(err)
+	}
+	if err := store.SetLaunchPhase(ctx, issue, domain.LaunchPhaseAuthorized, "LOCAL_SYSTEM", 42); err != nil {
+		t.Fatal(err)
+	}
+	recovered, err := store.RecoverLaunchUncertainty(ctx, now.Add(time.Second))
+	if err != nil || len(recovered) != 1 || recovered[0] != issue {
+		t.Fatalf("recovered = %v, %v", recovered, err)
+	}
+	var phase, launchPhase uint32
+	if err := store.db.QueryRow(`SELECT phase, launch_phase FROM commands WHERE issue_uuid = ?`, issue[:]).Scan(&phase, &launchPhase); err != nil {
+		t.Fatal(err)
+	}
+	if phase != uint32(rvboxv1.CommandLifecycle_COMMAND_INTERRUPTED) || launchPhase != 0 {
+		t.Fatalf("recovered phase/barrier = %d/%d", phase, launchPhase)
+	}
+	if second, err := store.RecoverLaunchUncertainty(ctx, now.Add(2*time.Second)); err != nil || len(second) != 0 {
+		t.Fatalf("recovery repeated = %v, %v", second, err)
+	}
+}

internal/client/spool/script.go

@@ -20,6 +20,7 @@ var (
 	ErrScriptBounds   = errors.New("script chunk is outside declared bounds")
 	ErrScriptState    = errors.New("stored script state is corrupt")
 	ErrScriptTerminal = errors.New("terminal command cannot accept script data")
+	ErrScriptNotReady = errors.New("script has not been durably committed")
 )
 
 type ScriptDescriptor struct {
@@ -33,6 +34,39 @@ type ScriptStatus struct {
 	Duplicate     bool
 }
 
+// ScriptBody returns a copy of the verified script body only after the
+// contiguous upload has been committed. It is the sole spool read used by the
+// supervisor; callers never reconstruct script bytes from individual chunks.
+func (store *Store) ScriptBody(ctx context.Context, issueUUID domain.UUID) ([]byte, error) {
+	if !validUUID(issueUUID) {
+		return nil, ErrUnknownCommand
+	}
+	var row storedScript
+	var digest []byte
+	var storedBytes uint64
+	var compression uint32
+	var committed int
+	err := store.db.QueryRowContext(ctx, `SELECT declared_raw_bytes, declared_sha256, received_raw_bytes, stored_bytes, compression, stored_data, charged_bytes, committed FROM scripts WHERE issue_uuid = ?`, issueUUID[:]).Scan(&row.DeclaredBytes, &digest, &row.ReceivedBytes, &storedBytes, &compression, &row.Stored, &row.ChargedBytes, &committed)
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, ErrScriptNotReady
+	}
+	if err != nil {
+		return nil, err
+	}
+	if len(digest) != sha256.Size || storedBytes != uint64(len(row.Stored)) || compression != 2 || committed == 0 || row.ReceivedBytes != row.DeclaredBytes {
+		if committed == 0 {
+			return nil, ErrScriptNotReady
+		}
+		return nil, ErrScriptState
+	}
+	copy(row.DeclaredSHA256[:], digest)
+	body, err := decompressScript(row.Stored, row.ReceivedBytes, store.maxScriptBytes)
+	if err != nil || sha256.Sum256(body) != row.DeclaredSHA256 {
+		return nil, ErrScriptState
+	}
+	return append([]byte(nil), body...), nil
+}
+
 // BeginScript persists the immutable descriptor at command acceptance time. A
 // matching replay is harmless; a different descriptor is a protocol conflict.
 func (store *Store) BeginScript(ctx context.Context, issueUUID domain.UUID, descriptor ScriptDescriptor) (ScriptStatus, error) {

internal/client/spool/spool_test.go

@@ -10,6 +10,8 @@ import (
 	"testing"
 	"time"
 
+	rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
+
 	"github.com/rvbox/rvbox/internal/domain"
 )
 
@@ -186,6 +188,43 @@ func TestSpoolAcceptanceSequencingAndAck_HP_CLIENT_07(t *testing.T) {
 	}
 }
 
+func TestAppendLifecycleAtomicallyUpdatesPhaseAndEvent_HP_CLIENT_11(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	store := openTestStore(t, ctx, filepath.Join(t.TempDir(), "spool"), DefaultTombstoneLimit)
+	issue := testUUID(t, "019c46f1-1d02-7000-8000-000000000021")
+	command := testCommand(issue, []byte("lifecycle"))
+	command.Phase = uint32(rvboxv1.CommandLifecycle_COMMAND_ACCEPTED)
+	now := time.Date(2026, time.September, 6, 12, 0, 0, 0, time.UTC)
+	if _, err := store.AcceptCommand(ctx, command, now); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := store.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), 1, "launch authorized", now); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := store.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED), 1, "exit 0", now.Add(time.Second)); err != nil {
+		t.Fatal(err)
+	}
+	var phase uint32
+	var terminal int
+	if err := store.db.QueryRow(`SELECT phase, terminal FROM commands WHERE issue_uuid = ?`, issue[:]).Scan(&phase, &terminal); err != nil {
+		t.Fatal(err)
+	}
+	if phase != uint32(rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED) || terminal != 1 {
+		t.Fatalf("phase/terminal = %d/%d", phase, terminal)
+	}
+	if _, err := store.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), 1, "illegal", now.Add(2*time.Second)); err == nil {
+		t.Fatal("terminal lifecycle regressed")
+	}
+	if _, err := store.AssignSendWindow(ctx, issue, 8, 1<<20); err != nil {
+		t.Fatal(err)
+	}
+	events, err := store.PendingEvents(ctx, issue)
+	if err != nil || len(events) != 2 {
+		t.Fatalf("lifecycle events = %#v, %v", events, err)
+	}
+}
+
 func TestTerminalCleanupTombstonesAndConflicts_BH_CLIENT_02(t *testing.T) {
 	t.Parallel()
 	ctx := context.Background()