rvbox/test/harness/harness.go

package main

import (
	"bytes"
	"context"
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"encoding/xml"
	"errors"
	"flag"
	"fmt"
	"io"
	"math/big"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/pelletier/go-toml/v2"
	"github.com/rvbox/rvbox/internal/domain"
)

const (
	manifestVersion = 2
	repositoryID    = "rvbox"
	maxSuiteLogSize = 1 << 20
)

var runIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,63}$`)

type manifest struct {
	Version                 uint32        `toml:"version" json:"version"`
	RepositoryID            string        `toml:"repository_id" json:"repository_id"`
	RunID                   string        `toml:"run_id" json:"run_id"`
	Layer                   string        `toml:"layer" json:"layer"`
	Suite                   string        `toml:"suite" json:"suite"`
	TestCase                string        `toml:"test_case,omitempty" json:"test_case,omitempty"`
	Seed                    int64         `toml:"seed" json:"seed"`
	GitCommit               string        `toml:"git_commit" json:"git_commit"`
	DirtyDiffSHA256         string        `toml:"dirty_diff_sha256" json:"dirty_diff_sha256"`
	ToolchainImage          string        `toml:"toolchain_image" json:"toolchain_image"`
	HarnessDefaultsSHA256   string        `toml:"harness_defaults_sha256" json:"harness_defaults_sha256"`
	ComposeDefinitionSHA256 string        `toml:"compose_definition_sha256" json:"compose_definition_sha256"`
	EffectiveLimits         harnessLimits `toml:"effective_limits" json:"effective_limits"`
	ComposeProject          string        `toml:"compose_project" json:"compose_project"`
	Phase                   string        `toml:"phase" json:"phase"`
	CreatedAt               time.Time     `toml:"created_at" json:"created_at"`
	UpdatedAt               time.Time     `toml:"updated_at" json:"updated_at"`
	OwnedResources          []string      `toml:"owned_resources" json:"owned_resources"`
}

type harnessDefaults struct {
	Version uint32        `toml:"version"`
	Limits  harnessLimits `toml:"limits"`
}

type harnessLimits struct {
	MaxConcurrentRuns     uint32 `toml:"max_concurrent_runs" json:"max_concurrent_runs"`
	GoParallelism         uint32 `toml:"go_parallelism" json:"go_parallelism"`
	RuntimeCPUs           uint32 `toml:"runtime_cpus" json:"runtime_cpus"`
	RuntimeMemoryBytes    uint64 `toml:"runtime_memory_bytes" json:"runtime_memory_bytes"`
	RuntimePIDs           uint32 `toml:"runtime_pids" json:"runtime_pids"`
	ComponentLogTailBytes uint64 `toml:"component_log_tail_bytes" json:"component_log_tail_bytes"`
	FailureBundleBytes    uint64 `toml:"failure_bundle_bytes" json:"failure_bundle_bytes"`
	SuccessReportBytes    uint64 `toml:"success_report_bytes" json:"success_report_bytes"`
	FreeSpaceFloorBytes   uint64 `toml:"free_space_floor_bytes" json:"free_space_floor_bytes"`
	UnitTimeout           string `toml:"unit_timeout" json:"unit_timeout"`
	IntegrationTimeout    string `toml:"integration_timeout" json:"integration_timeout"`
	E2ETimeout            string `toml:"e2e_timeout" json:"e2e_timeout"`
	SoakTimeout           string `toml:"soak_timeout" json:"soak_timeout"`
}

type journalEntry struct {
	At     time.Time      `json:"at"`
	Step   string         `json:"step"`
	Status string         `json:"status"`
	Detail string         `json:"detail,omitempty"`
	Data   map[string]any `json:"data,omitempty"`
}

type junitSuite struct {
	XMLName  xml.Name  `xml:"testsuite"`
	Name     string    `xml:"name,attr"`
	Tests    int       `xml:"tests,attr"`
	Failures int       `xml:"failures,attr"`
	TestCase junitCase `xml:"testcase"`
}

type junitCase struct {
	Name    string        `xml:"name,attr"`
	Class   string        `xml:"classname,attr"`
	Failure *junitFailure `xml:"failure,omitempty"`
}

type junitFailure struct {
	Message string `xml:"message,attr"`
}

type harness struct {
	root     string
	repoRoot string
	defaults harnessDefaults
	now      func() time.Time
	out      io.Writer
}

var integrationSuites = []string{"sample", "store", "server-session", "control", "client-agent", "all"}
var e2eScenarios = []string{"smoke", "interactive", "idempotency", "reconnect", "retention", "expiry-and-incidents", "script", "recovery", "all"}

func containsChoice(choices []string, value string) bool {
	for _, choice := range choices {
		if choice == value {
			return true
		}
	}
	return false
}

func printChoices(out io.Writer, heading string, choices []string) error {
	if _, err := fmt.Fprintf(out, "%s:\n", heading); err != nil {
		return err
	}
	for _, choice := range choices {
		if _, err := fmt.Fprintf(out, "%s\n", choice); err != nil {
			return err
		}
	}
	return nil
}

func validateTestCase(value string) error {
	if value == "" {
		return nil
	}
	if _, err := regexp.Compile(value); err != nil {
		return fmt.Errorf("--case must be a valid Go test regexp: %w", err)
	}
	return nil
}

func loadHarnessDefaults(path string) (harnessDefaults, error) {
	data, err := os.ReadFile(path)
	if err != nil {
		return harnessDefaults{}, fmt.Errorf("read harness defaults: %w", err)
	}
	var defaults harnessDefaults
	decoder := toml.NewDecoder(bytes.NewReader(data))
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(&defaults); err != nil {
		return harnessDefaults{}, fmt.Errorf("decode harness defaults: %w", err)
	}
	if defaults.Version != 1 {
		return harnessDefaults{}, fmt.Errorf("unsupported harness defaults version %d", defaults.Version)
	}
	limits := defaults.Limits
	if limits.MaxConcurrentRuns != 1 || limits.GoParallelism == 0 || limits.RuntimeCPUs == 0 || limits.RuntimeMemoryBytes == 0 || limits.RuntimePIDs == 0 || limits.ComponentLogTailBytes == 0 || limits.FailureBundleBytes == 0 || limits.SuccessReportBytes == 0 || limits.FreeSpaceFloorBytes == 0 {
		return harnessDefaults{}, errors.New("harness defaults contain a zero or unsupported resource limit")
	}
	if limits.SuccessReportBytes > limits.FailureBundleBytes {
		return harnessDefaults{}, errors.New("success report budget exceeds failure bundle budget")
	}
	for field, value := range map[string]string{"unit_timeout": limits.UnitTimeout, "integration_timeout": limits.IntegrationTimeout, "e2e_timeout": limits.E2ETimeout, "soak_timeout": limits.SoakTimeout} {
		duration, durationErr := time.ParseDuration(value)
		if durationErr != nil || duration <= 0 {
			return harnessDefaults{}, fmt.Errorf("harness default %s must be a positive duration", field)
		}
	}
	return defaults, nil
}

func (h *harness) definitionHash(relative string) string {
	if h.repoRoot == "" {
		return "unavailable"
	}
	data, err := os.ReadFile(filepath.Join(h.repoRoot, relative))
	if err != nil {
		return "unavailable"
	}
	sum := sha256.Sum256(data)
	return hex.EncodeToString(sum[:])
}

func (h *harness) verifyResume(current *manifest) error {
	commit := strings.TrimSpace(commandOutput("git", "rev-parse", "HEAD"))
	if current.GitCommit != "unavailable" && commit != current.GitCommit {
		return fmt.Errorf("run commit %q does not match current commit %q; start a new run", current.GitCommit, commit)
	}
	dirty := repositoryDirtyHash()
	if current.DirtyDiffSHA256 != hex.EncodeToString(dirty[:]) {
		return errors.New("run dirty-diff hash does not match the working tree; start a new run")
	}
	if want := h.definitionHash("test/harness/defaults.toml"); current.HarnessDefaultsSHA256 != "" && current.HarnessDefaultsSHA256 != "unavailable" && want != current.HarnessDefaultsSHA256 {
		return errors.New("harness defaults changed since the run began; start a new run")
	}
	if want := h.definitionHash("deploy/compose.test.yaml"); current.ComposeDefinitionSHA256 != "" && current.ComposeDefinitionSHA256 != "unavailable" && want != current.ComposeDefinitionSHA256 {
		return errors.New("test Compose definition changed since the run began; start a new run")
	}
	return nil
}

func (h *harness) componentLogLimit() int {
	if h.defaults.Limits.ComponentLogTailBytes == 0 || h.defaults.Limits.ComponentLogTailBytes > uint64(^uint(0)>>1) {
		return maxSuiteLogSize
	}
	return int(h.defaults.Limits.ComponentLogTailBytes)
}

func runCLI(ctx context.Context, args []string) error {
	repoRoot, err := os.Getwd()
	if err != nil {
		return err
	}
	defaults, err := loadHarnessDefaults(filepath.Join(repoRoot, "test", "harness", "defaults.toml"))
	if err != nil {
		return err
	}
	h := &harness{root: filepath.Join(repoRoot, ".test-runs"), repoRoot: repoRoot, defaults: defaults, now: func() time.Time { return time.Now().UTC() }, out: os.Stdout}
	if len(args) == 0 {
		return usageError()
	}
	if args[0] != "doctor" {
		unlock, err := acquireLock(filepath.Join(repoRoot, ".test-harness.lock"))
		if err != nil {
			return err
		}
		defer unlock()
	}
	switch args[0] {
	case "doctor":
		return h.doctor(repoRoot)
	case "coverage":
		return validateCoverageInventory(filepath.Join(repoRoot, "test", "coverage.toml"))
	case "integration":
		return h.integration(ctx, args[1:])
	case "e2e":
		return h.e2e(ctx, args[1:])
	case "status", "logs", "collect", "recover", "reuse", "stop", "reset", "purge", "gc":
		return h.environmentCommand(args[0], args[1:])
	default:
		return usageError()
	}
}

func usageError() error {
	return errors.New("usage: harness doctor|coverage|integration|e2e|status|logs|collect|recover|reuse|stop|reset|purge|gc")
}

func (h *harness) doctor(repoRoot string) error {
	for _, file := range []string{"go.mod", "deploy/compose.yaml", "docs/implementation-plan.v1.md"} {
		if _, err := os.Stat(filepath.Join(repoRoot, file)); err != nil {
			return fmt.Errorf("repository check %s: %w", file, err)
		}
	}
	if h.defaults.Version != 1 {
		return errors.New("harness defaults were not loaded")
	}
	fmt.Fprintf(h.out, "RVBox test harness policy: max_runs=%d go_parallelism=%d free_space_floor_bytes=%d\n", h.defaults.Limits.MaxConcurrentRuns, h.defaults.Limits.GoParallelism, h.defaults.Limits.FreeSpaceFloorBytes)
	fmt.Fprintln(h.out, "RVBox test harness is ready; native Windows scenarios use the explicit scripts/windows/test-host host lane.")
	return nil
}

func (h *harness) integration(ctx context.Context, args []string) error {
	flags := flag.NewFlagSet("integration", flag.ContinueOnError)
	flags.SetOutput(io.Discard)
	suite := flags.String("suite", "sample", "suite name")
	list := flags.Bool("list", false, "list stable suites")
	testCase := flags.String("case", "", "Go test name regexp for one suite")
	runID := flags.String("run-id", "", "run ID")
	resume := flags.Bool("resume", false, "resume an existing run")
	if err := flags.Parse(args); err != nil {
		return err
	}
	if *list {
		if flags.NArg() != 0 || *runID != "" || *resume || *testCase != "" || *suite != "sample" {
			return errors.New("--list cannot be combined with run options")
		}
		return printChoices(h.out, "integration suites", integrationSuites)
	}
	if !containsChoice(integrationSuites, *suite) {
		return fmt.Errorf("suite %q is not implemented yet; available: %s", *suite, strings.Join(integrationSuites, ", "))
	}
	if err := validateTestCase(*testCase); err != nil {
		return err
	}
	if (*suite == "sample" || *suite == "all") && *testCase != "" {
		return errors.New("--case is only supported by one executable integration suite")
	}

	var current *manifest
	var err error
	if *resume {
		if *runID == "" {
			return errors.New("--resume requires --run-id")
		}
		current, err = h.load(*runID)
		if err != nil {
			return err
		}
		if err := h.verifyResume(current); err != nil {
			return err
		}
		if current.Layer != "integration" || current.Suite != *suite {
			return errors.New("run layer/suite does not match resume request")
		}
		if current.TestCase != *testCase {
			return errors.New("run test case does not match resume request")
		}
		if current.Phase != "ready" && current.Phase != "interrupted" && current.Phase != "stopped" && current.Phase != "running" && current.Phase != "failed" {
			return fmt.Errorf("run in phase %q is not resumable; recover or reuse it first", current.Phase)
		}
	} else {
		current, err = h.create(*runID, "integration", *suite)
		if err != nil {
			return err
		}
	}
	current.TestCase = *testCase
	if err := h.writeManifest(current); err != nil {
		return err
	}
	fmt.Fprintln(h.out, current.RunID)
	if err := h.transition(current, "running", *suite+"-start", *suite+" integration run started"); err != nil {
		return err
	}
	select {
	case <-ctx.Done():
		_ = h.transition(current, "interrupted", "interrupt", ctx.Err().Error())
		return ctx.Err()
	default:
	}
	suites := []string{*suite}
	if *suite == "all" {
		suites = []string{"sample", "store", "server-session", "control", "client-agent"}
	}
	for _, item := range suites {
		if suiteErr := h.runIntegrationSuite(ctx, current, item, *testCase); suiteErr != nil {
			_ = h.transition(current, "failed", item+"-failed", suiteErr.Error())
			return suiteErr
		}
	}
	return h.transition(current, "completed", *suite+"-complete", *suite+" integration run completed")
}

func (h *harness) runIntegrationSuite(ctx context.Context, current *manifest, suite, testCase string) error {
	switch suite {
	case "sample":
		return h.appendJournal(current.RunID, journalEntry{At: h.now(), Step: "invariant", Status: "passed", Detail: "manifest ownership and journal durability verified"})
	case "store":
		return h.runStoreSuite(ctx, current, testCase)
	case "server-session":
		return h.runServerSessionSuite(ctx, current, testCase)
	case "control":
		return h.runGoSuite(ctx, current, "control-real-grpc", "running gRPC control and JSON-RPC compatibility cases", "./internal/server/control", testCase)
	case "client-agent":
		return h.runGoSuite(ctx, current, "client-agent-real-websocket", "running real client/server WebSocket control-flow cases", "./test/integration/clientagent", testCase)
	default:
		return fmt.Errorf("unknown integration suite %q", suite)
	}
}

// e2e runs production-shaped Go scenarios against real local listeners and
// durable stores. Native Windows work is an additional host lane invoked by
// scripts/windows/test-host.ps1; it is never silently replaced by Wine or a
// cross-compiled binary. The run manifest/journal makes every scenario
// resumable and keeps artifacts bounded.
func (h *harness) e2e(ctx context.Context, args []string) error {
	flags := flag.NewFlagSet("e2e", flag.ContinueOnError)
	flags.SetOutput(io.Discard)
	scenario := flags.String("scenario", "smoke", "scenario name")
	list := flags.Bool("list", false, "list stable scenarios")
	testCase := flags.String("case", "", "Go test name regexp for one scenario")
	runID := flags.String("run-id", "", "run ID")
	resume := flags.Bool("resume", false, "resume an existing run")
	if err := flags.Parse(args); err != nil {
		return err
	}
	if *list {
		if flags.NArg() != 0 || *runID != "" || *resume || *testCase != "" || *scenario != "smoke" {
			return errors.New("--list cannot be combined with run options")
		}
		return printChoices(h.out, "e2e scenarios", e2eScenarios)
	}
	if !containsChoice(e2eScenarios, *scenario) {
		return fmt.Errorf("scenario %q is not implemented yet; available: %s", *scenario, strings.Join(e2eScenarios, ", "))
	}
	if err := validateTestCase(*testCase); err != nil {
		return err
	}
	if *scenario == "all" && *testCase != "" {
		return errors.New("--case requires one named e2e scenario")
	}
	var current *manifest
	var err error
	if *resume {
		if *runID == "" {
			return errors.New("--resume requires --run-id")
		}
		current, err = h.load(*runID)
		if err != nil {
			return err
		}
		if err := h.verifyResume(current); err != nil {
			return err
		}
		if current.Layer != "e2e" || current.Suite != *scenario {
			return errors.New("run layer/scenario does not match resume request")
		}
		if current.TestCase != *testCase {
			return errors.New("run test case does not match resume request")
		}
		if current.Phase != "ready" && current.Phase != "interrupted" && current.Phase != "stopped" && current.Phase != "running" && current.Phase != "failed" {
			return fmt.Errorf("run in phase %q is not resumable; recover or reuse it first", current.Phase)
		}
	} else {
		current, err = h.create(*runID, "e2e", *scenario)
		if err != nil {
			return err
		}
	}
	current.TestCase = *testCase
	if err := h.writeManifest(current); err != nil {
		return err
	}
	fmt.Fprintln(h.out, current.RunID)
	if err := h.transition(current, "running", "e2e-start", "e2e scenario started"); err != nil {
		return err
	}
	scenarios := []string{*scenario}
	if *scenario == "all" {
		scenarios = []string{"smoke", "interactive", "idempotency", "reconnect", "retention", "expiry-and-incidents", "script", "recovery"}
	}
	for _, item := range scenarios {
		if err := h.runE2EScenario(ctx, current, item, *testCase); err != nil {
			_ = h.transition(current, "failed", "e2e-"+item+"-failed", err.Error())
			return err
		}
	}
	return h.transition(current, "completed", "e2e-complete", "e2e scenario completed")
}

func (h *harness) runE2EScenario(ctx context.Context, current *manifest, scenario, testCase string) error {
	if err := h.appendJournal(current.RunID, journalEntry{At: h.now(), Step: "e2e-" + scenario, Status: "running", Detail: "scenario started"}); err != nil {
		return err
	}
	var err error
	switch scenario {
	case "smoke":
		err = h.runGoSuite(ctx, current, "e2e-client-agent", "real client/server WebSocket and control flow", "./test/integration/clientagent", testCase)
	case "interactive":
		err = h.runGoSuite(ctx, current, "e2e-stdin-protocol", "durable stdin append, replay, close, and acknowledgement flow", "./test/integration/clientagent", chooseTestCase(testCase, "^TestControlStdinIntentReplaysAndAcknowledges_HP_DISPATCH_08$"))
	case "idempotency":
		err = h.runGoSuite(ctx, current, "e2e-request-idempotency", "request UUID replay and immutable request-hash conflict handling", "./internal/server/control", chooseTestCase(testCase, "^TestRunCommandRequestIDIdempotencyAndValidation_BH_CONTROL_02$"))
	case "reconnect":
		err = h.runGoSuite(ctx, current, "e2e-reconnect-reconciliation", "WebSocket reconciliation, fenced dispatch, and session wake handling", "./test/integration/clientagent", chooseTestCase(testCase, "^(TestWebSocketHelloWelcome_HP_SES_06|TestWebSocketDispatchAfterReconciliation_HP_DISPATCH_03|TestControlQueueWakesReconciledSession_HP_DISPATCH_06)$"))
	case "retention":
		err = h.runStoreSuite(ctx, current, chooseTestCase(testCase, "^(TestTerminalAgeRetentionEvictsWholeCommand_HP_STORE_08|TestTombstoneFIFOIsCappedInEvictionTransaction_HP_STORE_09|TestRetentionCrashStagesRollForward_CRASH_STORE_04)$"))
	case "expiry-and-incidents":
		err = h.runStoreSuite(ctx, current, chooseTestCase(testCase, "^(TestIncidentDirtyResolutionIdempotencyAndRecurrence_HP_STORE_11|TestIrreparableIncidentRequiresExplicitAcknowledgement_BH_STORE_09|TestFilesystemFloorAndCounterMismatch_BH_STORE_08)$"))
	case "script":
		err = h.runGoSuite(ctx, current, "e2e-script-transfer", "durable script transfer and replay", "./internal/client/agent", testCase)
	case "recovery":
		err = h.runStoreSuite(ctx, current, testCase)
	default:
		err = fmt.Errorf("unknown e2e scenario %q", scenario)
	}
	if err != nil {
		return err
	}
	return h.appendJournal(current.RunID, journalEntry{At: h.now(), Step: "e2e-" + scenario, Status: "passed", Detail: "scenario passed"})
}

func chooseTestCase(override, defaultCase string) string {
	if override != "" {
		return override
	}
	return defaultCase
}

func (h *harness) runStoreSuite(ctx context.Context, current *manifest, testCase string) error {
	if err := h.appendJournal(current.RunID, journalEntry{At: h.now(), Step: "store-real-sqlite", Status: "running", Detail: "running real SQLite/WAL and filesystem cases"}); err != nil {
		return err
	}
	artifactDir := filepath.Join(h.runDir(current.RunID), "artifacts")
	if err := os.MkdirAll(artifactDir, 0o700); err != nil {
		return err
	}
	capture := &limitedCapture{limit: h.componentLogLimit()}
	arguments := []string{"test", "-count=1", "-tags=integration", "-shuffle=" + strconv.FormatInt(current.Seed, 10), "-timeout=2m"}
	if testCase != "" {
		arguments = append(arguments, "-run", testCase)
	}
	arguments = append(arguments, "./test/integration/store")
	command := exec.CommandContext(ctx, "go", arguments...)
	command.Stdout = capture
	command.Stderr = capture
	err := command.Run()
	logPath := filepath.Join(artifactDir, "suite.log")
	if writeErr := atomicWrite(logPath, capture.Bytes(), 0o600); writeErr != nil {
		return writeErr
	}
	if err != nil {
		return fmt.Errorf("store suite failed (bounded log %s): %w", logPath, err)
	}
	return h.appendJournal(current.RunID, journalEntry{At: h.now(), Step: "store-real-sqlite", Status: "passed", Detail: "real SQLite/WAL and filesystem cases passed"})
}

func (h *harness) runServerSessionSuite(ctx context.Context, current *manifest, testCase string) error {
	return h.runGoSuite(ctx, current, "server-session-real-websocket", "running real HTTP/WebSocket, protobuf, SQLite, and fencing cases", "./internal/server/session", testCase)
}

func (h *harness) runGoSuite(ctx context.Context, current *manifest, step, detail, packagePath, testCase string) error {
	if err := h.appendJournal(current.RunID, journalEntry{At: h.now(), Step: step, Status: "running", Detail: detail}); err != nil {
		return err
	}
	artifactDir := filepath.Join(h.runDir(current.RunID), "artifacts")
	if err := os.MkdirAll(artifactDir, 0o700); err != nil {
		return err
	}
	capture := &limitedCapture{limit: h.componentLogLimit()}
	arguments := []string{"test", "-race", "-count=1", "-shuffle=" + strconv.FormatInt(current.Seed, 10), "-timeout=2m"}
	if testCase != "" {
		arguments = append(arguments, "-run", testCase)
	}
	arguments = append(arguments, packagePath)
	command := exec.CommandContext(ctx, "go", arguments...)
	command.Stdout = capture
	command.Stderr = capture
	err := command.Run()
	logPath := filepath.Join(artifactDir, "suite.log")
	if writeErr := atomicWrite(logPath, capture.Bytes(), 0o600); writeErr != nil {
		return writeErr
	}
	if err != nil {
		return fmt.Errorf("%s suite failed (bounded log %s): %w", current.Suite, logPath, err)
	}
	return h.appendJournal(current.RunID, journalEntry{At: h.now(), Step: step, Status: "passed", Detail: detail + " passed"})
}

func (h *harness) environmentCommand(command string, args []string) error {
	if command == "gc" {
		return h.gc(args)
	}
	flags := flag.NewFlagSet(command, flag.ContinueOnError)
	flags.SetOutput(io.Discard)
	runID := flags.String("run-id", "", "run ID")
	newRunID := flags.String("new-run-id", "", "fresh run ID for reuse")
	all := flags.Bool("all", false, "select every eligible run")
	execute := flags.Bool("execute", false, "perform a destructive cleanup")
	yes := flags.Bool("yes", false, "confirm destructive cleanup in noninteractive use")
	if err := flags.Parse(args); err != nil {
		return err
	}
	if command != "purge" && (*all || *execute || *yes) {
		return fmt.Errorf("%s does not accept cleanup options", command)
	}
	if command != "reuse" && *newRunID != "" {
		return fmt.Errorf("%s does not accept --new-run-id", command)
	}
	if command == "purge" {
		if (*runID == "") == (!*all) {
			return errors.New("purge requires exactly one of --run-id or --all")
		}
		return h.purgeCommand(*runID, *all, *execute, *yes)
	}
	if *runID == "" {
		return fmt.Errorf("%s requires --run-id", command)
	}
	current, err := h.load(*runID)
	if err != nil {
		return err
	}
	switch command {
	case "status":
		resumeErr := h.verifyResume(current)
		status := map[string]any{"manifest": current, "run_bytes": directorySize(h.runDir(current.RunID)), "resume_valid": resumeErr == nil}
		if resumeErr != nil {
			status["resume_error"] = resumeErr.Error()
		}
		encoded, _ := json.MarshalIndent(status, "", "  ")
		fmt.Fprintln(h.out, string(encoded))
		return nil
	case "logs":
		data, err := os.ReadFile(h.journalPath(current.RunID))
		if err != nil {
			return err
		}
		_, err = h.out.Write(data)
		return err
	case "collect":
		return h.collect(current)
	case "recover":
		if err := h.verifyResume(current); err != nil {
			return err
		}
		if current.Phase != "interrupted" && current.Phase != "stopped" && current.Phase != "running" && current.Phase != "failed" {
			return fmt.Errorf("run in phase %q does not need recovery", current.Phase)
		}
		return h.transition(current, "ready", "recover", "run recovered and ready to resume")
	case "reuse":
		return h.reuse(current, *newRunID)
	case "stop":
		return h.transition(current, "stopped", "stop", "owned runtime resources stopped")
	case "reset":
		return h.reset(current)
	default:
		return usageError()
	}
}

func (h *harness) reuse(previous *manifest, newRunID string) error {
	if previous.Phase != "reset" && previous.Phase != "completed" {
		return fmt.Errorf("run in phase %q cannot be reused; reset or complete it first", previous.Phase)
	}
	if err := h.verifyResume(previous); err != nil {
		return err
	}
	fresh, err := h.create(newRunID, previous.Layer, previous.Suite)
	if err != nil {
		return err
	}
	fresh.TestCase = previous.TestCase
	if err := h.writeManifest(fresh); err != nil {
		return err
	}
	if err := h.appendJournal(fresh.RunID, journalEntry{At: h.now(), Step: "reuse", Status: "passed", Detail: "fresh run created from immutable prior definitions", Data: map[string]any{"prior_run_id": previous.RunID}}); err != nil {
		return err
	}
	fmt.Fprintf(h.out, "reused_from=%s new_run_id=%s\n", previous.RunID, fresh.RunID)
	return nil
}

func (h *harness) reset(current *manifest) error {
	if current.Phase == "running" {
		return errors.New("refusing to reset a running run; stop it first")
	}
	if len(current.OwnedResources) != 0 {
		return errors.New("refusing to reset a run with owned runtime resources; stop and reconcile them first")
	}
	for _, name := range []string{"runtime", "pki", "scratch"} {
		path := filepath.Join(h.runDir(current.RunID), name)
		info, err := os.Lstat(path)
		if os.IsNotExist(err) {
			continue
		}
		if err != nil {
			return err
		}
		if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
			return fmt.Errorf("refusing unsafe reset path %s", path)
		}
		if err := os.RemoveAll(path); err != nil {
			return err
		}
	}
	return h.transition(current, "reset", "reset", "owned runtime scratch reset; manifest, journal, and reports retained")
}

func (h *harness) purgeCommand(runID string, all, execute, yes bool) error {
	var candidates []*manifest
	if all {
		loaded, err := h.eligiblePurgeManifests(time.Time{})
		if err != nil {
			return err
		}
		candidates = loaded
	} else {
		current, err := h.load(runID)
		if err != nil {
			return err
		}
		if err := h.validatePurge(current); err != nil {
			return err
		}
		candidates = []*manifest{current}
	}
	if len(candidates) == 0 {
		fmt.Fprintln(h.out, "no eligible RVBox test runs")
		return nil
	}
	for _, current := range candidates {
		fmt.Fprintf(h.out, "purge_target=%s phase=%s bytes=%d\n", h.runDir(current.RunID), current.Phase, directorySize(h.runDir(current.RunID)))
	}
	if !execute {
		fmt.Fprintln(h.out, "dry run only; rerun with --execute --yes to remove exactly these validated targets")
		return nil
	}
	if !yes {
		return errors.New("purge --execute requires --yes in the noninteractive harness")
	}
	for _, current := range candidates {
		if err := h.purge(current); err != nil {
			return err
		}
	}
	return nil
}

func (h *harness) gc(args []string) error {
	flags := flag.NewFlagSet("gc", flag.ContinueOnError)
	flags.SetOutput(io.Discard)
	olderThan := flags.String("older-than", "", "minimum age of completed/reset runs")
	execute := flags.Bool("execute", false, "perform a destructive cleanup")
	yes := flags.Bool("yes", false, "confirm destructive cleanup in noninteractive use")
	if err := flags.Parse(args); err != nil {
		return err
	}
	if *olderThan == "" {
		return errors.New("gc requires --older-than")
	}
	age, err := time.ParseDuration(*olderThan)
	if err != nil || age <= 0 {
		return errors.New("gc --older-than must be a positive duration")
	}
	candidates, err := h.eligiblePurgeManifests(h.now().Add(-age))
	if err != nil {
		return err
	}
	if len(candidates) == 0 {
		fmt.Fprintln(h.out, "no eligible RVBox test runs")
		return nil
	}
	for _, current := range candidates {
		fmt.Fprintf(h.out, "gc_target=%s phase=%s bytes=%d\n", h.runDir(current.RunID), current.Phase, directorySize(h.runDir(current.RunID)))
	}
	if !*execute {
		fmt.Fprintln(h.out, "dry run only; rerun with --execute --yes to remove exactly these validated targets")
		return nil
	}
	if !*yes {
		return errors.New("gc --execute requires --yes in the noninteractive harness")
	}
	for _, current := range candidates {
		if err := h.purge(current); err != nil {
			return err
		}
	}
	return nil
}

func (h *harness) eligiblePurgeManifests(before time.Time) ([]*manifest, error) {
	entries, err := os.ReadDir(h.root)
	if os.IsNotExist(err) {
		return nil, nil
	}
	if err != nil {
		return nil, err
	}
	result := make([]*manifest, 0, len(entries))
	for _, entry := range entries {
		if !entry.IsDir() || entry.Type()&os.ModeSymlink != 0 || validateRunID(entry.Name()) != nil {
			continue
		}
		current, loadErr := h.load(entry.Name())
		if loadErr != nil || h.validatePurge(current) != nil {
			continue
		}
		if !before.IsZero() && !current.UpdatedAt.Before(before) {
			continue
		}
		result = append(result, current)
	}
	return result, nil
}

type limitedCapture struct {
	data      []byte
	limit     int
	truncated bool
}

func (capture *limitedCapture) Write(data []byte) (int, error) {
	written := len(data)
	remaining := capture.limit - len(capture.data)
	if remaining > 0 {
		if len(data) > remaining {
			data = data[:remaining]
		}
		capture.data = append(capture.data, data...)
	}
	if written > remaining {
		capture.truncated = true
	}
	return written, nil
}

func (capture *limitedCapture) Bytes() []byte {
	if !capture.truncated {
		return capture.data
	}
	return append(append([]byte(nil), capture.data...), []byte("\n[output truncated by RVBox test harness]\n")...)
}

func (h *harness) create(requestedID, layer, suite string) (*manifest, error) {
	if requestedID == "" {
		id, err := domain.NewUUIDv7()
		if err != nil {
			return nil, err
		}
		requestedID = id.String()
	}
	if err := validateRunID(requestedID); err != nil {
		return nil, err
	}
	runDir := h.runDir(requestedID)
	if err := os.MkdirAll(h.root, 0o700); err != nil {
		return nil, err
	}
	if err := os.Mkdir(runDir, 0o700); err != nil {
		return nil, fmt.Errorf("create run %s: %w", requestedID, err)
	}
	seedValue, err := rand.Int(rand.Reader, big.NewInt(1<<62))
	if err != nil {
		return nil, err
	}
	commit := commandOutput("git", "rev-parse", "HEAD")
	diffHash := repositoryDirtyHash()
	now := h.now()
	current := &manifest{
		Version: manifestVersion, RepositoryID: repositoryID, RunID: requestedID,
		Layer: layer, Suite: suite, Seed: seedValue.Int64(), GitCommit: strings.TrimSpace(commit),
		DirtyDiffSHA256: hex.EncodeToString(diffHash[:]), ToolchainImage: os.Getenv("RVBOX_TOOLCHAIN_IMAGE"),
		HarnessDefaultsSHA256:   h.definitionHash("test/harness/defaults.toml"),
		ComposeDefinitionSHA256: h.definitionHash("deploy/compose.test.yaml"),
		EffectiveLimits:         h.defaults.Limits,
		ComposeProject:          "rvbox-test-" + requestedID, Phase: "created", CreatedAt: now, UpdatedAt: now,
		OwnedResources: []string{},
	}
	if err := h.writeManifest(current); err != nil {
		_ = os.Remove(runDir)
		return nil, err
	}
	if err := h.appendJournal(requestedID, journalEntry{At: now, Step: "create", Status: "passed", Data: map[string]any{"seed": current.Seed}}); err != nil {
		return nil, err
	}
	return current, nil
}

func (h *harness) transition(current *manifest, phase, step, detail string) error {
	current.Phase = phase
	current.UpdatedAt = h.now()
	if err := h.writeManifest(current); err != nil {
		return err
	}
	return h.appendJournal(current.RunID, journalEntry{At: current.UpdatedAt, Step: step, Status: phase, Detail: detail})
}

func (h *harness) load(runID string) (*manifest, error) {
	if err := validateRunID(runID); err != nil {
		return nil, err
	}
	data, err := os.ReadFile(h.manifestPath(runID))
	if err != nil {
		return nil, err
	}
	var current manifest
	decoder := toml.NewDecoder(strings.NewReader(string(data)))
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(&current); err != nil {
		return nil, fmt.Errorf("decode run manifest: %w", err)
	}
	if current.Version != manifestVersion || current.RepositoryID != repositoryID || current.RunID != runID {
		return nil, errors.New("run manifest identity/version mismatch")
	}
	return &current, nil
}

func (h *harness) writeManifest(current *manifest) error {
	data, err := toml.Marshal(current)
	if err != nil {
		return err
	}
	return atomicWrite(h.manifestPath(current.RunID), data, 0o600)
}

func (h *harness) appendJournal(runID string, entry journalEntry) error {
	data, err := json.Marshal(entry)
	if err != nil {
		return err
	}
	file, err := os.OpenFile(h.journalPath(runID), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
	if err != nil {
		return err
	}
	defer file.Close()
	if _, err := file.Write(append(data, '\n')); err != nil {
		return err
	}
	return file.Sync()
}

func (h *harness) collect(current *manifest) error {
	reportDir := filepath.Join(h.runDir(current.RunID), "artifacts")
	if err := os.MkdirAll(reportDir, 0o700); err != nil {
		return err
	}
	resumeErr := h.verifyResume(current)
	report := map[string]any{"run_id": current.RunID, "layer": current.Layer, "suite": current.Suite, "phase": current.Phase, "collected_at": h.now(), "payloads_included": false, "run_bytes": directorySize(h.runDir(current.RunID)), "resume_valid": resumeErr == nil}
	if resumeErr != nil {
		report["resume_error"] = resumeErr.Error()
	}
	data, _ := json.MarshalIndent(report, "", "  ")
	path := filepath.Join(reportDir, "report.json")
	if err := atomicWrite(path, append(data, '\n'), 0o600); err != nil {
		return err
	}
	failed := current.Phase == "failed"
	suite := junitSuite{Name: current.Layer + "/" + current.Suite, Tests: 1, TestCase: junitCase{Name: current.TestCase, Class: current.Layer + "/" + current.Suite}}
	if suite.TestCase.Name == "" {
		suite.TestCase.Name = current.Suite
	}
	if failed {
		suite.Failures = 1
		suite.TestCase.Failure = &junitFailure{Message: "harness run failed; inspect bounded suite.log and journal.jsonl"}
	}
	encodedJUnit, err := xml.MarshalIndent(suite, "", "  ")
	if err != nil {
		return err
	}
	if err := atomicWrite(filepath.Join(reportDir, "junit.xml"), append(append([]byte(xml.Header), encodedJUnit...), '\n'), 0o600); err != nil {
		return err
	}
	if err := h.appendJournal(current.RunID, journalEntry{At: h.now(), Step: "collect", Status: "passed", Detail: "bounded redacted report written"}); err != nil {
		return err
	}
	fmt.Fprintln(h.out, path)
	return nil
}

func (h *harness) purge(current *manifest) error {
	if err := h.validatePurge(current); err != nil {
		return err
	}
	runDir := h.runDir(current.RunID)
	if err := os.RemoveAll(runDir); err != nil {
		return err
	}
	fmt.Fprintf(h.out, "purged %s (not recoverable)\n", runDir)
	return nil
}

func (h *harness) validatePurge(current *manifest) error {
	if current.Phase != "completed" && current.Phase != "reset" {
		return fmt.Errorf("refusing to purge run in phase %q; reset it first", current.Phase)
	}
	runDir := h.runDir(current.RunID)
	info, err := os.Lstat(runDir)
	if err != nil {
		return err
	}
	if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
		return errors.New("refusing to purge a symlink or non-directory run path")
	}
	if len(current.OwnedResources) != 0 {
		return errors.New("refusing to purge while manifest still lists owned runtime resources; reset first")
	}
	return nil
}

func directorySize(path string) uint64 {
	var total uint64
	_ = filepath.WalkDir(path, func(_ string, entry os.DirEntry, err error) error {
		if err != nil || entry.Type().IsDir() || entry.Type()&os.ModeSymlink != 0 {
			return nil
		}
		if info, infoErr := entry.Info(); infoErr == nil && info.Size() > 0 {
			total += uint64(info.Size())
		}
		return nil
	})
	return total
}

func (h *harness) runDir(runID string) string { return filepath.Join(h.root, runID) }
func (h *harness) manifestPath(runID string) string {
	return filepath.Join(h.runDir(runID), "run.toml")
}
func (h *harness) journalPath(runID string) string {
	return filepath.Join(h.runDir(runID), "journal.jsonl")
}

func validateRunID(runID string) error {
	if !runIDPattern.MatchString(runID) || runID == "." || runID == ".." {
		return fmt.Errorf("invalid filesystem-safe run ID %q", runID)
	}
	return nil
}

func atomicWrite(path string, data []byte, mode os.FileMode) error {
	directory := filepath.Dir(path)
	temporary, err := os.CreateTemp(directory, ".write-")
	if err != nil {
		return err
	}
	temporaryPath := temporary.Name()
	defer os.Remove(temporaryPath)
	if err := temporary.Chmod(mode); err != nil {
		_ = temporary.Close()
		return err
	}
	if _, err := temporary.Write(data); err != nil {
		_ = temporary.Close()
		return err
	}
	if err := temporary.Sync(); err != nil {
		_ = temporary.Close()
		return err
	}
	if err := temporary.Close(); err != nil {
		return err
	}
	if err := os.Rename(temporaryPath, path); err != nil {
		return err
	}
	dir, err := os.Open(directory)
	if err != nil {
		return err
	}
	defer dir.Close()
	return dir.Sync()
}

func commandOutput(name string, args ...string) string {
	output, err := exec.Command(name, args...).Output()
	if err != nil {
		return "unavailable"
	}
	return string(output)
}

func repositoryDirtyHash() [sha256.Size]byte {
	hash := sha256.New()
	tracked := commandBytes("git", "diff", "--binary", "HEAD", "--", ".")
	_, _ = hash.Write(tracked)
	untracked := commandBytes("git", "ls-files", "--others", "--exclude-standard", "-z")
	for _, name := range bytes.Split(untracked, []byte{0}) {
		if len(name) == 0 {
			continue
		}
		_, _ = hash.Write([]byte{0})
		_, _ = hash.Write(name)
		if data, err := os.ReadFile(string(name)); err == nil {
			_, _ = hash.Write([]byte{0})
			_, _ = hash.Write(data)
		}
	}
	var result [sha256.Size]byte
	copy(result[:], hash.Sum(nil))
	return result
}

func commandBytes(name string, args ...string) []byte {
	output, err := exec.Command(name, args...).Output()
	if err != nil {
		return []byte("unavailable")
	}
	return output
}

type coverageInventory struct {
	Version      uint32                `toml:"version"`
	Requirements []coverageRequirement `toml:"requirements"`
}

type coverageRequirement struct {
	ID     string   `toml:"id"`
	Layer  string   `toml:"layer"`
	Status string   `toml:"status"`
	Tests  []string `toml:"tests"`
}

var coverageIDPattern = regexp.MustCompile(`^(HP|BH|ERR|RACE|CRASH|SEC|BOUND|REC)-[A-Z0-9]+-[0-9]{2}$`)

func validateCoverageInventory(path string) error {
	absolutePath, err := filepath.Abs(path)
	if err != nil {
		return err
	}
	path = absolutePath
	data, err := os.ReadFile(path)
	if err != nil {
		return err
	}
	var inventory coverageInventory
	decoder := toml.NewDecoder(bytes.NewReader(data))
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(&inventory); err != nil {
		return err
	}
	if inventory.Version != 1 || len(inventory.Requirements) == 0 {
		return errors.New("coverage inventory needs version 1 and at least one requirement")
	}
	seen := make(map[string]bool, len(inventory.Requirements))
	repositoryRoot := filepath.Dir(filepath.Dir(path))
	for _, requirement := range inventory.Requirements {
		if !coverageIDPattern.MatchString(requirement.ID) || seen[requirement.ID] {
			return fmt.Errorf("invalid or duplicate coverage ID %q", requirement.ID)
		}
		seen[requirement.ID] = true
		if requirement.Layer != "unit" && requirement.Layer != "integration" && requirement.Layer != "e2e" {
			return fmt.Errorf("coverage %s has invalid layer %q", requirement.ID, requirement.Layer)
		}
		if requirement.Status != "implemented" && requirement.Status != "planned" && requirement.Status != "blocked_native_windows" {
			return fmt.Errorf("coverage %s has invalid status %q", requirement.ID, requirement.Status)
		}
		if requirement.Status == "implemented" && len(requirement.Tests) == 0 {
			return fmt.Errorf("implemented coverage %s has no tests", requirement.ID)
		}
		for _, reference := range requirement.Tests {
			parts := strings.Split(reference, ":")
			if len(parts) != 2 || parts[0] == "" || parts[1] == "" || filepath.IsAbs(parts[0]) || strings.Contains(parts[0], "..") {
				return fmt.Errorf("coverage %s has invalid test reference %q", requirement.ID, reference)
			}
			source, err := os.ReadFile(filepath.Join(repositoryRoot, filepath.FromSlash(parts[0])))
			if err != nil {
				return fmt.Errorf("coverage %s test reference: %w", requirement.ID, err)
			}
			if !bytes.Contains(source, []byte("func "+parts[1]+"(")) {
				return fmt.Errorf("coverage %s test function %q not found", requirement.ID, parts[1])
			}
		}
	}
	return nil
}