package main import ( "bytes" "context" "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "encoding/xml" "errors" "flag" "fmt" "io" "math/big" "os" "os/exec" "path/filepath" "regexp" "strconv" "strings" "time" "github.com/pelletier/go-toml/v2" "github.com/rvbox/rvbox/internal/domain" ) const ( manifestVersion = 2 repositoryID = "rvbox" maxSuiteLogSize = 1 << 20 ) var runIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,63}$`) type manifest struct { Version uint32 `toml:"version" json:"version"` RepositoryID string `toml:"repository_id" json:"repository_id"` RunID string `toml:"run_id" json:"run_id"` Layer string `toml:"layer" json:"layer"` Suite string `toml:"suite" json:"suite"` TestCase string `toml:"test_case,omitempty" json:"test_case,omitempty"` Seed int64 `toml:"seed" json:"seed"` GitCommit string `toml:"git_commit" json:"git_commit"` DirtyDiffSHA256 string `toml:"dirty_diff_sha256" json:"dirty_diff_sha256"` ToolchainImage string `toml:"toolchain_image" json:"toolchain_image"` HarnessDefaultsSHA256 string `toml:"harness_defaults_sha256" json:"harness_defaults_sha256"` ComposeDefinitionSHA256 string `toml:"compose_definition_sha256" json:"compose_definition_sha256"` EffectiveLimits harnessLimits `toml:"effective_limits" json:"effective_limits"` ComposeProject string `toml:"compose_project" json:"compose_project"` Phase string `toml:"phase" json:"phase"` CreatedAt time.Time `toml:"created_at" json:"created_at"` UpdatedAt time.Time `toml:"updated_at" json:"updated_at"` OwnedResources []string `toml:"owned_resources" json:"owned_resources"` } type harnessDefaults struct { Version uint32 `toml:"version"` Limits harnessLimits `toml:"limits"` } type harnessLimits struct { MaxConcurrentRuns uint32 `toml:"max_concurrent_runs" json:"max_concurrent_runs"` GoParallelism uint32 `toml:"go_parallelism" json:"go_parallelism"` RuntimeCPUs uint32 `toml:"runtime_cpus" json:"runtime_cpus"` RuntimeMemoryBytes uint64 `toml:"runtime_memory_bytes" json:"runtime_memory_bytes"` RuntimePIDs uint32 `toml:"runtime_pids" json:"runtime_pids"` ComponentLogTailBytes uint64 `toml:"component_log_tail_bytes" json:"component_log_tail_bytes"` FailureBundleBytes uint64 `toml:"failure_bundle_bytes" json:"failure_bundle_bytes"` SuccessReportBytes uint64 `toml:"success_report_bytes" json:"success_report_bytes"` FreeSpaceFloorBytes uint64 `toml:"free_space_floor_bytes" json:"free_space_floor_bytes"` UnitTimeout string `toml:"unit_timeout" json:"unit_timeout"` IntegrationTimeout string `toml:"integration_timeout" json:"integration_timeout"` E2ETimeout string `toml:"e2e_timeout" json:"e2e_timeout"` SoakTimeout string `toml:"soak_timeout" json:"soak_timeout"` } type journalEntry struct { At time.Time `json:"at"` Step string `json:"step"` Status string `json:"status"` Detail string `json:"detail,omitempty"` Data map[string]any `json:"data,omitempty"` } type junitSuite struct { XMLName xml.Name `xml:"testsuite"` Name string `xml:"name,attr"` Tests int `xml:"tests,attr"` Failures int `xml:"failures,attr"` TestCase junitCase `xml:"testcase"` } type junitCase struct { Name string `xml:"name,attr"` Class string `xml:"classname,attr"` Failure *junitFailure `xml:"failure,omitempty"` } type junitFailure struct { Message string `xml:"message,attr"` } type harness struct { root string repoRoot string defaults harnessDefaults now func() time.Time out io.Writer } var integrationSuites = []string{"sample", "store", "server-session", "control", "client-agent", "all"} var e2eScenarios = []string{"smoke", "interactive", "idempotency", "reconnect", "retention", "expiry-and-incidents", "script", "recovery", "all"} func containsChoice(choices []string, value string) bool { for _, choice := range choices { if choice == value { return true } } return false } func printChoices(out io.Writer, heading string, choices []string) error { if _, err := fmt.Fprintf(out, "%s:\n", heading); err != nil { return err } for _, choice := range choices { if _, err := fmt.Fprintf(out, "%s\n", choice); err != nil { return err } } return nil } func validateTestCase(value string) error { if value == "" { return nil } if _, err := regexp.Compile(value); err != nil { return fmt.Errorf("--case must be a valid Go test regexp: %w", err) } return nil } func loadHarnessDefaults(path string) (harnessDefaults, error) { data, err := os.ReadFile(path) if err != nil { return harnessDefaults{}, fmt.Errorf("read harness defaults: %w", err) } var defaults harnessDefaults decoder := toml.NewDecoder(bytes.NewReader(data)) decoder.DisallowUnknownFields() if err := decoder.Decode(&defaults); err != nil { return harnessDefaults{}, fmt.Errorf("decode harness defaults: %w", err) } if defaults.Version != 1 { return harnessDefaults{}, fmt.Errorf("unsupported harness defaults version %d", defaults.Version) } limits := defaults.Limits if limits.MaxConcurrentRuns != 1 || limits.GoParallelism == 0 || limits.RuntimeCPUs == 0 || limits.RuntimeMemoryBytes == 0 || limits.RuntimePIDs == 0 || limits.ComponentLogTailBytes == 0 || limits.FailureBundleBytes == 0 || limits.SuccessReportBytes == 0 || limits.FreeSpaceFloorBytes == 0 { return harnessDefaults{}, errors.New("harness defaults contain a zero or unsupported resource limit") } if limits.SuccessReportBytes > limits.FailureBundleBytes { return harnessDefaults{}, errors.New("success report budget exceeds failure bundle budget") } for field, value := range map[string]string{"unit_timeout": limits.UnitTimeout, "integration_timeout": limits.IntegrationTimeout, "e2e_timeout": limits.E2ETimeout, "soak_timeout": limits.SoakTimeout} { duration, durationErr := time.ParseDuration(value) if durationErr != nil || duration <= 0 { return harnessDefaults{}, fmt.Errorf("harness default %s must be a positive duration", field) } } return defaults, nil } func (h *harness) definitionHash(relative string) string { if h.repoRoot == "" { return "unavailable" } data, err := os.ReadFile(filepath.Join(h.repoRoot, relative)) if err != nil { return "unavailable" } sum := sha256.Sum256(data) return hex.EncodeToString(sum[:]) } func (h *harness) verifyResume(current *manifest) error { commit := strings.TrimSpace(commandOutput("git", "rev-parse", "HEAD")) if current.GitCommit != "unavailable" && commit != current.GitCommit { return fmt.Errorf("run commit %q does not match current commit %q; start a new run", current.GitCommit, commit) } dirty := repositoryDirtyHash() if current.DirtyDiffSHA256 != hex.EncodeToString(dirty[:]) { return errors.New("run dirty-diff hash does not match the working tree; start a new run") } if want := h.definitionHash("test/harness/defaults.toml"); current.HarnessDefaultsSHA256 != "" && current.HarnessDefaultsSHA256 != "unavailable" && want != current.HarnessDefaultsSHA256 { return errors.New("harness defaults changed since the run began; start a new run") } if want := h.definitionHash("deploy/compose.test.yaml"); current.ComposeDefinitionSHA256 != "" && current.ComposeDefinitionSHA256 != "unavailable" && want != current.ComposeDefinitionSHA256 { return errors.New("test Compose definition changed since the run began; start a new run") } return nil } func (h *harness) componentLogLimit() int { if h.defaults.Limits.ComponentLogTailBytes == 0 || h.defaults.Limits.ComponentLogTailBytes > uint64(^uint(0)>>1) { return maxSuiteLogSize } return int(h.defaults.Limits.ComponentLogTailBytes) } func runCLI(ctx context.Context, args []string) error { repoRoot, err := os.Getwd() if err != nil { return err } defaults, err := loadHarnessDefaults(filepath.Join(repoRoot, "test", "harness", "defaults.toml")) if err != nil { return err } h := &harness{root: filepath.Join(repoRoot, ".test-runs"), repoRoot: repoRoot, defaults: defaults, now: func() time.Time { return time.Now().UTC() }, out: os.Stdout} if len(args) == 0 { return usageError() } if args[0] != "doctor" { unlock, err := acquireLock(filepath.Join(repoRoot, ".test-harness.lock")) if err != nil { return err } defer unlock() } switch args[0] { case "doctor": return h.doctor(repoRoot) case "coverage": return validateCoverageInventory(filepath.Join(repoRoot, "test", "coverage.toml")) case "integration": return h.integration(ctx, args[1:]) case "e2e": return h.e2e(ctx, args[1:]) case "status", "logs", "collect", "recover", "reuse", "stop", "reset", "purge", "gc": return h.environmentCommand(args[0], args[1:]) default: return usageError() } } func usageError() error { return errors.New("usage: harness doctor|coverage|integration|e2e|status|logs|collect|recover|reuse|stop|reset|purge|gc") } func (h *harness) doctor(repoRoot string) error { for _, file := range []string{"go.mod", "deploy/compose.yaml", "docs/implementation-plan.v1.md"} { if _, err := os.Stat(filepath.Join(repoRoot, file)); err != nil { return fmt.Errorf("repository check %s: %w", file, err) } } if h.defaults.Version != 1 { return errors.New("harness defaults were not loaded") } fmt.Fprintf(h.out, "RVBox test harness policy: max_runs=%d go_parallelism=%d free_space_floor_bytes=%d\n", h.defaults.Limits.MaxConcurrentRuns, h.defaults.Limits.GoParallelism, h.defaults.Limits.FreeSpaceFloorBytes) fmt.Fprintln(h.out, "RVBox test harness is ready; native Windows scenarios use the explicit scripts/windows/test-host host lane.") return nil } func (h *harness) integration(ctx context.Context, args []string) error { flags := flag.NewFlagSet("integration", flag.ContinueOnError) flags.SetOutput(io.Discard) suite := flags.String("suite", "sample", "suite name") list := flags.Bool("list", false, "list stable suites") testCase := flags.String("case", "", "Go test name regexp for one suite") runID := flags.String("run-id", "", "run ID") resume := flags.Bool("resume", false, "resume an existing run") if err := flags.Parse(args); err != nil { return err } if *list { if flags.NArg() != 0 || *runID != "" || *resume || *testCase != "" || *suite != "sample" { return errors.New("--list cannot be combined with run options") } return printChoices(h.out, "integration suites", integrationSuites) } if !containsChoice(integrationSuites, *suite) { return fmt.Errorf("suite %q is not implemented yet; available: %s", *suite, strings.Join(integrationSuites, ", ")) } if err := validateTestCase(*testCase); err != nil { return err } if (*suite == "sample" || *suite == "all") && *testCase != "" { return errors.New("--case is only supported by one executable integration suite") } var current *manifest var err error if *resume { if *runID == "" { return errors.New("--resume requires --run-id") } current, err = h.load(*runID) if err != nil { return err } if err := h.verifyResume(current); err != nil { return err } if current.Layer != "integration" || current.Suite != *suite { return errors.New("run layer/suite does not match resume request") } if current.TestCase != *testCase { return errors.New("run test case does not match resume request") } if current.Phase != "ready" && current.Phase != "interrupted" && current.Phase != "stopped" && current.Phase != "running" && current.Phase != "failed" { return fmt.Errorf("run in phase %q is not resumable; recover or reuse it first", current.Phase) } } else { current, err = h.create(*runID, "integration", *suite) if err != nil { return err } } current.TestCase = *testCase if err := h.writeManifest(current); err != nil { return err } fmt.Fprintln(h.out, current.RunID) if err := h.transition(current, "running", *suite+"-start", *suite+" integration run started"); err != nil { return err } select { case <-ctx.Done(): _ = h.transition(current, "interrupted", "interrupt", ctx.Err().Error()) return ctx.Err() default: } suites := []string{*suite} if *suite == "all" { suites = []string{"sample", "store", "server-session", "control", "client-agent"} } for _, item := range suites { if suiteErr := h.runIntegrationSuite(ctx, current, item, *testCase); suiteErr != nil { _ = h.transition(current, "failed", item+"-failed", suiteErr.Error()) return suiteErr } } return h.transition(current, "completed", *suite+"-complete", *suite+" integration run completed") } func (h *harness) runIntegrationSuite(ctx context.Context, current *manifest, suite, testCase string) error { switch suite { case "sample": return h.appendJournal(current.RunID, journalEntry{At: h.now(), Step: "invariant", Status: "passed", Detail: "manifest ownership and journal durability verified"}) case "store": return h.runStoreSuite(ctx, current, testCase) case "server-session": return h.runServerSessionSuite(ctx, current, testCase) case "control": return h.runGoSuite(ctx, current, "control-compiled-server-rvc", "running compiled server/rvc control-plane boundary cases", "./test/integration/controlplane", testCase) case "client-agent": return h.runGoSuite(ctx, current, "client-agent-real-websocket", "running real client/server WebSocket control-flow cases", "./test/integration/clientagent", testCase) default: return fmt.Errorf("unknown integration suite %q", suite) } } // e2e runs production-shaped Go scenarios against real local listeners and // durable stores. Native Windows work is an additional host lane invoked by // scripts/windows/test-host.ps1; it is never silently replaced by Wine or a // cross-compiled binary. The run manifest/journal makes every scenario // resumable and keeps artifacts bounded. func (h *harness) e2e(ctx context.Context, args []string) error { flags := flag.NewFlagSet("e2e", flag.ContinueOnError) flags.SetOutput(io.Discard) scenario := flags.String("scenario", "smoke", "scenario name") list := flags.Bool("list", false, "list stable scenarios") testCase := flags.String("case", "", "Go test name regexp for one scenario") runID := flags.String("run-id", "", "run ID") resume := flags.Bool("resume", false, "resume an existing run") if err := flags.Parse(args); err != nil { return err } if *list { if flags.NArg() != 0 || *runID != "" || *resume || *testCase != "" || *scenario != "smoke" { return errors.New("--list cannot be combined with run options") } return printChoices(h.out, "e2e scenarios", e2eScenarios) } if !containsChoice(e2eScenarios, *scenario) { return fmt.Errorf("scenario %q is not implemented yet; available: %s", *scenario, strings.Join(e2eScenarios, ", ")) } if err := validateTestCase(*testCase); err != nil { return err } if *scenario == "all" && *testCase != "" { return errors.New("--case requires one named e2e scenario") } var current *manifest var err error if *resume { if *runID == "" { return errors.New("--resume requires --run-id") } current, err = h.load(*runID) if err != nil { return err } if err := h.verifyResume(current); err != nil { return err } if current.Layer != "e2e" || current.Suite != *scenario { return errors.New("run layer/scenario does not match resume request") } if current.TestCase != *testCase { return errors.New("run test case does not match resume request") } if current.Phase != "ready" && current.Phase != "interrupted" && current.Phase != "stopped" && current.Phase != "running" && current.Phase != "failed" { return fmt.Errorf("run in phase %q is not resumable; recover or reuse it first", current.Phase) } } else { current, err = h.create(*runID, "e2e", *scenario) if err != nil { return err } } current.TestCase = *testCase if err := h.writeManifest(current); err != nil { return err } fmt.Fprintln(h.out, current.RunID) if err := h.transition(current, "running", "e2e-start", "e2e scenario started"); err != nil { return err } scenarios := []string{*scenario} if *scenario == "all" { scenarios = []string{"smoke", "interactive", "idempotency", "reconnect", "retention", "expiry-and-incidents", "script", "recovery"} } for _, item := range scenarios { if err := h.runE2EScenario(ctx, current, item, *testCase); err != nil { _ = h.transition(current, "failed", "e2e-"+item+"-failed", err.Error()) return err } } return h.transition(current, "completed", "e2e-complete", "e2e scenario completed") } func (h *harness) runE2EScenario(ctx context.Context, current *manifest, scenario, testCase string) error { if err := h.appendJournal(current.RunID, journalEntry{At: h.now(), Step: "e2e-" + scenario, Status: "running", Detail: "scenario started"}); err != nil { return err } var err error switch scenario { case "smoke": err = h.runGoSuite(ctx, current, "e2e-client-agent", "real client/server WebSocket and control flow", "./test/integration/clientagent", testCase) case "interactive": err = h.runGoSuite(ctx, current, "e2e-stdin-protocol", "durable stdin append, replay, close, and acknowledgement flow", "./test/integration/clientagent", chooseTestCase(testCase, "^TestControlStdinIntentReplaysAndAcknowledges_HP_DISPATCH_08$")) case "idempotency": err = h.runGoSuite(ctx, current, "e2e-request-idempotency", "request UUID replay and immutable request-hash conflict handling", "./internal/server/control", chooseTestCase(testCase, "^TestRunCommandRequestIDIdempotencyAndValidation_BH_CONTROL_02$")) case "reconnect": err = h.runGoSuite(ctx, current, "e2e-reconnect-reconciliation", "WebSocket reconciliation, fenced dispatch, and session wake handling", "./test/integration/clientagent", chooseTestCase(testCase, "^(TestWebSocketHelloWelcome_HP_SES_06|TestWebSocketDispatchAfterReconciliation_HP_DISPATCH_03|TestControlQueueWakesReconciledSession_HP_DISPATCH_06)$")) case "retention": err = h.runStoreSuite(ctx, current, chooseTestCase(testCase, "^(TestTerminalAgeRetentionEvictsWholeCommand_HP_STORE_08|TestTombstoneFIFOIsCappedInEvictionTransaction_HP_STORE_09|TestRetentionCrashStagesRollForward_CRASH_STORE_04)$")) case "expiry-and-incidents": err = h.runStoreSuite(ctx, current, chooseTestCase(testCase, "^(TestIncidentDirtyResolutionIdempotencyAndRecurrence_HP_STORE_11|TestIrreparableIncidentRequiresExplicitAcknowledgement_BH_STORE_09|TestFilesystemFloorAndCounterMismatch_BH_STORE_08)$")) case "script": err = h.runGoSuite(ctx, current, "e2e-script-transfer", "durable script transfer and replay", "./internal/client/agent", testCase) case "recovery": err = h.runStoreSuite(ctx, current, testCase) default: err = fmt.Errorf("unknown e2e scenario %q", scenario) } if err != nil { return err } return h.appendJournal(current.RunID, journalEntry{At: h.now(), Step: "e2e-" + scenario, Status: "passed", Detail: "scenario passed"}) } func chooseTestCase(override, defaultCase string) string { if override != "" { return override } return defaultCase } func (h *harness) runStoreSuite(ctx context.Context, current *manifest, testCase string) error { if err := h.appendJournal(current.RunID, journalEntry{At: h.now(), Step: "store-real-sqlite", Status: "running", Detail: "running real SQLite/WAL and filesystem cases"}); err != nil { return err } artifactDir := filepath.Join(h.runDir(current.RunID), "artifacts") if err := os.MkdirAll(artifactDir, 0o700); err != nil { return err } capture := &limitedCapture{limit: h.componentLogLimit()} arguments := []string{"test", "-count=1", "-tags=integration", "-shuffle=" + strconv.FormatInt(current.Seed, 10), "-timeout=2m"} if testCase != "" { arguments = append(arguments, "-run", testCase) } arguments = append(arguments, "./test/integration/store") command := exec.CommandContext(ctx, "go", arguments...) command.Stdout = capture command.Stderr = capture err := command.Run() logPath := filepath.Join(artifactDir, "suite.log") if writeErr := atomicWrite(logPath, capture.Bytes(), 0o600); writeErr != nil { return writeErr } if err != nil { return fmt.Errorf("store suite failed (bounded log %s): %w", logPath, err) } return h.appendJournal(current.RunID, journalEntry{At: h.now(), Step: "store-real-sqlite", Status: "passed", Detail: "real SQLite/WAL and filesystem cases passed"}) } func (h *harness) runServerSessionSuite(ctx context.Context, current *manifest, testCase string) error { return h.runGoSuite(ctx, current, "server-session-real-websocket", "running real HTTP/WebSocket, protobuf, SQLite, and fencing cases", "./internal/server/session", testCase) } func (h *harness) runGoSuite(ctx context.Context, current *manifest, step, detail, packagePath, testCase string) error { if err := h.appendJournal(current.RunID, journalEntry{At: h.now(), Step: step, Status: "running", Detail: detail}); err != nil { return err } artifactDir := filepath.Join(h.runDir(current.RunID), "artifacts") if err := os.MkdirAll(artifactDir, 0o700); err != nil { return err } capture := &limitedCapture{limit: h.componentLogLimit()} arguments := []string{"test", "-race", "-count=1", "-shuffle=" + strconv.FormatInt(current.Seed, 10), "-timeout=2m"} if testCase != "" { arguments = append(arguments, "-run", testCase) } arguments = append(arguments, packagePath) command := exec.CommandContext(ctx, "go", arguments...) command.Stdout = capture command.Stderr = capture err := command.Run() logPath := filepath.Join(artifactDir, "suite.log") if writeErr := atomicWrite(logPath, capture.Bytes(), 0o600); writeErr != nil { return writeErr } if err != nil { return fmt.Errorf("%s suite failed (bounded log %s): %w", current.Suite, logPath, err) } return h.appendJournal(current.RunID, journalEntry{At: h.now(), Step: step, Status: "passed", Detail: detail + " passed"}) } func (h *harness) environmentCommand(command string, args []string) error { if command == "gc" { return h.gc(args) } flags := flag.NewFlagSet(command, flag.ContinueOnError) flags.SetOutput(io.Discard) runID := flags.String("run-id", "", "run ID") newRunID := flags.String("new-run-id", "", "fresh run ID for reuse") all := flags.Bool("all", false, "select every eligible run") execute := flags.Bool("execute", false, "perform a destructive cleanup") yes := flags.Bool("yes", false, "confirm destructive cleanup in noninteractive use") if err := flags.Parse(args); err != nil { return err } if command != "purge" && (*all || *execute || *yes) { return fmt.Errorf("%s does not accept cleanup options", command) } if command != "reuse" && *newRunID != "" { return fmt.Errorf("%s does not accept --new-run-id", command) } if command == "purge" { if (*runID == "") == (!*all) { return errors.New("purge requires exactly one of --run-id or --all") } return h.purgeCommand(*runID, *all, *execute, *yes) } if *runID == "" { return fmt.Errorf("%s requires --run-id", command) } current, err := h.load(*runID) if err != nil { return err } switch command { case "status": resumeErr := h.verifyResume(current) status := map[string]any{"manifest": current, "run_bytes": directorySize(h.runDir(current.RunID)), "resume_valid": resumeErr == nil} if resumeErr != nil { status["resume_error"] = resumeErr.Error() } encoded, _ := json.MarshalIndent(status, "", " ") fmt.Fprintln(h.out, string(encoded)) return nil case "logs": data, err := os.ReadFile(h.journalPath(current.RunID)) if err != nil { return err } _, err = h.out.Write(data) return err case "collect": return h.collect(current) case "recover": if err := h.verifyResume(current); err != nil { return err } if current.Phase != "interrupted" && current.Phase != "stopped" && current.Phase != "running" && current.Phase != "failed" { return fmt.Errorf("run in phase %q does not need recovery", current.Phase) } return h.transition(current, "ready", "recover", "run recovered and ready to resume") case "reuse": return h.reuse(current, *newRunID) case "stop": return h.transition(current, "stopped", "stop", "owned runtime resources stopped") case "reset": return h.reset(current) default: return usageError() } } func (h *harness) reuse(previous *manifest, newRunID string) error { if previous.Phase != "reset" && previous.Phase != "completed" { return fmt.Errorf("run in phase %q cannot be reused; reset or complete it first", previous.Phase) } if err := h.verifyResume(previous); err != nil { return err } fresh, err := h.create(newRunID, previous.Layer, previous.Suite) if err != nil { return err } fresh.TestCase = previous.TestCase if err := h.writeManifest(fresh); err != nil { return err } if err := h.appendJournal(fresh.RunID, journalEntry{At: h.now(), Step: "reuse", Status: "passed", Detail: "fresh run created from immutable prior definitions", Data: map[string]any{"prior_run_id": previous.RunID}}); err != nil { return err } fmt.Fprintf(h.out, "reused_from=%s new_run_id=%s\n", previous.RunID, fresh.RunID) return nil } func (h *harness) reset(current *manifest) error { if current.Phase == "running" { return errors.New("refusing to reset a running run; stop it first") } if len(current.OwnedResources) != 0 { return errors.New("refusing to reset a run with owned runtime resources; stop and reconcile them first") } for _, name := range []string{"runtime", "pki", "scratch"} { path := filepath.Join(h.runDir(current.RunID), name) info, err := os.Lstat(path) if os.IsNotExist(err) { continue } if err != nil { return err } if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { return fmt.Errorf("refusing unsafe reset path %s", path) } if err := os.RemoveAll(path); err != nil { return err } } return h.transition(current, "reset", "reset", "owned runtime scratch reset; manifest, journal, and reports retained") } func (h *harness) purgeCommand(runID string, all, execute, yes bool) error { var candidates []*manifest if all { loaded, err := h.eligiblePurgeManifests(time.Time{}) if err != nil { return err } candidates = loaded } else { current, err := h.load(runID) if err != nil { return err } if err := h.validatePurge(current); err != nil { return err } candidates = []*manifest{current} } if len(candidates) == 0 { fmt.Fprintln(h.out, "no eligible RVBox test runs") return nil } for _, current := range candidates { fmt.Fprintf(h.out, "purge_target=%s phase=%s bytes=%d\n", h.runDir(current.RunID), current.Phase, directorySize(h.runDir(current.RunID))) } if !execute { fmt.Fprintln(h.out, "dry run only; rerun with --execute --yes to remove exactly these validated targets") return nil } if !yes { return errors.New("purge --execute requires --yes in the noninteractive harness") } for _, current := range candidates { if err := h.purge(current); err != nil { return err } } return nil } func (h *harness) gc(args []string) error { flags := flag.NewFlagSet("gc", flag.ContinueOnError) flags.SetOutput(io.Discard) olderThan := flags.String("older-than", "", "minimum age of completed/reset runs") execute := flags.Bool("execute", false, "perform a destructive cleanup") yes := flags.Bool("yes", false, "confirm destructive cleanup in noninteractive use") if err := flags.Parse(args); err != nil { return err } if *olderThan == "" { return errors.New("gc requires --older-than") } age, err := time.ParseDuration(*olderThan) if err != nil || age <= 0 { return errors.New("gc --older-than must be a positive duration") } candidates, err := h.eligiblePurgeManifests(h.now().Add(-age)) if err != nil { return err } if len(candidates) == 0 { fmt.Fprintln(h.out, "no eligible RVBox test runs") return nil } for _, current := range candidates { fmt.Fprintf(h.out, "gc_target=%s phase=%s bytes=%d\n", h.runDir(current.RunID), current.Phase, directorySize(h.runDir(current.RunID))) } if !*execute { fmt.Fprintln(h.out, "dry run only; rerun with --execute --yes to remove exactly these validated targets") return nil } if !*yes { return errors.New("gc --execute requires --yes in the noninteractive harness") } for _, current := range candidates { if err := h.purge(current); err != nil { return err } } return nil } func (h *harness) eligiblePurgeManifests(before time.Time) ([]*manifest, error) { entries, err := os.ReadDir(h.root) if os.IsNotExist(err) { return nil, nil } if err != nil { return nil, err } result := make([]*manifest, 0, len(entries)) for _, entry := range entries { if !entry.IsDir() || entry.Type()&os.ModeSymlink != 0 || validateRunID(entry.Name()) != nil { continue } current, loadErr := h.load(entry.Name()) if loadErr != nil || h.validatePurge(current) != nil { continue } if !before.IsZero() && !current.UpdatedAt.Before(before) { continue } result = append(result, current) } return result, nil } type limitedCapture struct { data []byte limit int truncated bool } func (capture *limitedCapture) Write(data []byte) (int, error) { written := len(data) remaining := capture.limit - len(capture.data) if remaining > 0 { if len(data) > remaining { data = data[:remaining] } capture.data = append(capture.data, data...) } if written > remaining { capture.truncated = true } return written, nil } func (capture *limitedCapture) Bytes() []byte { if !capture.truncated { return capture.data } return append(append([]byte(nil), capture.data...), []byte("\n[output truncated by RVBox test harness]\n")...) } func (h *harness) create(requestedID, layer, suite string) (*manifest, error) { if requestedID == "" { id, err := domain.NewUUIDv7() if err != nil { return nil, err } requestedID = id.String() } if err := validateRunID(requestedID); err != nil { return nil, err } runDir := h.runDir(requestedID) if err := os.MkdirAll(h.root, 0o700); err != nil { return nil, err } if err := os.Mkdir(runDir, 0o700); err != nil { return nil, fmt.Errorf("create run %s: %w", requestedID, err) } seedValue, err := rand.Int(rand.Reader, big.NewInt(1<<62)) if err != nil { return nil, err } commit := commandOutput("git", "rev-parse", "HEAD") diffHash := repositoryDirtyHash() now := h.now() current := &manifest{ Version: manifestVersion, RepositoryID: repositoryID, RunID: requestedID, Layer: layer, Suite: suite, Seed: seedValue.Int64(), GitCommit: strings.TrimSpace(commit), DirtyDiffSHA256: hex.EncodeToString(diffHash[:]), ToolchainImage: os.Getenv("RVBOX_TOOLCHAIN_IMAGE"), HarnessDefaultsSHA256: h.definitionHash("test/harness/defaults.toml"), ComposeDefinitionSHA256: h.definitionHash("deploy/compose.test.yaml"), EffectiveLimits: h.defaults.Limits, ComposeProject: "rvbox-test-" + requestedID, Phase: "created", CreatedAt: now, UpdatedAt: now, OwnedResources: []string{}, } if err := h.writeManifest(current); err != nil { _ = os.Remove(runDir) return nil, err } if err := h.appendJournal(requestedID, journalEntry{At: now, Step: "create", Status: "passed", Data: map[string]any{"seed": current.Seed}}); err != nil { return nil, err } return current, nil } func (h *harness) transition(current *manifest, phase, step, detail string) error { current.Phase = phase current.UpdatedAt = h.now() if err := h.writeManifest(current); err != nil { return err } return h.appendJournal(current.RunID, journalEntry{At: current.UpdatedAt, Step: step, Status: phase, Detail: detail}) } func (h *harness) load(runID string) (*manifest, error) { if err := validateRunID(runID); err != nil { return nil, err } data, err := os.ReadFile(h.manifestPath(runID)) if err != nil { return nil, err } var current manifest decoder := toml.NewDecoder(strings.NewReader(string(data))) decoder.DisallowUnknownFields() if err := decoder.Decode(¤t); err != nil { return nil, fmt.Errorf("decode run manifest: %w", err) } if current.Version != manifestVersion || current.RepositoryID != repositoryID || current.RunID != runID { return nil, errors.New("run manifest identity/version mismatch") } return ¤t, nil } func (h *harness) writeManifest(current *manifest) error { data, err := toml.Marshal(current) if err != nil { return err } return atomicWrite(h.manifestPath(current.RunID), data, 0o600) } func (h *harness) appendJournal(runID string, entry journalEntry) error { data, err := json.Marshal(entry) if err != nil { return err } file, err := os.OpenFile(h.journalPath(runID), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) if err != nil { return err } defer file.Close() if _, err := file.Write(append(data, '\n')); err != nil { return err } return file.Sync() } func (h *harness) collect(current *manifest) error { reportDir := filepath.Join(h.runDir(current.RunID), "artifacts") if err := os.MkdirAll(reportDir, 0o700); err != nil { return err } resumeErr := h.verifyResume(current) report := map[string]any{"run_id": current.RunID, "layer": current.Layer, "suite": current.Suite, "phase": current.Phase, "collected_at": h.now(), "payloads_included": false, "run_bytes": directorySize(h.runDir(current.RunID)), "resume_valid": resumeErr == nil} if resumeErr != nil { report["resume_error"] = resumeErr.Error() } data, _ := json.MarshalIndent(report, "", " ") path := filepath.Join(reportDir, "report.json") if err := atomicWrite(path, append(data, '\n'), 0o600); err != nil { return err } failed := current.Phase == "failed" suite := junitSuite{Name: current.Layer + "/" + current.Suite, Tests: 1, TestCase: junitCase{Name: current.TestCase, Class: current.Layer + "/" + current.Suite}} if suite.TestCase.Name == "" { suite.TestCase.Name = current.Suite } if failed { suite.Failures = 1 suite.TestCase.Failure = &junitFailure{Message: "harness run failed; inspect bounded suite.log and journal.jsonl"} } encodedJUnit, err := xml.MarshalIndent(suite, "", " ") if err != nil { return err } if err := atomicWrite(filepath.Join(reportDir, "junit.xml"), append(append([]byte(xml.Header), encodedJUnit...), '\n'), 0o600); err != nil { return err } if err := h.appendJournal(current.RunID, journalEntry{At: h.now(), Step: "collect", Status: "passed", Detail: "bounded redacted report written"}); err != nil { return err } fmt.Fprintln(h.out, path) return nil } func (h *harness) purge(current *manifest) error { if err := h.validatePurge(current); err != nil { return err } runDir := h.runDir(current.RunID) if err := os.RemoveAll(runDir); err != nil { return err } fmt.Fprintf(h.out, "purged %s (not recoverable)\n", runDir) return nil } func (h *harness) validatePurge(current *manifest) error { if current.Phase != "completed" && current.Phase != "reset" { return fmt.Errorf("refusing to purge run in phase %q; reset it first", current.Phase) } runDir := h.runDir(current.RunID) info, err := os.Lstat(runDir) if err != nil { return err } if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { return errors.New("refusing to purge a symlink or non-directory run path") } if len(current.OwnedResources) != 0 { return errors.New("refusing to purge while manifest still lists owned runtime resources; reset first") } return nil } func directorySize(path string) uint64 { var total uint64 _ = filepath.WalkDir(path, func(_ string, entry os.DirEntry, err error) error { if err != nil || entry.Type().IsDir() || entry.Type()&os.ModeSymlink != 0 { return nil } if info, infoErr := entry.Info(); infoErr == nil && info.Size() > 0 { total += uint64(info.Size()) } return nil }) return total } func (h *harness) runDir(runID string) string { return filepath.Join(h.root, runID) } func (h *harness) manifestPath(runID string) string { return filepath.Join(h.runDir(runID), "run.toml") } func (h *harness) journalPath(runID string) string { return filepath.Join(h.runDir(runID), "journal.jsonl") } func validateRunID(runID string) error { if !runIDPattern.MatchString(runID) || runID == "." || runID == ".." { return fmt.Errorf("invalid filesystem-safe run ID %q", runID) } return nil } func atomicWrite(path string, data []byte, mode os.FileMode) error { directory := filepath.Dir(path) temporary, err := os.CreateTemp(directory, ".write-") if err != nil { return err } temporaryPath := temporary.Name() defer os.Remove(temporaryPath) if err := temporary.Chmod(mode); err != nil { _ = temporary.Close() return err } if _, err := temporary.Write(data); err != nil { _ = temporary.Close() return err } if err := temporary.Sync(); err != nil { _ = temporary.Close() return err } if err := temporary.Close(); err != nil { return err } if err := os.Rename(temporaryPath, path); err != nil { return err } dir, err := os.Open(directory) if err != nil { return err } defer dir.Close() return dir.Sync() } func commandOutput(name string, args ...string) string { output, err := exec.Command(name, args...).Output() if err != nil { return "unavailable" } return string(output) } func repositoryDirtyHash() [sha256.Size]byte { hash := sha256.New() tracked := commandBytes("git", "diff", "--binary", "HEAD", "--", ".") _, _ = hash.Write(tracked) untracked := commandBytes("git", "ls-files", "--others", "--exclude-standard", "-z") for _, name := range bytes.Split(untracked, []byte{0}) { if len(name) == 0 { continue } _, _ = hash.Write([]byte{0}) _, _ = hash.Write(name) if data, err := os.ReadFile(string(name)); err == nil { _, _ = hash.Write([]byte{0}) _, _ = hash.Write(data) } } var result [sha256.Size]byte copy(result[:], hash.Sum(nil)) return result } func commandBytes(name string, args ...string) []byte { output, err := exec.Command(name, args...).Output() if err != nil { return []byte("unavailable") } return output } type coverageInventory struct { Version uint32 `toml:"version"` Requirements []coverageRequirement `toml:"requirements"` } type coverageRequirement struct { ID string `toml:"id"` Layer string `toml:"layer"` Status string `toml:"status"` Tests []string `toml:"tests"` } var coverageIDPattern = regexp.MustCompile(`^(HP|BH|ERR|RACE|CRASH|SEC|BOUND|REC)-[A-Z0-9]+-[0-9]{2}$`) func validateCoverageInventory(path string) error { absolutePath, err := filepath.Abs(path) if err != nil { return err } path = absolutePath data, err := os.ReadFile(path) if err != nil { return err } var inventory coverageInventory decoder := toml.NewDecoder(bytes.NewReader(data)) decoder.DisallowUnknownFields() if err := decoder.Decode(&inventory); err != nil { return err } if inventory.Version != 1 || len(inventory.Requirements) == 0 { return errors.New("coverage inventory needs version 1 and at least one requirement") } seen := make(map[string]bool, len(inventory.Requirements)) repositoryRoot := filepath.Dir(filepath.Dir(path)) for _, requirement := range inventory.Requirements { if !coverageIDPattern.MatchString(requirement.ID) || seen[requirement.ID] { return fmt.Errorf("invalid or duplicate coverage ID %q", requirement.ID) } seen[requirement.ID] = true if requirement.Layer != "unit" && requirement.Layer != "integration" && requirement.Layer != "e2e" { return fmt.Errorf("coverage %s has invalid layer %q", requirement.ID, requirement.Layer) } if requirement.Status != "implemented" && requirement.Status != "planned" && requirement.Status != "blocked_native_windows" { return fmt.Errorf("coverage %s has invalid status %q", requirement.ID, requirement.Status) } if requirement.Status == "implemented" && len(requirement.Tests) == 0 { return fmt.Errorf("implemented coverage %s has no tests", requirement.ID) } for _, reference := range requirement.Tests { parts := strings.Split(reference, ":") if len(parts) != 2 || parts[0] == "" || parts[1] == "" || filepath.IsAbs(parts[0]) || strings.Contains(parts[0], "..") { return fmt.Errorf("coverage %s has invalid test reference %q", requirement.ID, reference) } source, err := os.ReadFile(filepath.Join(repositoryRoot, filepath.FromSlash(parts[0]))) if err != nil { return fmt.Errorf("coverage %s test reference: %w", requirement.ID, err) } if !bytes.Contains(source, []byte("func "+parts[1]+"(")) { return fmt.Errorf("coverage %s test function %q not found", requirement.ID, parts[1]) } } } return nil }