feat: complete Windows client control and recovery paths
This commit is contained in:
@@ -219,9 +219,8 @@ func (service *Service) GetCommand(ctx context.Context, request *rvboxv1.GetComm
|
||||
return &rvboxv1.GetCommandResponse{Command: record}, nil
|
||||
}
|
||||
|
||||
// RunCommand durably admits command-text work. Script payload persistence and
|
||||
// chunk dispatch are intentionally kept behind the same immutable boundary and
|
||||
// are rejected until their command_payloads path is wired to the dispatcher.
|
||||
// RunCommand durably admits command-text and script work. Both forms share one
|
||||
// immutable request boundary so replaying the same request ID is idempotent.
|
||||
func (service *Service) RunCommand(ctx context.Context, request *rvboxv1.RunCommandRequest) (*rvboxv1.RunCommandResponse, error) {
|
||||
if request == nil || request.GetTargetClientId() == "" || request.GetSpec() == nil {
|
||||
return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "target_client_id and spec are required")
|
||||
@@ -375,7 +374,10 @@ func (service *Service) GetOutput(ctx context.Context, request *rvboxv1.GetOutpu
|
||||
if maxBytes > 16<<20 {
|
||||
return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "max_bytes exceeds the control limit")
|
||||
}
|
||||
filterBytes := []byte(fmt.Sprintf("output\x00%s\x00%v", request.GetClientId(), streams))
|
||||
// Bind the cursor to every selector that affects the result. In particular,
|
||||
// issue_uuid must be included: otherwise a token issued for one command
|
||||
// could be replayed against another command owned by the same client.
|
||||
filterBytes := []byte(fmt.Sprintf("output\x00%s\x00%s\x00%v", request.GetClientId(), issue.String(), streams))
|
||||
filter := domain.HashCursorFilters(filterBytes)
|
||||
var eventSeq, offset, boundary uint64
|
||||
if request.GetPageToken() != "" {
|
||||
@@ -793,7 +795,14 @@ func commandRecord(view store.CommandView) (*rvboxv1.CommandRecord, error) {
|
||||
if err := proto.Unmarshal(view.ExecutionSpec, spec); err != nil {
|
||||
return nil, fmt.Errorf("decode stored execution spec: %w", err)
|
||||
}
|
||||
result := &rvboxv1.CommandRecord{IssueUuid: view.IssueUUID.String(), TargetClientId: view.ClientID, IssueTime: timestamppb.New(view.IssueTime), ServerReceiptTime: timestamppb.New(view.ServerReceiptTime), Spec: spec, Lifecycle: rvboxv1.CommandLifecycle(view.Lifecycle), LastEventSeq: view.LastEventSeq, OutputTruncated: view.OutputTruncated, OutputIncomplete: view.OutputIncomplete, RetainedCompressedBytes: view.RetainedCompressedBytes, CommandRevision: view.Revision}
|
||||
result := &rvboxv1.CommandRecord{IssueUuid: view.IssueUUID.String(), TargetClientId: view.ClientID, IssueTime: timestamppb.New(view.IssueTime), ServerReceiptTime: timestamppb.New(view.ServerReceiptTime), Spec: spec, Lifecycle: rvboxv1.CommandLifecycle(view.Lifecycle), LastEventSeq: view.LastEventSeq, OutputTruncated: view.OutputTruncated, OutputIncomplete: view.OutputIncomplete, RetainedCompressedBytes: view.RetainedCompressedBytes, CommandRevision: view.Revision, LateAfterExpiry: view.LateAfterExpiry}
|
||||
if len(view.Rejection) > 0 {
|
||||
rejection := &rvboxv1.ControlError{}
|
||||
if err := proto.Unmarshal(view.Rejection, rejection); err != nil {
|
||||
return nil, fmt.Errorf("decode stored command rejection: %w", err)
|
||||
}
|
||||
result.Rejection = rejection
|
||||
}
|
||||
if view.QueueExpiryTime != nil {
|
||||
result.QueueExpiryTime = timestamppb.New(*view.QueueExpiryTime)
|
||||
}
|
||||
|
||||
@@ -240,6 +240,10 @@ func TestGetOutputSlicesWithAuthenticatedCursor_HP_CONTROL_12(t *testing.T) {
|
||||
if _, err := service.GetOutput(context.Background(), &rvboxv1.GetOutputRequest{ClientId: "win-a", IssueUuid: issue.String(), Streams: []rvboxv1.StreamKind{rvboxv1.StreamKind_STREAM_STDERR}, MaxBytes: 3, PageToken: first.GetNextPageToken()}); status.Code(err) != codes.InvalidArgument {
|
||||
t.Fatalf("changed output filter code = %v", status.Code(err))
|
||||
}
|
||||
otherIssue := fixedIssue(0xa8)
|
||||
if _, err := service.GetOutput(context.Background(), &rvboxv1.GetOutputRequest{ClientId: "win-a", IssueUuid: otherIssue.String(), Streams: []rvboxv1.StreamKind{rvboxv1.StreamKind_STREAM_STDOUT}, MaxBytes: 3, PageToken: first.GetNextPageToken()}); status.Code(err) != codes.InvalidArgument {
|
||||
t.Fatalf("changed output command code = %v", status.Code(err))
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlMutationIdempotencyAndQueuedCancellation_BH_CONTROL_13(t *testing.T) {
|
||||
|
||||
@@ -64,18 +64,30 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
|
||||
http.Error(response, "agent session registry is unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
started := time.Now()
|
||||
heartbeat := newSynchronizedHeartbeat(server.heartbeatIdle(), server.livenessTimeout(), 0)
|
||||
connection, err := websocket.Accept(response, request, &websocket.AcceptOptions{
|
||||
CompressionMode: websocket.CompressionDisabled,
|
||||
// coder/websocket consumes control frames inside Read. Count ping and
|
||||
// pong callbacks as inbound activity so a healthy, otherwise idle agent
|
||||
// is not mistaken for a dead peer; application Read results are observed
|
||||
// by serveConnection as usual.
|
||||
OnPingReceived: func(context.Context, []byte) bool {
|
||||
heartbeat.Observe(time.Since(started))
|
||||
return true
|
||||
},
|
||||
OnPongReceived: func(context.Context, []byte) {
|
||||
heartbeat.Observe(time.Since(started))
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer connection.CloseNow()
|
||||
server.serveConnection(request.Context(), connection)
|
||||
server.serveConnection(request.Context(), connection, heartbeat, started)
|
||||
}
|
||||
|
||||
func (server *AgentServer) serveConnection(parent context.Context, connection *websocket.Conn) {
|
||||
started := time.Now()
|
||||
func (server *AgentServer) serveConnection(parent context.Context, connection *websocket.Conn, heartbeat *synchronizedHeartbeat, started time.Time) {
|
||||
messageType, payload, err := connection.Read(parent)
|
||||
if err != nil {
|
||||
return
|
||||
@@ -194,7 +206,6 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
|
||||
return
|
||||
}
|
||||
|
||||
heartbeat := newSynchronizedHeartbeat(server.heartbeatIdle(), server.livenessTimeout(), 0)
|
||||
writerDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(writerDone)
|
||||
@@ -266,7 +277,7 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
|
||||
server.close(connection, websocket.StatusPolicyViolation, "invalid command acceptance")
|
||||
return
|
||||
}
|
||||
if _, acceptErr := server.Store.RecordCommandAcceptance(sessionContext, issue, hello.GetClientId(), registration.Generation, acknowledgement.GetCommandRevision(), acknowledgement.GetAccepted(), server.now()); acceptErr != nil {
|
||||
if _, acceptErr := server.Store.RecordCommandAcceptanceWithRejection(sessionContext, issue, hello.GetClientId(), registration.Generation, acknowledgement.GetCommandRevision(), acknowledgement.GetAccepted(), acknowledgement.GetRejection(), server.now()); acceptErr != nil {
|
||||
server.close(connection, websocket.StatusPolicyViolation, "invalid command acceptance")
|
||||
return
|
||||
}
|
||||
@@ -360,6 +371,7 @@ func eventAppendFromWire(event *rvboxv1.CommandEvent, clientID string, generatio
|
||||
if lifecycle := event.GetLifecycle(); lifecycle != nil {
|
||||
value := lifecycle.GetLifecycle()
|
||||
result.Lifecycle, result.LifecycleRevision = &value, lifecycle.GetCommandRevision()
|
||||
result.WindowsIdentity = lifecycle.GetWindowsExecutionIdentity()
|
||||
}
|
||||
if output := event.GetOutput(); output != nil {
|
||||
result.Stream = uint16(output.GetStream())
|
||||
@@ -707,7 +719,6 @@ func (server *AgentServer) writeLoop(ctx context.Context, connection *websocket.
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
heartbeat.Observe(time.Since(started))
|
||||
case HeartbeatClose:
|
||||
return
|
||||
}
|
||||
|
||||
@@ -6,7 +6,9 @@ import (
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||
"github.com/rvbox/rvbox/internal/domain"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
var ErrDispatchMismatch = errors.New("command acceptance does not match dispatched generation or revision")
|
||||
@@ -15,9 +17,31 @@ var ErrDispatchMismatch = errors.New("command acceptance does not match dispatch
|
||||
// produced it. Repeated acknowledgements are harmless; a stale session cannot
|
||||
// advance a newer dispatch or overwrite a terminal decision.
|
||||
func (store *Store) RecordCommandAcceptance(ctx context.Context, issueUUID domain.UUID, clientID string, generation, revision uint64, accepted bool, now time.Time) (bool, error) {
|
||||
return store.RecordCommandAcceptanceWithRejection(ctx, issueUUID, clientID, generation, revision, accepted, nil, now)
|
||||
}
|
||||
|
||||
// RecordCommandAcceptanceWithRejection is the wire-facing variant that keeps
|
||||
// a structured client admission rejection on the command row for status and
|
||||
// history. The legacy method above remains convenient for reconciliation and
|
||||
// tests that only have the accepted bit.
|
||||
func (store *Store) RecordCommandAcceptanceWithRejection(ctx context.Context, issueUUID domain.UUID, clientID string, generation, revision uint64, accepted bool, rejection *rvboxv1.ControlError, now time.Time) (bool, error) {
|
||||
if isZeroUUID([16]byte(issueUUID)) || clientID == "" || generation == 0 || revision == 0 || now.IsZero() {
|
||||
return false, ErrDispatchMismatch
|
||||
}
|
||||
if accepted && rejection != nil {
|
||||
return false, ErrDispatchMismatch
|
||||
}
|
||||
var storedRejection []byte
|
||||
var err error
|
||||
if rejection != nil {
|
||||
if len(rejection.GetMessage()) > 4096 || len(rejection.GetIssueUuid()) > 128 {
|
||||
return false, ErrDispatchMismatch
|
||||
}
|
||||
storedRejection, err = proto.MarshalOptions{Deterministic: true}.Marshal(rejection)
|
||||
if err != nil || len(storedRejection) > 16<<10 {
|
||||
return false, ErrDispatchMismatch
|
||||
}
|
||||
}
|
||||
store.writeMu.Lock()
|
||||
defer store.writeMu.Unlock()
|
||||
database, err := store.openDatabase()
|
||||
@@ -28,15 +52,37 @@ func (store *Store) RecordCommandAcceptance(ctx context.Context, issueUUID domai
|
||||
if !accepted {
|
||||
targetLifecycle = 11
|
||||
}
|
||||
tx, err := database.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var queueExpiry sql.NullInt64
|
||||
var lateAfterExpiry int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT queue_expiry_time, late_after_expiry FROM commands WHERE issue_uuid = ? AND client_id = ? AND target_session_generation = ? AND revision = ?`, issueUUID[:], clientID, generation, revision).Scan(&queueExpiry, &lateAfterExpiry); errors.Is(err, sql.ErrNoRows) {
|
||||
_ = tx.Rollback()
|
||||
return false, ErrDispatchMismatch
|
||||
} else if err != nil {
|
||||
return false, err
|
||||
}
|
||||
query := `UPDATE commands SET lifecycle = ?`
|
||||
args := []any{targetLifecycle}
|
||||
if !accepted {
|
||||
query += `, terminal_time = ?`
|
||||
args = append(args, now.UTC().UnixNano())
|
||||
}
|
||||
if rejection != nil {
|
||||
query += `, rejection = ?`
|
||||
args = append(args, storedRejection)
|
||||
}
|
||||
// The command may have been dispatched just before its queue deadline and
|
||||
// its acceptance can arrive after that deadline. Preserve the eventual
|
||||
// client truth, but durably retain the contradiction for status/audit.
|
||||
query += `, late_after_expiry = CASE WHEN queue_expiry_time IS NOT NULL AND queue_expiry_time <= ? THEN 1 ELSE late_after_expiry END`
|
||||
args = append(args, now.UTC().UnixNano())
|
||||
query += ` WHERE issue_uuid = ? AND client_id = ? AND lifecycle = 2 AND target_session_generation = ? AND revision = ?`
|
||||
args = append(args, issueUUID[:], clientID, generation, revision)
|
||||
result, err := database.ExecContext(ctx, query, args...)
|
||||
result, err := tx.ExecContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -45,8 +91,20 @@ func (store *Store) RecordCommandAcceptance(ctx context.Context, issueUUID domai
|
||||
return false, err
|
||||
}
|
||||
if changed == 1 {
|
||||
late := queueExpiry.Valid && now.UTC().UnixNano() >= queueExpiry.Int64
|
||||
if late && lateAfterExpiry == 0 {
|
||||
if err := appendLateExpiryAudit(ctx, tx, now.UTC().UnixNano(), clientID, [16]byte(issueUUID), now.UTC().UnixNano()); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
var lifecycle uint32
|
||||
var storedGeneration, storedRevision uint64
|
||||
err = database.QueryRowContext(ctx, `SELECT lifecycle, COALESCE(target_session_generation, 0), revision FROM commands WHERE issue_uuid = ? AND client_id = ?`, issueUUID[:], clientID).Scan(&lifecycle, &storedGeneration, &storedRevision)
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// appendLateExpiryAudit records the first observed queue-deadline
|
||||
// contradiction without copying command text, script bodies, or output into
|
||||
// the separate audit budget. The command row remains the fast status path;
|
||||
// this row is the durable operator/audit history.
|
||||
func appendLateExpiryAudit(ctx context.Context, tx *sql.Tx, occurredAt int64, clientID string, issue [16]byte, receiptUnixNano int64) error {
|
||||
payload := []byte(fmt.Sprintf("receipt_unix_nano=%d", receiptUnixNano))
|
||||
digest := sha256.Sum256(payload)
|
||||
_, err := tx.ExecContext(ctx, `INSERT INTO audit_events (
|
||||
occurred_at, source, client_id, issue_uuid, action, outcome, compression, payload, raw_bytes, stored_bytes, sha256
|
||||
) VALUES (?, 'session', ?, ?, 'late_after_expiry', 'observed', 1, ?, ?, ?, ?)`, occurredAt, clientID, issue[:], payload, len(payload), len(payload), digest[:])
|
||||
return err
|
||||
}
|
||||
@@ -126,6 +126,51 @@ func TestClaimDispatchExpiresAndFencesRequeue_HP_DISPATCH_02(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLateDispatchAcceptanceAndEventRetainExpiryContradiction_BH_DISPATCH_09(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
opened, err := Open(ctx, Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = opened.Close() })
|
||||
base := time.Date(2026, time.September, 6, 13, 0, 0, 0, time.UTC)
|
||||
if _, err := opened.RegisterClientSession(ctx, ClientRegistration{
|
||||
ClientID: "late-client", Platform: 2, Architecture: "amd64", DaemonVersion: "test", DaemonCWD: `C:\`, SupportedShells: []byte{1},
|
||||
ClientInstanceID: [16]byte{21}, SessionID: [16]byte{22}, ConnectedAt: base,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
issue := fixedStoreIssue(0xd6)
|
||||
expiry := base.Add(time.Second)
|
||||
if _, err := opened.QueueCommand(ctx, QueueCommandInput{IssueUUID: issue, ClientID: "late-client", IssueTime: base, ReceiptTime: base, QueueExpiryTime: &expiry, ImmutableSHA256: sha256.Sum256([]byte("late")), ExecutionSpec: []byte("spec")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if candidate, err := opened.ClaimNextDispatch(ctx, "late-client", 7, base.Add(500*time.Millisecond)); err != nil || candidate == nil {
|
||||
t.Fatalf("dispatch claim = %#v, %v", candidate, err)
|
||||
}
|
||||
if changed, err := opened.RecordCommandAcceptance(ctx, issue, "late-client", 7, 1, true, base.Add(2*time.Second)); err != nil || !changed {
|
||||
t.Fatalf("late acceptance = %t, %v", changed, err)
|
||||
}
|
||||
var late int
|
||||
if err := opened.DB().QueryRow(`SELECT late_after_expiry FROM commands WHERE issue_uuid = ?`, issue[:]).Scan(&late); err != nil || late != 1 {
|
||||
t.Fatalf("late acceptance flag = %d, %v", late, err)
|
||||
}
|
||||
var audits int
|
||||
if err := opened.DB().QueryRow(`SELECT COUNT(*) FROM audit_events WHERE action = 'late_after_expiry' AND issue_uuid = ?`, issue[:]).Scan(&audits); err != nil || audits != 1 {
|
||||
t.Fatalf("late acceptance audit rows = %d, %v", audits, err)
|
||||
}
|
||||
if _, err := opened.AppendCommandEvent(ctx, EventAppend{IssueUUID: [16]byte(issue), ClientID: "late-client", SessionGeneration: 7, EventSeq: 1, ObservedUnixNano: base.Add(2 * time.Second).UnixNano(), ReceiptUnixNano: base.Add(2 * time.Second).UnixNano(), EventType: 5, Stream: 1, Compression: 1, RawLength: 1, Payload: []byte("x"), ImmutableSHA256: sha256.Sum256([]byte("event")), Output: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := opened.DB().QueryRow(`SELECT late_after_expiry FROM commands WHERE issue_uuid = ?`, issue[:]).Scan(&late); err != nil || late != 1 {
|
||||
t.Fatalf("late event flag = %d, %v", late, err)
|
||||
}
|
||||
if err := opened.DB().QueryRow(`SELECT COUNT(*) FROM audit_events WHERE action = 'late_after_expiry' AND issue_uuid = ?`, issue[:]).Scan(&audits); err != nil || audits != 1 {
|
||||
t.Fatalf("late event duplicated audit rows = %d, %v", audits, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueAndClaimScriptPayloadIsDurable_HP_SCRIPT_02(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
@@ -195,6 +240,25 @@ func TestRecordCommandAcceptanceFencesGeneration_HP_DISPATCH_05(t *testing.T) {
|
||||
if changed, err := opened.RecordCommandAcceptance(ctx, issue, "win-client", 3, 1, true, time.Now()); err != nil || changed {
|
||||
t.Fatalf("duplicate acceptance = %t, %v", changed, err)
|
||||
}
|
||||
rejectedIssue := fixedStoreIssue(0x96)
|
||||
if _, err := opened.QueueCommand(ctx, QueueCommandInput{IssueUUID: rejectedIssue, ClientID: "win-client", IssueTime: time.Now(), ReceiptTime: time.Now(), ImmutableSHA256: sha256.Sum256([]byte("reject")), ExecutionSpec: []byte("spec")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := opened.ClaimNextDispatch(ctx, "win-client", 3, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rejection := &rvboxv1.ControlError{Code: rvboxv1.ControlError_INVALID_ARGUMENT, Message: "invalid command"}
|
||||
if changed, err := opened.RecordCommandAcceptanceWithRejection(ctx, rejectedIssue, "win-client", 3, 1, false, rejection, time.Now()); err != nil || !changed {
|
||||
t.Fatalf("rejection acceptance = %t, %v", changed, err)
|
||||
}
|
||||
view, err := opened.GetCommandView(ctx, "win-client", rejectedIssue)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stored := &rvboxv1.ControlError{}
|
||||
if err := proto.Unmarshal(view.Rejection, stored); err != nil || stored.GetCode() != rejection.GetCode() || stored.GetMessage() != rejection.GetMessage() {
|
||||
t.Fatalf("stored rejection = %v, %v", stored, err)
|
||||
}
|
||||
}
|
||||
|
||||
func timePtr(value time.Time) *time.Time { return &value }
|
||||
|
||||
@@ -50,6 +50,7 @@ type EventAppend struct {
|
||||
ImmutableSHA256 [32]byte
|
||||
Lifecycle *rvboxv1.CommandLifecycle
|
||||
LifecycleRevision uint64
|
||||
WindowsIdentity *rvboxv1.WindowsExecutionIdentity
|
||||
Output bool
|
||||
UseCloseout bool
|
||||
}
|
||||
@@ -79,6 +80,16 @@ func (store *Store) AppendCommandEvent(ctx context.Context, event EventAppend) (
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
var windowsIdentity []byte
|
||||
if event.WindowsIdentity != nil {
|
||||
if len(event.WindowsIdentity.GetAttemptedContexts()) > 8 || len(event.WindowsIdentity.GetSelectionDetail()) > 4096 {
|
||||
return result, ErrInvalidSegmentRecord
|
||||
}
|
||||
windowsIdentity, err = proto.MarshalOptions{Deterministic: true}.Marshal(event.WindowsIdentity)
|
||||
if err != nil || len(windowsIdentity) > 16<<10 {
|
||||
return result, ErrInvalidSegmentRecord
|
||||
}
|
||||
}
|
||||
|
||||
store.writeMu.Lock()
|
||||
defer store.writeMu.Unlock()
|
||||
@@ -89,10 +100,13 @@ func (store *Store) AppendCommandEvent(ctx context.Context, event EventAppend) (
|
||||
var lastSequence, commandOutputCharged, commandCharged, clientCharged, serverCharged, closeoutRemaining uint64
|
||||
var currentLifecycle uint32
|
||||
var currentRevision uint64
|
||||
var lateAfterExpiry int
|
||||
var queueExpiry sql.NullInt64
|
||||
var clientID string
|
||||
query := `SELECT commands.last_event_seq, commands.output_charged_bytes,
|
||||
commands.charged_bytes, commands.closeout_remaining_bytes, commands.client_id, clients.charged_bytes,
|
||||
storage_counters.command_charged_bytes, commands.lifecycle, commands.revision
|
||||
storage_counters.command_charged_bytes, commands.lifecycle, commands.revision,
|
||||
commands.queue_expiry_time, commands.late_after_expiry
|
||||
FROM commands JOIN clients ON clients.client_id = commands.client_id
|
||||
JOIN storage_counters ON storage_counters.singleton = 1 WHERE commands.issue_uuid = ?`
|
||||
arguments := []any{event.IssueUUID[:]}
|
||||
@@ -105,7 +119,7 @@ JOIN storage_counters ON storage_counters.singleton = 1 WHERE commands.issue_uui
|
||||
arguments = append(arguments, event.SessionGeneration)
|
||||
}
|
||||
err = database.QueryRowContext(ctx, query, arguments...).Scan(
|
||||
&lastSequence, &commandOutputCharged, &commandCharged, &closeoutRemaining, &clientID, &clientCharged, &serverCharged, ¤tLifecycle, ¤tRevision)
|
||||
&lastSequence, &commandOutputCharged, &commandCharged, &closeoutRemaining, &clientID, &clientCharged, &serverCharged, ¤tLifecycle, ¤tRevision, &queueExpiry, &lateAfterExpiry)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return result, ErrCommandNotFound
|
||||
}
|
||||
@@ -133,6 +147,11 @@ JOIN storage_counters ON storage_counters.singleton = 1 WHERE commands.issue_uui
|
||||
return result, domain.ValidateTransition(rvboxv1.CommandLifecycle(currentLifecycle), *event.Lifecycle)
|
||||
}
|
||||
}
|
||||
late := queueExpiry.Valid && event.ReceiptUnixNano >= queueExpiry.Int64
|
||||
lateValue := 0
|
||||
if late {
|
||||
lateValue = 1
|
||||
}
|
||||
rowsCharged, indexesCharged := uint64(1), uint64(1)
|
||||
if store.willCreateSegment(event.IssueUUID, uint64(len(encoded))) {
|
||||
rowsCharged++
|
||||
@@ -215,8 +234,12 @@ payload, segment_ordinal, segment_record_offset, segment_record_length, immutabl
|
||||
var update sql.Result
|
||||
commandUpdate := `UPDATE commands SET last_event_seq = ?,
|
||||
retained_compressed_bytes = retained_compressed_bytes + ?, output_charged_bytes = ?, charged_bytes = ?,
|
||||
closeout_remaining_bytes = ?`
|
||||
commandArgs := []any{event.EventSeq, len(event.Payload), reservation.CommandOutputCharged, reservation.CommandTotalCharged, reservation.CloseoutRemaining}
|
||||
closeout_remaining_bytes = ?, late_after_expiry = CASE WHEN ? = 1 THEN 1 ELSE late_after_expiry END`
|
||||
commandArgs := []any{event.EventSeq, len(event.Payload), reservation.CommandOutputCharged, reservation.CommandTotalCharged, reservation.CloseoutRemaining, lateValue}
|
||||
if event.WindowsIdentity != nil {
|
||||
commandUpdate = `UPDATE commands SET windows_execution_identity = ?, ` + commandUpdate[len("UPDATE commands SET "):]
|
||||
commandArgs = append([]any{windowsIdentity}, commandArgs...)
|
||||
}
|
||||
if event.Lifecycle != nil {
|
||||
terminal := 0
|
||||
if domain.IsTerminal(*event.Lifecycle) {
|
||||
@@ -236,6 +259,11 @@ retained_compressed_bytes = retained_compressed_bytes + ?, output_charged_bytes
|
||||
}
|
||||
}
|
||||
}
|
||||
if err == nil && late && lateAfterExpiry == 0 {
|
||||
if err = appendLateExpiryAudit(ctx, tx, event.ReceiptUnixNano, clientID, event.IssueUUID, event.ReceiptUnixNano); err == nil {
|
||||
lateAfterExpiry = 1
|
||||
}
|
||||
}
|
||||
if err == nil && event.EventType == 7 {
|
||||
// Stdin acknowledgements are command events, but their durable delivery
|
||||
// cursor lives in stdin_writes. Mark the cumulative prefix in the same
|
||||
|
||||
@@ -34,10 +34,22 @@ func TestReadCommandEventsValidatesSegmentAndPreservesOrder_HP_EVENT_04(t *testi
|
||||
}
|
||||
for sequence, lifecycle := range []rvboxv1.CommandLifecycle{rvboxv1.CommandLifecycle_COMMAND_RUNNING, rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED} {
|
||||
event := []byte{byte(sequence + 1), 'e', 'v', 't'}
|
||||
if _, err := opened.AppendCommandEvent(ctx, EventAppend{IssueUUID: [16]byte(issue), ClientID: "win-client", SessionGeneration: 1, EventSeq: uint64(sequence + 1), ObservedUnixNano: now.UnixNano(), ReceiptUnixNano: now.Add(time.Duration(sequence) * time.Millisecond).UnixNano(), EventType: 4, Compression: 1, RawLength: uint64(len(event)), Payload: event, ImmutableSHA256: sha256.Sum256(event), Lifecycle: &lifecycle, LifecycleRevision: 1}); err != nil {
|
||||
var identity *rvboxv1.WindowsExecutionIdentity
|
||||
if sequence == 0 {
|
||||
effective := rvboxv1.WindowsExecutionContext_WINDOWS_EXECUTION_CONTEXT_ACTIVE_USER
|
||||
identity = &rvboxv1.WindowsExecutionIdentity{EffectiveContext: &effective, AttemptedContexts: []rvboxv1.WindowsExecutionContext{effective}, SelectionDetail: "selected"}
|
||||
}
|
||||
if _, err := opened.AppendCommandEvent(ctx, EventAppend{IssueUUID: [16]byte(issue), ClientID: "win-client", SessionGeneration: 1, EventSeq: uint64(sequence + 1), ObservedUnixNano: now.UnixNano(), ReceiptUnixNano: now.Add(time.Duration(sequence) * time.Millisecond).UnixNano(), EventType: 4, Compression: 1, RawLength: uint64(len(event)), Payload: event, ImmutableSHA256: sha256.Sum256(event), Lifecycle: &lifecycle, LifecycleRevision: 1, WindowsIdentity: identity}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
view, err := opened.GetCommandView(ctx, "win-client", issue)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(view.WindowsIdentity) == 0 {
|
||||
t.Fatal("running lifecycle did not persist Windows identity")
|
||||
}
|
||||
events, err := opened.ReadCommandEvents(ctx, issue, 0, 10)
|
||||
if err != nil || len(events) != 2 || events[0].EventSeq != 1 || events[1].EventSeq != 2 || string(events[1].Payload) != "\x02evt" {
|
||||
t.Fatalf("read events = %#v, %v", events, err)
|
||||
|
||||
@@ -16,6 +16,8 @@ type migration struct {
|
||||
var migrations = []migration{
|
||||
{version: 1, sql: schemaV1},
|
||||
{version: 2, sql: schemaV2},
|
||||
{version: 3, sql: schemaV3},
|
||||
{version: 4, sql: schemaV4},
|
||||
}
|
||||
|
||||
func applyMigrations(ctx context.Context, db *sql.DB) error {
|
||||
@@ -196,3 +198,17 @@ CREATE TABLE signal_intents (
|
||||
) STRICT;
|
||||
CREATE INDEX signal_intents_pending ON signal_intents(issue_uuid, command_revision, signal, acknowledged, created_at, request_uuid);
|
||||
`
|
||||
|
||||
// schemaV3 records the accepted/executed-after-deadline contradiction on the
|
||||
// command itself. It is separate from the public lifecycle: the eventual
|
||||
// client-reported terminal result remains authoritative while operators can
|
||||
// still see that acceptance or execution arrived after the queue deadline.
|
||||
const schemaV3 = `
|
||||
ALTER TABLE commands ADD COLUMN late_after_expiry INTEGER NOT NULL DEFAULT 0 CHECK(late_after_expiry IN (0,1));
|
||||
`
|
||||
|
||||
// schemaV4 retains the structured client admission rejection so status and
|
||||
// history do not have to reconstruct an error from a free-form event detail.
|
||||
const schemaV4 = `
|
||||
ALTER TABLE commands ADD COLUMN rejection BLOB;
|
||||
`
|
||||
|
||||
@@ -46,6 +46,8 @@ type CommandView struct {
|
||||
OutputTruncated bool
|
||||
OutputIncomplete bool
|
||||
RetainedCompressedBytes uint64
|
||||
LateAfterExpiry bool
|
||||
Rejection []byte
|
||||
ExecutionSpec []byte
|
||||
WindowsIdentity []byte
|
||||
}
|
||||
@@ -182,7 +184,7 @@ func (store *Store) ListCommandViews(ctx context.Context, page CommandPage) ([]C
|
||||
query := `SELECT issue_uuid, client_id, issue_time, server_receipt_time,
|
||||
queue_expiry_time, terminal_time, lifecycle, revision, last_event_seq,
|
||||
exit_code, output_truncated, output_incomplete, retained_compressed_bytes,
|
||||
execution_spec, execution_spec_raw_bytes, windows_execution_identity
|
||||
late_after_expiry, rejection, execution_spec, execution_spec_raw_bytes, windows_execution_identity
|
||||
FROM commands WHERE client_id = ? AND issue_time <= ?`
|
||||
args := []any{page.ClientID, page.SnapshotBoundary}
|
||||
if !page.IncludeTerminal {
|
||||
@@ -231,7 +233,7 @@ func (store *Store) GetCommandView(ctx context.Context, clientID string, issue d
|
||||
query := `SELECT issue_uuid, client_id, issue_time, server_receipt_time,
|
||||
queue_expiry_time, terminal_time, lifecycle, revision, last_event_seq,
|
||||
exit_code, output_truncated, output_incomplete, retained_compressed_bytes,
|
||||
execution_spec, execution_spec_raw_bytes, windows_execution_identity
|
||||
late_after_expiry, rejection, execution_spec, execution_spec_raw_bytes, windows_execution_identity
|
||||
FROM commands WHERE issue_uuid = ?`
|
||||
args := []any{issue[:]}
|
||||
if clientID != "" {
|
||||
@@ -252,10 +254,12 @@ func scanCommandView(scanner interface{ Scan(...any) error }) (CommandView, erro
|
||||
var expiry, terminal sql.NullInt64
|
||||
var exit sql.NullInt64
|
||||
var truncated, incomplete int
|
||||
var lateAfterExpiry int
|
||||
var rejection []byte
|
||||
var rawBytes uint64
|
||||
if err := scanner.Scan(&issue, &view.ClientID, &issueTime, &receipt, &expiry, &terminal,
|
||||
&view.Lifecycle, &view.Revision, &view.LastEventSeq, &exit, &truncated, &incomplete,
|
||||
&view.RetainedCompressedBytes, &stored, &rawBytes, &view.WindowsIdentity); err != nil {
|
||||
&view.RetainedCompressedBytes, &lateAfterExpiry, &rejection, &stored, &rawBytes, &view.WindowsIdentity); err != nil {
|
||||
return view, err
|
||||
}
|
||||
if len(issue) != 16 {
|
||||
@@ -267,6 +271,8 @@ func scanCommandView(scanner interface{ Scan(...any) error }) (CommandView, erro
|
||||
view.QueueExpiryTime = nullableTime(expiry)
|
||||
view.TerminalTime = nullableTime(terminal)
|
||||
view.OutputTruncated, view.OutputIncomplete = truncated == 1, incomplete == 1
|
||||
view.LateAfterExpiry = lateAfterExpiry == 1
|
||||
view.Rejection = append([]byte(nil), rejection...)
|
||||
if exit.Valid {
|
||||
if exit.Int64 < -1<<31 || exit.Int64 > 1<<31-1 {
|
||||
return view, ErrInvalidSegmentRecord
|
||||
|
||||
Reference in New Issue
Block a user