feat: implement control history and mutation APIs

2026-09-06 10:18:37 +00:00
parent ef43e9592c
commit 8df56263fb
11 changed files with 1515 additions and 3 deletions
@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"sort"
 	"time"

 	rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
@@ -41,6 +42,7 @@ type Options struct {
 	// documented indefinite-queue setting) from an omitted option in tests or
 	// embedders that want the compiled 15-minute default.
 	DefaultQueueTTLSet bool
+	TakeoverTTL        time.Duration
 	Limits             agentproto.Limits
 	Now                func() time.Time
 	CursorKey          []byte
@@ -51,6 +53,7 @@ type Service struct {
 	rvboxv1.UnimplementedControlServer
 	store           *store.Store
 	defaultQueueTTL time.Duration
+	takeoverTTL     time.Duration
 	limits          agentproto.Limits
 	now             func() time.Time
 	cursors         *domain.CursorCodec
@@ -67,6 +70,12 @@ func NewService(options Options) (*Service, error) {
 	if !options.DefaultQueueTTLSet && options.DefaultQueueTTL == 0 {
 		options.DefaultQueueTTL = defaultQueueTTL
 	}
+	if options.TakeoverTTL < 0 {
+		return nil, errors.New("takeover TTL must be non-negative")
+	}
+	if options.TakeoverTTL == 0 {
+		options.TakeoverTTL = 5 * time.Minute
+	}
 	if options.Limits.MaxEnvelopeBytes == 0 {
 		options.Limits = agentproto.DefaultLimits()
 	}
@@ -84,7 +93,7 @@ func NewService(options Options) (*Service, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &Service{store: options.Store, defaultQueueTTL: options.DefaultQueueTTL, limits: options.Limits, now: options.Now, cursors: codec, wakeClient: options.WakeClient}, nil
+	return &Service{store: options.Store, defaultQueueTTL: options.DefaultQueueTTL, takeoverTTL: options.TakeoverTTL, limits: options.Limits, now: options.Now, cursors: codec, wakeClient: options.WakeClient}, nil
 }

 func (service *Service) ListClients(ctx context.Context, request *rvboxv1.ListClientsRequest) (*rvboxv1.ListClientsResponse, error) {
@@ -272,6 +281,428 @@ func (service *Service) RunCommand(ctx context.Context, request *rvboxv1.RunComm
 	return &rvboxv1.RunCommandResponse{IssueUuid: issue.String(), Lifecycle: rvboxv1.CommandLifecycle(queued.Lifecycle)}, nil
 }

+// FollowCommand streams the durable event history and then waits for newly
+// committed events. It polls the store rather than maintaining a second event
+// bus, so a reconnect can resume from the last event sequence without losing a
+// commit that raced the stream cancellation.
+func (service *Service) FollowCommand(request *rvboxv1.FollowCommandRequest, stream rvboxv1.Control_FollowCommandServer) error {
+	if request == nil || request.GetIssueUuid() == "" || request.GetClientId() == "" {
+		return controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "client_id and issue_uuid are required")
+	}
+	issue, err := domain.ParseUUIDv7(request.GetIssueUuid())
+	if err != nil {
+		return controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "issue_uuid must be canonical UUIDv7")
+	}
+	view, err := service.store.GetCommandView(stream.Context(), request.GetClientId(), issue)
+	if err != nil {
+		return mapStoreError(err)
+	}
+	after := request.GetAfterEventSeq()
+	if !request.GetIncludeExisting() && after == 0 {
+		after = view.LastEventSeq
+	}
+	for {
+		events, readErr := service.store.ReadCommandEvents(stream.Context(), issue, after, 1000)
+		if readErr != nil {
+			return mapStoreError(readErr)
+		}
+		for _, stored := range events {
+			payload, decodeErr := store.DecodeEventPayload(stored)
+			if decodeErr != nil {
+				return mapStoreError(decodeErr)
+			}
+			event := &rvboxv1.CommandEvent{}
+			if unmarshalErr := proto.Unmarshal(payload, event); unmarshalErr != nil || event.GetIssueUuid() != issue.String() || event.GetEventSeq() != stored.EventSeq || len(event.GetImmutableEventSha256()) != sha256.Size {
+				return controlError(codes.Internal, rvboxv1.ControlError_INTERNAL, "stored command event is invalid")
+			}
+			digest, digestErr := agentproto.CommandEventDigest(event)
+			if digestErr != nil || string(digest[:]) != string(event.GetImmutableEventSha256()) {
+				return controlError(codes.Internal, rvboxv1.ControlError_INTERNAL, "stored command event digest is invalid")
+			}
+			if err := stream.Send(&rvboxv1.FollowCommandResponse{Item: &rvboxv1.FollowCommandResponse_Event{Event: event}, ServerReceiptTime: timestamppb.New(time.Unix(0, stored.ReceiptUnixNano).UTC())}); err != nil {
+				return err
+			}
+			after = stored.EventSeq
+		}
+		view, err = service.store.GetCommandView(stream.Context(), request.GetClientId(), issue)
+		if err != nil {
+			return mapStoreError(err)
+		}
+		if domain.IsTerminal(rvboxv1.CommandLifecycle(view.Lifecycle)) && after >= view.LastEventSeq {
+			return nil
+		}
+		timer := time.NewTimer(100 * time.Millisecond)
+		select {
+		case <-stream.Context().Done():
+			timer.Stop()
+			return stream.Context().Err()
+		case <-timer.C:
+		}
+	}
+}
+
+// GetOutput returns byte-bounded, cursor-resumable slices from output events.
+// The cursor binds the selected streams and a command-event snapshot boundary;
+// concurrent appends therefore cannot reorder or duplicate earlier bytes.
+func (service *Service) GetOutput(ctx context.Context, request *rvboxv1.GetOutputRequest) (*rvboxv1.GetOutputResponse, error) {
+	if request == nil || request.GetClientId() == "" || request.GetIssueUuid() == "" {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "client_id and issue_uuid are required")
+	}
+	issue, err := domain.ParseUUIDv7(request.GetIssueUuid())
+	if err != nil {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "issue_uuid must be canonical UUIDv7")
+	}
+	if _, err := service.store.GetClientView(ctx, request.GetClientId()); err != nil {
+		return nil, mapStoreError(err)
+	}
+	streams, err := normalizeStreams(request.GetStreams())
+	if err != nil {
+		return nil, err
+	}
+	maxBytes := request.GetMaxBytes()
+	if maxBytes == 0 {
+		maxBytes = 1 << 20
+	}
+	if maxBytes > 16<<20 {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "max_bytes exceeds the control limit")
+	}
+	filterBytes := []byte(fmt.Sprintf("output\x00%s\x00%v", request.GetClientId(), streams))
+	filter := domain.HashCursorFilters(filterBytes)
+	var eventSeq, offset, boundary uint64
+	if request.GetPageToken() != "" {
+		cursor, decodeErr := service.cursors.Decode(request.GetPageToken(), domain.CursorKindOutput, filter)
+		if decodeErr != nil || len(cursor.Position) != 16 || len(cursor.SnapshotBoundary) != 8 {
+			return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "invalid output page token")
+		}
+		eventSeq = binary.BigEndian.Uint64(cursor.Position[:8])
+		offset = binary.BigEndian.Uint64(cursor.Position[8:])
+		boundary = binary.BigEndian.Uint64(cursor.SnapshotBoundary)
+	} else {
+		view, viewErr := service.store.GetCommandView(ctx, request.GetClientId(), issue)
+		if viewErr != nil {
+			return nil, mapStoreError(viewErr)
+		}
+		boundary = view.LastEventSeq
+		eventSeq = request.GetAfterEventSeq()
+	}
+	queryAfter := eventSeq
+	if offset > 0 && eventSeq > 0 {
+		queryAfter = eventSeq - 1
+	}
+	storedEvents, err := service.store.ReadCommandEvents(ctx, issue, queryAfter, 1000)
+	if err != nil {
+		return nil, mapStoreError(err)
+	}
+	response := &rvboxv1.GetOutputResponse{}
+	remaining := maxBytes
+	hasMore := false
+	for _, stored := range storedEvents {
+		if stored.EventSeq > boundary {
+			break
+		}
+		payload, decodeErr := store.DecodeEventPayload(stored)
+		if decodeErr != nil {
+			return nil, mapStoreError(decodeErr)
+		}
+		event := &rvboxv1.CommandEvent{}
+		if unmarshalErr := proto.Unmarshal(payload, event); unmarshalErr != nil || event.GetIssueUuid() != issue.String() || event.GetEventSeq() != stored.EventSeq {
+			return nil, controlError(codes.Internal, rvboxv1.ControlError_INTERNAL, "stored command event is invalid")
+		}
+		output := event.GetOutput()
+		if output == nil || !containsStream(streams, output.GetStream()) {
+			if stored.EventSeq == eventSeq {
+				offset = 0
+			}
+			continue
+		}
+		data, decodeErr := agentproto.DecodeOutputChunk(output, service.limits.MaxRawChunkBytes)
+		if decodeErr != nil {
+			return nil, controlError(codes.Internal, rvboxv1.ControlError_INTERNAL, "stored output chunk is invalid")
+		}
+		start := uint64(0)
+		if stored.EventSeq == eventSeq {
+			start = offset
+		}
+		if start > uint64(len(data)) {
+			return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "output cursor is outside its event")
+		}
+		if start == uint64(len(data)) {
+			eventSeq, offset = stored.EventSeq, 0
+			continue
+		}
+		if remaining == 0 {
+			hasMore = true
+			break
+		}
+		end := uint64(len(data))
+		if end-start > remaining {
+			end = start + remaining
+			hasMore = true
+		}
+		response.Output = append(response.Output, &rvboxv1.OutputSlice{EventSeq: stored.EventSeq, ObservedAt: timestamppb.New(time.Unix(0, stored.ObservedUnixNano).UTC()), Stream: output.GetStream(), Data: append([]byte(nil), data[start:end]...), EventByteOffset: start, EndOfEvent: end == uint64(len(data)), ServerReceiptTime: timestamppb.New(time.Unix(0, stored.ReceiptUnixNano).UTC())})
+		remaining -= end - start
+		eventSeq, offset = stored.EventSeq, end
+		if end < uint64(len(data)) {
+			break
+		}
+		if remaining == 0 {
+			hasMore = true
+			break
+		}
+	}
+	if len(storedEvents) == 1000 {
+		hasMore = true
+	}
+	view, viewErr := service.store.GetCommandView(ctx, request.GetClientId(), issue)
+	if viewErr == nil {
+		response.OutputTruncated = view.OutputTruncated
+		if view.OutputIncomplete {
+			response.Incomplete = &rvboxv1.OutputIncomplete{Reason: "persisted output capture is incomplete"}
+		}
+	}
+	if hasMore {
+		position := make([]byte, 16)
+		binary.BigEndian.PutUint64(position[:8], eventSeq)
+		binary.BigEndian.PutUint64(position[8:], offset)
+		snapshot := make([]byte, 8)
+		binary.BigEndian.PutUint64(snapshot, boundary)
+		response.NextPageToken, err = service.cursors.Encode(domain.Cursor{Kind: domain.CursorKindOutput, FilterHash: filter, Position: position, SnapshotBoundary: snapshot})
+		if err != nil {
+			return nil, mapStoreError(err)
+		}
+	}
+	return response, nil
+}
+
+func (service *Service) AppendStdin(ctx context.Context, request *rvboxv1.AppendStdinRequest) (*rvboxv1.AppendStdinResponse, error) {
+	if request == nil || request.GetClientId() == "" || request.GetIssueUuid() == "" || len(request.GetData()) == 0 {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "client_id, issue_uuid, and non-empty data are required")
+	}
+	if uint64(len(request.GetData())) > service.limits.MaxRawChunkBytes {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "stdin data exceeds the raw chunk limit")
+	}
+	issue, err := domain.ParseUUIDv7(request.GetIssueUuid())
+	if err != nil {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "issue_uuid must be canonical UUIDv7")
+	}
+	requestUUID, err := service.requestID(request.GetRequestId())
+	if err != nil {
+		return nil, err
+	}
+	canonical := proto.Clone(request).(*rvboxv1.AppendStdinRequest)
+	canonical.RequestId = ""
+	encoded, err := proto.MarshalOptions{Deterministic: true}.Marshal(canonical)
+	if err != nil {
+		return nil, controlError(codes.Internal, rvboxv1.ControlError_INTERNAL, "canonicalize stdin request")
+	}
+	result, err := service.store.AppendStdin(ctx, store.StdinWriteInput{IssueUUID: issue, ClientID: request.GetClientId(), RequestUUID: requestUUID, Data: request.GetData(), AppendNewline: request.GetAppendNewline(), ImmutableHash: sha256.Sum256(encoded), OccurredAt: service.now()})
+	if err != nil {
+		return nil, mapStoreError(err)
+	}
+	if service.wakeClient != nil {
+		_ = service.wakeClient(request.GetClientId())
+	}
+	return &rvboxv1.AppendStdinResponse{WriteSeq: result.WriteSeq}, nil
+}
+
+func (service *Service) CloseStdin(ctx context.Context, request *rvboxv1.CloseStdinRequest) (*rvboxv1.CloseStdinResponse, error) {
+	if request == nil || request.GetClientId() == "" || request.GetIssueUuid() == "" {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "client_id and issue_uuid are required")
+	}
+	issue, err := domain.ParseUUIDv7(request.GetIssueUuid())
+	if err != nil {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "issue_uuid must be canonical UUIDv7")
+	}
+	requestUUID, err := service.requestID(request.GetRequestId())
+	if err != nil {
+		return nil, err
+	}
+	canonical := proto.Clone(request).(*rvboxv1.CloseStdinRequest)
+	canonical.RequestId = ""
+	encoded, err := proto.MarshalOptions{Deterministic: true}.Marshal(canonical)
+	if err != nil {
+		return nil, controlError(codes.Internal, rvboxv1.ControlError_INTERNAL, "canonicalize close-stdin request")
+	}
+	result, err := service.store.CloseStdin(ctx, store.StdinWriteInput{IssueUUID: issue, ClientID: request.GetClientId(), RequestUUID: requestUUID, ImmutableHash: sha256.Sum256(encoded), OccurredAt: service.now()})
+	if err != nil {
+		return nil, mapStoreError(err)
+	}
+	if service.wakeClient != nil {
+		_ = service.wakeClient(request.GetClientId())
+	}
+	return &rvboxv1.CloseStdinResponse{WriteSeq: result.WriteSeq}, nil
+}
+
+func (service *Service) SignalCommand(ctx context.Context, request *rvboxv1.ControlSignalCommandRequest) (*rvboxv1.ControlSignalCommandResponse, error) {
+	if request == nil || request.GetClientId() == "" || request.GetIssueUuid() == "" || request.GetSignal() < rvboxv1.SignalKind_SIGNAL_HUP || request.GetSignal() > rvboxv1.SignalKind_SIGNAL_USR2 {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "client_id, issue_uuid, and a portable signal are required")
+	}
+	issue, err := domain.ParseUUIDv7(request.GetIssueUuid())
+	if err != nil {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "issue_uuid must be canonical UUIDv7")
+	}
+	requestUUID, err := service.requestID(request.GetRequestId())
+	if err != nil {
+		return nil, err
+	}
+	canonical := proto.Clone(request).(*rvboxv1.ControlSignalCommandRequest)
+	canonical.RequestId = ""
+	encoded, err := proto.MarshalOptions{Deterministic: true}.Marshal(canonical)
+	if err != nil {
+		return nil, controlError(codes.Internal, rvboxv1.ControlError_INTERNAL, "canonicalize signal request")
+	}
+	result, err := service.store.SignalCommand(ctx, store.SignalInput{IssueUUID: issue, ClientID: request.GetClientId(), RequestUUID: requestUUID, Signal: request.GetSignal(), Hash: sha256.Sum256(encoded), OccurredAt: service.now()})
+	if err != nil {
+		return nil, mapStoreError(err)
+	}
+	return &rvboxv1.ControlSignalCommandResponse{CommandRevision: result.CommandRevision}, nil
+}
+
+func (service *Service) ListStorageIncidents(ctx context.Context, request *rvboxv1.ListStorageIncidentsRequest) (*rvboxv1.ListStorageIncidentsResponse, error) {
+	if request == nil {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "request is required")
+	}
+	limit, err := pageSize(request.GetPageSize())
+	if err != nil {
+		return nil, err
+	}
+	filterBytes := []byte(fmt.Sprintf("incidents\x00%t", request.GetIncludeResolved()))
+	filter := domain.HashCursorFilters(filterBytes)
+	page := store.IncidentPage{IncludeResolved: request.GetIncludeResolved(), Limit: limit, SnapshotBoundary: service.now().UnixNano()}
+	if request.GetPageToken() != "" {
+		cursor, decodeErr := service.cursors.Decode(request.GetPageToken(), domain.CursorKindIncidents, filter)
+		if decodeErr != nil || len(cursor.Position) != 24 || len(cursor.SnapshotBoundary) != 8 {
+			return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "invalid incident page token")
+		}
+		page.AfterDetectedAt = int64(binary.BigEndian.Uint64(cursor.Position[:8]))
+		copy(page.AfterUUID[:], cursor.Position[8:])
+		page.SnapshotBoundary = int64(binary.BigEndian.Uint64(cursor.SnapshotBoundary))
+		page.HasAfter = true
+	}
+	incidents, hasNext, err := service.store.ListIncidentViews(ctx, page)
+	if err != nil {
+		return nil, mapStoreError(err)
+	}
+	response := &rvboxv1.ListStorageIncidentsResponse{Incidents: make([]*rvboxv1.StorageIncident, 0, len(incidents))}
+	for _, incident := range incidents {
+		response.Incidents = append(response.Incidents, incidentRecord(incident))
+	}
+	if hasNext && len(incidents) > 0 {
+		position := make([]byte, 24)
+		binary.BigEndian.PutUint64(position[:8], uint64(incidents[len(incidents)-1].DetectedAt.UnixNano()))
+		copy(position[8:], incidents[len(incidents)-1].IncidentUUID[:])
+		snapshot := make([]byte, 8)
+		binary.BigEndian.PutUint64(snapshot, uint64(page.SnapshotBoundary))
+		response.NextPageToken, err = service.cursors.Encode(domain.Cursor{Kind: domain.CursorKindIncidents, FilterHash: filter, Position: position, SnapshotBoundary: snapshot})
+		if err != nil {
+			return nil, mapStoreError(err)
+		}
+	}
+	return response, nil
+}
+
+func (service *Service) RepairStorageIncident(ctx context.Context, request *rvboxv1.RepairStorageIncidentRequest) (*rvboxv1.RepairStorageIncidentResponse, error) {
+	if request == nil || request.GetIncidentId() == "" {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "incident_id is required")
+	}
+	incidentID, err := domain.ParseUUIDv7(request.GetIncidentId())
+	if err != nil {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "incident_id must be canonical UUIDv7")
+	}
+	requestID, err := service.requestID(request.GetRequestId())
+	if err != nil {
+		return nil, err
+	}
+	resolved, err := service.store.ResolveIncident(ctx, store.IncidentResolution{RequestUUID: [16]byte(requestID), IncidentUUID: [16]byte(incidentID), State: store.IncidentRepaired, ResolvedAt: service.now()})
+	if err != nil {
+		return nil, mapStoreError(err)
+	}
+	view, err := service.store.GetIncidentView(ctx, resolved.IncidentUUID)
+	if err != nil {
+		return nil, mapStoreError(err)
+	}
+	return &rvboxv1.RepairStorageIncidentResponse{Incident: incidentRecord(view)}, nil
+}
+
+func (service *Service) AcknowledgeStorageIncident(ctx context.Context, request *rvboxv1.AcknowledgeStorageIncidentRequest) (*rvboxv1.AcknowledgeStorageIncidentResponse, error) {
+	if request == nil || request.GetIncidentId() == "" || request.GetNote() == "" {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "incident_id and note are required")
+	}
+	incidentID, err := domain.ParseUUIDv7(request.GetIncidentId())
+	if err != nil {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "incident_id must be canonical UUIDv7")
+	}
+	requestID, err := service.requestID(request.GetRequestId())
+	if err != nil {
+		return nil, err
+	}
+	resolved, err := service.store.ResolveIncident(ctx, store.IncidentResolution{RequestUUID: [16]byte(requestID), IncidentUUID: [16]byte(incidentID), State: store.IncidentAcknowledged, ResolvedAt: service.now(), Note: request.GetNote()})
+	if err != nil {
+		return nil, mapStoreError(err)
+	}
+	view, err := service.store.GetIncidentView(ctx, resolved.IncidentUUID)
+	if err != nil {
+		return nil, mapStoreError(err)
+	}
+	return &rvboxv1.AcknowledgeStorageIncidentResponse{Incident: incidentRecord(view)}, nil
+}
+
+func (service *Service) AuthorizeClientTakeover(ctx context.Context, request *rvboxv1.AuthorizeClientTakeoverRequest) (*rvboxv1.AuthorizeClientTakeoverResponse, error) {
+	if request == nil || request.GetClientId() == "" || request.GetClientInstanceId() == "" {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "client_id and client_instance_id are required")
+	}
+	instance, err := domain.ParseUUIDv7(request.GetClientInstanceId())
+	if err != nil {
+		return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "client_instance_id must be canonical UUIDv7")
+	}
+	requestID, err := service.requestID(request.GetRequestId())
+	if err != nil {
+		return nil, err
+	}
+	expires := service.now().UTC().Add(service.takeoverTTL)
+	value, _, err := service.store.AuthorizeClientTakeover(ctx, store.TakeoverAuthorization{ClientID: request.GetClientId(), ClientInstanceID: [16]byte(instance), RequestID: [16]byte(requestID), ExpiresAt: expires})
+	if err != nil {
+		return nil, mapStoreError(err)
+	}
+	return &rvboxv1.AuthorizeClientTakeoverResponse{ExpiresAt: timestamppb.New(value)}, nil
+}
+
+func incidentRecord(view store.IncidentView) *rvboxv1.StorageIncident {
+	result := &rvboxv1.StorageIncident{IncidentId: domain.UUID(view.IncidentUUID).String(), DetectedAt: timestamppb.New(view.DetectedAt), State: rvboxv1.StorageIncidentState(view.State), Scope: string(view.Scope), ClientId: view.ClientID, Summary: view.Summary, DataLoss: view.DataLoss, AutomaticallyRepairable: view.AutomaticallyRepairable}
+	if view.ResolvedAt != nil {
+		result.ResolvedAt = timestamppb.New(*view.ResolvedAt)
+	}
+	if view.IssueUUID != nil {
+		result.IssueUuid = domain.UUID(*view.IssueUUID).String()
+	}
+	return result
+}
+
+func normalizeStreams(streams []rvboxv1.StreamKind) ([]rvboxv1.StreamKind, error) {
+	if len(streams) == 0 {
+		return []rvboxv1.StreamKind{rvboxv1.StreamKind_STREAM_STDOUT, rvboxv1.StreamKind_STREAM_STDERR}, nil
+	}
+	seen := make(map[rvboxv1.StreamKind]bool, len(streams))
+	result := append([]rvboxv1.StreamKind(nil), streams...)
+	for _, stream := range result {
+		if stream != rvboxv1.StreamKind_STREAM_STDOUT && stream != rvboxv1.StreamKind_STREAM_STDERR || seen[stream] {
+			return nil, controlError(codes.InvalidArgument, rvboxv1.ControlError_INVALID_ARGUMENT, "streams must contain unique stdout/stderr values")
+		}
+		seen[stream] = true
+	}
+	sort.Slice(result, func(left, right int) bool { return result[left] < result[right] })
+	return result, nil
+}
+
+func containsStream(streams []rvboxv1.StreamKind, value rvboxv1.StreamKind) bool {
+	for _, stream := range streams {
+		if stream == value {
+			return true
+		}
+	}
+	return false
+}
+
 func (service *Service) requestID(value string) (domain.UUID, error) {
 	if value == "" {
 		issue, err := domain.NewUUIDv7()
@@ -403,6 +834,10 @@ func mapStoreError(err error) error {
 		return controlError(codes.AlreadyExists, rvboxv1.ControlError_CONFLICT, err.Error())
 	case errors.Is(err, store.ErrCapacityExhausted):
 		return controlError(codes.ResourceExhausted, rvboxv1.ControlError_CAPACITY_EXHAUSTED, err.Error())
+	case errors.Is(err, store.ErrCommandTerminal), errors.Is(err, store.ErrSignalDeliveryUnavailable):
+		return controlError(codes.FailedPrecondition, rvboxv1.ControlError_UNSUPPORTED, err.Error())
+	case errors.Is(err, store.ErrTakeoverRequired), errors.Is(err, store.ErrTakeoverMismatch), errors.Is(err, store.ErrTakeoverAlreadyGranted):
+		return controlError(codes.FailedPrecondition, rvboxv1.ControlError_CONFLICT, err.Error())
 	case errors.Is(err, store.ErrStoreClosed):
 		return controlError(codes.Unavailable, rvboxv1.ControlError_OFFLINE, err.Error())
 	default:
@@ -15,10 +15,12 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 	"google.golang.org/grpc/test/bufconn"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )

 func TestControlListAndGetViews_HP_CONTROL_01(t *testing.T) {
@@ -147,6 +149,174 @@ func TestControlGRPCRoundTrip_HP_CONTROL_06(t *testing.T) {
 	}
 }

+func TestFollowCommandStreamsDurableEvents_HP_CONTROL_10(t *testing.T) {
+	service, persistence := newTestService(t)
+	defer persistence.Close()
+	registerControlClient(t, persistence, "win-a", rvboxv1.Platform_PLATFORM_WINDOWS, rvboxv1.ShellType_SHELL_CMD, 6)
+	issue := fixedIssue(0xa6)
+	now := time.Now().UTC()
+	specBytes, err := proto.Marshal(&rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "echo"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := persistence.QueueCommand(context.Background(), store.QueueCommandInput{IssueUUID: issue, ClientID: "win-a", IssueTime: now, ReceiptTime: now, ImmutableSHA256: sha256.Sum256([]byte("follow")), ExecutionSpec: specBytes}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := persistence.ClaimNextDispatch(context.Background(), "win-a", 1, now); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := persistence.RecordCommandAcceptance(context.Background(), issue, "win-a", 1, 1, true, now); err != nil {
+		t.Fatal(err)
+	}
+	for index, lifecycle := range []rvboxv1.CommandLifecycle{rvboxv1.CommandLifecycle_COMMAND_RUNNING, rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED} {
+		event := &rvboxv1.CommandEvent{IssueUuid: issue.String(), EventSeq: uint64(index + 1), ObservedAt: timestamppb.New(now.Add(time.Duration(index) * time.Millisecond)), Payload: &rvboxv1.CommandEvent_Lifecycle{Lifecycle: &rvboxv1.LifecycleChange{Lifecycle: lifecycle, CommandRevision: 1}}}
+		digest, digestErr := agentproto.CommandEventDigest(event)
+		if digestErr != nil {
+			t.Fatal(digestErr)
+		}
+		event.ImmutableEventSha256 = digest[:]
+		payload, marshalErr := proto.MarshalOptions{Deterministic: true}.Marshal(event)
+		if marshalErr != nil {
+			t.Fatal(marshalErr)
+		}
+		if _, appendErr := persistence.AppendCommandEvent(context.Background(), store.EventAppend{IssueUUID: [16]byte(issue), ClientID: "win-a", SessionGeneration: 1, EventSeq: uint64(index + 1), ObservedUnixNano: event.GetObservedAt().AsTime().UnixNano(), ReceiptUnixNano: now.Add(time.Duration(index) * time.Millisecond).UnixNano(), EventType: 4, Compression: 1, RawLength: uint64(len(payload)), Payload: payload, ImmutableSHA256: digest, Lifecycle: &lifecycle, LifecycleRevision: 1}); appendErr != nil {
+			t.Fatal(appendErr)
+		}
+	}
+	stream := &testFollowStream{ctx: context.Background()}
+	if err := service.FollowCommand(&rvboxv1.FollowCommandRequest{ClientId: "win-a", IssueUuid: issue.String(), IncludeExisting: true}, stream); err != nil {
+		t.Fatal(err)
+	}
+	if len(stream.responses) != 2 || stream.responses[0].GetEvent().GetEventSeq() != 1 || stream.responses[1].GetEvent().GetLifecycle().GetLifecycle() != rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED {
+		t.Fatalf("follow responses = %#v", stream.responses)
+	}
+}
+
+func TestGetOutputSlicesWithAuthenticatedCursor_HP_CONTROL_12(t *testing.T) {
+	service, persistence := newTestService(t)
+	defer persistence.Close()
+	registerControlClient(t, persistence, "win-a", rvboxv1.Platform_PLATFORM_WINDOWS, rvboxv1.ShellType_SHELL_CMD, 7)
+	issue := fixedIssue(0xa7)
+	now := time.Now().UTC()
+	specBytes, _ := proto.Marshal(&rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "echo"}})
+	if _, err := persistence.QueueCommand(context.Background(), store.QueueCommandInput{IssueUUID: issue, ClientID: "win-a", IssueTime: now, ReceiptTime: now, ImmutableSHA256: sha256.Sum256([]byte("output")), ExecutionSpec: specBytes}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := persistence.ClaimNextDispatch(context.Background(), "win-a", 1, now); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := persistence.RecordCommandAcceptance(context.Background(), issue, "win-a", 1, 1, true, now); err != nil {
+		t.Fatal(err)
+	}
+	event := &rvboxv1.CommandEvent{IssueUuid: issue.String(), EventSeq: 1, ObservedAt: timestamppb.New(now), Payload: &rvboxv1.CommandEvent_Output{Output: &rvboxv1.OutputChunk{Stream: rvboxv1.StreamKind_STREAM_STDOUT, Compression: rvboxv1.Compression_COMPRESSION_NONE, Data: []byte("abcdef"), UncompressedSize: 6, CompressedSize: 6}}}
+	digest, err := agentproto.CommandEventDigest(event)
+	if err != nil {
+		t.Fatal(err)
+	}
+	event.ImmutableEventSha256 = digest[:]
+	payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(event)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := persistence.AppendCommandEvent(context.Background(), store.EventAppend{IssueUUID: [16]byte(issue), ClientID: "win-a", SessionGeneration: 1, EventSeq: 1, ObservedUnixNano: now.UnixNano(), ReceiptUnixNano: now.UnixNano(), EventType: 5, Stream: 1, Compression: 1, RawLength: uint64(len(payload)), Payload: payload, ImmutableSHA256: digest, Output: true}); err != nil {
+		t.Fatal(err)
+	}
+	first, err := service.GetOutput(context.Background(), &rvboxv1.GetOutputRequest{ClientId: "win-a", IssueUuid: issue.String(), Streams: []rvboxv1.StreamKind{rvboxv1.StreamKind_STREAM_STDOUT}, MaxBytes: 3})
+	if err != nil || len(first.GetOutput()) != 1 || string(first.GetOutput()[0].GetData()) != "abc" || first.GetNextPageToken() == "" {
+		t.Fatalf("first output page = %#v, %v", first, err)
+	}
+	second, err := service.GetOutput(context.Background(), &rvboxv1.GetOutputRequest{ClientId: "win-a", IssueUuid: issue.String(), Streams: []rvboxv1.StreamKind{rvboxv1.StreamKind_STREAM_STDOUT}, MaxBytes: 3, PageToken: first.GetNextPageToken()})
+	if err != nil || len(second.GetOutput()) != 1 || string(second.GetOutput()[0].GetData()) != "def" || !second.GetOutput()[0].GetEndOfEvent() {
+		t.Fatalf("second output page = %#v, %v", second, err)
+	}
+	if _, err := service.GetOutput(context.Background(), &rvboxv1.GetOutputRequest{ClientId: "win-a", IssueUuid: issue.String(), Streams: []rvboxv1.StreamKind{rvboxv1.StreamKind_STREAM_STDERR}, MaxBytes: 3, PageToken: first.GetNextPageToken()}); status.Code(err) != codes.InvalidArgument {
+		t.Fatalf("changed output filter code = %v", status.Code(err))
+	}
+}
+
+func TestControlMutationIdempotencyAndQueuedCancellation_BH_CONTROL_13(t *testing.T) {
+	service, persistence := newTestService(t)
+	defer persistence.Close()
+	registerControlClient(t, persistence, "win-a", rvboxv1.Platform_PLATFORM_WINDOWS, rvboxv1.ShellType_SHELL_CMD, 8)
+	issue := fixedIssue(0xa8)
+	if _, err := service.RunCommand(context.Background(), &rvboxv1.RunCommandRequest{TargetClientId: "win-a", RequestId: issue.String(), Spec: &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "wait"}}}); err != nil {
+		t.Fatal(err)
+	}
+	stdinID := fixedIssue(0xa9)
+	appendRequest := &rvboxv1.AppendStdinRequest{ClientId: "win-a", IssueUuid: issue.String(), RequestId: stdinID.String(), Data: []byte("hello"), AppendNewline: true}
+	first, err := service.AppendStdin(context.Background(), appendRequest)
+	if err != nil || first.GetWriteSeq() != 1 {
+		t.Fatalf("append stdin = %#v, %v", first, err)
+	}
+	second, err := service.AppendStdin(context.Background(), proto.Clone(appendRequest).(*rvboxv1.AppendStdinRequest))
+	if err != nil || second.GetWriteSeq() != first.GetWriteSeq() {
+		t.Fatalf("append replay = %#v, %v", second, err)
+	}
+	conflict := proto.Clone(appendRequest).(*rvboxv1.AppendStdinRequest)
+	conflict.Data = []byte("different")
+	if _, err := service.AppendStdin(context.Background(), conflict); status.Code(err) != codes.AlreadyExists {
+		t.Fatalf("append conflict code = %v", status.Code(err))
+	}
+	closeID := fixedIssue(0xaa)
+	closed, err := service.CloseStdin(context.Background(), &rvboxv1.CloseStdinRequest{ClientId: "win-a", IssueUuid: issue.String(), RequestId: closeID.String()})
+	if err != nil || closed.GetWriteSeq() != 2 {
+		t.Fatalf("close stdin = %#v, %v", closed, err)
+	}
+	signalID := fixedIssue(0xab)
+	cancelled, err := service.SignalCommand(context.Background(), &rvboxv1.ControlSignalCommandRequest{ClientId: "win-a", IssueUuid: issue.String(), RequestId: signalID.String(), Signal: rvboxv1.SignalKind_SIGNAL_TERM})
+	if err != nil || cancelled.GetCommandRevision() != 2 {
+		t.Fatalf("queued cancellation = %#v, %v", cancelled, err)
+	}
+	replay, err := service.SignalCommand(context.Background(), &rvboxv1.ControlSignalCommandRequest{ClientId: "win-a", IssueUuid: issue.String(), RequestId: signalID.String(), Signal: rvboxv1.SignalKind_SIGNAL_TERM})
+	if err != nil || replay.GetCommandRevision() != cancelled.GetCommandRevision() {
+		t.Fatalf("cancel replay = %#v, %v", replay, err)
+	}
+	command, err := service.GetCommand(context.Background(), &rvboxv1.GetCommandRequest{ClientId: "win-a", IssueUuid: issue.String()})
+	if err != nil || command.GetCommand().GetLifecycle() != rvboxv1.CommandLifecycle_COMMAND_CANCELLED {
+		t.Fatalf("cancelled command = %#v, %v", command, err)
+	}
+}
+
+func TestStorageIncidentControlLifecycle_HP_CONTROL_14(t *testing.T) {
+	service, persistence := newTestService(t)
+	defer persistence.Close()
+	incidentID := fixedIssue(0xac)
+	if _, err := persistence.RecordIncident(context.Background(), store.IncidentInput{IncidentUUID: [16]byte(incidentID), DetectedAt: time.Now().UTC(), Kind: store.IncidentChecksumMismatch, Scope: store.IncidentScopeGlobal, ScopeKey: "segments", Summary: "checksum mismatch", Evidence: []byte("evidence"), AutomaticallyRepairable: true}); err != nil {
+		t.Fatal(err)
+	}
+	listed, err := service.ListStorageIncidents(context.Background(), &rvboxv1.ListStorageIncidentsRequest{PageSize: 1})
+	if err != nil || len(listed.GetIncidents()) != 1 || listed.GetIncidents()[0].GetState() != rvboxv1.StorageIncidentState_STORAGE_INCIDENT_STATE_OPEN {
+		t.Fatalf("incident list = %#v, %v", listed, err)
+	}
+	repaired, err := service.RepairStorageIncident(context.Background(), &rvboxv1.RepairStorageIncidentRequest{IncidentId: incidentID.String(), RequestId: fixedIssue(0xad).String()})
+	if err != nil || repaired.GetIncident().GetState() != rvboxv1.StorageIncidentState_STORAGE_INCIDENT_STATE_REPAIRED {
+		t.Fatalf("incident repair = %#v, %v", repaired, err)
+	}
+	if repeat, err := service.RepairStorageIncident(context.Background(), &rvboxv1.RepairStorageIncidentRequest{IncidentId: incidentID.String(), RequestId: fixedIssue(0xae).String()}); err != nil || repeat.GetIncident().GetState() != rvboxv1.StorageIncidentState_STORAGE_INCIDENT_STATE_REPAIRED {
+		t.Fatalf("second repair = %#v, %v", repeat, err)
+	}
+	resolved, err := service.ListStorageIncidents(context.Background(), &rvboxv1.ListStorageIncidentsRequest{IncludeResolved: true})
+	if err != nil || len(resolved.GetIncidents()) != 1 || resolved.GetIncidents()[0].GetResolvedAt() == nil {
+		t.Fatalf("resolved incident list = %#v, %v", resolved, err)
+	}
+}
+
+type testFollowStream struct {
+	ctx       context.Context
+	responses []*rvboxv1.FollowCommandResponse
+}
+
+func (stream *testFollowStream) Send(response *rvboxv1.FollowCommandResponse) error {
+	stream.responses = append(stream.responses, response)
+	return nil
+}
+func (stream *testFollowStream) SetHeader(metadata.MD) error  { return nil }
+func (stream *testFollowStream) SendHeader(metadata.MD) error { return nil }
+func (stream *testFollowStream) SetTrailer(metadata.MD)       {}
+func (stream *testFollowStream) Context() context.Context     { return stream.ctx }
+func (stream *testFollowStream) SendMsg(any) error            { return nil }
+func (stream *testFollowStream) RecvMsg(any) error            { return nil }
+
 func newTestService(t *testing.T) (*Service, *store.Store) {
 	t.Helper()
 	persistence, err := store.Open(context.Background(), store.Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second})