feat: add bounded operational metrics

internal/client/agent/runner.go

@@ -12,6 +12,7 @@ import (
 	"github.com/rvbox/rvbox/internal/agentproto"
 	"github.com/rvbox/rvbox/internal/client/spool"
 	"github.com/rvbox/rvbox/internal/domain"
+	"github.com/rvbox/rvbox/internal/observability"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/timestamppb"
 )
@@ -39,6 +40,12 @@ type RunnerOptions struct {
 	// transport session before normal reconnect backoff. It must not block; the
 	// durable spool and retry policy remain owned by Run.
 	OnSessionError func(error)
+	// OnSessionActive reports whether a reconciled transport is currently able
+	// to exchange command data. It lets daemon readiness represent connection
+	// state without making reconnect behavior depend on the observer.
+	OnSessionActive func(bool)
+	// Metrics is optional. It only receives static, bounded metric names.
+	Metrics *observability.Health
 	// EventReady wakes the active session after a supervisor worker appends a
 	// durable event. The network loop remains the sole writer; a reconnect can
 	// safely ignore a stale notification because replay reads the spool again.
@@ -98,6 +105,10 @@ func Run(ctx context.Context, options RunnerOptions) error {
 		if err := waitUntil(ctx, delay); err != nil {
 			return nil
 		}
+		if failures > 0 {
+			options.incMetric("reconnect_attempt")
+		}
+		options.incMetric("connection_attempt")
 		sessionStarted := options.Now()
 		sessionErr := runOnce(ctx, options)
 		if ctx.Err() != nil {
@@ -106,6 +117,9 @@ func Run(ctx context.Context, options RunnerOptions) error {
 		if sessionErr != nil && options.OnSessionError != nil {
 			options.OnSessionError(sessionErr)
 		}
+		if sessionErr != nil {
+			options.incMetric("connection_failure")
+		}
 		if options.Now().Sub(sessionStarted) >= options.Backoff.StableReset {
 			failures = 0
 		}
@@ -173,6 +187,7 @@ func runOnce(ctx context.Context, options RunnerOptions) (resultErr error) {
 	}
 	session, err := Handshake(ctx, transport, options.Hello, limits)
 	if err != nil {
+		options.incMetric("protocol_error")
 		return fmt.Errorf("agent handshake: %w", err)
 	}
 	snapshot, err := options.Store.ReconcileSnapshot(ctx)
@@ -201,6 +216,17 @@ func runOnce(ctx context.Context, options RunnerOptions) (resultErr error) {
 	if err := sendCapacity(ctx, transport, session, options.Hello, snapshot, limits); err != nil {
 		return fmt.Errorf("advertise client capacity: %w", err)
 	}
+	activeStarted := options.Now()
+	options.incMetric("session_established")
+	if options.OnSessionActive != nil {
+		options.OnSessionActive(true)
+	}
+	defer func() {
+		if options.OnSessionActive != nil {
+			options.OnSessionActive(false)
+		}
+		options.observeMetric("session_duration", options.Now().Sub(activeStarted))
+	}()
 	return serveActive(ctx, transport, options, session, sentEvents)
 }
 
@@ -343,9 +369,11 @@ func serveActive(ctx context.Context, transport Transport, options RunnerOptions
 		}
 		envelope, err := agentproto.DecodeEnvelope(encoded, limits, rvboxv1.Platform_PLATFORM_WINDOWS)
 		if err != nil {
+			options.incMetric("protocol_error")
 			return fmt.Errorf("decode active agent frame: %w", err)
 		}
 		if envelope.GetSessionId() != session.ID || envelope.GetSessionGeneration() != session.Generation {
+			options.incMetric("stale_message")
 			return ErrProtocolHandshake
 		}
 		switch {
@@ -452,6 +480,11 @@ func flushEvents(ctx context.Context, transport Transport, store *spool.Store, s
 func handleDispatch(ctx context.Context, transport Transport, options RunnerOptions, session Session, dispatch *rvboxv1.CommandDispatch, limits agentproto.Limits) error {
 	acceptance, err := PersistDispatch(ctx, options.Store, session, dispatch, options.Now(), limits)
 	accepted := err == nil
+	if accepted {
+		options.incMetric("command_dispatch_accepted")
+	} else {
+		options.incMetric("command_dispatch_rejected")
+	}
 	ack := &rvboxv1.CommandAccepted{IssueUuid: dispatch.GetIssueUuid(), CommandRevision: dispatch.GetCommandRevision(), Accepted: accepted}
 	if err != nil {
 		ack.Rejection = rejectionForError(err, dispatch.GetIssueUuid())
@@ -474,6 +507,18 @@ func handleDispatch(ctx context.Context, transport Transport, options RunnerOpti
 	return nil
 }
 
+func (options RunnerOptions) incMetric(name string) {
+	if options.Metrics != nil {
+		options.Metrics.Inc(name)
+	}
+}
+
+func (options RunnerOptions) observeMetric(name string, duration time.Duration) {
+	if options.Metrics != nil {
+		options.Metrics.ObserveDuration(name, duration)
+	}
+}
+
 func rejectionForError(err error, issue string) *rvboxv1.ControlError {
 	code := rvboxv1.ControlError_TRANSIENT
 	if errors.Is(err, spool.ErrCommandConflict) || errors.Is(err, spool.ErrAlreadyExecuted) {