cabbage/rvbox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add bounded operational metrics

Browse Source
This commit is contained in:
cabbage
2026-09-11 09:09:52 +00:00
parent 46185f1f6c
commit 2cf563d88e
12 changed files with 342 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/client/agent/executor.go
+24 -1
View File
@@ -12,6 +12,7 @@ import (
"github.com/rvbox/rvbox/internal/client/spool"
"github.com/rvbox/rvbox/internal/client/supervisor"
"github.com/rvbox/rvbox/internal/domain"
"github.com/rvbox/rvbox/internal/observability"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -25,6 +26,7 @@ type Executor struct {
WorkDir string
Now func() time.Time
Notify func(domain.UUID)
Metrics *observability.Health
mu sync.Mutex
active map[domain.UUID]supervisor.Process
@@ -38,6 +40,7 @@ type ExecutorOptions struct {
WorkDir string
Now func() time.Time
Notify func(domain.UUID)
Metrics *observability.Health
}
func NewExecutor(options ExecutorOptions) (*Executor, error) {
@@ -47,7 +50,7 @@ func NewExecutor(options ExecutorOptions) (*Executor, error) {
if options.Now == nil {
options.Now = func() time.Time { return time.Now().UTC() }
}
return &Executor{Store: options.Store, Supervisor: options.Supervisor, WorkDir: options.WorkDir, Now: options.Now, Notify: options.Notify, active: make(map[domain.UUID]supervisor.Process), cancel: make(map[domain.UUID]context.CancelFunc), forced: make(map[domain.UUID]bool)}, nil
return &Executor{Store: options.Store, Supervisor: options.Supervisor, WorkDir: options.WorkDir, Now: options.Now, Notify: options.Notify, Metrics: options.Metrics, active: make(map[domain.UUID]supervisor.Process), cancel: make(map[domain.UUID]context.CancelFunc), forced: make(map[domain.UUID]bool)}, nil
}
// Dispatch is safe to invoke after CommandAccepted has been sent. Script
@@ -158,6 +161,7 @@ func (executor *Executor) launch(ctx context.Context, issue domain.UUID, revisio
executor.remove(issue)
return err
}
executor.incMetric("command_running")
executor.notify(issue)
go executor.watch(runContext, issue, revision, process, cancel)
return nil
@@ -171,6 +175,7 @@ func (executor *Executor) watch(ctx context.Context, issue domain.UUID, revision
break
}
if err != nil {
executor.incMetric("output_loss")
_ = executor.appendIncomplete(context.Background(), issue, revision, err.Error())
break
}
@@ -178,10 +183,13 @@ func (executor *Executor) watch(ctx context.Context, issue domain.UUID, revision
continue
}
if _, err := executor.Store.AppendOutput(context.Background(), issue, spool.OutputInput{Stream: chunk.Stream, Raw: chunk.Data, ObservedAt: executor.Now()}); err != nil {
executor.incMetric("output_loss")
_ = executor.appendIncomplete(context.Background(), issue, revision, err.Error())
_, _ = executor.Supervisor.Signal(context.Background(), process, supervisor.SignalKill)
break
}
executor.incMetric("output_chunk")
executor.incMetricBy("output_bytes", uint64(len(chunk.Data)))
executor.notify(issue)
}
status, waitErr := process.Wait(context.Background())
@@ -210,9 +218,24 @@ func (executor *Executor) appendLifecycle(ctx context.Context, issue domain.UUID
func (executor *Executor) appendLifecycleWithIdentity(ctx context.Context, issue domain.UUID, revision uint64, phase rvboxv1.CommandLifecycle, detail string, identity *rvboxv1.WindowsExecutionIdentity) error {
_, err := executor.Store.AppendLifecycleWithIdentity(ctx, issue, uint32(phase), revision, detail, executor.Now(), identity)
if err == nil {
executor.incMetric("command_transition")
}
return err
}
func (executor *Executor) incMetric(name string) {
if executor != nil && executor.Metrics != nil {
executor.Metrics.Inc(name)
}
}
func (executor *Executor) incMetricBy(name string, delta uint64) {
if executor != nil && executor.Metrics != nil {
executor.Metrics.IncBy(name, delta)
}
}
func (executor *Executor) reject(ctx context.Context, issue domain.UUID, revision uint64, cause error) error {
return executor.rejectWithIdentity(ctx, issue, revision, cause, nil)
}
internal/client/agent/runner.go
+45
View File
@@ -12,6 +12,7 @@ import (
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/client/spool"
"github.com/rvbox/rvbox/internal/domain"
"github.com/rvbox/rvbox/internal/observability"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -39,6 +40,12 @@ type RunnerOptions struct {
// transport session before normal reconnect backoff. It must not block; the
// durable spool and retry policy remain owned by Run.
OnSessionError func(error)
// OnSessionActive reports whether a reconciled transport is currently able
// to exchange command data. It lets daemon readiness represent connection
// state without making reconnect behavior depend on the observer.
OnSessionActive func(bool)
// Metrics is optional. It only receives static, bounded metric names.
Metrics *observability.Health
// EventReady wakes the active session after a supervisor worker appends a
// durable event. The network loop remains the sole writer; a reconnect can
// safely ignore a stale notification because replay reads the spool again.
@@ -98,6 +105,10 @@ func Run(ctx context.Context, options RunnerOptions) error {
if err := waitUntil(ctx, delay); err != nil {
return nil
}
if failures > 0 {
options.incMetric("reconnect_attempt")
}
options.incMetric("connection_attempt")
sessionStarted := options.Now()
sessionErr := runOnce(ctx, options)
if ctx.Err() != nil {
@@ -106,6 +117,9 @@ func Run(ctx context.Context, options RunnerOptions) error {
if sessionErr != nil && options.OnSessionError != nil {
options.OnSessionError(sessionErr)
}
if sessionErr != nil {
options.incMetric("connection_failure")
}
if options.Now().Sub(sessionStarted) >= options.Backoff.StableReset {
failures = 0
}
@@ -173,6 +187,7 @@ func runOnce(ctx context.Context, options RunnerOptions) (resultErr error) {
}
session, err := Handshake(ctx, transport, options.Hello, limits)
if err != nil {
options.incMetric("protocol_error")
return fmt.Errorf("agent handshake: %w", err)
}
snapshot, err := options.Store.ReconcileSnapshot(ctx)
@@ -201,6 +216,17 @@ func runOnce(ctx context.Context, options RunnerOptions) (resultErr error) {
if err := sendCapacity(ctx, transport, session, options.Hello, snapshot, limits); err != nil {
return fmt.Errorf("advertise client capacity: %w", err)
}
activeStarted := options.Now()
options.incMetric("session_established")
if options.OnSessionActive != nil {
options.OnSessionActive(true)
}
defer func() {
if options.OnSessionActive != nil {
options.OnSessionActive(false)
}
options.observeMetric("session_duration", options.Now().Sub(activeStarted))
}()
return serveActive(ctx, transport, options, session, sentEvents)
}
@@ -343,9 +369,11 @@ func serveActive(ctx context.Context, transport Transport, options RunnerOptions
}
envelope, err := agentproto.DecodeEnvelope(encoded, limits, rvboxv1.Platform_PLATFORM_WINDOWS)
if err != nil {
options.incMetric("protocol_error")
return fmt.Errorf("decode active agent frame: %w", err)
}
if envelope.GetSessionId() != session.ID || envelope.GetSessionGeneration() != session.Generation {
options.incMetric("stale_message")
return ErrProtocolHandshake
}
switch {
@@ -452,6 +480,11 @@ func flushEvents(ctx context.Context, transport Transport, store *spool.Store, s
func handleDispatch(ctx context.Context, transport Transport, options RunnerOptions, session Session, dispatch *rvboxv1.CommandDispatch, limits agentproto.Limits) error {
acceptance, err := PersistDispatch(ctx, options.Store, session, dispatch, options.Now(), limits)
accepted := err == nil
if accepted {
options.incMetric("command_dispatch_accepted")
} else {
options.incMetric("command_dispatch_rejected")
}
ack := &rvboxv1.CommandAccepted{IssueUuid: dispatch.GetIssueUuid(), CommandRevision: dispatch.GetCommandRevision(), Accepted: accepted}
if err != nil {
ack.Rejection = rejectionForError(err, dispatch.GetIssueUuid())
@@ -474,6 +507,18 @@ func handleDispatch(ctx context.Context, transport Transport, options RunnerOpti
return nil
}
func (options RunnerOptions) incMetric(name string) {
if options.Metrics != nil {
options.Metrics.Inc(name)
}
}
func (options RunnerOptions) observeMetric(name string, duration time.Duration) {
if options.Metrics != nil {
options.Metrics.ObserveDuration(name, duration)
}
}
func rejectionForError(err error, issue string) *rvboxv1.ControlError {
code := rvboxv1.ControlError_TRANSIENT
if errors.Is(err, spool.ErrCommandConflict) || errors.Is(err, spool.ErrAlreadyExecuted) {
internal/client/agent/runner_test.go
+11 -1
View File
@@ -12,6 +12,7 @@ import (
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/client/spool"
"github.com/rvbox/rvbox/internal/domain"
"github.com/rvbox/rvbox/internal/observability"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -46,7 +47,9 @@ func TestRunOnceReconcilesReplaysAndAdvertisesCapacity_HP_RUNTIME_01(t *testing.
reconcileResult, _ := proto.Marshal(&rvboxv1.AgentEnvelope{SessionId: "session", SessionGeneration: 1, Payload: &rvboxv1.AgentEnvelope_ReconcileResult{ReconcileResult: &rvboxv1.ReconcileResult{}}})
transport := &runnerTransport{reads: [][]byte{welcome, reconcileRequest, reconcileResult}, terminal: errors.New("transport closed")}
hello := &rvboxv1.ClientHello{ClientId: "runner-client", SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1, MinMinor: 0, MaxMinor: 0}, DaemonVersion: "test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS, Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_POWERSHELL}, ClientInstanceId: store.ClientInstanceID().String(), MaxRunningCommands: 1, MaxQueuedCommands: 1, SentAt: timestamppb.Now()}
err = RunOnce(ctx, RunnerOptions{Store: store, Dial: func(context.Context) (Transport, error) { return transport, nil }, Hello: hello, Limits: agentproto.DefaultLimits(), Backoff: BackoffOptions{Initial: time.Millisecond, Maximum: time.Millisecond, StableReset: time.Second}, Jitter: func(value time.Duration) time.Duration { return 0 }, Now: func() time.Time { return time.Now().UTC() }})
metrics := observability.New()
var active []bool
err = RunOnce(ctx, RunnerOptions{Store: store, Dial: func(context.Context) (Transport, error) { return transport, nil }, Hello: hello, Limits: agentproto.DefaultLimits(), Backoff: BackoffOptions{Initial: time.Millisecond, Maximum: time.Millisecond, StableReset: time.Second}, Jitter: func(value time.Duration) time.Duration { return 0 }, Now: func() time.Time { return time.Now().UTC() }, Metrics: metrics, OnSessionActive: func(value bool) { active = append(active, value) }})
if !errors.Is(err, transport.terminal) {
t.Fatalf("RunOnce error = %v, want transport close", err)
}
@@ -64,6 +67,13 @@ func TestRunOnceReconcilesReplaysAndAdvertisesCapacity_HP_RUNTIME_01(t *testing.
if err != nil || eventEnvelope.GetCommandEvent() == nil || eventEnvelope.GetCommandEvent().GetEventSeq() != 1 || eventEnvelope.GetCommandEvent().GetIssueUuid() != issue.String() {
t.Fatalf("replayed event = %#v, %v", eventEnvelope, err)
}
_, _, counters := metrics.Snapshot()
if counters["session_established"] != 1 {
t.Fatalf("session metrics = %#v", counters)
}
if len(active) != 2 || !active[0] || active[1] {
t.Fatalf("active transitions = %#v", active)
}
}
func TestRunnerOptionsRejectMissingJitter_HP_RUNTIME_02(t *testing.T) {