cabbage/rvbox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add bounded operational metrics

Browse Source
This commit is contained in:
cabbage
2026-09-11 09:09:52 +00:00
parent 46185f1f6c
commit 2cf563d88e
12 changed files with 342 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/client/agent/executor.go
+24 -1
View File
@@ -12,6 +12,7 @@ import (
"github.com/rvbox/rvbox/internal/client/spool"
"github.com/rvbox/rvbox/internal/client/supervisor"
"github.com/rvbox/rvbox/internal/domain"
"github.com/rvbox/rvbox/internal/observability"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -25,6 +26,7 @@ type Executor struct {
WorkDir string
Now func() time.Time
Notify func(domain.UUID)
Metrics *observability.Health
mu sync.Mutex
active map[domain.UUID]supervisor.Process
@@ -38,6 +40,7 @@ type ExecutorOptions struct {
WorkDir string
Now func() time.Time
Notify func(domain.UUID)
Metrics *observability.Health
}
func NewExecutor(options ExecutorOptions) (*Executor, error) {
@@ -47,7 +50,7 @@ func NewExecutor(options ExecutorOptions) (*Executor, error) {
if options.Now == nil {
options.Now = func() time.Time { return time.Now().UTC() }
}
return &Executor{Store: options.Store, Supervisor: options.Supervisor, WorkDir: options.WorkDir, Now: options.Now, Notify: options.Notify, active: make(map[domain.UUID]supervisor.Process), cancel: make(map[domain.UUID]context.CancelFunc), forced: make(map[domain.UUID]bool)}, nil
return &Executor{Store: options.Store, Supervisor: options.Supervisor, WorkDir: options.WorkDir, Now: options.Now, Notify: options.Notify, Metrics: options.Metrics, active: make(map[domain.UUID]supervisor.Process), cancel: make(map[domain.UUID]context.CancelFunc), forced: make(map[domain.UUID]bool)}, nil
}
// Dispatch is safe to invoke after CommandAccepted has been sent. Script
@@ -158,6 +161,7 @@ func (executor *Executor) launch(ctx context.Context, issue domain.UUID, revisio
executor.remove(issue)
return err
}
executor.incMetric("command_running")
executor.notify(issue)
go executor.watch(runContext, issue, revision, process, cancel)
return nil
@@ -171,6 +175,7 @@ func (executor *Executor) watch(ctx context.Context, issue domain.UUID, revision
break
}
if err != nil {
executor.incMetric("output_loss")
_ = executor.appendIncomplete(context.Background(), issue, revision, err.Error())
break
}
@@ -178,10 +183,13 @@ func (executor *Executor) watch(ctx context.Context, issue domain.UUID, revision
continue
}
if _, err := executor.Store.AppendOutput(context.Background(), issue, spool.OutputInput{Stream: chunk.Stream, Raw: chunk.Data, ObservedAt: executor.Now()}); err != nil {
executor.incMetric("output_loss")
_ = executor.appendIncomplete(context.Background(), issue, revision, err.Error())
_, _ = executor.Supervisor.Signal(context.Background(), process, supervisor.SignalKill)
break
}
executor.incMetric("output_chunk")
executor.incMetricBy("output_bytes", uint64(len(chunk.Data)))
executor.notify(issue)
}
status, waitErr := process.Wait(context.Background())
@@ -210,9 +218,24 @@ func (executor *Executor) appendLifecycle(ctx context.Context, issue domain.UUID
func (executor *Executor) appendLifecycleWithIdentity(ctx context.Context, issue domain.UUID, revision uint64, phase rvboxv1.CommandLifecycle, detail string, identity *rvboxv1.WindowsExecutionIdentity) error {
_, err := executor.Store.AppendLifecycleWithIdentity(ctx, issue, uint32(phase), revision, detail, executor.Now(), identity)
if err == nil {
executor.incMetric("command_transition")
}
return err
}
func (executor *Executor) incMetric(name string) {
if executor != nil && executor.Metrics != nil {
executor.Metrics.Inc(name)
}
}
func (executor *Executor) incMetricBy(name string, delta uint64) {
if executor != nil && executor.Metrics != nil {
executor.Metrics.IncBy(name, delta)
}
}
func (executor *Executor) reject(ctx context.Context, issue domain.UUID, revision uint64, cause error) error {
return executor.rejectWithIdentity(ctx, issue, revision, cause, nil)
}