cabbage/rvbox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add bounded operational metrics

Browse Source
This commit is contained in:
cabbage
2026-09-11 09:09:52 +00:00
parent 46185f1f6c
commit 2cf563d88e
12 changed files with 342 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/server/control/service.go
+13 -1
View File
@@ -18,6 +18,7 @@ import (
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/domain"
"github.com/rvbox/rvbox/internal/observability"
"github.com/rvbox/rvbox/internal/server/store"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@@ -48,6 +49,8 @@ type Options struct {
Now func() time.Time
CursorKey []byte
WakeClient func(string) bool
// Metrics is optional and receives only static metric names.
Metrics *observability.Health
}
type Service struct {
@@ -59,6 +62,7 @@ type Service struct {
now func() time.Time
cursors *domain.CursorCodec
wakeClient func(string) bool
metrics *observability.Health
}
func NewService(options Options) (*Service, error) {
@@ -94,7 +98,7 @@ func NewService(options Options) (*Service, error) {
if err != nil {
return nil, err
}
return &Service{store: options.Store, defaultQueueTTL: options.DefaultQueueTTL, takeoverTTL: options.TakeoverTTL, limits: options.Limits, now: options.Now, cursors: codec, wakeClient: options.WakeClient}, nil
return &Service{store: options.Store, defaultQueueTTL: options.DefaultQueueTTL, takeoverTTL: options.TakeoverTTL, limits: options.Limits, now: options.Now, cursors: codec, wakeClient: options.WakeClient, metrics: options.Metrics}, nil
}
func (service *Service) ListClients(ctx context.Context, request *rvboxv1.ListClientsRequest) (*rvboxv1.ListClientsResponse, error) {
@@ -676,11 +680,19 @@ func (service *Service) AuthorizeClientTakeover(ctx context.Context, request *rv
expires := service.now().UTC().Add(service.takeoverTTL)
value, _, err := service.store.AuthorizeClientTakeover(ctx, store.TakeoverAuthorization{ClientID: request.GetClientId(), ClientInstanceID: [16]byte(instance), RequestID: [16]byte(requestID), ExpiresAt: expires})
if err != nil {
service.incMetric("client_takeover_rejected")
return nil, mapStoreError(err)
}
service.incMetric("client_takeover_authorized")
return &rvboxv1.AuthorizeClientTakeoverResponse{ExpiresAt: timestamppb.New(value)}, nil
}
func (service *Service) incMetric(name string) {
if service != nil && service.metrics != nil {
service.metrics.Inc(name)
}
}
func incidentRecord(view store.IncidentView) *rvboxv1.StorageIncident {
result := &rvboxv1.StorageIncident{IncidentId: domain.UUID(view.IncidentUUID).String(), DetectedAt: timestamppb.New(view.DetectedAt), State: rvboxv1.StorageIncidentState(view.State), Scope: string(view.Scope), ClientId: view.ClientID, Summary: view.Summary, DataLoss: view.DataLoss, AutomaticallyRepairable: view.AutomaticallyRepairable}
if view.ResolvedAt != nil {
internal/server/session/agent_server.go
+40
View File
@@ -16,6 +16,7 @@ import (
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/domain"
"github.com/rvbox/rvbox/internal/observability"
"github.com/rvbox/rvbox/internal/server/store"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
@@ -45,6 +46,9 @@ type AgentServer struct {
HeartbeatIdle time.Duration
LivenessTimeout time.Duration
Now func() time.Time
// Metrics is optional so transport tests and embeddings do not need an
// observability endpoint. Every emitted name is a static bounded value.
Metrics *observability.Health
}
func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http.Request) {
@@ -53,6 +57,7 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
return
}
if request.Header.Get("Origin") != "" {
server.incMetric("protocol_error")
http.Error(response, ErrUnexpectedOrigin.Error(), http.StatusForbidden)
return
}
@@ -83,6 +88,7 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
if err != nil {
return
}
server.incMetric("agent_connection")
defer connection.CloseNow()
server.serveConnection(request.Context(), connection, heartbeat, started)
}
@@ -98,6 +104,7 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
}
helloEnvelope, err := agentproto.DecodeEnvelope(payload, server.limits(), rvboxv1.Platform_PLATFORM_UNSPECIFIED)
if err != nil || helloEnvelope.GetClientHello() == nil {
server.incMetric("protocol_error")
server.close(connection, websocket.StatusPolicyViolation, "invalid ClientHello")
return
}
@@ -108,11 +115,13 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
reconcileBoundary := server.now()
instanceID, err := domain.ParseUUIDv7(hello.GetClientInstanceId())
if err != nil {
server.incMetric("protocol_error")
server.close(connection, websocket.StatusPolicyViolation, "invalid client instance ID")
return
}
selected, err := domain.SelectProtocol(server.protocolRange(), hello.GetSupportedProtocol())
if err != nil {
server.incMetric("protocol_error")
server.close(connection, websocket.StatusPolicyViolation, "unsupported protocol")
return
}
@@ -126,15 +135,22 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
server.close(connection, websocket.StatusInternalError, "could not persist client capabilities")
return
}
registrationStarted := time.Now()
registration, err := server.Store.RegisterClientSession(parent, store.ClientRegistration{
ClientID: hello.GetClientId(), Platform: uint32(hello.GetPlatform()), Architecture: hello.GetArchitecture(),
DaemonVersion: hello.GetDaemonVersion(), DaemonCWD: hello.GetDaemonCwd(), SupportedShells: shells,
ClientInstanceID: [16]byte(instanceID), SessionID: sessionID, ConnectedAt: server.now(),
})
server.observeMetric("sqlite_write_latency", time.Since(registrationStarted))
if err != nil {
server.incMetric("client_registration_failure")
if errors.Is(err, store.ErrTakeoverRequired) {
server.incMetric("client_takeover_required")
}
server.close(connection, websocket.StatusPolicyViolation, sessionCloseReason(err))
return
}
server.incMetric("client_registration")
registry := server.Registry
handle, err := registry.Install(hello.GetClientId(), sessionID, registration.Generation)
if err != nil {
@@ -227,16 +243,19 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
// to determine liveness.
heartbeat.Observe(time.Since(started))
if messageType != websocket.MessageBinary {
server.incMetric("protocol_error")
server.close(connection, websocket.StatusUnsupportedData, ErrUnexpectedMessage.Error())
return
}
envelope, err := agentproto.DecodeEnvelope(payload, server.limits(), hello.GetPlatform())
if err != nil || envelope.GetClientHello() != nil || envelope.GetSessionId() != encodedSessionID || envelope.GetSessionGeneration() != registration.Generation {
server.incMetric("stale_message")
server.close(connection, websocket.StatusPolicyViolation, ErrStaleSession.Error())
return
}
clientID, err := server.Store.ValidateLiveSession(sessionContext, sessionID, registration.Generation)
if err != nil || clientID != hello.GetClientId() {
server.incMetric("stale_message")
server.close(connection, websocket.StatusPolicyViolation, ErrStaleSession.Error())
return
}
@@ -317,14 +336,20 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
}
appendEvent, eventErr := eventAppendFromWire(event, hello.GetClientId(), registration.Generation, server.now())
if eventErr != nil {
server.incMetric("protocol_error")
server.close(connection, websocket.StatusPolicyViolation, "invalid command event")
return
}
appended, eventErr := server.Store.AppendCommandEvent(sessionContext, appendEvent)
if eventErr != nil {
server.incMetric("command_event_rejected")
server.close(connection, websocket.StatusPolicyViolation, "command event was not accepted")
return
}
server.incMetric("command_event")
if event.GetLifecycle() != nil {
server.incMetric("command_transition")
}
ack, eventErr := proto.Marshal(&rvboxv1.AgentEnvelope{SessionId: encodedSessionID, SessionGeneration: registration.Generation, Payload: &rvboxv1.AgentEnvelope_EventAck{EventAck: &rvboxv1.EventAck{IssueUuid: event.GetIssueUuid(), ThroughEventSeq: appended.ThroughEventSeq}}})
if eventErr != nil || queue.EnqueueControl(Frame{Kind: FrameControl, Payload: ack}) != nil {
server.close(connection, websocket.StatusInternalError, "could not acknowledge command event")
@@ -452,6 +477,8 @@ func (server *AgentServer) enqueueNextDispatch(ctx context.Context, queue *Write
beforeEnqueue(candidate)
}
if !queue.EnqueueData(Frame{Kind: FrameData, Payload: encoded, OnWritten: func() {
server.incMetric("command_dispatch")
server.observeMetric("dispatch_latency", server.now().Sub(candidate.IssueTime))
if onWritten != nil {
onWritten(candidate.IssueUUID)
}
@@ -756,11 +783,24 @@ func (server *AgentServer) writeLoop(ctx context.Context, connection *websocket.
return
}
case HeartbeatClose:
server.incMetric("heartbeat_timeout")
return
}
}
}
func (server *AgentServer) incMetric(name string) {
if server != nil && server.Metrics != nil {
server.Metrics.Inc(name)
}
}
func (server *AgentServer) observeMetric(name string, duration time.Duration) {
if server != nil && server.Metrics != nil {
server.Metrics.ObserveDuration(name, duration)
}
}
func (server *AgentServer) close(connection *websocket.Conn, status websocket.StatusCode, reason string) {
_ = connection.Close(status, reason)
}
internal/server/session/agent_server_test.go
+14
View File
@@ -18,6 +18,7 @@ import (
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/domain"
"github.com/rvbox/rvbox/internal/observability"
"github.com/rvbox/rvbox/internal/server/store"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
@@ -98,6 +99,19 @@ func TestAgentServerRegistrationAndReplacement_HP_SES_05(t *testing.T) {
}
}
func TestAgentServerEmitsBoundedRegistrationMetrics_HP_OPS_05(t *testing.T) {
server, agent, cleanup := newTestAgentServerWithStore(t)
defer cleanup()
metrics := observability.New()
agent.Metrics = metrics
connection, _ := dialAndHello(t, server, "metric-client", "019c46f1-1d02-7000-8000-000000000073")
defer connection.CloseNow()
_, _, counters := metrics.Snapshot()
if counters["agent_connection"] != 1 || counters["client_registration"] != 1 {
t.Fatalf("registration metrics = %#v", counters)
}
}
func TestAgentServerDispatchesCommandAdmittedDuringReconcile_HP_SES_11(t *testing.T) {
server, agent, cleanup := newTestAgentServerWithStore(t)
defer cleanup()