cabbage/rvbox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add bounded operational metrics

Browse Source
This commit is contained in:
cabbage
2026-09-11 09:09:52 +00:00
parent 46185f1f6c
commit 2cf563d88e
12 changed files with 342 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/server/session/agent_server.go
+40
View File
@@ -16,6 +16,7 @@ import (
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/domain"
"github.com/rvbox/rvbox/internal/observability"
"github.com/rvbox/rvbox/internal/server/store"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
@@ -45,6 +46,9 @@ type AgentServer struct {
HeartbeatIdle time.Duration
LivenessTimeout time.Duration
Now func() time.Time
// Metrics is optional so transport tests and embeddings do not need an
// observability endpoint. Every emitted name is a static bounded value.
Metrics *observability.Health
}
func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http.Request) {
@@ -53,6 +57,7 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
return
}
if request.Header.Get("Origin") != "" {
server.incMetric("protocol_error")
http.Error(response, ErrUnexpectedOrigin.Error(), http.StatusForbidden)
return
}
@@ -83,6 +88,7 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
if err != nil {
return
}
server.incMetric("agent_connection")
defer connection.CloseNow()
server.serveConnection(request.Context(), connection, heartbeat, started)
}
@@ -98,6 +104,7 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
}
helloEnvelope, err := agentproto.DecodeEnvelope(payload, server.limits(), rvboxv1.Platform_PLATFORM_UNSPECIFIED)
if err != nil || helloEnvelope.GetClientHello() == nil {
server.incMetric("protocol_error")
server.close(connection, websocket.StatusPolicyViolation, "invalid ClientHello")
return
}
@@ -108,11 +115,13 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
reconcileBoundary := server.now()
instanceID, err := domain.ParseUUIDv7(hello.GetClientInstanceId())
if err != nil {
server.incMetric("protocol_error")
server.close(connection, websocket.StatusPolicyViolation, "invalid client instance ID")
return
}
selected, err := domain.SelectProtocol(server.protocolRange(), hello.GetSupportedProtocol())
if err != nil {
server.incMetric("protocol_error")
server.close(connection, websocket.StatusPolicyViolation, "unsupported protocol")
return
}
@@ -126,15 +135,22 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
server.close(connection, websocket.StatusInternalError, "could not persist client capabilities")
return
}
registrationStarted := time.Now()
registration, err := server.Store.RegisterClientSession(parent, store.ClientRegistration{
ClientID: hello.GetClientId(), Platform: uint32(hello.GetPlatform()), Architecture: hello.GetArchitecture(),
DaemonVersion: hello.GetDaemonVersion(), DaemonCWD: hello.GetDaemonCwd(), SupportedShells: shells,
ClientInstanceID: [16]byte(instanceID), SessionID: sessionID, ConnectedAt: server.now(),
})
server.observeMetric("sqlite_write_latency", time.Since(registrationStarted))
if err != nil {
server.incMetric("client_registration_failure")
if errors.Is(err, store.ErrTakeoverRequired) {
server.incMetric("client_takeover_required")
}
server.close(connection, websocket.StatusPolicyViolation, sessionCloseReason(err))
return
}
server.incMetric("client_registration")
registry := server.Registry
handle, err := registry.Install(hello.GetClientId(), sessionID, registration.Generation)
if err != nil {
@@ -227,16 +243,19 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
// to determine liveness.
heartbeat.Observe(time.Since(started))
if messageType != websocket.MessageBinary {
server.incMetric("protocol_error")
server.close(connection, websocket.StatusUnsupportedData, ErrUnexpectedMessage.Error())
return
}
envelope, err := agentproto.DecodeEnvelope(payload, server.limits(), hello.GetPlatform())
if err != nil || envelope.GetClientHello() != nil || envelope.GetSessionId() != encodedSessionID || envelope.GetSessionGeneration() != registration.Generation {
server.incMetric("stale_message")
server.close(connection, websocket.StatusPolicyViolation, ErrStaleSession.Error())
return
}
clientID, err := server.Store.ValidateLiveSession(sessionContext, sessionID, registration.Generation)
if err != nil || clientID != hello.GetClientId() {
server.incMetric("stale_message")
server.close(connection, websocket.StatusPolicyViolation, ErrStaleSession.Error())
return
}
@@ -317,14 +336,20 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
}
appendEvent, eventErr := eventAppendFromWire(event, hello.GetClientId(), registration.Generation, server.now())
if eventErr != nil {
server.incMetric("protocol_error")
server.close(connection, websocket.StatusPolicyViolation, "invalid command event")
return
}
appended, eventErr := server.Store.AppendCommandEvent(sessionContext, appendEvent)
if eventErr != nil {
server.incMetric("command_event_rejected")
server.close(connection, websocket.StatusPolicyViolation, "command event was not accepted")
return
}
server.incMetric("command_event")
if event.GetLifecycle() != nil {
server.incMetric("command_transition")
}
ack, eventErr := proto.Marshal(&rvboxv1.AgentEnvelope{SessionId: encodedSessionID, SessionGeneration: registration.Generation, Payload: &rvboxv1.AgentEnvelope_EventAck{EventAck: &rvboxv1.EventAck{IssueUuid: event.GetIssueUuid(), ThroughEventSeq: appended.ThroughEventSeq}}})
if eventErr != nil || queue.EnqueueControl(Frame{Kind: FrameControl, Payload: ack}) != nil {
server.close(connection, websocket.StatusInternalError, "could not acknowledge command event")
@@ -452,6 +477,8 @@ func (server *AgentServer) enqueueNextDispatch(ctx context.Context, queue *Write
beforeEnqueue(candidate)
}
if !queue.EnqueueData(Frame{Kind: FrameData, Payload: encoded, OnWritten: func() {
server.incMetric("command_dispatch")
server.observeMetric("dispatch_latency", server.now().Sub(candidate.IssueTime))
if onWritten != nil {
onWritten(candidate.IssueUUID)
}
@@ -756,11 +783,24 @@ func (server *AgentServer) writeLoop(ctx context.Context, connection *websocket.
return
}
case HeartbeatClose:
server.incMetric("heartbeat_timeout")
return
}
}
}
func (server *AgentServer) incMetric(name string) {
if server != nil && server.Metrics != nil {
server.Metrics.Inc(name)
}
}
func (server *AgentServer) observeMetric(name string, duration time.Duration) {
if server != nil && server.Metrics != nil {
server.Metrics.ObserveDuration(name, duration)
}
}
func (server *AgentServer) close(connection *websocket.Conn, status websocket.StatusCode, reason string) {
_ = connection.Close(status, reason)
}