264 lines
9.8 KiB
Go
264 lines
9.8 KiB
Go
// Command rvbox-server is the RVBox control and agent server.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
|
|
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
|
"github.com/rvbox/rvbox/internal/agentproto"
|
|
"github.com/rvbox/rvbox/internal/config"
|
|
"github.com/rvbox/rvbox/internal/domain"
|
|
"github.com/rvbox/rvbox/internal/observability"
|
|
"github.com/rvbox/rvbox/internal/server/control"
|
|
"github.com/rvbox/rvbox/internal/server/session"
|
|
"github.com/rvbox/rvbox/internal/server/store"
|
|
"google.golang.org/grpc"
|
|
)
|
|
|
|
// buildVersion is set by scripts/release for published artifacts. Development
|
|
// and test builds intentionally retain the explicit non-release value.
|
|
var buildVersion = "dev"
|
|
|
|
func main() {
|
|
var configPath string
|
|
var checkConfig bool
|
|
var version bool
|
|
flag.StringVar(&configPath, "config", "", "absolute server TOML configuration path")
|
|
flag.BoolVar(&checkConfig, "check-config", false, "validate server configuration and exit")
|
|
flag.BoolVar(&version, "version", false, "print build version and exit")
|
|
flag.Parse()
|
|
if version {
|
|
fmt.Fprintln(os.Stdout, buildVersion)
|
|
return
|
|
}
|
|
if configPath == "" {
|
|
log.Print("rvbox-server: --config is required")
|
|
os.Exit(2)
|
|
}
|
|
if checkConfig {
|
|
configured, err := config.LoadServer(configPath)
|
|
if err != nil {
|
|
log.Printf("rvbox-server: invalid configuration: %v", err)
|
|
os.Exit(1)
|
|
}
|
|
if _, err := fmt.Fprintf(os.Stdout, "valid server configuration: data_dir=%s agent_listen=%s control_socket=%s observability_listen=%s json_rpc_enabled=%t\n", configured.Server.DataDir, configured.Server.AgentListen, configured.Server.ControlSocket, configured.Observability.Listen, configured.JSONRPC.Enabled); err != nil {
|
|
log.Printf("rvbox-server: write check result: %v", err)
|
|
os.Exit(1)
|
|
}
|
|
return
|
|
}
|
|
if err := run(configPath); err != nil {
|
|
log.Printf("rvbox-server: %v", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func run(configPath string) error {
|
|
configured, err := config.LoadServer(configPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
runtimeLog, err := observability.OpenRotatingFile(configured.Observability.LogFile, configured.Observability.LogMaxBytes, configured.Observability.LogMaxFiles)
|
|
if err != nil {
|
|
return fmt.Errorf("open server rotating log: %w", err)
|
|
}
|
|
defer runtimeLog.Close()
|
|
logger := log.New(io.MultiWriter(os.Stderr, observability.FormatLog(runtimeLog, configured.Observability.LogFormat)), "rvbox-server: ", log.LstdFlags|log.LUTC)
|
|
persistence, err := store.Open(context.Background(), store.Options{
|
|
DataDir: configured.Server.DataDir, BusyTimeout: configured.Storage.SQLiteBusyTimeout,
|
|
SegmentTargetSize: configured.Storage.SegmentTargetBytes,
|
|
QuotaLimits: store.QuotaLimits{
|
|
HardAllocationBytes: store.DefaultSegmentLimit,
|
|
CommandOutputBytes: configured.Storage.CommandOutputLimitBytes,
|
|
CommandTotalBytes: configured.Storage.CommandTotalLimitBytes,
|
|
ClientTotalBytes: configured.Storage.ClientTotalLimitBytes,
|
|
ServerTotalBytes: configured.Storage.ServerTotalLimitBytes,
|
|
CloseoutReserveBytes: configured.Storage.CommandCloseoutReserveBytes,
|
|
FilesystemFloorBytes: configured.Storage.FreeSpaceFloorBytes,
|
|
},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer persistence.Close()
|
|
health := observability.New()
|
|
healthServer := &http.Server{Handler: health.Handler(observability.Paths{Liveness: configured.Observability.LivenessPath, Readiness: configured.Observability.ReadinessPath, Metrics: configured.Observability.MetricsPath}), ReadHeaderTimeout: configured.Flow.WriteDeadline}
|
|
// Health is deliberately best effort. A port collision or a temporary
|
|
// listener failure must not prevent the control/agent service from starting;
|
|
// operators still get the failure in the daemon log.
|
|
var healthListener net.Listener
|
|
if configured.Observability.Listen != "" {
|
|
healthListener, err = net.Listen("tcp", configured.Observability.Listen)
|
|
if err != nil {
|
|
logger.Printf("observability endpoint unavailable (continuing without it): %v", err)
|
|
}
|
|
}
|
|
if healthListener != nil {
|
|
defer healthListener.Close()
|
|
}
|
|
recoveryContext, cancelRecovery := context.WithCancel(context.Background())
|
|
defer cancelRecovery()
|
|
go publishServerTelemetry(recoveryContext, persistence, health)
|
|
// Recovery is deliberately asynchronous: liveness and incident inspection
|
|
// remain available while committed-range checks run. Readiness becomes
|
|
// true only after the real SQLite/segment recovery completes.
|
|
go func() {
|
|
if _, recoverErr := persistence.RecoverCommandSegments(recoveryContext); recoverErr != nil {
|
|
health.SetDirty(true)
|
|
if incidentID, idErr := domain.NewUUIDv7(); idErr == nil {
|
|
if _, incidentErr := persistence.RecordIncident(context.Background(), store.IncidentInput{
|
|
IncidentUUID: incidentID, DetectedAt: time.Now().UTC(), Kind: store.IncidentSQLiteIntegrity,
|
|
Scope: store.IncidentScopeGlobal, ScopeKey: "server-startup-recovery", Summary: "startup storage recovery failed",
|
|
Evidence: []byte(recoverErr.Error()), AutomaticallyRepairable: false,
|
|
}); incidentErr != nil {
|
|
logger.Printf("could not persist recovery incident: %v", incidentErr)
|
|
}
|
|
}
|
|
logger.Printf("storage recovery left readiness disabled: %v", recoverErr)
|
|
return
|
|
}
|
|
health.SetReady(true)
|
|
}()
|
|
|
|
listener, err := net.Listen("tcp", configured.Server.AgentListen)
|
|
if err != nil {
|
|
return fmt.Errorf("listen for agents: %w", err)
|
|
}
|
|
defer listener.Close()
|
|
controlListener, cleanupControl, err := control.ListenUnix(configured.Server.ControlSocket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = cleanupControl() }()
|
|
registry := session.NewRegistry()
|
|
controlService, err := control.NewService(control.Options{
|
|
Store: persistence, DefaultQueueTTL: configured.Queue.DefaultTTL, DefaultQueueTTLSet: true, TakeoverTTL: configured.Protocol.TakeoverTTL,
|
|
WakeClient: registry.Wake,
|
|
Metrics: health,
|
|
Limits: agentproto.Limits{MaxEnvelopeBytes: configured.Protocol.MaxAgentEnvelopeBytes, MaxExecutionSpecBytes: configured.Protocol.MaxExecutionSpecBytes, MaxRawChunkBytes: configured.Protocol.MaxRawChunkBytes, MaxScriptBytes: configured.Protocol.MaxScriptBytes, MaxDetailBytes: configured.Storage.ProtocolDetailMaxBytes},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
grpcServer := grpc.NewServer(grpc.MaxRecvMsgSize(int(configured.Protocol.MaxControlRequestBytes)), grpc.MaxSendMsgSize(int(configured.Protocol.MaxControlRequestBytes)))
|
|
rvboxv1.RegisterControlServer(grpcServer, controlService)
|
|
var rpcListener net.Listener
|
|
var rpcServer *http.Server
|
|
if configured.JSONRPC.Enabled {
|
|
rpcListener, err = net.Listen("tcp", configured.JSONRPC.Listen)
|
|
if err != nil {
|
|
return fmt.Errorf("listen for JSON-RPC: %w", err)
|
|
}
|
|
defer rpcListener.Close()
|
|
if configured.JSONRPC.NonLoopbackBind {
|
|
logger.Printf("WARNING JSON-RPC is unauthenticated and bound to non-loopback address %s", configured.JSONRPC.Listen)
|
|
}
|
|
rpcServer = &http.Server{Handler: control.NewJSONRPCHandler(controlService, int64(configured.Protocol.MaxJSONRPCBodyBytes)), ReadHeaderTimeout: configured.Flow.WriteDeadline}
|
|
}
|
|
agent := &session.AgentServer{
|
|
Store: persistence, Registry: registry, Path: configured.Server.AgentPath,
|
|
Metrics: health,
|
|
Limits: agentproto.Limits{
|
|
MaxEnvelopeBytes: configured.Protocol.MaxAgentEnvelopeBytes,
|
|
MaxExecutionSpecBytes: configured.Protocol.MaxExecutionSpecBytes,
|
|
MaxRawChunkBytes: configured.Protocol.MaxRawChunkBytes,
|
|
MaxScriptBytes: configured.Protocol.MaxScriptBytes,
|
|
MaxDetailBytes: configured.Storage.ProtocolDetailMaxBytes,
|
|
},
|
|
WriteDeadline: configured.Flow.WriteDeadline,
|
|
HeartbeatIdle: configured.Protocol.HeartbeatIdle, LivenessTimeout: configured.Protocol.LivenessTimeout,
|
|
}
|
|
httpServer := &http.Server{Handler: agent, ReadHeaderTimeout: configured.Flow.WriteDeadline}
|
|
serveError := make(chan error, 4)
|
|
go func() { serveError <- httpServer.Serve(listener) }()
|
|
go func() { serveError <- grpcServer.Serve(controlListener) }()
|
|
if healthListener != nil {
|
|
go func() { serveError <- healthServer.Serve(healthListener) }()
|
|
}
|
|
if rpcServer != nil {
|
|
go func() { serveError <- rpcServer.Serve(rpcListener) }()
|
|
}
|
|
|
|
signals := make(chan os.Signal, 1)
|
|
signal.Notify(signals, os.Interrupt, syscall.SIGTERM)
|
|
defer signal.Stop(signals)
|
|
select {
|
|
case err := <-serveError:
|
|
if errors.Is(err, http.ErrServerClosed) {
|
|
if healthListener != nil {
|
|
_ = healthServer.Close()
|
|
}
|
|
if rpcServer != nil {
|
|
_ = rpcServer.Close()
|
|
}
|
|
grpcServer.Stop()
|
|
return nil
|
|
}
|
|
_ = httpServer.Close()
|
|
if healthListener != nil {
|
|
_ = healthServer.Close()
|
|
}
|
|
if rpcServer != nil {
|
|
_ = rpcServer.Close()
|
|
}
|
|
grpcServer.Stop()
|
|
return err
|
|
case <-signals:
|
|
shutdownContext, cancel := context.WithTimeout(context.Background(), configured.Server.ShutdownGrace)
|
|
defer cancel()
|
|
httpErr := httpServer.Shutdown(shutdownContext)
|
|
var healthErr error
|
|
if healthListener != nil {
|
|
healthErr = healthServer.Shutdown(shutdownContext)
|
|
}
|
|
if httpErr == nil {
|
|
httpErr = healthErr
|
|
}
|
|
if rpcServer != nil {
|
|
if err := rpcServer.Shutdown(shutdownContext); httpErr == nil {
|
|
httpErr = err
|
|
}
|
|
}
|
|
grpcDone := make(chan struct{})
|
|
go func() { grpcServer.GracefulStop(); close(grpcDone) }()
|
|
select {
|
|
case <-grpcDone:
|
|
case <-shutdownContext.Done():
|
|
grpcServer.Stop()
|
|
}
|
|
return httpErr
|
|
}
|
|
}
|
|
|
|
func publishServerTelemetry(ctx context.Context, persistence *store.Store, health *observability.Health) {
|
|
ticker := time.NewTicker(10 * time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
sampleContext, cancel := context.WithTimeout(ctx, time.Second)
|
|
telemetry, err := persistence.Telemetry(sampleContext)
|
|
cancel()
|
|
if err != nil {
|
|
health.Inc("telemetry_read_failure")
|
|
} else {
|
|
health.SetGauge("queue_depth", float64(telemetry.QueuedCommands))
|
|
health.SetGauge("server_charged_bytes", float64(telemetry.ChargedBytes))
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|