fix(transport): align WebSocket read limits with agent protocol

This commit is contained in:
cabbage committed 2026-11-09 07:53:18 +00:00
1 parent ef6e65e6f1
commit 6984a09486
11 files changed
+376 -4

No files matched your search

+1 -1
View File
@@ -168,7 +168,7 @@ func run(configPath string) error {
} }
agent := &session.AgentServer{ agent := &session.AgentServer{
Store: persistence, Registry: registry, Path: configured.Server.AgentPath, Store: persistence, Registry: registry, Path: configured.Server.AgentPath,
Metrics: health, Metrics: health, Logf: logger.Printf,
Limits: agentproto.Limits{ Limits: agentproto.Limits{
MaxEnvelopeBytes: configured.Protocol.MaxAgentEnvelopeBytes, MaxEnvelopeBytes: configured.Protocol.MaxAgentEnvelopeBytes,
MaxExecutionSpecBytes: configured.Protocol.MaxExecutionSpecBytes, MaxExecutionSpecBytes: configured.Protocol.MaxExecutionSpecBytes,
+2 -2
View File
@@ -249,7 +249,7 @@ func runClientDaemon(ctx context.Context, configPath string, diagnostics io.Writ
if runErr := agent.Run(ctx, agent.RunnerOptions{ if runErr := agent.Run(ctx, agent.RunnerOptions{
Store: state, Store: state,
Dial: func(dialContext context.Context) (agent.Transport, error) { Dial: func(dialContext context.Context) (agent.Transport, error) {
return agent.DialWebSocket(dialContext, configured.Client.ServerURL, httpClient) return agent.DialWebSocketWithLimits(dialContext, configured.Client.ServerURL, httpClient, limits)
}, },
Hello: hello, Limits: limits, Hello: hello, Limits: limits,
Backoff: agent.BackoffOptions{Initial: configured.Network.ReconnectInitial, Maximum: configured.Network.ReconnectMax, StableReset: configured.Network.StableSessionReset}, Backoff: agent.BackoffOptions{Initial: configured.Network.ReconnectInitial, Maximum: configured.Network.ReconnectMax, StableReset: configured.Network.StableSessionReset},
@@ -389,7 +389,7 @@ func clientHello(configured *config.Client, instance domain.UUID) *rvboxv1.Clien
} }
return &rvboxv1.ClientHello{ return &rvboxv1.ClientHello{
ClientId: clientID, SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1, MinMinor: 0, MaxMinor: 0}, ClientId: clientID, SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1, MinMinor: 0, MaxMinor: 0},
DaemonVersion: "v1", Platform: platform, Architecture: runtime.GOARCH, DaemonCwd: configured.Client.DaemonCWD, DaemonVersion: buildVersion, Platform: platform, Architecture: runtime.GOARCH, DaemonCwd: configured.Client.DaemonCWD,
SupportedShells: shells, ClientInstanceId: instance.String(), MaxRunningCommands: configured.Client.MaxRunningCommands, SupportedShells: shells, ClientInstanceId: instance.String(), MaxRunningCommands: configured.Client.MaxRunningCommands,
MaxQueuedCommands: configured.Client.MaxQueuedCommands, SentAt: timestamppb.New(time.Now().UTC()), MaxQueuedCommands: configured.Client.MaxQueuedCommands, SentAt: timestamppb.New(time.Now().UTC()),
} }
+10
View File
@@ -126,6 +126,16 @@ exact ignored `dist/` directory after preserving any evidence.
## Common incidents ## Common incidents
- **Reconnect loop after a command produces substantial output:** inspect
`C:\ProgramData\RVBox\service-startup.log` on Windows and the server log for
`StatusMessageTooBig` / `read limited at 32769 bytes`. Early v1 transports
inherited a 32 KiB WebSocket ceiling despite permitting 64 KiB chunks and
1 MiB envelopes. Both transports now set the configured envelope ceiling
before reading any messages. Upgrade the server first to drain pending
client output, then the client for large dispatch/reconciliation responses.
Preserve spool/history: rebooting cannot remove a replayable oversized event.
New server logs include bounded read/reconciliation errors and an
`agent_message_too_big` counter; new clients advertise the actual build version.
- **No client / stale session:** verify nginx has WebSocket `101` entries for - **No client / stale session:** verify nginx has WebSocket `101` entries for
`/v1/agent`, server readiness is true, and the Windows service is running. `/v1/agent`, server readiness is true, and the Windows service is running.
Use `rvc stat CLIENT_ID`; do not restart the client merely to clear history. Use `rvc stat CLIENT_ID`; do not restart the client merely to clear history.
+18
View File
@@ -3,9 +3,12 @@ package agent
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"math"
"net/http" "net/http"
"github.com/coder/websocket" "github.com/coder/websocket"
"github.com/rvbox/rvbox/internal/agentproto"
) )
var ErrUnexpectedMessage = errors.New("agent transport received a non-binary WebSocket message") var ErrUnexpectedMessage = errors.New("agent transport received a non-binary WebSocket message")
@@ -13,10 +16,25 @@ var ErrUnexpectedMessage = errors.New("agent transport received a non-binary Web
type WebSocketTransport struct{ connection *websocket.Conn } type WebSocketTransport struct{ connection *websocket.Conn }
func DialWebSocket(ctx context.Context, address string, client *http.Client) (*WebSocketTransport, error) { func DialWebSocket(ctx context.Context, address string, client *http.Client) (*WebSocketTransport, error) {
return DialWebSocketWithLimits(ctx, address, client, agentproto.DefaultLimits())
}
// DialWebSocketWithLimits applies the same message ceiling as envelope
// validation. The library's 32 KiB default is smaller than ordinary RVBox
// chunks, dispatches, and reconciliation history.
func DialWebSocketWithLimits(ctx context.Context, address string, client *http.Client, limits agentproto.Limits) (*WebSocketTransport, error) {
readLimit := limits.MaxEnvelopeBytes
if readLimit == 0 {
readLimit = agentproto.DefaultLimits().MaxEnvelopeBytes
}
if readLimit > math.MaxInt64-1 {
return nil, fmt.Errorf("invalid WebSocket envelope limit: %d", readLimit)
}
connection, _, err := websocket.Dial(ctx, address, &websocket.DialOptions{HTTPClient: client, CompressionMode: websocket.CompressionDisabled}) connection, _, err := websocket.Dial(ctx, address, &websocket.DialOptions{HTTPClient: client, CompressionMode: websocket.CompressionDisabled})
if err != nil { if err != nil {
return nil, err return nil, err
} }
connection.SetReadLimit(int64(readLimit))
return &WebSocketTransport{connection: connection}, nil return &WebSocketTransport{connection: connection}, nil
} }
+68
View File
@@ -0,0 +1,68 @@
package agent
import (
"bytes"
"context"
"errors"
"math"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/coder/websocket"
"github.com/rvbox/rvbox/internal/agentproto"
)
func TestWebSocketConfiguredReadCeiling_BH_WS_01(t *testing.T) {
t.Parallel()
for _, scenario := range []struct {
name string
limit uint64
bytes int
wantLarge bool
}{
{name: "above library default", bytes: (32 << 10) + 1},
{name: "exact default ceiling", bytes: 1 << 20},
{name: "over default ceiling", bytes: (1 << 20) + 1, wantLarge: true},
{name: "exact configured ceiling", limit: 2048, bytes: 2048},
{name: "over configured ceiling", limit: 2048, bytes: 2049, wantLarge: true},
} {
t.Run(scenario.name, func(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
payload := bytes.Repeat([]byte{0x51}, scenario.bytes)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
connection, err := websocket.Accept(w, r, nil)
if err != nil {
return
}
defer connection.CloseNow()
if err := connection.Write(ctx, websocket.MessageBinary, payload); err != nil {
return
}
_, _, _ = connection.Read(ctx) // Complete the peer's close handshake.
}))
defer server.Close()
transport, err := DialWebSocketWithLimits(ctx, "ws"+strings.TrimPrefix(server.URL, "http"), nil,
agentproto.Limits{MaxEnvelopeBytes: scenario.limit})
if err != nil {
t.Fatal(err)
}
defer transport.Close()
got, err := transport.Read(ctx)
if scenario.wantLarge {
if !errors.Is(err, websocket.ErrMessageTooBig) {
t.Fatalf("oversize read = %v", err)
}
} else if err != nil || !bytes.Equal(got, payload) {
t.Fatalf("read bytes=%d error=%v", len(got), err)
}
})
}
if _, err := DialWebSocketWithLimits(context.Background(), "invalid", nil, agentproto.Limits{MaxEnvelopeBytes: math.MaxUint64}); err == nil {
t.Fatal("overflowing read ceiling accepted")
}
}
+30
View File
@@ -7,6 +7,7 @@ import (
"encoding/base64" "encoding/base64"
"errors" "errors"
"fmt" "fmt"
"math"
"net/http" "net/http"
"sort" "sort"
"sync" "sync"
@@ -49,6 +50,8 @@ type AgentServer struct {
// Metrics is optional so transport tests and embeddings do not need an // Metrics is optional so transport tests and embeddings do not need an
// observability endpoint. Every emitted name is a static bounded value. // observability endpoint. Every emitted name is a static bounded value.
Metrics *observability.Health Metrics *observability.Health
// Logf receives bounded transport diagnostics without command payloads.
Logf func(string, ...any)
} }
func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http.Request) { func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http.Request) {
@@ -69,6 +72,10 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
http.Error(response, "agent session registry is unavailable", http.StatusServiceUnavailable) http.Error(response, "agent session registry is unavailable", http.StatusServiceUnavailable)
return return
} }
if server.limits().MaxEnvelopeBytes > math.MaxInt64-1 {
http.Error(response, "invalid agent envelope limit", http.StatusServiceUnavailable)
return
}
started := time.Now() started := time.Now()
heartbeat := newSynchronizedHeartbeat(server.heartbeatIdle(), server.livenessTimeout(), 0) heartbeat := newSynchronizedHeartbeat(server.heartbeatIdle(), server.livenessTimeout(), 0)
connection, err := websocket.Accept(response, request, &websocket.AcceptOptions{ connection, err := websocket.Accept(response, request, &websocket.AcceptOptions{
@@ -88,6 +95,7 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
if err != nil { if err != nil {
return return
} }
connection.SetReadLimit(int64(server.limits().MaxEnvelopeBytes))
server.incMetric("agent_connection") server.incMetric("agent_connection")
defer connection.CloseNow() defer connection.CloseNow()
server.serveConnection(request.Context(), connection, heartbeat, started) server.serveConnection(request.Context(), connection, heartbeat, started)
@@ -96,6 +104,10 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
func (server *AgentServer) serveConnection(parent context.Context, connection *websocket.Conn, heartbeat *synchronizedHeartbeat, started time.Time) { func (server *AgentServer) serveConnection(parent context.Context, connection *websocket.Conn, heartbeat *synchronizedHeartbeat, started time.Time) {
messageType, payload, err := connection.Read(parent) messageType, payload, err := connection.Read(parent)
if err != nil { if err != nil {
if errors.Is(err, websocket.ErrMessageTooBig) {
server.incMetric("agent_message_too_big")
}
server.logSessionError("", 0, "hello-read", err)
return return
} }
if messageType != websocket.MessageBinary { if messageType != websocket.MessageBinary {
@@ -143,6 +155,7 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
}) })
server.observeMetric("sqlite_write_latency", time.Since(registrationStarted)) server.observeMetric("sqlite_write_latency", time.Since(registrationStarted))
if err != nil { if err != nil {
server.logSessionError(hello.GetClientId(), 0, "register", err)
server.incMetric("client_registration_failure") server.incMetric("client_registration_failure")
if errors.Is(err, store.ErrTakeoverRequired) { if errors.Is(err, store.ErrTakeoverRequired) {
server.incMetric("client_takeover_required") server.incMetric("client_takeover_required")
@@ -242,6 +255,10 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
for { for {
messageType, payload, err = connection.Read(sessionContext) messageType, payload, err = connection.Read(sessionContext)
if err != nil { if err != nil {
if errors.Is(err, websocket.ErrMessageTooBig) {
server.incMetric("agent_message_too_big")
}
server.logSessionError(hello.GetClientId(), registration.Generation, "read", err)
return return
} }
// Elapsed time is monotonic; peer-provided wall timestamps are never used // Elapsed time is monotonic; peer-provided wall timestamps are never used
@@ -267,6 +284,7 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
if snapshot := envelope.GetReconcileSnapshot(); snapshot != nil { if snapshot := envelope.GetReconcileSnapshot(); snapshot != nil {
result, reconcileErr := server.Store.ReconcileClientSnapshotForSessionAt(sessionContext, hello.GetClientId(), registration.Generation, snapshot, reconcileBoundary) result, reconcileErr := server.Store.ReconcileClientSnapshotForSessionAt(sessionContext, hello.GetClientId(), registration.Generation, snapshot, reconcileBoundary)
if reconcileErr != nil { if reconcileErr != nil {
server.logSessionError(hello.GetClientId(), registration.Generation, "reconcile", reconcileErr)
server.close(connection, websocket.StatusPolicyViolation, "reconciliation failed") server.close(connection, websocket.StatusPolicyViolation, "reconciliation failed")
return return
} }
@@ -348,6 +366,7 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
appended, eventErr := server.Store.AppendCommandEvent(sessionContext, appendEvent) appended, eventErr := server.Store.AppendCommandEvent(sessionContext, appendEvent)
if eventErr != nil { if eventErr != nil {
server.incMetric("command_event_rejected") server.incMetric("command_event_rejected")
server.logSessionError(hello.GetClientId(), registration.Generation, "command-event", eventErr)
server.close(connection, websocket.StatusPolicyViolation, "command event was not accepted") server.close(connection, websocket.StatusPolicyViolation, "command event was not accepted")
return return
} }
@@ -810,6 +829,17 @@ func (server *AgentServer) close(connection *websocket.Conn, status websocket.St
_ = connection.Close(status, reason) _ = connection.Close(status, reason)
} }
func (server *AgentServer) logSessionError(clientID string, generation uint64, stage string, err error) {
if server.Logf == nil || err == nil {
return
}
detail := err.Error()
if len(detail) > 1024 {
detail = detail[:1024] + "..."
}
server.Logf("agent session client=%q generation=%d stage=%s error=%q", clientID, generation, stage, detail)
}
func (server *AgentServer) agentPath() string { func (server *AgentServer) agentPath() string {
if server.Path == "" { if server.Path == "" {
return defaultAgentPath return defaultAgentPath
@@ -5,6 +5,7 @@ import (
"context" "context"
"crypto/sha256" "crypto/sha256"
"errors" "errors"
"fmt"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"os" "os"
@@ -225,6 +226,37 @@ func TestAgentServerRejectsHostileWireInputs_BH_SES_04(t *testing.T) {
} }
} }
func TestAgentServerEnforcesConfiguredReadCeiling_BH_WS_02(t *testing.T) {
t.Parallel()
server, edge, cleanup := newTestAgentServerWithStore(t)
defer cleanup()
edge.Limits = agentproto.DefaultLimits()
edge.Limits.MaxEnvelopeBytes = 2048
logs := make(chan string, 1)
edge.Logf = func(format string, args ...any) { logs <- fmt.Sprintf(format, args...) }
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
connection, _, err := websocket.Dial(ctx, websocketURL(server.URL), nil)
if err != nil {
t.Fatal(err)
}
defer connection.CloseNow()
if err := connection.Write(ctx, websocket.MessageBinary, bytes.Repeat([]byte{0x51}, 2049)); err != nil {
t.Fatal(err)
}
if _, _, err := connection.Read(ctx); websocket.CloseStatus(err) != websocket.StatusMessageTooBig {
t.Fatalf("oversize close = %v", err)
}
select {
case diagnostic := <-logs:
if !strings.Contains(diagnostic, "stage=hello-read") || !strings.Contains(diagnostic, "2049") {
t.Fatalf("missing bounded transport diagnostic: %s", diagnostic)
}
case <-ctx.Done():
t.Fatal("oversize failure was not logged")
}
}
func TestWireEventAppendCarriesClientBinding_HP_EVENT_01(t *testing.T) { func TestWireEventAppendCarriesClientBinding_HP_EVENT_01(t *testing.T) {
event := &rvboxv1.CommandEvent{IssueUuid: "019c46f1-1d02-7000-8000-000000000096", EventSeq: 1, ObservedAt: timestamppb.Now(), Payload: &rvboxv1.CommandEvent_Lifecycle{Lifecycle: &rvboxv1.LifecycleChange{Lifecycle: rvboxv1.CommandLifecycle_COMMAND_RUNNING, CommandRevision: 1}}} event := &rvboxv1.CommandEvent{IssueUuid: "019c46f1-1d02-7000-8000-000000000096", EventSeq: 1, ObservedAt: timestamppb.Now(), Payload: &rvboxv1.CommandEvent_Lifecycle{Lifecycle: &rvboxv1.LifecycleChange{Lifecycle: rvboxv1.CommandLifecycle_COMMAND_RUNNING, CommandRevision: 1}}}
digest, err := agentproto.CommandEventDigest(event) digest, err := agentproto.CommandEventDigest(event)
+1 -1
View File
@@ -8,7 +8,7 @@ description: "Operate and troubleshoot RVBox deployments through rvc and the Con
Use the deployment already present. Discover its addresses, paths, accounts, Use the deployment already present. Discover its addresses, paths, accounts,
versions, and supported capabilities; never substitute a remembered test host, versions, and supported capabilities; never substitute a remembered test host,
container name, credential, client ID, or filesystem layout. This reference container name, credential, client ID, or filesystem layout. This reference
matches RVBox v1 source reviewed on 2026-10-27: Linux server and Windows client. matches RVBox v1 source reviewed on 2026-11-09: Linux server and Windows client.
Unix-like client implementation is outside that release. Unix-like client implementation is outside that release.
## Establish the control route ## Establish the control route
@@ -19,6 +19,7 @@ initial evidence; do not restart services or resubmit work merely to inspect it.
| Socket permission denied | Run as the existing socket owner through the deployment route; inspect directory traversal and mount permissions. Do not chmod the control socket world-accessible. | | Socket permission denied | Run as the existing socket owner through the deployment route; inspect directory traversal and mount permissions. Do not chmod the control socket world-accessible. |
| Client missing or NOT_FOUND | List all client pages; verify the exact registered ID and server instance. A hostname assumption or wrong Compose project can select different state. Retention can also remove command history. | | Client missing or NOT_FOUND | List all client pages; verify the exact registered ID and server instance. A hostname assumption or wrong Compose project can select different state. Retention can also remove command history. |
| Disconnected / OFFLINE / UNAVAILABLE | Separate server recovery from client connectivity. Inspect SCM, WSS hostname/path, proxy WebSocket upgrade, certificate/time/CA, DNS, reconnect logs, and pending instance conflicts. Missing optional HTTP health is not proof of a stopped daemon. | | Disconnected / OFFLINE / UNAVAILABLE | Separate server recovery from client connectivity. Inspect SCM, WSS hostname/path, proxy WebSocket upgrade, certificate/time/CA, DNS, reconnect logs, and pending instance conflicts. Missing optional HTTP health is not proof of a stopped daemon. |
| `StatusMessageTooBig` / `read limited at 32769 bytes` after output or during reconnect | Early v1 WebSocket readers kept the library's 32 KiB default despite allowing larger protocol envelopes. Upgrade the server first to accept pending output, then the client for large incoming dispatch/history. Preserve spool/history; rebooting replays the same pending message. Corrected readers use the configured envelope ceiling, normally 1 MiB. |
| Repeated `signal command ...: command is not running` | An older client disconnects on a late signal instead of persisting its rejection. Upgrade the client while retaining config/spool/history; corrected builds acknowledge the signal without disconnecting. Do not wipe state or reissue the original command. | | Repeated `signal command ...: command is not running` | An older client disconnects on a late signal instead of persisting its rejection. Upgrade the client while retaining config/spool/history; corrected builds acknowledge the signal without disconnecting. Do not wipe state or reissue the original command. |
| QUEUED persists | Check connection, supported shell, queue/running capacity, acceptance events, and queue expiry. Queuing does not promise execution, and queue TTL is not a runtime timeout. | | QUEUED persists | Check connection, supported shell, queue/running capacity, acceptance events, and queue expiry. Queuing does not promise execution, and queue TTL is not a runtime timeout. |
| CLI deadline / broken stream | Work may still run. Query the captured issue and resume history via the API. For uncertain admission, retry only with the same request ID and payload. | | CLI deadline / broken stream | Work may still run. Query the captured issue and resume history via the API. For uncertain admission, retry only with the same request ID and payload. |
@@ -38,6 +39,16 @@ immediate repeated calls will succeed. Do not parse error text into invented
stable exit-code classes: the current CLI normally exits 1 for its own errors stable exit-code classes: the current CLI normally exits 1 for its own errors
and exposes remote lifecycle separately. and exposes remote lifecycle separately.
On Windows, SCM startup and reconnect diagnostics are written to
`%ProgramData%\RVBox\service-startup.log`, independently of optional
observability logging. Inspect recent entries around the failure before
restarting. The tray's log action may select the configured runtime log instead;
an empty/missing runtime log does not mean the service recorded no diagnostics.
Corrected servers log bounded WebSocket read/reconciliation errors without
command payloads, and count `agent_message_too_big` failures. Earlier clients
reported the placeholder daemon version `v1`; use the installed executable's
`--version` or PE product version when the server cannot identify the release.
## Windows identity, GUI, and shells ## Windows identity, GUI, and shells
`--elevated` is the only execution-context request toggle. Selection and `--elevated` is the only execution-context request toggle. Selection and
+24
View File
@@ -1157,6 +1157,30 @@ layer = "integration"
status = "implemented" status = "implemented"
tests = ["test/integration/store/store_integration_test.go:TestIncidentDirtyResolutionIdempotencyAndRecurrence_HP_STORE_11"] tests = ["test/integration/store/store_integration_test.go:TestIncidentDirtyResolutionIdempotencyAndRecurrence_HP_STORE_11"]
[[requirements]]
id = "REC-WS-01"
layer = "integration"
status = "implemented"
tests = ["test/integration/clientagent/websocket_limits_integration_test.go:TestLargeOutputReplaysAfterReconnect_REC_WS_01"]
[[requirements]]
id = "HP-WS-01"
layer = "integration"
status = "implemented"
tests = ["test/integration/clientagent/websocket_limits_integration_test.go:TestLargeDispatchAndReconciliation_HP_WS_01"]
[[requirements]]
id = "BH-WS-01"
layer = "unit"
status = "implemented"
tests = ["internal/client/agent/websocket_test.go:TestWebSocketConfiguredReadCeiling_BH_WS_01"]
[[requirements]]
id = "BH-WS-02"
layer = "unit"
status = "implemented"
tests = ["internal/server/session/agent_server_test.go:TestAgentServerEnforcesConfiguredReadCeiling_BH_WS_02"]
[[requirements]] [[requirements]]
id = "REC-STORE-03" id = "REC-STORE-03"
layer = "unit" layer = "unit"
@@ -0,0 +1,179 @@
package clientagent_test
import (
"bytes"
"context"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"time"
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/client/agent"
"github.com/rvbox/rvbox/internal/domain"
"github.com/rvbox/rvbox/internal/server/control"
"github.com/rvbox/rvbox/internal/server/session"
"github.com/rvbox/rvbox/internal/server/store"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
func TestLargeOutputReplaysAfterReconnect_REC_WS_01(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
fixture := newFrameFixture(t)
transport, accepted := fixture.connect(t, ctx, &rvboxv1.ReconcileSnapshot{})
dispatch := fixture.dispatch(t, ctx, transport, "echo fixture")
send := func(event *rvboxv1.CommandEvent, sequence uint64) {
t.Helper()
if err := agent.SendCommandEvent(ctx, transport, accepted, event, agentproto.DefaultLimits()); err != nil {
t.Fatal(err)
}
ack := readFrame(t, ctx, transport).GetEventAck()
if ack == nil || ack.IssueUuid != dispatch.IssueUuid || ack.ThroughEventSeq != sequence {
t.Fatalf("event acknowledgement = %v", ack)
}
}
send(&rvboxv1.CommandEvent{IssueUuid: dispatch.IssueUuid, EventSeq: 1, ObservedAt: timestamppb.Now(),
Payload: &rvboxv1.CommandEvent_Lifecycle{Lifecycle: &rvboxv1.LifecycleChange{
Lifecycle: rvboxv1.CommandLifecycle_COMMAND_RUNNING, CommandRevision: 1,
}}}, 1)
data := bytes.Repeat([]byte("large output\n"), 5000)
chunk := &rvboxv1.OutputChunk{Stream: rvboxv1.StreamKind_STREAM_STDOUT, Compression: rvboxv1.Compression_COMPRESSION_NONE,
Data: data, UncompressedSize: uint64(len(data)), CompressedSize: uint64(len(data))}
event := &rvboxv1.CommandEvent{IssueUuid: dispatch.IssueUuid, EventSeq: 2, ObservedAt: timestamppb.Now(),
Payload: &rvboxv1.CommandEvent_Output{Output: chunk}}
send(event, 2)
if err := transport.Close(); err != nil {
t.Fatal(err)
}
// Simulate a lost acknowledgement: replay the same durable bytes in a new
// session. They must be accepted once without rerunning the command.
transport, accepted = fixture.connect(t, ctx, &rvboxv1.ReconcileSnapshot{RetainedCommands: []*rvboxv1.ReconcileCommandState{{
IssueUuid: dispatch.IssueUuid, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_RUNNING,
CommandRevision: 1, LastClientEventSeq: 2, ImmutableRequestSha256: dispatch.ImmutableRequestSha256,
}}})
send(event, 2)
send(&rvboxv1.CommandEvent{IssueUuid: dispatch.IssueUuid, EventSeq: 3, ObservedAt: timestamppb.Now(),
Payload: &rvboxv1.CommandEvent_Lifecycle{Lifecycle: &rvboxv1.LifecycleChange{
Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED, CommandRevision: 1,
}}}, 3)
issue, _ := domain.ParseUUIDv7(dispatch.IssueUuid)
events, err := fixture.persistence.ReadCommandEvents(ctx, issue, 0, 10)
if err != nil || len(events) != 3 {
t.Fatalf("deduplicated events=%d error=%v", len(events), err)
}
}
func TestLargeDispatchAndReconciliation_HP_WS_01(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
fixture := newFrameFixture(t)
snapshot := &rvboxv1.ReconcileSnapshot{}
for i := 0; i < 1100; i++ {
issue, err := domain.NewUUIDv7()
if err != nil {
t.Fatal(err)
}
snapshot.RetainedCommands = append(snapshot.RetainedCommands, &rvboxv1.ReconcileCommandState{
IssueUuid: issue.String(), Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
CommandRevision: 1, Tombstoned: true, ImmutableRequestSha256: make([]byte, 32),
})
}
transport, _ := fixture.connect(t, ctx, snapshot)
command := "echo " + strings.Repeat("x", 64<<10)
if got := fixture.dispatch(t, ctx, transport, command).GetSpec().GetCommandText(); got != command {
t.Fatal("large dispatch text changed")
}
}
type frameFixture struct {
persistence *store.Store
control *control.Service
server *httptest.Server
hello *rvboxv1.ClientHello
}
func newFrameFixture(t *testing.T) *frameFixture {
t.Helper()
persistence, err := store.Open(context.Background(), store.Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = persistence.Close() })
registry := session.NewRegistry()
server := httptest.NewTLSServer(&session.AgentServer{Store: persistence, Registry: registry, Limits: agentproto.DefaultLimits()})
t.Cleanup(server.Close)
controlService, err := control.NewService(control.Options{Store: persistence, WakeClient: registry.Wake})
if err != nil {
t.Fatal(err)
}
return &frameFixture{persistence: persistence, control: controlService, server: server,
hello: &rvboxv1.ClientHello{ClientId: "large-frame-client", ClientInstanceId: fixedIssueID(0x91),
SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1}, DaemonVersion: "frame-test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS,
Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_CMD},
MaxRunningCommands: 2, MaxQueuedCommands: 4, SentAt: timestamppb.Now()}}
}
func (fixture *frameFixture) connect(t *testing.T, ctx context.Context, snapshot *rvboxv1.ReconcileSnapshot) (*agent.WebSocketTransport, agent.Session) {
t.Helper()
transport, err := agent.DialWebSocket(ctx, "wss"+strings.TrimPrefix(fixture.server.URL, "https")+"/v1/agent", fixture.server.Client())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = transport.Close() })
accepted, err := agent.Handshake(ctx, transport, fixture.hello, agentproto.DefaultLimits())
if err != nil {
t.Fatal(err)
}
result, err := agent.Reconcile(ctx, transport, accepted, snapshot, agentproto.DefaultLimits())
if err != nil {
t.Fatal(err)
}
if len(snapshot.RetainedCommands) > 1 && len(result.DiscardLocalTerminalIssueUuids) != len(snapshot.RetainedCommands) {
t.Fatal("large reconciliation result incomplete")
}
return transport, accepted
}
func (fixture *frameFixture) dispatch(t *testing.T, ctx context.Context, transport *agent.WebSocketTransport, text string) *rvboxv1.CommandDispatch {
t.Helper()
if _, err := fixture.control.RunCommand(ctx, &rvboxv1.RunCommandRequest{TargetClientId: fixture.hello.ClientId,
Spec: &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: text}},
}); err != nil {
t.Fatal(err)
}
envelope := readFrame(t, ctx, transport)
dispatch := envelope.GetCommandDispatch()
if dispatch == nil {
t.Fatal("expected command dispatch")
}
ack, err := proto.Marshal(&rvboxv1.AgentEnvelope{SessionId: envelope.SessionId, SessionGeneration: envelope.SessionGeneration,
Payload: &rvboxv1.AgentEnvelope_CommandAccepted{CommandAccepted: &rvboxv1.CommandAccepted{
IssueUuid: dispatch.IssueUuid, CommandRevision: dispatch.CommandRevision, Accepted: true,
}}})
if err != nil {
t.Fatal(err)
}
if err := transport.Write(ctx, ack); err != nil {
t.Fatal(err)
}
return dispatch
}
func readFrame(t *testing.T, ctx context.Context, transport *agent.WebSocketTransport) *rvboxv1.AgentEnvelope {
t.Helper()
encoded, err := transport.Read(ctx)
if err != nil {
t.Fatal(err)
}
envelope, err := agentproto.DecodeEnvelope(encoded, agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
if err != nil {
t.Fatal(err)
}
return envelope
}