fix(transport): align WebSocket read limits with agent protocol
This commit is contained in:
1 parent
ef6e65e6f1
commit
6984a09486
11 files changed
+376
-4
No files matched your search
@@ -168,7 +168,7 @@ func run(configPath string) error {
|
||||
}
|
||||
agent := &session.AgentServer{
|
||||
Store: persistence, Registry: registry, Path: configured.Server.AgentPath,
|
||||
Metrics: health,
|
||||
Metrics: health, Logf: logger.Printf,
|
||||
Limits: agentproto.Limits{
|
||||
MaxEnvelopeBytes: configured.Protocol.MaxAgentEnvelopeBytes,
|
||||
MaxExecutionSpecBytes: configured.Protocol.MaxExecutionSpecBytes,
|
||||
|
||||
+2
-2
@@ -249,7 +249,7 @@ func runClientDaemon(ctx context.Context, configPath string, diagnostics io.Writ
|
||||
if runErr := agent.Run(ctx, agent.RunnerOptions{
|
||||
Store: state,
|
||||
Dial: func(dialContext context.Context) (agent.Transport, error) {
|
||||
return agent.DialWebSocket(dialContext, configured.Client.ServerURL, httpClient)
|
||||
return agent.DialWebSocketWithLimits(dialContext, configured.Client.ServerURL, httpClient, limits)
|
||||
},
|
||||
Hello: hello, Limits: limits,
|
||||
Backoff: agent.BackoffOptions{Initial: configured.Network.ReconnectInitial, Maximum: configured.Network.ReconnectMax, StableReset: configured.Network.StableSessionReset},
|
||||
@@ -389,7 +389,7 @@ func clientHello(configured *config.Client, instance domain.UUID) *rvboxv1.Clien
|
||||
}
|
||||
return &rvboxv1.ClientHello{
|
||||
ClientId: clientID, SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1, MinMinor: 0, MaxMinor: 0},
|
||||
DaemonVersion: "v1", Platform: platform, Architecture: runtime.GOARCH, DaemonCwd: configured.Client.DaemonCWD,
|
||||
DaemonVersion: buildVersion, Platform: platform, Architecture: runtime.GOARCH, DaemonCwd: configured.Client.DaemonCWD,
|
||||
SupportedShells: shells, ClientInstanceId: instance.String(), MaxRunningCommands: configured.Client.MaxRunningCommands,
|
||||
MaxQueuedCommands: configured.Client.MaxQueuedCommands, SentAt: timestamppb.New(time.Now().UTC()),
|
||||
}
|
||||
|
||||
@@ -126,6 +126,16 @@ exact ignored `dist/` directory after preserving any evidence.
|
||||
|
||||
## Common incidents
|
||||
|
||||
- **Reconnect loop after a command produces substantial output:** inspect
|
||||
`C:\ProgramData\RVBox\service-startup.log` on Windows and the server log for
|
||||
`StatusMessageTooBig` / `read limited at 32769 bytes`. Early v1 transports
|
||||
inherited a 32 KiB WebSocket ceiling despite permitting 64 KiB chunks and
|
||||
1 MiB envelopes. Both transports now set the configured envelope ceiling
|
||||
before reading any messages. Upgrade the server first to drain pending
|
||||
client output, then the client for large dispatch/reconciliation responses.
|
||||
Preserve spool/history: rebooting cannot remove a replayable oversized event.
|
||||
New server logs include bounded read/reconciliation errors and an
|
||||
`agent_message_too_big` counter; new clients advertise the actual build version.
|
||||
- **No client / stale session:** verify nginx has WebSocket `101` entries for
|
||||
`/v1/agent`, server readiness is true, and the Windows service is running.
|
||||
Use `rvc stat CLIENT_ID`; do not restart the client merely to clear history.
|
||||
|
||||
@@ -3,9 +3,12 @@ package agent
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
"github.com/rvbox/rvbox/internal/agentproto"
|
||||
)
|
||||
|
||||
var ErrUnexpectedMessage = errors.New("agent transport received a non-binary WebSocket message")
|
||||
@@ -13,10 +16,25 @@ var ErrUnexpectedMessage = errors.New("agent transport received a non-binary Web
|
||||
type WebSocketTransport struct{ connection *websocket.Conn }
|
||||
|
||||
func DialWebSocket(ctx context.Context, address string, client *http.Client) (*WebSocketTransport, error) {
|
||||
return DialWebSocketWithLimits(ctx, address, client, agentproto.DefaultLimits())
|
||||
}
|
||||
|
||||
// DialWebSocketWithLimits applies the same message ceiling as envelope
|
||||
// validation. The library's 32 KiB default is smaller than ordinary RVBox
|
||||
// chunks, dispatches, and reconciliation history.
|
||||
func DialWebSocketWithLimits(ctx context.Context, address string, client *http.Client, limits agentproto.Limits) (*WebSocketTransport, error) {
|
||||
readLimit := limits.MaxEnvelopeBytes
|
||||
if readLimit == 0 {
|
||||
readLimit = agentproto.DefaultLimits().MaxEnvelopeBytes
|
||||
}
|
||||
if readLimit > math.MaxInt64-1 {
|
||||
return nil, fmt.Errorf("invalid WebSocket envelope limit: %d", readLimit)
|
||||
}
|
||||
connection, _, err := websocket.Dial(ctx, address, &websocket.DialOptions{HTTPClient: client, CompressionMode: websocket.CompressionDisabled})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
connection.SetReadLimit(int64(readLimit))
|
||||
return &WebSocketTransport{connection: connection}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
"github.com/rvbox/rvbox/internal/agentproto"
|
||||
)
|
||||
|
||||
func TestWebSocketConfiguredReadCeiling_BH_WS_01(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, scenario := range []struct {
|
||||
name string
|
||||
limit uint64
|
||||
bytes int
|
||||
wantLarge bool
|
||||
}{
|
||||
{name: "above library default", bytes: (32 << 10) + 1},
|
||||
{name: "exact default ceiling", bytes: 1 << 20},
|
||||
{name: "over default ceiling", bytes: (1 << 20) + 1, wantLarge: true},
|
||||
{name: "exact configured ceiling", limit: 2048, bytes: 2048},
|
||||
{name: "over configured ceiling", limit: 2048, bytes: 2049, wantLarge: true},
|
||||
} {
|
||||
t.Run(scenario.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
payload := bytes.Repeat([]byte{0x51}, scenario.bytes)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
connection, err := websocket.Accept(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer connection.CloseNow()
|
||||
if err := connection.Write(ctx, websocket.MessageBinary, payload); err != nil {
|
||||
return
|
||||
}
|
||||
_, _, _ = connection.Read(ctx) // Complete the peer's close handshake.
|
||||
}))
|
||||
defer server.Close()
|
||||
transport, err := DialWebSocketWithLimits(ctx, "ws"+strings.TrimPrefix(server.URL, "http"), nil,
|
||||
agentproto.Limits{MaxEnvelopeBytes: scenario.limit})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer transport.Close()
|
||||
got, err := transport.Read(ctx)
|
||||
if scenario.wantLarge {
|
||||
if !errors.Is(err, websocket.ErrMessageTooBig) {
|
||||
t.Fatalf("oversize read = %v", err)
|
||||
}
|
||||
} else if err != nil || !bytes.Equal(got, payload) {
|
||||
t.Fatalf("read bytes=%d error=%v", len(got), err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := DialWebSocketWithLimits(context.Background(), "invalid", nil, agentproto.Limits{MaxEnvelopeBytes: math.MaxUint64}); err == nil {
|
||||
t.Fatal("overflowing read ceiling accepted")
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"sort"
|
||||
"sync"
|
||||
@@ -49,6 +50,8 @@ type AgentServer struct {
|
||||
// Metrics is optional so transport tests and embeddings do not need an
|
||||
// observability endpoint. Every emitted name is a static bounded value.
|
||||
Metrics *observability.Health
|
||||
// Logf receives bounded transport diagnostics without command payloads.
|
||||
Logf func(string, ...any)
|
||||
}
|
||||
|
||||
func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http.Request) {
|
||||
@@ -69,6 +72,10 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
|
||||
http.Error(response, "agent session registry is unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
if server.limits().MaxEnvelopeBytes > math.MaxInt64-1 {
|
||||
http.Error(response, "invalid agent envelope limit", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
started := time.Now()
|
||||
heartbeat := newSynchronizedHeartbeat(server.heartbeatIdle(), server.livenessTimeout(), 0)
|
||||
connection, err := websocket.Accept(response, request, &websocket.AcceptOptions{
|
||||
@@ -88,6 +95,7 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
connection.SetReadLimit(int64(server.limits().MaxEnvelopeBytes))
|
||||
server.incMetric("agent_connection")
|
||||
defer connection.CloseNow()
|
||||
server.serveConnection(request.Context(), connection, heartbeat, started)
|
||||
@@ -96,6 +104,10 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
|
||||
func (server *AgentServer) serveConnection(parent context.Context, connection *websocket.Conn, heartbeat *synchronizedHeartbeat, started time.Time) {
|
||||
messageType, payload, err := connection.Read(parent)
|
||||
if err != nil {
|
||||
if errors.Is(err, websocket.ErrMessageTooBig) {
|
||||
server.incMetric("agent_message_too_big")
|
||||
}
|
||||
server.logSessionError("", 0, "hello-read", err)
|
||||
return
|
||||
}
|
||||
if messageType != websocket.MessageBinary {
|
||||
@@ -143,6 +155,7 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
|
||||
})
|
||||
server.observeMetric("sqlite_write_latency", time.Since(registrationStarted))
|
||||
if err != nil {
|
||||
server.logSessionError(hello.GetClientId(), 0, "register", err)
|
||||
server.incMetric("client_registration_failure")
|
||||
if errors.Is(err, store.ErrTakeoverRequired) {
|
||||
server.incMetric("client_takeover_required")
|
||||
@@ -242,6 +255,10 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
|
||||
for {
|
||||
messageType, payload, err = connection.Read(sessionContext)
|
||||
if err != nil {
|
||||
if errors.Is(err, websocket.ErrMessageTooBig) {
|
||||
server.incMetric("agent_message_too_big")
|
||||
}
|
||||
server.logSessionError(hello.GetClientId(), registration.Generation, "read", err)
|
||||
return
|
||||
}
|
||||
// Elapsed time is monotonic; peer-provided wall timestamps are never used
|
||||
@@ -267,6 +284,7 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
|
||||
if snapshot := envelope.GetReconcileSnapshot(); snapshot != nil {
|
||||
result, reconcileErr := server.Store.ReconcileClientSnapshotForSessionAt(sessionContext, hello.GetClientId(), registration.Generation, snapshot, reconcileBoundary)
|
||||
if reconcileErr != nil {
|
||||
server.logSessionError(hello.GetClientId(), registration.Generation, "reconcile", reconcileErr)
|
||||
server.close(connection, websocket.StatusPolicyViolation, "reconciliation failed")
|
||||
return
|
||||
}
|
||||
@@ -348,6 +366,7 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
|
||||
appended, eventErr := server.Store.AppendCommandEvent(sessionContext, appendEvent)
|
||||
if eventErr != nil {
|
||||
server.incMetric("command_event_rejected")
|
||||
server.logSessionError(hello.GetClientId(), registration.Generation, "command-event", eventErr)
|
||||
server.close(connection, websocket.StatusPolicyViolation, "command event was not accepted")
|
||||
return
|
||||
}
|
||||
@@ -810,6 +829,17 @@ func (server *AgentServer) close(connection *websocket.Conn, status websocket.St
|
||||
_ = connection.Close(status, reason)
|
||||
}
|
||||
|
||||
func (server *AgentServer) logSessionError(clientID string, generation uint64, stage string, err error) {
|
||||
if server.Logf == nil || err == nil {
|
||||
return
|
||||
}
|
||||
detail := err.Error()
|
||||
if len(detail) > 1024 {
|
||||
detail = detail[:1024] + "..."
|
||||
}
|
||||
server.Logf("agent session client=%q generation=%d stage=%s error=%q", clientID, generation, stage, detail)
|
||||
}
|
||||
|
||||
func (server *AgentServer) agentPath() string {
|
||||
if server.Path == "" {
|
||||
return defaultAgentPath
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
@@ -225,6 +226,37 @@ func TestAgentServerRejectsHostileWireInputs_BH_SES_04(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentServerEnforcesConfiguredReadCeiling_BH_WS_02(t *testing.T) {
|
||||
t.Parallel()
|
||||
server, edge, cleanup := newTestAgentServerWithStore(t)
|
||||
defer cleanup()
|
||||
edge.Limits = agentproto.DefaultLimits()
|
||||
edge.Limits.MaxEnvelopeBytes = 2048
|
||||
logs := make(chan string, 1)
|
||||
edge.Logf = func(format string, args ...any) { logs <- fmt.Sprintf(format, args...) }
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
connection, _, err := websocket.Dial(ctx, websocketURL(server.URL), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer connection.CloseNow()
|
||||
if err := connection.Write(ctx, websocket.MessageBinary, bytes.Repeat([]byte{0x51}, 2049)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := connection.Read(ctx); websocket.CloseStatus(err) != websocket.StatusMessageTooBig {
|
||||
t.Fatalf("oversize close = %v", err)
|
||||
}
|
||||
select {
|
||||
case diagnostic := <-logs:
|
||||
if !strings.Contains(diagnostic, "stage=hello-read") || !strings.Contains(diagnostic, "2049") {
|
||||
t.Fatalf("missing bounded transport diagnostic: %s", diagnostic)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("oversize failure was not logged")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWireEventAppendCarriesClientBinding_HP_EVENT_01(t *testing.T) {
|
||||
event := &rvboxv1.CommandEvent{IssueUuid: "019c46f1-1d02-7000-8000-000000000096", EventSeq: 1, ObservedAt: timestamppb.Now(), Payload: &rvboxv1.CommandEvent_Lifecycle{Lifecycle: &rvboxv1.LifecycleChange{Lifecycle: rvboxv1.CommandLifecycle_COMMAND_RUNNING, CommandRevision: 1}}}
|
||||
digest, err := agentproto.CommandEventDigest(event)
|
||||
|
||||
@@ -8,7 +8,7 @@ description: "Operate and troubleshoot RVBox deployments through rvc and the Con
|
||||
Use the deployment already present. Discover its addresses, paths, accounts,
|
||||
versions, and supported capabilities; never substitute a remembered test host,
|
||||
container name, credential, client ID, or filesystem layout. This reference
|
||||
matches RVBox v1 source reviewed on 2026-10-27: Linux server and Windows client.
|
||||
matches RVBox v1 source reviewed on 2026-11-09: Linux server and Windows client.
|
||||
Unix-like client implementation is outside that release.
|
||||
|
||||
## Establish the control route
|
||||
|
||||
@@ -19,6 +19,7 @@ initial evidence; do not restart services or resubmit work merely to inspect it.
|
||||
| Socket permission denied | Run as the existing socket owner through the deployment route; inspect directory traversal and mount permissions. Do not chmod the control socket world-accessible. |
|
||||
| Client missing or NOT_FOUND | List all client pages; verify the exact registered ID and server instance. A hostname assumption or wrong Compose project can select different state. Retention can also remove command history. |
|
||||
| Disconnected / OFFLINE / UNAVAILABLE | Separate server recovery from client connectivity. Inspect SCM, WSS hostname/path, proxy WebSocket upgrade, certificate/time/CA, DNS, reconnect logs, and pending instance conflicts. Missing optional HTTP health is not proof of a stopped daemon. |
|
||||
| `StatusMessageTooBig` / `read limited at 32769 bytes` after output or during reconnect | Early v1 WebSocket readers kept the library's 32 KiB default despite allowing larger protocol envelopes. Upgrade the server first to accept pending output, then the client for large incoming dispatch/history. Preserve spool/history; rebooting replays the same pending message. Corrected readers use the configured envelope ceiling, normally 1 MiB. |
|
||||
| Repeated `signal command ...: command is not running` | An older client disconnects on a late signal instead of persisting its rejection. Upgrade the client while retaining config/spool/history; corrected builds acknowledge the signal without disconnecting. Do not wipe state or reissue the original command. |
|
||||
| QUEUED persists | Check connection, supported shell, queue/running capacity, acceptance events, and queue expiry. Queuing does not promise execution, and queue TTL is not a runtime timeout. |
|
||||
| CLI deadline / broken stream | Work may still run. Query the captured issue and resume history via the API. For uncertain admission, retry only with the same request ID and payload. |
|
||||
@@ -38,6 +39,16 @@ immediate repeated calls will succeed. Do not parse error text into invented
|
||||
stable exit-code classes: the current CLI normally exits 1 for its own errors
|
||||
and exposes remote lifecycle separately.
|
||||
|
||||
On Windows, SCM startup and reconnect diagnostics are written to
|
||||
`%ProgramData%\RVBox\service-startup.log`, independently of optional
|
||||
observability logging. Inspect recent entries around the failure before
|
||||
restarting. The tray's log action may select the configured runtime log instead;
|
||||
an empty/missing runtime log does not mean the service recorded no diagnostics.
|
||||
Corrected servers log bounded WebSocket read/reconciliation errors without
|
||||
command payloads, and count `agent_message_too_big` failures. Earlier clients
|
||||
reported the placeholder daemon version `v1`; use the installed executable's
|
||||
`--version` or PE product version when the server cannot identify the release.
|
||||
|
||||
## Windows identity, GUI, and shells
|
||||
|
||||
`--elevated` is the only execution-context request toggle. Selection and
|
||||
|
||||
@@ -1157,6 +1157,30 @@ layer = "integration"
|
||||
status = "implemented"
|
||||
tests = ["test/integration/store/store_integration_test.go:TestIncidentDirtyResolutionIdempotencyAndRecurrence_HP_STORE_11"]
|
||||
|
||||
[[requirements]]
|
||||
id = "REC-WS-01"
|
||||
layer = "integration"
|
||||
status = "implemented"
|
||||
tests = ["test/integration/clientagent/websocket_limits_integration_test.go:TestLargeOutputReplaysAfterReconnect_REC_WS_01"]
|
||||
|
||||
[[requirements]]
|
||||
id = "HP-WS-01"
|
||||
layer = "integration"
|
||||
status = "implemented"
|
||||
tests = ["test/integration/clientagent/websocket_limits_integration_test.go:TestLargeDispatchAndReconciliation_HP_WS_01"]
|
||||
|
||||
[[requirements]]
|
||||
id = "BH-WS-01"
|
||||
layer = "unit"
|
||||
status = "implemented"
|
||||
tests = ["internal/client/agent/websocket_test.go:TestWebSocketConfiguredReadCeiling_BH_WS_01"]
|
||||
|
||||
[[requirements]]
|
||||
id = "BH-WS-02"
|
||||
layer = "unit"
|
||||
status = "implemented"
|
||||
tests = ["internal/server/session/agent_server_test.go:TestAgentServerEnforcesConfiguredReadCeiling_BH_WS_02"]
|
||||
|
||||
[[requirements]]
|
||||
id = "REC-STORE-03"
|
||||
layer = "unit"
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
package clientagent_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||
"github.com/rvbox/rvbox/internal/agentproto"
|
||||
"github.com/rvbox/rvbox/internal/client/agent"
|
||||
"github.com/rvbox/rvbox/internal/domain"
|
||||
"github.com/rvbox/rvbox/internal/server/control"
|
||||
"github.com/rvbox/rvbox/internal/server/session"
|
||||
"github.com/rvbox/rvbox/internal/server/store"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
func TestLargeOutputReplaysAfterReconnect_REC_WS_01(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
fixture := newFrameFixture(t)
|
||||
transport, accepted := fixture.connect(t, ctx, &rvboxv1.ReconcileSnapshot{})
|
||||
dispatch := fixture.dispatch(t, ctx, transport, "echo fixture")
|
||||
send := func(event *rvboxv1.CommandEvent, sequence uint64) {
|
||||
t.Helper()
|
||||
if err := agent.SendCommandEvent(ctx, transport, accepted, event, agentproto.DefaultLimits()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ack := readFrame(t, ctx, transport).GetEventAck()
|
||||
if ack == nil || ack.IssueUuid != dispatch.IssueUuid || ack.ThroughEventSeq != sequence {
|
||||
t.Fatalf("event acknowledgement = %v", ack)
|
||||
}
|
||||
}
|
||||
send(&rvboxv1.CommandEvent{IssueUuid: dispatch.IssueUuid, EventSeq: 1, ObservedAt: timestamppb.Now(),
|
||||
Payload: &rvboxv1.CommandEvent_Lifecycle{Lifecycle: &rvboxv1.LifecycleChange{
|
||||
Lifecycle: rvboxv1.CommandLifecycle_COMMAND_RUNNING, CommandRevision: 1,
|
||||
}}}, 1)
|
||||
data := bytes.Repeat([]byte("large output\n"), 5000)
|
||||
chunk := &rvboxv1.OutputChunk{Stream: rvboxv1.StreamKind_STREAM_STDOUT, Compression: rvboxv1.Compression_COMPRESSION_NONE,
|
||||
Data: data, UncompressedSize: uint64(len(data)), CompressedSize: uint64(len(data))}
|
||||
event := &rvboxv1.CommandEvent{IssueUuid: dispatch.IssueUuid, EventSeq: 2, ObservedAt: timestamppb.Now(),
|
||||
Payload: &rvboxv1.CommandEvent_Output{Output: chunk}}
|
||||
send(event, 2)
|
||||
if err := transport.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Simulate a lost acknowledgement: replay the same durable bytes in a new
|
||||
// session. They must be accepted once without rerunning the command.
|
||||
transport, accepted = fixture.connect(t, ctx, &rvboxv1.ReconcileSnapshot{RetainedCommands: []*rvboxv1.ReconcileCommandState{{
|
||||
IssueUuid: dispatch.IssueUuid, Lifecycle: rvboxv1.CommandLifecycle_COMMAND_RUNNING,
|
||||
CommandRevision: 1, LastClientEventSeq: 2, ImmutableRequestSha256: dispatch.ImmutableRequestSha256,
|
||||
}}})
|
||||
send(event, 2)
|
||||
send(&rvboxv1.CommandEvent{IssueUuid: dispatch.IssueUuid, EventSeq: 3, ObservedAt: timestamppb.Now(),
|
||||
Payload: &rvboxv1.CommandEvent_Lifecycle{Lifecycle: &rvboxv1.LifecycleChange{
|
||||
Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED, CommandRevision: 1,
|
||||
}}}, 3)
|
||||
issue, _ := domain.ParseUUIDv7(dispatch.IssueUuid)
|
||||
events, err := fixture.persistence.ReadCommandEvents(ctx, issue, 0, 10)
|
||||
if err != nil || len(events) != 3 {
|
||||
t.Fatalf("deduplicated events=%d error=%v", len(events), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLargeDispatchAndReconciliation_HP_WS_01(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
fixture := newFrameFixture(t)
|
||||
snapshot := &rvboxv1.ReconcileSnapshot{}
|
||||
for i := 0; i < 1100; i++ {
|
||||
issue, err := domain.NewUUIDv7()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snapshot.RetainedCommands = append(snapshot.RetainedCommands, &rvboxv1.ReconcileCommandState{
|
||||
IssueUuid: issue.String(), Lifecycle: rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED,
|
||||
CommandRevision: 1, Tombstoned: true, ImmutableRequestSha256: make([]byte, 32),
|
||||
})
|
||||
}
|
||||
transport, _ := fixture.connect(t, ctx, snapshot)
|
||||
command := "echo " + strings.Repeat("x", 64<<10)
|
||||
if got := fixture.dispatch(t, ctx, transport, command).GetSpec().GetCommandText(); got != command {
|
||||
t.Fatal("large dispatch text changed")
|
||||
}
|
||||
}
|
||||
|
||||
type frameFixture struct {
|
||||
persistence *store.Store
|
||||
control *control.Service
|
||||
server *httptest.Server
|
||||
hello *rvboxv1.ClientHello
|
||||
}
|
||||
|
||||
func newFrameFixture(t *testing.T) *frameFixture {
|
||||
t.Helper()
|
||||
persistence, err := store.Open(context.Background(), store.Options{DataDir: filepath.Join(t.TempDir(), "state"), BusyTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = persistence.Close() })
|
||||
registry := session.NewRegistry()
|
||||
server := httptest.NewTLSServer(&session.AgentServer{Store: persistence, Registry: registry, Limits: agentproto.DefaultLimits()})
|
||||
t.Cleanup(server.Close)
|
||||
controlService, err := control.NewService(control.Options{Store: persistence, WakeClient: registry.Wake})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &frameFixture{persistence: persistence, control: controlService, server: server,
|
||||
hello: &rvboxv1.ClientHello{ClientId: "large-frame-client", ClientInstanceId: fixedIssueID(0x91),
|
||||
SupportedProtocol: &rvboxv1.ProtocolRange{Major: 1}, DaemonVersion: "frame-test", Platform: rvboxv1.Platform_PLATFORM_WINDOWS,
|
||||
Architecture: "amd64", DaemonCwd: `C:\`, SupportedShells: []rvboxv1.ShellType{rvboxv1.ShellType_SHELL_CMD},
|
||||
MaxRunningCommands: 2, MaxQueuedCommands: 4, SentAt: timestamppb.Now()}}
|
||||
}
|
||||
|
||||
func (fixture *frameFixture) connect(t *testing.T, ctx context.Context, snapshot *rvboxv1.ReconcileSnapshot) (*agent.WebSocketTransport, agent.Session) {
|
||||
t.Helper()
|
||||
transport, err := agent.DialWebSocket(ctx, "wss"+strings.TrimPrefix(fixture.server.URL, "https")+"/v1/agent", fixture.server.Client())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = transport.Close() })
|
||||
accepted, err := agent.Handshake(ctx, transport, fixture.hello, agentproto.DefaultLimits())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := agent.Reconcile(ctx, transport, accepted, snapshot, agentproto.DefaultLimits())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(snapshot.RetainedCommands) > 1 && len(result.DiscardLocalTerminalIssueUuids) != len(snapshot.RetainedCommands) {
|
||||
t.Fatal("large reconciliation result incomplete")
|
||||
}
|
||||
return transport, accepted
|
||||
}
|
||||
|
||||
func (fixture *frameFixture) dispatch(t *testing.T, ctx context.Context, transport *agent.WebSocketTransport, text string) *rvboxv1.CommandDispatch {
|
||||
t.Helper()
|
||||
if _, err := fixture.control.RunCommand(ctx, &rvboxv1.RunCommandRequest{TargetClientId: fixture.hello.ClientId,
|
||||
Spec: &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: text}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
envelope := readFrame(t, ctx, transport)
|
||||
dispatch := envelope.GetCommandDispatch()
|
||||
if dispatch == nil {
|
||||
t.Fatal("expected command dispatch")
|
||||
}
|
||||
ack, err := proto.Marshal(&rvboxv1.AgentEnvelope{SessionId: envelope.SessionId, SessionGeneration: envelope.SessionGeneration,
|
||||
Payload: &rvboxv1.AgentEnvelope_CommandAccepted{CommandAccepted: &rvboxv1.CommandAccepted{
|
||||
IssueUuid: dispatch.IssueUuid, CommandRevision: dispatch.CommandRevision, Accepted: true,
|
||||
}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := transport.Write(ctx, ack); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dispatch
|
||||
}
|
||||
|
||||
func readFrame(t *testing.T, ctx context.Context, transport *agent.WebSocketTransport) *rvboxv1.AgentEnvelope {
|
||||
t.Helper()
|
||||
encoded, err := transport.Read(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
envelope, err := agentproto.DecodeEnvelope(encoded, agentproto.DefaultLimits(), rvboxv1.Platform_PLATFORM_WINDOWS)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return envelope
|
||||
}
|
||||
Reference in new issue
Block a user