feat(transport): negotiate bounded automatic message chunking

This commit is contained in:
cabbage committed 2026-11-09 08:50:51 +00:00
1 parent 474afb3484
commit 8acffac985
20 files changed
+1332 -38

No files matched your search

+136
View File
@@ -0,0 +1,136 @@
package agentwire
import (
"context"
"errors"
"io"
"math"
"sync"
"github.com/coder/websocket"
)
type incoming struct {
kind websocket.MessageType
payload []byte
err error
}
// Conn owns one bounded read pump so WebSocket Ping/Pong is processed even
// while a caller is writing a large hello/reconciliation message. Application
// readers see only complete logical messages. Legacy peers retain raw protobuf.
type Conn struct {
*websocket.Conn
maximum uint64
chunked bool
cancel context.CancelFunc
read chan incoming
writeMu sync.Mutex
}
func Wrap(raw *websocket.Conn, maximum uint64, activity func()) (*Conn, error) {
if raw == nil || maximum == 0 || maximum > math.MaxInt64-1 {
return nil, ErrMessageTooBig
}
chunked := raw.Subprotocol() == Subprotocol
if chunked {
raw.SetReadLimit(PacketBytes)
} else {
raw.SetReadLimit(int64(maximum))
}
ctx, cancel := context.WithCancel(context.Background())
connection := &Conn{Conn: raw, maximum: maximum, chunked: chunked, cancel: cancel, read: make(chan incoming, 1)}
go connection.readLoop(ctx, activity)
return connection, nil
}
func (connection *Conn) readLoop(ctx context.Context, activity func()) {
defer close(connection.read)
decoder := Decoder{Maximum: connection.maximum}
for {
kind, payload, err := connection.Conn.Read(ctx)
if err == nil && kind == websocket.MessageBinary && connection.chunked {
payload, err = decoder.Push(payload)
if err != nil {
code := websocket.StatusPolicyViolation
if errors.Is(err, ErrMessageTooBig) {
code = websocket.StatusMessageTooBig
}
_ = connection.Conn.Close(code, err.Error())
}
}
if err != nil {
_ = connection.Conn.CloseNow()
select {
case connection.read <- incoming{err: err}:
case <-ctx.Done():
}
return
}
if activity != nil {
activity()
}
if connection.chunked && kind == websocket.MessageBinary && payload == nil { // Incomplete fragment.
continue
}
select {
case connection.read <- incoming{kind: kind, payload: payload}:
case <-ctx.Done():
return
}
}
}
func (connection *Conn) Read(ctx context.Context) (websocket.MessageType, []byte, error) {
select {
case frame, ok := <-connection.read:
if !ok {
return 0, nil, io.EOF
}
return frame.kind, frame.payload, frame.err
case <-ctx.Done():
return 0, nil, ctx.Err()
}
}
// Encode is used by the server's sole queue writer so it can yield to control
// messages and check heartbeats between data packets. Each lane stays ordered.
func (connection *Conn) Encode(payload []byte, priority bool) (*Encoder, error) {
return NewEncoder(payload, connection.maximum, connection.chunked, priority)
}
func (connection *Conn) WritePacket(ctx context.Context, packet []byte) error {
return connection.Conn.Write(ctx, websocket.MessageBinary, packet)
}
func (connection *Conn) Write(ctx context.Context, kind websocket.MessageType, payload []byte) error {
if kind != websocket.MessageBinary {
return errors.New("agent transport writes must be binary")
}
encoder, err := connection.Encode(payload, false)
if err != nil {
return err
}
connection.writeMu.Lock()
defer connection.writeMu.Unlock()
for {
packet, last := encoder.Next()
if err := connection.WritePacket(ctx, packet); err != nil {
_ = connection.CloseNow()
return err
}
if last {
return nil
}
}
}
func (connection *Conn) Close(code websocket.StatusCode, reason string) error {
defer connection.cancel()
return connection.Conn.Close(code, reason)
}
func (connection *Conn) CloseNow() error {
connection.cancel()
return connection.Conn.CloseNow()
}
+216
View File
@@ -0,0 +1,216 @@
package agentwire
import (
"bytes"
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/coder/websocket"
)
func TestNegotiationFallbackAndBackgroundPong_HP_CHUNK_02(t *testing.T) {
t.Parallel()
for _, scenario := range []struct {
name string
server, client bool
}{
{"both", true, true}, {"old server", false, true}, {"old client", true, false},
} {
t.Run(scenario.name, func(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
payload := bytes.Repeat([]byte{0x53}, 64<<10)
pong := make(chan error, 1)
var activity atomic.Uint32
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
options := &websocket.AcceptOptions{}
if scenario.server {
options.Subprotocols = []string{Subprotocol}
}
raw, err := websocket.Accept(w, r, options)
if err != nil {
pong <- err
return
}
connection, err := Wrap(raw, 1<<20, func() { activity.Add(1) })
if err != nil {
_ = raw.CloseNow()
pong <- err
return
}
defer connection.CloseNow()
_, got, err := connection.Read(ctx)
if err == nil && !bytes.Equal(got, payload) {
err = errors.New("payload changed")
}
if err == nil {
err = connection.Write(ctx, websocket.MessageBinary, got)
}
if err == nil {
err = connection.Ping(ctx)
}
pong <- err
_, _, _ = connection.Read(ctx)
}))
defer server.Close()
options := &websocket.DialOptions{HTTPClient: server.Client()}
if scenario.client {
options.Subprotocols = []string{Subprotocol}
}
raw, _, err := websocket.Dial(ctx, "wss"+strings.TrimPrefix(server.URL, "https"), options)
if err != nil {
t.Fatal(err)
}
connection, err := Wrap(raw, 1<<20, nil)
if err != nil {
_ = raw.CloseNow()
t.Fatal(err)
}
defer connection.Close(websocket.StatusNormalClosure, "")
if connection.chunked != (scenario.client && scenario.server) {
t.Fatal("wrong negotiated mode")
}
if err := connection.Write(ctx, websocket.MessageBinary, payload); err != nil {
t.Fatal(err)
}
// Do not call Read yet: the read pump must still service Ping while
// the echoed complete message waits for the application.
select {
case err := <-pong:
if err != nil {
t.Fatal(err)
}
case <-ctx.Done():
t.Fatal("Ping blocked behind application read")
}
_, got, err := connection.Read(ctx)
if err != nil || !bytes.Equal(got, payload) {
t.Fatalf("echo bytes=%d error=%v", len(got), err)
}
if connection.chunked && activity.Load() <= 1 {
t.Fatal("large message was not chunked")
}
})
}
}
func TestConcurrentLogicalWrites_RACE_CHUNK_01(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)
defer cancel()
result := make(chan error, 1)
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
raw, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{Subprotocol}})
if err != nil {
result <- err
return
}
connection, err := Wrap(raw, 1<<20, nil)
if err != nil {
_ = raw.CloseNow()
result <- err
return
}
defer connection.CloseNow()
seen := map[byte]bool{}
for len(seen) < 8 {
_, payload, err := connection.Read(ctx)
if err != nil {
result <- err
return
}
if len(payload) != 128<<10 || seen[payload[0]] || !bytes.Equal(payload, bytes.Repeat(payload[:1], len(payload))) {
result <- errors.New("concurrent messages interleaved or duplicated")
return
}
seen[payload[0]] = true
}
result <- nil
}))
defer server.Close()
raw, _, err := websocket.Dial(ctx, "wss"+strings.TrimPrefix(server.URL, "https"), &websocket.DialOptions{HTTPClient: server.Client(), Subprotocols: []string{Subprotocol}})
if err != nil {
t.Fatal(err)
}
connection, err := Wrap(raw, 1<<20, nil)
if err != nil {
t.Fatal(err)
}
defer connection.CloseNow()
var workers sync.WaitGroup
for i := byte(0); i < 8; i++ {
workers.Go(func() {
if err := connection.Write(ctx, websocket.MessageBinary, bytes.Repeat([]byte{i}, 128<<10)); err != nil {
t.Error(err)
}
})
}
workers.Wait()
select {
case err := <-result:
if err != nil {
t.Fatal(err)
}
case <-ctx.Done():
t.Fatal("concurrent write test stalled")
}
}
func TestInterruptedPacketNeverDelivered_REC_CHUNK_02(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
result := make(chan incoming, 2)
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
raw, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{Subprotocol}})
if err != nil {
result <- incoming{err: err}
return
}
connection, err := Wrap(raw, 1<<20, nil)
if err != nil {
_ = raw.CloseNow()
result <- incoming{err: err}
return
}
defer connection.CloseNow()
_, payload, err := connection.Read(ctx)
result <- incoming{payload: payload, err: err}
}))
defer server.Close()
connect := func() *websocket.Conn {
t.Helper()
connection, _, err := websocket.Dial(ctx, "wss"+strings.TrimPrefix(server.URL, "https"), &websocket.DialOptions{HTTPClient: server.Client(), Subprotocols: []string{Subprotocol}})
if err != nil {
t.Fatal(err)
}
return connection
}
payload := bytes.Repeat([]byte{0x55}, 64<<10)
first := connect()
if err := first.Write(ctx, websocket.MessageBinary, packets(t, payload, false)[0]); err != nil {
t.Fatal(err)
}
_ = first.CloseNow()
if got := <-result; got.err == nil || got.payload != nil {
t.Fatal("partial message delivered")
}
second := connect()
defer second.CloseNow()
for _, packet := range packets(t, payload, false) {
if err := second.Write(ctx, websocket.MessageBinary, packet); err != nil {
t.Fatal(err)
}
}
if got := <-result; got.err != nil || !bytes.Equal(got.payload, payload) {
t.Fatalf("replay: %v", got.err)
}
}
+177
View File
@@ -0,0 +1,177 @@
package agentwire
import (
"bytes"
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/coder/websocket"
)
func TestLogicalSendCeilingBeforeAnyPacket_BH_CHUNK_02(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
result := make(chan incoming, 1)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
raw, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{Subprotocol}})
if err != nil {
result <- incoming{err: err}
return
}
connection, err := Wrap(raw, 2048, nil)
if err != nil {
_ = raw.CloseNow()
result <- incoming{err: err}
return
}
defer connection.CloseNow()
_, payload, err := connection.Read(ctx)
result <- incoming{payload: payload, err: err}
}))
defer server.Close()
raw, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http"), &websocket.DialOptions{Subprotocols: []string{Subprotocol}})
if err != nil {
t.Fatal(err)
}
connection, err := Wrap(raw, 2048, nil)
if err != nil {
t.Fatal(err)
}
defer connection.CloseNow()
if err := connection.Write(ctx, websocket.MessageBinary, make([]byte, 2049)); !errors.Is(err, ErrMessageTooBig) {
t.Fatal("logical send limit bypassed")
}
if err := connection.Write(ctx, websocket.MessageBinary, []byte("still usable")); err != nil {
t.Fatal(err)
}
select {
case frame := <-result:
if frame.err != nil || !bytes.Equal(frame.payload, []byte("still usable")) {
t.Fatal("oversized prefix was sent or connection was poisoned")
}
case <-ctx.Done():
t.Fatal("receiver did not receive next valid message")
}
}
func TestCloseStopsBlockedReadPump_RACE_CHUNK_03(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
sent := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
connection, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{Subprotocol}})
if err != nil {
return
}
defer connection.CloseNow()
for i := 0; i < 3; i++ {
if err := connection.Write(ctx, websocket.MessageBinary, []byte{0, 0x51}); err != nil {
return
}
}
close(sent)
_, _, _ = connection.Read(ctx)
}))
defer server.Close()
raw, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http"), &websocket.DialOptions{Subprotocols: []string{Subprotocol}})
if err != nil {
t.Fatal(err)
}
connection, err := Wrap(raw, 1<<20, nil)
if err != nil {
t.Fatal(err)
}
defer connection.CloseNow()
select {
case <-sent:
case <-ctx.Done():
t.Fatal("peer did not fill delivery queue")
}
for len(connection.read) == 0 {
select {
case <-time.After(time.Millisecond):
case <-ctx.Done():
t.Fatal("delivery queue never filled")
}
}
if err := connection.Close(websocket.StatusNormalClosure, ""); err != nil {
t.Fatal(err)
}
for {
select {
case _, ok := <-connection.read:
if !ok {
return
}
case <-ctx.Done():
t.Fatal("read pump leaked after close with stopped consumer")
}
}
}
func TestMalformedWireClosesWithoutDelivery_BH_CHUNK_03(t *testing.T) {
t.Parallel()
for _, scenario := range []struct {
name string
packet []byte
want error
}{
{"invalid kind", []byte{0xff, 1}, ErrPacket},
{"oversized declaration", []byte{1, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 1}, ErrMessageTooBig},
{"oversized wire packet", make([]byte, PacketBytes+1), websocket.ErrMessageTooBig},
} {
t.Run(scenario.name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
result := make(chan incoming, 1)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
raw, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{Subprotocol}})
if err != nil {
result <- incoming{err: err}
return
}
connection, err := Wrap(raw, 1<<20, nil)
if err != nil {
_ = raw.CloseNow()
result <- incoming{err: err}
return
}
defer connection.CloseNow()
_, payload, err := connection.Read(ctx)
result <- incoming{payload: payload, err: err}
}))
defer server.Close()
raw, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http"), &websocket.DialOptions{Subprotocols: []string{Subprotocol}})
if err != nil {
t.Fatal(err)
}
defer raw.CloseNow()
if err := raw.Write(ctx, websocket.MessageBinary, scenario.packet); err != nil {
t.Fatal(err)
}
_, _, closed := raw.Read(ctx)
wantCode := websocket.StatusPolicyViolation
if !errors.Is(scenario.want, ErrPacket) {
wantCode = websocket.StatusMessageTooBig
}
if websocket.CloseStatus(closed) != wantCode {
t.Fatalf("close code = %v, want %v", closed, wantCode)
}
select {
case frame := <-result:
if frame.payload != nil || !errors.Is(frame.err, scenario.want) {
t.Fatalf("malformed frame delivered or wrong error: %v", frame.err)
}
case <-ctx.Done():
t.Fatal("malformed packet did not close reader")
}
})
}
}
+130
View File
@@ -0,0 +1,130 @@
// Package agentwire carries bounded logical Agent messages over WebSocket.
// It does not assign command IDs, event sequences, or durable acknowledgements.
package agentwire
import (
"encoding/binary"
"errors"
"math"
)
const (
Subprotocol = "rvbox.agent.v1.chunked"
PacketBytes = 16 << 10
headerBytes = 9
complete = byte(0)
data = byte(1)
control = byte(2)
)
var (
ErrPacket = errors.New("invalid agent transport packet")
ErrMessageTooBig = errors.New("agent transport message exceeds logical limit")
)
// Encoder retains one immutable logical message and allocates one wire packet
// at a time. Two fragment lanes reuse the server's control/data distinction;
// no fragment IDs are needed on an ordered, connection-scoped stream.
type Encoder struct {
payload []byte
offset int
kind byte
chunked bool
}
func NewEncoder(payload []byte, maximum uint64, chunked, priority bool) (*Encoder, error) {
if len(payload) == 0 {
return nil, ErrPacket
}
if uint64(len(payload)) > maximum || uint64(len(payload)) > math.MaxUint32 {
return nil, ErrMessageTooBig
}
kind := data
if priority {
kind = control
}
return &Encoder{payload: payload, chunked: chunked, kind: kind}, nil
}
// Next returns a packet and whether it finishes this logical message.
func (encoder *Encoder) Next() ([]byte, bool) {
if encoder.offset == len(encoder.payload) {
return nil, true
}
if !encoder.chunked {
encoder.offset = len(encoder.payload)
return encoder.payload, true
}
if len(encoder.payload) < PacketBytes {
packet := make([]byte, len(encoder.payload)+1)
copy(packet[1:], encoder.payload)
encoder.offset = len(encoder.payload)
return packet, true
}
end := min(len(encoder.payload), encoder.offset+PacketBytes-headerBytes)
packet := make([]byte, headerBytes+end-encoder.offset)
packet[0] = encoder.kind
binary.BigEndian.PutUint32(packet[1:5], uint32(len(encoder.payload)))
binary.BigEndian.PutUint32(packet[5:9], uint32(encoder.offset))
copy(packet[headerBytes:], encoder.payload[encoder.offset:end])
encoder.offset = end
return packet, end == len(encoder.payload)
}
type partial struct {
total uint32
bytes []byte
}
// Decoder is owned by one network reader. At most one incomplete message per
// lane is retained, each bounded by Maximum before allocation. Complete small
// messages can bypass either partial lane. Errors are connection-fatal.
type Decoder struct {
Maximum uint64
lanes [2]partial
}
func (decoder *Decoder) Push(packet []byte) ([]byte, error) {
if len(packet) == 0 || len(packet) > PacketBytes {
return nil, ErrPacket
}
if packet[0] == complete {
if len(packet) == 1 {
return nil, ErrPacket
}
if uint64(len(packet)-1) > decoder.Maximum {
return nil, ErrMessageTooBig
}
return packet[1:], nil
}
if (packet[0] != data && packet[0] != control) || len(packet) <= headerBytes {
return nil, ErrPacket
}
total := binary.BigEndian.Uint32(packet[1:5])
offset := binary.BigEndian.Uint32(packet[5:9])
if uint64(total) > decoder.Maximum {
return nil, ErrMessageTooBig
}
length := uint32(len(packet) - headerBytes)
if total < PacketBytes || offset > total || length > total-offset {
return nil, ErrPacket
}
lane := &decoder.lanes[packet[0]-1]
if lane.bytes == nil {
if offset != 0 {
return nil, ErrPacket
}
lane.total = total
lane.bytes = make([]byte, 0, int(total))
}
if lane.total != total || uint64(offset) != uint64(len(lane.bytes)) {
return nil, ErrPacket
}
lane.bytes = append(lane.bytes, packet[headerBytes:]...)
if uint32(len(lane.bytes)) != total {
return nil, nil
}
result := lane.bytes
*lane = partial{}
return result, nil
}
+169
View File
@@ -0,0 +1,169 @@
package agentwire
import (
"bytes"
"encoding/binary"
"errors"
"testing"
)
func packets(t *testing.T, payload []byte, priority bool) [][]byte {
t.Helper()
encoder, err := NewEncoder(payload, 1<<20, true, priority)
if err != nil {
t.Fatal(err)
}
var result [][]byte
for {
packet, last := encoder.Next()
if len(packet) > PacketBytes {
t.Fatal("wire packet exceeds ceiling")
}
result = append(result, packet)
if last {
return result
}
}
}
func TestMessageRoundTripAndBoundaries_HP_CHUNK_01(t *testing.T) {
t.Parallel()
for _, size := range []int{1, PacketBytes - 1, PacketBytes, PacketBytes + 1, 64 << 10, 1 << 20} {
payload := bytes.Repeat([]byte{0x53}, size)
decoder := Decoder{Maximum: 1 << 20}
encoded := packets(t, payload, false)
for index, packet := range encoded {
result, err := decoder.Push(packet)
if err != nil {
t.Fatal(err)
}
if index != len(encoded)-1 && result != nil {
t.Fatal("partial logical message delivered")
}
if index == len(encoded)-1 && !bytes.Equal(result, payload) {
t.Fatalf("message size %d changed", size)
}
}
}
if _, err := NewEncoder(make([]byte, (1<<20)+1), 1<<20, true, false); !errors.Is(err, ErrMessageTooBig) {
t.Fatalf("logical ceiling bypassed: %v", err)
}
if _, err := NewEncoder(nil, 1<<20, true, false); !errors.Is(err, ErrPacket) {
t.Fatal("empty logical message accepted")
}
}
func TestIndependentControlLaneAndDisconnect_REC_CHUNK_01(t *testing.T) {
t.Parallel()
large := bytes.Repeat([]byte{0x51}, 200<<10)
priority := bytes.Repeat([]byte{0x52}, 48<<10)
dataPackets := packets(t, large, false)
controlPackets := packets(t, priority, true)
decoder := Decoder{Maximum: 1 << 20}
if value, err := decoder.Push(dataPackets[0]); err != nil || value != nil {
t.Fatalf("first partial packet: %v", err)
}
for index, packet := range controlPackets {
value, err := decoder.Push(packet)
if err != nil || index == len(controlPackets)-1 && !bytes.Equal(value, priority) {
t.Fatalf("control lane did not overtake partial data: %v", err)
}
}
ack := []byte("ack")
if value, err := decoder.Push(packets(t, ack, false)[0]); err != nil || !bytes.Equal(value, ack) {
t.Fatal("small message blocked behind partial data")
}
for index, packet := range dataPackets[1:] {
value, err := decoder.Push(packet)
if err != nil || index == len(dataPackets)-2 && !bytes.Equal(value, large) {
t.Fatalf("preemption changed data: %v", err)
}
}
// A new connection has no partial state. A tail without its prefix fails;
// a complete durable replay from offset zero succeeds.
restarted := Decoder{Maximum: 1 << 20}
if _, err := restarted.Push(dataPackets[1]); !errors.Is(err, ErrPacket) {
t.Fatal("new connection accepted an old tail")
}
restarted = Decoder{Maximum: 1 << 20}
var result []byte
for _, packet := range dataPackets {
var err error
result, err = restarted.Push(packet)
if err != nil {
t.Fatal(err)
}
}
if !bytes.Equal(result, large) {
t.Fatal("durable replay changed message")
}
}
func TestRejectMalformedPacketsWithoutAllocation_BH_CHUNK_01(t *testing.T) {
t.Parallel()
base := packets(t, bytes.Repeat([]byte{0x53}, 64<<10), false)
changed := func(packet []byte, position int, value uint32) []byte {
result := bytes.Clone(packet)
binary.BigEndian.PutUint32(result[position:position+4], value)
return result
}
for _, scenario := range []struct {
name string
input [][]byte
want error
max uint64
}{
{name: "empty packet", input: [][]byte{nil}, want: ErrPacket},
{name: "empty complete", input: [][]byte{{0}}, want: ErrPacket},
{name: "unknown kind", input: [][]byte{{3, 1}}, want: ErrPacket},
{name: "short header", input: [][]byte{{1, 0}}, want: ErrPacket},
{name: "oversized packet", input: [][]byte{make([]byte, PacketBytes+1)}, want: ErrPacket},
{name: "oversized declaration", input: [][]byte{changed(base[0], 1, 0xffffffff)}, want: ErrMessageTooBig},
{name: "small fragmented declaration", input: [][]byte{changed(base[0], 1, 1)}, want: ErrPacket},
{name: "tail first", input: [][]byte{base[1]}, want: ErrPacket},
{name: "duplicate prefix", input: [][]byte{base[0], base[0]}, want: ErrPacket},
{name: "gap", input: [][]byte{base[0], base[2]}, want: ErrPacket},
{name: "changed total", input: [][]byte{base[0], changed(base[1], 1, 65537)}, want: ErrPacket},
{name: "offset overflow", input: [][]byte{changed(base[0], 5, 0xffffffff)}, want: ErrPacket},
{name: "configured complete limit", input: [][]byte{{0, 1, 2}}, max: 1, want: ErrMessageTooBig},
} {
t.Run(scenario.name, func(t *testing.T) {
maximum := scenario.max
if maximum == 0 {
maximum = 1 << 20
}
decoder := Decoder{Maximum: maximum}
var err error
for _, packet := range scenario.input {
_, err = decoder.Push(packet)
if err != nil {
break
}
}
if !errors.Is(err, scenario.want) {
t.Fatalf("error = %v, want %v", err, scenario.want)
}
if scenario.name == "oversized declaration" && decoder.lanes[0].bytes != nil {
t.Fatal("allocated an oversized declared message")
}
})
}
}
func FuzzDecoder(f *testing.F) {
f.Add([]byte{0, 1}, []byte{1})
f.Add([]byte{1, 0, 1, 0, 0, 0, 0, 0, 0, 1}, []byte{0, 1})
f.Fuzz(func(t *testing.T, first, second []byte) {
decoder := Decoder{Maximum: 64 << 10}
for _, packet := range [][]byte{first, second} {
if _, err := decoder.Push(packet); err != nil {
return
}
for _, lane := range decoder.lanes {
if cap(lane.bytes) > 64<<10 {
t.Fatal("reassembly exceeded configured bound")
}
}
}
})
}
+10 -3
View File
@@ -9,11 +9,12 @@ import (
"github.com/coder/websocket"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/agentwire"
)
var ErrUnexpectedMessage = errors.New("agent transport received a non-binary WebSocket message")
type WebSocketTransport struct{ connection *websocket.Conn }
type WebSocketTransport struct{ connection *agentwire.Conn }
func DialWebSocket(ctx context.Context, address string, client *http.Client) (*WebSocketTransport, error) {
return DialWebSocketWithLimits(ctx, address, client, agentproto.DefaultLimits())
@@ -30,11 +31,17 @@ func DialWebSocketWithLimits(ctx context.Context, address string, client *http.C
if readLimit > math.MaxInt64-1 {
return nil, fmt.Errorf("invalid WebSocket envelope limit: %d", readLimit)
}
connection, _, err := websocket.Dial(ctx, address, &websocket.DialOptions{HTTPClient: client, CompressionMode: websocket.CompressionDisabled})
raw, _, err := websocket.Dial(ctx, address, &websocket.DialOptions{
HTTPClient: client, CompressionMode: websocket.CompressionDisabled, Subprotocols: []string{agentwire.Subprotocol},
})
if err != nil {
return nil, err
}
connection.SetReadLimit(int64(readLimit))
connection, err := agentwire.Wrap(raw, readLimit, nil)
if err != nil {
_ = raw.CloseNow()
return nil, err
}
return &WebSocketTransport{connection: connection}, nil
}
+75 -27
View File
@@ -16,6 +16,7 @@ import (
"github.com/coder/websocket"
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/agentwire"
"github.com/rvbox/rvbox/internal/domain"
"github.com/rvbox/rvbox/internal/observability"
"github.com/rvbox/rvbox/internal/server/store"
@@ -78,8 +79,9 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
}
started := time.Now()
heartbeat := newSynchronizedHeartbeat(server.heartbeatIdle(), server.livenessTimeout(), 0)
connection, err := websocket.Accept(response, request, &websocket.AcceptOptions{
raw, err := websocket.Accept(response, request, &websocket.AcceptOptions{
CompressionMode: websocket.CompressionDisabled,
Subprotocols: []string{agentwire.Subprotocol},
// coder/websocket consumes control frames inside Read. Count ping and
// pong callbacks as inbound activity so a healthy, otherwise idle agent
// is not mistaken for a dead peer; application Read results are observed
@@ -95,16 +97,20 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
if err != nil {
return
}
connection.SetReadLimit(int64(server.limits().MaxEnvelopeBytes))
connection, err := agentwire.Wrap(raw, server.limits().MaxEnvelopeBytes, func() { heartbeat.Observe(time.Since(started)) })
if err != nil {
_ = raw.CloseNow()
return
}
server.incMetric("agent_connection")
defer connection.CloseNow()
server.serveConnection(request.Context(), connection, heartbeat, started)
}
func (server *AgentServer) serveConnection(parent context.Context, connection *websocket.Conn, heartbeat *synchronizedHeartbeat, started time.Time) {
func (server *AgentServer) serveConnection(parent context.Context, connection *agentwire.Conn, heartbeat *synchronizedHeartbeat, started time.Time) {
messageType, payload, err := connection.Read(parent)
if err != nil {
if errors.Is(err, websocket.ErrMessageTooBig) {
if errors.Is(err, websocket.ErrMessageTooBig) || errors.Is(err, agentwire.ErrMessageTooBig) {
server.incMetric("agent_message_too_big")
}
server.logSessionError("", 0, "hello-read", err)
@@ -255,7 +261,7 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
for {
messageType, payload, err = connection.Read(sessionContext)
if err != nil {
if errors.Is(err, websocket.ErrMessageTooBig) {
if errors.Is(err, websocket.ErrMessageTooBig) || errors.Is(err, agentwire.ErrMessageTooBig) {
server.incMetric("agent_message_too_big")
}
server.logSessionError(hello.GetClientId(), registration.Generation, "read", err)
@@ -784,39 +790,81 @@ func (server *AgentServer) closeForDispatchFailure(cancel context.CancelFunc) {
}
}
func (server *AgentServer) writeLoop(ctx context.Context, connection *websocket.Conn, queue *WriterQueue, heartbeat *synchronizedHeartbeat, started time.Time) {
type packetWriter interface {
Encode([]byte, bool) (*agentwire.Encoder, error)
WritePacket(context.Context, []byte) error
Ping(context.Context) error
}
func (server *AgentServer) writeLoop(ctx context.Context, connection packetWriter, queue *WriterQueue, heartbeat *synchronizedHeartbeat, started time.Time) {
checkHeartbeat := func() error {
switch heartbeat.Check(time.Since(started)) {
case HeartbeatPing:
pingContext, cancel := context.WithTimeout(ctx, server.writeDeadline())
defer cancel()
return connection.Ping(pingContext)
case HeartbeatClose:
server.incMetric("heartbeat_timeout")
return ErrSessionClosed
}
return ctx.Err()
}
var writeFrame func(Frame) error
writeFrame = func(frame Frame) error {
encoder, err := connection.Encode(frame.Payload, frame.Kind == FrameControl)
if err != nil {
return err
}
for {
packet, last := encoder.Next()
writeContext, cancel := context.WithTimeout(ctx, server.writeDeadline())
err := connection.WritePacket(writeContext, packet)
cancel()
if err != nil {
return err
}
if err := checkHeartbeat(); err != nil {
return err
}
if last {
break
}
if frame.Kind == FrameData {
// A bounded control burst may overtake a partial data message.
// Its independent fragment lane cannot corrupt data reassembly.
for burst := uint32(0); burst < queue.controlBurst; burst++ {
priority, ok := queue.TryControl()
if !ok {
break
}
if err := writeFrame(priority); err != nil {
return err
}
}
}
}
if frame.Written != nil {
close(frame.Written)
}
if frame.OnWritten != nil {
frame.OnWritten()
}
return nil
}
for {
frameContext, cancel := context.WithTimeout(ctx, server.heartbeatPollInterval())
frame, err := queue.Next(frameContext)
cancel()
if err == nil {
writeContext, writeCancel := context.WithTimeout(ctx, server.writeDeadline())
err = connection.Write(writeContext, websocket.MessageBinary, frame.Payload)
writeCancel()
if err != nil {
if err := writeFrame(frame); err != nil {
return
}
if frame.Written != nil {
close(frame.Written)
}
if frame.OnWritten != nil {
frame.OnWritten()
}
continue
}
if !errors.Is(err, context.DeadlineExceeded) {
return
}
switch heartbeat.Check(time.Since(started)) {
case HeartbeatPing:
pingContext, pingCancel := context.WithTimeout(ctx, server.writeDeadline())
err = connection.Ping(pingContext)
pingCancel()
if err != nil {
return
}
case HeartbeatClose:
server.incMetric("heartbeat_timeout")
if err := checkHeartbeat(); err != nil {
return
}
}
@@ -834,7 +882,7 @@ func (server *AgentServer) observeMetric(name string, duration time.Duration) {
}
}
func (server *AgentServer) close(connection *websocket.Conn, status websocket.StatusCode, reason string) {
func (server *AgentServer) close(connection *agentwire.Conn, status websocket.StatusCode, reason string) {
_ = connection.Close(status, reason)
}
@@ -0,0 +1,116 @@
package session
import (
"bytes"
"context"
"sync/atomic"
"testing"
"time"
"github.com/rvbox/rvbox/internal/agentwire"
)
func TestChunkedWriterPreemptionAndBarrier_RACE_CHUNK_02(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
queue := NewWriterQueue(8, 2, 4)
defer queue.Close()
data := bytes.Repeat([]byte{0x51}, 128<<10)
priority := bytes.Repeat([]byte{0x52}, 48<<10)
dataWritten, controlWritten := make(chan struct{}), make(chan struct{})
queue.EnqueueData(Frame{Kind: FrameData, Payload: data, Written: dataWritten})
writer := &blockedPacketWriter{packets: make(chan []byte, 32), gate: make(chan struct{})}
done := make(chan struct{})
started := time.Now()
go func() {
defer close(done)
(&AgentServer{}).writeLoop(ctx, writer, queue, newSynchronizedHeartbeat(time.Millisecond, time.Second, 0), started)
}()
var first []byte
select {
case first = <-writer.packets:
case <-ctx.Done():
t.Fatal("first packet not written")
}
if err := queue.EnqueueControl(Frame{Kind: FrameControl, Payload: []byte("ack")}); err != nil {
t.Fatal(err)
}
if err := queue.EnqueueControl(Frame{Kind: FrameControl, Payload: priority, Written: controlWritten}); err != nil {
t.Fatal(err)
}
select {
case <-dataWritten:
t.Fatal("dispatch barrier opened on a partial message")
default:
}
time.Sleep(2 * time.Millisecond) // Cross the heartbeat idle boundary.
close(writer.gate)
decoder := agentwire.Decoder{Maximum: 1 << 20}
if value, err := decoder.Push(first); err != nil || value != nil {
t.Fatal("invalid first partial packet")
}
var received [][]byte
for len(received) < 3 {
select {
case packet := <-writer.packets:
value, err := decoder.Push(packet)
if err != nil {
t.Fatal(err)
}
if value != nil {
received = append(received, value)
}
case <-ctx.Done():
t.Fatal("preempted writer stalled")
}
}
if !bytes.Equal(received[0], []byte("ack")) || !bytes.Equal(received[1], priority) || !bytes.Equal(received[2], data) {
t.Fatal("control did not preempt partial data intact")
}
select {
case <-dataWritten:
case <-ctx.Done():
t.Fatal("completed data barrier never opened")
}
select {
case <-controlWritten:
default:
t.Fatal("completed control barrier never opened")
}
cancel()
<-done
if writer.pings.Load() == 0 {
t.Fatal("heartbeat skipped during chunked transfer")
}
}
type blockedPacketWriter struct {
packets chan []byte
gate chan struct{}
writes int
pings atomic.Uint32
}
func (writer *blockedPacketWriter) Encode(payload []byte, priority bool) (*agentwire.Encoder, error) {
return agentwire.NewEncoder(payload, 1<<20, true, priority)
}
func (writer *blockedPacketWriter) WritePacket(ctx context.Context, packet []byte) error {
select {
case writer.packets <- bytes.Clone(packet):
case <-ctx.Done():
return ctx.Err()
}
writer.writes++
if writer.writes == 1 {
select {
case <-writer.gate:
case <-ctx.Done():
return ctx.Err()
}
}
return nil
}
func (writer *blockedPacketWriter) Ping(context.Context) error { writer.pings.Add(1); return nil }
+11
View File
@@ -126,6 +126,17 @@ func (queue *WriterQueue) Next(ctx context.Context) (Frame, error) {
}
}
// TryControl is used only by the sole writer between packets of a data frame.
func (queue *WriterQueue) TryControl() (Frame, bool) {
select {
case frame := <-queue.control:
queue.controlRun++
return frame, true
default:
return Frame{}, false
}
}
func (queue *WriterQueue) Close() { queue.closeOnce.Do(func() { close(queue.closed) }) }
type HeartbeatAction uint8