feat(transport): negotiate bounded automatic message chunking

This commit is contained in:
cabbage committed 2026-11-09 08:50:51 +00:00
1 parent 474afb3484
commit 8acffac985
20 files changed
+1332 -38

No files matched your search

+75 -27
View File
@@ -16,6 +16,7 @@ import (
"github.com/coder/websocket"
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
"github.com/rvbox/rvbox/internal/agentproto"
"github.com/rvbox/rvbox/internal/agentwire"
"github.com/rvbox/rvbox/internal/domain"
"github.com/rvbox/rvbox/internal/observability"
"github.com/rvbox/rvbox/internal/server/store"
@@ -78,8 +79,9 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
}
started := time.Now()
heartbeat := newSynchronizedHeartbeat(server.heartbeatIdle(), server.livenessTimeout(), 0)
connection, err := websocket.Accept(response, request, &websocket.AcceptOptions{
raw, err := websocket.Accept(response, request, &websocket.AcceptOptions{
CompressionMode: websocket.CompressionDisabled,
Subprotocols: []string{agentwire.Subprotocol},
// coder/websocket consumes control frames inside Read. Count ping and
// pong callbacks as inbound activity so a healthy, otherwise idle agent
// is not mistaken for a dead peer; application Read results are observed
@@ -95,16 +97,20 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
if err != nil {
return
}
connection.SetReadLimit(int64(server.limits().MaxEnvelopeBytes))
connection, err := agentwire.Wrap(raw, server.limits().MaxEnvelopeBytes, func() { heartbeat.Observe(time.Since(started)) })
if err != nil {
_ = raw.CloseNow()
return
}
server.incMetric("agent_connection")
defer connection.CloseNow()
server.serveConnection(request.Context(), connection, heartbeat, started)
}
func (server *AgentServer) serveConnection(parent context.Context, connection *websocket.Conn, heartbeat *synchronizedHeartbeat, started time.Time) {
func (server *AgentServer) serveConnection(parent context.Context, connection *agentwire.Conn, heartbeat *synchronizedHeartbeat, started time.Time) {
messageType, payload, err := connection.Read(parent)
if err != nil {
if errors.Is(err, websocket.ErrMessageTooBig) {
if errors.Is(err, websocket.ErrMessageTooBig) || errors.Is(err, agentwire.ErrMessageTooBig) {
server.incMetric("agent_message_too_big")
}
server.logSessionError("", 0, "hello-read", err)
@@ -255,7 +261,7 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
for {
messageType, payload, err = connection.Read(sessionContext)
if err != nil {
if errors.Is(err, websocket.ErrMessageTooBig) {
if errors.Is(err, websocket.ErrMessageTooBig) || errors.Is(err, agentwire.ErrMessageTooBig) {
server.incMetric("agent_message_too_big")
}
server.logSessionError(hello.GetClientId(), registration.Generation, "read", err)
@@ -784,39 +790,81 @@ func (server *AgentServer) closeForDispatchFailure(cancel context.CancelFunc) {
}
}
func (server *AgentServer) writeLoop(ctx context.Context, connection *websocket.Conn, queue *WriterQueue, heartbeat *synchronizedHeartbeat, started time.Time) {
type packetWriter interface {
Encode([]byte, bool) (*agentwire.Encoder, error)
WritePacket(context.Context, []byte) error
Ping(context.Context) error
}
func (server *AgentServer) writeLoop(ctx context.Context, connection packetWriter, queue *WriterQueue, heartbeat *synchronizedHeartbeat, started time.Time) {
checkHeartbeat := func() error {
switch heartbeat.Check(time.Since(started)) {
case HeartbeatPing:
pingContext, cancel := context.WithTimeout(ctx, server.writeDeadline())
defer cancel()
return connection.Ping(pingContext)
case HeartbeatClose:
server.incMetric("heartbeat_timeout")
return ErrSessionClosed
}
return ctx.Err()
}
var writeFrame func(Frame) error
writeFrame = func(frame Frame) error {
encoder, err := connection.Encode(frame.Payload, frame.Kind == FrameControl)
if err != nil {
return err
}
for {
packet, last := encoder.Next()
writeContext, cancel := context.WithTimeout(ctx, server.writeDeadline())
err := connection.WritePacket(writeContext, packet)
cancel()
if err != nil {
return err
}
if err := checkHeartbeat(); err != nil {
return err
}
if last {
break
}
if frame.Kind == FrameData {
// A bounded control burst may overtake a partial data message.
// Its independent fragment lane cannot corrupt data reassembly.
for burst := uint32(0); burst < queue.controlBurst; burst++ {
priority, ok := queue.TryControl()
if !ok {
break
}
if err := writeFrame(priority); err != nil {
return err
}
}
}
}
if frame.Written != nil {
close(frame.Written)
}
if frame.OnWritten != nil {
frame.OnWritten()
}
return nil
}
for {
frameContext, cancel := context.WithTimeout(ctx, server.heartbeatPollInterval())
frame, err := queue.Next(frameContext)
cancel()
if err == nil {
writeContext, writeCancel := context.WithTimeout(ctx, server.writeDeadline())
err = connection.Write(writeContext, websocket.MessageBinary, frame.Payload)
writeCancel()
if err != nil {
if err := writeFrame(frame); err != nil {
return
}
if frame.Written != nil {
close(frame.Written)
}
if frame.OnWritten != nil {
frame.OnWritten()
}
continue
}
if !errors.Is(err, context.DeadlineExceeded) {
return
}
switch heartbeat.Check(time.Since(started)) {
case HeartbeatPing:
pingContext, pingCancel := context.WithTimeout(ctx, server.writeDeadline())
err = connection.Ping(pingContext)
pingCancel()
if err != nil {
return
}
case HeartbeatClose:
server.incMetric("heartbeat_timeout")
if err := checkHeartbeat(); err != nil {
return
}
}
@@ -834,7 +882,7 @@ func (server *AgentServer) observeMetric(name string, duration time.Duration) {
}
}
func (server *AgentServer) close(connection *websocket.Conn, status websocket.StatusCode, reason string) {
func (server *AgentServer) close(connection *agentwire.Conn, status websocket.StatusCode, reason string) {
_ = connection.Close(status, reason)
}
@@ -0,0 +1,116 @@
package session
import (
"bytes"
"context"
"sync/atomic"
"testing"
"time"
"github.com/rvbox/rvbox/internal/agentwire"
)
func TestChunkedWriterPreemptionAndBarrier_RACE_CHUNK_02(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
queue := NewWriterQueue(8, 2, 4)
defer queue.Close()
data := bytes.Repeat([]byte{0x51}, 128<<10)
priority := bytes.Repeat([]byte{0x52}, 48<<10)
dataWritten, controlWritten := make(chan struct{}), make(chan struct{})
queue.EnqueueData(Frame{Kind: FrameData, Payload: data, Written: dataWritten})
writer := &blockedPacketWriter{packets: make(chan []byte, 32), gate: make(chan struct{})}
done := make(chan struct{})
started := time.Now()
go func() {
defer close(done)
(&AgentServer{}).writeLoop(ctx, writer, queue, newSynchronizedHeartbeat(time.Millisecond, time.Second, 0), started)
}()
var first []byte
select {
case first = <-writer.packets:
case <-ctx.Done():
t.Fatal("first packet not written")
}
if err := queue.EnqueueControl(Frame{Kind: FrameControl, Payload: []byte("ack")}); err != nil {
t.Fatal(err)
}
if err := queue.EnqueueControl(Frame{Kind: FrameControl, Payload: priority, Written: controlWritten}); err != nil {
t.Fatal(err)
}
select {
case <-dataWritten:
t.Fatal("dispatch barrier opened on a partial message")
default:
}
time.Sleep(2 * time.Millisecond) // Cross the heartbeat idle boundary.
close(writer.gate)
decoder := agentwire.Decoder{Maximum: 1 << 20}
if value, err := decoder.Push(first); err != nil || value != nil {
t.Fatal("invalid first partial packet")
}
var received [][]byte
for len(received) < 3 {
select {
case packet := <-writer.packets:
value, err := decoder.Push(packet)
if err != nil {
t.Fatal(err)
}
if value != nil {
received = append(received, value)
}
case <-ctx.Done():
t.Fatal("preempted writer stalled")
}
}
if !bytes.Equal(received[0], []byte("ack")) || !bytes.Equal(received[1], priority) || !bytes.Equal(received[2], data) {
t.Fatal("control did not preempt partial data intact")
}
select {
case <-dataWritten:
case <-ctx.Done():
t.Fatal("completed data barrier never opened")
}
select {
case <-controlWritten:
default:
t.Fatal("completed control barrier never opened")
}
cancel()
<-done
if writer.pings.Load() == 0 {
t.Fatal("heartbeat skipped during chunked transfer")
}
}
type blockedPacketWriter struct {
packets chan []byte
gate chan struct{}
writes int
pings atomic.Uint32
}
func (writer *blockedPacketWriter) Encode(payload []byte, priority bool) (*agentwire.Encoder, error) {
return agentwire.NewEncoder(payload, 1<<20, true, priority)
}
func (writer *blockedPacketWriter) WritePacket(ctx context.Context, packet []byte) error {
select {
case writer.packets <- bytes.Clone(packet):
case <-ctx.Done():
return ctx.Err()
}
writer.writes++
if writer.writes == 1 {
select {
case <-writer.gate:
case <-ctx.Done():
return ctx.Err()
}
}
return nil
}
func (writer *blockedPacketWriter) Ping(context.Context) error { writer.pings.Add(1); return nil }
+11
View File
@@ -126,6 +126,17 @@ func (queue *WriterQueue) Next(ctx context.Context) (Frame, error) {
}
}
// TryControl is used only by the sole writer between packets of a data frame.
func (queue *WriterQueue) TryControl() (Frame, bool) {
select {
case frame := <-queue.control:
queue.controlRun++
return frame, true
default:
return Frame{}, false
}
}
func (queue *WriterQueue) Close() { queue.closeOnce.Do(func() { close(queue.closed) }) }
type HeartbeatAction uint8