feat(transport): negotiate bounded automatic message chunking
This commit is contained in:
1 parent
474afb3484
commit
8acffac985
20 files changed
+1332
-38
No files matched your search
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/coder/websocket"
|
||||
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||
"github.com/rvbox/rvbox/internal/agentproto"
|
||||
"github.com/rvbox/rvbox/internal/agentwire"
|
||||
"github.com/rvbox/rvbox/internal/domain"
|
||||
"github.com/rvbox/rvbox/internal/observability"
|
||||
"github.com/rvbox/rvbox/internal/server/store"
|
||||
@@ -78,8 +79,9 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
|
||||
}
|
||||
started := time.Now()
|
||||
heartbeat := newSynchronizedHeartbeat(server.heartbeatIdle(), server.livenessTimeout(), 0)
|
||||
connection, err := websocket.Accept(response, request, &websocket.AcceptOptions{
|
||||
raw, err := websocket.Accept(response, request, &websocket.AcceptOptions{
|
||||
CompressionMode: websocket.CompressionDisabled,
|
||||
Subprotocols: []string{agentwire.Subprotocol},
|
||||
// coder/websocket consumes control frames inside Read. Count ping and
|
||||
// pong callbacks as inbound activity so a healthy, otherwise idle agent
|
||||
// is not mistaken for a dead peer; application Read results are observed
|
||||
@@ -95,16 +97,20 @@ func (server *AgentServer) ServeHTTP(response http.ResponseWriter, request *http
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
connection.SetReadLimit(int64(server.limits().MaxEnvelopeBytes))
|
||||
connection, err := agentwire.Wrap(raw, server.limits().MaxEnvelopeBytes, func() { heartbeat.Observe(time.Since(started)) })
|
||||
if err != nil {
|
||||
_ = raw.CloseNow()
|
||||
return
|
||||
}
|
||||
server.incMetric("agent_connection")
|
||||
defer connection.CloseNow()
|
||||
server.serveConnection(request.Context(), connection, heartbeat, started)
|
||||
}
|
||||
|
||||
func (server *AgentServer) serveConnection(parent context.Context, connection *websocket.Conn, heartbeat *synchronizedHeartbeat, started time.Time) {
|
||||
func (server *AgentServer) serveConnection(parent context.Context, connection *agentwire.Conn, heartbeat *synchronizedHeartbeat, started time.Time) {
|
||||
messageType, payload, err := connection.Read(parent)
|
||||
if err != nil {
|
||||
if errors.Is(err, websocket.ErrMessageTooBig) {
|
||||
if errors.Is(err, websocket.ErrMessageTooBig) || errors.Is(err, agentwire.ErrMessageTooBig) {
|
||||
server.incMetric("agent_message_too_big")
|
||||
}
|
||||
server.logSessionError("", 0, "hello-read", err)
|
||||
@@ -255,7 +261,7 @@ func (server *AgentServer) serveConnection(parent context.Context, connection *w
|
||||
for {
|
||||
messageType, payload, err = connection.Read(sessionContext)
|
||||
if err != nil {
|
||||
if errors.Is(err, websocket.ErrMessageTooBig) {
|
||||
if errors.Is(err, websocket.ErrMessageTooBig) || errors.Is(err, agentwire.ErrMessageTooBig) {
|
||||
server.incMetric("agent_message_too_big")
|
||||
}
|
||||
server.logSessionError(hello.GetClientId(), registration.Generation, "read", err)
|
||||
@@ -784,39 +790,81 @@ func (server *AgentServer) closeForDispatchFailure(cancel context.CancelFunc) {
|
||||
}
|
||||
}
|
||||
|
||||
func (server *AgentServer) writeLoop(ctx context.Context, connection *websocket.Conn, queue *WriterQueue, heartbeat *synchronizedHeartbeat, started time.Time) {
|
||||
type packetWriter interface {
|
||||
Encode([]byte, bool) (*agentwire.Encoder, error)
|
||||
WritePacket(context.Context, []byte) error
|
||||
Ping(context.Context) error
|
||||
}
|
||||
|
||||
func (server *AgentServer) writeLoop(ctx context.Context, connection packetWriter, queue *WriterQueue, heartbeat *synchronizedHeartbeat, started time.Time) {
|
||||
checkHeartbeat := func() error {
|
||||
switch heartbeat.Check(time.Since(started)) {
|
||||
case HeartbeatPing:
|
||||
pingContext, cancel := context.WithTimeout(ctx, server.writeDeadline())
|
||||
defer cancel()
|
||||
return connection.Ping(pingContext)
|
||||
case HeartbeatClose:
|
||||
server.incMetric("heartbeat_timeout")
|
||||
return ErrSessionClosed
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
var writeFrame func(Frame) error
|
||||
writeFrame = func(frame Frame) error {
|
||||
encoder, err := connection.Encode(frame.Payload, frame.Kind == FrameControl)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
packet, last := encoder.Next()
|
||||
writeContext, cancel := context.WithTimeout(ctx, server.writeDeadline())
|
||||
err := connection.WritePacket(writeContext, packet)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkHeartbeat(); err != nil {
|
||||
return err
|
||||
}
|
||||
if last {
|
||||
break
|
||||
}
|
||||
if frame.Kind == FrameData {
|
||||
// A bounded control burst may overtake a partial data message.
|
||||
// Its independent fragment lane cannot corrupt data reassembly.
|
||||
for burst := uint32(0); burst < queue.controlBurst; burst++ {
|
||||
priority, ok := queue.TryControl()
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
if err := writeFrame(priority); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if frame.Written != nil {
|
||||
close(frame.Written)
|
||||
}
|
||||
if frame.OnWritten != nil {
|
||||
frame.OnWritten()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for {
|
||||
frameContext, cancel := context.WithTimeout(ctx, server.heartbeatPollInterval())
|
||||
frame, err := queue.Next(frameContext)
|
||||
cancel()
|
||||
if err == nil {
|
||||
writeContext, writeCancel := context.WithTimeout(ctx, server.writeDeadline())
|
||||
err = connection.Write(writeContext, websocket.MessageBinary, frame.Payload)
|
||||
writeCancel()
|
||||
if err != nil {
|
||||
if err := writeFrame(frame); err != nil {
|
||||
return
|
||||
}
|
||||
if frame.Written != nil {
|
||||
close(frame.Written)
|
||||
}
|
||||
if frame.OnWritten != nil {
|
||||
frame.OnWritten()
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
return
|
||||
}
|
||||
switch heartbeat.Check(time.Since(started)) {
|
||||
case HeartbeatPing:
|
||||
pingContext, pingCancel := context.WithTimeout(ctx, server.writeDeadline())
|
||||
err = connection.Ping(pingContext)
|
||||
pingCancel()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
case HeartbeatClose:
|
||||
server.incMetric("heartbeat_timeout")
|
||||
if err := checkHeartbeat(); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -834,7 +882,7 @@ func (server *AgentServer) observeMetric(name string, duration time.Duration) {
|
||||
}
|
||||
}
|
||||
|
||||
func (server *AgentServer) close(connection *websocket.Conn, status websocket.StatusCode, reason string) {
|
||||
func (server *AgentServer) close(connection *agentwire.Conn, status websocket.StatusCode, reason string) {
|
||||
_ = connection.Close(status, reason)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package session
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rvbox/rvbox/internal/agentwire"
|
||||
)
|
||||
|
||||
func TestChunkedWriterPreemptionAndBarrier_RACE_CHUNK_02(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
queue := NewWriterQueue(8, 2, 4)
|
||||
defer queue.Close()
|
||||
data := bytes.Repeat([]byte{0x51}, 128<<10)
|
||||
priority := bytes.Repeat([]byte{0x52}, 48<<10)
|
||||
dataWritten, controlWritten := make(chan struct{}), make(chan struct{})
|
||||
queue.EnqueueData(Frame{Kind: FrameData, Payload: data, Written: dataWritten})
|
||||
writer := &blockedPacketWriter{packets: make(chan []byte, 32), gate: make(chan struct{})}
|
||||
done := make(chan struct{})
|
||||
started := time.Now()
|
||||
go func() {
|
||||
defer close(done)
|
||||
(&AgentServer{}).writeLoop(ctx, writer, queue, newSynchronizedHeartbeat(time.Millisecond, time.Second, 0), started)
|
||||
}()
|
||||
var first []byte
|
||||
select {
|
||||
case first = <-writer.packets:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("first packet not written")
|
||||
}
|
||||
if err := queue.EnqueueControl(Frame{Kind: FrameControl, Payload: []byte("ack")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := queue.EnqueueControl(Frame{Kind: FrameControl, Payload: priority, Written: controlWritten}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-dataWritten:
|
||||
t.Fatal("dispatch barrier opened on a partial message")
|
||||
default:
|
||||
}
|
||||
time.Sleep(2 * time.Millisecond) // Cross the heartbeat idle boundary.
|
||||
close(writer.gate)
|
||||
decoder := agentwire.Decoder{Maximum: 1 << 20}
|
||||
if value, err := decoder.Push(first); err != nil || value != nil {
|
||||
t.Fatal("invalid first partial packet")
|
||||
}
|
||||
var received [][]byte
|
||||
for len(received) < 3 {
|
||||
select {
|
||||
case packet := <-writer.packets:
|
||||
value, err := decoder.Push(packet)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value != nil {
|
||||
received = append(received, value)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("preempted writer stalled")
|
||||
}
|
||||
}
|
||||
if !bytes.Equal(received[0], []byte("ack")) || !bytes.Equal(received[1], priority) || !bytes.Equal(received[2], data) {
|
||||
t.Fatal("control did not preempt partial data intact")
|
||||
}
|
||||
select {
|
||||
case <-dataWritten:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("completed data barrier never opened")
|
||||
}
|
||||
select {
|
||||
case <-controlWritten:
|
||||
default:
|
||||
t.Fatal("completed control barrier never opened")
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
if writer.pings.Load() == 0 {
|
||||
t.Fatal("heartbeat skipped during chunked transfer")
|
||||
}
|
||||
}
|
||||
|
||||
type blockedPacketWriter struct {
|
||||
packets chan []byte
|
||||
gate chan struct{}
|
||||
writes int
|
||||
pings atomic.Uint32
|
||||
}
|
||||
|
||||
func (writer *blockedPacketWriter) Encode(payload []byte, priority bool) (*agentwire.Encoder, error) {
|
||||
return agentwire.NewEncoder(payload, 1<<20, true, priority)
|
||||
}
|
||||
|
||||
func (writer *blockedPacketWriter) WritePacket(ctx context.Context, packet []byte) error {
|
||||
select {
|
||||
case writer.packets <- bytes.Clone(packet):
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
writer.writes++
|
||||
if writer.writes == 1 {
|
||||
select {
|
||||
case <-writer.gate:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (writer *blockedPacketWriter) Ping(context.Context) error { writer.pings.Add(1); return nil }
|
||||
@@ -126,6 +126,17 @@ func (queue *WriterQueue) Next(ctx context.Context) (Frame, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// TryControl is used only by the sole writer between packets of a data frame.
|
||||
func (queue *WriterQueue) TryControl() (Frame, bool) {
|
||||
select {
|
||||
case frame := <-queue.control:
|
||||
queue.controlRun++
|
||||
return frame, true
|
||||
default:
|
||||
return Frame{}, false
|
||||
}
|
||||
}
|
||||
|
||||
func (queue *WriterQueue) Close() { queue.closeOnce.Do(func() { close(queue.closed) }) }
|
||||
|
||||
type HeartbeatAction uint8
|
||||
|
||||
Reference in new issue
Block a user