fix(client): tolerate script replay after command completion
This commit is contained in:
1 parent
6984a09486
commit
1b5c6dbf98
5 files changed
+178
No files matched your search
@@ -136,6 +136,13 @@ exact ignored `dist/` directory after preserving any evidence.
|
||||
Preserve spool/history: rebooting cannot remove a replayable oversized event.
|
||||
New server logs include bounded read/reconciliation errors and an
|
||||
`agent_message_too_big` counter; new clients advertise the actual build version.
|
||||
- **`terminal command cannot accept script data` during reconnect:** the server
|
||||
may replay an upload while the client is still delivering that command's
|
||||
terminal events. Corrected clients ignore these frames only after verifying
|
||||
the command is locally terminal; existing event replay reports the outcome.
|
||||
They neither change script bytes nor invoke launch again. The same error for
|
||||
a committed but still-live script, unknown owners, and malformed transfers
|
||||
remains a failure. Upgrade the client while preserving its spool.
|
||||
- **No client / stale session:** verify nginx has WebSocket `101` entries for
|
||||
`/v1/agent`, server readiness is true, and the Windows service is running.
|
||||
Use `rvc stat CLIENT_ID`; do not restart the client merely to clear history.
|
||||
|
||||
@@ -416,12 +416,22 @@ func serveActive(ctx context.Context, transport Transport, options RunnerOptions
|
||||
}
|
||||
case envelope.GetScriptChunk() != nil:
|
||||
if err := handleScriptChunk(ctx, transport, options.Store, session, envelope.GetScriptChunk(), limits, options.Now, sent); err != nil {
|
||||
err = finishedScriptReplay(ctx, options.Store, envelope.GetScriptChunk().GetIssueUuid(), err)
|
||||
if err != nil {
|
||||
return fmt.Errorf("apply script chunk for %s: %w", envelope.GetScriptChunk().GetIssueUuid(), err)
|
||||
}
|
||||
options.incMetric("terminal_script_frame")
|
||||
continue
|
||||
}
|
||||
case envelope.GetScriptCommit() != nil:
|
||||
if err := handleScriptCommit(ctx, transport, options.Store, session, envelope.GetScriptCommit(), limits, options.Now, sent); err != nil {
|
||||
err = finishedScriptReplay(ctx, options.Store, envelope.GetScriptCommit().GetIssueUuid(), err)
|
||||
if err != nil {
|
||||
return fmt.Errorf("commit script for %s: %w", envelope.GetScriptCommit().GetIssueUuid(), err)
|
||||
}
|
||||
options.incMetric("terminal_script_frame")
|
||||
continue
|
||||
}
|
||||
if options.OnScriptReady != nil {
|
||||
issue, err := domain.ParseUUIDv7(envelope.GetScriptCommit().GetIssueUuid())
|
||||
if err != nil {
|
||||
@@ -563,6 +573,27 @@ func boundedError(err error) string {
|
||||
return message
|
||||
}
|
||||
|
||||
// A reconnect can race the server's upload replay with the client's terminal
|
||||
// event replay. Leave the finished command unchanged and let its pending
|
||||
// events drain; no script acknowledgement or launch should be fabricated.
|
||||
func finishedScriptReplay(ctx context.Context, store *spool.Store, text string, transferErr error) error {
|
||||
if !errors.Is(transferErr, spool.ErrScriptTerminal) {
|
||||
return transferErr
|
||||
}
|
||||
issue, err := domain.ParseUUIDv7(text)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
command, err := store.GetCommand(ctx, issue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !command.Terminal {
|
||||
return transferErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func handleScriptChunk(ctx context.Context, transport Transport, store *spool.Store, session Session, chunk *rvboxv1.ScriptChunk, limits agentproto.Limits, now func() time.Time, sent map[domain.UUID]uint64) error {
|
||||
status, err := ApplyScriptChunk(ctx, store, session, chunk, limits)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
rvboxv1 "github.com/rvbox/rvbox/gen/go/rvbox/v1"
|
||||
"github.com/rvbox/rvbox/internal/agentproto"
|
||||
"github.com/rvbox/rvbox/internal/client/spool"
|
||||
"github.com/rvbox/rvbox/internal/domain"
|
||||
"github.com/rvbox/rvbox/internal/observability"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
func TestFinishedScriptReplayKeepsSessionUsable_REC_RUNTIME_06(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
store, err := spool.Open(ctx, spool.Options{DataDir: filepath.Join(t.TempDir(), "spool"), BusyTimeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
issue, _ := domain.ParseUUIDv7("019c46f1-1d02-7000-8000-0000000000e1")
|
||||
body := []byte("echo finished script")
|
||||
digest := sha256.Sum256(body)
|
||||
descriptor := spool.ScriptDescriptor{SizeBytes: uint64(len(body)), SHA256: digest}
|
||||
spec := &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD, Source: &rvboxv1.ExecutionSpec_Script{
|
||||
Script: &rvboxv1.ScriptDescriptor{Filename: "done.cmd", SizeBytes: uint64(len(body)), Sha256: digest[:]},
|
||||
}}
|
||||
encoded, _ := proto.Marshal(spec)
|
||||
if _, err := store.AcceptCommand(ctx, spool.Command{IssueUUID: issue, ImmutableSHA256: digest, Revision: 1,
|
||||
Phase: uint32(rvboxv1.CommandLifecycle_COMMAND_ACCEPTED), ExecutionSpec: encoded, Script: &descriptor}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.AppendScriptChunk(ctx, issue, 0, body, digest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.CommitScript(ctx, issue, descriptor); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A committed but live script must still reject new bytes. The shared
|
||||
// ErrScriptTerminal value alone is not sufficient to ignore a frame.
|
||||
if err := finishedScriptReplay(ctx, store, issue.String(), spool.ErrScriptTerminal); !errors.Is(err, spool.ErrScriptTerminal) {
|
||||
t.Fatalf("live script error suppressed: %v", err)
|
||||
}
|
||||
for _, phase := range []rvboxv1.CommandLifecycle{rvboxv1.CommandLifecycle_COMMAND_RUNNING, rvboxv1.CommandLifecycle_COMMAND_SUCCEEDED} {
|
||||
if _, err := store.AppendLifecycle(ctx, issue, uint32(phase), 1, "finished offline", time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := finishedScriptReplay(ctx, store, issue.String(), spool.ErrScriptDigest); !errors.Is(err, spool.ErrScriptDigest) {
|
||||
t.Fatalf("checksum failure suppressed: %v", err)
|
||||
}
|
||||
original, err := store.ScriptBody(ctx, issue)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
marshal := func(payload *rvboxv1.AgentEnvelope) []byte {
|
||||
t.Helper()
|
||||
payload.SessionId, payload.SessionGeneration = "session", 1
|
||||
encoded, err := proto.Marshal(payload)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return encoded
|
||||
}
|
||||
fresh, _ := domain.ParseUUIDv7("019c46f1-1d02-7000-8000-0000000000e2")
|
||||
transport := &scriptReplayTransport{runnerTransport: runnerTransport{reads: [][]byte{
|
||||
marshal(&rvboxv1.AgentEnvelope{Payload: &rvboxv1.AgentEnvelope_ScriptChunk{ScriptChunk: &rvboxv1.ScriptChunk{
|
||||
IssueUuid: issue.String(), Data: body, Sha256: digest[:],
|
||||
}}}),
|
||||
marshal(&rvboxv1.AgentEnvelope{Payload: &rvboxv1.AgentEnvelope_ScriptCommit{ScriptCommit: &rvboxv1.ScriptCommit{
|
||||
IssueUuid: issue.String(), SizeBytes: uint64(len(body)), Sha256: digest[:],
|
||||
}}}),
|
||||
marshal(&rvboxv1.AgentEnvelope{Payload: &rvboxv1.AgentEnvelope_CommandDispatch{CommandDispatch: &rvboxv1.CommandDispatch{
|
||||
IssueUuid: fresh.String(), CommandRevision: 1, TargetSessionGeneration: 1, IssueTime: timestamppb.Now(),
|
||||
ImmutableRequestSha256: digest[:], Spec: &rvboxv1.ExecutionSpec{ShellType: rvboxv1.ShellType_SHELL_CMD,
|
||||
Source: &rvboxv1.ExecutionSpec_CommandText{CommandText: "echo next"}},
|
||||
}}}),
|
||||
}}}
|
||||
runCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
dispatched, done := make(chan struct{}, 1), make(chan error, 1)
|
||||
readyCalls := 0
|
||||
metrics := observability.New()
|
||||
go func() {
|
||||
done <- serveActive(runCtx, transport, RunnerOptions{Store: store, Limits: agentproto.DefaultLimits(), Now: time.Now, Metrics: metrics,
|
||||
OnScriptReady: func(context.Context, Session, domain.UUID) error { readyCalls++; return nil },
|
||||
OnDispatch: func(context.Context, Session, *rvboxv1.CommandDispatch) error { dispatched <- struct{}{}; return nil },
|
||||
}, Session{ID: "session", Generation: 1}, map[domain.UUID]uint64{})
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("stale upload closed session: %v", err)
|
||||
case <-dispatched:
|
||||
cancel()
|
||||
<-done
|
||||
case <-runCtx.Done():
|
||||
t.Fatal("fresh command could not pass stale upload")
|
||||
}
|
||||
if readyCalls != 0 {
|
||||
t.Fatal("finished script was made launch-eligible again")
|
||||
}
|
||||
retained, err := store.ScriptBody(ctx, issue)
|
||||
if err != nil || !bytes.Equal(original, retained) {
|
||||
t.Fatalf("finished script bytes changed: %v", err)
|
||||
}
|
||||
_, _, counters := metrics.Snapshot()
|
||||
if counters["terminal_script_frame"] != 2 {
|
||||
t.Fatalf("stale frame count: %v", counters)
|
||||
}
|
||||
}
|
||||
|
||||
type scriptReplayTransport struct{ runnerTransport }
|
||||
|
||||
func (transport *scriptReplayTransport) Read(ctx context.Context) ([]byte, error) {
|
||||
transport.mu.Lock()
|
||||
if len(transport.reads) != 0 {
|
||||
frame := transport.reads[0]
|
||||
transport.reads = transport.reads[1:]
|
||||
transport.mu.Unlock()
|
||||
return frame, nil
|
||||
}
|
||||
transport.mu.Unlock()
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
@@ -20,6 +20,7 @@ initial evidence; do not restart services or resubmit work merely to inspect it.
|
||||
| Client missing or NOT_FOUND | List all client pages; verify the exact registered ID and server instance. A hostname assumption or wrong Compose project can select different state. Retention can also remove command history. |
|
||||
| Disconnected / OFFLINE / UNAVAILABLE | Separate server recovery from client connectivity. Inspect SCM, WSS hostname/path, proxy WebSocket upgrade, certificate/time/CA, DNS, reconnect logs, and pending instance conflicts. Missing optional HTTP health is not proof of a stopped daemon. |
|
||||
| `StatusMessageTooBig` / `read limited at 32769 bytes` after output or during reconnect | Early v1 WebSocket readers kept the library's 32 KiB default despite allowing larger protocol envelopes. Upgrade the server first to accept pending output, then the client for large incoming dispatch/history. Preserve spool/history; rebooting replays the same pending message. Corrected readers use the configured envelope ceiling, normally 1 MiB. |
|
||||
| Repeated `terminal command cannot accept script data` | Upload replay can race the finished command's pending event replay. Corrected clients ignore stale script frames for confirmed local terminal commands and let events drain, without modifying script bytes or relaunching. Upgrade the client and retain spool/history. A terminal-script error for a still-live command remains a protocol failure. |
|
||||
| Repeated `signal command ...: command is not running` | An older client disconnects on a late signal instead of persisting its rejection. Upgrade the client while retaining config/spool/history; corrected builds acknowledge the signal without disconnecting. Do not wipe state or reissue the original command. |
|
||||
| QUEUED persists | Check connection, supported shell, queue/running capacity, acceptance events, and queue expiry. Queuing does not promise execution, and queue TTL is not a runtime timeout. |
|
||||
| CLI deadline / broken stream | Work may still run. Query the captured issue and resume history via the API. For uncertain admission, retry only with the same request ID and payload. |
|
||||
|
||||
@@ -1163,6 +1163,12 @@ layer = "integration"
|
||||
status = "implemented"
|
||||
tests = ["test/integration/clientagent/websocket_limits_integration_test.go:TestLargeOutputReplaysAfterReconnect_REC_WS_01"]
|
||||
|
||||
[[requirements]]
|
||||
id = "REC-RUNTIME-06"
|
||||
layer = "unit"
|
||||
status = "implemented"
|
||||
tests = ["internal/client/agent/script_replay_test.go:TestFinishedScriptReplayKeepsSessionUsable_REC_RUNTIME_06"]
|
||||
|
||||
[[requirements]]
|
||||
id = "HP-WS-01"
|
||||
layer = "integration"
|
||||
|
||||
Reference in new issue
Block a user