feat: complete Windows client control and recovery paths
This commit is contained in:
@@ -129,6 +129,10 @@ func (executor *Executor) launch(ctx context.Context, issue domain.UUID, revisio
|
||||
if err != nil {
|
||||
cancel()
|
||||
_ = executor.Store.SetLaunchPhase(context.Background(), issue, domain.LaunchPhaseNone, "", 0)
|
||||
var startErr *supervisor.StartError
|
||||
if errors.As(err, &startErr) {
|
||||
return executor.rejectWithIdentity(ctx, issue, revision, err, startErr.WindowsIdentity)
|
||||
}
|
||||
return executor.reject(ctx, issue, revision, err)
|
||||
}
|
||||
identity := process.Identity()
|
||||
@@ -141,7 +145,7 @@ func (executor *Executor) launch(ctx context.Context, issue domain.UUID, revisio
|
||||
executor.active[issue] = process
|
||||
executor.cancel[issue] = cancel
|
||||
executor.mu.Unlock()
|
||||
if _, err := executor.Store.AppendLifecycle(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), revision, "process started", executor.Now()); err != nil {
|
||||
if _, err := executor.Store.AppendLifecycleWithIdentity(ctx, issue, uint32(rvboxv1.CommandLifecycle_COMMAND_RUNNING), revision, "process started", executor.Now(), identity.WindowsIdentity()); err != nil {
|
||||
_, _ = executor.Supervisor.Signal(context.Background(), process, supervisor.SignalKill)
|
||||
cancel()
|
||||
executor.remove(issue)
|
||||
@@ -187,22 +191,30 @@ func (executor *Executor) watch(ctx context.Context, issue domain.UUID, revision
|
||||
if waitErr != nil {
|
||||
detail = boundedError(waitErr)
|
||||
}
|
||||
if err := executor.appendLifecycle(context.Background(), issue, revision, phase, detail); err == nil {
|
||||
if err := executor.appendLifecycleWithIdentity(context.Background(), issue, revision, phase, detail, process.Identity().WindowsIdentity()); err == nil {
|
||||
executor.notify(issue)
|
||||
}
|
||||
executor.remove(issue)
|
||||
}
|
||||
|
||||
func (executor *Executor) appendLifecycle(ctx context.Context, issue domain.UUID, revision uint64, phase rvboxv1.CommandLifecycle, detail string) error {
|
||||
_, err := executor.Store.AppendLifecycle(ctx, issue, uint32(phase), revision, detail, executor.Now())
|
||||
return executor.appendLifecycleWithIdentity(ctx, issue, revision, phase, detail, nil)
|
||||
}
|
||||
|
||||
func (executor *Executor) appendLifecycleWithIdentity(ctx context.Context, issue domain.UUID, revision uint64, phase rvboxv1.CommandLifecycle, detail string, identity *rvboxv1.WindowsExecutionIdentity) error {
|
||||
_, err := executor.Store.AppendLifecycleWithIdentity(ctx, issue, uint32(phase), revision, detail, executor.Now(), identity)
|
||||
return err
|
||||
}
|
||||
|
||||
func (executor *Executor) reject(ctx context.Context, issue domain.UUID, revision uint64, cause error) error {
|
||||
return executor.rejectWithIdentity(ctx, issue, revision, cause, nil)
|
||||
}
|
||||
|
||||
func (executor *Executor) rejectWithIdentity(ctx context.Context, issue domain.UUID, revision uint64, cause error, identity *rvboxv1.WindowsExecutionIdentity) error {
|
||||
if revision == 0 {
|
||||
return cause
|
||||
}
|
||||
if err := executor.appendLifecycle(ctx, issue, revision, rvboxv1.CommandLifecycle_COMMAND_REJECTED, boundedError(cause)); err != nil {
|
||||
if err := executor.appendLifecycleWithIdentity(ctx, issue, revision, rvboxv1.CommandLifecycle_COMMAND_REJECTED, boundedError(cause), identity); err != nil {
|
||||
return err
|
||||
}
|
||||
executor.notify(issue)
|
||||
|
||||
@@ -94,12 +94,12 @@ func Run(ctx context.Context, options RunnerOptions) error {
|
||||
if err := waitUntil(ctx, delay); err != nil {
|
||||
return nil
|
||||
}
|
||||
sessionStarted := time.Now()
|
||||
sessionStarted := options.Now()
|
||||
_ = runOnce(ctx, options)
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if time.Since(sessionStarted) >= options.Backoff.StableReset {
|
||||
if options.Now().Sub(sessionStarted) >= options.Backoff.StableReset {
|
||||
failures = 0
|
||||
}
|
||||
if failures < ^uint32(0) {
|
||||
@@ -357,11 +357,11 @@ func serveActive(ctx context.Context, transport Transport, options RunnerOptions
|
||||
return err
|
||||
}
|
||||
case envelope.GetScriptChunk() != nil:
|
||||
if err := handleScriptChunk(ctx, transport, options.Store, session, envelope.GetScriptChunk(), limits); err != nil {
|
||||
if err := handleScriptChunk(ctx, transport, options.Store, session, envelope.GetScriptChunk(), limits, options.Now, sent); err != nil {
|
||||
return err
|
||||
}
|
||||
case envelope.GetScriptCommit() != nil:
|
||||
if err := handleScriptCommit(ctx, transport, options.Store, session, envelope.GetScriptCommit(), limits); err != nil {
|
||||
if err := handleScriptCommit(ctx, transport, options.Store, session, envelope.GetScriptCommit(), limits, options.Now, sent); err != nil {
|
||||
return err
|
||||
}
|
||||
if options.OnScriptReady != nil {
|
||||
@@ -488,28 +488,38 @@ func boundedError(err error) string {
|
||||
return message
|
||||
}
|
||||
|
||||
func handleScriptChunk(ctx context.Context, transport Transport, store *spool.Store, session Session, chunk *rvboxv1.ScriptChunk, limits agentproto.Limits) error {
|
||||
func handleScriptChunk(ctx context.Context, transport Transport, store *spool.Store, session Session, chunk *rvboxv1.ScriptChunk, limits agentproto.Limits, now func() time.Time, sent map[domain.UUID]uint64) error {
|
||||
status, err := ApplyScriptChunk(ctx, store, session, chunk, limits)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return appendAndSendScriptStatus(ctx, transport, store, session, chunk.GetIssueUuid(), status, limits)
|
||||
return appendAndSendScriptStatus(ctx, transport, store, session, chunk.GetIssueUuid(), status, limits, now, sent)
|
||||
}
|
||||
|
||||
func handleScriptCommit(ctx context.Context, transport Transport, store *spool.Store, session Session, commit *rvboxv1.ScriptCommit, limits agentproto.Limits) error {
|
||||
func handleScriptCommit(ctx context.Context, transport Transport, store *spool.Store, session Session, commit *rvboxv1.ScriptCommit, limits agentproto.Limits, now func() time.Time, sent map[domain.UUID]uint64) error {
|
||||
status, err := ApplyScriptCommit(ctx, store, session, commit, limits)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return appendAndSendScriptStatus(ctx, transport, store, session, commit.GetIssueUuid(), status, limits)
|
||||
return appendAndSendScriptStatus(ctx, transport, store, session, commit.GetIssueUuid(), status, limits, now, sent)
|
||||
}
|
||||
|
||||
func appendAndSendScriptStatus(ctx context.Context, transport Transport, store *spool.Store, session Session, issueText string, status spool.ScriptStatus, limits agentproto.Limits) error {
|
||||
func appendAndSendScriptStatus(ctx context.Context, transport Transport, store *spool.Store, session Session, issueText string, status spool.ScriptStatus, limits agentproto.Limits, now func() time.Time, sent map[domain.UUID]uint64) error {
|
||||
issue, err := domain.ParseUUIDv7(issueText)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
event := &rvboxv1.CommandEvent{IssueUuid: issueText, ObservedAt: timestamppb.New(time.Now().UTC()), Payload: &rvboxv1.CommandEvent_ScriptStatus{ScriptStatus: &rvboxv1.ScriptUploadStatus{ReceivedBytes: status.ReceivedBytes, Complete: status.Committed}}}
|
||||
if sent == nil {
|
||||
sent = make(map[domain.UUID]uint64)
|
||||
}
|
||||
if now == nil {
|
||||
now = func() time.Time { return time.Now().UTC() }
|
||||
}
|
||||
observedAt := now()
|
||||
if observedAt.IsZero() {
|
||||
return errors.New("script status clock returned zero")
|
||||
}
|
||||
event := &rvboxv1.CommandEvent{IssueUuid: issueText, ObservedAt: timestamppb.New(observedAt), Payload: &rvboxv1.CommandEvent_ScriptStatus{ScriptStatus: &rvboxv1.ScriptUploadStatus{ReceivedBytes: status.ReceivedBytes, Complete: status.Committed}}}
|
||||
payload, err := proto.MarshalOptions{Deterministic: true}.Marshal(event)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -522,9 +532,13 @@ func appendAndSendScriptStatus(ctx context.Context, transport Transport, store *
|
||||
return err
|
||||
}
|
||||
for _, item := range assigned {
|
||||
if item.EventSeq == 0 || item.EventSeq <= sent[issue] {
|
||||
continue
|
||||
}
|
||||
if err := SendStoredEvent(ctx, transport, session, item, limits); err != nil {
|
||||
return err
|
||||
}
|
||||
sent[issue] = item.EventSeq
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user