fix: bind client job events to command leases

src/archive_clients/jobs.py

@@ -97,33 +97,25 @@ class ClientJobExecutor:
     ) -> list[control_pb2.JobEvent]:
         definition = command.job
         self._validate_definition(definition)
-        replay = self._replay(
-            definition.job_id, command.expected_last_event_sequence
+        self.store.ensure_job_definition(
+            definition.job_id, encode_message(definition)
         )
-        if replay:
-            return replay
-        event = self._event(
-            definition,
-            sequence=command.expected_last_event_sequence + 1,
-            revision=command.expected_job_revision,
-            event_type=control_pb2.JOB_EVENT_TYPE_ASSIGNED,
-            state=job_pb2.JOB_STATE_PREPARING,
-            committed=False,
-        )
-        self._record(definition, event)
-        return [event]
+        # Assignment succeeds through CommandAck. It must not let a client
+        # allocate a globally ordered JobEvent cursor.
+        return []
 
     def execute(
         self,
         command: control_pb2.ExecuteStepCommand,
         event_callback: Callable[[control_pb2.JobEvent], None] | None = None,
+        command_id: str = "",
     ) -> list[control_pb2.JobEvent]:
         # asyncio cancellation of a connection-bound task cannot stop the
         # synchronous filesystem/qB operation already running in its worker
         # thread. A replay after reconnect therefore waits for that operation
         # and then reads its durable event journal instead of executing twice.
         with self._execution_lock(command.job_id):
-            return self._execute_locked(command, event_callback)
+            return self._execute_locked(command, event_callback, command_id)
 
     def _execution_lock(self, job_id: str) -> threading.Lock:
         with self._execution_locks_guard:
@@ -133,6 +125,7 @@ class ClientJobExecutor:
         self,
         command: control_pb2.ExecuteStepCommand,
         event_callback: Callable[[control_pb2.JobEvent], None] | None = None,
+        command_id: str = "",
     ) -> list[control_pb2.JobEvent]:
         definition = self._definition(command.job_id)
         replay = self._replay(
@@ -186,6 +179,7 @@ class ClientJobExecutor:
                 ),
                 step=command.step,
                 step_state=job_pb2.STEP_STATE_RUNNING,
+                command_id=command_id,
             )
             self._record(definition, started)
             cursor = started
@@ -228,6 +222,7 @@ class ClientJobExecutor:
                 committed=cursor.committed,
                 step=command.step,
                 step_state=job_pb2.STEP_STATE_RUNNING,
+                command_id=command_id,
             )
             event.progress.fraction_complete = max(
                 0.0, min(float(fraction), 1.0)
@@ -258,6 +253,7 @@ class ClientJobExecutor:
                 committed=started.committed,
                 step=command.step,
                 step_state=job_pb2.STEP_STATE_CANCELLED,
+                command_id=command_id,
             )
             cancelling.error.code = common_pb2.ERROR_CODE_CANCELLED
             cancelling.error.message = str(error)
@@ -315,6 +311,7 @@ class ClientJobExecutor:
                 committed=started.committed,
                 step=command.step,
                 step_state=job_pb2.STEP_STATE_FAILED,
+                command_id=command_id,
             )
             failed.error.code = _job_error_code(error)
             failed.error.message = str(error) or type(error).__name__
@@ -357,6 +354,7 @@ class ClientJobExecutor:
             committed=committed,
             step=command.step,
             step_state=job_pb2.STEP_STATE_SUCCEEDED,
+            command_id=command_id,
         )
         if result is not None:
             succeeded.observed_placement.CopyFrom(result)
@@ -367,7 +365,7 @@ class ClientJobExecutor:
         return emitted
 
     def cancel(
-        self, command: control_pb2.CancelJobCommand
+        self, command: control_pb2.CancelJobCommand, command_id: str = ""
     ) -> list[control_pb2.JobEvent]:
         definition = self._definition(command.job_id)
         self.request_cancel(command.job_id)
@@ -418,6 +416,7 @@ class ClientJobExecutor:
             committed=committed,
             step=job_pb2.JOB_STEP_KIND_ROLLBACK,
             step_state=job_pb2.STEP_STATE_SUCCEEDED,
+            command_id=command_id,
         )
         if placement is not None:
             event.observed_placement.CopyFrom(placement)
@@ -1050,6 +1049,7 @@ class ClientJobExecutor:
         committed: bool,
         step: int = job_pb2.JOB_STEP_KIND_UNSPECIFIED,
         step_state: int = job_pb2.STEP_STATE_UNSPECIFIED,
+        command_id: str = "",
     ) -> control_pb2.JobEvent:
         event = control_pb2.JobEvent(
             event_id=str(uuid.uuid4()),
@@ -1059,6 +1059,7 @@ class ClientJobExecutor:
             type=event_type,
             state=state,
             committed=committed,
+            command_id=command_id,
         )
         event.occurred_at.GetCurrentTime()
         if step != job_pb2.JOB_STEP_KIND_UNSPECIFIED: