feat: complete transfer and eviction execution

src/archive_clients/daemon.py

@@ -105,6 +105,11 @@ class ArchiveClientDaemon:
                 route_path=self._route_path,
                 syncthing_transport=self.routes.transport,
                 sparse_supported=all(probe.sparse_files for probe in probes),
+                poll_interval=config.jobs.poll_interval,
+                verification_timeout=config.jobs.verification_timeout,
+                free_space_reserve_bytes=(
+                    config.jobs.free_space_reserve_bytes
+                ),
             )
             if resource_reader is not None and self.routes is not None
             else None
@@ -415,9 +420,13 @@ class ArchiveClientDaemon:
         elif (
             accepted is not None
             and accepted_for_execution
-            and command.WhichOneof("payload") in {"assign_job", "execute_step"}
+            and command.WhichOneof("payload") in {
+                "assign_job", "execute_step", "cancel_job"
+            }
             and self.jobs is not None
         ):
+            if command.WhichOneof("payload") == "cancel_job":
+                self.jobs.request_cancel(command.cancel_job.job_id)
             self._schedule_job_command(
                 command,
                 envelope.message_id,
@@ -445,9 +454,13 @@ class ArchiveClientDaemon:
                     command, "", outbound, command_tasks
                 )
             elif (
-                command.WhichOneof("payload") in {"assign_job", "execute_step"}
+                command.WhichOneof("payload") in {
+                    "assign_job", "execute_step", "cancel_job"
+                }
                 and self.jobs is not None
             ):
+                if command.WhichOneof("payload") == "cancel_job":
+                    self.jobs.request_cancel(command.cancel_job.job_id)
                 job_commands.append(command)
         if job_commands:
             task = asyncio.create_task(
@@ -543,17 +556,39 @@ class ArchiveClientDaemon:
     ) -> None:
         assert self.jobs is not None
         payload = command.WhichOneof("payload")
+        streamed = False
         if payload == "assign_job":
             events = await asyncio.to_thread(
                 self.jobs.assign, command.assign_job
             )
         elif payload == "execute_step":
+            loop = asyncio.get_running_loop()
+
+            def emit(event):
+                response = new_envelope()
+                response.correlation_id = correlation_id
+                response.job_event.CopyFrom(event)
+                future = asyncio.run_coroutine_threadsafe(
+                    outbound.put(encode(response)), loop
+                )
+                try:
+                    future.result(timeout=30)
+                except Exception:
+                    # The event is already durable in the client DB and will
+                    # be replayed after reconnect.
+                    pass
+
             events = await asyncio.to_thread(
-                self.jobs.execute, command.execute_step
+                self.jobs.execute, command.execute_step, emit
+            )
+            streamed = True
+        elif payload == "cancel_job":
+            events = await asyncio.to_thread(
+                self.jobs.cancel, command.cancel_job
             )
         else:
             raise JobExecutionError("job command payload is unsupported")
-        for event in events:
+        for event in (() if streamed else events):
             response = new_envelope()
             response.correlation_id = correlation_id
             response.job_event.CopyFrom(event)
@@ -880,17 +915,25 @@ class ArchiveClientDaemon:
                 acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED
         elif command.WhichOneof("payload") == "assign_job":
             definition = command.assign_job.job
+            specification = definition.WhichOneof("spec")
+            assigned_to_client = (
+                specification == "transfer"
+                and self.config.client_id
+                in {
+                    definition.transfer.source_client_id,
+                    definition.transfer.target_client_id,
+                }
+            ) or (
+                specification == "eviction"
+                and self.config.client_id == definition.eviction.cache_client_id
+            )
             if self.jobs is None:
                 acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
                 acknowledgement.error.code = common_pb2.ERROR_CODE_UNAVAILABLE
                 acknowledgement.error.message = "job executor is unavailable"
             elif (
                 not definition.job_id
-                or definition.WhichOneof("spec") != "transfer"
-                or self.config.client_id not in {
-                    definition.transfer.source_client_id,
-                    definition.transfer.target_client_id,
-                }
+                or not assigned_to_client
             ):
                 acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
                 acknowledgement.error.code = common_pb2.ERROR_CODE_INVALID_ARGUMENT
@@ -911,6 +954,9 @@ class ArchiveClientDaemon:
                     job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE,
                     job_pb2.JOB_STEP_KIND_QB_VERIFY,
                     job_pb2.JOB_STEP_KIND_STAGING_CLEANUP,
+                    job_pb2.JOB_STEP_KIND_VERIFY_ARCHIVE_COVERAGE,
+                    job_pb2.JOB_STEP_KIND_QB_REMOVE_ENTRY,
+                    job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK,
                 }
             ):
                 acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED