feat: execute durable archive transfers

src/archive_clients/daemon.py

@@ -7,6 +7,7 @@ import logging
 import random
 import time
 import uuid
+from pathlib import Path, PurePosixPath
 from typing import Any
 
 from websockets.asyncio.client import connect
@@ -14,6 +15,7 @@ from websockets.asyncio.client import connect
 from archive_clients.backup import SQLiteBackupManager
 from archive_clients.config import ClientConfig
 from archive_clients.inventory import InventoryService
+from archive_clients.jobs import ClientJobExecutor, JobExecutionError
 from archive_clients.locking import DatabaseLease
 from archive_clients.probes import FilesystemProbe
 from archive_clients.protocol import (
@@ -73,12 +75,40 @@ class ArchiveClientDaemon:
             if healthy_syncthing and probes[1].writable
             else None
         )
+        self._known_route_paths: dict[str, Path] = {}
+        for probe in service_probes:
+            if probe.service != "syncthing":
+                continue
+            for route in probe.routes:
+                api_path = (
+                    config.syncthing.api_root
+                    / PurePosixPath(route.local_relative_path)
+                ).as_posix()
+                self._known_route_paths[route.route_id] = (
+                    config.syncthing.roots.api_to_local(api_path)
+                )
         self.store = ClientStore(config.state_db)
         self.backups = SQLiteBackupManager(
             config.state_db, config.backup_dir, config.backup
         )
         self._lease = DatabaseLease(config.state_db)
         self._active_route_commands: set[str] = set()
+        self._active_job_commands: set[str] = set()
+        self._job_execution_lock = asyncio.Lock()
+        self.jobs = (
+            ClientJobExecutor(
+                client_id=config.client_id,
+                qbittorrent=resource_reader,
+                store=self.store,
+                qb_root=config.qbittorrent.local_root,
+                qb_api_root=config.qbittorrent.api_root,
+                route_path=self._route_path,
+                syncthing_transport=self.routes.transport,
+                sparse_supported=all(probe.sparse_files for probe in probes),
+            )
+            if resource_reader is not None and self.routes is not None
+            else None
+        )
 
     async def run(self) -> None:
         await asyncio.to_thread(self._lease.acquire)
@@ -169,7 +199,7 @@ class ArchiveClientDaemon:
             writer = asyncio.create_task(self._writer(websocket, outbound))
             command_tasks: set[asyncio.Task[None]] = set()
             try:
-                await self._resume_route_commands(outbound, command_tasks)
+                await self._resume_commands(outbound, command_tasks)
                 async for frame in websocket:
                     await self._handle(decode(frame), outbound, command_tasks)
             finally:
@@ -382,12 +412,25 @@ class ArchiveClientDaemon:
                 outbound,
                 command_tasks,
             )
+        elif (
+            accepted is not None
+            and accepted_for_execution
+            and command.WhichOneof("payload") in {"assign_job", "execute_step"}
+            and self.jobs is not None
+        ):
+            self._schedule_job_command(
+                command,
+                envelope.message_id,
+                outbound,
+                command_tasks,
+            )
 
-    async def _resume_route_commands(
+    async def _resume_commands(
         self,
         outbound: asyncio.Queue[str],
         command_tasks: set[asyncio.Task[None]],
     ) -> None:
+        job_commands: list[control_pb2.Command] = []
         for row in await asyncio.to_thread(self.store.list_accepted_commands):
             acknowledgement = decode_message(
                 str(row["acknowledgement_json"]), control_pb2.CommandAck()
@@ -401,6 +444,120 @@ class ArchiveClientDaemon:
                 self._schedule_route_command(
                     command, "", outbound, command_tasks
                 )
+            elif (
+                command.WhichOneof("payload") in {"assign_job", "execute_step"}
+                and self.jobs is not None
+            ):
+                job_commands.append(command)
+        if job_commands:
+            task = asyncio.create_task(
+                self._resume_job_commands(job_commands, outbound),
+                name="resume-job-commands",
+            )
+            command_tasks.add(task)
+            task.add_done_callback(
+                lambda completed: self._command_finished(
+                    completed, command_tasks
+                )
+            )
+
+    async def _resume_route_commands(
+        self,
+        outbound: asyncio.Queue[str],
+        command_tasks: set[asyncio.Task[None]],
+    ) -> None:
+        """Backward-compatible test hook."""
+
+        await self._resume_commands(outbound, command_tasks)
+
+    def _schedule_job_command(
+        self,
+        command: control_pb2.Command,
+        correlation_id: str,
+        outbound: asyncio.Queue[str],
+        command_tasks: set[asyncio.Task[None]],
+    ) -> None:
+        if command.command_id in self._active_job_commands:
+            return
+        self._active_job_commands.add(command.command_id)
+        task = asyncio.create_task(
+            self._execute_job_command(command, correlation_id, outbound),
+            name=f"job-command-{command.command_id}",
+        )
+        command_tasks.add(task)
+        task.add_done_callback(
+            lambda completed: self._job_command_finished(
+                command.command_id, completed, command_tasks
+            )
+        )
+
+    async def _resume_job_commands(
+        self,
+        commands: list[control_pb2.Command],
+        outbound: asyncio.Queue[str],
+    ) -> None:
+        for command in commands:
+            if command.command_id in self._active_job_commands:
+                continue
+            self._active_job_commands.add(command.command_id)
+            try:
+                await self._execute_job_command(command, "", outbound)
+            except asyncio.CancelledError:
+                raise
+            except Exception as error:
+                logger.error(
+                    "background_command_failed",
+                    extra={
+                        "error_type": type(error).__name__,
+                        "error_detail": str(error),
+                    },
+                )
+            finally:
+                self._active_job_commands.discard(command.command_id)
+
+    def _job_command_finished(
+        self,
+        command_id: str,
+        task: asyncio.Task[None],
+        command_tasks: set[asyncio.Task[None]],
+    ) -> None:
+        self._active_job_commands.discard(command_id)
+        self._command_finished(task, command_tasks)
+
+    async def _execute_job_command(
+        self,
+        command: control_pb2.Command,
+        correlation_id: str,
+        outbound: asyncio.Queue[str],
+    ) -> None:
+        async with self._job_execution_lock:
+            await self._execute_job_command_locked(
+                command, correlation_id, outbound
+            )
+
+    async def _execute_job_command_locked(
+        self,
+        command: control_pb2.Command,
+        correlation_id: str,
+        outbound: asyncio.Queue[str],
+    ) -> None:
+        assert self.jobs is not None
+        payload = command.WhichOneof("payload")
+        if payload == "assign_job":
+            events = await asyncio.to_thread(
+                self.jobs.assign, command.assign_job
+            )
+        elif payload == "execute_step":
+            events = await asyncio.to_thread(
+                self.jobs.execute, command.execute_step
+            )
+        else:
+            raise JobExecutionError("job command payload is unsupported")
+        for event in events:
+            response = new_envelope()
+            response.correlation_id = correlation_id
+            response.job_event.CopyFrom(event)
+            await outbound.put(encode(response))
 
     def _schedule_route_command(
         self,
@@ -495,6 +652,7 @@ class ArchiveClientDaemon:
             configured = await asyncio.to_thread(
                 self.routes.configure, spec, deadline
             )
+            self._known_route_paths[spec.route_id] = configured.local_path
             await asyncio.to_thread(
                 self.store.record_route_ownership,
                 command.command_id,
@@ -658,7 +816,10 @@ class ArchiveClientDaemon:
         if error is not None:
             logger.error(
                 "background_command_failed",
-                extra={"error_type": type(error).__name__},
+                extra={
+                    "error_type": type(error).__name__,
+                    "error_detail": str(error),
+                },
             )
 
     def _initial_acknowledgement(
@@ -717,6 +878,46 @@ class ArchiveClientDaemon:
                 acknowledgement.error.message = "ensure route specification is invalid"
             else:
                 acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED
+        elif command.WhichOneof("payload") == "assign_job":
+            definition = command.assign_job.job
+            if self.jobs is None:
+                acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
+                acknowledgement.error.code = common_pb2.ERROR_CODE_UNAVAILABLE
+                acknowledgement.error.message = "job executor is unavailable"
+            elif (
+                not definition.job_id
+                or definition.WhichOneof("spec") != "transfer"
+                or self.config.client_id not in {
+                    definition.transfer.source_client_id,
+                    definition.transfer.target_client_id,
+                }
+            ):
+                acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
+                acknowledgement.error.code = common_pb2.ERROR_CODE_INVALID_ARGUMENT
+                acknowledgement.error.message = "job assignment is invalid"
+            else:
+                acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED
+        elif command.WhichOneof("payload") == "execute_step":
+            step = command.execute_step
+            if self.jobs is None:
+                acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
+                acknowledgement.error.code = common_pb2.ERROR_CODE_UNAVAILABLE
+                acknowledgement.error.message = "job executor is unavailable"
+            elif (
+                not step.job_id
+                or step.step not in {
+                    job_pb2.JOB_STEP_KIND_SOURCE_STAGE,
+                    job_pb2.JOB_STEP_KIND_SYNCTHING_TRANSFER,
+                    job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE,
+                    job_pb2.JOB_STEP_KIND_QB_VERIFY,
+                    job_pb2.JOB_STEP_KIND_STAGING_CLEANUP,
+                }
+            ):
+                acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
+                acknowledgement.error.code = common_pb2.ERROR_CODE_INVALID_ARGUMENT
+                acknowledgement.error.message = "job step is invalid"
+            else:
+                acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED
         else:
             acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
             acknowledgement.error.code = common_pb2.ERROR_CODE_UNSUPPORTED
@@ -725,6 +926,12 @@ class ArchiveClientDaemon:
             )
         return acknowledgement
 
+    def _route_path(self, route_id: str) -> Path:
+        try:
+            return self._known_route_paths[route_id]
+        except KeyError as exc:
+            raise JobExecutionError("job route is not configured locally") from exc
+
 
 def _route_error(exc: Exception) -> tuple[int, bool]:
     if isinstance(exc, RoutePathConflict):