"""Reconnectable Archive Control client transport.""" from __future__ import annotations import asyncio import logging import random import time import uuid from pathlib import Path, PurePosixPath from typing import Any from websockets.asyncio.client import connect from websockets.exceptions import ConnectionClosedOK from archive_clients.backup import SQLiteBackupManager from archive_clients.config import ClientConfig from archive_clients.inventory import InventoryService from archive_clients.jobs import ClientJobExecutor, JobExecutionError from archive_clients.locking import DatabaseLease from archive_clients.probes import FilesystemProbe from archive_clients.protocol import ( decode, decode_message, encode, encode_message, new_envelope, ) from archive_clients.qbittorrent import QBittorrentReader from archive_clients.services import ServiceProbe from archive_clients.state import ClientStore, CommandConflict from archive_clients.syncthing import ( RoutePathConflict, RouteSetupError, RouteSetupTimeout, SyncthingRouteManager, ) from archive_control.v1 import ( client_pb2, common_pb2, control_pb2, inventory_pb2, job_pb2, route_pb2, ) logger = logging.getLogger(__name__) class ArchiveClientDaemon: def __init__( self, config: ClientConfig, probes: list[FilesystemProbe], service_probes: list[ServiceProbe], resource_reader: QBittorrentReader | None = None, route_manager: SyncthingRouteManager | None = None, ): if len(probes) != 2: raise ValueError( "qBittorrent and Syncthing filesystem probes are required" ) self.config = config self.probes = probes self.service_probes = service_probes self.inventory = ( InventoryService(resource_reader, config.client_id) if resource_reader is not None else None ) healthy_syncthing = any( probe.service == "syncthing" and probe.state == common_pb2.HEALTH_STATE_HEALTHY for probe in service_probes ) self.routes = route_manager or ( SyncthingRouteManager( config.syncthing, sparse_supported=probes[1].sparse_files, ) if healthy_syncthing and probes[1].writable else None ) self._known_route_paths: dict[str, Path] = {} for probe in service_probes: if probe.service != "syncthing": continue for route in probe.routes: api_path = ( config.syncthing.api_root / PurePosixPath(route.local_relative_path) ).as_posix() self._known_route_paths[route.route_id] = ( config.syncthing.roots.api_to_local(api_path) ) self.store = ClientStore(config.state_db) self.backups = SQLiteBackupManager( config.state_db, config.backup_dir, config.backup ) self._lease = DatabaseLease(config.state_db) self._active_route_commands: set[str] = set() self._active_job_commands: set[str] = set() self._job_execution_lock = asyncio.Lock() self.jobs = ( ClientJobExecutor( client_id=config.client_id, qbittorrent=resource_reader, store=self.store, qb_root=config.qbittorrent.local_root, qb_api_root=config.qbittorrent.api_root, route_path=self._route_path, syncthing_transport=self.routes.transport, sparse_supported=all(probe.sparse_files for probe in probes), poll_interval=config.jobs.poll_interval, verification_timeout=config.jobs.verification_timeout, free_space_reserve_bytes=( config.jobs.free_space_reserve_bytes ), ) if resource_reader is not None and self.routes is not None else None ) async def run(self) -> None: await asyncio.to_thread(self._lease.acquire) backup_task: asyncio.Task[None] | None = None try: await asyncio.to_thread(self.store.initialize) await asyncio.to_thread(self._warn_if_backup_shares_filesystem) backup_task = asyncio.create_task( self._backup_loop(), name="archive-client-backups" ) await self._connection_loop() finally: if backup_task is not None: backup_task.cancel() await asyncio.gather(backup_task, return_exceptions=True) await asyncio.to_thread(self._lease.release) def _warn_if_backup_shares_filesystem(self) -> None: if ( self.config.state_db.parent.stat().st_dev == self.config.backup_dir.stat().st_dev ): logger.warning("database_and_backup_share_filesystem") async def _connection_loop(self) -> None: delay = self.config.connection.reconnect_initial while True: started = time.monotonic() try: await self._connection() except asyncio.CancelledError: raise except Exception as exc: logger.warning( "control_connection_ended", extra={"error_type": type(exc).__name__}, ) if ( time.monotonic() - started >= self.config.connection.reconnect_reset_after ): delay = self.config.connection.reconnect_initial wait = ( random.uniform(0, delay) if self.config.connection.reconnect_jitter else delay ) await asyncio.sleep(wait) delay = min(delay * 2, self.config.connection.reconnect_max) async def _backup_loop(self) -> None: while True: await asyncio.sleep(self.config.backup.interval) try: record = await asyncio.to_thread(self.backups.create, "scheduled") logger.info( "database_backup_created", extra={"backup_name": record.database.name}, ) except asyncio.CancelledError: raise except Exception as exc: logger.error( "database_backup_failed", extra={"error_type": type(exc).__name__}, ) async def _connection(self) -> None: async with connect( self.config.control_endpoint, ping_interval=None, compression=None, max_size=1024 * 1024, ) as websocket: registration = self._registration() await websocket.send(encode(registration)) registration.register_request.shared_token = "" response = decode(await asyncio.wait_for( websocket.recv(), self.config.connection.registration_timeout )) if response.WhichOneof("payload") != "register_response": raise RuntimeError("control did not answer registration") if response.correlation_id != registration.message_id: raise RuntimeError("registration response correlation mismatch") if response.register_response.status != client_pb2.REGISTRATION_STATUS_ACCEPTED: raise RuntimeError("control rejected registration") if response.register_response.negotiated_version.major != 1: raise RuntimeError("control negotiated an unsupported protocol version") logger.info("control_connection_registered") outbound: asyncio.Queue[str] = asyncio.Queue(maxsize=100) writer = asyncio.create_task(self._writer(websocket, outbound)) command_tasks: set[asyncio.Task[None]] = set() try: await self._resume_commands(outbound, command_tasks) # A proxy can leave the TCP/WebSocket socket apparently open # after the control server has discarded its session. The # server then cannot deliver durable commands and its pending # outbox remains stranded unless the client independently # detects the missing application heartbeats and reconnects. while True: try: frame = await asyncio.wait_for( websocket.recv(), self.config.connection.offline_timeout, ) except ConnectionClosedOK: return except asyncio.TimeoutError as exc: raise RuntimeError( "control heartbeat timed out" ) from exc await self._handle(decode(frame), outbound, command_tasks) finally: writer.cancel() for task in command_tasks: task.cancel() await asyncio.gather( writer, *command_tasks, return_exceptions=True ) def _registration(self): envelope = new_envelope() request = envelope.register_request request.protocol_version.CopyFrom(envelope.protocol_version) request.client.client_id = self.config.client_id request.client.display_name = self.config.display_name request.client.role = ( common_pb2.CLIENT_ROLE_ARCHIVE if self.config.role == "archive" else common_pb2.CLIENT_ROLE_CACHE ) request.connection_instance_id = str(uuid.uuid4()) request.shared_token = self.config.read_shared_token() request.capabilities.max_envelope_bytes = 1024 * 1024 request.capabilities.syncthing_advertised_addresses.extend( self.config.syncthing.advertised_addresses ) for probe in self.service_probes: health = request.capabilities.services.add() health.service = probe.service health.state = probe.state health.version = probe.version health.api_version = probe.api_version health.detail = probe.detail health.checked_at.FromDatetime(probe.checked_at) if probe.service == "qbittorrent": request.capabilities.qbittorrent_version = probe.version request.capabilities.qbittorrent_web_api_version = ( probe.api_version ) request.capabilities.libtorrent_version = probe.libtorrent_version elif probe.service == "syncthing": request.capabilities.syncthing_version = probe.version request.capabilities.syncthing_device_id = probe.device_id request.capabilities.routes.extend(probe.routes) for root_name, probe in zip( ("qbittorrent", "syncthing"), self.probes, strict=True ): filesystem = request.capabilities.filesystems.add() filesystem.root_name = root_name filesystem.readable = probe.readable filesystem.writable = probe.writable filesystem.hard_link = probe.hard_link filesystem.reflink = probe.reflink filesystem.sparse_files = probe.sparse_files if all(probe.hard_link for probe in self.probes): request.capabilities.features.append(client_pb2.CLIENT_FEATURE_HARD_LINK) if all(probe.reflink for probe in self.probes): request.capabilities.features.append(client_pb2.CLIENT_FEATURE_REFLINK) if all(probe.sparse_files for probe in self.probes): request.capabilities.features.append(client_pb2.CLIENT_FEATURE_SPARSE_FILES) request.capabilities.features.append(client_pb2.CLIENT_FEATURE_DB_BACKUP) if self.inventory is not None: request.capabilities.features.extend(( client_pb2.CLIENT_FEATURE_INVENTORY_CHUNKS, client_pb2.CLIENT_FEATURE_CONTENT_TREE, )) if self.routes is not None: request.capabilities.features.append( client_pb2.CLIENT_FEATURE_ROUTE_PROVISIONING ) for cursor in self.store.list_active_job_cursors(): active = request.active_jobs.add() active.job_id = str(cursor["job_id"]) active.job_revision = int(cursor["revision"]) active.last_event_sequence = int(cursor["last_event_sequence"]) active.state = job_pb2.JobState.Value(str(cursor["state"])) active.committed = bool(cursor["committed"]) return envelope async def _writer( self, websocket: Any, outbound: asyncio.Queue[str] ) -> None: while True: await websocket.send(await outbound.get()) async def _handle( self, envelope: Any, outbound: asyncio.Queue[str], command_tasks: set[asyncio.Task[None]], ) -> None: payload = envelope.WhichOneof("payload") if payload == "heartbeat": response = new_envelope() response.correlation_id = envelope.message_id response.heartbeat_ack.sequence = envelope.heartbeat.sequence await outbound.put(encode(response)) elif payload == "command": await self._accept_command(envelope, outbound, command_tasks) elif payload == "protocol_error": logger.warning( "control_reported_protocol_error", extra={"error_code": envelope.protocol_error.error.code}, ) async def _accept_command( self, envelope: Any, outbound: asyncio.Queue[str], command_tasks: set[asyncio.Task[None]], ) -> None: command = envelope.command snapshot_rows: list[dict[str, object]] = [] if command.WhichOneof("payload") == "request_job_snapshot": requested = list(command.request_job_snapshot.job_ids) snapshot_rows = await asyncio.to_thread( self.store.job_snapshot_rows, requested ) missing = set(requested) - { str(row["job_id"]) for row in snapshot_rows } else: requested = [] missing = set() acknowledgement = self._initial_acknowledgement( command, missing, bool(requested) ) accepted = None accepted_for_execution = False try: accepted = await asyncio.to_thread( self.store.accept_command, command.command_id, encode_message(command), encode_message(acknowledgement), ) if accepted.duplicate: acknowledgement = decode_message( accepted.acknowledgement_json, control_pb2.CommandAck() ) if ( acknowledgement.status == control_pb2.COMMAND_ACK_STATUS_ACCEPTED ): accepted_for_execution = True acknowledgement.status = ( control_pb2.COMMAND_ACK_STATUS_DUPLICATE ) else: accepted_for_execution = ( acknowledgement.status == control_pb2.COMMAND_ACK_STATUS_ACCEPTED ) except CommandConflict: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.error.code = common_pb2.ERROR_CODE_CONFLICT acknowledgement.error.message = "command ID content conflict" response = new_envelope() response.correlation_id = envelope.message_id response.command_ack.CopyFrom(acknowledgement) await outbound.put(encode(response)) if ( accepted is not None and accepted_for_execution and command.WhichOneof("payload") == "request_job_snapshot" ): for row in snapshot_rows: snapshot = new_envelope() snapshot.correlation_id = envelope.message_id job_snapshot = snapshot.job_snapshot job_snapshot.job.definition.CopyFrom(decode_message( str(row["definition_json"]), job_pb2.JobDefinition() )) job_snapshot.job.state = job_pb2.JobState.Value(str(row["state"])) job_snapshot.job.revision = int(row["revision"]) job_snapshot.job.committed = bool(row["committed"]) job_snapshot.job.updated_at.CopyFrom(snapshot.sent_at) job_snapshot.last_event_sequence = int( row["last_event_sequence"] ) await outbound.put(encode(snapshot)) elif ( accepted is not None and accepted_for_execution and command.WhichOneof("payload") == "inventory_query" and self.inventory is not None ): task = asyncio.create_task( self._send_inventory( command.inventory_query, envelope.message_id, outbound ), name=f"inventory-{command.inventory_query.query_id}", ) command_tasks.add(task) task.add_done_callback( lambda completed: self._command_finished( completed, command_tasks ) ) elif ( accepted is not None and accepted_for_execution and command.WhichOneof("payload") == "ensure_route" and self.routes is not None ): self._schedule_route_command( command, envelope.message_id, outbound, command_tasks, ) elif ( accepted is not None and accepted_for_execution and command.WhichOneof("payload") in { "assign_job", "execute_step", "cancel_job" } and self.jobs is not None ): if command.WhichOneof("payload") == "cancel_job": self.jobs.request_cancel(command.cancel_job.job_id) self._schedule_job_command( command, envelope.message_id, outbound, command_tasks, ) async def _resume_commands( self, outbound: asyncio.Queue[str], command_tasks: set[asyncio.Task[None]], ) -> None: job_commands: list[control_pb2.Command] = [] for row in await asyncio.to_thread(self.store.list_accepted_commands): acknowledgement = decode_message( str(row["acknowledgement_json"]), control_pb2.CommandAck() ) if acknowledgement.status != control_pb2.COMMAND_ACK_STATUS_ACCEPTED: continue command = decode_message( str(row["command_json"]), control_pb2.Command() ) if command.WhichOneof("payload") == "ensure_route": self._schedule_route_command( command, "", outbound, command_tasks ) elif ( command.WhichOneof("payload") in { "assign_job", "execute_step", "cancel_job" } and self.jobs is not None ): if command.WhichOneof("payload") == "cancel_job": self.jobs.request_cancel(command.cancel_job.job_id) job_commands.append(command) if job_commands: task = asyncio.create_task( self._resume_job_commands(job_commands, outbound), name="resume-job-commands", ) command_tasks.add(task) task.add_done_callback( lambda completed: self._command_finished( completed, command_tasks ) ) async def _resume_route_commands( self, outbound: asyncio.Queue[str], command_tasks: set[asyncio.Task[None]], ) -> None: """Backward-compatible test hook.""" await self._resume_commands(outbound, command_tasks) def _schedule_job_command( self, command: control_pb2.Command, correlation_id: str, outbound: asyncio.Queue[str], command_tasks: set[asyncio.Task[None]], ) -> None: if command.command_id in self._active_job_commands: return self._active_job_commands.add(command.command_id) task = asyncio.create_task( self._execute_job_command(command, correlation_id, outbound), name=f"job-command-{command.command_id}", ) command_tasks.add(task) task.add_done_callback( lambda completed: self._job_command_finished( command.command_id, completed, command_tasks ) ) async def _resume_job_commands( self, commands: list[control_pb2.Command], outbound: asyncio.Queue[str], ) -> None: for command in commands: if command.command_id in self._active_job_commands: continue self._active_job_commands.add(command.command_id) try: await self._execute_job_command(command, "", outbound) except asyncio.CancelledError: raise except Exception as error: logger.error( "background_command_failed", extra={ "error_type": type(error).__name__, "error_detail": str(error), }, ) finally: self._active_job_commands.discard(command.command_id) def _job_command_finished( self, command_id: str, task: asyncio.Task[None], command_tasks: set[asyncio.Task[None]], ) -> None: self._active_job_commands.discard(command_id) self._command_finished(task, command_tasks) async def _execute_job_command( self, command: control_pb2.Command, correlation_id: str, outbound: asyncio.Queue[str], ) -> None: async with self._job_execution_lock: await self._execute_job_command_locked( command, correlation_id, outbound ) async def _execute_job_command_locked( self, command: control_pb2.Command, correlation_id: str, outbound: asyncio.Queue[str], ) -> None: assert self.jobs is not None payload = command.WhichOneof("payload") streamed = False if payload == "assign_job": events = await asyncio.to_thread( self.jobs.assign, command.assign_job ) elif payload == "execute_step": loop = asyncio.get_running_loop() def emit(event): response = new_envelope() response.correlation_id = correlation_id response.job_event.CopyFrom(event) future = asyncio.run_coroutine_threadsafe( outbound.put(encode(response)), loop ) try: future.result(timeout=30) except Exception: # The event is already durable in the client DB and will # be replayed after reconnect. pass events = await asyncio.to_thread( self.jobs.execute, command.execute_step, emit ) streamed = True elif payload == "cancel_job": events = await asyncio.to_thread( self.jobs.cancel, command.cancel_job ) else: raise JobExecutionError("job command payload is unsupported") for event in (() if streamed else events): response = new_envelope() response.correlation_id = correlation_id response.job_event.CopyFrom(event) await outbound.put(encode(response)) def _schedule_route_command( self, command: control_pb2.Command, correlation_id: str, outbound: asyncio.Queue[str], command_tasks: set[asyncio.Task[None]], ) -> None: if command.command_id in self._active_route_commands: return self._active_route_commands.add(command.command_id) task = asyncio.create_task( self._execute_route(command, correlation_id, outbound), name=f"route-{command.ensure_route.route.route_id}", ) command_tasks.add(task) task.add_done_callback( lambda completed: self._route_command_finished( command.command_id, completed, command_tasks ) ) def _route_command_finished( self, command_id: str, task: asyncio.Task[None], command_tasks: set[asyncio.Task[None]], ) -> None: self._active_route_commands.discard(command_id) self._command_finished(task, command_tasks) async def _send_inventory( self, query: Any, correlation_id: str, outbound: asyncio.Queue[str], ) -> None: assert self.inventory is not None chunks = await asyncio.to_thread(self.inventory.execute, query) for chunk in chunks: response = new_envelope() response.correlation_id = correlation_id response.inventory_chunk.CopyFrom(chunk) await outbound.put(encode(response)) async def _execute_route( self, command: Any, correlation_id: str, outbound: asyncio.Queue[str], ) -> None: assert self.routes is not None spec = command.ensure_route.route attempt = await asyncio.to_thread( self.store.begin_route_attempt, command.command_id, spec.route_id, encode_message(spec), str(uuid.uuid4()), ) for row in await asyncio.to_thread( self.store.route_update_rows, command.command_id ): response = new_envelope() response.correlation_id = correlation_id response.route_update.CopyFrom( decode_message(str(row["update_json"]), control_pb2.RouteUpdate()) ) await outbound.put(encode(response)) if attempt["state"] in {"ready", "failed"}: return deadline = time.monotonic() + spec.setup_timeout_seconds configured = None try: attempt = await asyncio.to_thread( self.store.get_route_attempt, command.command_id ) if int(attempt["last_sequence"]) == 0: await self._emit_route_update( command.command_id, correlation_id, outbound, sequence=1, state="provisioning", local_route=self._local_route( spec, route_pb2.ROUTE_STATE_PROVISIONING ), nonce=str(attempt["nonce"]), detail="route command accepted; configuring Syncthing", ) configured = await asyncio.to_thread( self.routes.configure, spec, deadline ) self._known_route_paths[spec.route_id] = configured.local_path await asyncio.to_thread( self.store.record_route_ownership, command.command_id, configured.created_device, configured.created_folder, ) attempt = await asyncio.to_thread( self.store.get_route_attempt, command.command_id ) configured.local_route.archive_control_created = bool( attempt["created_folder"] ) if int(attempt["last_sequence"]) < 2: configured.local_route.state = route_pb2.ROUTE_STATE_VERIFYING configured.local_route.observed_at.GetCurrentTime() await self._emit_route_update( command.command_id, correlation_id, outbound, sequence=2, state="verifying", local_route=configured.local_route, nonce=str(attempt["nonce"]), detail="Syncthing configuration verified; exchanging nonces", ) local_seen, peer_seen = await asyncio.to_thread( self.routes.verify, configured, spec, self.config.client_id, str(attempt["nonce"]), deadline, ) attempt = await asyncio.to_thread( self.store.get_route_attempt, command.command_id ) if int(attempt["last_sequence"]) < 3: configured.local_route.state = route_pb2.ROUTE_STATE_READY configured.local_route.detail = "bidirectional nonce verification passed" configured.local_route.observed_at.GetCurrentTime() await self._emit_route_update( command.command_id, correlation_id, outbound, sequence=3, state="ready", local_route=configured.local_route, nonce=str(attempt["nonce"]), detail="bidirectional nonce verification passed", local_nonce_seen_by_peer=local_seen, peer_nonce_seen_locally=peer_seen, ) except Exception as exc: attempt = await asyncio.to_thread( self.store.get_route_attempt, command.command_id ) if attempt is None or attempt["state"] == "failed": return code, retryable = _route_error(exc) detail = str(exc) or type(exc).__name__ local_route = ( configured.local_route if configured is not None else self._local_route(spec, route_pb2.ROUTE_STATE_UNHEALTHY) ) local_route.state = route_pb2.ROUTE_STATE_UNHEALTHY local_route.detail = detail local_route.observed_at.GetCurrentTime() await self._emit_route_update( command.command_id, correlation_id, outbound, sequence=int(attempt["last_sequence"]) + 1, state="failed", local_route=local_route, nonce=str(attempt["nonce"]), detail=detail, error_code=code, retryable=retryable, ) async def _emit_route_update( self, command_id: str, correlation_id: str, outbound: asyncio.Queue[str], *, sequence: int, state: str, local_route: route_pb2.LocalRoute, nonce: str, detail: str, local_nonce_seen_by_peer: bool = False, peer_nonce_seen_locally: bool = False, error_code: int = common_pb2.ERROR_CODE_UNSPECIFIED, retryable: bool = False, ) -> None: update = control_pb2.RouteUpdate( command_id=command_id, update_id=str(uuid.uuid4()), sequence=sequence, ) update.route.CopyFrom(local_route) update.verification.route_id = local_route.route_id update.verification.nonce = nonce update.verification.local_nonce_seen_by_peer = local_nonce_seen_by_peer update.verification.peer_nonce_seen_locally = peer_nonce_seen_locally update.verification.state = local_route.state update.verification.detail = detail update.verification.observed_at.GetCurrentTime() if error_code != common_pb2.ERROR_CODE_UNSPECIFIED: update.error.code = error_code update.error.message = detail update.error.retryable = retryable encoded_update = encode_message(update) await asyncio.to_thread( self.store.record_route_update, command_id, sequence, state, encoded_update, ) response = new_envelope() response.correlation_id = correlation_id response.route_update.CopyFrom(update) await outbound.put(encode(response)) def _local_route( self, spec: route_pb2.EnsureRouteSpec, state: int ) -> route_pb2.LocalRoute: device_id = next( ( probe.device_id for probe in self.service_probes if probe.service == "syncthing" ), "", ) route = route_pb2.LocalRoute( route_id=spec.route_id, local_relative_path=spec.local_relative_path, folder_type=route_pb2.SYNCTHING_FOLDER_TYPE_SEND_RECEIVE, local_syncthing_device_id=device_id, peer_syncthing_device_ids=[spec.peer_syncthing_device_id], state=state, writable=self.probes[1].writable, sparse_supported=self.probes[1].sparse_files, detail="route provisioning", ) route.observed_at.GetCurrentTime() return route @staticmethod def _command_finished( task: asyncio.Task[None], command_tasks: set[asyncio.Task[None]] ) -> None: command_tasks.discard(task) if task.cancelled(): return error = task.exception() if error is not None: logger.error( "background_command_failed", extra={ "error_type": type(error).__name__, "error_detail": str(error), }, ) def _initial_acknowledgement( self, command: Any, missing_snapshot_jobs: set[str], has_snapshot_jobs: bool, ) -> control_pb2.CommandAck: acknowledgement = control_pb2.CommandAck(command_id=command.command_id) if not command.command_id or command.WhichOneof("payload") is None: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.error.code = common_pb2.ERROR_CODE_INVALID_ARGUMENT acknowledgement.error.message = "command ID and payload are required" elif command.WhichOneof("payload") == "request_job_snapshot": if not has_snapshot_jobs: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.error.code = common_pb2.ERROR_CODE_INVALID_ARGUMENT acknowledgement.error.message = "snapshot job IDs are required" elif missing_snapshot_jobs: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.error.code = common_pb2.ERROR_CODE_NOT_FOUND acknowledgement.error.message = "requested client job is not found" else: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED elif command.WhichOneof("payload") == "inventory_query": scope = command.inventory_query.scope if self.inventory is None: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.error.code = common_pb2.ERROR_CODE_UNAVAILABLE acknowledgement.error.message = "inventory adapter is unavailable" elif scope not in { inventory_pb2.INVENTORY_SCOPE_RESOURCE_SUMMARIES, inventory_pb2.INVENTORY_SCOPE_RESOURCE_LOOKUP, inventory_pb2.INVENTORY_SCOPE_CONTENT_TREE, }: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.error.code = common_pb2.ERROR_CODE_UNSUPPORTED acknowledgement.error.message = "inventory scope is unsupported" else: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED elif command.WhichOneof("payload") == "ensure_route": spec = command.ensure_route.route if self.routes is None: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.error.code = common_pb2.ERROR_CODE_UNAVAILABLE acknowledgement.error.message = "Syncthing route adapter is unavailable" elif ( not spec.route_id or not spec.peer_client_id or not spec.peer_syncthing_device_id or not spec.local_relative_path or spec.setup_timeout_seconds <= 0 ): acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.error.code = common_pb2.ERROR_CODE_INVALID_ARGUMENT acknowledgement.error.message = "ensure route specification is invalid" else: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED elif command.WhichOneof("payload") == "assign_job": definition = command.assign_job.job specification = definition.WhichOneof("spec") assigned_to_client = ( specification == "transfer" and self.config.client_id in { definition.transfer.source_client_id, definition.transfer.target_client_id, } ) or ( specification == "eviction" and self.config.client_id == definition.eviction.cache_client_id ) if self.jobs is None: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.error.code = common_pb2.ERROR_CODE_UNAVAILABLE acknowledgement.error.message = "job executor is unavailable" elif ( not definition.job_id or not assigned_to_client ): acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.error.code = common_pb2.ERROR_CODE_INVALID_ARGUMENT acknowledgement.error.message = "job assignment is invalid" else: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED elif command.WhichOneof("payload") == "execute_step": step = command.execute_step if self.jobs is None: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.error.code = common_pb2.ERROR_CODE_UNAVAILABLE acknowledgement.error.message = "job executor is unavailable" elif ( not step.job_id or step.step not in { job_pb2.JOB_STEP_KIND_SOURCE_STAGE, job_pb2.JOB_STEP_KIND_SYNCTHING_TRANSFER, job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE, job_pb2.JOB_STEP_KIND_QB_VERIFY, job_pb2.JOB_STEP_KIND_STAGING_CLEANUP, job_pb2.JOB_STEP_KIND_VERIFY_ARCHIVE_COVERAGE, job_pb2.JOB_STEP_KIND_QB_REMOVE_ENTRY, job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK, } ): acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.error.code = common_pb2.ERROR_CODE_INVALID_ARGUMENT acknowledgement.error.message = "job step is invalid" else: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED elif command.WhichOneof("payload") == "cancel_job": if self.jobs is None: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.error.code = common_pb2.ERROR_CODE_UNAVAILABLE acknowledgement.error.message = ( "job executor is unavailable" ) elif not command.cancel_job.job_id: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.error.code = ( common_pb2.ERROR_CODE_INVALID_ARGUMENT ) acknowledgement.error.message = "cancel job ID is required" else: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED else: acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.error.code = common_pb2.ERROR_CODE_UNSUPPORTED acknowledgement.error.message = ( "command is not supported by this client build" ) return acknowledgement def _route_path(self, route_id: str) -> Path: try: return self._known_route_paths[route_id] except KeyError as exc: raise JobExecutionError("job route is not configured locally") from exc def _route_error(exc: Exception) -> tuple[int, bool]: if isinstance(exc, RoutePathConflict): return common_pb2.ERROR_CODE_PATH_CONFLICT, False if isinstance(exc, RouteSetupTimeout): return common_pb2.ERROR_CODE_TIMEOUT, True if isinstance(exc, PermissionError): return common_pb2.ERROR_CODE_PERMISSION_DENIED, False if isinstance(exc, RouteSetupError): return common_pb2.ERROR_CODE_UNAVAILABLE, True return common_pb2.ERROR_CODE_INTERNAL, False