1024 lines
41 KiB
Python
1024 lines
41 KiB
Python
"""Reconnectable Archive Control client transport."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import random
|
|
import time
|
|
import uuid
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Any
|
|
|
|
from websockets.asyncio.client import connect
|
|
from websockets.exceptions import ConnectionClosedOK
|
|
|
|
from archive_clients.backup import SQLiteBackupManager
|
|
from archive_clients.config import ClientConfig
|
|
from archive_clients.inventory import InventoryService
|
|
from archive_clients.jobs import ClientJobExecutor, JobExecutionError
|
|
from archive_clients.locking import DatabaseLease
|
|
from archive_clients.probes import FilesystemProbe
|
|
from archive_clients.protocol import (
|
|
decode,
|
|
decode_message,
|
|
encode,
|
|
encode_message,
|
|
new_envelope,
|
|
)
|
|
from archive_clients.qbittorrent import QBittorrentReader
|
|
from archive_clients.services import ServiceProbe
|
|
from archive_clients.state import ClientStore, CommandConflict
|
|
from archive_clients.syncthing import (
|
|
RoutePathConflict,
|
|
RouteSetupError,
|
|
RouteSetupTimeout,
|
|
SyncthingRouteManager,
|
|
)
|
|
from archive_control.v1 import (
|
|
client_pb2, common_pb2, control_pb2, inventory_pb2, job_pb2, route_pb2,
|
|
)
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ArchiveClientDaemon:
|
|
def __init__(
|
|
self,
|
|
config: ClientConfig,
|
|
probes: list[FilesystemProbe],
|
|
service_probes: list[ServiceProbe],
|
|
resource_reader: QBittorrentReader | None = None,
|
|
route_manager: SyncthingRouteManager | None = None,
|
|
):
|
|
if len(probes) != 2:
|
|
raise ValueError(
|
|
"qBittorrent and Syncthing filesystem probes are required"
|
|
)
|
|
self.config = config
|
|
self.probes = probes
|
|
self.service_probes = service_probes
|
|
self.inventory = (
|
|
InventoryService(resource_reader, config.client_id)
|
|
if resource_reader is not None else None
|
|
)
|
|
healthy_syncthing = any(
|
|
probe.service == "syncthing"
|
|
and probe.state == common_pb2.HEALTH_STATE_HEALTHY
|
|
for probe in service_probes
|
|
)
|
|
self.routes = route_manager or (
|
|
SyncthingRouteManager(
|
|
config.syncthing,
|
|
sparse_supported=probes[1].sparse_files,
|
|
)
|
|
if healthy_syncthing and probes[1].writable
|
|
else None
|
|
)
|
|
self._known_route_paths: dict[str, Path] = {}
|
|
for probe in service_probes:
|
|
if probe.service != "syncthing":
|
|
continue
|
|
for route in probe.routes:
|
|
api_path = (
|
|
config.syncthing.api_root
|
|
/ PurePosixPath(route.local_relative_path)
|
|
).as_posix()
|
|
self._known_route_paths[route.route_id] = (
|
|
config.syncthing.roots.api_to_local(api_path)
|
|
)
|
|
self.store = ClientStore(config.state_db)
|
|
self.backups = SQLiteBackupManager(
|
|
config.state_db, config.backup_dir, config.backup
|
|
)
|
|
self._lease = DatabaseLease(config.state_db)
|
|
self._active_route_commands: set[str] = set()
|
|
self._active_job_commands: set[str] = set()
|
|
self._job_execution_lock = asyncio.Lock()
|
|
self.jobs = (
|
|
ClientJobExecutor(
|
|
client_id=config.client_id,
|
|
qbittorrent=resource_reader,
|
|
store=self.store,
|
|
qb_root=config.qbittorrent.local_root,
|
|
qb_api_root=config.qbittorrent.api_root,
|
|
route_path=self._route_path,
|
|
syncthing_transport=self.routes.transport,
|
|
sparse_supported=all(probe.sparse_files for probe in probes),
|
|
poll_interval=config.jobs.poll_interval,
|
|
verification_timeout=config.jobs.verification_timeout,
|
|
free_space_reserve_bytes=(
|
|
config.jobs.free_space_reserve_bytes
|
|
),
|
|
)
|
|
if resource_reader is not None and self.routes is not None
|
|
else None
|
|
)
|
|
|
|
async def run(self) -> None:
|
|
await asyncio.to_thread(self._lease.acquire)
|
|
backup_task: asyncio.Task[None] | None = None
|
|
try:
|
|
await asyncio.to_thread(self.store.initialize)
|
|
await asyncio.to_thread(self._warn_if_backup_shares_filesystem)
|
|
backup_task = asyncio.create_task(
|
|
self._backup_loop(), name="archive-client-backups"
|
|
)
|
|
await self._connection_loop()
|
|
finally:
|
|
if backup_task is not None:
|
|
backup_task.cancel()
|
|
await asyncio.gather(backup_task, return_exceptions=True)
|
|
await asyncio.to_thread(self._lease.release)
|
|
|
|
def _warn_if_backup_shares_filesystem(self) -> None:
|
|
if (
|
|
self.config.state_db.parent.stat().st_dev
|
|
== self.config.backup_dir.stat().st_dev
|
|
):
|
|
logger.warning("database_and_backup_share_filesystem")
|
|
|
|
async def _connection_loop(self) -> None:
|
|
delay = self.config.connection.reconnect_initial
|
|
while True:
|
|
started = time.monotonic()
|
|
try:
|
|
await self._connection()
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"control_connection_ended",
|
|
extra={"error_type": type(exc).__name__},
|
|
)
|
|
if (
|
|
time.monotonic() - started
|
|
>= self.config.connection.reconnect_reset_after
|
|
):
|
|
delay = self.config.connection.reconnect_initial
|
|
wait = (
|
|
random.uniform(0, delay)
|
|
if self.config.connection.reconnect_jitter else delay
|
|
)
|
|
await asyncio.sleep(wait)
|
|
delay = min(delay * 2, self.config.connection.reconnect_max)
|
|
|
|
async def _backup_loop(self) -> None:
|
|
while True:
|
|
await asyncio.sleep(self.config.backup.interval)
|
|
try:
|
|
record = await asyncio.to_thread(self.backups.create, "scheduled")
|
|
logger.info(
|
|
"database_backup_created",
|
|
extra={"backup_name": record.database.name},
|
|
)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as exc:
|
|
logger.error(
|
|
"database_backup_failed",
|
|
extra={"error_type": type(exc).__name__},
|
|
)
|
|
|
|
async def _connection(self) -> None:
|
|
async with connect(
|
|
self.config.control_endpoint, ping_interval=None, compression=None,
|
|
max_size=1024 * 1024,
|
|
) as websocket:
|
|
registration = self._registration()
|
|
await websocket.send(encode(registration))
|
|
registration.register_request.shared_token = ""
|
|
response = decode(await asyncio.wait_for(
|
|
websocket.recv(), self.config.connection.registration_timeout
|
|
))
|
|
if response.WhichOneof("payload") != "register_response":
|
|
raise RuntimeError("control did not answer registration")
|
|
if response.correlation_id != registration.message_id:
|
|
raise RuntimeError("registration response correlation mismatch")
|
|
if response.register_response.status != client_pb2.REGISTRATION_STATUS_ACCEPTED:
|
|
raise RuntimeError("control rejected registration")
|
|
if response.register_response.negotiated_version.major != 1:
|
|
raise RuntimeError("control negotiated an unsupported protocol version")
|
|
logger.info("control_connection_registered")
|
|
outbound: asyncio.Queue[str] = asyncio.Queue(maxsize=100)
|
|
writer = asyncio.create_task(self._writer(websocket, outbound))
|
|
command_tasks: set[asyncio.Task[None]] = set()
|
|
try:
|
|
await self._resume_commands(outbound, command_tasks)
|
|
# A proxy can leave the TCP/WebSocket socket apparently open
|
|
# after the control server has discarded its session. The
|
|
# server then cannot deliver durable commands and its pending
|
|
# outbox remains stranded unless the client independently
|
|
# detects the missing application heartbeats and reconnects.
|
|
while True:
|
|
try:
|
|
frame = await asyncio.wait_for(
|
|
websocket.recv(),
|
|
self.config.connection.offline_timeout,
|
|
)
|
|
except ConnectionClosedOK:
|
|
return
|
|
except asyncio.TimeoutError as exc:
|
|
raise RuntimeError(
|
|
"control heartbeat timed out"
|
|
) from exc
|
|
await self._handle(decode(frame), outbound, command_tasks)
|
|
finally:
|
|
writer.cancel()
|
|
for task in command_tasks:
|
|
task.cancel()
|
|
await asyncio.gather(
|
|
writer, *command_tasks, return_exceptions=True
|
|
)
|
|
|
|
def _registration(self):
|
|
envelope = new_envelope()
|
|
request = envelope.register_request
|
|
request.protocol_version.CopyFrom(envelope.protocol_version)
|
|
request.client.client_id = self.config.client_id
|
|
request.client.display_name = self.config.display_name
|
|
request.client.role = (
|
|
common_pb2.CLIENT_ROLE_ARCHIVE
|
|
if self.config.role == "archive" else common_pb2.CLIENT_ROLE_CACHE
|
|
)
|
|
request.connection_instance_id = str(uuid.uuid4())
|
|
request.shared_token = self.config.read_shared_token()
|
|
request.capabilities.max_envelope_bytes = 1024 * 1024
|
|
request.capabilities.syncthing_advertised_addresses.extend(
|
|
self.config.syncthing.advertised_addresses
|
|
)
|
|
for probe in self.service_probes:
|
|
health = request.capabilities.services.add()
|
|
health.service = probe.service
|
|
health.state = probe.state
|
|
health.version = probe.version
|
|
health.api_version = probe.api_version
|
|
health.detail = probe.detail
|
|
health.checked_at.FromDatetime(probe.checked_at)
|
|
if probe.service == "qbittorrent":
|
|
request.capabilities.qbittorrent_version = probe.version
|
|
request.capabilities.qbittorrent_web_api_version = (
|
|
probe.api_version
|
|
)
|
|
request.capabilities.libtorrent_version = probe.libtorrent_version
|
|
elif probe.service == "syncthing":
|
|
request.capabilities.syncthing_version = probe.version
|
|
request.capabilities.syncthing_device_id = probe.device_id
|
|
request.capabilities.routes.extend(probe.routes)
|
|
for root_name, probe in zip(
|
|
("qbittorrent", "syncthing"), self.probes, strict=True
|
|
):
|
|
filesystem = request.capabilities.filesystems.add()
|
|
filesystem.root_name = root_name
|
|
filesystem.readable = probe.readable
|
|
filesystem.writable = probe.writable
|
|
filesystem.hard_link = probe.hard_link
|
|
filesystem.reflink = probe.reflink
|
|
filesystem.sparse_files = probe.sparse_files
|
|
if all(probe.hard_link for probe in self.probes):
|
|
request.capabilities.features.append(client_pb2.CLIENT_FEATURE_HARD_LINK)
|
|
if all(probe.reflink for probe in self.probes):
|
|
request.capabilities.features.append(client_pb2.CLIENT_FEATURE_REFLINK)
|
|
if all(probe.sparse_files for probe in self.probes):
|
|
request.capabilities.features.append(client_pb2.CLIENT_FEATURE_SPARSE_FILES)
|
|
request.capabilities.features.append(client_pb2.CLIENT_FEATURE_DB_BACKUP)
|
|
if self.inventory is not None:
|
|
request.capabilities.features.extend((
|
|
client_pb2.CLIENT_FEATURE_INVENTORY_CHUNKS,
|
|
client_pb2.CLIENT_FEATURE_CONTENT_TREE,
|
|
))
|
|
if self.routes is not None:
|
|
request.capabilities.features.append(
|
|
client_pb2.CLIENT_FEATURE_ROUTE_PROVISIONING
|
|
)
|
|
for cursor in self.store.list_active_job_cursors():
|
|
active = request.active_jobs.add()
|
|
active.job_id = str(cursor["job_id"])
|
|
active.job_revision = int(cursor["revision"])
|
|
active.last_event_sequence = int(cursor["last_event_sequence"])
|
|
active.state = job_pb2.JobState.Value(str(cursor["state"]))
|
|
active.committed = bool(cursor["committed"])
|
|
return envelope
|
|
|
|
async def _writer(
|
|
self, websocket: Any, outbound: asyncio.Queue[str]
|
|
) -> None:
|
|
while True:
|
|
await websocket.send(await outbound.get())
|
|
|
|
async def _handle(
|
|
self,
|
|
envelope: Any,
|
|
outbound: asyncio.Queue[str],
|
|
command_tasks: set[asyncio.Task[None]],
|
|
) -> None:
|
|
payload = envelope.WhichOneof("payload")
|
|
if payload == "heartbeat":
|
|
response = new_envelope()
|
|
response.correlation_id = envelope.message_id
|
|
response.heartbeat_ack.sequence = envelope.heartbeat.sequence
|
|
await outbound.put(encode(response))
|
|
elif payload == "command":
|
|
await self._accept_command(envelope, outbound, command_tasks)
|
|
elif payload == "protocol_error":
|
|
logger.warning(
|
|
"control_reported_protocol_error",
|
|
extra={"error_code": envelope.protocol_error.error.code},
|
|
)
|
|
|
|
async def _accept_command(
|
|
self,
|
|
envelope: Any,
|
|
outbound: asyncio.Queue[str],
|
|
command_tasks: set[asyncio.Task[None]],
|
|
) -> None:
|
|
command = envelope.command
|
|
snapshot_rows: list[dict[str, object]] = []
|
|
if command.WhichOneof("payload") == "request_job_snapshot":
|
|
requested = list(command.request_job_snapshot.job_ids)
|
|
snapshot_rows = await asyncio.to_thread(
|
|
self.store.job_snapshot_rows, requested
|
|
)
|
|
missing = set(requested) - {
|
|
str(row["job_id"]) for row in snapshot_rows
|
|
}
|
|
else:
|
|
requested = []
|
|
missing = set()
|
|
acknowledgement = self._initial_acknowledgement(
|
|
command, missing, bool(requested)
|
|
)
|
|
accepted = None
|
|
accepted_for_execution = False
|
|
try:
|
|
accepted = await asyncio.to_thread(
|
|
self.store.accept_command,
|
|
command.command_id,
|
|
encode_message(command),
|
|
encode_message(acknowledgement),
|
|
)
|
|
if accepted.duplicate:
|
|
acknowledgement = decode_message(
|
|
accepted.acknowledgement_json, control_pb2.CommandAck()
|
|
)
|
|
if (
|
|
acknowledgement.status
|
|
== control_pb2.COMMAND_ACK_STATUS_ACCEPTED
|
|
):
|
|
accepted_for_execution = True
|
|
acknowledgement.status = (
|
|
control_pb2.COMMAND_ACK_STATUS_DUPLICATE
|
|
)
|
|
else:
|
|
accepted_for_execution = (
|
|
acknowledgement.status
|
|
== control_pb2.COMMAND_ACK_STATUS_ACCEPTED
|
|
)
|
|
except CommandConflict:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
|
|
acknowledgement.error.code = common_pb2.ERROR_CODE_CONFLICT
|
|
acknowledgement.error.message = "command ID content conflict"
|
|
response = new_envelope()
|
|
response.correlation_id = envelope.message_id
|
|
response.command_ack.CopyFrom(acknowledgement)
|
|
await outbound.put(encode(response))
|
|
|
|
if (
|
|
accepted is not None
|
|
and accepted_for_execution
|
|
and command.WhichOneof("payload") == "request_job_snapshot"
|
|
):
|
|
for row in snapshot_rows:
|
|
snapshot = new_envelope()
|
|
snapshot.correlation_id = envelope.message_id
|
|
job_snapshot = snapshot.job_snapshot
|
|
job_snapshot.job.definition.CopyFrom(decode_message(
|
|
str(row["definition_json"]), job_pb2.JobDefinition()
|
|
))
|
|
job_snapshot.job.state = job_pb2.JobState.Value(str(row["state"]))
|
|
job_snapshot.job.revision = int(row["revision"])
|
|
job_snapshot.job.committed = bool(row["committed"])
|
|
job_snapshot.job.updated_at.CopyFrom(snapshot.sent_at)
|
|
job_snapshot.last_event_sequence = int(
|
|
row["last_event_sequence"]
|
|
)
|
|
await outbound.put(encode(snapshot))
|
|
elif (
|
|
accepted is not None
|
|
and accepted_for_execution
|
|
and command.WhichOneof("payload") == "inventory_query"
|
|
and self.inventory is not None
|
|
):
|
|
task = asyncio.create_task(
|
|
self._send_inventory(
|
|
command.inventory_query, envelope.message_id, outbound
|
|
),
|
|
name=f"inventory-{command.inventory_query.query_id}",
|
|
)
|
|
command_tasks.add(task)
|
|
task.add_done_callback(
|
|
lambda completed: self._command_finished(
|
|
completed, command_tasks
|
|
)
|
|
)
|
|
elif (
|
|
accepted is not None
|
|
and accepted_for_execution
|
|
and command.WhichOneof("payload") == "ensure_route"
|
|
and self.routes is not None
|
|
):
|
|
self._schedule_route_command(
|
|
command,
|
|
envelope.message_id,
|
|
outbound,
|
|
command_tasks,
|
|
)
|
|
elif (
|
|
accepted is not None
|
|
and accepted_for_execution
|
|
and command.WhichOneof("payload") in {
|
|
"assign_job", "execute_step", "cancel_job"
|
|
}
|
|
and self.jobs is not None
|
|
):
|
|
if command.WhichOneof("payload") == "cancel_job":
|
|
self.jobs.request_cancel(command.cancel_job.job_id)
|
|
self._schedule_job_command(
|
|
command,
|
|
envelope.message_id,
|
|
outbound,
|
|
command_tasks,
|
|
)
|
|
|
|
async def _resume_commands(
|
|
self,
|
|
outbound: asyncio.Queue[str],
|
|
command_tasks: set[asyncio.Task[None]],
|
|
) -> None:
|
|
job_commands: list[control_pb2.Command] = []
|
|
for row in await asyncio.to_thread(self.store.list_accepted_commands):
|
|
acknowledgement = decode_message(
|
|
str(row["acknowledgement_json"]), control_pb2.CommandAck()
|
|
)
|
|
if acknowledgement.status != control_pb2.COMMAND_ACK_STATUS_ACCEPTED:
|
|
continue
|
|
command = decode_message(
|
|
str(row["command_json"]), control_pb2.Command()
|
|
)
|
|
if command.WhichOneof("payload") == "ensure_route":
|
|
self._schedule_route_command(
|
|
command, "", outbound, command_tasks
|
|
)
|
|
elif (
|
|
command.WhichOneof("payload") in {
|
|
"assign_job", "execute_step", "cancel_job"
|
|
}
|
|
and self.jobs is not None
|
|
):
|
|
if command.WhichOneof("payload") == "cancel_job":
|
|
self.jobs.request_cancel(command.cancel_job.job_id)
|
|
job_commands.append(command)
|
|
if job_commands:
|
|
task = asyncio.create_task(
|
|
self._resume_job_commands(job_commands, outbound),
|
|
name="resume-job-commands",
|
|
)
|
|
command_tasks.add(task)
|
|
task.add_done_callback(
|
|
lambda completed: self._command_finished(
|
|
completed, command_tasks
|
|
)
|
|
)
|
|
|
|
async def _resume_route_commands(
|
|
self,
|
|
outbound: asyncio.Queue[str],
|
|
command_tasks: set[asyncio.Task[None]],
|
|
) -> None:
|
|
"""Backward-compatible test hook."""
|
|
|
|
await self._resume_commands(outbound, command_tasks)
|
|
|
|
def _schedule_job_command(
|
|
self,
|
|
command: control_pb2.Command,
|
|
correlation_id: str,
|
|
outbound: asyncio.Queue[str],
|
|
command_tasks: set[asyncio.Task[None]],
|
|
) -> None:
|
|
if command.command_id in self._active_job_commands:
|
|
return
|
|
self._active_job_commands.add(command.command_id)
|
|
task = asyncio.create_task(
|
|
self._execute_job_command(command, correlation_id, outbound),
|
|
name=f"job-command-{command.command_id}",
|
|
)
|
|
command_tasks.add(task)
|
|
task.add_done_callback(
|
|
lambda completed: self._job_command_finished(
|
|
command.command_id, completed, command_tasks
|
|
)
|
|
)
|
|
|
|
async def _resume_job_commands(
|
|
self,
|
|
commands: list[control_pb2.Command],
|
|
outbound: asyncio.Queue[str],
|
|
) -> None:
|
|
for command in commands:
|
|
if command.command_id in self._active_job_commands:
|
|
continue
|
|
self._active_job_commands.add(command.command_id)
|
|
try:
|
|
await self._execute_job_command(command, "", outbound)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as error:
|
|
logger.error(
|
|
"background_command_failed",
|
|
extra={
|
|
"error_type": type(error).__name__,
|
|
"error_detail": str(error),
|
|
},
|
|
)
|
|
finally:
|
|
self._active_job_commands.discard(command.command_id)
|
|
|
|
def _job_command_finished(
|
|
self,
|
|
command_id: str,
|
|
task: asyncio.Task[None],
|
|
command_tasks: set[asyncio.Task[None]],
|
|
) -> None:
|
|
self._active_job_commands.discard(command_id)
|
|
self._command_finished(task, command_tasks)
|
|
|
|
async def _execute_job_command(
|
|
self,
|
|
command: control_pb2.Command,
|
|
correlation_id: str,
|
|
outbound: asyncio.Queue[str],
|
|
) -> None:
|
|
async with self._job_execution_lock:
|
|
await self._execute_job_command_locked(
|
|
command, correlation_id, outbound
|
|
)
|
|
|
|
async def _execute_job_command_locked(
|
|
self,
|
|
command: control_pb2.Command,
|
|
correlation_id: str,
|
|
outbound: asyncio.Queue[str],
|
|
) -> None:
|
|
assert self.jobs is not None
|
|
payload = command.WhichOneof("payload")
|
|
streamed = False
|
|
if payload == "assign_job":
|
|
events = await asyncio.to_thread(
|
|
self.jobs.assign, command.assign_job
|
|
)
|
|
elif payload == "execute_step":
|
|
loop = asyncio.get_running_loop()
|
|
|
|
def emit(event):
|
|
response = new_envelope()
|
|
response.correlation_id = correlation_id
|
|
response.job_event.CopyFrom(event)
|
|
future = asyncio.run_coroutine_threadsafe(
|
|
outbound.put(encode(response)), loop
|
|
)
|
|
try:
|
|
future.result(timeout=30)
|
|
except Exception:
|
|
# The event is already durable in the client DB and will
|
|
# be replayed after reconnect.
|
|
pass
|
|
|
|
events = await asyncio.to_thread(
|
|
self.jobs.execute, command.execute_step, emit
|
|
)
|
|
streamed = True
|
|
elif payload == "cancel_job":
|
|
events = await asyncio.to_thread(
|
|
self.jobs.cancel, command.cancel_job
|
|
)
|
|
else:
|
|
raise JobExecutionError("job command payload is unsupported")
|
|
for event in (() if streamed else events):
|
|
response = new_envelope()
|
|
response.correlation_id = correlation_id
|
|
response.job_event.CopyFrom(event)
|
|
await outbound.put(encode(response))
|
|
|
|
def _schedule_route_command(
|
|
self,
|
|
command: control_pb2.Command,
|
|
correlation_id: str,
|
|
outbound: asyncio.Queue[str],
|
|
command_tasks: set[asyncio.Task[None]],
|
|
) -> None:
|
|
if command.command_id in self._active_route_commands:
|
|
return
|
|
self._active_route_commands.add(command.command_id)
|
|
task = asyncio.create_task(
|
|
self._execute_route(command, correlation_id, outbound),
|
|
name=f"route-{command.ensure_route.route.route_id}",
|
|
)
|
|
command_tasks.add(task)
|
|
task.add_done_callback(
|
|
lambda completed: self._route_command_finished(
|
|
command.command_id, completed, command_tasks
|
|
)
|
|
)
|
|
|
|
def _route_command_finished(
|
|
self,
|
|
command_id: str,
|
|
task: asyncio.Task[None],
|
|
command_tasks: set[asyncio.Task[None]],
|
|
) -> None:
|
|
self._active_route_commands.discard(command_id)
|
|
self._command_finished(task, command_tasks)
|
|
|
|
async def _send_inventory(
|
|
self,
|
|
query: Any,
|
|
correlation_id: str,
|
|
outbound: asyncio.Queue[str],
|
|
) -> None:
|
|
assert self.inventory is not None
|
|
chunks = await asyncio.to_thread(self.inventory.execute, query)
|
|
for chunk in chunks:
|
|
response = new_envelope()
|
|
response.correlation_id = correlation_id
|
|
response.inventory_chunk.CopyFrom(chunk)
|
|
await outbound.put(encode(response))
|
|
|
|
async def _execute_route(
|
|
self,
|
|
command: Any,
|
|
correlation_id: str,
|
|
outbound: asyncio.Queue[str],
|
|
) -> None:
|
|
assert self.routes is not None
|
|
spec = command.ensure_route.route
|
|
attempt = await asyncio.to_thread(
|
|
self.store.begin_route_attempt,
|
|
command.command_id,
|
|
spec.route_id,
|
|
encode_message(spec),
|
|
str(uuid.uuid4()),
|
|
)
|
|
for row in await asyncio.to_thread(
|
|
self.store.route_update_rows, command.command_id
|
|
):
|
|
response = new_envelope()
|
|
response.correlation_id = correlation_id
|
|
response.route_update.CopyFrom(
|
|
decode_message(str(row["update_json"]), control_pb2.RouteUpdate())
|
|
)
|
|
await outbound.put(encode(response))
|
|
if attempt["state"] in {"ready", "failed"}:
|
|
return
|
|
|
|
deadline = time.monotonic() + spec.setup_timeout_seconds
|
|
configured = None
|
|
try:
|
|
attempt = await asyncio.to_thread(
|
|
self.store.get_route_attempt, command.command_id
|
|
)
|
|
if int(attempt["last_sequence"]) == 0:
|
|
await self._emit_route_update(
|
|
command.command_id,
|
|
correlation_id,
|
|
outbound,
|
|
sequence=1,
|
|
state="provisioning",
|
|
local_route=self._local_route(
|
|
spec, route_pb2.ROUTE_STATE_PROVISIONING
|
|
),
|
|
nonce=str(attempt["nonce"]),
|
|
detail="route command accepted; configuring Syncthing",
|
|
)
|
|
configured = await asyncio.to_thread(
|
|
self.routes.configure, spec, deadline
|
|
)
|
|
self._known_route_paths[spec.route_id] = configured.local_path
|
|
await asyncio.to_thread(
|
|
self.store.record_route_ownership,
|
|
command.command_id,
|
|
configured.created_device,
|
|
configured.created_folder,
|
|
)
|
|
attempt = await asyncio.to_thread(
|
|
self.store.get_route_attempt, command.command_id
|
|
)
|
|
configured.local_route.archive_control_created = bool(
|
|
attempt["created_folder"]
|
|
)
|
|
if int(attempt["last_sequence"]) < 2:
|
|
configured.local_route.state = route_pb2.ROUTE_STATE_VERIFYING
|
|
configured.local_route.observed_at.GetCurrentTime()
|
|
await self._emit_route_update(
|
|
command.command_id,
|
|
correlation_id,
|
|
outbound,
|
|
sequence=2,
|
|
state="verifying",
|
|
local_route=configured.local_route,
|
|
nonce=str(attempt["nonce"]),
|
|
detail="Syncthing configuration verified; exchanging nonces",
|
|
)
|
|
local_seen, peer_seen = await asyncio.to_thread(
|
|
self.routes.verify,
|
|
configured,
|
|
spec,
|
|
self.config.client_id,
|
|
str(attempt["nonce"]),
|
|
deadline,
|
|
)
|
|
attempt = await asyncio.to_thread(
|
|
self.store.get_route_attempt, command.command_id
|
|
)
|
|
if int(attempt["last_sequence"]) < 3:
|
|
configured.local_route.state = route_pb2.ROUTE_STATE_READY
|
|
configured.local_route.detail = "bidirectional nonce verification passed"
|
|
configured.local_route.observed_at.GetCurrentTime()
|
|
await self._emit_route_update(
|
|
command.command_id,
|
|
correlation_id,
|
|
outbound,
|
|
sequence=3,
|
|
state="ready",
|
|
local_route=configured.local_route,
|
|
nonce=str(attempt["nonce"]),
|
|
detail="bidirectional nonce verification passed",
|
|
local_nonce_seen_by_peer=local_seen,
|
|
peer_nonce_seen_locally=peer_seen,
|
|
)
|
|
except Exception as exc:
|
|
attempt = await asyncio.to_thread(
|
|
self.store.get_route_attempt, command.command_id
|
|
)
|
|
if attempt is None or attempt["state"] == "failed":
|
|
return
|
|
code, retryable = _route_error(exc)
|
|
detail = str(exc) or type(exc).__name__
|
|
local_route = (
|
|
configured.local_route
|
|
if configured is not None
|
|
else self._local_route(spec, route_pb2.ROUTE_STATE_UNHEALTHY)
|
|
)
|
|
local_route.state = route_pb2.ROUTE_STATE_UNHEALTHY
|
|
local_route.detail = detail
|
|
local_route.observed_at.GetCurrentTime()
|
|
await self._emit_route_update(
|
|
command.command_id,
|
|
correlation_id,
|
|
outbound,
|
|
sequence=int(attempt["last_sequence"]) + 1,
|
|
state="failed",
|
|
local_route=local_route,
|
|
nonce=str(attempt["nonce"]),
|
|
detail=detail,
|
|
error_code=code,
|
|
retryable=retryable,
|
|
)
|
|
|
|
async def _emit_route_update(
|
|
self,
|
|
command_id: str,
|
|
correlation_id: str,
|
|
outbound: asyncio.Queue[str],
|
|
*,
|
|
sequence: int,
|
|
state: str,
|
|
local_route: route_pb2.LocalRoute,
|
|
nonce: str,
|
|
detail: str,
|
|
local_nonce_seen_by_peer: bool = False,
|
|
peer_nonce_seen_locally: bool = False,
|
|
error_code: int = common_pb2.ERROR_CODE_UNSPECIFIED,
|
|
retryable: bool = False,
|
|
) -> None:
|
|
update = control_pb2.RouteUpdate(
|
|
command_id=command_id,
|
|
update_id=str(uuid.uuid4()),
|
|
sequence=sequence,
|
|
)
|
|
update.route.CopyFrom(local_route)
|
|
update.verification.route_id = local_route.route_id
|
|
update.verification.nonce = nonce
|
|
update.verification.local_nonce_seen_by_peer = local_nonce_seen_by_peer
|
|
update.verification.peer_nonce_seen_locally = peer_nonce_seen_locally
|
|
update.verification.state = local_route.state
|
|
update.verification.detail = detail
|
|
update.verification.observed_at.GetCurrentTime()
|
|
if error_code != common_pb2.ERROR_CODE_UNSPECIFIED:
|
|
update.error.code = error_code
|
|
update.error.message = detail
|
|
update.error.retryable = retryable
|
|
encoded_update = encode_message(update)
|
|
await asyncio.to_thread(
|
|
self.store.record_route_update,
|
|
command_id,
|
|
sequence,
|
|
state,
|
|
encoded_update,
|
|
)
|
|
response = new_envelope()
|
|
response.correlation_id = correlation_id
|
|
response.route_update.CopyFrom(update)
|
|
await outbound.put(encode(response))
|
|
|
|
def _local_route(
|
|
self, spec: route_pb2.EnsureRouteSpec, state: int
|
|
) -> route_pb2.LocalRoute:
|
|
device_id = next(
|
|
(
|
|
probe.device_id
|
|
for probe in self.service_probes
|
|
if probe.service == "syncthing"
|
|
),
|
|
"",
|
|
)
|
|
route = route_pb2.LocalRoute(
|
|
route_id=spec.route_id,
|
|
local_relative_path=spec.local_relative_path,
|
|
folder_type=route_pb2.SYNCTHING_FOLDER_TYPE_SEND_RECEIVE,
|
|
local_syncthing_device_id=device_id,
|
|
peer_syncthing_device_ids=[spec.peer_syncthing_device_id],
|
|
state=state,
|
|
writable=self.probes[1].writable,
|
|
sparse_supported=self.probes[1].sparse_files,
|
|
detail="route provisioning",
|
|
)
|
|
route.observed_at.GetCurrentTime()
|
|
return route
|
|
|
|
@staticmethod
|
|
def _command_finished(
|
|
task: asyncio.Task[None], command_tasks: set[asyncio.Task[None]]
|
|
) -> None:
|
|
command_tasks.discard(task)
|
|
if task.cancelled():
|
|
return
|
|
error = task.exception()
|
|
if error is not None:
|
|
logger.error(
|
|
"background_command_failed",
|
|
extra={
|
|
"error_type": type(error).__name__,
|
|
"error_detail": str(error),
|
|
},
|
|
)
|
|
|
|
def _initial_acknowledgement(
|
|
self,
|
|
command: Any,
|
|
missing_snapshot_jobs: set[str],
|
|
has_snapshot_jobs: bool,
|
|
) -> control_pb2.CommandAck:
|
|
acknowledgement = control_pb2.CommandAck(command_id=command.command_id)
|
|
if not command.command_id or command.WhichOneof("payload") is None:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
|
|
acknowledgement.error.code = common_pb2.ERROR_CODE_INVALID_ARGUMENT
|
|
acknowledgement.error.message = "command ID and payload are required"
|
|
elif command.WhichOneof("payload") == "request_job_snapshot":
|
|
if not has_snapshot_jobs:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
|
|
acknowledgement.error.code = common_pb2.ERROR_CODE_INVALID_ARGUMENT
|
|
acknowledgement.error.message = "snapshot job IDs are required"
|
|
elif missing_snapshot_jobs:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
|
|
acknowledgement.error.code = common_pb2.ERROR_CODE_NOT_FOUND
|
|
acknowledgement.error.message = "requested client job is not found"
|
|
else:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED
|
|
elif command.WhichOneof("payload") == "inventory_query":
|
|
scope = command.inventory_query.scope
|
|
if self.inventory is None:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
|
|
acknowledgement.error.code = common_pb2.ERROR_CODE_UNAVAILABLE
|
|
acknowledgement.error.message = "inventory adapter is unavailable"
|
|
elif scope not in {
|
|
inventory_pb2.INVENTORY_SCOPE_RESOURCE_SUMMARIES,
|
|
inventory_pb2.INVENTORY_SCOPE_RESOURCE_LOOKUP,
|
|
inventory_pb2.INVENTORY_SCOPE_CONTENT_TREE,
|
|
}:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
|
|
acknowledgement.error.code = common_pb2.ERROR_CODE_UNSUPPORTED
|
|
acknowledgement.error.message = "inventory scope is unsupported"
|
|
else:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED
|
|
elif command.WhichOneof("payload") == "ensure_route":
|
|
spec = command.ensure_route.route
|
|
if self.routes is None:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
|
|
acknowledgement.error.code = common_pb2.ERROR_CODE_UNAVAILABLE
|
|
acknowledgement.error.message = "Syncthing route adapter is unavailable"
|
|
elif (
|
|
not spec.route_id
|
|
or not spec.peer_client_id
|
|
or not spec.peer_syncthing_device_id
|
|
or not spec.local_relative_path
|
|
or spec.setup_timeout_seconds <= 0
|
|
):
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
|
|
acknowledgement.error.code = common_pb2.ERROR_CODE_INVALID_ARGUMENT
|
|
acknowledgement.error.message = "ensure route specification is invalid"
|
|
else:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED
|
|
elif command.WhichOneof("payload") == "assign_job":
|
|
definition = command.assign_job.job
|
|
specification = definition.WhichOneof("spec")
|
|
assigned_to_client = (
|
|
specification == "transfer"
|
|
and self.config.client_id
|
|
in {
|
|
definition.transfer.source_client_id,
|
|
definition.transfer.target_client_id,
|
|
}
|
|
) or (
|
|
specification == "eviction"
|
|
and self.config.client_id == definition.eviction.cache_client_id
|
|
)
|
|
if self.jobs is None:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
|
|
acknowledgement.error.code = common_pb2.ERROR_CODE_UNAVAILABLE
|
|
acknowledgement.error.message = "job executor is unavailable"
|
|
elif (
|
|
not definition.job_id
|
|
or not assigned_to_client
|
|
):
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
|
|
acknowledgement.error.code = common_pb2.ERROR_CODE_INVALID_ARGUMENT
|
|
acknowledgement.error.message = "job assignment is invalid"
|
|
else:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED
|
|
elif command.WhichOneof("payload") == "execute_step":
|
|
step = command.execute_step
|
|
if self.jobs is None:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
|
|
acknowledgement.error.code = common_pb2.ERROR_CODE_UNAVAILABLE
|
|
acknowledgement.error.message = "job executor is unavailable"
|
|
elif (
|
|
not step.job_id
|
|
or step.step not in {
|
|
job_pb2.JOB_STEP_KIND_SOURCE_STAGE,
|
|
job_pb2.JOB_STEP_KIND_SYNCTHING_TRANSFER,
|
|
job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE,
|
|
job_pb2.JOB_STEP_KIND_QB_VERIFY,
|
|
job_pb2.JOB_STEP_KIND_STAGING_CLEANUP,
|
|
job_pb2.JOB_STEP_KIND_VERIFY_ARCHIVE_COVERAGE,
|
|
job_pb2.JOB_STEP_KIND_QB_REMOVE_ENTRY,
|
|
job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK,
|
|
}
|
|
):
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
|
|
acknowledgement.error.code = common_pb2.ERROR_CODE_INVALID_ARGUMENT
|
|
acknowledgement.error.message = "job step is invalid"
|
|
else:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED
|
|
elif command.WhichOneof("payload") == "cancel_job":
|
|
if self.jobs is None:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
|
|
acknowledgement.error.code = common_pb2.ERROR_CODE_UNAVAILABLE
|
|
acknowledgement.error.message = (
|
|
"job executor is unavailable"
|
|
)
|
|
elif not command.cancel_job.job_id:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
|
|
acknowledgement.error.code = (
|
|
common_pb2.ERROR_CODE_INVALID_ARGUMENT
|
|
)
|
|
acknowledgement.error.message = "cancel job ID is required"
|
|
else:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED
|
|
else:
|
|
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
|
|
acknowledgement.error.code = common_pb2.ERROR_CODE_UNSUPPORTED
|
|
acknowledgement.error.message = (
|
|
"command is not supported by this client build"
|
|
)
|
|
return acknowledgement
|
|
|
|
def _route_path(self, route_id: str) -> Path:
|
|
try:
|
|
return self._known_route_paths[route_id]
|
|
except KeyError as exc:
|
|
raise JobExecutionError("job route is not configured locally") from exc
|
|
|
|
|
|
def _route_error(exc: Exception) -> tuple[int, bool]:
|
|
if isinstance(exc, RoutePathConflict):
|
|
return common_pb2.ERROR_CODE_PATH_CONFLICT, False
|
|
if isinstance(exc, RouteSetupTimeout):
|
|
return common_pb2.ERROR_CODE_TIMEOUT, True
|
|
if isinstance(exc, PermissionError):
|
|
return common_pb2.ERROR_CODE_PERMISSION_DENIED, False
|
|
if isinstance(exc, RouteSetupError):
|
|
return common_pb2.ERROR_CODE_UNAVAILABLE, True
|
|
return common_pb2.ERROR_CODE_INTERNAL, False
|