"""Scoped Syncthing route provisioning and nonce verification.""" from __future__ import annotations import json import os import stat import time import uuid from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Any, Protocol from urllib import error, parse, request from archive_clients.config import ConfigError, ServiceConfig from archive_clients.transfer import ( TransferIntegrityError, UnsafeTransferPath, load_published_transfer, ) from archive_control.v1 import route_pb2 class RouteSetupError(RuntimeError): pass class RoutePathConflict(RouteSetupError): pass class RouteSetupTimeout(RouteSetupError): pass class SyncthingTransport(Protocol): def get_json(self, path: str) -> dict[str, Any]: ... def put_json(self, path: str, payload: dict[str, Any]) -> None: ... def post(self, path: str) -> None: ... class SyncthingHttp: def __init__(self, config: ServiceConfig, timeout: float = 10): self.endpoint = config.endpoint.rstrip("/") self.api_key = config.read_api_key() or "" self.timeout = timeout self.opener = request.build_opener(_NoRedirect()) def get_json(self, path: str) -> dict[str, Any]: response = self._open(path, "GET") try: value = json.loads(response.decode("utf-8")) except (UnicodeError, json.JSONDecodeError) as exc: raise RouteSetupError("Syncthing returned invalid JSON") from exc if not isinstance(value, dict): raise RouteSetupError("Syncthing returned a non-object response") return value def put_json(self, path: str, payload: dict[str, Any]) -> None: self._open( path, "PUT", json.dumps(payload, separators=(",", ":")).encode("utf-8"), ) def post(self, path: str) -> None: self._open(path, "POST", b"") def _open(self, path: str, method: str, body: bytes | None = None) -> bytes: headers = {"X-API-Key": self.api_key} if body is not None: headers["Content-Type"] = "application/json" req = request.Request( f"{self.endpoint}{path}", data=body, headers=headers, method=method ) try: with self.opener.open(req, timeout=self.timeout) as response: return response.read() except error.HTTPError as exc: if exc.code in {401, 403}: raise PermissionError("Syncthing authentication failed") from exc raise RouteSetupError( f"Syncthing HTTP request failed with status {exc.code}" ) from exc except (error.URLError, OSError, UnicodeError, json.JSONDecodeError) as exc: raise RouteSetupError("Syncthing API is unavailable") from exc @dataclass(frozen=True) class ConfiguredRoute: local_device_id: str local_path: Path local_route: route_pb2.LocalRoute created_device: bool created_folder: bool @dataclass(frozen=True) class SyncthingTransferStatus: fraction_complete: float bytes_complete: int bytes_total: int complete: bool needed_items: int class SyncthingTransferObserver: """Observe one exact Archive Control job prefix within a route folder.""" def __init__( self, transport: SyncthingTransport, folder_id: str, job_relative_path: str, local_job_directory: Path, ): relative = PurePosixPath(job_relative_path) if ( not job_relative_path or relative.is_absolute() or any(part in {"", ".", ".."} for part in relative.parts) ): raise UnsafeTransferPath("Syncthing job prefix is unsafe") self.transport = transport self.folder_id = folder_id self.job_relative_path = relative.as_posix() self.local_job_directory = local_job_directory def rescan(self) -> None: self.transport.post( "/rest/db/scan?" + parse.urlencode({ "folder": self.folder_id, "sub": self.job_relative_path, }) ) def status(self) -> SyncthingTransferStatus: published = load_published_transfer(self.local_job_directory) total = 0 for entry in published.manifest.files: total += _verified_job_file_size( self.local_job_directory, entry.payload_relative_path, entry.logical_bytes, ) for artifact in published.manifest.artifacts: total += _verified_job_file_size( self.local_job_directory, artifact.payload_relative_path, artifact.logical_bytes, ) completion = self.transport.get_json( "/rest/db/completion?" + parse.urlencode({"folder": self.folder_id}) ) raw_fraction = completion.get("completion") if ( isinstance(raw_fraction, bool) or not isinstance(raw_fraction, (int, float)) or not 0 <= float(raw_fraction) <= 100 ): raise RouteSetupError("Syncthing completion response is invalid") need = self.transport.get_json( "/rest/db/need?" + parse.urlencode({ "folder": self.folder_id, "page": 1, "perpage": 1000, }) ) needed_names = _needed_names(need) prefix = self.job_relative_path.rstrip("/") + "/" relevant = { name for name in needed_names if name == self.job_relative_path or name.startswith(prefix) } fraction = float(raw_fraction) / 100 complete = fraction == 1 and not relevant return SyncthingTransferStatus( fraction, total if complete else int(total * fraction), total, complete, len(relevant), ) class SyncthingRouteManager: def __init__( self, config: ServiceConfig, sparse_supported: bool, transport: SyncthingTransport | None = None, poll_interval: float = 1, ): self.config = config self.sparse_supported = sparse_supported self.transport = transport or SyncthingHttp(config) self.poll_interval = poll_interval def configure( self, spec: route_pb2.EnsureRouteSpec, deadline: float ) -> ConfiguredRoute: local_path, api_path = self._route_paths(spec.local_relative_path) status = self.transport.get_json("/rest/system/status") local_device_id = status.get("myID") if not isinstance(local_device_id, str) or not local_device_id: raise RouteSetupError("Syncthing returned no local device ID") configuration = self.transport.get_json("/rest/config") devices = _object_list(configuration, "devices") folders = _object_list(configuration, "folders") folder = _by_key(folders, "id", spec.route_id) if folder is not None: self._validate_folder( folder, api_path, local_device_id, spec.peer_syncthing_device_id, ) peer = _by_key(devices, "deviceID", spec.peer_syncthing_device_id) created_device = peer is None if peer is None: addresses = list(spec.peer_addresses) or ["dynamic"] self.transport.put_json( "/rest/config/devices/" + parse.quote(spec.peer_syncthing_device_id, safe=""), { "deviceID": spec.peer_syncthing_device_id, "name": spec.peer_client_id, "addresses": addresses, }, ) if folder is None: self._prepare_new_directory(local_path) self.transport.put_json( "/rest/config/folders/" + parse.quote(spec.route_id, safe=""), { "id": spec.route_id, "label": f"archive-control:{spec.route_id}", "path": api_path, "type": "sendreceive", "devices": [ {"deviceID": local_device_id}, {"deviceID": spec.peer_syncthing_device_id}, ], }, ) else: local_path.mkdir(parents=True, exist_ok=True) while True: current = self.transport.get_json("/rest/config") current_folder = _by_key( _object_list(current, "folders"), "id", spec.route_id ) if current_folder is not None: self._validate_folder( current_folder, api_path, local_device_id, spec.peer_syncthing_device_id, ) break self._wait(deadline) route = route_pb2.LocalRoute( route_id=spec.route_id, local_relative_path=spec.local_relative_path, folder_type=route_pb2.SYNCTHING_FOLDER_TYPE_SEND_RECEIVE, local_syncthing_device_id=local_device_id, peer_syncthing_device_ids=[spec.peer_syncthing_device_id], state=route_pb2.ROUTE_STATE_PROVISIONING, writable=True, sparse_supported=self.sparse_supported, archive_control_created=folder is None, detail="Syncthing device and folder configuration verified", ) route.observed_at.GetCurrentTime() return ConfiguredRoute( local_device_id, local_path, route, created_device, folder is None, ) def verify( self, configured: ConfiguredRoute, spec: route_pb2.EnsureRouteSpec, local_client_id: str, nonce: str, deadline: float, ) -> tuple[bool, bool]: local_nonce = configured.local_path / _nonce_name(local_client_id) _atomic_json( local_nonce, { "route_id": spec.route_id, "client_id": local_client_id, "nonce": nonce, }, ) peer_seen_locally = False local_seen_by_peer = False while not (peer_seen_locally and local_seen_by_peer): peer_nonce_path = configured.local_path / _nonce_name( spec.peer_client_id ) peer_nonce = _read_nonce( peer_nonce_path, spec.route_id, spec.peer_client_id ) if peer_nonce is not None: peer_seen_locally = True _atomic_json( configured.local_path / _ack_name(spec.peer_client_id, local_client_id), { "route_id": spec.route_id, "nonce": peer_nonce, }, ) acknowledgement = _read_ack( configured.local_path / _ack_name(local_client_id, spec.peer_client_id), spec.route_id, ) local_seen_by_peer = acknowledgement == nonce if peer_seen_locally and local_seen_by_peer: break try: self.transport.post( "/rest/db/scan?" + parse.urlencode({"folder": spec.route_id}) ) except RouteSetupError: pass self._wait(deadline) return local_seen_by_peer, peer_seen_locally def _route_paths(self, relative_path: str) -> tuple[Path, str]: relative = PurePosixPath(relative_path) if ( not relative_path or relative.is_absolute() or any(part in {"", ".", ".."} for part in relative.parts) ): raise RoutePathConflict("route path is unsafe") api_path = (self.config.api_root / relative).as_posix() try: local_path = self.config.roots.api_to_local(api_path) except ConfigError as exc: raise RoutePathConflict("route path is outside the sync root") from exc resolved_root = self.config.roots.local_root_for_api(api_path).resolve( strict=False ) try: local_path.resolve(strict=False).relative_to(resolved_root) except ValueError as exc: raise RoutePathConflict("route path escapes through a symlink") from exc return local_path, api_path @staticmethod def _prepare_new_directory(path: Path) -> None: if path.exists() and any(path.iterdir()): raise RoutePathConflict("new route path contains unrelated data") path.mkdir(parents=True, exist_ok=True) def _validate_folder( self, folder: dict[str, Any], api_path: str, local_device_id: str, peer_device_id: str, ) -> None: raw_devices = folder.get("devices") device_ids = { item.get("deviceID") for item in raw_devices if isinstance(item, dict) and isinstance(item.get("deviceID"), str) } if isinstance(raw_devices, list) else set() try: configured_api_path = self._normalized_folder_api_path( folder.get("path") ) except RoutePathConflict as exc: raise RoutePathConflict( "existing route ID uses a different path" ) from exc if configured_api_path != api_path: raise RoutePathConflict("existing route ID uses a different path") if folder.get("type") != "sendreceive": raise RouteSetupError("existing route folder is not sendreceive") if device_ids != {local_device_id, peer_device_id}: raise RouteSetupError("existing route folder has different devices") if folder.get("paused") is True: raise RouteSetupError("existing route folder is paused") def _normalized_folder_api_path(self, value: Any) -> str: """Normalize Syncthing's absolute and home-relative path spellings.""" if not isinstance(value, str) or not value: raise RoutePathConflict("existing route folder path is invalid") candidate = PurePosixPath(value) if candidate.parts and candidate.parts[0] == "~": candidate = self.config.api_root.joinpath(*candidate.parts[1:]) if not candidate.is_absolute() or any( part in {"", ".", ".."} for part in candidate.parts ): raise RoutePathConflict("existing route folder path is unsafe") normalized = candidate.as_posix() try: self.config.roots.api_to_local(normalized) except ConfigError as exc: raise RoutePathConflict( "existing route folder path is outside the sync root" ) from exc return normalized def _wait(self, deadline: float) -> None: if time.monotonic() >= deadline: raise RouteSetupTimeout("route setup timed out") time.sleep(min(self.poll_interval, max(0, deadline - time.monotonic()))) def _object_list(value: dict[str, Any], key: str) -> list[dict[str, Any]]: items = value.get(key) if not isinstance(items, list) or any(not isinstance(item, dict) for item in items): raise RouteSetupError(f"Syncthing configuration {key} is invalid") return items def _by_key( items: list[dict[str, Any]], key: str, wanted: str ) -> dict[str, Any] | None: matches = [item for item in items if item.get(key) == wanted] if len(matches) > 1: raise RouteSetupError(f"Syncthing configuration contains duplicate {key}") return matches[0] if matches else None def _verified_job_file_size( job_directory: Path, relative_path: str, expected_bytes: int, ) -> int: relative = PurePosixPath(relative_path) if ( not relative_path or relative.is_absolute() or any(part in {"", ".", ".."} for part in relative.parts) ): raise TransferIntegrityError("manifest payload path is unsafe") current = job_directory for component in relative.parts[:-1]: current = current / component metadata = current.lstat() if not stat.S_ISDIR(metadata.st_mode): raise TransferIntegrityError( "manifest payload parent is not a directory" ) metadata = (current / relative.name).lstat() if not stat.S_ISREG(metadata.st_mode) or metadata.st_size != expected_bytes: raise TransferIntegrityError( "manifest payload does not match its declared size" ) return metadata.st_size def _needed_names(value: dict[str, Any]) -> set[str]: result: set[str] = set() for key in ("progress", "queued", "rest"): items = value.get(key, []) if not isinstance(items, list): raise RouteSetupError("Syncthing need response is invalid") for item in items: if not isinstance(item, dict) or not isinstance(item.get("name"), str): raise RouteSetupError("Syncthing need item is invalid") result.add(item["name"]) return result def _nonce_name(client_id: str) -> str: return f".archive-control-route-nonce.{client_id}" def _ack_name(nonce_owner: str, observer: str) -> str: return f".archive-control-route-ack.{nonce_owner}.{observer}" def _atomic_json(path: Path, value: dict[str, Any]) -> None: encoded = json.dumps(value, sort_keys=True, separators=(",", ":")) try: if path.read_text(encoding="utf-8") == encoded: return except FileNotFoundError: pass except (OSError, UnicodeError) as exc: raise RouteSetupError("route verification file is unreadable") from exc temporary = path.with_name( f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp" ) with temporary.open("x", encoding="utf-8") as target: target.write(encoded) target.flush() os.fsync(target.fileno()) os.replace(temporary, path) directory = os.open(path.parent, os.O_RDONLY) try: os.fsync(directory) finally: os.close(directory) def _read_nonce(path: Path, route_id: str, client_id: str) -> str | None: value = _read_json(path) if value is None: return None if value.get("route_id") != route_id or value.get("client_id") != client_id: raise RouteSetupError("peer route nonce identity is invalid") nonce = value.get("nonce") if not isinstance(nonce, str) or not nonce: raise RouteSetupError("peer route nonce is invalid") return nonce def _read_ack(path: Path, route_id: str) -> str | None: value = _read_json(path) if value is None: return None if value.get("route_id") != route_id: raise RouteSetupError("peer route acknowledgement identity is invalid") nonce = value.get("nonce") if not isinstance(nonce, str) or not nonce: raise RouteSetupError("peer route acknowledgement is invalid") return nonce def _read_json(path: Path) -> dict[str, Any] | None: try: value = json.loads(path.read_text(encoding="utf-8")) except FileNotFoundError: return None except (OSError, UnicodeError, json.JSONDecodeError) as exc: raise RouteSetupError("route verification file is unreadable") from exc if not isinstance(value, dict): raise RouteSetupError("route verification file is invalid") return value class _NoRedirect(request.HTTPRedirectHandler): def redirect_request(self, req, fp, code, msg, headers, newurl): return None