feat: add transfer service adapters

This commit is contained in:
2026-07-23 03:39:33 +00:00
parent 2ec90b6a2d
commit 68db3ec4c5
5 changed files with 610 additions and 4 deletions
+9 -3
View File
@@ -19,9 +19,12 @@ derives canonical resource identities and selection ranges, builds lazy content
trees, and reports only safe Syncthing folders below the configured root.
Renamed/noncanonical torrent paths remain visible but are marked noncanonical
so later job preflight can reject them without hiding the resource.
The qBittorrent read adapter uses cookie authentication, bounded responses,
one reauthentication attempt on session expiry, hash-scoped file/metainfo
fetches, and never logs credentials, cookies, response bodies, or endpoints.
The qBittorrent adapter uses cookie authentication, bounded responses, one
reauthentication attempt on session expiry, and hash-scoped file/metainfo
fetches. Its mutation boundary adds targets stopped, applies only selected vs
skipped state, supports qBittorrent 4 pause and qBittorrent 5 stop semantics,
performs guarded full rechecks, and removes entries without requesting data
deletion. It never logs credentials, cookies, response bodies, or endpoints.
Durable inventory commands now stream bounded atomic summary, lookup, and
content-tree chunks. Page tokens are guarded by timestamp-independent snapshot
revisions, stale trees fail explicitly, and slow scans run outside the socket
@@ -39,6 +42,9 @@ then sparse-aware copy fallback; every intent/result is journaled in SQLite.
Receiver materialization never overwrites a path, can reuse a same-size regular
file for later qBittorrent verification, and replays completed operations
idempotently after a database reopen.
The Syncthing transfer observer requests scoped rescans and combines verified
local manifest payloads, folder completion, and outstanding job-prefix needs;
an advisory percentage alone is never treated as transfer completion.
```bash
archive-client --config /etc/archive-control/client.toml --check-config
+278
View File
@@ -4,7 +4,11 @@ from __future__ import annotations
import json
import threading
import time
import uuid
from dataclasses import dataclass
from http.cookiejar import CookieJar
from pathlib import PurePosixPath
from typing import Any
from urllib import error, parse, request
@@ -16,6 +20,25 @@ class QBittorrentError(RuntimeError):
pass
class QBittorrentDownloadAttempt(QBittorrentError):
pass
class QBittorrentHttpError(QBittorrentError):
def __init__(self, status: int):
super().__init__(f"qBittorrent HTTP request failed with status {status}")
self.status = status
@dataclass(frozen=True)
class RecheckResult:
torrent_hash: str
selected_file_indices: tuple[int, ...]
downloaded_bytes_before: int
downloaded_bytes_after: int
final_state: str
class _NoRedirect(request.HTTPRedirectHandler):
def redirect_request(self, *_: Any, **__: Any) -> None:
return None
@@ -68,6 +91,174 @@ class QBittorrentReader:
raise QBittorrentError("qBittorrent lookup is ambiguous")
return self._normalize(exact[0], torrent_hash)
def add_stopped(
self,
metainfo: bytes,
save_path: str,
) -> None:
if not metainfo:
raise QBittorrentError("torrent metainfo is empty")
candidate = PurePosixPath(save_path)
try:
candidate.relative_to(self.config.api_root)
except ValueError as exc:
raise QBittorrentError(
"qBittorrent save path is outside its configured root"
) from exc
if (
not candidate.is_absolute()
or ".." in candidate.parts
or "." in candidate.parts
):
raise QBittorrentError("qBittorrent save path is unsafe")
response = self._multipart(
"/api/v2/torrents/add",
{
"savepath": save_path,
"autoTMM": "false",
# qBittorrent 4.x calls this paused and 5.x calls it stopped.
"paused": "true",
"stopped": "true",
},
"torrents",
"source.torrent",
metainfo,
)
_require_mutation_success(response, "add torrent")
def set_selection(
self,
torrent_hash: str,
selected_file_indices: list[int] | tuple[int, ...],
total_file_count: int,
) -> None:
selected = sorted(set(selected_file_indices))
if total_file_count < 1:
raise QBittorrentError("torrent file count must be positive")
if not selected or selected[0] < 0 or selected[-1] >= total_file_count:
raise QBittorrentError("selected torrent file index is invalid")
self._post_form(
"/api/v2/torrents/filePrio",
{
"hash": torrent_hash,
"id": f"0-{total_file_count - 1}",
"priority": "0",
},
)
self._post_form(
"/api/v2/torrents/filePrio",
{
"hash": torrent_hash,
"id": "|".join(str(index) for index in selected),
"priority": "1",
},
)
def stop(self, torrent_hash: str) -> None:
try:
self._post_form(
"/api/v2/torrents/stop", {"hashes": torrent_hash}
)
except QBittorrentHttpError as exc:
if exc.status != 404:
raise
self._post_form(
"/api/v2/torrents/pause", {"hashes": torrent_hash}
)
def delete_entry(self, torrent_hash: str) -> None:
self._post_form(
"/api/v2/torrents/delete",
{"hashes": torrent_hash, "deleteFiles": "false"},
)
def recheck_and_wait(
self,
torrent_hash: str,
selected_file_indices: list[int] | tuple[int, ...],
*,
timeout: float,
poll_interval: float = 1,
) -> RecheckResult:
selected = tuple(sorted(set(selected_file_indices)))
if not selected:
raise QBittorrentError("recheck selection cannot be empty")
if timeout <= 0 or poll_interval < 0:
raise QBittorrentError("recheck timing values are invalid")
self.stop(torrent_hash)
baseline = self._downloaded_bytes(torrent_hash)
self._post_form(
"/api/v2/torrents/recheck", {"hashes": torrent_hash}
)
deadline = time.monotonic() + timeout
while True:
record = self._torrent_record(torrent_hash)
state = record.get("state")
if not isinstance(state, str):
raise QBittorrentError("qBittorrent torrent state is invalid")
downloaded = self._downloaded_bytes(torrent_hash)
if downloaded > baseline or _is_download_state(state):
try:
self.stop(torrent_hash)
finally:
raise QBittorrentDownloadAttempt(
"qBittorrent attempted to download during verification"
)
files = self._json(
"/api/v2/torrents/files", {"hash": torrent_hash}
)
if not isinstance(files, list) or not all(
isinstance(item, dict) for item in files
):
raise QBittorrentError("qBittorrent file list is invalid")
progress = {
item.get("index"): item.get("progress")
for item in files
}
if all(
isinstance(progress.get(index), (int, float))
and float(progress[index]) >= 1
for index in selected
) and state not in {"checkingDL", "checkingUP", "checkingResumeData"}:
return RecheckResult(
torrent_hash,
selected,
baseline,
downloaded,
state,
)
if time.monotonic() >= deadline:
raise QBittorrentError("qBittorrent recheck timed out")
time.sleep(min(poll_interval, max(0, deadline - time.monotonic())))
def _torrent_record(self, torrent_hash: str) -> dict[str, Any]:
torrents = self._json(
"/api/v2/torrents/info", {"hashes": torrent_hash}
)
if not isinstance(torrents, list):
raise QBittorrentError("qBittorrent lookup response is invalid")
exact = [
item for item in torrents
if isinstance(item, dict)
and str(item.get("hash", "")).lower() == torrent_hash.lower()
]
if len(exact) != 1:
raise QBittorrentError("qBittorrent target torrent is missing")
return exact[0]
def _downloaded_bytes(self, torrent_hash: str) -> int:
properties = self._json(
"/api/v2/torrents/properties", {"hash": torrent_hash}
)
if not isinstance(properties, dict):
raise QBittorrentError("qBittorrent properties response is invalid")
value = properties.get("total_downloaded")
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
raise QBittorrentError(
"qBittorrent downloaded byte counter is invalid"
)
return value
def _normalize(
self, torrent: dict[str, Any], torrent_hash: str
) -> NormalizedResource:
@@ -115,6 +306,81 @@ class QBittorrentReader:
"qBittorrent request failed after reauthentication"
) from retry_exc
def _post_form(self, path: str, parameters: dict[str, str]) -> None:
body = parse.urlencode(parameters).encode("utf-8")
call = request.Request(
self._url(path),
data=body,
method="POST",
headers={"Content-Type": "application/x-www-form-urlencoded"},
)
response = self._authorized_request(call, maximum=1024 * 1024)
_require_mutation_success(response, "mutate torrent")
def _multipart(
self,
path: str,
fields: dict[str, str],
file_field: str,
filename: str,
content: bytes,
) -> bytes:
boundary = f"archive-control-{uuid.uuid4().hex}"
chunks: list[bytes] = []
for name, value in fields.items():
chunks.extend((
f"--{boundary}\r\n".encode(),
(
f'Content-Disposition: form-data; name="{name}"\r\n\r\n'
).encode(),
value.encode(),
b"\r\n",
))
chunks.extend((
f"--{boundary}\r\n".encode(),
(
"Content-Disposition: form-data; "
f'name="{file_field}"; filename="{filename}"\r\n'
).encode(),
b"Content-Type: application/x-bittorrent\r\n\r\n",
content,
b"\r\n",
f"--{boundary}--\r\n".encode(),
))
call = request.Request(
self._url(path),
data=b"".join(chunks),
method="POST",
headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
)
return self._authorized_request(call, maximum=1024 * 1024)
def _authorized_request(self, call: request.Request, maximum: int) -> bytes:
with self._lock:
if not self._authenticated:
self._login()
try:
return self._read(
self._opener.open(call, timeout=self.timeout), maximum
)
except error.HTTPError as exc:
if exc.code not in {401, 403}:
raise QBittorrentHttpError(exc.code) from exc
self._authenticated = False
self._login()
try:
return self._read(
self._opener.open(call, timeout=self.timeout), maximum
)
except (error.HTTPError, error.URLError, OSError) as retry_exc:
raise QBittorrentError(
"qBittorrent request failed after reauthentication"
) from retry_exc
except (error.URLError, OSError) as exc:
raise QBittorrentError(
"qBittorrent service is unavailable"
) from exc
def _login(self) -> None:
body = parse.urlencode({
"username": self.config.username,
@@ -161,3 +427,15 @@ class QBittorrentReader:
if len(data) > maximum:
raise QBittorrentError("qBittorrent response exceeds limit")
return data
def _is_download_state(state: str) -> bool:
return state not in {"checkingDL"} and (
state.endswith("DL")
or state in {"downloading", "metaDL", "forcedMetaDL"}
)
def _require_mutation_success(response: bytes, operation: str) -> None:
if response.strip() not in {b"", b"Ok."}:
raise QBittorrentError(f"qBittorrent refused to {operation}")
+139
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import json
import os
import stat
import time
import uuid
from dataclasses import dataclass
@@ -12,6 +13,11 @@ from typing import Any, Protocol
from urllib import error, parse, request
from archive_clients.config import ConfigError, ServiceConfig
from archive_clients.transfer import (
TransferIntegrityError,
UnsafeTransferPath,
load_published_transfer,
)
from archive_control.v1 import route_pb2
@@ -91,6 +97,98 @@ class ConfiguredRoute:
created_folder: bool
@dataclass(frozen=True)
class SyncthingTransferStatus:
fraction_complete: float
bytes_complete: int
bytes_total: int
complete: bool
needed_items: int
class SyncthingTransferObserver:
"""Observe one exact Archive Control job prefix within a route folder."""
def __init__(
self,
transport: SyncthingTransport,
folder_id: str,
job_relative_path: str,
local_job_directory: Path,
):
relative = PurePosixPath(job_relative_path)
if (
not job_relative_path
or relative.is_absolute()
or any(part in {"", ".", ".."} for part in relative.parts)
):
raise UnsafeTransferPath("Syncthing job prefix is unsafe")
self.transport = transport
self.folder_id = folder_id
self.job_relative_path = relative.as_posix()
self.local_job_directory = local_job_directory
def rescan(self) -> None:
self.transport.post(
"/rest/db/scan?"
+ parse.urlencode({
"folder": self.folder_id,
"sub": self.job_relative_path,
})
)
def status(self) -> SyncthingTransferStatus:
published = load_published_transfer(self.local_job_directory)
total = 0
for entry in published.manifest.files:
total += _verified_job_file_size(
self.local_job_directory,
entry.payload_relative_path,
entry.logical_bytes,
)
for artifact in published.manifest.artifacts:
total += _verified_job_file_size(
self.local_job_directory,
artifact.payload_relative_path,
artifact.logical_bytes,
)
completion = self.transport.get_json(
"/rest/db/completion?"
+ parse.urlencode({"folder": self.folder_id})
)
raw_fraction = completion.get("completion")
if (
isinstance(raw_fraction, bool)
or not isinstance(raw_fraction, (int, float))
or not 0 <= float(raw_fraction) <= 100
):
raise RouteSetupError("Syncthing completion response is invalid")
need = self.transport.get_json(
"/rest/db/need?"
+ parse.urlencode({
"folder": self.folder_id,
"page": 1,
"perpage": 1000,
})
)
needed_names = _needed_names(need)
prefix = self.job_relative_path.rstrip("/") + "/"
relevant = {
name for name in needed_names
if name == self.job_relative_path or name.startswith(prefix)
}
fraction = float(raw_fraction) / 100
complete = fraction == 1 and not relevant
return SyncthingTransferStatus(
fraction,
total if complete else int(total * fraction),
total,
complete,
len(relevant),
)
class SyncthingRouteManager:
def __init__(
self,
@@ -315,6 +413,47 @@ def _by_key(
return matches[0] if matches else None
def _verified_job_file_size(
job_directory: Path,
relative_path: str,
expected_bytes: int,
) -> int:
relative = PurePosixPath(relative_path)
if (
not relative_path
or relative.is_absolute()
or any(part in {"", ".", ".."} for part in relative.parts)
):
raise TransferIntegrityError("manifest payload path is unsafe")
current = job_directory
for component in relative.parts[:-1]:
current = current / component
metadata = current.lstat()
if not stat.S_ISDIR(metadata.st_mode):
raise TransferIntegrityError(
"manifest payload parent is not a directory"
)
metadata = (current / relative.name).lstat()
if not stat.S_ISREG(metadata.st_mode) or metadata.st_size != expected_bytes:
raise TransferIntegrityError(
"manifest payload does not match its declared size"
)
return metadata.st_size
def _needed_names(value: dict[str, Any]) -> set[str]:
result: set[str] = set()
for key in ("progress", "queued", "rest"):
items = value.get(key, [])
if not isinstance(items, list):
raise RouteSetupError("Syncthing need response is invalid")
for item in items:
if not isinstance(item, dict) or not isinstance(item.get("name"), str):
raise RouteSetupError("Syncthing need item is invalid")
result.add(item["name"])
return result
def _nonce_name(client_id: str) -> str:
return f".archive-control-route-nonce.{client_id}"
+87
View File
@@ -6,6 +6,7 @@ import tempfile
import unittest
from pathlib import Path, PurePosixPath
from unittest.mock import patch
from urllib import error
from archive_clients.bencode import encode
from archive_clients.config import ServiceConfig
@@ -35,6 +36,8 @@ class _Opener:
def open(self, call, timeout):
self.calls.append(call)
response = next(self.values)
if isinstance(response, BaseException):
raise response
if isinstance(response, tuple):
return _Response(*response)
return _Response(response)
@@ -95,6 +98,90 @@ class QBittorrentReaderTests(unittest.TestCase):
self.assertEqual(resources, [])
self.assertEqual(len(opener.calls), 2)
def test_stopped_add_selection_recheck_and_entry_only_delete(self):
torrent_hash = "a" * 40
responses = [
b"Ok.", # login
b"Ok.", # multipart add
b"", # skip all files
b"", # select requested files
b"", # stop before recheck
b'{"total_downloaded":0}',
b"", # recheck
json.dumps([{
"hash": torrent_hash, "state": "checkingUP",
}]).encode(),
b'{"total_downloaded":0}',
b'[{"index":0,"progress":1},{"index":1,"progress":0}]',
json.dumps([{
"hash": torrent_hash, "state": "stoppedUP",
}]).encode(),
b'{"total_downloaded":0}',
b'[{"index":0,"progress":1},{"index":1,"progress":0}]',
b"", # entry-only delete
]
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
password = root / "password"
password.write_text("secret", encoding="utf-8")
os.chmod(password, 0o600)
config = ServiceConfig(
"http://qb", PurePosixPath("/downloads"), root,
username="admin", password_file=password,
)
opener = _Opener(responses)
with patch(
"archive_clients.qbittorrent.request.build_opener",
return_value=opener,
):
adapter = QBittorrentReader(config)
adapter.add_stopped(b"torrent", "/downloads/archive")
adapter.set_selection(torrent_hash, [0], 2)
result = adapter.recheck_and_wait(
torrent_hash, [0], timeout=1, poll_interval=0
)
adapter.delete_entry(torrent_hash)
self.assertEqual(result.final_state, "stoppedUP")
self.assertEqual(result.selected_file_indices, (0,))
calls = [
call for call in opener.calls
if not isinstance(call, str)
]
add = calls[1]
self.assertIn(b'name="stopped"\r\n\r\ntrue', add.data)
form_bodies = [
call.data.decode()
for call in calls[2:]
if getattr(call, "data", None)
]
self.assertIn("priority=0", form_bodies[0])
self.assertIn("priority=1", form_bodies[1])
self.assertIn("deleteFiles=false", form_bodies[-1])
def test_stop_falls_back_to_qbittorrent_4_pause_endpoint(self):
missing = error.HTTPError(
"http://qb/api/v2/torrents/stop", 404, "not found", {}, None
)
responses = [b"Ok.", missing, b""]
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
password = root / "password"
password.write_text("secret", encoding="utf-8")
os.chmod(password, 0o600)
config = ServiceConfig(
"http://qb", PurePosixPath("/downloads"), root,
username="admin", password_file=password,
)
opener = _Opener(responses)
with patch(
"archive_clients.qbittorrent.request.build_opener",
return_value=opener,
):
QBittorrentReader(config).stop("a" * 40)
self.assertTrue(opener.calls[-1].full_url.endswith("/torrents/pause"))
if __name__ == "__main__":
unittest.main()
+97 -1
View File
@@ -3,14 +3,18 @@ import tempfile
import time
import unittest
from pathlib import Path, PurePosixPath
from uuid import uuid4
from archive_clients.config import ServiceConfig
from archive_clients.state import ClientStore
from archive_clients.syncthing import (
RoutePathConflict,
RouteSetupTimeout,
SyncthingRouteManager,
SyncthingTransferObserver,
)
from archive_control.v1 import route_pb2
from archive_clients.transfer import stage_transfer
from archive_control.v1 import route_pb2, transfer_pb2
class FakeTransport:
@@ -40,6 +44,26 @@ class FakeTransport:
self.posts.append(path)
class FakeTransferTransport:
def __init__(self, completion, need):
self.completion = completion
self.need = need
self.posts = []
def get_json(self, path):
if path.startswith("/rest/db/completion?"):
return self.completion
if path.startswith("/rest/db/need?"):
return self.need
raise AssertionError(path)
def put_json(self, path, payload):
raise AssertionError((path, payload))
def post(self, path):
self.posts.append(path)
class SyncthingRouteManagerTests(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
@@ -164,6 +188,78 @@ class SyncthingRouteManagerTests(unittest.TestCase):
)
self.assertTrue(self.transport.posts)
def test_job_prefix_rescan_progress_and_completion(self):
source = Path(self.temp_dir.name) / "source"
source.mkdir()
(source / "payload.bin").write_bytes(b"x" * 8192)
metainfo = Path(self.temp_dir.name) / "source.torrent"
metainfo.write_bytes(b"torrent")
store = ClientStore(Path(self.temp_dir.name) / "state.db")
store.initialize()
manifest = transfer_pb2.TransferManifest(
manifest_version=1,
job_id=str(uuid4()),
source_client_id="cache-1",
target_client_id="archive-1",
route_id="route-1",
)
manifest.resource_id.info_hash_v1_hex = "a" * 40
manifest.created_at.seconds = 1_700_000_000
manifest.files.add(
file_index=0,
payload_relative_path="payload/payload.bin",
target_canonical_path="payload.bin",
logical_bytes=8192,
)
manifest.artifacts.add(
kind=transfer_pb2.ARTIFACT_KIND_TORRENT_FILE,
payload_relative_path="metainfo/source.torrent",
logical_bytes=metainfo.stat().st_size,
)
published = stage_transfer(
manifest,
source_root=source,
sync_root=self.root,
store=store,
artifact_sources={"metainfo/source.torrent": metainfo},
)
relative = (
f".archive-control/jobs/{manifest.job_id}"
)
in_progress_transport = FakeTransferTransport(
{"completion": 50},
{
"progress": [{"name": f"{relative}/payload/payload.bin"}],
"queued": [],
"rest": [],
},
)
observer = SyncthingTransferObserver(
in_progress_transport,
"route-1",
relative,
published.job_directory,
)
observer.rescan()
in_progress = observer.status()
self.assertEqual(in_progress.fraction_complete, 0.5)
self.assertFalse(in_progress.complete)
self.assertEqual(in_progress.needed_items, 1)
self.assertIn("folder=route-1", in_progress_transport.posts[0])
self.assertIn("sub=.archive-control%2Fjobs%2F", in_progress_transport.posts[0])
complete = SyncthingTransferObserver(
FakeTransferTransport(
{"completion": 100},
{"progress": [], "queued": [], "rest": []},
),
"route-1",
relative,
published.job_directory,
).status()
self.assertTrue(complete.complete)
self.assertEqual(complete.bytes_complete, complete.bytes_total)
if __name__ == "__main__":
unittest.main()