feat: add read-only resource and route discovery
This commit is contained in:
@@ -14,6 +14,12 @@ Startup probes report qBittorrent/Web API/libtorrent versions, Syncthing
|
|||||||
version/device identity, service health, and per-root hardlink/reflink/sparse
|
version/device identity, service health, and per-root hardlink/reflink/sparse
|
||||||
support without exposing local paths to control.
|
support without exposing local paths to control.
|
||||||
|
|
||||||
|
The first read-only discovery layer strictly decodes v1/v2/hybrid metainfo,
|
||||||
|
derives canonical resource identities and selection ranges, builds lazy content
|
||||||
|
trees, and reports only safe Syncthing folders below the configured root.
|
||||||
|
Renamed/noncanonical torrent paths remain visible but are marked noncanonical
|
||||||
|
so later job preflight can reject them without hiding the resource.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
archive-client --config /etc/archive-control/client.toml --check-config
|
archive-client --config /etc/archive-control/client.toml --check-config
|
||||||
archive-client --config /etc/archive-control/client.toml --mode archive
|
archive-client --config /etc/archive-control/client.toml --mode archive
|
||||||
|
|||||||
@@ -0,0 +1,225 @@
|
|||||||
|
"""Strict BitTorrent metainfo decoding with exact info-dictionary hashing."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
class BencodeError(ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class MetaFile:
|
||||||
|
path: str
|
||||||
|
logical_bytes: int
|
||||||
|
padding: bool
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Metainfo:
|
||||||
|
info_hash_v1_hex: str
|
||||||
|
info_hash_v2_hex: str
|
||||||
|
files: tuple[MetaFile, ...]
|
||||||
|
|
||||||
|
|
||||||
|
class _Parser:
|
||||||
|
def __init__(self, data: bytes):
|
||||||
|
self.data = data
|
||||||
|
self.position = 0
|
||||||
|
self.info_bytes: bytes | None = None
|
||||||
|
|
||||||
|
def parse(self, depth: int = 0) -> Any:
|
||||||
|
if depth > 100 or self.position >= len(self.data):
|
||||||
|
raise BencodeError("invalid or excessively nested bencode")
|
||||||
|
marker = self.data[self.position]
|
||||||
|
if marker == ord("i"):
|
||||||
|
return self._integer()
|
||||||
|
if marker == ord("l"):
|
||||||
|
return self._list(depth)
|
||||||
|
if marker == ord("d"):
|
||||||
|
return self._dictionary(depth)
|
||||||
|
if ord("0") <= marker <= ord("9"):
|
||||||
|
return self._bytes()
|
||||||
|
raise BencodeError("invalid bencode marker")
|
||||||
|
|
||||||
|
def _integer(self) -> int:
|
||||||
|
end = self.data.find(b"e", self.position + 1)
|
||||||
|
if end < 0:
|
||||||
|
raise BencodeError("unterminated integer")
|
||||||
|
raw = self.data[self.position + 1:end]
|
||||||
|
if (
|
||||||
|
not raw
|
||||||
|
or raw == b"-0"
|
||||||
|
or raw.startswith(b"+")
|
||||||
|
or (raw.startswith(b"0") and len(raw) > 1)
|
||||||
|
or (raw.startswith(b"-0"))
|
||||||
|
):
|
||||||
|
raise BencodeError("non-canonical integer")
|
||||||
|
try:
|
||||||
|
value = int(raw)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise BencodeError("invalid integer") from exc
|
||||||
|
self.position = end + 1
|
||||||
|
return value
|
||||||
|
|
||||||
|
def _bytes(self) -> bytes:
|
||||||
|
separator = self.data.find(b":", self.position)
|
||||||
|
if separator < 0:
|
||||||
|
raise BencodeError("invalid byte string")
|
||||||
|
raw_length = self.data[self.position:separator]
|
||||||
|
if not raw_length or (raw_length.startswith(b"0") and len(raw_length) > 1):
|
||||||
|
raise BencodeError("non-canonical byte-string length")
|
||||||
|
try:
|
||||||
|
length = int(raw_length)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise BencodeError("invalid byte-string length") from exc
|
||||||
|
start = separator + 1
|
||||||
|
end = start + length
|
||||||
|
if end > len(self.data):
|
||||||
|
raise BencodeError("truncated byte string")
|
||||||
|
self.position = end
|
||||||
|
return self.data[start:end]
|
||||||
|
|
||||||
|
def _list(self, depth: int) -> list[Any]:
|
||||||
|
self.position += 1
|
||||||
|
result = []
|
||||||
|
while self.position < len(self.data) and self.data[self.position] != ord("e"):
|
||||||
|
result.append(self.parse(depth + 1))
|
||||||
|
if self.position >= len(self.data):
|
||||||
|
raise BencodeError("unterminated list")
|
||||||
|
self.position += 1
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _dictionary(self, depth: int) -> dict[bytes, Any]:
|
||||||
|
self.position += 1
|
||||||
|
result: dict[bytes, Any] = {}
|
||||||
|
previous: bytes | None = None
|
||||||
|
while self.position < len(self.data) and self.data[self.position] != ord("e"):
|
||||||
|
key = self._bytes()
|
||||||
|
if previous is not None and key <= previous:
|
||||||
|
raise BencodeError("dictionary keys are duplicate or unsorted")
|
||||||
|
previous = key
|
||||||
|
value_start = self.position
|
||||||
|
value = self.parse(depth + 1)
|
||||||
|
if depth == 0 and key == b"info":
|
||||||
|
self.info_bytes = self.data[value_start:self.position]
|
||||||
|
result[key] = value
|
||||||
|
if self.position >= len(self.data):
|
||||||
|
raise BencodeError("unterminated dictionary")
|
||||||
|
self.position += 1
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def decode_metainfo(data: bytes) -> Metainfo:
|
||||||
|
parser = _Parser(data)
|
||||||
|
root = parser.parse()
|
||||||
|
if parser.position != len(data) or not isinstance(root, dict):
|
||||||
|
raise BencodeError("metainfo must be one top-level dictionary")
|
||||||
|
info = root.get(b"info")
|
||||||
|
if not isinstance(info, dict) or parser.info_bytes is None:
|
||||||
|
raise BencodeError("metainfo has no info dictionary")
|
||||||
|
is_v1 = isinstance(info.get(b"pieces"), bytes)
|
||||||
|
is_v2 = info.get(b"meta version") == 2
|
||||||
|
if not is_v1 and not is_v2:
|
||||||
|
raise BencodeError("unsupported torrent metainfo version")
|
||||||
|
files = _v1_files(info) if is_v1 else _v2_files(info)
|
||||||
|
return Metainfo(
|
||||||
|
hashlib.sha1(parser.info_bytes).hexdigest() if is_v1 else "",
|
||||||
|
hashlib.sha256(parser.info_bytes).hexdigest() if is_v2 else "",
|
||||||
|
tuple(files),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def encode(value: Any) -> bytes:
|
||||||
|
if isinstance(value, bytes):
|
||||||
|
return str(len(value)).encode("ascii") + b":" + value
|
||||||
|
if isinstance(value, str):
|
||||||
|
return encode(value.encode("utf-8"))
|
||||||
|
if isinstance(value, bool) or not isinstance(value, (int, list, dict)):
|
||||||
|
raise BencodeError("unsupported bencode value")
|
||||||
|
if isinstance(value, int):
|
||||||
|
return f"i{value}e".encode("ascii")
|
||||||
|
if isinstance(value, list):
|
||||||
|
return b"l" + b"".join(encode(item) for item in value) + b"e"
|
||||||
|
keys = sorted(value)
|
||||||
|
if any(not isinstance(key, bytes) for key in keys):
|
||||||
|
raise BencodeError("dictionary keys must be bytes")
|
||||||
|
return b"d" + b"".join(encode(key) + encode(value[key]) for key in keys) + b"e"
|
||||||
|
|
||||||
|
|
||||||
|
def _v1_files(info: dict[bytes, Any]) -> list[MetaFile]:
|
||||||
|
name = _component(info.get(b"name.utf-8", info.get(b"name")))
|
||||||
|
raw_files = info.get(b"files")
|
||||||
|
if raw_files is None:
|
||||||
|
length = _length(info.get(b"length"))
|
||||||
|
return [MetaFile(name, length, _padding(info))]
|
||||||
|
if not isinstance(raw_files, list):
|
||||||
|
raise BencodeError("v1 files must be a list")
|
||||||
|
result = []
|
||||||
|
for item in raw_files:
|
||||||
|
if not isinstance(item, dict) or not isinstance(item.get(b"path"), list):
|
||||||
|
raise BencodeError("invalid v1 file record")
|
||||||
|
raw_path = item.get(b"path.utf-8", item[b"path"])
|
||||||
|
if not isinstance(raw_path, list):
|
||||||
|
raise BencodeError("invalid v1 UTF-8 file path")
|
||||||
|
components = [name] + [_component(part) for part in raw_path]
|
||||||
|
result.append(MetaFile(
|
||||||
|
"/".join(components), _length(item.get(b"length")),
|
||||||
|
_padding(item),
|
||||||
|
))
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _v2_files(info: dict[bytes, Any]) -> list[MetaFile]:
|
||||||
|
name = _component(info.get(b"name.utf-8", info.get(b"name")))
|
||||||
|
tree = info.get(b"file tree")
|
||||||
|
if not isinstance(tree, dict):
|
||||||
|
raise BencodeError("v2 metainfo has no file tree")
|
||||||
|
result: list[MetaFile] = []
|
||||||
|
|
||||||
|
def visit(node: dict[bytes, Any], components: list[str]) -> None:
|
||||||
|
leaf = node.get(b"")
|
||||||
|
if leaf is not None:
|
||||||
|
if not isinstance(leaf, dict):
|
||||||
|
raise BencodeError("invalid v2 file leaf")
|
||||||
|
result.append(MetaFile(
|
||||||
|
"/".join([name] + components),
|
||||||
|
_length(leaf.get(b"length")),
|
||||||
|
_padding(leaf),
|
||||||
|
))
|
||||||
|
for key in sorted(item for item in node if item != b""):
|
||||||
|
child = node[key]
|
||||||
|
if not isinstance(child, dict):
|
||||||
|
raise BencodeError("invalid v2 file tree node")
|
||||||
|
visit(child, components + [_component(key)])
|
||||||
|
|
||||||
|
visit(tree, [])
|
||||||
|
if not result:
|
||||||
|
raise BencodeError("v2 file tree is empty")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _component(value: Any) -> str:
|
||||||
|
if not isinstance(value, bytes):
|
||||||
|
raise BencodeError("torrent path component must be bytes")
|
||||||
|
try:
|
||||||
|
decoded = value.decode("utf-8")
|
||||||
|
except UnicodeDecodeError as exc:
|
||||||
|
raise BencodeError("torrent path component is not UTF-8") from exc
|
||||||
|
if not decoded or decoded in {".", ".."} or "/" in decoded or "\x00" in decoded:
|
||||||
|
raise BencodeError("unsafe torrent path component")
|
||||||
|
return decoded
|
||||||
|
|
||||||
|
|
||||||
|
def _length(value: Any) -> int:
|
||||||
|
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
||||||
|
raise BencodeError("invalid torrent file length")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _padding(value: dict[bytes, Any]) -> bool:
|
||||||
|
attributes = value.get(b"attr", b"")
|
||||||
|
return isinstance(attributes, bytes) and b"p" in attributes
|
||||||
@@ -52,7 +52,9 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||||||
try:
|
try:
|
||||||
service_probes = [
|
service_probes = [
|
||||||
probe_qbittorrent(config.qbittorrent),
|
probe_qbittorrent(config.qbittorrent),
|
||||||
probe_syncthing(config.syncthing),
|
probe_syncthing(
|
||||||
|
config.syncthing, sparse_supported=probes[1].sparse_files
|
||||||
|
),
|
||||||
]
|
]
|
||||||
for service in service_probes:
|
for service in service_probes:
|
||||||
logger.info(
|
logger.info(
|
||||||
|
|||||||
@@ -177,6 +177,7 @@ class ArchiveClientDaemon:
|
|||||||
elif probe.service == "syncthing":
|
elif probe.service == "syncthing":
|
||||||
request.capabilities.syncthing_version = probe.version
|
request.capabilities.syncthing_version = probe.version
|
||||||
request.capabilities.syncthing_device_id = probe.device_id
|
request.capabilities.syncthing_device_id = probe.device_id
|
||||||
|
request.capabilities.routes.extend(probe.routes)
|
||||||
for root_name, probe in zip(
|
for root_name, probe in zip(
|
||||||
("qbittorrent", "syncthing"), self.probes, strict=True
|
("qbittorrent", "syncthing"), self.probes, strict=True
|
||||||
):
|
):
|
||||||
|
|||||||
@@ -0,0 +1,219 @@
|
|||||||
|
"""Normalize qBittorrent observations into the stable protocol model."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import PurePosixPath
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from archive_clients.bencode import Metainfo, decode_metainfo
|
||||||
|
from archive_control.v1 import resource_pb2
|
||||||
|
|
||||||
|
|
||||||
|
class ResourceError(ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class NormalizedResource:
|
||||||
|
summary: resource_pb2.ResourceSummary
|
||||||
|
files: tuple[resource_pb2.TorrentFile, ...]
|
||||||
|
metainfo: Metainfo
|
||||||
|
|
||||||
|
|
||||||
|
def build_content_tree(
|
||||||
|
files: tuple[resource_pb2.TorrentFile, ...],
|
||||||
|
available_file_indices: set[int],
|
||||||
|
) -> list[resource_pb2.ContentTreeEntry]:
|
||||||
|
directories: dict[str, set[int]] = {}
|
||||||
|
entries: list[resource_pb2.ContentTreeEntry] = []
|
||||||
|
by_index = {file.file_index: file for file in files}
|
||||||
|
for file in files:
|
||||||
|
path = PurePosixPath(file.canonical_path)
|
||||||
|
for parent in path.parents:
|
||||||
|
if parent == PurePosixPath("."):
|
||||||
|
continue
|
||||||
|
directories.setdefault(parent.as_posix(), set()).add(file.file_index)
|
||||||
|
entry = resource_pb2.ContentTreeEntry(
|
||||||
|
canonical_path=file.canonical_path,
|
||||||
|
parent_path=(
|
||||||
|
"" if path.parent == PurePosixPath(".")
|
||||||
|
else path.parent.as_posix()
|
||||||
|
),
|
||||||
|
display_name=path.name,
|
||||||
|
type=(
|
||||||
|
resource_pb2.CONTENT_ENTRY_TYPE_PADDING_FILE
|
||||||
|
if file.padding else resource_pb2.CONTENT_ENTRY_TYPE_FILE
|
||||||
|
),
|
||||||
|
file_index=file.file_index,
|
||||||
|
)
|
||||||
|
entry.file.CopyFrom(file)
|
||||||
|
if file.file_index in available_file_indices:
|
||||||
|
_set_selection(entry.available_file_indices, [file.file_index])
|
||||||
|
entry.available_logical_bytes = file.logical_bytes
|
||||||
|
entry.available_file_count = 1
|
||||||
|
entries.append(entry)
|
||||||
|
for path_string, descendant_indices in directories.items():
|
||||||
|
path = PurePosixPath(path_string)
|
||||||
|
available = sorted(descendant_indices & available_file_indices)
|
||||||
|
entry = resource_pb2.ContentTreeEntry(
|
||||||
|
canonical_path=path_string,
|
||||||
|
parent_path=(
|
||||||
|
"" if path.parent == PurePosixPath(".")
|
||||||
|
else path.parent.as_posix()
|
||||||
|
),
|
||||||
|
display_name=path.name,
|
||||||
|
type=resource_pb2.CONTENT_ENTRY_TYPE_DIRECTORY,
|
||||||
|
available_logical_bytes=sum(
|
||||||
|
by_index[index].logical_bytes for index in available
|
||||||
|
),
|
||||||
|
available_file_count=len(available),
|
||||||
|
)
|
||||||
|
_set_selection(entry.available_file_indices, available)
|
||||||
|
entries.append(entry)
|
||||||
|
return sorted(
|
||||||
|
entries,
|
||||||
|
key=lambda entry: (
|
||||||
|
tuple(PurePosixPath(entry.canonical_path).parts),
|
||||||
|
entry.type != resource_pb2.CONTENT_ENTRY_TYPE_DIRECTORY,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_resource(
|
||||||
|
torrent: dict[str, Any],
|
||||||
|
raw_files: list[dict[str, Any]],
|
||||||
|
metainfo_bytes: bytes,
|
||||||
|
observed_at: datetime | None = None,
|
||||||
|
) -> NormalizedResource:
|
||||||
|
metainfo = decode_metainfo(metainfo_bytes)
|
||||||
|
if len(raw_files) != len(metainfo.files):
|
||||||
|
raise ResourceError("qBittorrent and metainfo file counts differ")
|
||||||
|
files = []
|
||||||
|
canonical = True
|
||||||
|
for expected_index, (raw, meta_file) in enumerate(
|
||||||
|
zip(raw_files, metainfo.files, strict=True)
|
||||||
|
):
|
||||||
|
index = _integer(raw.get("index"), "file index")
|
||||||
|
if index != expected_index:
|
||||||
|
raise ResourceError("qBittorrent file indices are not contiguous")
|
||||||
|
path = _path(raw.get("name"))
|
||||||
|
size = _integer(raw.get("size"), "file size")
|
||||||
|
if size != meta_file.logical_bytes or path != meta_file.path:
|
||||||
|
canonical = False
|
||||||
|
completed = raw.get("completed")
|
||||||
|
if completed is None:
|
||||||
|
progress = raw.get("progress", 0)
|
||||||
|
if isinstance(progress, bool) or not isinstance(progress, (int, float)):
|
||||||
|
raise ResourceError("file progress is invalid")
|
||||||
|
completed = round(size * max(0.0, min(float(progress), 1.0)))
|
||||||
|
completed = min(_integer(completed, "completed bytes"), size)
|
||||||
|
priority = _integer(raw.get("priority", 0), "file priority")
|
||||||
|
file = resource_pb2.TorrentFile(
|
||||||
|
file_index=index,
|
||||||
|
canonical_path=path,
|
||||||
|
logical_bytes=size,
|
||||||
|
completed_bytes=completed,
|
||||||
|
selected=priority > 0,
|
||||||
|
padding=meta_file.padding,
|
||||||
|
)
|
||||||
|
files.append(file)
|
||||||
|
observed_at = observed_at or datetime.now(timezone.utc)
|
||||||
|
qb_torrent_id = _string(torrent.get("hash"), "torrent hash").lower()
|
||||||
|
if len(qb_torrent_id) not in {40, 64} or any(
|
||||||
|
character not in "0123456789abcdef" for character in qb_torrent_id
|
||||||
|
):
|
||||||
|
raise ResourceError("torrent hash is invalid")
|
||||||
|
summary = resource_pb2.ResourceSummary(
|
||||||
|
qb_torrent_id=qb_torrent_id,
|
||||||
|
display_name=_string(torrent.get("name"), "torrent name"),
|
||||||
|
runtime_state=_runtime_state(_string(torrent.get("state"), "state")),
|
||||||
|
total_logical_bytes=sum(file.logical_bytes for file in files),
|
||||||
|
total_file_count=len(files),
|
||||||
|
canonical_paths=canonical,
|
||||||
|
)
|
||||||
|
summary.resource_id.info_hash_v1_hex = metainfo.info_hash_v1_hex
|
||||||
|
summary.resource_id.info_hash_v2_hex = metainfo.info_hash_v2_hex
|
||||||
|
selected = [file.file_index for file in files if file.selected]
|
||||||
|
complete = [
|
||||||
|
file.file_index for file in files
|
||||||
|
if file.selected and file.completed_bytes == file.logical_bytes
|
||||||
|
]
|
||||||
|
_set_selection(summary.selected_files, selected)
|
||||||
|
_set_selection(summary.selected_complete_files, complete)
|
||||||
|
summary.selected_logical_bytes = sum(
|
||||||
|
file.logical_bytes for file in files if file.selected
|
||||||
|
)
|
||||||
|
summary.selected_complete_bytes = sum(
|
||||||
|
file.logical_bytes for file in files
|
||||||
|
if file.selected and file.completed_bytes == file.logical_bytes
|
||||||
|
)
|
||||||
|
revision_data = [{
|
||||||
|
"index": file.file_index,
|
||||||
|
"path": file.canonical_path,
|
||||||
|
"size": file.logical_bytes,
|
||||||
|
"completed": file.completed_bytes,
|
||||||
|
"selected": file.selected,
|
||||||
|
} for file in files]
|
||||||
|
summary.content_revision = hashlib.sha256(json.dumps(
|
||||||
|
revision_data, sort_keys=True, separators=(",", ":"),
|
||||||
|
).encode("utf-8")).hexdigest()
|
||||||
|
summary.observed_at.FromDatetime(observed_at)
|
||||||
|
return NormalizedResource(summary, tuple(files), metainfo)
|
||||||
|
|
||||||
|
|
||||||
|
def _set_selection(target: Any, indices: list[int]) -> None:
|
||||||
|
if not indices:
|
||||||
|
return
|
||||||
|
first = previous = indices[0]
|
||||||
|
for index in indices[1:]:
|
||||||
|
if index == previous + 1:
|
||||||
|
previous = index
|
||||||
|
continue
|
||||||
|
target.ranges.add(first=first, last=previous)
|
||||||
|
first = previous = index
|
||||||
|
target.ranges.add(first=first, last=previous)
|
||||||
|
|
||||||
|
|
||||||
|
def _runtime_state(state: str) -> int:
|
||||||
|
lowered = state.lower()
|
||||||
|
if "missing" in lowered:
|
||||||
|
return resource_pb2.TORRENT_RUNTIME_STATE_MISSING
|
||||||
|
if "error" in lowered:
|
||||||
|
return resource_pb2.TORRENT_RUNTIME_STATE_ERROR
|
||||||
|
if "check" in lowered:
|
||||||
|
return resource_pb2.TORRENT_RUNTIME_STATE_CHECKING
|
||||||
|
if "stalled" in lowered:
|
||||||
|
return resource_pb2.TORRENT_RUNTIME_STATE_STALLED
|
||||||
|
if "pause" in lowered or "stop" in lowered:
|
||||||
|
return resource_pb2.TORRENT_RUNTIME_STATE_STOPPED
|
||||||
|
if "queue" in lowered:
|
||||||
|
return resource_pb2.TORRENT_RUNTIME_STATE_QUEUED
|
||||||
|
if "download" in lowered or lowered in {"metadl", "forceddl", "allocating"}:
|
||||||
|
return resource_pb2.TORRENT_RUNTIME_STATE_DOWNLOADING
|
||||||
|
if lowered.endswith("up") or "upload" in lowered:
|
||||||
|
return resource_pb2.TORRENT_RUNTIME_STATE_SEEDING
|
||||||
|
return resource_pb2.TORRENT_RUNTIME_STATE_UNSPECIFIED
|
||||||
|
|
||||||
|
|
||||||
|
def _path(value: Any) -> str:
|
||||||
|
path = _string(value, "file path")
|
||||||
|
candidate = PurePosixPath(path)
|
||||||
|
if candidate.is_absolute() or ".." in candidate.parts or "\\" in path:
|
||||||
|
raise ResourceError("qBittorrent file path is unsafe")
|
||||||
|
return candidate.as_posix()
|
||||||
|
|
||||||
|
|
||||||
|
def _integer(value: Any, name: str) -> int:
|
||||||
|
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
||||||
|
raise ResourceError(f"{name} is invalid")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _string(value: Any, name: str) -> str:
|
||||||
|
if not isinstance(value, str) or not value:
|
||||||
|
raise ResourceError(f"{name} is invalid")
|
||||||
|
return value
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
"""Normalize safe, pairwise Syncthing folders into local route records."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from archive_clients.config import ConfigError, RootMapping
|
||||||
|
from archive_control.v1 import route_pb2
|
||||||
|
|
||||||
|
|
||||||
|
def discover_routes(
|
||||||
|
configuration: dict[str, Any],
|
||||||
|
local_device_id: str,
|
||||||
|
roots: RootMapping,
|
||||||
|
sparse_supported: bool,
|
||||||
|
observed_at: datetime | None = None,
|
||||||
|
) -> list[route_pb2.LocalRoute]:
|
||||||
|
folders = configuration.get("folders")
|
||||||
|
if not isinstance(folders, list) or not local_device_id:
|
||||||
|
raise ValueError("Syncthing configuration or local device ID is invalid")
|
||||||
|
observed_at = observed_at or datetime.now(timezone.utc)
|
||||||
|
routes = []
|
||||||
|
for folder in folders:
|
||||||
|
if not isinstance(folder, dict):
|
||||||
|
continue
|
||||||
|
folder_id = folder.get("id")
|
||||||
|
api_path = folder.get("path")
|
||||||
|
if not isinstance(folder_id, str) or not folder_id:
|
||||||
|
continue
|
||||||
|
if not isinstance(api_path, str):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
local_path = roots.api_to_local(api_path)
|
||||||
|
relative = PurePosixPath(api_path).relative_to(roots.api_root)
|
||||||
|
except (ConfigError, ValueError):
|
||||||
|
continue
|
||||||
|
if not _lexically_within(local_path, roots.local_root):
|
||||||
|
continue
|
||||||
|
if relative == PurePosixPath("."):
|
||||||
|
continue
|
||||||
|
route = route_pb2.LocalRoute(
|
||||||
|
route_id=folder_id,
|
||||||
|
local_relative_path=relative.as_posix(),
|
||||||
|
local_syncthing_device_id=local_device_id,
|
||||||
|
writable=bool(folder.get("type") == "sendreceive"),
|
||||||
|
sparse_supported=sparse_supported,
|
||||||
|
# Ownership is set only from a durable provisioning journal. A
|
||||||
|
# folder label is never sufficient deletion attribution.
|
||||||
|
archive_control_created=False,
|
||||||
|
)
|
||||||
|
route.observed_at.FromDatetime(observed_at)
|
||||||
|
folder_type = folder.get("type")
|
||||||
|
route.folder_type = {
|
||||||
|
"sendreceive": route_pb2.SYNCTHING_FOLDER_TYPE_SEND_RECEIVE,
|
||||||
|
"sendonly": route_pb2.SYNCTHING_FOLDER_TYPE_SEND_ONLY,
|
||||||
|
"receiveonly": route_pb2.SYNCTHING_FOLDER_TYPE_RECEIVE_ONLY,
|
||||||
|
}.get(folder_type, route_pb2.SYNCTHING_FOLDER_TYPE_OTHER)
|
||||||
|
raw_devices = folder.get("devices")
|
||||||
|
device_ids = [] if not isinstance(raw_devices, list) else [
|
||||||
|
item.get("deviceID") for item in raw_devices
|
||||||
|
if isinstance(item, dict) and isinstance(item.get("deviceID"), str)
|
||||||
|
]
|
||||||
|
peers = sorted(set(device_ids) - {local_device_id})
|
||||||
|
route.peer_syncthing_device_ids.extend(peers)
|
||||||
|
if folder.get("paused") is True:
|
||||||
|
route.state = route_pb2.ROUTE_STATE_PAUSED
|
||||||
|
route.detail = "Syncthing folder is paused"
|
||||||
|
elif folder_type != "sendreceive":
|
||||||
|
route.state = route_pb2.ROUTE_STATE_UNSUPPORTED
|
||||||
|
route.detail = "route folder is not sendreceive"
|
||||||
|
elif (
|
||||||
|
not isinstance(raw_devices, list)
|
||||||
|
or len(raw_devices) != 2
|
||||||
|
or len(device_ids) != 2
|
||||||
|
or len(set(device_ids)) != 2
|
||||||
|
):
|
||||||
|
route.state = route_pb2.ROUTE_STATE_UNSUPPORTED
|
||||||
|
route.detail = "route folder must contain exactly two devices"
|
||||||
|
elif local_device_id not in device_ids:
|
||||||
|
route.state = route_pb2.ROUTE_STATE_UNSUPPORTED
|
||||||
|
route.detail = "route folder does not include the local device"
|
||||||
|
else:
|
||||||
|
route.state = route_pb2.ROUTE_STATE_DISCOVERED
|
||||||
|
routes.append(route)
|
||||||
|
return sorted(routes, key=lambda route: route.route_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _lexically_within(path: Path, root: Path) -> bool:
|
||||||
|
try:
|
||||||
|
path.relative_to(root)
|
||||||
|
return True
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
@@ -11,7 +11,8 @@ from typing import Any
|
|||||||
from urllib import error, parse, request
|
from urllib import error, parse, request
|
||||||
|
|
||||||
from archive_clients.config import ServiceConfig
|
from archive_clients.config import ServiceConfig
|
||||||
from archive_control.v1 import common_pb2
|
from archive_clients.routes import discover_routes
|
||||||
|
from archive_control.v1 import common_pb2, route_pb2
|
||||||
|
|
||||||
|
|
||||||
_MAX_RESPONSE_BYTES = 1024 * 1024
|
_MAX_RESPONSE_BYTES = 1024 * 1024
|
||||||
@@ -28,6 +29,7 @@ class ServiceProbe:
|
|||||||
detail: str = ""
|
detail: str = ""
|
||||||
device_id: str = ""
|
device_id: str = ""
|
||||||
libtorrent_version: str = ""
|
libtorrent_version: str = ""
|
||||||
|
routes: tuple[route_pb2.LocalRoute, ...] = ()
|
||||||
|
|
||||||
|
|
||||||
class _NoRedirect(request.HTTPRedirectHandler):
|
class _NoRedirect(request.HTTPRedirectHandler):
|
||||||
@@ -95,7 +97,9 @@ def probe_qbittorrent(
|
|||||||
|
|
||||||
|
|
||||||
def probe_syncthing(
|
def probe_syncthing(
|
||||||
config: ServiceConfig, timeout: float = 10
|
config: ServiceConfig,
|
||||||
|
timeout: float = 10,
|
||||||
|
sparse_supported: bool = False,
|
||||||
) -> ServiceProbe:
|
) -> ServiceProbe:
|
||||||
checked_at = datetime.now(timezone.utc)
|
checked_at = datetime.now(timezone.utc)
|
||||||
try:
|
try:
|
||||||
@@ -107,15 +111,23 @@ def probe_syncthing(
|
|||||||
status = _json_get(
|
status = _json_get(
|
||||||
opener, config.endpoint, "/rest/system/status", timeout, headers
|
opener, config.endpoint, "/rest/system/status", timeout, headers
|
||||||
)
|
)
|
||||||
|
configuration = _json_get(
|
||||||
|
opener, config.endpoint, "/rest/config", timeout, headers
|
||||||
|
)
|
||||||
service_version = version.get("longVersion") or version.get("version")
|
service_version = version.get("longVersion") or version.get("version")
|
||||||
device_id = status.get("myID")
|
device_id = status.get("myID")
|
||||||
if not isinstance(service_version, str) or not service_version:
|
if not isinstance(service_version, str) or not service_version:
|
||||||
raise ValueError("Syncthing returned no version")
|
raise ValueError("Syncthing returned no version")
|
||||||
if not isinstance(device_id, str) or not device_id:
|
if not isinstance(device_id, str) or not device_id:
|
||||||
raise ValueError("Syncthing returned no device ID")
|
raise ValueError("Syncthing returned no device ID")
|
||||||
|
routes = discover_routes(
|
||||||
|
configuration, device_id, config.roots, sparse_supported,
|
||||||
|
checked_at,
|
||||||
|
)
|
||||||
return ServiceProbe(
|
return ServiceProbe(
|
||||||
"syncthing", common_pb2.HEALTH_STATE_HEALTHY, checked_at,
|
"syncthing", common_pb2.HEALTH_STATE_HEALTHY, checked_at,
|
||||||
version=service_version, device_id=device_id,
|
version=service_version, device_id=device_id,
|
||||||
|
routes=tuple(routes),
|
||||||
)
|
)
|
||||||
except error.HTTPError as exc:
|
except error.HTTPError as exc:
|
||||||
detail = (
|
detail = (
|
||||||
|
|||||||
@@ -0,0 +1,104 @@
|
|||||||
|
import hashlib
|
||||||
|
import unittest
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from archive_clients.bencode import BencodeError, decode_metainfo, encode
|
||||||
|
from archive_clients.resources import (
|
||||||
|
ResourceError,
|
||||||
|
build_content_tree,
|
||||||
|
normalize_resource,
|
||||||
|
)
|
||||||
|
from archive_control.v1 import resource_pb2
|
||||||
|
|
||||||
|
|
||||||
|
class ResourceTests(unittest.TestCase):
|
||||||
|
def test_hybrid_identity_and_selection_normalization(self):
|
||||||
|
info = {
|
||||||
|
b"file tree": {
|
||||||
|
b"a.txt": {b"": {b"length": 3}},
|
||||||
|
b"b.bin": {b"": {b"attr": b"p", b"length": 5}},
|
||||||
|
},
|
||||||
|
b"files": [
|
||||||
|
{b"length": 3, b"path": [b"a.txt"]},
|
||||||
|
{b"attr": b"p", b"length": 5, b"path": [b"b.bin"]},
|
||||||
|
],
|
||||||
|
b"meta version": 2,
|
||||||
|
b"name": b"resource",
|
||||||
|
b"piece length": 16384,
|
||||||
|
b"pieces": b"x" * 20,
|
||||||
|
}
|
||||||
|
metainfo_bytes = encode({b"announce": b"http://tracker", b"info": info})
|
||||||
|
decoded = decode_metainfo(metainfo_bytes)
|
||||||
|
encoded_info = encode(info)
|
||||||
|
self.assertEqual(
|
||||||
|
decoded.info_hash_v1_hex, hashlib.sha1(encoded_info).hexdigest()
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
decoded.info_hash_v2_hex, hashlib.sha256(encoded_info).hexdigest()
|
||||||
|
)
|
||||||
|
self.assertEqual(decoded.files[1].path, "resource/b.bin")
|
||||||
|
self.assertTrue(decoded.files[1].padding)
|
||||||
|
|
||||||
|
observed = datetime(2026, 1, 2, tzinfo=timezone.utc)
|
||||||
|
normalized = normalize_resource(
|
||||||
|
{
|
||||||
|
"hash": decoded.info_hash_v1_hex,
|
||||||
|
"name": "resource",
|
||||||
|
"state": "stoppedUP",
|
||||||
|
},
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"index": 0, "name": "resource/a.txt", "size": 3,
|
||||||
|
"progress": 1.0, "priority": 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"index": 1, "name": "resource/b.bin", "size": 5,
|
||||||
|
"progress": 0.5, "priority": 0,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
metainfo_bytes,
|
||||||
|
observed,
|
||||||
|
)
|
||||||
|
summary = normalized.summary
|
||||||
|
self.assertEqual(
|
||||||
|
summary.runtime_state, resource_pb2.TORRENT_RUNTIME_STATE_STOPPED
|
||||||
|
)
|
||||||
|
self.assertEqual(summary.selected_files.ranges[0].first, 0)
|
||||||
|
self.assertEqual(summary.selected_complete_files.ranges[0].last, 0)
|
||||||
|
self.assertEqual(summary.selected_logical_bytes, 3)
|
||||||
|
self.assertTrue(summary.canonical_paths)
|
||||||
|
tree = build_content_tree(normalized.files, {0})
|
||||||
|
root = next(entry for entry in tree if entry.canonical_path == "resource")
|
||||||
|
self.assertEqual(root.available_file_count, 1)
|
||||||
|
self.assertEqual(root.available_logical_bytes, 3)
|
||||||
|
|
||||||
|
def test_noncanonical_and_unsafe_paths_are_distinct(self):
|
||||||
|
info = {
|
||||||
|
b"length": 3,
|
||||||
|
b"name": b"a.txt",
|
||||||
|
b"piece length": 16384,
|
||||||
|
b"pieces": b"x" * 20,
|
||||||
|
}
|
||||||
|
metainfo = encode({b"info": info})
|
||||||
|
torrent = {
|
||||||
|
"hash": hashlib.sha1(encode(info)).hexdigest(),
|
||||||
|
"name": "renamed", "state": "uploading",
|
||||||
|
}
|
||||||
|
renamed = normalize_resource(torrent, [{
|
||||||
|
"index": 0, "name": "renamed.txt", "size": 3,
|
||||||
|
"progress": 1.0, "priority": 1,
|
||||||
|
}], metainfo)
|
||||||
|
self.assertFalse(renamed.summary.canonical_paths)
|
||||||
|
with self.assertRaisesRegex(ResourceError, "unsafe"):
|
||||||
|
normalize_resource(torrent, [{
|
||||||
|
"index": 0, "name": "../escape", "size": 3,
|
||||||
|
"progress": 1.0, "priority": 1,
|
||||||
|
}], metainfo)
|
||||||
|
|
||||||
|
def test_noncanonical_bencode_is_rejected(self):
|
||||||
|
with self.assertRaisesRegex(BencodeError, "unsorted"):
|
||||||
|
decode_metainfo(b"d4:infod1:b1:x1:a1:yee")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
|
|
||||||
|
from archive_clients.config import RootMapping
|
||||||
|
from archive_clients.routes import discover_routes
|
||||||
|
from archive_control.v1 import route_pb2
|
||||||
|
|
||||||
|
|
||||||
|
class RouteDiscoveryTests(unittest.TestCase):
|
||||||
|
def test_pairwise_routes_are_discovered_and_unsafe_folders_ignored(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
roots = RootMapping(PurePosixPath("/sync"), Path(directory))
|
||||||
|
configuration = {"folders": [
|
||||||
|
{
|
||||||
|
"id": "route-b", "path": "/sync/routes/b",
|
||||||
|
"type": "sendreceive",
|
||||||
|
"devices": [{"deviceID": "LOCAL"}, {"deviceID": "PEER"}],
|
||||||
|
"label": "Archive Control route-b",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "route-a", "path": "/sync/routes/a",
|
||||||
|
"type": "sendonly",
|
||||||
|
"devices": [{"deviceID": "LOCAL"}, {"deviceID": "PEER"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "outside", "path": "/other/path",
|
||||||
|
"type": "sendreceive", "devices": [],
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
routes = discover_routes(
|
||||||
|
configuration, "LOCAL", roots, True,
|
||||||
|
datetime(2026, 1, 2, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
self.assertEqual([route.route_id for route in routes], ["route-a", "route-b"])
|
||||||
|
self.assertEqual(routes[0].state, route_pb2.ROUTE_STATE_UNSUPPORTED)
|
||||||
|
self.assertEqual(routes[1].state, route_pb2.ROUTE_STATE_DISCOVERED)
|
||||||
|
self.assertEqual(list(routes[1].peer_syncthing_device_ids), ["PEER"])
|
||||||
|
self.assertEqual(routes[1].local_relative_path, "routes/b")
|
||||||
|
self.assertFalse(routes[1].archive_control_created)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -68,6 +68,7 @@ class ServiceProbeTests(unittest.TestCase):
|
|||||||
opener = _Opener([
|
opener = _Opener([
|
||||||
'{"version":"v2.0.1","longVersion":"syncthing v2.0.1"}',
|
'{"version":"v2.0.1","longVersion":"syncthing v2.0.1"}',
|
||||||
'{"myID":"DEVICE-ID"}',
|
'{"myID":"DEVICE-ID"}',
|
||||||
|
'{"folders":[]}',
|
||||||
])
|
])
|
||||||
with patch(
|
with patch(
|
||||||
"archive_clients.services.request.build_opener",
|
"archive_clients.services.request.build_opener",
|
||||||
|
|||||||
Reference in New Issue
Block a user