Compare commits

...
3 Commits
Author SHA1 Message Date
cabbage 87cca59d3a fix: wait for release builder platforms 2026-08-15 08:19:15 +00:00
cabbage 618bb3d206 docs: clarify qB save path resolution 2026-08-15 08:12:11 +00:00
cabbage a1bf3e4315 fix: resolve per-torrent qb save paths 2026-08-14 11:56:06 +00:00
19 changed files with 509 additions and 66 deletions
+15 -4
View File
@@ -120,6 +120,8 @@ username = "${QB_USER}"
password_file = "/run/secrets/qb_password" password_file = "/run/secrets/qb_password"
api_root = "/downloads" api_root = "/downloads"
local_root = "/data/qb" local_root = "/data/qb"
# Optional only when a nested qB path uses a different client-visible mount.
# local_path_overrides = { "/downloads/fast" = "/data/qb-fast" }
[syncthing] [syncthing]
endpoint = "http://syncthing:8384" endpoint = "http://syncthing:8384"
@@ -139,6 +141,14 @@ accounting. The override key is the normalized Syncthing folder path beneath
`api_root`, not its folder ID. See the production deployment README for the `api_root`, not its folder ID. See the production deployment README for the
required compose and override pattern. required compose and override pattern.
qBittorrent content is resolved differently: every existing torrent keeps its
qB-reported `save_path`. The client maps that path beneath `qbittorrent.api_root`
to its own mount before a source, existing-target, permission, or eviction
operation. Thus `/downloads/Downloading` naturally maps below `/data/qb`;
there is no migration or per-resource configuration. Add a qB
`local_path_overrides` entry only when that nested API prefix is a separate
client mount.
The remaining node examples omit optional `[connection]`, `[jobs]`, and The remaining node examples omit optional `[connection]`, `[jobs]`, and
`[backup]` tables and therefore use these same defaults; deployments may `[backup]` tables and therefore use these same defaults; deployments may
override them per node. override them per node.
@@ -277,10 +287,11 @@ The publisher deliberately uses `moby/buildkit:rootless` with
`--oci-worker-no-process-sandbox`. On nested Docker hosts, the default OCI `--oci-worker-no-process-sandbox`. On nested Docker hosts, the default OCI
sandbox can fail while masking `/proc/acpi` for an emulated build; rootless sandbox can fail while masking `/proc/acpi` for an emulated build; rootless
BuildKit confines that compatibility setting to the disposable builder. It BuildKit confines that compatibility setting to the disposable builder. It
refuses to publish unless `docker buildx inspect` reports both `linux/amd64` waits briefly for the new worker to observe binfmt, then refuses to publish
and `linux/arm64`, and removes the builder and binfmt handler on success, unless `docker buildx inspect` reports both `linux/amd64` and `linux/arm64`.
failure, or interruption. Retain the displayed manifest digest in release On capability failure it prints that inspection output and removes the builder
notes and deploy the immutable tag or digest. and binfmt handler on success, failure, or interruption. Retain the displayed
manifest digest in release notes and deploy the immutable tag or digest.
## Operator usage ## Operator usage
+14 -1
View File
@@ -86,6 +86,9 @@ python3 scripts/preflight-deployment.py ... \
- That future route path and qBittorrent content root use one client bind - That future route path and qBittorrent content root use one client bind
mount, so hard-link staging remains possible rather than silently falling mount, so hard-link staging remains possible rather than silently falling
back to a space-consuming copy. back to a space-consuming copy.
- Every explicit `qbittorrent.local_path_overrides` entry maps the same host
path in the qBittorrent and client containers. This protects nested qB save
paths that use a dedicated bind mount.
- A real `link(2)` operation between a unique zero-byte file in the qB root - A real `link(2)` operation between a unique zero-byte file in the qB root
and one in the future automatic-route root. The probe verifies that both and one in the future automatic-route root. The probe verifies that both
names refer to the same inode and removes them unconditionally. names refer to the same inode and removes them unconditionally.
@@ -96,7 +99,8 @@ python3 scripts/preflight-deployment.py ... \
- The token, qB password, and Syncthing API-key files are non-empty regular - The token, qB password, and Syncthing API-key files are non-empty regular
files with no group/world permissions. files with no group/world permissions.
- The client image can read its configuration and reports usable permissions, - The client image can read its configuration and reports usable permissions,
sparse-file support, and filesystem capabilities for both roots. sparse-file support, and filesystem capabilities for both primary roots and
every configured qBittorrent local-path override.
- qBittorrent authentication/version compatibility and Syncthing - qBittorrent authentication/version compatibility and Syncthing
authentication/device identity are healthy from the client container. authentication/device identity are healthy from the client container.
@@ -106,6 +110,15 @@ root and add a `local_path_overrides` mapping for that exact API path. This
prevents automatic route folders being created on a small configuration prevents automatic route folders being created on a small configuration
filesystem while the client expects to hardlink from the qB data mount. filesystem while the client expects to hardlink from the qB data mount.
qBittorrent resources may use any `save_path` below `qbittorrent.api_root`.
At job preflight the client maps that qB API path to its local mount and uses
it as the resource root; data is never moved to fit Archive Control. A resource
outside the configured qB API root, or whose resolved directory is unavailable
or not a real directory in the client container, fails that job before staging
or eviction. Use `qbittorrent.local_path_overrides` only for a nested qB API
prefix backed by a distinct client mount; this preflight verifies the Docker
bind topology for each such override.
When converting an existing node, stop its client and Syncthing containers, When converting an existing node, stop its client and Syncthing containers,
move each existing `routes/<route-id>` directory from the old Syncthing config move each existing `routes/<route-id>` directory from the old Syncthing config
tree into the new qB-backed route-root directory, then recreate Syncthing and tree into the new qB-backed route-root directory, then recreate Syncthing and
+19 -2
View File
@@ -42,8 +42,8 @@ and exported-metainfo data, keeps the qB-local hash separately, and calculates:
- normalized selected and selected-complete file-index sets; - normalized selected and selected-complete file-index sets;
- torrent runtime state; - torrent runtime state;
- canonical path flag and content revision; - canonical path flag and content revision;
- a save-path fingerprint based on validated path mapping, never a leaked - a validated qBittorrent `save_path`, retained only in the client's local
absolute host path. normalized observation.
qBittorrent 5 may expose a pure-v2 or hybrid torrent under the first 20 bytes qBittorrent 5 may expose a pure-v2 or hybrid torrent under the first 20 bytes
of its v2 hash while separately advertising full `infohash_v1` and of its v2 hash while separately advertising full `infohash_v1` and
@@ -56,6 +56,23 @@ All entries retain qBittorrent's stable torrent file index. Renamed/noncanonical
content paths are rejected in v1 because they cannot be transported and merged content paths are rejected in v1 because they cannot be transported and merged
without ambiguity. without ambiguity.
### Per-torrent local content roots
`save_path` is not inventory, placement, or protocol data. It is qBittorrent
metadata used only by the daemon that queried qBittorrent. Before staging a
source, merging into an existing target, applying post-recheck permissions, or
evicting files, that daemon maps the torrent's API-visible `save_path` through
`qbittorrent.api_root`/`local_root`. The most-specific
`qbittorrent.local_path_overrides` mapping wins when a nested path is exposed
through a distinct client container mount.
An ordinary nested qBittorrent path such as `/media/Data/Downloading` needs no
per-resource configuration: it resolves beneath the configured root. A path
outside that root, an unmapped distinct mount, or a mapped local path that is
not a visible real directory fails the affected job before filesystem mutation.
This check is intentionally per resource; an unrelated malformed qBittorrent
entry cannot prevent normal resources from being staged or transferred.
### Verification guard ### Verification guard
The client captures transfer counters and state before recheck, issues recheck, The client captures transfer counters and state before recheck, issues recheck,
+7 -1
View File
@@ -14,6 +14,13 @@ client:
a regular file or an explicitly created directory; a regular file or an explicitly created directory;
5. verifies every operation remains beneath the local configured root. 5. verifies every operation remains beneath the local configured root.
For qBittorrent content, the root is resolved per resource: qB's authoritative
API-visible `save_path` is mapped beneath `qbittorrent.api_root` (or a more
specific configured local override) into the client namespace. This permits
existing nested save paths without moving data, but does not permit paths
outside the configured boundary. The resolved path remains local client state
and is never sent to the control daemon.
Sockets, devices, FIFOs, symlinks, and other special entries fail preflight. Sockets, devices, FIFOs, symlinks, and other special entries fail preflight.
Permission or ownership mismatch is fail-fast. Archive Control never changes Permission or ownership mismatch is fail-fast. Archive Control never changes
source ownership or mode to make a job pass. source ownership or mode to make a job pass.
@@ -150,4 +157,3 @@ Offline tooling provides list, verify, and restore. Restore requires stopped
daemon access, verifies the chosen backup, preserves the suspect database under daemon access, verifies the chosen backup, preserves the suspect database under
a timestamped name, installs the replacement atomically, and runs integrity and a timestamped name, installs the replacement atomically, and runs integrity and
schema checks before normal startup. Backups contain no configured secrets. schema checks before normal startup. Backups contain no configured secrets.
+22 -1
View File
@@ -134,7 +134,7 @@ from pathlib import Path
from archive_clients.config import ClientConfig from archive_clients.config import ClientConfig
config=ClientConfig.load(Path(sys.argv[1])) config=ClientConfig.load(Path(sys.argv[1]))
value={"shared_token_file":str(config.shared_token_file)} value={"shared_token_file":str(config.shared_token_file)}
value["qbittorrent"]={"api_root":str(config.qbittorrent.api_root),"local_root":str(config.qbittorrent.local_root),"password_file":str(config.qbittorrent.password_file)} value["qbittorrent"]={"api_root":str(config.qbittorrent.api_root),"local_root":str(config.qbittorrent.local_root),"password_file":str(config.qbittorrent.password_file),"local_path_overrides":{str(api):str(local) for api,local in config.qbittorrent.local_path_overrides}}
value["syncthing"]={"api_root":str(config.syncthing.api_root),"local_root":str(config.syncthing.local_root),"api_key_file":str(config.syncthing.api_key_file),"local_path_overrides":{str(api):str(local) for api,local in config.syncthing.local_path_overrides}} value["syncthing"]={"api_root":str(config.syncthing.api_root),"local_root":str(config.syncthing.local_root),"api_key_file":str(config.syncthing.api_key_file),"local_path_overrides":{str(api):str(local) for api,local in config.syncthing.local_path_overrides}}
print(json.dumps(value,sort_keys=True))''' print(json.dumps(value,sort_keys=True))'''
result = subprocess.run( result = subprocess.run(
@@ -174,6 +174,26 @@ raise SystemExit(0 if all(p.state == 1 for p in probes) else 1)'''
print(result.stdout.strip()) print(result.stdout.strip())
def require_qb_override_mappings(
client: dict[str, Any], qbittorrent: dict[str, Any], qb: dict[str, Any],
) -> None:
"""Prove every explicit qB API/local override sees the same host bytes."""
overrides = qb.get("local_path_overrides", {})
if not isinstance(overrides, dict):
raise CheckFailure("qbittorrent.local_path_overrides must be a table")
for api_path, local_path in overrides.items():
if not isinstance(api_path, str) or not isinstance(local_path, str):
raise CheckFailure(
"qbittorrent.local_path_overrides entries are invalid"
)
require_same_path(
f"qBittorrent override {api_path}/local mapping",
map_path(qbittorrent, api_path),
map_path(client, local_path),
)
def run_hardlink_probe( def run_hardlink_probe(
container: str, qb_root: str, route_root: str, user: str | None = None, container: str, qb_root: str, route_root: str, user: str | None = None,
) -> None: ) -> None:
@@ -298,6 +318,7 @@ def main(argv: list[str] | None = None) -> int:
syncthing_route, client_route, syncthing_route, client_route,
) )
require_same_path("qBittorrent api_root/local_root", qb_api, client_qb) require_same_path("qBittorrent api_root/local_root", qb_api, client_qb)
require_qb_override_mappings(client, qbittorrent, qb)
if client_qb.destination != client_route.destination: if client_qb.destination != client_route.destination:
raise CheckFailure( raise CheckFailure(
"qBittorrent and future route roots use separate client bind " "qBittorrent and future route roots use separate client bind "
+23 -4
View File
@@ -69,13 +69,32 @@ docker buildx create --name "${builder}" --driver docker-container \
--buildkitd-flags "${buildkitd_flags}" --use >/dev/null --buildkitd-flags "${buildkitd_flags}" --use >/dev/null
builder_created=true builder_created=true
platforms=$(docker buildx inspect "${builder}" --bootstrap 2>&1) # A newly-created rootless worker can publish its native platform before it has
for platform in linux/amd64 linux/arm64; do # observed the just-registered binfmt handler. Do not mistake that brief
# startup state for a partial-release-capable builder.
platforms=''
supports_all=false
for attempt in {1..10}; do
platforms=$(docker buildx inspect "${builder}" --bootstrap 2>&1)
supports_all=true
for platform in linux/amd64 linux/arm64; do
if ! grep -Fq "${platform}" <<<"${platforms}"; then if ! grep -Fq "${platform}" <<<"${platforms}"; then
echo "Builder ${builder} does not support ${platform}; refusing partial release." >&2 supports_all=false
exit 1 break
fi
done
if [[ ${supports_all} == true ]]; then
break
fi
if [[ ${attempt} -lt 10 ]]; then
sleep 1
fi fi
done done
if [[ ${supports_all} != true ]]; then
echo "Builder ${builder} does not support both required platforms; refusing partial release." >&2
printf '%s\n' "${platforms}" >&2
exit 1
fi
tags=(--tag "${repository}:${tag}") tags=(--tag "${repository}:${tag}")
if [[ ${also_latest} == true ]]; then if [[ ${also_latest} == true ]]; then
+11 -2
View File
@@ -27,10 +27,16 @@ def main(argv: Sequence[str] | None = None) -> int:
parser.add_argument("--check-config", action="store_true") parser.add_argument("--check-config", action="store_true")
arguments = parser.parse_args(argv) arguments = parser.parse_args(argv)
config = ClientConfig.load(arguments.config, arguments.mode) config = ClientConfig.load(arguments.config, arguments.mode)
qb_extra_roots = tuple(sorted({
root
for _, root in config.qbittorrent.local_path_overrides
if root != config.qbittorrent.local_root
}))
probes = [ probes = [
probe_root(config.qbittorrent.local_root), probe_root(config.qbittorrent.local_root),
probe_root(config.syncthing.local_root), probe_root(config.syncthing.local_root),
] ]
qb_extra_probes = tuple(probe_root(root) for root in qb_extra_roots)
probe_writable_directory(config.state_db.parent) probe_writable_directory(config.state_db.parent)
probe_writable_directory(config.backup_dir) probe_writable_directory(config.backup_dir)
shared_token = config.read_shared_token() shared_token = config.read_shared_token()
@@ -40,7 +46,10 @@ def main(argv: Sequence[str] | None = None) -> int:
print(json.dumps({ print(json.dumps({
"client_id": config.client_id, "client_id": config.client_id,
"role": config.role, "role": config.role,
"filesystems": [probe.__dict__ | {"root": str(probe.root)} for probe in probes], "filesystems": [
probe.__dict__ | {"root": str(probe.root)}
for probe in (*probes[:1], *qb_extra_probes, *probes[1:])
],
}, sort_keys=True)) }, sort_keys=True))
return 0 return 0
configure_logging( configure_logging(
@@ -68,7 +77,7 @@ def main(argv: Sequence[str] | None = None) -> int:
}, },
) )
asyncio.run(ArchiveClientDaemon( asyncio.run(ArchiveClientDaemon(
config, probes, service_probes, config, probes, service_probes, qb_extra_probes=qb_extra_probes,
resource_reader=QBittorrentReader(config.qbittorrent), resource_reader=QBittorrentReader(config.qbittorrent),
).run()) ).run())
except KeyboardInterrupt: except KeyboardInterrupt:
-2
View File
@@ -213,8 +213,6 @@ def _local_path_overrides(
value: dict[str, Any], api_root: PurePosixPath, name: str value: dict[str, Any], api_root: PurePosixPath, name: str
) -> tuple[tuple[PurePosixPath, Path], ...]: ) -> tuple[tuple[PurePosixPath, Path], ...]:
raw = value.get("local_path_overrides", {}) raw = value.get("local_path_overrides", {})
if name != "syncthing" and raw:
raise ConfigError(f"{name}.local_path_overrides is unsupported")
if not isinstance(raw, dict): if not isinstance(raw, dict):
raise ConfigError(f"{name}.local_path_overrides must be a table") raise ConfigError(f"{name}.local_path_overrides must be a table")
parsed: list[tuple[PurePosixPath, Path]] = [] parsed: list[tuple[PurePosixPath, Path]] = []
+7 -1
View File
@@ -64,6 +64,7 @@ class ArchiveClientDaemon:
service_probes: list[ServiceProbe], service_probes: list[ServiceProbe],
resource_reader: QBittorrentReader | None = None, resource_reader: QBittorrentReader | None = None,
route_manager: SyncthingRouteManager | None = None, route_manager: SyncthingRouteManager | None = None,
qb_extra_probes: tuple[FilesystemProbe, ...] = (),
): ):
if len(probes) != 2: if len(probes) != 2:
raise ValueError( raise ValueError(
@@ -71,6 +72,7 @@ class ArchiveClientDaemon:
) )
self.config = config self.config = config
self.probes = probes self.probes = probes
self.qb_probes = (probes[0], *qb_extra_probes)
self.service_probes = service_probes self.service_probes = service_probes
self.inventory = ( self.inventory = (
InventoryService(resource_reader, config.client_id) InventoryService(resource_reader, config.client_id)
@@ -119,9 +121,13 @@ class ArchiveClientDaemon:
store=self.store, store=self.store,
qb_root=config.qbittorrent.local_root, qb_root=config.qbittorrent.local_root,
qb_api_root=config.qbittorrent.api_root, qb_api_root=config.qbittorrent.api_root,
qb_roots=config.qbittorrent.roots,
route_path=self._route_path, route_path=self._route_path,
syncthing_transport=self.routes.transport, syncthing_transport=self.routes.transport,
sparse_supported=all(probe.sparse_files for probe in probes), sparse_supported=all(
probe.sparse_files
for probe in (*self.qb_probes, probes[1])
),
poll_interval=config.jobs.poll_interval, poll_interval=config.jobs.poll_interval,
verification_timeout=config.jobs.verification_timeout, verification_timeout=config.jobs.verification_timeout,
free_space_reserve_bytes=( free_space_reserve_bytes=(
+35 -15
View File
@@ -7,7 +7,7 @@ import hashlib
import os import os
import stat import stat
from pathlib import Path, PurePosixPath from pathlib import Path, PurePosixPath
from typing import Iterable from typing import Callable, Iterable
from archive_clients.qbittorrent import QBittorrentReader from archive_clients.qbittorrent import QBittorrentReader
from archive_clients.resources import NormalizedResource from archive_clients.resources import NormalizedResource
@@ -23,7 +23,7 @@ def verify_and_snapshot(
job_id: str, job_id: str,
resource: NormalizedResource, resource: NormalizedResource,
selected_indices: Iterable[int], selected_indices: Iterable[int],
qb_root: Path, content_root: Path,
store: ClientStore, store: ClientStore,
) -> dict[str, object]: ) -> dict[str, object]:
selected = set(selected_indices) selected = set(selected_indices)
@@ -38,7 +38,7 @@ def verify_and_snapshot(
f"cache file {index} is not selected and complete" f"cache file {index} is not selected and complete"
) )
relative = _relative(item.canonical_path) relative = _relative(item.canonical_path)
path = qb_root.joinpath(*relative.parts) path = content_root.joinpath(*relative.parts)
try: try:
metadata = path.lstat() metadata = path.lstat()
except FileNotFoundError as exc: except FileNotFoundError as exc:
@@ -65,7 +65,7 @@ def verify_and_snapshot(
"sha256": _sha256(path), "sha256": _sha256(path),
} }
) )
snapshot = {"files": files} snapshot = {"content_root": str(content_root), "files": files}
return store.put_job_artifact(job_id, "eviction-snapshot", snapshot)["value"] return store.put_job_artifact(job_id, "eviction-snapshot", snapshot)["value"]
@@ -90,9 +90,9 @@ def remove_qb_entry(
def safe_unlink( def safe_unlink(
*, *,
job_id: str, job_id: str,
qb_root: Path,
qbittorrent: QBittorrentReader, qbittorrent: QBittorrentReader,
store: ClientStore, store: ClientStore,
resource_root: Callable[[NormalizedResource], Path],
) -> dict[str, object]: ) -> dict[str, object]:
completed = store.get_job_artifact(job_id, "eviction-unlinked") completed = store.get_job_artifact(job_id, "eviction-unlinked")
if completed is not None: if completed is not None:
@@ -101,11 +101,19 @@ def safe_unlink(
if snapshot_row is None: if snapshot_row is None:
raise EvictionError("eviction snapshot is missing") raise EvictionError("eviction snapshot is missing")
snapshot = snapshot_row["value"] snapshot = snapshot_row["value"]
if not isinstance(snapshot, dict):
raise EvictionError("eviction snapshot is invalid")
content_root_value = snapshot.get("content_root")
if not isinstance(content_root_value, str):
raise EvictionError("eviction snapshot content root is missing")
content_root = Path(content_root_value)
if not content_root.is_absolute() or content_root.is_symlink():
raise EvictionError("eviction snapshot content root is unsafe")
files = snapshot.get("files") files = snapshot.get("files")
if not isinstance(files, list): if not isinstance(files, list):
raise EvictionError("eviction snapshot is invalid") raise EvictionError("eviction snapshot is invalid")
referenced = _remaining_paths(qbittorrent.list_resources()) referenced = _remaining_paths(qbittorrent.list_resources(), resource_root)
removed: list[str] = [] removed: list[str] = []
retained: list[dict[str, str]] = [] retained: list[dict[str, str]] = []
directories: set[Path] = set() directories: set[Path] = set()
@@ -113,10 +121,10 @@ def safe_unlink(
if not isinstance(record, dict) or not isinstance(record.get("path"), str): if not isinstance(record, dict) or not isinstance(record.get("path"), str):
raise EvictionError("eviction file record is invalid") raise EvictionError("eviction file record is invalid")
relative = _relative(record["path"]) relative = _relative(record["path"])
if relative.as_posix() in referenced: path = content_root.joinpath(*relative.parts)
if path in referenced:
retained.append({"path": relative.as_posix(), "reason": "shared"}) retained.append({"path": relative.as_posix(), "reason": "shared"})
continue continue
path = qb_root.joinpath(*relative.parts)
try: try:
metadata = path.lstat() metadata = path.lstat()
except FileNotFoundError: except FileNotFoundError:
@@ -142,7 +150,7 @@ def safe_unlink(
path.unlink() path.unlink()
removed.append(relative.as_posix()) removed.append(relative.as_posix())
parent = path.parent parent = path.parent
while parent != qb_root: while parent != content_root:
directories.add(parent) directories.add(parent)
parent = parent.parent parent = parent.parent
@@ -153,7 +161,7 @@ def safe_unlink(
try: try:
directory.rmdir() directory.rmdir()
removed_directories.append( removed_directories.append(
directory.relative_to(qb_root).as_posix() directory.relative_to(content_root).as_posix()
) )
except OSError as exc: except OSError as exc:
if exc.errno not in {errno.ENOTEMPTY, errno.ENOENT}: if exc.errno not in {errno.ENOTEMPTY, errno.ENOENT}:
@@ -214,12 +222,24 @@ def compensate_materialized_files(
return removed return removed
def _remaining_paths(resources: Iterable[NormalizedResource]) -> set[str]: def _remaining_paths(
return { resources: Iterable[NormalizedResource],
item.canonical_path resource_root: Callable[[NormalizedResource], Path],
for resource in resources ) -> set[Path]:
result: set[Path] = set()
for resource in resources:
try:
root = resource_root(resource)
except (OSError, RuntimeError, ValueError):
# A malformed unrelated qB entry must not stop an otherwise
# safe eviction. Its unresolvable path cannot be considered a
# shared path under the verified eviction root.
continue
result.update(
root.joinpath(*_relative(item.canonical_path).parts)
for item in resource.files for item in resource.files
} )
return result
def _relative(value: str) -> PurePosixPath: def _relative(value: str) -> PurePosixPath:
+78 -16
View File
@@ -14,6 +14,7 @@ import uuid
from pathlib import Path, PurePosixPath from pathlib import Path, PurePosixPath
from typing import Callable, Iterable from typing import Callable, Iterable
from archive_clients.config import ConfigError, RootMapping
from archive_clients.protocol import decode_message, encode_message from archive_clients.protocol import decode_message, encode_message
from archive_clients.eviction import ( from archive_clients.eviction import (
EvictionError, EvictionError,
@@ -67,6 +68,7 @@ class ClientJobExecutor:
store: ClientStore, store: ClientStore,
qb_root: Path, qb_root: Path,
qb_api_root: PurePosixPath, qb_api_root: PurePosixPath,
qb_roots: RootMapping | None = None,
route_path: Callable[[str], Path], route_path: Callable[[str], Path],
syncthing_transport: object, syncthing_transport: object,
sparse_supported: bool, sparse_supported: bool,
@@ -78,7 +80,10 @@ class ClientJobExecutor:
self.qbittorrent = qbittorrent self.qbittorrent = qbittorrent
self.store = store self.store = store
self.qb_root = qb_root self.qb_root = qb_root
self.qb_api_root = qb_api_root self.qb_api_root = PurePosixPath(qb_api_root)
self.qb_roots = qb_roots or RootMapping(
self.qb_api_root, self.qb_root
)
self.route_path = route_path self.route_path = route_path
self.syncthing_transport = syncthing_transport self.syncthing_transport = syncthing_transport
self.sparse_supported = sparse_supported self.sparse_supported = sparse_supported
@@ -479,6 +484,7 @@ class ClientJobExecutor:
"archive coverage no longer covers the cache selection" "archive coverage no longer covers the cache selection"
) )
resource = self._resource(definition) resource = self._resource(definition)
resource_root = self._resource_root(resource)
current = _resource_fingerprint(resource, self.client_id) current = _resource_fingerprint(resource, self.client_id)
if not _fingerprint_matches( if not _fingerprint_matches(
current, definition.eviction.cache_fingerprint current, definition.eviction.cache_fingerprint
@@ -490,7 +496,7 @@ class ClientJobExecutor:
job_id=definition.job_id, job_id=definition.job_id,
resource=resource, resource=resource,
selected_indices=requested, selected_indices=requested,
qb_root=self.qb_root, content_root=resource_root,
store=self.store, store=self.store,
) )
return None return None
@@ -505,9 +511,9 @@ class ClientJobExecutor:
if step == job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK: if step == job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK:
safe_unlink( safe_unlink(
job_id=definition.job_id, job_id=definition.job_id,
qb_root=self.qb_root,
qbittorrent=self.qbittorrent, qbittorrent=self.qbittorrent,
store=self.store, store=self.store,
resource_root=self._resource_root,
) )
placement = resource_pb2.Placement( placement = resource_pb2.Placement(
client_id=self.client_id, client_id=self.client_id,
@@ -538,7 +544,8 @@ class ClientJobExecutor:
raise JobExecutionError( raise JobExecutionError(
"source resource changed after job confirmation" "source resource changed after job confirmation"
) )
self._reject_unsafe_partfile(definition, resource) source_root = self._resource_root(resource)
self._reject_unsafe_partfile(definition, resource, source_root)
indices = _selection_indices(definition.transfer.transfer_delta_files) indices = _selection_indices(definition.transfer.transfer_delta_files)
by_index = {item.file_index: item for item in resource.files} by_index = {item.file_index: item for item in resource.files}
if not indices or any(index not in by_index for index in indices): if not indices or any(index not in by_index for index in indices):
@@ -602,7 +609,7 @@ class ClientJobExecutor:
self._require_space( self._require_space(
route_root, route_root,
self._copy_required_bytes( self._copy_required_bytes(
self.qb_root, source_root,
route_root, route_root,
( (
(entry.target_canonical_path, entry.logical_bytes) (entry.target_canonical_path, entry.logical_bytes)
@@ -613,7 +620,7 @@ class ClientJobExecutor:
) )
stage_transfer( stage_transfer(
manifest, manifest,
source_root=self.qb_root, source_root=source_root,
sync_root=route_root, sync_root=route_root,
store=self.store, store=self.store,
artifact_sources={"metainfo/source.torrent": metainfo_path}, artifact_sources={"metainfo/source.torrent": metainfo_path},
@@ -691,23 +698,27 @@ class ClientJobExecutor:
"target materialization was sent to the wrong client" "target materialization was sent to the wrong client"
) )
published = load_published_transfer(self._job_directory(definition)) published = load_published_transfer(self._job_directory(definition))
info_hash = _info_hash(definition)
resource = self.qbittorrent.get_resource(info_hash)
target_root = (
self._resource_root(resource)
if resource is not None else self.qb_root
)
# The target can likewise hardlink an arrived Syncthing payload into # The target can likewise hardlink an arrived Syncthing payload into
# qB's content root when those directories share a filesystem. # qB's content root when those directories share a filesystem.
self._require_space( self._require_space(
self.qb_root, target_root,
self._copy_required_bytes( self._copy_required_bytes(
published.job_directory, published.job_directory,
self.qb_root, target_root,
( (
(entry.payload_relative_path, entry.logical_bytes) (entry.payload_relative_path, entry.logical_bytes)
for entry in published.manifest.files for entry in published.manifest.files
), ),
), ),
) )
info_hash = _info_hash(definition)
resource = self.qbittorrent.get_resource(info_hash)
if resource is not None: if resource is not None:
self._reject_unsafe_partfile(definition, resource) self._reject_unsafe_partfile(definition, resource, target_root)
if definition.transfer.HasField("target_baseline_fingerprint"): if definition.transfer.HasField("target_baseline_fingerprint"):
if resource is None or not _fingerprint_matches( if resource is None or not _fingerprint_matches(
_resource_fingerprint(resource, self.client_id), _resource_fingerprint(resource, self.client_id),
@@ -732,13 +743,14 @@ class ClientJobExecutor:
== resource_pb2.TORRENT_RUNTIME_STATE_STOPPED == resource_pb2.TORRENT_RUNTIME_STATE_STOPPED
), ),
"total_file_count": len(published.manifest.files), "total_file_count": len(published.manifest.files),
"content_root": str(target_root),
} }
self.store.put_job_artifact( self.store.put_job_artifact(
definition.job_id, "target-baseline", baseline definition.job_id, "target-baseline", baseline
) )
materialize_transfer( materialize_transfer(
published, published,
target_root=self.qb_root, target_root=target_root,
store=self.store, store=self.store,
sparse_supported=self.sparse_supported, sparse_supported=self.sparse_supported,
cancel_check=lambda: self._raise_if_cancelled( cancel_check=lambda: self._raise_if_cancelled(
@@ -813,7 +825,9 @@ class ClientJobExecutor:
raise JobExecutionError( raise JobExecutionError(
"verified qBittorrent resource disappeared" "verified qBittorrent resource disappeared"
) )
_normalize_verified_resource_permissions(self.qb_root, verified) _normalize_verified_resource_permissions(
self._resource_root(verified), verified
)
if should_start: if should_start:
self.qbittorrent.start(qb_torrent_id) self.qbittorrent.start(qb_torrent_id)
placement = resource_pb2.Placement( placement = resource_pb2.Placement(
@@ -852,6 +866,7 @@ class ClientJobExecutor:
) )
return return
baseline = baseline_row["value"] baseline = baseline_row["value"]
content_root = self._artifact_content_root(baseline)
info_hash = _info_hash(definition) info_hash = _info_hash(definition)
present = self.qbittorrent.get_resource(info_hash) present = self.qbittorrent.get_resource(info_hash)
if present is not None: if present is not None:
@@ -870,7 +885,7 @@ class ClientJobExecutor:
self.qbittorrent.delete_entry(qb_torrent_id) self.qbittorrent.delete_entry(qb_torrent_id)
compensate_materialized_files( compensate_materialized_files(
job_id=definition.job_id, job_id=definition.job_id,
qb_root=self.qb_root, qb_root=content_root,
store=self.store, store=self.store,
) )
@@ -906,10 +921,11 @@ class ClientJobExecutor:
self, self,
definition: job_pb2.JobDefinition, definition: job_pb2.JobDefinition,
resource: NormalizedResource, resource: NormalizedResource,
content_root: Path,
) -> None: ) -> None:
roots = {self.qb_root} roots = {content_root}
for item in resource.files: for item in resource.files:
candidate = self.qb_root / PurePosixPath(item.canonical_path).parts[0] candidate = content_root / PurePosixPath(item.canonical_path).parts[0]
roots.add(candidate if candidate.is_dir() else candidate.parent) roots.add(candidate if candidate.is_dir() else candidate.parent)
hashes = { hashes = {
value.lower() for value in ( value.lower() for value in (
@@ -928,6 +944,52 @@ class ClientJobExecutor:
"safely transferred by this client" "safely transferred by this client"
) )
def _resource_root(self, resource: NormalizedResource) -> Path:
"""Resolve a qB resource's save path within this client's mounts.
qB returns paths in its own API/container namespace. The configured
root mapping translates those paths into the client namespace and
supports nested save paths and explicit prefix overrides. A missing
save path is retained only for older in-process test fixtures; every
normalized qB API resource has one.
"""
if resource.save_path is None:
return self.qb_root
try:
root = self.qb_roots.api_to_local(resource.save_path.as_posix())
except ConfigError as exc:
raise JobExecutionError(
"qBittorrent resource save path is outside the configured "
"API root or has no local path mapping"
) from exc
try:
metadata = root.lstat()
except FileNotFoundError as exc:
raise JobExecutionError(
"qBittorrent resource save path is not visible in the "
"client container"
) from exc
if not stat.S_ISDIR(metadata.st_mode):
raise JobExecutionError(
"qBittorrent resource save path is not a real directory in "
"the client container"
)
return root
def _artifact_content_root(self, baseline: object) -> Path:
if not isinstance(baseline, dict):
raise JobExecutionError("target baseline is invalid")
value = baseline.get("content_root")
if not isinstance(value, str):
# A pre-existing durable baseline predates per-resource roots.
# It can only have materialized to the configured target root.
return self.qb_root
root = Path(value)
if not root.is_absolute() or root.is_symlink():
raise JobExecutionError("target baseline content root is unsafe")
return root
def _raise_if_cancelled(self, job_id: str) -> None: def _raise_if_cancelled(self, job_id: str) -> None:
event = self._cancel_events.get(job_id) event = self._cancel_events.get(job_id)
if event is not None and event.is_set(): if event is not None and event.is_set():
+29 -1
View File
@@ -19,10 +19,21 @@ class ResourceError(ValueError):
@dataclass(frozen=True) @dataclass(frozen=True)
class NormalizedResource: class NormalizedResource:
"""A normalized qB observation for protocol data and local file work.
``save_path`` is qBittorrent's API-visible per-torrent content root. It
is deliberately local-only: clients resolve it through their qB path
mapping immediately before filesystem work, and it is never serialized in
inventory, placements, or control protocol messages.
"""
summary: resource_pb2.ResourceSummary summary: resource_pb2.ResourceSummary
files: tuple[resource_pb2.TorrentFile, ...] files: tuple[resource_pb2.TorrentFile, ...]
metainfo: Metainfo metainfo: Metainfo
metainfo_bytes: bytes = b"" metainfo_bytes: bytes = b""
# qBittorrent's API-visible save path is intentionally local-only. It
# must never become part of inventory or placement protocol messages.
save_path: PurePosixPath | None = None
def build_content_tree( def build_content_tree(
@@ -137,6 +148,7 @@ def normalize_resource(
character not in "0123456789abcdef" for character in qb_torrent_id character not in "0123456789abcdef" for character in qb_torrent_id
): ):
raise ResourceError("torrent hash is invalid") raise ResourceError("torrent hash is invalid")
save_path = _save_path(torrent.get("save_path"))
summary = resource_pb2.ResourceSummary( summary = resource_pb2.ResourceSummary(
qb_torrent_id=qb_torrent_id, qb_torrent_id=qb_torrent_id,
display_name=_string(torrent.get("name"), "torrent name"), display_name=_string(torrent.get("name"), "torrent name"),
@@ -172,7 +184,9 @@ def normalize_resource(
revision_data, sort_keys=True, separators=(",", ":"), revision_data, sort_keys=True, separators=(",", ":"),
).encode("utf-8")).hexdigest() ).encode("utf-8")).hexdigest()
summary.observed_at.FromDatetime(observed_at) summary.observed_at.FromDatetime(observed_at)
return NormalizedResource(summary, tuple(files), metainfo, metainfo_bytes) return NormalizedResource(
summary, tuple(files), metainfo, metainfo_bytes, save_path
)
def _set_selection(target: Any, indices: list[int]) -> None: def _set_selection(target: Any, indices: list[int]) -> None:
@@ -217,6 +231,20 @@ def _path(value: Any) -> str:
return candidate.as_posix() return candidate.as_posix()
def _save_path(value: Any) -> PurePosixPath:
"""Validate qBittorrent's API-visible per-torrent content root."""
path = _string(value, "torrent save path")
candidate = PurePosixPath(path)
if (
not candidate.is_absolute()
or ".." in candidate.parts
or "." in candidate.parts
):
raise ResourceError("qBittorrent torrent save path is unsafe")
return candidate
def _integer(value: Any, name: str) -> int: def _integer(value: Any, name: str) -> int:
if isinstance(value, bool) or not isinstance(value, int) or value < 0: if isinstance(value, bool) or not isinstance(value, int) or value < 0:
raise ResourceError(f"{name} is invalid") raise ResourceError(f"{name} is invalid")
+59
View File
@@ -0,0 +1,59 @@
import io
import json
import os
import tempfile
import unittest
from contextlib import redirect_stdout
from pathlib import Path
from archive_clients.cli import main
class ClientCliTests(unittest.TestCase):
def test_check_config_probes_every_qb_override_root(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
for name in ("token", "qb-password", "syncthing-key"):
path = root / name
path.write_text(name, encoding="utf-8")
os.chmod(path, 0o600)
for name in ("qb", "qb-fast", "sync", "backups"):
(root / name).mkdir()
config = root / "client.toml"
config.write_text(
f'''client_id = "cache-1"
display_name = "Cache 1"
role = "cache"
control_endpoint = "ws://control/archive_control"
shared_token_file = "{root / "token"}"
state_db = "{root / "state.db"}"
backup_dir = "{root / "backups"}"
[qbittorrent]
endpoint = "http://qb"
username = "admin"
password_file = "{root / "qb-password"}"
api_root = "/downloads"
local_root = "{root / "qb"}"
local_path_overrides = {{ "/downloads/fast" = "{root / "qb-fast"}" }}
[syncthing]
endpoint = "http://syncthing"
api_key_file = "{root / "syncthing-key"}"
api_root = "/sync"
local_root = "{root / "sync"}"
''',
encoding="utf-8",
)
output = io.StringIO()
with redirect_stdout(output):
self.assertEqual(main(["--config", str(config), "--check-config"]), 0)
reported = json.loads(output.getvalue())
self.assertEqual(
[item["root"] for item in reported["filesystems"]],
[str(root / "qb"), str(root / "qb-fast"), str(root / "sync")],
)
if __name__ == "__main__":
unittest.main()
+29
View File
@@ -78,6 +78,35 @@ class ConfigTests(unittest.TestCase):
Path("/local/qb/Sync"), Path("/local/qb/Sync"),
) )
def test_qbittorrent_can_override_a_nested_save_path(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
for name in ("token", "qb-password", "syncthing-key"):
path = root / name
path.write_text(name, encoding="utf-8")
os.chmod(path, 0o600)
(root / "qb").mkdir()
(root / "fast").mkdir()
(root / "sync").mkdir()
config_path = root / "client.toml"
config_path.write_text(
_config(root).replace(
f'local_root = "{root / "qb"}"',
f'local_root = "{root / "qb"}"\n'
"local_path_overrides = { \"/downloads/fast\" = "
f'"{root / "fast"}" }}',
1,
),
encoding="utf-8",
)
config = ClientConfig.load(config_path)
self.assertEqual(
config.qbittorrent.roots.api_to_local(
"/downloads/fast/resource/file.bin"
),
root / "fast/resource/file.bin",
)
def test_endpoint_scheme_and_job_keys_are_strict(self): def test_endpoint_scheme_and_job_keys_are_strict(self):
with tempfile.TemporaryDirectory() as directory: with tempfile.TemporaryDirectory() as directory:
root = Path(directory) root = Path(directory)
+20
View File
@@ -50,6 +50,26 @@ class DeploymentPreflightTests(unittest.TestCase):
"/data/qb/.archive-control-routes", "/data/qb/.archive-control-routes",
) )
def test_qb_override_must_map_to_the_same_host_path(self):
client = {"Mounts": [
{"Type": "bind", "Source": "/srv/fast", "Destination": "/data/fast"},
]}
qbittorrent = {"Mounts": [
{"Type": "bind", "Source": "/srv/fast", "Destination": "/downloads/fast"},
]}
preflight.require_qb_override_mappings(
client, qbittorrent,
{"local_path_overrides": {"/downloads/fast": "/data/fast"}},
)
with self.assertRaisesRegex(preflight.CheckFailure, "host paths differ"):
preflight.require_qb_override_mappings(
client,
{"Mounts": [{
"Type": "bind", "Source": "/other", "Destination": "/downloads/fast",
}]},
{"local_path_overrides": {"/downloads/fast": "/data/fast"}},
)
def test_runs_hardlink_probe_with_qb_and_route_roots(self): def test_runs_hardlink_probe_with_qb_and_route_roots(self):
completed = __import__("subprocess").CompletedProcess( completed = __import__("subprocess").CompletedProcess(
args=[], returncode=0, stdout="hard-link staging probe passed\n", stderr="" args=[], returncode=0, stdout="hard-link staging probe passed\n", stderr=""
+47 -7
View File
@@ -1,5 +1,7 @@
import tempfile import tempfile
import unittest import unittest
from dataclasses import replace
from pathlib import PurePosixPath
from pathlib import Path from pathlib import Path
from archive_clients.eviction import ( from archive_clients.eviction import (
@@ -86,7 +88,7 @@ class EvictionTests(unittest.TestCase):
job_id="job-1", job_id="job-1",
resource=evicted, resource=evicted,
selected_indices=[0, 1], selected_indices=[0, 1],
qb_root=self.root, content_root=self.root,
store=self.store, store=self.store,
) )
remove_qb_entry( remove_qb_entry(
@@ -97,9 +99,9 @@ class EvictionTests(unittest.TestCase):
) )
result = safe_unlink( result = safe_unlink(
job_id="job-1", job_id="job-1",
qb_root=self.root,
qbittorrent=qb, qbittorrent=qb,
store=self.store, store=self.store,
resource_root=lambda _: self.root,
) )
self.assertTrue((self.root / "tree/shared.bin").exists()) self.assertTrue((self.root / "tree/shared.bin").exists())
self.assertFalse((self.root / "tree/owned.bin").exists()) self.assertFalse((self.root / "tree/owned.bin").exists())
@@ -121,7 +123,7 @@ class EvictionTests(unittest.TestCase):
job_id="job-1", job_id="job-1",
resource=evicted, resource=evicted,
selected_indices=[0], selected_indices=[0],
qb_root=self.root, content_root=self.root,
store=self.store, store=self.store,
) )
path.unlink() path.unlink()
@@ -134,9 +136,9 @@ class EvictionTests(unittest.TestCase):
) )
result = safe_unlink( result = safe_unlink(
job_id="job-1", job_id="job-1",
qb_root=self.root,
qbittorrent=qb, qbittorrent=qb,
store=self.store, store=self.store,
resource_root=lambda _: self.root,
) )
self.assertTrue(path.exists()) self.assertTrue(path.exists())
self.assertEqual( self.assertEqual(
@@ -152,7 +154,7 @@ class EvictionTests(unittest.TestCase):
job_id="job-1", job_id="job-1",
resource=evicted, resource=evicted,
selected_indices=[0], selected_indices=[0],
qb_root=self.root, content_root=self.root,
store=self.store, store=self.store,
) )
remove_qb_entry( remove_qb_entry(
@@ -163,19 +165,57 @@ class EvictionTests(unittest.TestCase):
) )
first = safe_unlink( first = safe_unlink(
job_id="job-1", job_id="job-1",
qb_root=self.root,
qbittorrent=qb, qbittorrent=qb,
store=self.store, store=self.store,
resource_root=lambda _: self.root,
) )
second = safe_unlink( second = safe_unlink(
job_id="job-1", job_id="job-1",
qb_root=self.root,
qbittorrent=qb, qbittorrent=qb,
store=self.store, store=self.store,
resource_root=lambda _: self.root,
) )
self.assertEqual(first, second) self.assertEqual(first, second)
self.assertEqual(qb.deleted, ["a" * 40]) self.assertEqual(qb.deleted, ["a" * 40])
def test_nested_save_path_uses_its_own_root_and_not_a_same_named_peer(self):
nested = self.root / "Downloading"
nested.mkdir()
evicted = replace(
normalized("a" * 40, ["resource/file.bin"]),
save_path=PurePosixPath("/downloads/Downloading"),
)
peer = replace(
normalized("b" * 40, ["resource/file.bin"]),
save_path=PurePosixPath("/downloads"),
)
nested_file = nested / "resource/file.bin"
nested_file.parent.mkdir()
nested_file.write_bytes(b"x" * evicted.files[0].logical_bytes)
root_file = self.root / "resource/file.bin"
root_file.parent.mkdir()
root_file.write_bytes(b"x" * peer.files[0].logical_bytes)
qb = FakeQB(evicted, [peer])
verify_and_snapshot(
job_id="job-1", resource=evicted, selected_indices=[0],
content_root=nested, store=self.store,
)
remove_qb_entry(
job_id="job-1", torrent_hash="a" * 40,
qbittorrent=qb, store=self.store,
)
safe_unlink(
job_id="job-1", qbittorrent=qb, store=self.store,
resource_root=lambda item: (
nested
if item.save_path == PurePosixPath("/downloads/Downloading")
else self.root
),
)
self.assertFalse(nested_file.exists())
self.assertTrue(root_file.exists())
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+64 -3
View File
@@ -4,11 +4,13 @@ import stat
import tempfile import tempfile
import threading import threading
import unittest import unittest
from pathlib import Path from dataclasses import replace
from pathlib import Path, PurePosixPath
from unittest.mock import Mock, patch from unittest.mock import Mock, patch
from uuid import uuid4 from uuid import uuid4
from archive_clients.bencode import encode from archive_clients.bencode import encode
from archive_clients.config import RootMapping
from archive_clients.jobs import ( from archive_clients.jobs import (
ClientJobExecutor, ClientJobExecutor,
JobExecutionError, JobExecutionError,
@@ -246,6 +248,55 @@ class ClientJobHappyPathTests(unittest.TestCase):
content=b"x" * 4096, content=b"x" * 4096,
) )
def test_nested_qb_save_path_stages_from_mapped_subdirectory(self):
with tempfile.TemporaryDirectory() as directory:
self._run_transfer(
Path(directory),
job_pb2.JOB_OPERATION_UNARCHIVE,
source_save_path="/downloads/Downloading",
)
def test_qb_save_path_preflight_rejects_unmapped_path(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
executor = ClientJobExecutor(
client_id="cache-1", qbittorrent=Mock(),
store=ClientStore(root / "state.db"), qb_root=root,
qb_api_root=PurePosixPath("/downloads"),
route_path=lambda _: root, syncthing_transport=Mock(),
sparse_supported=True,
)
resource = NormalizedResource(
resource_pb2.ResourceSummary(), (), Mock(),
save_path=PurePosixPath("/outside"),
)
with self.assertRaisesRegex(JobExecutionError, "outside"):
executor._resource_root(resource)
def test_qb_save_path_override_uses_its_dedicated_local_mount(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
primary = root / "primary"
override = root / "override"
primary.mkdir()
override.mkdir()
executor = ClientJobExecutor(
client_id="cache-1", qbittorrent=Mock(),
store=ClientStore(root / "state.db"), qb_root=primary,
qb_api_root=PurePosixPath("/downloads"),
qb_roots=RootMapping(
PurePosixPath("/downloads"), primary,
((PurePosixPath("/downloads/slow"), override),),
),
route_path=lambda _: root, syncthing_transport=Mock(),
sparse_supported=True,
)
resource = NormalizedResource(
resource_pb2.ResourceSummary(), (), Mock(),
save_path=PurePosixPath("/downloads/slow"),
)
self.assertEqual(executor._resource_root(resource), override)
def test_mount_boundary_requires_copy_space_even_with_same_device(self): def test_mount_boundary_requires_copy_space_even_with_same_device(self):
with tempfile.TemporaryDirectory() as directory: with tempfile.TemporaryDirectory() as directory:
root = Path(directory) root = Path(directory)
@@ -390,6 +441,7 @@ class ClientJobHappyPathTests(unittest.TestCase):
source_stage_free_bytes: int | None = None, source_stage_free_bytes: int | None = None,
content: bytes = b"archive-control-happy-path", content: bytes = b"archive-control-happy-path",
syncthing: CompleteSyncthing | None = None, syncthing: CompleteSyncthing | None = None,
source_save_path: str = "/downloads",
): ):
source_root = root / "source" source_root = root / "source"
target_root = root / "target" target_root = root / "target"
@@ -397,7 +449,12 @@ class ClientJobHappyPathTests(unittest.TestCase):
source_root.mkdir() source_root.mkdir()
target_root.mkdir() target_root.mkdir()
route_root.mkdir() route_root.mkdir()
(source_root / "fixture.bin").write_bytes(content) save_relative = PurePosixPath(source_save_path).relative_to(
PurePosixPath("/downloads")
)
source_content_root = source_root.joinpath(*save_relative.parts)
source_content_root.mkdir(parents=True, exist_ok=True)
(source_content_root / "fixture.bin").write_bytes(content)
info = { info = {
b"length": len(content), b"length": len(content),
b"name": b"fixture.bin", b"name": b"fixture.bin",
@@ -411,6 +468,7 @@ class ClientJobHappyPathTests(unittest.TestCase):
"hash": torrent_hash, "hash": torrent_hash,
"name": "fixture.bin", "name": "fixture.bin",
"state": "uploading", "state": "uploading",
"save_path": source_save_path,
}, },
[{ [{
"index": 0, "index": 0,
@@ -454,8 +512,11 @@ class ClientJobHappyPathTests(unittest.TestCase):
source_qb = Mock() source_qb = Mock()
source_qb.get_resource.return_value = resource source_qb.get_resource.return_value = resource
target_qb = Mock() target_qb = Mock()
target_resource = replace(
resource, save_path=PurePosixPath("/downloads")
)
target_qb.get_resource.side_effect = [ target_qb.get_resource.side_effect = [
None, None, resource, resource, None, None, target_resource, target_resource,
] ]
syncthing = syncthing or CompleteSyncthing() syncthing = syncthing or CompleteSyncthing()
source = ClientJobExecutor( source = ClientJobExecutor(
+6 -3
View File
@@ -55,6 +55,7 @@ class QBittorrentReaderTests(unittest.TestCase):
b"Ok.", b"Ok.",
json.dumps([{ json.dumps([{
"hash": torrent_hash, "name": "a.txt", "state": "uploading", "hash": torrent_hash, "name": "a.txt", "state": "uploading",
"save_path": "/downloads",
}]).encode(), }]).encode(),
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]', b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
torrent_bytes, torrent_bytes,
@@ -102,6 +103,7 @@ class QBittorrentReaderTests(unittest.TestCase):
b"Ok.", b"Ok.",
json.dumps([{ json.dumps([{
"hash": qb_hash, "name": "a.txt", "state": "uploading", "hash": qb_hash, "name": "a.txt", "state": "uploading",
"save_path": "/downloads",
}]).encode(), }]).encode(),
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]', b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
torrent_bytes, torrent_bytes,
@@ -157,6 +159,7 @@ class QBittorrentReaderTests(unittest.TestCase):
"infohash_v2": v2_hash, "infohash_v2": v2_hash,
"name": "a.txt", "name": "a.txt",
"state": "uploading", "state": "uploading",
"save_path": "/downloads",
}]).encode(), }]).encode(),
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]', b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
torrent_bytes, torrent_bytes,
@@ -217,9 +220,9 @@ class QBittorrentReaderTests(unittest.TestCase):
malformed_bytes, malformed_hash = torrent("broken.txt", b"bad") malformed_bytes, malformed_hash = torrent("broken.txt", b"bad")
second_bytes, second_hash = torrent("second.txt", b"two") second_bytes, second_hash = torrent("second.txt", b"two")
torrents = [ torrents = [
{"hash": first_hash, "name": "first.txt", "state": "uploading"}, {"hash": first_hash, "name": "first.txt", "state": "uploading", "save_path": "/downloads"},
{"hash": malformed_hash, "name": "broken.txt", "state": "stalledUP"}, {"hash": malformed_hash, "name": "broken.txt", "state": "stalledUP", "save_path": "/downloads"},
{"hash": second_hash, "name": "second.txt", "state": "uploading"}, {"hash": second_hash, "name": "second.txt", "state": "uploading", "save_path": "/downloads"},
] ]
responses = [ responses = [
b"Ok.", json.dumps(torrents).encode(), b"Ok.", json.dumps(torrents).encode(),
+22 -1
View File
@@ -40,6 +40,7 @@ class ResourceTests(unittest.TestCase):
"hash": decoded.info_hash_v2_hex, "hash": decoded.info_hash_v2_hex,
"name": "v2.bin", "name": "v2.bin",
"state": "stoppedUP", "state": "stoppedUP",
"save_path": "/downloads",
}, },
[{ [{
"index": 0, "index": 0,
@@ -91,6 +92,7 @@ class ResourceTests(unittest.TestCase):
"hash": decoded.info_hash_v1_hex, "hash": decoded.info_hash_v1_hex,
"name": "resource", "name": "resource",
"state": "stoppedUP", "state": "stoppedUP",
"save_path": "/downloads/nested",
}, },
[ [
{ {
@@ -106,6 +108,7 @@ class ResourceTests(unittest.TestCase):
observed, observed,
) )
summary = normalized.summary summary = normalized.summary
self.assertEqual(normalized.save_path.as_posix(), "/downloads/nested")
self.assertEqual( self.assertEqual(
summary.runtime_state, resource_pb2.TORRENT_RUNTIME_STATE_STOPPED summary.runtime_state, resource_pb2.TORRENT_RUNTIME_STATE_STOPPED
) )
@@ -139,6 +142,7 @@ class ResourceTests(unittest.TestCase):
"hash": torrent_hash, "hash": torrent_hash,
"name": "with-padding", "name": "with-padding",
"state": "stalledUP", "state": "stalledUP",
"save_path": "/downloads",
}, },
[ [
{ {
@@ -169,7 +173,7 @@ class ResourceTests(unittest.TestCase):
metainfo = encode({b"info": info}) metainfo = encode({b"info": info})
torrent = { torrent = {
"hash": hashlib.sha1(encode(info)).hexdigest(), "hash": hashlib.sha1(encode(info)).hexdigest(),
"name": "renamed", "state": "uploading", "name": "renamed", "state": "uploading", "save_path": "/downloads",
} }
renamed = normalize_resource(torrent, [{ renamed = normalize_resource(torrent, [{
"index": 0, "name": "renamed.txt", "size": 3, "index": 0, "name": "renamed.txt", "size": 3,
@@ -182,6 +186,23 @@ class ResourceTests(unittest.TestCase):
"progress": 1.0, "priority": 1, "progress": 1.0, "priority": 1,
}], metainfo) }], metainfo)
def test_missing_or_out_of_shape_save_path_is_rejected(self):
info = {
b"length": 3, b"name": b"a.txt", b"piece length": 16384,
b"pieces": b"x" * 20,
}
metainfo = encode({b"info": info})
torrent = {
"hash": hashlib.sha1(encode(info)).hexdigest(),
"name": "a.txt", "state": "uploading", "save_path": "relative",
}
with self.assertRaisesRegex(ResourceError, "save path is unsafe"):
normalize_resource(torrent, [{
"index": 0, "name": "a.txt", "size": 3,
"completed": 3, "priority": 1,
}], metainfo)
def test_noncanonical_bencode_is_rejected(self): def test_noncanonical_bencode_is_rejected(self):
with self.assertRaisesRegex(BencodeError, "unsorted"): with self.assertRaisesRegex(BencodeError, "unsorted"):
decode_metainfo(b"d4:infod1:b1:x1:a1:yee") decode_metainfo(b"d4:infod1:b1:x1:a1:yee")