Compare commits

..
3 Commits
Author SHA1 Message Date
cabbage 87cca59d3a fix: wait for release builder platforms 2026-08-15 08:19:15 +00:00
cabbage 618bb3d206 docs: clarify qB save path resolution 2026-08-15 08:12:11 +00:00
cabbage a1bf3e4315 fix: resolve per-torrent qb save paths 2026-08-14 11:56:06 +00:00
19 changed files with 509 additions and 66 deletions
+15 -4
View File
@@ -120,6 +120,8 @@ username = "${QB_USER}"
password_file = "/run/secrets/qb_password"
api_root = "/downloads"
local_root = "/data/qb"
# Optional only when a nested qB path uses a different client-visible mount.
# local_path_overrides = { "/downloads/fast" = "/data/qb-fast" }
[syncthing]
endpoint = "http://syncthing:8384"
@@ -139,6 +141,14 @@ accounting. The override key is the normalized Syncthing folder path beneath
`api_root`, not its folder ID. See the production deployment README for the
required compose and override pattern.
qBittorrent content is resolved differently: every existing torrent keeps its
qB-reported `save_path`. The client maps that path beneath `qbittorrent.api_root`
to its own mount before a source, existing-target, permission, or eviction
operation. Thus `/downloads/Downloading` naturally maps below `/data/qb`;
there is no migration or per-resource configuration. Add a qB
`local_path_overrides` entry only when that nested API prefix is a separate
client mount.
The remaining node examples omit optional `[connection]`, `[jobs]`, and
`[backup]` tables and therefore use these same defaults; deployments may
override them per node.
@@ -277,10 +287,11 @@ The publisher deliberately uses `moby/buildkit:rootless` with
`--oci-worker-no-process-sandbox`. On nested Docker hosts, the default OCI
sandbox can fail while masking `/proc/acpi` for an emulated build; rootless
BuildKit confines that compatibility setting to the disposable builder. It
refuses to publish unless `docker buildx inspect` reports both `linux/amd64`
and `linux/arm64`, and removes the builder and binfmt handler on success,
failure, or interruption. Retain the displayed manifest digest in release
notes and deploy the immutable tag or digest.
waits briefly for the new worker to observe binfmt, then refuses to publish
unless `docker buildx inspect` reports both `linux/amd64` and `linux/arm64`.
On capability failure it prints that inspection output and removes the builder
and binfmt handler on success, failure, or interruption. Retain the displayed
manifest digest in release notes and deploy the immutable tag or digest.
## Operator usage
+14 -1
View File
@@ -86,6 +86,9 @@ python3 scripts/preflight-deployment.py ... \
- That future route path and qBittorrent content root use one client bind
mount, so hard-link staging remains possible rather than silently falling
back to a space-consuming copy.
- Every explicit `qbittorrent.local_path_overrides` entry maps the same host
path in the qBittorrent and client containers. This protects nested qB save
paths that use a dedicated bind mount.
- A real `link(2)` operation between a unique zero-byte file in the qB root
and one in the future automatic-route root. The probe verifies that both
names refer to the same inode and removes them unconditionally.
@@ -96,7 +99,8 @@ python3 scripts/preflight-deployment.py ... \
- The token, qB password, and Syncthing API-key files are non-empty regular
files with no group/world permissions.
- The client image can read its configuration and reports usable permissions,
sparse-file support, and filesystem capabilities for both roots.
sparse-file support, and filesystem capabilities for both primary roots and
every configured qBittorrent local-path override.
- qBittorrent authentication/version compatibility and Syncthing
authentication/device identity are healthy from the client container.
@@ -106,6 +110,15 @@ root and add a `local_path_overrides` mapping for that exact API path. This
prevents automatic route folders being created on a small configuration
filesystem while the client expects to hardlink from the qB data mount.
qBittorrent resources may use any `save_path` below `qbittorrent.api_root`.
At job preflight the client maps that qB API path to its local mount and uses
it as the resource root; data is never moved to fit Archive Control. A resource
outside the configured qB API root, or whose resolved directory is unavailable
or not a real directory in the client container, fails that job before staging
or eviction. Use `qbittorrent.local_path_overrides` only for a nested qB API
prefix backed by a distinct client mount; this preflight verifies the Docker
bind topology for each such override.
When converting an existing node, stop its client and Syncthing containers,
move each existing `routes/<route-id>` directory from the old Syncthing config
tree into the new qB-backed route-root directory, then recreate Syncthing and
+19 -2
View File
@@ -42,8 +42,8 @@ and exported-metainfo data, keeps the qB-local hash separately, and calculates:
- normalized selected and selected-complete file-index sets;
- torrent runtime state;
- canonical path flag and content revision;
- a save-path fingerprint based on validated path mapping, never a leaked
absolute host path.
- a validated qBittorrent `save_path`, retained only in the client's local
normalized observation.
qBittorrent 5 may expose a pure-v2 or hybrid torrent under the first 20 bytes
of its v2 hash while separately advertising full `infohash_v1` and
@@ -56,6 +56,23 @@ All entries retain qBittorrent's stable torrent file index. Renamed/noncanonical
content paths are rejected in v1 because they cannot be transported and merged
without ambiguity.
### Per-torrent local content roots
`save_path` is not inventory, placement, or protocol data. It is qBittorrent
metadata used only by the daemon that queried qBittorrent. Before staging a
source, merging into an existing target, applying post-recheck permissions, or
evicting files, that daemon maps the torrent's API-visible `save_path` through
`qbittorrent.api_root`/`local_root`. The most-specific
`qbittorrent.local_path_overrides` mapping wins when a nested path is exposed
through a distinct client container mount.
An ordinary nested qBittorrent path such as `/media/Data/Downloading` needs no
per-resource configuration: it resolves beneath the configured root. A path
outside that root, an unmapped distinct mount, or a mapped local path that is
not a visible real directory fails the affected job before filesystem mutation.
This check is intentionally per resource; an unrelated malformed qBittorrent
entry cannot prevent normal resources from being staged or transferred.
### Verification guard
The client captures transfer counters and state before recheck, issues recheck,
+7 -1
View File
@@ -14,6 +14,13 @@ client:
a regular file or an explicitly created directory;
5. verifies every operation remains beneath the local configured root.
For qBittorrent content, the root is resolved per resource: qB's authoritative
API-visible `save_path` is mapped beneath `qbittorrent.api_root` (or a more
specific configured local override) into the client namespace. This permits
existing nested save paths without moving data, but does not permit paths
outside the configured boundary. The resolved path remains local client state
and is never sent to the control daemon.
Sockets, devices, FIFOs, symlinks, and other special entries fail preflight.
Permission or ownership mismatch is fail-fast. Archive Control never changes
source ownership or mode to make a job pass.
@@ -150,4 +157,3 @@ Offline tooling provides list, verify, and restore. Restore requires stopped
daemon access, verifies the chosen backup, preserves the suspect database under
a timestamped name, installs the replacement atomically, and runs integrity and
schema checks before normal startup. Backups contain no configured secrets.
+22 -1
View File
@@ -134,7 +134,7 @@ from pathlib import Path
from archive_clients.config import ClientConfig
config=ClientConfig.load(Path(sys.argv[1]))
value={"shared_token_file":str(config.shared_token_file)}
value["qbittorrent"]={"api_root":str(config.qbittorrent.api_root),"local_root":str(config.qbittorrent.local_root),"password_file":str(config.qbittorrent.password_file)}
value["qbittorrent"]={"api_root":str(config.qbittorrent.api_root),"local_root":str(config.qbittorrent.local_root),"password_file":str(config.qbittorrent.password_file),"local_path_overrides":{str(api):str(local) for api,local in config.qbittorrent.local_path_overrides}}
value["syncthing"]={"api_root":str(config.syncthing.api_root),"local_root":str(config.syncthing.local_root),"api_key_file":str(config.syncthing.api_key_file),"local_path_overrides":{str(api):str(local) for api,local in config.syncthing.local_path_overrides}}
print(json.dumps(value,sort_keys=True))'''
result = subprocess.run(
@@ -174,6 +174,26 @@ raise SystemExit(0 if all(p.state == 1 for p in probes) else 1)'''
print(result.stdout.strip())
def require_qb_override_mappings(
client: dict[str, Any], qbittorrent: dict[str, Any], qb: dict[str, Any],
) -> None:
"""Prove every explicit qB API/local override sees the same host bytes."""
overrides = qb.get("local_path_overrides", {})
if not isinstance(overrides, dict):
raise CheckFailure("qbittorrent.local_path_overrides must be a table")
for api_path, local_path in overrides.items():
if not isinstance(api_path, str) or not isinstance(local_path, str):
raise CheckFailure(
"qbittorrent.local_path_overrides entries are invalid"
)
require_same_path(
f"qBittorrent override {api_path}/local mapping",
map_path(qbittorrent, api_path),
map_path(client, local_path),
)
def run_hardlink_probe(
container: str, qb_root: str, route_root: str, user: str | None = None,
) -> None:
@@ -298,6 +318,7 @@ def main(argv: list[str] | None = None) -> int:
syncthing_route, client_route,
)
require_same_path("qBittorrent api_root/local_root", qb_api, client_qb)
require_qb_override_mappings(client, qbittorrent, qb)
if client_qb.destination != client_route.destination:
raise CheckFailure(
"qBittorrent and future route roots use separate client bind "
+23 -4
View File
@@ -69,13 +69,32 @@ docker buildx create --name "${builder}" --driver docker-container \
--buildkitd-flags "${buildkitd_flags}" --use >/dev/null
builder_created=true
platforms=$(docker buildx inspect "${builder}" --bootstrap 2>&1)
for platform in linux/amd64 linux/arm64; do
# A newly-created rootless worker can publish its native platform before it has
# observed the just-registered binfmt handler. Do not mistake that brief
# startup state for a partial-release-capable builder.
platforms=''
supports_all=false
for attempt in {1..10}; do
platforms=$(docker buildx inspect "${builder}" --bootstrap 2>&1)
supports_all=true
for platform in linux/amd64 linux/arm64; do
if ! grep -Fq "${platform}" <<<"${platforms}"; then
echo "Builder ${builder} does not support ${platform}; refusing partial release." >&2
exit 1
supports_all=false
break
fi
done
if [[ ${supports_all} == true ]]; then
break
fi
if [[ ${attempt} -lt 10 ]]; then
sleep 1
fi
done
if [[ ${supports_all} != true ]]; then
echo "Builder ${builder} does not support both required platforms; refusing partial release." >&2
printf '%s\n' "${platforms}" >&2
exit 1
fi
tags=(--tag "${repository}:${tag}")
if [[ ${also_latest} == true ]]; then
+11 -2
View File
@@ -27,10 +27,16 @@ def main(argv: Sequence[str] | None = None) -> int:
parser.add_argument("--check-config", action="store_true")
arguments = parser.parse_args(argv)
config = ClientConfig.load(arguments.config, arguments.mode)
qb_extra_roots = tuple(sorted({
root
for _, root in config.qbittorrent.local_path_overrides
if root != config.qbittorrent.local_root
}))
probes = [
probe_root(config.qbittorrent.local_root),
probe_root(config.syncthing.local_root),
]
qb_extra_probes = tuple(probe_root(root) for root in qb_extra_roots)
probe_writable_directory(config.state_db.parent)
probe_writable_directory(config.backup_dir)
shared_token = config.read_shared_token()
@@ -40,7 +46,10 @@ def main(argv: Sequence[str] | None = None) -> int:
print(json.dumps({
"client_id": config.client_id,
"role": config.role,
"filesystems": [probe.__dict__ | {"root": str(probe.root)} for probe in probes],
"filesystems": [
probe.__dict__ | {"root": str(probe.root)}
for probe in (*probes[:1], *qb_extra_probes, *probes[1:])
],
}, sort_keys=True))
return 0
configure_logging(
@@ -68,7 +77,7 @@ def main(argv: Sequence[str] | None = None) -> int:
},
)
asyncio.run(ArchiveClientDaemon(
config, probes, service_probes,
config, probes, service_probes, qb_extra_probes=qb_extra_probes,
resource_reader=QBittorrentReader(config.qbittorrent),
).run())
except KeyboardInterrupt:
-2
View File
@@ -213,8 +213,6 @@ def _local_path_overrides(
value: dict[str, Any], api_root: PurePosixPath, name: str
) -> tuple[tuple[PurePosixPath, Path], ...]:
raw = value.get("local_path_overrides", {})
if name != "syncthing" and raw:
raise ConfigError(f"{name}.local_path_overrides is unsupported")
if not isinstance(raw, dict):
raise ConfigError(f"{name}.local_path_overrides must be a table")
parsed: list[tuple[PurePosixPath, Path]] = []
+7 -1
View File
@@ -64,6 +64,7 @@ class ArchiveClientDaemon:
service_probes: list[ServiceProbe],
resource_reader: QBittorrentReader | None = None,
route_manager: SyncthingRouteManager | None = None,
qb_extra_probes: tuple[FilesystemProbe, ...] = (),
):
if len(probes) != 2:
raise ValueError(
@@ -71,6 +72,7 @@ class ArchiveClientDaemon:
)
self.config = config
self.probes = probes
self.qb_probes = (probes[0], *qb_extra_probes)
self.service_probes = service_probes
self.inventory = (
InventoryService(resource_reader, config.client_id)
@@ -119,9 +121,13 @@ class ArchiveClientDaemon:
store=self.store,
qb_root=config.qbittorrent.local_root,
qb_api_root=config.qbittorrent.api_root,
qb_roots=config.qbittorrent.roots,
route_path=self._route_path,
syncthing_transport=self.routes.transport,
sparse_supported=all(probe.sparse_files for probe in probes),
sparse_supported=all(
probe.sparse_files
for probe in (*self.qb_probes, probes[1])
),
poll_interval=config.jobs.poll_interval,
verification_timeout=config.jobs.verification_timeout,
free_space_reserve_bytes=(
+35 -15
View File
@@ -7,7 +7,7 @@ import hashlib
import os
import stat
from pathlib import Path, PurePosixPath
from typing import Iterable
from typing import Callable, Iterable
from archive_clients.qbittorrent import QBittorrentReader
from archive_clients.resources import NormalizedResource
@@ -23,7 +23,7 @@ def verify_and_snapshot(
job_id: str,
resource: NormalizedResource,
selected_indices: Iterable[int],
qb_root: Path,
content_root: Path,
store: ClientStore,
) -> dict[str, object]:
selected = set(selected_indices)
@@ -38,7 +38,7 @@ def verify_and_snapshot(
f"cache file {index} is not selected and complete"
)
relative = _relative(item.canonical_path)
path = qb_root.joinpath(*relative.parts)
path = content_root.joinpath(*relative.parts)
try:
metadata = path.lstat()
except FileNotFoundError as exc:
@@ -65,7 +65,7 @@ def verify_and_snapshot(
"sha256": _sha256(path),
}
)
snapshot = {"files": files}
snapshot = {"content_root": str(content_root), "files": files}
return store.put_job_artifact(job_id, "eviction-snapshot", snapshot)["value"]
@@ -90,9 +90,9 @@ def remove_qb_entry(
def safe_unlink(
*,
job_id: str,
qb_root: Path,
qbittorrent: QBittorrentReader,
store: ClientStore,
resource_root: Callable[[NormalizedResource], Path],
) -> dict[str, object]:
completed = store.get_job_artifact(job_id, "eviction-unlinked")
if completed is not None:
@@ -101,11 +101,19 @@ def safe_unlink(
if snapshot_row is None:
raise EvictionError("eviction snapshot is missing")
snapshot = snapshot_row["value"]
if not isinstance(snapshot, dict):
raise EvictionError("eviction snapshot is invalid")
content_root_value = snapshot.get("content_root")
if not isinstance(content_root_value, str):
raise EvictionError("eviction snapshot content root is missing")
content_root = Path(content_root_value)
if not content_root.is_absolute() or content_root.is_symlink():
raise EvictionError("eviction snapshot content root is unsafe")
files = snapshot.get("files")
if not isinstance(files, list):
raise EvictionError("eviction snapshot is invalid")
referenced = _remaining_paths(qbittorrent.list_resources())
referenced = _remaining_paths(qbittorrent.list_resources(), resource_root)
removed: list[str] = []
retained: list[dict[str, str]] = []
directories: set[Path] = set()
@@ -113,10 +121,10 @@ def safe_unlink(
if not isinstance(record, dict) or not isinstance(record.get("path"), str):
raise EvictionError("eviction file record is invalid")
relative = _relative(record["path"])
if relative.as_posix() in referenced:
path = content_root.joinpath(*relative.parts)
if path in referenced:
retained.append({"path": relative.as_posix(), "reason": "shared"})
continue
path = qb_root.joinpath(*relative.parts)
try:
metadata = path.lstat()
except FileNotFoundError:
@@ -142,7 +150,7 @@ def safe_unlink(
path.unlink()
removed.append(relative.as_posix())
parent = path.parent
while parent != qb_root:
while parent != content_root:
directories.add(parent)
parent = parent.parent
@@ -153,7 +161,7 @@ def safe_unlink(
try:
directory.rmdir()
removed_directories.append(
directory.relative_to(qb_root).as_posix()
directory.relative_to(content_root).as_posix()
)
except OSError as exc:
if exc.errno not in {errno.ENOTEMPTY, errno.ENOENT}:
@@ -214,12 +222,24 @@ def compensate_materialized_files(
return removed
def _remaining_paths(resources: Iterable[NormalizedResource]) -> set[str]:
return {
item.canonical_path
for resource in resources
def _remaining_paths(
resources: Iterable[NormalizedResource],
resource_root: Callable[[NormalizedResource], Path],
) -> set[Path]:
result: set[Path] = set()
for resource in resources:
try:
root = resource_root(resource)
except (OSError, RuntimeError, ValueError):
# A malformed unrelated qB entry must not stop an otherwise
# safe eviction. Its unresolvable path cannot be considered a
# shared path under the verified eviction root.
continue
result.update(
root.joinpath(*_relative(item.canonical_path).parts)
for item in resource.files
}
)
return result
def _relative(value: str) -> PurePosixPath:
+78 -16
View File
@@ -14,6 +14,7 @@ import uuid
from pathlib import Path, PurePosixPath
from typing import Callable, Iterable
from archive_clients.config import ConfigError, RootMapping
from archive_clients.protocol import decode_message, encode_message
from archive_clients.eviction import (
EvictionError,
@@ -67,6 +68,7 @@ class ClientJobExecutor:
store: ClientStore,
qb_root: Path,
qb_api_root: PurePosixPath,
qb_roots: RootMapping | None = None,
route_path: Callable[[str], Path],
syncthing_transport: object,
sparse_supported: bool,
@@ -78,7 +80,10 @@ class ClientJobExecutor:
self.qbittorrent = qbittorrent
self.store = store
self.qb_root = qb_root
self.qb_api_root = qb_api_root
self.qb_api_root = PurePosixPath(qb_api_root)
self.qb_roots = qb_roots or RootMapping(
self.qb_api_root, self.qb_root
)
self.route_path = route_path
self.syncthing_transport = syncthing_transport
self.sparse_supported = sparse_supported
@@ -479,6 +484,7 @@ class ClientJobExecutor:
"archive coverage no longer covers the cache selection"
)
resource = self._resource(definition)
resource_root = self._resource_root(resource)
current = _resource_fingerprint(resource, self.client_id)
if not _fingerprint_matches(
current, definition.eviction.cache_fingerprint
@@ -490,7 +496,7 @@ class ClientJobExecutor:
job_id=definition.job_id,
resource=resource,
selected_indices=requested,
qb_root=self.qb_root,
content_root=resource_root,
store=self.store,
)
return None
@@ -505,9 +511,9 @@ class ClientJobExecutor:
if step == job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK:
safe_unlink(
job_id=definition.job_id,
qb_root=self.qb_root,
qbittorrent=self.qbittorrent,
store=self.store,
resource_root=self._resource_root,
)
placement = resource_pb2.Placement(
client_id=self.client_id,
@@ -538,7 +544,8 @@ class ClientJobExecutor:
raise JobExecutionError(
"source resource changed after job confirmation"
)
self._reject_unsafe_partfile(definition, resource)
source_root = self._resource_root(resource)
self._reject_unsafe_partfile(definition, resource, source_root)
indices = _selection_indices(definition.transfer.transfer_delta_files)
by_index = {item.file_index: item for item in resource.files}
if not indices or any(index not in by_index for index in indices):
@@ -602,7 +609,7 @@ class ClientJobExecutor:
self._require_space(
route_root,
self._copy_required_bytes(
self.qb_root,
source_root,
route_root,
(
(entry.target_canonical_path, entry.logical_bytes)
@@ -613,7 +620,7 @@ class ClientJobExecutor:
)
stage_transfer(
manifest,
source_root=self.qb_root,
source_root=source_root,
sync_root=route_root,
store=self.store,
artifact_sources={"metainfo/source.torrent": metainfo_path},
@@ -691,23 +698,27 @@ class ClientJobExecutor:
"target materialization was sent to the wrong client"
)
published = load_published_transfer(self._job_directory(definition))
info_hash = _info_hash(definition)
resource = self.qbittorrent.get_resource(info_hash)
target_root = (
self._resource_root(resource)
if resource is not None else self.qb_root
)
# The target can likewise hardlink an arrived Syncthing payload into
# qB's content root when those directories share a filesystem.
self._require_space(
self.qb_root,
target_root,
self._copy_required_bytes(
published.job_directory,
self.qb_root,
target_root,
(
(entry.payload_relative_path, entry.logical_bytes)
for entry in published.manifest.files
),
),
)
info_hash = _info_hash(definition)
resource = self.qbittorrent.get_resource(info_hash)
if resource is not None:
self._reject_unsafe_partfile(definition, resource)
self._reject_unsafe_partfile(definition, resource, target_root)
if definition.transfer.HasField("target_baseline_fingerprint"):
if resource is None or not _fingerprint_matches(
_resource_fingerprint(resource, self.client_id),
@@ -732,13 +743,14 @@ class ClientJobExecutor:
== resource_pb2.TORRENT_RUNTIME_STATE_STOPPED
),
"total_file_count": len(published.manifest.files),
"content_root": str(target_root),
}
self.store.put_job_artifact(
definition.job_id, "target-baseline", baseline
)
materialize_transfer(
published,
target_root=self.qb_root,
target_root=target_root,
store=self.store,
sparse_supported=self.sparse_supported,
cancel_check=lambda: self._raise_if_cancelled(
@@ -813,7 +825,9 @@ class ClientJobExecutor:
raise JobExecutionError(
"verified qBittorrent resource disappeared"
)
_normalize_verified_resource_permissions(self.qb_root, verified)
_normalize_verified_resource_permissions(
self._resource_root(verified), verified
)
if should_start:
self.qbittorrent.start(qb_torrent_id)
placement = resource_pb2.Placement(
@@ -852,6 +866,7 @@ class ClientJobExecutor:
)
return
baseline = baseline_row["value"]
content_root = self._artifact_content_root(baseline)
info_hash = _info_hash(definition)
present = self.qbittorrent.get_resource(info_hash)
if present is not None:
@@ -870,7 +885,7 @@ class ClientJobExecutor:
self.qbittorrent.delete_entry(qb_torrent_id)
compensate_materialized_files(
job_id=definition.job_id,
qb_root=self.qb_root,
qb_root=content_root,
store=self.store,
)
@@ -906,10 +921,11 @@ class ClientJobExecutor:
self,
definition: job_pb2.JobDefinition,
resource: NormalizedResource,
content_root: Path,
) -> None:
roots = {self.qb_root}
roots = {content_root}
for item in resource.files:
candidate = self.qb_root / PurePosixPath(item.canonical_path).parts[0]
candidate = content_root / PurePosixPath(item.canonical_path).parts[0]
roots.add(candidate if candidate.is_dir() else candidate.parent)
hashes = {
value.lower() for value in (
@@ -928,6 +944,52 @@ class ClientJobExecutor:
"safely transferred by this client"
)
def _resource_root(self, resource: NormalizedResource) -> Path:
"""Resolve a qB resource's save path within this client's mounts.
qB returns paths in its own API/container namespace. The configured
root mapping translates those paths into the client namespace and
supports nested save paths and explicit prefix overrides. A missing
save path is retained only for older in-process test fixtures; every
normalized qB API resource has one.
"""
if resource.save_path is None:
return self.qb_root
try:
root = self.qb_roots.api_to_local(resource.save_path.as_posix())
except ConfigError as exc:
raise JobExecutionError(
"qBittorrent resource save path is outside the configured "
"API root or has no local path mapping"
) from exc
try:
metadata = root.lstat()
except FileNotFoundError as exc:
raise JobExecutionError(
"qBittorrent resource save path is not visible in the "
"client container"
) from exc
if not stat.S_ISDIR(metadata.st_mode):
raise JobExecutionError(
"qBittorrent resource save path is not a real directory in "
"the client container"
)
return root
def _artifact_content_root(self, baseline: object) -> Path:
if not isinstance(baseline, dict):
raise JobExecutionError("target baseline is invalid")
value = baseline.get("content_root")
if not isinstance(value, str):
# A pre-existing durable baseline predates per-resource roots.
# It can only have materialized to the configured target root.
return self.qb_root
root = Path(value)
if not root.is_absolute() or root.is_symlink():
raise JobExecutionError("target baseline content root is unsafe")
return root
def _raise_if_cancelled(self, job_id: str) -> None:
event = self._cancel_events.get(job_id)
if event is not None and event.is_set():
+29 -1
View File
@@ -19,10 +19,21 @@ class ResourceError(ValueError):
@dataclass(frozen=True)
class NormalizedResource:
"""A normalized qB observation for protocol data and local file work.
``save_path`` is qBittorrent's API-visible per-torrent content root. It
is deliberately local-only: clients resolve it through their qB path
mapping immediately before filesystem work, and it is never serialized in
inventory, placements, or control protocol messages.
"""
summary: resource_pb2.ResourceSummary
files: tuple[resource_pb2.TorrentFile, ...]
metainfo: Metainfo
metainfo_bytes: bytes = b""
# qBittorrent's API-visible save path is intentionally local-only. It
# must never become part of inventory or placement protocol messages.
save_path: PurePosixPath | None = None
def build_content_tree(
@@ -137,6 +148,7 @@ def normalize_resource(
character not in "0123456789abcdef" for character in qb_torrent_id
):
raise ResourceError("torrent hash is invalid")
save_path = _save_path(torrent.get("save_path"))
summary = resource_pb2.ResourceSummary(
qb_torrent_id=qb_torrent_id,
display_name=_string(torrent.get("name"), "torrent name"),
@@ -172,7 +184,9 @@ def normalize_resource(
revision_data, sort_keys=True, separators=(",", ":"),
).encode("utf-8")).hexdigest()
summary.observed_at.FromDatetime(observed_at)
return NormalizedResource(summary, tuple(files), metainfo, metainfo_bytes)
return NormalizedResource(
summary, tuple(files), metainfo, metainfo_bytes, save_path
)
def _set_selection(target: Any, indices: list[int]) -> None:
@@ -217,6 +231,20 @@ def _path(value: Any) -> str:
return candidate.as_posix()
def _save_path(value: Any) -> PurePosixPath:
"""Validate qBittorrent's API-visible per-torrent content root."""
path = _string(value, "torrent save path")
candidate = PurePosixPath(path)
if (
not candidate.is_absolute()
or ".." in candidate.parts
or "." in candidate.parts
):
raise ResourceError("qBittorrent torrent save path is unsafe")
return candidate
def _integer(value: Any, name: str) -> int:
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
raise ResourceError(f"{name} is invalid")
+59
View File
@@ -0,0 +1,59 @@
import io
import json
import os
import tempfile
import unittest
from contextlib import redirect_stdout
from pathlib import Path
from archive_clients.cli import main
class ClientCliTests(unittest.TestCase):
def test_check_config_probes_every_qb_override_root(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
for name in ("token", "qb-password", "syncthing-key"):
path = root / name
path.write_text(name, encoding="utf-8")
os.chmod(path, 0o600)
for name in ("qb", "qb-fast", "sync", "backups"):
(root / name).mkdir()
config = root / "client.toml"
config.write_text(
f'''client_id = "cache-1"
display_name = "Cache 1"
role = "cache"
control_endpoint = "ws://control/archive_control"
shared_token_file = "{root / "token"}"
state_db = "{root / "state.db"}"
backup_dir = "{root / "backups"}"
[qbittorrent]
endpoint = "http://qb"
username = "admin"
password_file = "{root / "qb-password"}"
api_root = "/downloads"
local_root = "{root / "qb"}"
local_path_overrides = {{ "/downloads/fast" = "{root / "qb-fast"}" }}
[syncthing]
endpoint = "http://syncthing"
api_key_file = "{root / "syncthing-key"}"
api_root = "/sync"
local_root = "{root / "sync"}"
''',
encoding="utf-8",
)
output = io.StringIO()
with redirect_stdout(output):
self.assertEqual(main(["--config", str(config), "--check-config"]), 0)
reported = json.loads(output.getvalue())
self.assertEqual(
[item["root"] for item in reported["filesystems"]],
[str(root / "qb"), str(root / "qb-fast"), str(root / "sync")],
)
if __name__ == "__main__":
unittest.main()
+29
View File
@@ -78,6 +78,35 @@ class ConfigTests(unittest.TestCase):
Path("/local/qb/Sync"),
)
def test_qbittorrent_can_override_a_nested_save_path(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
for name in ("token", "qb-password", "syncthing-key"):
path = root / name
path.write_text(name, encoding="utf-8")
os.chmod(path, 0o600)
(root / "qb").mkdir()
(root / "fast").mkdir()
(root / "sync").mkdir()
config_path = root / "client.toml"
config_path.write_text(
_config(root).replace(
f'local_root = "{root / "qb"}"',
f'local_root = "{root / "qb"}"\n'
"local_path_overrides = { \"/downloads/fast\" = "
f'"{root / "fast"}" }}',
1,
),
encoding="utf-8",
)
config = ClientConfig.load(config_path)
self.assertEqual(
config.qbittorrent.roots.api_to_local(
"/downloads/fast/resource/file.bin"
),
root / "fast/resource/file.bin",
)
def test_endpoint_scheme_and_job_keys_are_strict(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
+20
View File
@@ -50,6 +50,26 @@ class DeploymentPreflightTests(unittest.TestCase):
"/data/qb/.archive-control-routes",
)
def test_qb_override_must_map_to_the_same_host_path(self):
client = {"Mounts": [
{"Type": "bind", "Source": "/srv/fast", "Destination": "/data/fast"},
]}
qbittorrent = {"Mounts": [
{"Type": "bind", "Source": "/srv/fast", "Destination": "/downloads/fast"},
]}
preflight.require_qb_override_mappings(
client, qbittorrent,
{"local_path_overrides": {"/downloads/fast": "/data/fast"}},
)
with self.assertRaisesRegex(preflight.CheckFailure, "host paths differ"):
preflight.require_qb_override_mappings(
client,
{"Mounts": [{
"Type": "bind", "Source": "/other", "Destination": "/downloads/fast",
}]},
{"local_path_overrides": {"/downloads/fast": "/data/fast"}},
)
def test_runs_hardlink_probe_with_qb_and_route_roots(self):
completed = __import__("subprocess").CompletedProcess(
args=[], returncode=0, stdout="hard-link staging probe passed\n", stderr=""
+47 -7
View File
@@ -1,5 +1,7 @@
import tempfile
import unittest
from dataclasses import replace
from pathlib import PurePosixPath
from pathlib import Path
from archive_clients.eviction import (
@@ -86,7 +88,7 @@ class EvictionTests(unittest.TestCase):
job_id="job-1",
resource=evicted,
selected_indices=[0, 1],
qb_root=self.root,
content_root=self.root,
store=self.store,
)
remove_qb_entry(
@@ -97,9 +99,9 @@ class EvictionTests(unittest.TestCase):
)
result = safe_unlink(
job_id="job-1",
qb_root=self.root,
qbittorrent=qb,
store=self.store,
resource_root=lambda _: self.root,
)
self.assertTrue((self.root / "tree/shared.bin").exists())
self.assertFalse((self.root / "tree/owned.bin").exists())
@@ -121,7 +123,7 @@ class EvictionTests(unittest.TestCase):
job_id="job-1",
resource=evicted,
selected_indices=[0],
qb_root=self.root,
content_root=self.root,
store=self.store,
)
path.unlink()
@@ -134,9 +136,9 @@ class EvictionTests(unittest.TestCase):
)
result = safe_unlink(
job_id="job-1",
qb_root=self.root,
qbittorrent=qb,
store=self.store,
resource_root=lambda _: self.root,
)
self.assertTrue(path.exists())
self.assertEqual(
@@ -152,7 +154,7 @@ class EvictionTests(unittest.TestCase):
job_id="job-1",
resource=evicted,
selected_indices=[0],
qb_root=self.root,
content_root=self.root,
store=self.store,
)
remove_qb_entry(
@@ -163,19 +165,57 @@ class EvictionTests(unittest.TestCase):
)
first = safe_unlink(
job_id="job-1",
qb_root=self.root,
qbittorrent=qb,
store=self.store,
resource_root=lambda _: self.root,
)
second = safe_unlink(
job_id="job-1",
qb_root=self.root,
qbittorrent=qb,
store=self.store,
resource_root=lambda _: self.root,
)
self.assertEqual(first, second)
self.assertEqual(qb.deleted, ["a" * 40])
def test_nested_save_path_uses_its_own_root_and_not_a_same_named_peer(self):
nested = self.root / "Downloading"
nested.mkdir()
evicted = replace(
normalized("a" * 40, ["resource/file.bin"]),
save_path=PurePosixPath("/downloads/Downloading"),
)
peer = replace(
normalized("b" * 40, ["resource/file.bin"]),
save_path=PurePosixPath("/downloads"),
)
nested_file = nested / "resource/file.bin"
nested_file.parent.mkdir()
nested_file.write_bytes(b"x" * evicted.files[0].logical_bytes)
root_file = self.root / "resource/file.bin"
root_file.parent.mkdir()
root_file.write_bytes(b"x" * peer.files[0].logical_bytes)
qb = FakeQB(evicted, [peer])
verify_and_snapshot(
job_id="job-1", resource=evicted, selected_indices=[0],
content_root=nested, store=self.store,
)
remove_qb_entry(
job_id="job-1", torrent_hash="a" * 40,
qbittorrent=qb, store=self.store,
)
safe_unlink(
job_id="job-1", qbittorrent=qb, store=self.store,
resource_root=lambda item: (
nested
if item.save_path == PurePosixPath("/downloads/Downloading")
else self.root
),
)
self.assertFalse(nested_file.exists())
self.assertTrue(root_file.exists())
if __name__ == "__main__":
unittest.main()
+64 -3
View File
@@ -4,11 +4,13 @@ import stat
import tempfile
import threading
import unittest
from pathlib import Path
from dataclasses import replace
from pathlib import Path, PurePosixPath
from unittest.mock import Mock, patch
from uuid import uuid4
from archive_clients.bencode import encode
from archive_clients.config import RootMapping
from archive_clients.jobs import (
ClientJobExecutor,
JobExecutionError,
@@ -246,6 +248,55 @@ class ClientJobHappyPathTests(unittest.TestCase):
content=b"x" * 4096,
)
def test_nested_qb_save_path_stages_from_mapped_subdirectory(self):
with tempfile.TemporaryDirectory() as directory:
self._run_transfer(
Path(directory),
job_pb2.JOB_OPERATION_UNARCHIVE,
source_save_path="/downloads/Downloading",
)
def test_qb_save_path_preflight_rejects_unmapped_path(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
executor = ClientJobExecutor(
client_id="cache-1", qbittorrent=Mock(),
store=ClientStore(root / "state.db"), qb_root=root,
qb_api_root=PurePosixPath("/downloads"),
route_path=lambda _: root, syncthing_transport=Mock(),
sparse_supported=True,
)
resource = NormalizedResource(
resource_pb2.ResourceSummary(), (), Mock(),
save_path=PurePosixPath("/outside"),
)
with self.assertRaisesRegex(JobExecutionError, "outside"):
executor._resource_root(resource)
def test_qb_save_path_override_uses_its_dedicated_local_mount(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
primary = root / "primary"
override = root / "override"
primary.mkdir()
override.mkdir()
executor = ClientJobExecutor(
client_id="cache-1", qbittorrent=Mock(),
store=ClientStore(root / "state.db"), qb_root=primary,
qb_api_root=PurePosixPath("/downloads"),
qb_roots=RootMapping(
PurePosixPath("/downloads"), primary,
((PurePosixPath("/downloads/slow"), override),),
),
route_path=lambda _: root, syncthing_transport=Mock(),
sparse_supported=True,
)
resource = NormalizedResource(
resource_pb2.ResourceSummary(), (), Mock(),
save_path=PurePosixPath("/downloads/slow"),
)
self.assertEqual(executor._resource_root(resource), override)
def test_mount_boundary_requires_copy_space_even_with_same_device(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
@@ -390,6 +441,7 @@ class ClientJobHappyPathTests(unittest.TestCase):
source_stage_free_bytes: int | None = None,
content: bytes = b"archive-control-happy-path",
syncthing: CompleteSyncthing | None = None,
source_save_path: str = "/downloads",
):
source_root = root / "source"
target_root = root / "target"
@@ -397,7 +449,12 @@ class ClientJobHappyPathTests(unittest.TestCase):
source_root.mkdir()
target_root.mkdir()
route_root.mkdir()
(source_root / "fixture.bin").write_bytes(content)
save_relative = PurePosixPath(source_save_path).relative_to(
PurePosixPath("/downloads")
)
source_content_root = source_root.joinpath(*save_relative.parts)
source_content_root.mkdir(parents=True, exist_ok=True)
(source_content_root / "fixture.bin").write_bytes(content)
info = {
b"length": len(content),
b"name": b"fixture.bin",
@@ -411,6 +468,7 @@ class ClientJobHappyPathTests(unittest.TestCase):
"hash": torrent_hash,
"name": "fixture.bin",
"state": "uploading",
"save_path": source_save_path,
},
[{
"index": 0,
@@ -454,8 +512,11 @@ class ClientJobHappyPathTests(unittest.TestCase):
source_qb = Mock()
source_qb.get_resource.return_value = resource
target_qb = Mock()
target_resource = replace(
resource, save_path=PurePosixPath("/downloads")
)
target_qb.get_resource.side_effect = [
None, None, resource, resource,
None, None, target_resource, target_resource,
]
syncthing = syncthing or CompleteSyncthing()
source = ClientJobExecutor(
+6 -3
View File
@@ -55,6 +55,7 @@ class QBittorrentReaderTests(unittest.TestCase):
b"Ok.",
json.dumps([{
"hash": torrent_hash, "name": "a.txt", "state": "uploading",
"save_path": "/downloads",
}]).encode(),
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
torrent_bytes,
@@ -102,6 +103,7 @@ class QBittorrentReaderTests(unittest.TestCase):
b"Ok.",
json.dumps([{
"hash": qb_hash, "name": "a.txt", "state": "uploading",
"save_path": "/downloads",
}]).encode(),
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
torrent_bytes,
@@ -157,6 +159,7 @@ class QBittorrentReaderTests(unittest.TestCase):
"infohash_v2": v2_hash,
"name": "a.txt",
"state": "uploading",
"save_path": "/downloads",
}]).encode(),
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
torrent_bytes,
@@ -217,9 +220,9 @@ class QBittorrentReaderTests(unittest.TestCase):
malformed_bytes, malformed_hash = torrent("broken.txt", b"bad")
second_bytes, second_hash = torrent("second.txt", b"two")
torrents = [
{"hash": first_hash, "name": "first.txt", "state": "uploading"},
{"hash": malformed_hash, "name": "broken.txt", "state": "stalledUP"},
{"hash": second_hash, "name": "second.txt", "state": "uploading"},
{"hash": first_hash, "name": "first.txt", "state": "uploading", "save_path": "/downloads"},
{"hash": malformed_hash, "name": "broken.txt", "state": "stalledUP", "save_path": "/downloads"},
{"hash": second_hash, "name": "second.txt", "state": "uploading", "save_path": "/downloads"},
]
responses = [
b"Ok.", json.dumps(torrents).encode(),
+22 -1
View File
@@ -40,6 +40,7 @@ class ResourceTests(unittest.TestCase):
"hash": decoded.info_hash_v2_hex,
"name": "v2.bin",
"state": "stoppedUP",
"save_path": "/downloads",
},
[{
"index": 0,
@@ -91,6 +92,7 @@ class ResourceTests(unittest.TestCase):
"hash": decoded.info_hash_v1_hex,
"name": "resource",
"state": "stoppedUP",
"save_path": "/downloads/nested",
},
[
{
@@ -106,6 +108,7 @@ class ResourceTests(unittest.TestCase):
observed,
)
summary = normalized.summary
self.assertEqual(normalized.save_path.as_posix(), "/downloads/nested")
self.assertEqual(
summary.runtime_state, resource_pb2.TORRENT_RUNTIME_STATE_STOPPED
)
@@ -139,6 +142,7 @@ class ResourceTests(unittest.TestCase):
"hash": torrent_hash,
"name": "with-padding",
"state": "stalledUP",
"save_path": "/downloads",
},
[
{
@@ -169,7 +173,7 @@ class ResourceTests(unittest.TestCase):
metainfo = encode({b"info": info})
torrent = {
"hash": hashlib.sha1(encode(info)).hexdigest(),
"name": "renamed", "state": "uploading",
"name": "renamed", "state": "uploading", "save_path": "/downloads",
}
renamed = normalize_resource(torrent, [{
"index": 0, "name": "renamed.txt", "size": 3,
@@ -182,6 +186,23 @@ class ResourceTests(unittest.TestCase):
"progress": 1.0, "priority": 1,
}], metainfo)
def test_missing_or_out_of_shape_save_path_is_rejected(self):
info = {
b"length": 3, b"name": b"a.txt", b"piece length": 16384,
b"pieces": b"x" * 20,
}
metainfo = encode({b"info": info})
torrent = {
"hash": hashlib.sha1(encode(info)).hexdigest(),
"name": "a.txt", "state": "uploading", "save_path": "relative",
}
with self.assertRaisesRegex(ResourceError, "save path is unsafe"):
normalize_resource(torrent, [{
"index": 0, "name": "a.txt", "size": 3,
"completed": 3, "priority": 1,
}], metainfo)
def test_noncanonical_bencode_is_rejected(self):
with self.assertRaisesRegex(BencodeError, "unsorted"):
decode_metainfo(b"d4:infod1:b1:x1:a1:yee")