Compare commits
3
Commits
ae7175719b
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
87cca59d3a | ||
|
|
618bb3d206 | ||
|
|
a1bf3e4315 |
@@ -120,6 +120,8 @@ username = "${QB_USER}"
|
||||
password_file = "/run/secrets/qb_password"
|
||||
api_root = "/downloads"
|
||||
local_root = "/data/qb"
|
||||
# Optional only when a nested qB path uses a different client-visible mount.
|
||||
# local_path_overrides = { "/downloads/fast" = "/data/qb-fast" }
|
||||
|
||||
[syncthing]
|
||||
endpoint = "http://syncthing:8384"
|
||||
@@ -139,6 +141,14 @@ accounting. The override key is the normalized Syncthing folder path beneath
|
||||
`api_root`, not its folder ID. See the production deployment README for the
|
||||
required compose and override pattern.
|
||||
|
||||
qBittorrent content is resolved differently: every existing torrent keeps its
|
||||
qB-reported `save_path`. The client maps that path beneath `qbittorrent.api_root`
|
||||
to its own mount before a source, existing-target, permission, or eviction
|
||||
operation. Thus `/downloads/Downloading` naturally maps below `/data/qb`;
|
||||
there is no migration or per-resource configuration. Add a qB
|
||||
`local_path_overrides` entry only when that nested API prefix is a separate
|
||||
client mount.
|
||||
|
||||
The remaining node examples omit optional `[connection]`, `[jobs]`, and
|
||||
`[backup]` tables and therefore use these same defaults; deployments may
|
||||
override them per node.
|
||||
@@ -277,10 +287,11 @@ The publisher deliberately uses `moby/buildkit:rootless` with
|
||||
`--oci-worker-no-process-sandbox`. On nested Docker hosts, the default OCI
|
||||
sandbox can fail while masking `/proc/acpi` for an emulated build; rootless
|
||||
BuildKit confines that compatibility setting to the disposable builder. It
|
||||
refuses to publish unless `docker buildx inspect` reports both `linux/amd64`
|
||||
and `linux/arm64`, and removes the builder and binfmt handler on success,
|
||||
failure, or interruption. Retain the displayed manifest digest in release
|
||||
notes and deploy the immutable tag or digest.
|
||||
waits briefly for the new worker to observe binfmt, then refuses to publish
|
||||
unless `docker buildx inspect` reports both `linux/amd64` and `linux/arm64`.
|
||||
On capability failure it prints that inspection output and removes the builder
|
||||
and binfmt handler on success, failure, or interruption. Retain the displayed
|
||||
manifest digest in release notes and deploy the immutable tag or digest.
|
||||
|
||||
## Operator usage
|
||||
|
||||
|
||||
@@ -86,6 +86,9 @@ python3 scripts/preflight-deployment.py ... \
|
||||
- That future route path and qBittorrent content root use one client bind
|
||||
mount, so hard-link staging remains possible rather than silently falling
|
||||
back to a space-consuming copy.
|
||||
- Every explicit `qbittorrent.local_path_overrides` entry maps the same host
|
||||
path in the qBittorrent and client containers. This protects nested qB save
|
||||
paths that use a dedicated bind mount.
|
||||
- A real `link(2)` operation between a unique zero-byte file in the qB root
|
||||
and one in the future automatic-route root. The probe verifies that both
|
||||
names refer to the same inode and removes them unconditionally.
|
||||
@@ -96,7 +99,8 @@ python3 scripts/preflight-deployment.py ... \
|
||||
- The token, qB password, and Syncthing API-key files are non-empty regular
|
||||
files with no group/world permissions.
|
||||
- The client image can read its configuration and reports usable permissions,
|
||||
sparse-file support, and filesystem capabilities for both roots.
|
||||
sparse-file support, and filesystem capabilities for both primary roots and
|
||||
every configured qBittorrent local-path override.
|
||||
- qBittorrent authentication/version compatibility and Syncthing
|
||||
authentication/device identity are healthy from the client container.
|
||||
|
||||
@@ -106,6 +110,15 @@ root and add a `local_path_overrides` mapping for that exact API path. This
|
||||
prevents automatic route folders being created on a small configuration
|
||||
filesystem while the client expects to hardlink from the qB data mount.
|
||||
|
||||
qBittorrent resources may use any `save_path` below `qbittorrent.api_root`.
|
||||
At job preflight the client maps that qB API path to its local mount and uses
|
||||
it as the resource root; data is never moved to fit Archive Control. A resource
|
||||
outside the configured qB API root, or whose resolved directory is unavailable
|
||||
or not a real directory in the client container, fails that job before staging
|
||||
or eviction. Use `qbittorrent.local_path_overrides` only for a nested qB API
|
||||
prefix backed by a distinct client mount; this preflight verifies the Docker
|
||||
bind topology for each such override.
|
||||
|
||||
When converting an existing node, stop its client and Syncthing containers,
|
||||
move each existing `routes/<route-id>` directory from the old Syncthing config
|
||||
tree into the new qB-backed route-root directory, then recreate Syncthing and
|
||||
|
||||
+19
-2
@@ -42,8 +42,8 @@ and exported-metainfo data, keeps the qB-local hash separately, and calculates:
|
||||
- normalized selected and selected-complete file-index sets;
|
||||
- torrent runtime state;
|
||||
- canonical path flag and content revision;
|
||||
- a save-path fingerprint based on validated path mapping, never a leaked
|
||||
absolute host path.
|
||||
- a validated qBittorrent `save_path`, retained only in the client's local
|
||||
normalized observation.
|
||||
|
||||
qBittorrent 5 may expose a pure-v2 or hybrid torrent under the first 20 bytes
|
||||
of its v2 hash while separately advertising full `infohash_v1` and
|
||||
@@ -56,6 +56,23 @@ All entries retain qBittorrent's stable torrent file index. Renamed/noncanonical
|
||||
content paths are rejected in v1 because they cannot be transported and merged
|
||||
without ambiguity.
|
||||
|
||||
### Per-torrent local content roots
|
||||
|
||||
`save_path` is not inventory, placement, or protocol data. It is qBittorrent
|
||||
metadata used only by the daemon that queried qBittorrent. Before staging a
|
||||
source, merging into an existing target, applying post-recheck permissions, or
|
||||
evicting files, that daemon maps the torrent's API-visible `save_path` through
|
||||
`qbittorrent.api_root`/`local_root`. The most-specific
|
||||
`qbittorrent.local_path_overrides` mapping wins when a nested path is exposed
|
||||
through a distinct client container mount.
|
||||
|
||||
An ordinary nested qBittorrent path such as `/media/Data/Downloading` needs no
|
||||
per-resource configuration: it resolves beneath the configured root. A path
|
||||
outside that root, an unmapped distinct mount, or a mapped local path that is
|
||||
not a visible real directory fails the affected job before filesystem mutation.
|
||||
This check is intentionally per resource; an unrelated malformed qBittorrent
|
||||
entry cannot prevent normal resources from being staged or transferred.
|
||||
|
||||
### Verification guard
|
||||
|
||||
The client captures transfer counters and state before recheck, issues recheck,
|
||||
|
||||
@@ -14,6 +14,13 @@ client:
|
||||
a regular file or an explicitly created directory;
|
||||
5. verifies every operation remains beneath the local configured root.
|
||||
|
||||
For qBittorrent content, the root is resolved per resource: qB's authoritative
|
||||
API-visible `save_path` is mapped beneath `qbittorrent.api_root` (or a more
|
||||
specific configured local override) into the client namespace. This permits
|
||||
existing nested save paths without moving data, but does not permit paths
|
||||
outside the configured boundary. The resolved path remains local client state
|
||||
and is never sent to the control daemon.
|
||||
|
||||
Sockets, devices, FIFOs, symlinks, and other special entries fail preflight.
|
||||
Permission or ownership mismatch is fail-fast. Archive Control never changes
|
||||
source ownership or mode to make a job pass.
|
||||
@@ -150,4 +157,3 @@ Offline tooling provides list, verify, and restore. Restore requires stopped
|
||||
daemon access, verifies the chosen backup, preserves the suspect database under
|
||||
a timestamped name, installs the replacement atomically, and runs integrity and
|
||||
schema checks before normal startup. Backups contain no configured secrets.
|
||||
|
||||
|
||||
@@ -134,7 +134,7 @@ from pathlib import Path
|
||||
from archive_clients.config import ClientConfig
|
||||
config=ClientConfig.load(Path(sys.argv[1]))
|
||||
value={"shared_token_file":str(config.shared_token_file)}
|
||||
value["qbittorrent"]={"api_root":str(config.qbittorrent.api_root),"local_root":str(config.qbittorrent.local_root),"password_file":str(config.qbittorrent.password_file)}
|
||||
value["qbittorrent"]={"api_root":str(config.qbittorrent.api_root),"local_root":str(config.qbittorrent.local_root),"password_file":str(config.qbittorrent.password_file),"local_path_overrides":{str(api):str(local) for api,local in config.qbittorrent.local_path_overrides}}
|
||||
value["syncthing"]={"api_root":str(config.syncthing.api_root),"local_root":str(config.syncthing.local_root),"api_key_file":str(config.syncthing.api_key_file),"local_path_overrides":{str(api):str(local) for api,local in config.syncthing.local_path_overrides}}
|
||||
print(json.dumps(value,sort_keys=True))'''
|
||||
result = subprocess.run(
|
||||
@@ -174,6 +174,26 @@ raise SystemExit(0 if all(p.state == 1 for p in probes) else 1)'''
|
||||
print(result.stdout.strip())
|
||||
|
||||
|
||||
def require_qb_override_mappings(
|
||||
client: dict[str, Any], qbittorrent: dict[str, Any], qb: dict[str, Any],
|
||||
) -> None:
|
||||
"""Prove every explicit qB API/local override sees the same host bytes."""
|
||||
|
||||
overrides = qb.get("local_path_overrides", {})
|
||||
if not isinstance(overrides, dict):
|
||||
raise CheckFailure("qbittorrent.local_path_overrides must be a table")
|
||||
for api_path, local_path in overrides.items():
|
||||
if not isinstance(api_path, str) or not isinstance(local_path, str):
|
||||
raise CheckFailure(
|
||||
"qbittorrent.local_path_overrides entries are invalid"
|
||||
)
|
||||
require_same_path(
|
||||
f"qBittorrent override {api_path}/local mapping",
|
||||
map_path(qbittorrent, api_path),
|
||||
map_path(client, local_path),
|
||||
)
|
||||
|
||||
|
||||
def run_hardlink_probe(
|
||||
container: str, qb_root: str, route_root: str, user: str | None = None,
|
||||
) -> None:
|
||||
@@ -298,6 +318,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
syncthing_route, client_route,
|
||||
)
|
||||
require_same_path("qBittorrent api_root/local_root", qb_api, client_qb)
|
||||
require_qb_override_mappings(client, qbittorrent, qb)
|
||||
if client_qb.destination != client_route.destination:
|
||||
raise CheckFailure(
|
||||
"qBittorrent and future route roots use separate client bind "
|
||||
|
||||
@@ -69,13 +69,32 @@ docker buildx create --name "${builder}" --driver docker-container \
|
||||
--buildkitd-flags "${buildkitd_flags}" --use >/dev/null
|
||||
builder_created=true
|
||||
|
||||
# A newly-created rootless worker can publish its native platform before it has
|
||||
# observed the just-registered binfmt handler. Do not mistake that brief
|
||||
# startup state for a partial-release-capable builder.
|
||||
platforms=''
|
||||
supports_all=false
|
||||
for attempt in {1..10}; do
|
||||
platforms=$(docker buildx inspect "${builder}" --bootstrap 2>&1)
|
||||
supports_all=true
|
||||
for platform in linux/amd64 linux/arm64; do
|
||||
if ! grep -Fq "${platform}" <<<"${platforms}"; then
|
||||
echo "Builder ${builder} does not support ${platform}; refusing partial release." >&2
|
||||
exit 1
|
||||
supports_all=false
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ ${supports_all} == true ]]; then
|
||||
break
|
||||
fi
|
||||
if [[ ${attempt} -lt 10 ]]; then
|
||||
sleep 1
|
||||
fi
|
||||
done
|
||||
if [[ ${supports_all} != true ]]; then
|
||||
echo "Builder ${builder} does not support both required platforms; refusing partial release." >&2
|
||||
printf '%s\n' "${platforms}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tags=(--tag "${repository}:${tag}")
|
||||
if [[ ${also_latest} == true ]]; then
|
||||
|
||||
@@ -27,10 +27,16 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser.add_argument("--check-config", action="store_true")
|
||||
arguments = parser.parse_args(argv)
|
||||
config = ClientConfig.load(arguments.config, arguments.mode)
|
||||
qb_extra_roots = tuple(sorted({
|
||||
root
|
||||
for _, root in config.qbittorrent.local_path_overrides
|
||||
if root != config.qbittorrent.local_root
|
||||
}))
|
||||
probes = [
|
||||
probe_root(config.qbittorrent.local_root),
|
||||
probe_root(config.syncthing.local_root),
|
||||
]
|
||||
qb_extra_probes = tuple(probe_root(root) for root in qb_extra_roots)
|
||||
probe_writable_directory(config.state_db.parent)
|
||||
probe_writable_directory(config.backup_dir)
|
||||
shared_token = config.read_shared_token()
|
||||
@@ -40,7 +46,10 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
print(json.dumps({
|
||||
"client_id": config.client_id,
|
||||
"role": config.role,
|
||||
"filesystems": [probe.__dict__ | {"root": str(probe.root)} for probe in probes],
|
||||
"filesystems": [
|
||||
probe.__dict__ | {"root": str(probe.root)}
|
||||
for probe in (*probes[:1], *qb_extra_probes, *probes[1:])
|
||||
],
|
||||
}, sort_keys=True))
|
||||
return 0
|
||||
configure_logging(
|
||||
@@ -68,7 +77,7 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
},
|
||||
)
|
||||
asyncio.run(ArchiveClientDaemon(
|
||||
config, probes, service_probes,
|
||||
config, probes, service_probes, qb_extra_probes=qb_extra_probes,
|
||||
resource_reader=QBittorrentReader(config.qbittorrent),
|
||||
).run())
|
||||
except KeyboardInterrupt:
|
||||
|
||||
@@ -213,8 +213,6 @@ def _local_path_overrides(
|
||||
value: dict[str, Any], api_root: PurePosixPath, name: str
|
||||
) -> tuple[tuple[PurePosixPath, Path], ...]:
|
||||
raw = value.get("local_path_overrides", {})
|
||||
if name != "syncthing" and raw:
|
||||
raise ConfigError(f"{name}.local_path_overrides is unsupported")
|
||||
if not isinstance(raw, dict):
|
||||
raise ConfigError(f"{name}.local_path_overrides must be a table")
|
||||
parsed: list[tuple[PurePosixPath, Path]] = []
|
||||
|
||||
@@ -64,6 +64,7 @@ class ArchiveClientDaemon:
|
||||
service_probes: list[ServiceProbe],
|
||||
resource_reader: QBittorrentReader | None = None,
|
||||
route_manager: SyncthingRouteManager | None = None,
|
||||
qb_extra_probes: tuple[FilesystemProbe, ...] = (),
|
||||
):
|
||||
if len(probes) != 2:
|
||||
raise ValueError(
|
||||
@@ -71,6 +72,7 @@ class ArchiveClientDaemon:
|
||||
)
|
||||
self.config = config
|
||||
self.probes = probes
|
||||
self.qb_probes = (probes[0], *qb_extra_probes)
|
||||
self.service_probes = service_probes
|
||||
self.inventory = (
|
||||
InventoryService(resource_reader, config.client_id)
|
||||
@@ -119,9 +121,13 @@ class ArchiveClientDaemon:
|
||||
store=self.store,
|
||||
qb_root=config.qbittorrent.local_root,
|
||||
qb_api_root=config.qbittorrent.api_root,
|
||||
qb_roots=config.qbittorrent.roots,
|
||||
route_path=self._route_path,
|
||||
syncthing_transport=self.routes.transport,
|
||||
sparse_supported=all(probe.sparse_files for probe in probes),
|
||||
sparse_supported=all(
|
||||
probe.sparse_files
|
||||
for probe in (*self.qb_probes, probes[1])
|
||||
),
|
||||
poll_interval=config.jobs.poll_interval,
|
||||
verification_timeout=config.jobs.verification_timeout,
|
||||
free_space_reserve_bytes=(
|
||||
|
||||
@@ -7,7 +7,7 @@ import hashlib
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Iterable
|
||||
from typing import Callable, Iterable
|
||||
|
||||
from archive_clients.qbittorrent import QBittorrentReader
|
||||
from archive_clients.resources import NormalizedResource
|
||||
@@ -23,7 +23,7 @@ def verify_and_snapshot(
|
||||
job_id: str,
|
||||
resource: NormalizedResource,
|
||||
selected_indices: Iterable[int],
|
||||
qb_root: Path,
|
||||
content_root: Path,
|
||||
store: ClientStore,
|
||||
) -> dict[str, object]:
|
||||
selected = set(selected_indices)
|
||||
@@ -38,7 +38,7 @@ def verify_and_snapshot(
|
||||
f"cache file {index} is not selected and complete"
|
||||
)
|
||||
relative = _relative(item.canonical_path)
|
||||
path = qb_root.joinpath(*relative.parts)
|
||||
path = content_root.joinpath(*relative.parts)
|
||||
try:
|
||||
metadata = path.lstat()
|
||||
except FileNotFoundError as exc:
|
||||
@@ -65,7 +65,7 @@ def verify_and_snapshot(
|
||||
"sha256": _sha256(path),
|
||||
}
|
||||
)
|
||||
snapshot = {"files": files}
|
||||
snapshot = {"content_root": str(content_root), "files": files}
|
||||
return store.put_job_artifact(job_id, "eviction-snapshot", snapshot)["value"]
|
||||
|
||||
|
||||
@@ -90,9 +90,9 @@ def remove_qb_entry(
|
||||
def safe_unlink(
|
||||
*,
|
||||
job_id: str,
|
||||
qb_root: Path,
|
||||
qbittorrent: QBittorrentReader,
|
||||
store: ClientStore,
|
||||
resource_root: Callable[[NormalizedResource], Path],
|
||||
) -> dict[str, object]:
|
||||
completed = store.get_job_artifact(job_id, "eviction-unlinked")
|
||||
if completed is not None:
|
||||
@@ -101,11 +101,19 @@ def safe_unlink(
|
||||
if snapshot_row is None:
|
||||
raise EvictionError("eviction snapshot is missing")
|
||||
snapshot = snapshot_row["value"]
|
||||
if not isinstance(snapshot, dict):
|
||||
raise EvictionError("eviction snapshot is invalid")
|
||||
content_root_value = snapshot.get("content_root")
|
||||
if not isinstance(content_root_value, str):
|
||||
raise EvictionError("eviction snapshot content root is missing")
|
||||
content_root = Path(content_root_value)
|
||||
if not content_root.is_absolute() or content_root.is_symlink():
|
||||
raise EvictionError("eviction snapshot content root is unsafe")
|
||||
files = snapshot.get("files")
|
||||
if not isinstance(files, list):
|
||||
raise EvictionError("eviction snapshot is invalid")
|
||||
|
||||
referenced = _remaining_paths(qbittorrent.list_resources())
|
||||
referenced = _remaining_paths(qbittorrent.list_resources(), resource_root)
|
||||
removed: list[str] = []
|
||||
retained: list[dict[str, str]] = []
|
||||
directories: set[Path] = set()
|
||||
@@ -113,10 +121,10 @@ def safe_unlink(
|
||||
if not isinstance(record, dict) or not isinstance(record.get("path"), str):
|
||||
raise EvictionError("eviction file record is invalid")
|
||||
relative = _relative(record["path"])
|
||||
if relative.as_posix() in referenced:
|
||||
path = content_root.joinpath(*relative.parts)
|
||||
if path in referenced:
|
||||
retained.append({"path": relative.as_posix(), "reason": "shared"})
|
||||
continue
|
||||
path = qb_root.joinpath(*relative.parts)
|
||||
try:
|
||||
metadata = path.lstat()
|
||||
except FileNotFoundError:
|
||||
@@ -142,7 +150,7 @@ def safe_unlink(
|
||||
path.unlink()
|
||||
removed.append(relative.as_posix())
|
||||
parent = path.parent
|
||||
while parent != qb_root:
|
||||
while parent != content_root:
|
||||
directories.add(parent)
|
||||
parent = parent.parent
|
||||
|
||||
@@ -153,7 +161,7 @@ def safe_unlink(
|
||||
try:
|
||||
directory.rmdir()
|
||||
removed_directories.append(
|
||||
directory.relative_to(qb_root).as_posix()
|
||||
directory.relative_to(content_root).as_posix()
|
||||
)
|
||||
except OSError as exc:
|
||||
if exc.errno not in {errno.ENOTEMPTY, errno.ENOENT}:
|
||||
@@ -214,12 +222,24 @@ def compensate_materialized_files(
|
||||
return removed
|
||||
|
||||
|
||||
def _remaining_paths(resources: Iterable[NormalizedResource]) -> set[str]:
|
||||
return {
|
||||
item.canonical_path
|
||||
for resource in resources
|
||||
def _remaining_paths(
|
||||
resources: Iterable[NormalizedResource],
|
||||
resource_root: Callable[[NormalizedResource], Path],
|
||||
) -> set[Path]:
|
||||
result: set[Path] = set()
|
||||
for resource in resources:
|
||||
try:
|
||||
root = resource_root(resource)
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
# A malformed unrelated qB entry must not stop an otherwise
|
||||
# safe eviction. Its unresolvable path cannot be considered a
|
||||
# shared path under the verified eviction root.
|
||||
continue
|
||||
result.update(
|
||||
root.joinpath(*_relative(item.canonical_path).parts)
|
||||
for item in resource.files
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _relative(value: str) -> PurePosixPath:
|
||||
|
||||
+78
-16
@@ -14,6 +14,7 @@ import uuid
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Callable, Iterable
|
||||
|
||||
from archive_clients.config import ConfigError, RootMapping
|
||||
from archive_clients.protocol import decode_message, encode_message
|
||||
from archive_clients.eviction import (
|
||||
EvictionError,
|
||||
@@ -67,6 +68,7 @@ class ClientJobExecutor:
|
||||
store: ClientStore,
|
||||
qb_root: Path,
|
||||
qb_api_root: PurePosixPath,
|
||||
qb_roots: RootMapping | None = None,
|
||||
route_path: Callable[[str], Path],
|
||||
syncthing_transport: object,
|
||||
sparse_supported: bool,
|
||||
@@ -78,7 +80,10 @@ class ClientJobExecutor:
|
||||
self.qbittorrent = qbittorrent
|
||||
self.store = store
|
||||
self.qb_root = qb_root
|
||||
self.qb_api_root = qb_api_root
|
||||
self.qb_api_root = PurePosixPath(qb_api_root)
|
||||
self.qb_roots = qb_roots or RootMapping(
|
||||
self.qb_api_root, self.qb_root
|
||||
)
|
||||
self.route_path = route_path
|
||||
self.syncthing_transport = syncthing_transport
|
||||
self.sparse_supported = sparse_supported
|
||||
@@ -479,6 +484,7 @@ class ClientJobExecutor:
|
||||
"archive coverage no longer covers the cache selection"
|
||||
)
|
||||
resource = self._resource(definition)
|
||||
resource_root = self._resource_root(resource)
|
||||
current = _resource_fingerprint(resource, self.client_id)
|
||||
if not _fingerprint_matches(
|
||||
current, definition.eviction.cache_fingerprint
|
||||
@@ -490,7 +496,7 @@ class ClientJobExecutor:
|
||||
job_id=definition.job_id,
|
||||
resource=resource,
|
||||
selected_indices=requested,
|
||||
qb_root=self.qb_root,
|
||||
content_root=resource_root,
|
||||
store=self.store,
|
||||
)
|
||||
return None
|
||||
@@ -505,9 +511,9 @@ class ClientJobExecutor:
|
||||
if step == job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK:
|
||||
safe_unlink(
|
||||
job_id=definition.job_id,
|
||||
qb_root=self.qb_root,
|
||||
qbittorrent=self.qbittorrent,
|
||||
store=self.store,
|
||||
resource_root=self._resource_root,
|
||||
)
|
||||
placement = resource_pb2.Placement(
|
||||
client_id=self.client_id,
|
||||
@@ -538,7 +544,8 @@ class ClientJobExecutor:
|
||||
raise JobExecutionError(
|
||||
"source resource changed after job confirmation"
|
||||
)
|
||||
self._reject_unsafe_partfile(definition, resource)
|
||||
source_root = self._resource_root(resource)
|
||||
self._reject_unsafe_partfile(definition, resource, source_root)
|
||||
indices = _selection_indices(definition.transfer.transfer_delta_files)
|
||||
by_index = {item.file_index: item for item in resource.files}
|
||||
if not indices or any(index not in by_index for index in indices):
|
||||
@@ -602,7 +609,7 @@ class ClientJobExecutor:
|
||||
self._require_space(
|
||||
route_root,
|
||||
self._copy_required_bytes(
|
||||
self.qb_root,
|
||||
source_root,
|
||||
route_root,
|
||||
(
|
||||
(entry.target_canonical_path, entry.logical_bytes)
|
||||
@@ -613,7 +620,7 @@ class ClientJobExecutor:
|
||||
)
|
||||
stage_transfer(
|
||||
manifest,
|
||||
source_root=self.qb_root,
|
||||
source_root=source_root,
|
||||
sync_root=route_root,
|
||||
store=self.store,
|
||||
artifact_sources={"metainfo/source.torrent": metainfo_path},
|
||||
@@ -691,23 +698,27 @@ class ClientJobExecutor:
|
||||
"target materialization was sent to the wrong client"
|
||||
)
|
||||
published = load_published_transfer(self._job_directory(definition))
|
||||
info_hash = _info_hash(definition)
|
||||
resource = self.qbittorrent.get_resource(info_hash)
|
||||
target_root = (
|
||||
self._resource_root(resource)
|
||||
if resource is not None else self.qb_root
|
||||
)
|
||||
# The target can likewise hardlink an arrived Syncthing payload into
|
||||
# qB's content root when those directories share a filesystem.
|
||||
self._require_space(
|
||||
self.qb_root,
|
||||
target_root,
|
||||
self._copy_required_bytes(
|
||||
published.job_directory,
|
||||
self.qb_root,
|
||||
target_root,
|
||||
(
|
||||
(entry.payload_relative_path, entry.logical_bytes)
|
||||
for entry in published.manifest.files
|
||||
),
|
||||
),
|
||||
)
|
||||
info_hash = _info_hash(definition)
|
||||
resource = self.qbittorrent.get_resource(info_hash)
|
||||
if resource is not None:
|
||||
self._reject_unsafe_partfile(definition, resource)
|
||||
self._reject_unsafe_partfile(definition, resource, target_root)
|
||||
if definition.transfer.HasField("target_baseline_fingerprint"):
|
||||
if resource is None or not _fingerprint_matches(
|
||||
_resource_fingerprint(resource, self.client_id),
|
||||
@@ -732,13 +743,14 @@ class ClientJobExecutor:
|
||||
== resource_pb2.TORRENT_RUNTIME_STATE_STOPPED
|
||||
),
|
||||
"total_file_count": len(published.manifest.files),
|
||||
"content_root": str(target_root),
|
||||
}
|
||||
self.store.put_job_artifact(
|
||||
definition.job_id, "target-baseline", baseline
|
||||
)
|
||||
materialize_transfer(
|
||||
published,
|
||||
target_root=self.qb_root,
|
||||
target_root=target_root,
|
||||
store=self.store,
|
||||
sparse_supported=self.sparse_supported,
|
||||
cancel_check=lambda: self._raise_if_cancelled(
|
||||
@@ -813,7 +825,9 @@ class ClientJobExecutor:
|
||||
raise JobExecutionError(
|
||||
"verified qBittorrent resource disappeared"
|
||||
)
|
||||
_normalize_verified_resource_permissions(self.qb_root, verified)
|
||||
_normalize_verified_resource_permissions(
|
||||
self._resource_root(verified), verified
|
||||
)
|
||||
if should_start:
|
||||
self.qbittorrent.start(qb_torrent_id)
|
||||
placement = resource_pb2.Placement(
|
||||
@@ -852,6 +866,7 @@ class ClientJobExecutor:
|
||||
)
|
||||
return
|
||||
baseline = baseline_row["value"]
|
||||
content_root = self._artifact_content_root(baseline)
|
||||
info_hash = _info_hash(definition)
|
||||
present = self.qbittorrent.get_resource(info_hash)
|
||||
if present is not None:
|
||||
@@ -870,7 +885,7 @@ class ClientJobExecutor:
|
||||
self.qbittorrent.delete_entry(qb_torrent_id)
|
||||
compensate_materialized_files(
|
||||
job_id=definition.job_id,
|
||||
qb_root=self.qb_root,
|
||||
qb_root=content_root,
|
||||
store=self.store,
|
||||
)
|
||||
|
||||
@@ -906,10 +921,11 @@ class ClientJobExecutor:
|
||||
self,
|
||||
definition: job_pb2.JobDefinition,
|
||||
resource: NormalizedResource,
|
||||
content_root: Path,
|
||||
) -> None:
|
||||
roots = {self.qb_root}
|
||||
roots = {content_root}
|
||||
for item in resource.files:
|
||||
candidate = self.qb_root / PurePosixPath(item.canonical_path).parts[0]
|
||||
candidate = content_root / PurePosixPath(item.canonical_path).parts[0]
|
||||
roots.add(candidate if candidate.is_dir() else candidate.parent)
|
||||
hashes = {
|
||||
value.lower() for value in (
|
||||
@@ -928,6 +944,52 @@ class ClientJobExecutor:
|
||||
"safely transferred by this client"
|
||||
)
|
||||
|
||||
def _resource_root(self, resource: NormalizedResource) -> Path:
|
||||
"""Resolve a qB resource's save path within this client's mounts.
|
||||
|
||||
qB returns paths in its own API/container namespace. The configured
|
||||
root mapping translates those paths into the client namespace and
|
||||
supports nested save paths and explicit prefix overrides. A missing
|
||||
save path is retained only for older in-process test fixtures; every
|
||||
normalized qB API resource has one.
|
||||
"""
|
||||
|
||||
if resource.save_path is None:
|
||||
return self.qb_root
|
||||
try:
|
||||
root = self.qb_roots.api_to_local(resource.save_path.as_posix())
|
||||
except ConfigError as exc:
|
||||
raise JobExecutionError(
|
||||
"qBittorrent resource save path is outside the configured "
|
||||
"API root or has no local path mapping"
|
||||
) from exc
|
||||
try:
|
||||
metadata = root.lstat()
|
||||
except FileNotFoundError as exc:
|
||||
raise JobExecutionError(
|
||||
"qBittorrent resource save path is not visible in the "
|
||||
"client container"
|
||||
) from exc
|
||||
if not stat.S_ISDIR(metadata.st_mode):
|
||||
raise JobExecutionError(
|
||||
"qBittorrent resource save path is not a real directory in "
|
||||
"the client container"
|
||||
)
|
||||
return root
|
||||
|
||||
def _artifact_content_root(self, baseline: object) -> Path:
|
||||
if not isinstance(baseline, dict):
|
||||
raise JobExecutionError("target baseline is invalid")
|
||||
value = baseline.get("content_root")
|
||||
if not isinstance(value, str):
|
||||
# A pre-existing durable baseline predates per-resource roots.
|
||||
# It can only have materialized to the configured target root.
|
||||
return self.qb_root
|
||||
root = Path(value)
|
||||
if not root.is_absolute() or root.is_symlink():
|
||||
raise JobExecutionError("target baseline content root is unsafe")
|
||||
return root
|
||||
|
||||
def _raise_if_cancelled(self, job_id: str) -> None:
|
||||
event = self._cancel_events.get(job_id)
|
||||
if event is not None and event.is_set():
|
||||
|
||||
@@ -19,10 +19,21 @@ class ResourceError(ValueError):
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NormalizedResource:
|
||||
"""A normalized qB observation for protocol data and local file work.
|
||||
|
||||
``save_path`` is qBittorrent's API-visible per-torrent content root. It
|
||||
is deliberately local-only: clients resolve it through their qB path
|
||||
mapping immediately before filesystem work, and it is never serialized in
|
||||
inventory, placements, or control protocol messages.
|
||||
"""
|
||||
|
||||
summary: resource_pb2.ResourceSummary
|
||||
files: tuple[resource_pb2.TorrentFile, ...]
|
||||
metainfo: Metainfo
|
||||
metainfo_bytes: bytes = b""
|
||||
# qBittorrent's API-visible save path is intentionally local-only. It
|
||||
# must never become part of inventory or placement protocol messages.
|
||||
save_path: PurePosixPath | None = None
|
||||
|
||||
|
||||
def build_content_tree(
|
||||
@@ -137,6 +148,7 @@ def normalize_resource(
|
||||
character not in "0123456789abcdef" for character in qb_torrent_id
|
||||
):
|
||||
raise ResourceError("torrent hash is invalid")
|
||||
save_path = _save_path(torrent.get("save_path"))
|
||||
summary = resource_pb2.ResourceSummary(
|
||||
qb_torrent_id=qb_torrent_id,
|
||||
display_name=_string(torrent.get("name"), "torrent name"),
|
||||
@@ -172,7 +184,9 @@ def normalize_resource(
|
||||
revision_data, sort_keys=True, separators=(",", ":"),
|
||||
).encode("utf-8")).hexdigest()
|
||||
summary.observed_at.FromDatetime(observed_at)
|
||||
return NormalizedResource(summary, tuple(files), metainfo, metainfo_bytes)
|
||||
return NormalizedResource(
|
||||
summary, tuple(files), metainfo, metainfo_bytes, save_path
|
||||
)
|
||||
|
||||
|
||||
def _set_selection(target: Any, indices: list[int]) -> None:
|
||||
@@ -217,6 +231,20 @@ def _path(value: Any) -> str:
|
||||
return candidate.as_posix()
|
||||
|
||||
|
||||
def _save_path(value: Any) -> PurePosixPath:
|
||||
"""Validate qBittorrent's API-visible per-torrent content root."""
|
||||
|
||||
path = _string(value, "torrent save path")
|
||||
candidate = PurePosixPath(path)
|
||||
if (
|
||||
not candidate.is_absolute()
|
||||
or ".." in candidate.parts
|
||||
or "." in candidate.parts
|
||||
):
|
||||
raise ResourceError("qBittorrent torrent save path is unsafe")
|
||||
return candidate
|
||||
|
||||
|
||||
def _integer(value: Any, name: str) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
||||
raise ResourceError(f"{name} is invalid")
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import redirect_stdout
|
||||
from pathlib import Path
|
||||
|
||||
from archive_clients.cli import main
|
||||
|
||||
|
||||
class ClientCliTests(unittest.TestCase):
|
||||
def test_check_config_probes_every_qb_override_root(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
for name in ("token", "qb-password", "syncthing-key"):
|
||||
path = root / name
|
||||
path.write_text(name, encoding="utf-8")
|
||||
os.chmod(path, 0o600)
|
||||
for name in ("qb", "qb-fast", "sync", "backups"):
|
||||
(root / name).mkdir()
|
||||
config = root / "client.toml"
|
||||
config.write_text(
|
||||
f'''client_id = "cache-1"
|
||||
display_name = "Cache 1"
|
||||
role = "cache"
|
||||
control_endpoint = "ws://control/archive_control"
|
||||
shared_token_file = "{root / "token"}"
|
||||
state_db = "{root / "state.db"}"
|
||||
backup_dir = "{root / "backups"}"
|
||||
|
||||
[qbittorrent]
|
||||
endpoint = "http://qb"
|
||||
username = "admin"
|
||||
password_file = "{root / "qb-password"}"
|
||||
api_root = "/downloads"
|
||||
local_root = "{root / "qb"}"
|
||||
local_path_overrides = {{ "/downloads/fast" = "{root / "qb-fast"}" }}
|
||||
|
||||
[syncthing]
|
||||
endpoint = "http://syncthing"
|
||||
api_key_file = "{root / "syncthing-key"}"
|
||||
api_root = "/sync"
|
||||
local_root = "{root / "sync"}"
|
||||
''',
|
||||
encoding="utf-8",
|
||||
)
|
||||
output = io.StringIO()
|
||||
with redirect_stdout(output):
|
||||
self.assertEqual(main(["--config", str(config), "--check-config"]), 0)
|
||||
reported = json.loads(output.getvalue())
|
||||
self.assertEqual(
|
||||
[item["root"] for item in reported["filesystems"]],
|
||||
[str(root / "qb"), str(root / "qb-fast"), str(root / "sync")],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -78,6 +78,35 @@ class ConfigTests(unittest.TestCase):
|
||||
Path("/local/qb/Sync"),
|
||||
)
|
||||
|
||||
def test_qbittorrent_can_override_a_nested_save_path(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
for name in ("token", "qb-password", "syncthing-key"):
|
||||
path = root / name
|
||||
path.write_text(name, encoding="utf-8")
|
||||
os.chmod(path, 0o600)
|
||||
(root / "qb").mkdir()
|
||||
(root / "fast").mkdir()
|
||||
(root / "sync").mkdir()
|
||||
config_path = root / "client.toml"
|
||||
config_path.write_text(
|
||||
_config(root).replace(
|
||||
f'local_root = "{root / "qb"}"',
|
||||
f'local_root = "{root / "qb"}"\n'
|
||||
"local_path_overrides = { \"/downloads/fast\" = "
|
||||
f'"{root / "fast"}" }}',
|
||||
1,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
config = ClientConfig.load(config_path)
|
||||
self.assertEqual(
|
||||
config.qbittorrent.roots.api_to_local(
|
||||
"/downloads/fast/resource/file.bin"
|
||||
),
|
||||
root / "fast/resource/file.bin",
|
||||
)
|
||||
|
||||
def test_endpoint_scheme_and_job_keys_are_strict(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
|
||||
@@ -50,6 +50,26 @@ class DeploymentPreflightTests(unittest.TestCase):
|
||||
"/data/qb/.archive-control-routes",
|
||||
)
|
||||
|
||||
def test_qb_override_must_map_to_the_same_host_path(self):
|
||||
client = {"Mounts": [
|
||||
{"Type": "bind", "Source": "/srv/fast", "Destination": "/data/fast"},
|
||||
]}
|
||||
qbittorrent = {"Mounts": [
|
||||
{"Type": "bind", "Source": "/srv/fast", "Destination": "/downloads/fast"},
|
||||
]}
|
||||
preflight.require_qb_override_mappings(
|
||||
client, qbittorrent,
|
||||
{"local_path_overrides": {"/downloads/fast": "/data/fast"}},
|
||||
)
|
||||
with self.assertRaisesRegex(preflight.CheckFailure, "host paths differ"):
|
||||
preflight.require_qb_override_mappings(
|
||||
client,
|
||||
{"Mounts": [{
|
||||
"Type": "bind", "Source": "/other", "Destination": "/downloads/fast",
|
||||
}]},
|
||||
{"local_path_overrides": {"/downloads/fast": "/data/fast"}},
|
||||
)
|
||||
|
||||
def test_runs_hardlink_probe_with_qb_and_route_roots(self):
|
||||
completed = __import__("subprocess").CompletedProcess(
|
||||
args=[], returncode=0, stdout="hard-link staging probe passed\n", stderr=""
|
||||
|
||||
+47
-7
@@ -1,5 +1,7 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from dataclasses import replace
|
||||
from pathlib import PurePosixPath
|
||||
from pathlib import Path
|
||||
|
||||
from archive_clients.eviction import (
|
||||
@@ -86,7 +88,7 @@ class EvictionTests(unittest.TestCase):
|
||||
job_id="job-1",
|
||||
resource=evicted,
|
||||
selected_indices=[0, 1],
|
||||
qb_root=self.root,
|
||||
content_root=self.root,
|
||||
store=self.store,
|
||||
)
|
||||
remove_qb_entry(
|
||||
@@ -97,9 +99,9 @@ class EvictionTests(unittest.TestCase):
|
||||
)
|
||||
result = safe_unlink(
|
||||
job_id="job-1",
|
||||
qb_root=self.root,
|
||||
qbittorrent=qb,
|
||||
store=self.store,
|
||||
resource_root=lambda _: self.root,
|
||||
)
|
||||
self.assertTrue((self.root / "tree/shared.bin").exists())
|
||||
self.assertFalse((self.root / "tree/owned.bin").exists())
|
||||
@@ -121,7 +123,7 @@ class EvictionTests(unittest.TestCase):
|
||||
job_id="job-1",
|
||||
resource=evicted,
|
||||
selected_indices=[0],
|
||||
qb_root=self.root,
|
||||
content_root=self.root,
|
||||
store=self.store,
|
||||
)
|
||||
path.unlink()
|
||||
@@ -134,9 +136,9 @@ class EvictionTests(unittest.TestCase):
|
||||
)
|
||||
result = safe_unlink(
|
||||
job_id="job-1",
|
||||
qb_root=self.root,
|
||||
qbittorrent=qb,
|
||||
store=self.store,
|
||||
resource_root=lambda _: self.root,
|
||||
)
|
||||
self.assertTrue(path.exists())
|
||||
self.assertEqual(
|
||||
@@ -152,7 +154,7 @@ class EvictionTests(unittest.TestCase):
|
||||
job_id="job-1",
|
||||
resource=evicted,
|
||||
selected_indices=[0],
|
||||
qb_root=self.root,
|
||||
content_root=self.root,
|
||||
store=self.store,
|
||||
)
|
||||
remove_qb_entry(
|
||||
@@ -163,19 +165,57 @@ class EvictionTests(unittest.TestCase):
|
||||
)
|
||||
first = safe_unlink(
|
||||
job_id="job-1",
|
||||
qb_root=self.root,
|
||||
qbittorrent=qb,
|
||||
store=self.store,
|
||||
resource_root=lambda _: self.root,
|
||||
)
|
||||
second = safe_unlink(
|
||||
job_id="job-1",
|
||||
qb_root=self.root,
|
||||
qbittorrent=qb,
|
||||
store=self.store,
|
||||
resource_root=lambda _: self.root,
|
||||
)
|
||||
self.assertEqual(first, second)
|
||||
self.assertEqual(qb.deleted, ["a" * 40])
|
||||
|
||||
def test_nested_save_path_uses_its_own_root_and_not_a_same_named_peer(self):
|
||||
nested = self.root / "Downloading"
|
||||
nested.mkdir()
|
||||
evicted = replace(
|
||||
normalized("a" * 40, ["resource/file.bin"]),
|
||||
save_path=PurePosixPath("/downloads/Downloading"),
|
||||
)
|
||||
peer = replace(
|
||||
normalized("b" * 40, ["resource/file.bin"]),
|
||||
save_path=PurePosixPath("/downloads"),
|
||||
)
|
||||
nested_file = nested / "resource/file.bin"
|
||||
nested_file.parent.mkdir()
|
||||
nested_file.write_bytes(b"x" * evicted.files[0].logical_bytes)
|
||||
root_file = self.root / "resource/file.bin"
|
||||
root_file.parent.mkdir()
|
||||
root_file.write_bytes(b"x" * peer.files[0].logical_bytes)
|
||||
qb = FakeQB(evicted, [peer])
|
||||
|
||||
verify_and_snapshot(
|
||||
job_id="job-1", resource=evicted, selected_indices=[0],
|
||||
content_root=nested, store=self.store,
|
||||
)
|
||||
remove_qb_entry(
|
||||
job_id="job-1", torrent_hash="a" * 40,
|
||||
qbittorrent=qb, store=self.store,
|
||||
)
|
||||
safe_unlink(
|
||||
job_id="job-1", qbittorrent=qb, store=self.store,
|
||||
resource_root=lambda item: (
|
||||
nested
|
||||
if item.save_path == PurePosixPath("/downloads/Downloading")
|
||||
else self.root
|
||||
),
|
||||
)
|
||||
self.assertFalse(nested_file.exists())
|
||||
self.assertTrue(root_file.exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
+64
-3
@@ -4,11 +4,13 @@ import stat
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from dataclasses import replace
|
||||
from pathlib import Path, PurePosixPath
|
||||
from unittest.mock import Mock, patch
|
||||
from uuid import uuid4
|
||||
|
||||
from archive_clients.bencode import encode
|
||||
from archive_clients.config import RootMapping
|
||||
from archive_clients.jobs import (
|
||||
ClientJobExecutor,
|
||||
JobExecutionError,
|
||||
@@ -246,6 +248,55 @@ class ClientJobHappyPathTests(unittest.TestCase):
|
||||
content=b"x" * 4096,
|
||||
)
|
||||
|
||||
def test_nested_qb_save_path_stages_from_mapped_subdirectory(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
self._run_transfer(
|
||||
Path(directory),
|
||||
job_pb2.JOB_OPERATION_UNARCHIVE,
|
||||
source_save_path="/downloads/Downloading",
|
||||
)
|
||||
|
||||
def test_qb_save_path_preflight_rejects_unmapped_path(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
executor = ClientJobExecutor(
|
||||
client_id="cache-1", qbittorrent=Mock(),
|
||||
store=ClientStore(root / "state.db"), qb_root=root,
|
||||
qb_api_root=PurePosixPath("/downloads"),
|
||||
route_path=lambda _: root, syncthing_transport=Mock(),
|
||||
sparse_supported=True,
|
||||
)
|
||||
resource = NormalizedResource(
|
||||
resource_pb2.ResourceSummary(), (), Mock(),
|
||||
save_path=PurePosixPath("/outside"),
|
||||
)
|
||||
with self.assertRaisesRegex(JobExecutionError, "outside"):
|
||||
executor._resource_root(resource)
|
||||
|
||||
def test_qb_save_path_override_uses_its_dedicated_local_mount(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
primary = root / "primary"
|
||||
override = root / "override"
|
||||
primary.mkdir()
|
||||
override.mkdir()
|
||||
executor = ClientJobExecutor(
|
||||
client_id="cache-1", qbittorrent=Mock(),
|
||||
store=ClientStore(root / "state.db"), qb_root=primary,
|
||||
qb_api_root=PurePosixPath("/downloads"),
|
||||
qb_roots=RootMapping(
|
||||
PurePosixPath("/downloads"), primary,
|
||||
((PurePosixPath("/downloads/slow"), override),),
|
||||
),
|
||||
route_path=lambda _: root, syncthing_transport=Mock(),
|
||||
sparse_supported=True,
|
||||
)
|
||||
resource = NormalizedResource(
|
||||
resource_pb2.ResourceSummary(), (), Mock(),
|
||||
save_path=PurePosixPath("/downloads/slow"),
|
||||
)
|
||||
self.assertEqual(executor._resource_root(resource), override)
|
||||
|
||||
def test_mount_boundary_requires_copy_space_even_with_same_device(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
@@ -390,6 +441,7 @@ class ClientJobHappyPathTests(unittest.TestCase):
|
||||
source_stage_free_bytes: int | None = None,
|
||||
content: bytes = b"archive-control-happy-path",
|
||||
syncthing: CompleteSyncthing | None = None,
|
||||
source_save_path: str = "/downloads",
|
||||
):
|
||||
source_root = root / "source"
|
||||
target_root = root / "target"
|
||||
@@ -397,7 +449,12 @@ class ClientJobHappyPathTests(unittest.TestCase):
|
||||
source_root.mkdir()
|
||||
target_root.mkdir()
|
||||
route_root.mkdir()
|
||||
(source_root / "fixture.bin").write_bytes(content)
|
||||
save_relative = PurePosixPath(source_save_path).relative_to(
|
||||
PurePosixPath("/downloads")
|
||||
)
|
||||
source_content_root = source_root.joinpath(*save_relative.parts)
|
||||
source_content_root.mkdir(parents=True, exist_ok=True)
|
||||
(source_content_root / "fixture.bin").write_bytes(content)
|
||||
info = {
|
||||
b"length": len(content),
|
||||
b"name": b"fixture.bin",
|
||||
@@ -411,6 +468,7 @@ class ClientJobHappyPathTests(unittest.TestCase):
|
||||
"hash": torrent_hash,
|
||||
"name": "fixture.bin",
|
||||
"state": "uploading",
|
||||
"save_path": source_save_path,
|
||||
},
|
||||
[{
|
||||
"index": 0,
|
||||
@@ -454,8 +512,11 @@ class ClientJobHappyPathTests(unittest.TestCase):
|
||||
source_qb = Mock()
|
||||
source_qb.get_resource.return_value = resource
|
||||
target_qb = Mock()
|
||||
target_resource = replace(
|
||||
resource, save_path=PurePosixPath("/downloads")
|
||||
)
|
||||
target_qb.get_resource.side_effect = [
|
||||
None, None, resource, resource,
|
||||
None, None, target_resource, target_resource,
|
||||
]
|
||||
syncthing = syncthing or CompleteSyncthing()
|
||||
source = ClientJobExecutor(
|
||||
|
||||
@@ -55,6 +55,7 @@ class QBittorrentReaderTests(unittest.TestCase):
|
||||
b"Ok.",
|
||||
json.dumps([{
|
||||
"hash": torrent_hash, "name": "a.txt", "state": "uploading",
|
||||
"save_path": "/downloads",
|
||||
}]).encode(),
|
||||
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
|
||||
torrent_bytes,
|
||||
@@ -102,6 +103,7 @@ class QBittorrentReaderTests(unittest.TestCase):
|
||||
b"Ok.",
|
||||
json.dumps([{
|
||||
"hash": qb_hash, "name": "a.txt", "state": "uploading",
|
||||
"save_path": "/downloads",
|
||||
}]).encode(),
|
||||
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
|
||||
torrent_bytes,
|
||||
@@ -157,6 +159,7 @@ class QBittorrentReaderTests(unittest.TestCase):
|
||||
"infohash_v2": v2_hash,
|
||||
"name": "a.txt",
|
||||
"state": "uploading",
|
||||
"save_path": "/downloads",
|
||||
}]).encode(),
|
||||
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
|
||||
torrent_bytes,
|
||||
@@ -217,9 +220,9 @@ class QBittorrentReaderTests(unittest.TestCase):
|
||||
malformed_bytes, malformed_hash = torrent("broken.txt", b"bad")
|
||||
second_bytes, second_hash = torrent("second.txt", b"two")
|
||||
torrents = [
|
||||
{"hash": first_hash, "name": "first.txt", "state": "uploading"},
|
||||
{"hash": malformed_hash, "name": "broken.txt", "state": "stalledUP"},
|
||||
{"hash": second_hash, "name": "second.txt", "state": "uploading"},
|
||||
{"hash": first_hash, "name": "first.txt", "state": "uploading", "save_path": "/downloads"},
|
||||
{"hash": malformed_hash, "name": "broken.txt", "state": "stalledUP", "save_path": "/downloads"},
|
||||
{"hash": second_hash, "name": "second.txt", "state": "uploading", "save_path": "/downloads"},
|
||||
]
|
||||
responses = [
|
||||
b"Ok.", json.dumps(torrents).encode(),
|
||||
|
||||
+22
-1
@@ -40,6 +40,7 @@ class ResourceTests(unittest.TestCase):
|
||||
"hash": decoded.info_hash_v2_hex,
|
||||
"name": "v2.bin",
|
||||
"state": "stoppedUP",
|
||||
"save_path": "/downloads",
|
||||
},
|
||||
[{
|
||||
"index": 0,
|
||||
@@ -91,6 +92,7 @@ class ResourceTests(unittest.TestCase):
|
||||
"hash": decoded.info_hash_v1_hex,
|
||||
"name": "resource",
|
||||
"state": "stoppedUP",
|
||||
"save_path": "/downloads/nested",
|
||||
},
|
||||
[
|
||||
{
|
||||
@@ -106,6 +108,7 @@ class ResourceTests(unittest.TestCase):
|
||||
observed,
|
||||
)
|
||||
summary = normalized.summary
|
||||
self.assertEqual(normalized.save_path.as_posix(), "/downloads/nested")
|
||||
self.assertEqual(
|
||||
summary.runtime_state, resource_pb2.TORRENT_RUNTIME_STATE_STOPPED
|
||||
)
|
||||
@@ -139,6 +142,7 @@ class ResourceTests(unittest.TestCase):
|
||||
"hash": torrent_hash,
|
||||
"name": "with-padding",
|
||||
"state": "stalledUP",
|
||||
"save_path": "/downloads",
|
||||
},
|
||||
[
|
||||
{
|
||||
@@ -169,7 +173,7 @@ class ResourceTests(unittest.TestCase):
|
||||
metainfo = encode({b"info": info})
|
||||
torrent = {
|
||||
"hash": hashlib.sha1(encode(info)).hexdigest(),
|
||||
"name": "renamed", "state": "uploading",
|
||||
"name": "renamed", "state": "uploading", "save_path": "/downloads",
|
||||
}
|
||||
renamed = normalize_resource(torrent, [{
|
||||
"index": 0, "name": "renamed.txt", "size": 3,
|
||||
@@ -182,6 +186,23 @@ class ResourceTests(unittest.TestCase):
|
||||
"progress": 1.0, "priority": 1,
|
||||
}], metainfo)
|
||||
|
||||
def test_missing_or_out_of_shape_save_path_is_rejected(self):
|
||||
info = {
|
||||
b"length": 3, b"name": b"a.txt", b"piece length": 16384,
|
||||
b"pieces": b"x" * 20,
|
||||
}
|
||||
metainfo = encode({b"info": info})
|
||||
torrent = {
|
||||
"hash": hashlib.sha1(encode(info)).hexdigest(),
|
||||
"name": "a.txt", "state": "uploading", "save_path": "relative",
|
||||
}
|
||||
with self.assertRaisesRegex(ResourceError, "save path is unsafe"):
|
||||
normalize_resource(torrent, [{
|
||||
"index": 0, "name": "a.txt", "size": 3,
|
||||
"completed": 3, "priority": 1,
|
||||
}], metainfo)
|
||||
|
||||
|
||||
def test_noncanonical_bencode_is_rejected(self):
|
||||
with self.assertRaisesRegex(BencodeError, "unsorted"):
|
||||
decode_metainfo(b"d4:infod1:b1:x1:a1:yee")
|
||||
|
||||
Reference in New Issue
Block a user