Compare commits
3
Commits
ae7175719b
...
87cca59d3a
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
87cca59d3a | ||
|
|
618bb3d206 | ||
|
|
a1bf3e4315 |
@@ -120,6 +120,8 @@ username = "${QB_USER}"
|
|||||||
password_file = "/run/secrets/qb_password"
|
password_file = "/run/secrets/qb_password"
|
||||||
api_root = "/downloads"
|
api_root = "/downloads"
|
||||||
local_root = "/data/qb"
|
local_root = "/data/qb"
|
||||||
|
# Optional only when a nested qB path uses a different client-visible mount.
|
||||||
|
# local_path_overrides = { "/downloads/fast" = "/data/qb-fast" }
|
||||||
|
|
||||||
[syncthing]
|
[syncthing]
|
||||||
endpoint = "http://syncthing:8384"
|
endpoint = "http://syncthing:8384"
|
||||||
@@ -139,6 +141,14 @@ accounting. The override key is the normalized Syncthing folder path beneath
|
|||||||
`api_root`, not its folder ID. See the production deployment README for the
|
`api_root`, not its folder ID. See the production deployment README for the
|
||||||
required compose and override pattern.
|
required compose and override pattern.
|
||||||
|
|
||||||
|
qBittorrent content is resolved differently: every existing torrent keeps its
|
||||||
|
qB-reported `save_path`. The client maps that path beneath `qbittorrent.api_root`
|
||||||
|
to its own mount before a source, existing-target, permission, or eviction
|
||||||
|
operation. Thus `/downloads/Downloading` naturally maps below `/data/qb`;
|
||||||
|
there is no migration or per-resource configuration. Add a qB
|
||||||
|
`local_path_overrides` entry only when that nested API prefix is a separate
|
||||||
|
client mount.
|
||||||
|
|
||||||
The remaining node examples omit optional `[connection]`, `[jobs]`, and
|
The remaining node examples omit optional `[connection]`, `[jobs]`, and
|
||||||
`[backup]` tables and therefore use these same defaults; deployments may
|
`[backup]` tables and therefore use these same defaults; deployments may
|
||||||
override them per node.
|
override them per node.
|
||||||
@@ -277,10 +287,11 @@ The publisher deliberately uses `moby/buildkit:rootless` with
|
|||||||
`--oci-worker-no-process-sandbox`. On nested Docker hosts, the default OCI
|
`--oci-worker-no-process-sandbox`. On nested Docker hosts, the default OCI
|
||||||
sandbox can fail while masking `/proc/acpi` for an emulated build; rootless
|
sandbox can fail while masking `/proc/acpi` for an emulated build; rootless
|
||||||
BuildKit confines that compatibility setting to the disposable builder. It
|
BuildKit confines that compatibility setting to the disposable builder. It
|
||||||
refuses to publish unless `docker buildx inspect` reports both `linux/amd64`
|
waits briefly for the new worker to observe binfmt, then refuses to publish
|
||||||
and `linux/arm64`, and removes the builder and binfmt handler on success,
|
unless `docker buildx inspect` reports both `linux/amd64` and `linux/arm64`.
|
||||||
failure, or interruption. Retain the displayed manifest digest in release
|
On capability failure it prints that inspection output and removes the builder
|
||||||
notes and deploy the immutable tag or digest.
|
and binfmt handler on success, failure, or interruption. Retain the displayed
|
||||||
|
manifest digest in release notes and deploy the immutable tag or digest.
|
||||||
|
|
||||||
## Operator usage
|
## Operator usage
|
||||||
|
|
||||||
|
|||||||
@@ -86,6 +86,9 @@ python3 scripts/preflight-deployment.py ... \
|
|||||||
- That future route path and qBittorrent content root use one client bind
|
- That future route path and qBittorrent content root use one client bind
|
||||||
mount, so hard-link staging remains possible rather than silently falling
|
mount, so hard-link staging remains possible rather than silently falling
|
||||||
back to a space-consuming copy.
|
back to a space-consuming copy.
|
||||||
|
- Every explicit `qbittorrent.local_path_overrides` entry maps the same host
|
||||||
|
path in the qBittorrent and client containers. This protects nested qB save
|
||||||
|
paths that use a dedicated bind mount.
|
||||||
- A real `link(2)` operation between a unique zero-byte file in the qB root
|
- A real `link(2)` operation between a unique zero-byte file in the qB root
|
||||||
and one in the future automatic-route root. The probe verifies that both
|
and one in the future automatic-route root. The probe verifies that both
|
||||||
names refer to the same inode and removes them unconditionally.
|
names refer to the same inode and removes them unconditionally.
|
||||||
@@ -96,7 +99,8 @@ python3 scripts/preflight-deployment.py ... \
|
|||||||
- The token, qB password, and Syncthing API-key files are non-empty regular
|
- The token, qB password, and Syncthing API-key files are non-empty regular
|
||||||
files with no group/world permissions.
|
files with no group/world permissions.
|
||||||
- The client image can read its configuration and reports usable permissions,
|
- The client image can read its configuration and reports usable permissions,
|
||||||
sparse-file support, and filesystem capabilities for both roots.
|
sparse-file support, and filesystem capabilities for both primary roots and
|
||||||
|
every configured qBittorrent local-path override.
|
||||||
- qBittorrent authentication/version compatibility and Syncthing
|
- qBittorrent authentication/version compatibility and Syncthing
|
||||||
authentication/device identity are healthy from the client container.
|
authentication/device identity are healthy from the client container.
|
||||||
|
|
||||||
@@ -106,6 +110,15 @@ root and add a `local_path_overrides` mapping for that exact API path. This
|
|||||||
prevents automatic route folders being created on a small configuration
|
prevents automatic route folders being created on a small configuration
|
||||||
filesystem while the client expects to hardlink from the qB data mount.
|
filesystem while the client expects to hardlink from the qB data mount.
|
||||||
|
|
||||||
|
qBittorrent resources may use any `save_path` below `qbittorrent.api_root`.
|
||||||
|
At job preflight the client maps that qB API path to its local mount and uses
|
||||||
|
it as the resource root; data is never moved to fit Archive Control. A resource
|
||||||
|
outside the configured qB API root, or whose resolved directory is unavailable
|
||||||
|
or not a real directory in the client container, fails that job before staging
|
||||||
|
or eviction. Use `qbittorrent.local_path_overrides` only for a nested qB API
|
||||||
|
prefix backed by a distinct client mount; this preflight verifies the Docker
|
||||||
|
bind topology for each such override.
|
||||||
|
|
||||||
When converting an existing node, stop its client and Syncthing containers,
|
When converting an existing node, stop its client and Syncthing containers,
|
||||||
move each existing `routes/<route-id>` directory from the old Syncthing config
|
move each existing `routes/<route-id>` directory from the old Syncthing config
|
||||||
tree into the new qB-backed route-root directory, then recreate Syncthing and
|
tree into the new qB-backed route-root directory, then recreate Syncthing and
|
||||||
|
|||||||
+19
-2
@@ -42,8 +42,8 @@ and exported-metainfo data, keeps the qB-local hash separately, and calculates:
|
|||||||
- normalized selected and selected-complete file-index sets;
|
- normalized selected and selected-complete file-index sets;
|
||||||
- torrent runtime state;
|
- torrent runtime state;
|
||||||
- canonical path flag and content revision;
|
- canonical path flag and content revision;
|
||||||
- a save-path fingerprint based on validated path mapping, never a leaked
|
- a validated qBittorrent `save_path`, retained only in the client's local
|
||||||
absolute host path.
|
normalized observation.
|
||||||
|
|
||||||
qBittorrent 5 may expose a pure-v2 or hybrid torrent under the first 20 bytes
|
qBittorrent 5 may expose a pure-v2 or hybrid torrent under the first 20 bytes
|
||||||
of its v2 hash while separately advertising full `infohash_v1` and
|
of its v2 hash while separately advertising full `infohash_v1` and
|
||||||
@@ -56,6 +56,23 @@ All entries retain qBittorrent's stable torrent file index. Renamed/noncanonical
|
|||||||
content paths are rejected in v1 because they cannot be transported and merged
|
content paths are rejected in v1 because they cannot be transported and merged
|
||||||
without ambiguity.
|
without ambiguity.
|
||||||
|
|
||||||
|
### Per-torrent local content roots
|
||||||
|
|
||||||
|
`save_path` is not inventory, placement, or protocol data. It is qBittorrent
|
||||||
|
metadata used only by the daemon that queried qBittorrent. Before staging a
|
||||||
|
source, merging into an existing target, applying post-recheck permissions, or
|
||||||
|
evicting files, that daemon maps the torrent's API-visible `save_path` through
|
||||||
|
`qbittorrent.api_root`/`local_root`. The most-specific
|
||||||
|
`qbittorrent.local_path_overrides` mapping wins when a nested path is exposed
|
||||||
|
through a distinct client container mount.
|
||||||
|
|
||||||
|
An ordinary nested qBittorrent path such as `/media/Data/Downloading` needs no
|
||||||
|
per-resource configuration: it resolves beneath the configured root. A path
|
||||||
|
outside that root, an unmapped distinct mount, or a mapped local path that is
|
||||||
|
not a visible real directory fails the affected job before filesystem mutation.
|
||||||
|
This check is intentionally per resource; an unrelated malformed qBittorrent
|
||||||
|
entry cannot prevent normal resources from being staged or transferred.
|
||||||
|
|
||||||
### Verification guard
|
### Verification guard
|
||||||
|
|
||||||
The client captures transfer counters and state before recheck, issues recheck,
|
The client captures transfer counters and state before recheck, issues recheck,
|
||||||
|
|||||||
@@ -14,6 +14,13 @@ client:
|
|||||||
a regular file or an explicitly created directory;
|
a regular file or an explicitly created directory;
|
||||||
5. verifies every operation remains beneath the local configured root.
|
5. verifies every operation remains beneath the local configured root.
|
||||||
|
|
||||||
|
For qBittorrent content, the root is resolved per resource: qB's authoritative
|
||||||
|
API-visible `save_path` is mapped beneath `qbittorrent.api_root` (or a more
|
||||||
|
specific configured local override) into the client namespace. This permits
|
||||||
|
existing nested save paths without moving data, but does not permit paths
|
||||||
|
outside the configured boundary. The resolved path remains local client state
|
||||||
|
and is never sent to the control daemon.
|
||||||
|
|
||||||
Sockets, devices, FIFOs, symlinks, and other special entries fail preflight.
|
Sockets, devices, FIFOs, symlinks, and other special entries fail preflight.
|
||||||
Permission or ownership mismatch is fail-fast. Archive Control never changes
|
Permission or ownership mismatch is fail-fast. Archive Control never changes
|
||||||
source ownership or mode to make a job pass.
|
source ownership or mode to make a job pass.
|
||||||
@@ -150,4 +157,3 @@ Offline tooling provides list, verify, and restore. Restore requires stopped
|
|||||||
daemon access, verifies the chosen backup, preserves the suspect database under
|
daemon access, verifies the chosen backup, preserves the suspect database under
|
||||||
a timestamped name, installs the replacement atomically, and runs integrity and
|
a timestamped name, installs the replacement atomically, and runs integrity and
|
||||||
schema checks before normal startup. Backups contain no configured secrets.
|
schema checks before normal startup. Backups contain no configured secrets.
|
||||||
|
|
||||||
|
|||||||
@@ -134,7 +134,7 @@ from pathlib import Path
|
|||||||
from archive_clients.config import ClientConfig
|
from archive_clients.config import ClientConfig
|
||||||
config=ClientConfig.load(Path(sys.argv[1]))
|
config=ClientConfig.load(Path(sys.argv[1]))
|
||||||
value={"shared_token_file":str(config.shared_token_file)}
|
value={"shared_token_file":str(config.shared_token_file)}
|
||||||
value["qbittorrent"]={"api_root":str(config.qbittorrent.api_root),"local_root":str(config.qbittorrent.local_root),"password_file":str(config.qbittorrent.password_file)}
|
value["qbittorrent"]={"api_root":str(config.qbittorrent.api_root),"local_root":str(config.qbittorrent.local_root),"password_file":str(config.qbittorrent.password_file),"local_path_overrides":{str(api):str(local) for api,local in config.qbittorrent.local_path_overrides}}
|
||||||
value["syncthing"]={"api_root":str(config.syncthing.api_root),"local_root":str(config.syncthing.local_root),"api_key_file":str(config.syncthing.api_key_file),"local_path_overrides":{str(api):str(local) for api,local in config.syncthing.local_path_overrides}}
|
value["syncthing"]={"api_root":str(config.syncthing.api_root),"local_root":str(config.syncthing.local_root),"api_key_file":str(config.syncthing.api_key_file),"local_path_overrides":{str(api):str(local) for api,local in config.syncthing.local_path_overrides}}
|
||||||
print(json.dumps(value,sort_keys=True))'''
|
print(json.dumps(value,sort_keys=True))'''
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
@@ -174,6 +174,26 @@ raise SystemExit(0 if all(p.state == 1 for p in probes) else 1)'''
|
|||||||
print(result.stdout.strip())
|
print(result.stdout.strip())
|
||||||
|
|
||||||
|
|
||||||
|
def require_qb_override_mappings(
|
||||||
|
client: dict[str, Any], qbittorrent: dict[str, Any], qb: dict[str, Any],
|
||||||
|
) -> None:
|
||||||
|
"""Prove every explicit qB API/local override sees the same host bytes."""
|
||||||
|
|
||||||
|
overrides = qb.get("local_path_overrides", {})
|
||||||
|
if not isinstance(overrides, dict):
|
||||||
|
raise CheckFailure("qbittorrent.local_path_overrides must be a table")
|
||||||
|
for api_path, local_path in overrides.items():
|
||||||
|
if not isinstance(api_path, str) or not isinstance(local_path, str):
|
||||||
|
raise CheckFailure(
|
||||||
|
"qbittorrent.local_path_overrides entries are invalid"
|
||||||
|
)
|
||||||
|
require_same_path(
|
||||||
|
f"qBittorrent override {api_path}/local mapping",
|
||||||
|
map_path(qbittorrent, api_path),
|
||||||
|
map_path(client, local_path),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def run_hardlink_probe(
|
def run_hardlink_probe(
|
||||||
container: str, qb_root: str, route_root: str, user: str | None = None,
|
container: str, qb_root: str, route_root: str, user: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -298,6 +318,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
syncthing_route, client_route,
|
syncthing_route, client_route,
|
||||||
)
|
)
|
||||||
require_same_path("qBittorrent api_root/local_root", qb_api, client_qb)
|
require_same_path("qBittorrent api_root/local_root", qb_api, client_qb)
|
||||||
|
require_qb_override_mappings(client, qbittorrent, qb)
|
||||||
if client_qb.destination != client_route.destination:
|
if client_qb.destination != client_route.destination:
|
||||||
raise CheckFailure(
|
raise CheckFailure(
|
||||||
"qBittorrent and future route roots use separate client bind "
|
"qBittorrent and future route roots use separate client bind "
|
||||||
|
|||||||
@@ -69,13 +69,32 @@ docker buildx create --name "${builder}" --driver docker-container \
|
|||||||
--buildkitd-flags "${buildkitd_flags}" --use >/dev/null
|
--buildkitd-flags "${buildkitd_flags}" --use >/dev/null
|
||||||
builder_created=true
|
builder_created=true
|
||||||
|
|
||||||
|
# A newly-created rootless worker can publish its native platform before it has
|
||||||
|
# observed the just-registered binfmt handler. Do not mistake that brief
|
||||||
|
# startup state for a partial-release-capable builder.
|
||||||
|
platforms=''
|
||||||
|
supports_all=false
|
||||||
|
for attempt in {1..10}; do
|
||||||
platforms=$(docker buildx inspect "${builder}" --bootstrap 2>&1)
|
platforms=$(docker buildx inspect "${builder}" --bootstrap 2>&1)
|
||||||
|
supports_all=true
|
||||||
for platform in linux/amd64 linux/arm64; do
|
for platform in linux/amd64 linux/arm64; do
|
||||||
if ! grep -Fq "${platform}" <<<"${platforms}"; then
|
if ! grep -Fq "${platform}" <<<"${platforms}"; then
|
||||||
echo "Builder ${builder} does not support ${platform}; refusing partial release." >&2
|
supports_all=false
|
||||||
exit 1
|
break
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
if [[ ${supports_all} == true ]]; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
if [[ ${attempt} -lt 10 ]]; then
|
||||||
|
sleep 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [[ ${supports_all} != true ]]; then
|
||||||
|
echo "Builder ${builder} does not support both required platforms; refusing partial release." >&2
|
||||||
|
printf '%s\n' "${platforms}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
tags=(--tag "${repository}:${tag}")
|
tags=(--tag "${repository}:${tag}")
|
||||||
if [[ ${also_latest} == true ]]; then
|
if [[ ${also_latest} == true ]]; then
|
||||||
|
|||||||
@@ -27,10 +27,16 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||||||
parser.add_argument("--check-config", action="store_true")
|
parser.add_argument("--check-config", action="store_true")
|
||||||
arguments = parser.parse_args(argv)
|
arguments = parser.parse_args(argv)
|
||||||
config = ClientConfig.load(arguments.config, arguments.mode)
|
config = ClientConfig.load(arguments.config, arguments.mode)
|
||||||
|
qb_extra_roots = tuple(sorted({
|
||||||
|
root
|
||||||
|
for _, root in config.qbittorrent.local_path_overrides
|
||||||
|
if root != config.qbittorrent.local_root
|
||||||
|
}))
|
||||||
probes = [
|
probes = [
|
||||||
probe_root(config.qbittorrent.local_root),
|
probe_root(config.qbittorrent.local_root),
|
||||||
probe_root(config.syncthing.local_root),
|
probe_root(config.syncthing.local_root),
|
||||||
]
|
]
|
||||||
|
qb_extra_probes = tuple(probe_root(root) for root in qb_extra_roots)
|
||||||
probe_writable_directory(config.state_db.parent)
|
probe_writable_directory(config.state_db.parent)
|
||||||
probe_writable_directory(config.backup_dir)
|
probe_writable_directory(config.backup_dir)
|
||||||
shared_token = config.read_shared_token()
|
shared_token = config.read_shared_token()
|
||||||
@@ -40,7 +46,10 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||||||
print(json.dumps({
|
print(json.dumps({
|
||||||
"client_id": config.client_id,
|
"client_id": config.client_id,
|
||||||
"role": config.role,
|
"role": config.role,
|
||||||
"filesystems": [probe.__dict__ | {"root": str(probe.root)} for probe in probes],
|
"filesystems": [
|
||||||
|
probe.__dict__ | {"root": str(probe.root)}
|
||||||
|
for probe in (*probes[:1], *qb_extra_probes, *probes[1:])
|
||||||
|
],
|
||||||
}, sort_keys=True))
|
}, sort_keys=True))
|
||||||
return 0
|
return 0
|
||||||
configure_logging(
|
configure_logging(
|
||||||
@@ -68,7 +77,7 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
asyncio.run(ArchiveClientDaemon(
|
asyncio.run(ArchiveClientDaemon(
|
||||||
config, probes, service_probes,
|
config, probes, service_probes, qb_extra_probes=qb_extra_probes,
|
||||||
resource_reader=QBittorrentReader(config.qbittorrent),
|
resource_reader=QBittorrentReader(config.qbittorrent),
|
||||||
).run())
|
).run())
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
|
|||||||
@@ -213,8 +213,6 @@ def _local_path_overrides(
|
|||||||
value: dict[str, Any], api_root: PurePosixPath, name: str
|
value: dict[str, Any], api_root: PurePosixPath, name: str
|
||||||
) -> tuple[tuple[PurePosixPath, Path], ...]:
|
) -> tuple[tuple[PurePosixPath, Path], ...]:
|
||||||
raw = value.get("local_path_overrides", {})
|
raw = value.get("local_path_overrides", {})
|
||||||
if name != "syncthing" and raw:
|
|
||||||
raise ConfigError(f"{name}.local_path_overrides is unsupported")
|
|
||||||
if not isinstance(raw, dict):
|
if not isinstance(raw, dict):
|
||||||
raise ConfigError(f"{name}.local_path_overrides must be a table")
|
raise ConfigError(f"{name}.local_path_overrides must be a table")
|
||||||
parsed: list[tuple[PurePosixPath, Path]] = []
|
parsed: list[tuple[PurePosixPath, Path]] = []
|
||||||
|
|||||||
@@ -64,6 +64,7 @@ class ArchiveClientDaemon:
|
|||||||
service_probes: list[ServiceProbe],
|
service_probes: list[ServiceProbe],
|
||||||
resource_reader: QBittorrentReader | None = None,
|
resource_reader: QBittorrentReader | None = None,
|
||||||
route_manager: SyncthingRouteManager | None = None,
|
route_manager: SyncthingRouteManager | None = None,
|
||||||
|
qb_extra_probes: tuple[FilesystemProbe, ...] = (),
|
||||||
):
|
):
|
||||||
if len(probes) != 2:
|
if len(probes) != 2:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
@@ -71,6 +72,7 @@ class ArchiveClientDaemon:
|
|||||||
)
|
)
|
||||||
self.config = config
|
self.config = config
|
||||||
self.probes = probes
|
self.probes = probes
|
||||||
|
self.qb_probes = (probes[0], *qb_extra_probes)
|
||||||
self.service_probes = service_probes
|
self.service_probes = service_probes
|
||||||
self.inventory = (
|
self.inventory = (
|
||||||
InventoryService(resource_reader, config.client_id)
|
InventoryService(resource_reader, config.client_id)
|
||||||
@@ -119,9 +121,13 @@ class ArchiveClientDaemon:
|
|||||||
store=self.store,
|
store=self.store,
|
||||||
qb_root=config.qbittorrent.local_root,
|
qb_root=config.qbittorrent.local_root,
|
||||||
qb_api_root=config.qbittorrent.api_root,
|
qb_api_root=config.qbittorrent.api_root,
|
||||||
|
qb_roots=config.qbittorrent.roots,
|
||||||
route_path=self._route_path,
|
route_path=self._route_path,
|
||||||
syncthing_transport=self.routes.transport,
|
syncthing_transport=self.routes.transport,
|
||||||
sparse_supported=all(probe.sparse_files for probe in probes),
|
sparse_supported=all(
|
||||||
|
probe.sparse_files
|
||||||
|
for probe in (*self.qb_probes, probes[1])
|
||||||
|
),
|
||||||
poll_interval=config.jobs.poll_interval,
|
poll_interval=config.jobs.poll_interval,
|
||||||
verification_timeout=config.jobs.verification_timeout,
|
verification_timeout=config.jobs.verification_timeout,
|
||||||
free_space_reserve_bytes=(
|
free_space_reserve_bytes=(
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import hashlib
|
|||||||
import os
|
import os
|
||||||
import stat
|
import stat
|
||||||
from pathlib import Path, PurePosixPath
|
from pathlib import Path, PurePosixPath
|
||||||
from typing import Iterable
|
from typing import Callable, Iterable
|
||||||
|
|
||||||
from archive_clients.qbittorrent import QBittorrentReader
|
from archive_clients.qbittorrent import QBittorrentReader
|
||||||
from archive_clients.resources import NormalizedResource
|
from archive_clients.resources import NormalizedResource
|
||||||
@@ -23,7 +23,7 @@ def verify_and_snapshot(
|
|||||||
job_id: str,
|
job_id: str,
|
||||||
resource: NormalizedResource,
|
resource: NormalizedResource,
|
||||||
selected_indices: Iterable[int],
|
selected_indices: Iterable[int],
|
||||||
qb_root: Path,
|
content_root: Path,
|
||||||
store: ClientStore,
|
store: ClientStore,
|
||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
selected = set(selected_indices)
|
selected = set(selected_indices)
|
||||||
@@ -38,7 +38,7 @@ def verify_and_snapshot(
|
|||||||
f"cache file {index} is not selected and complete"
|
f"cache file {index} is not selected and complete"
|
||||||
)
|
)
|
||||||
relative = _relative(item.canonical_path)
|
relative = _relative(item.canonical_path)
|
||||||
path = qb_root.joinpath(*relative.parts)
|
path = content_root.joinpath(*relative.parts)
|
||||||
try:
|
try:
|
||||||
metadata = path.lstat()
|
metadata = path.lstat()
|
||||||
except FileNotFoundError as exc:
|
except FileNotFoundError as exc:
|
||||||
@@ -65,7 +65,7 @@ def verify_and_snapshot(
|
|||||||
"sha256": _sha256(path),
|
"sha256": _sha256(path),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
snapshot = {"files": files}
|
snapshot = {"content_root": str(content_root), "files": files}
|
||||||
return store.put_job_artifact(job_id, "eviction-snapshot", snapshot)["value"]
|
return store.put_job_artifact(job_id, "eviction-snapshot", snapshot)["value"]
|
||||||
|
|
||||||
|
|
||||||
@@ -90,9 +90,9 @@ def remove_qb_entry(
|
|||||||
def safe_unlink(
|
def safe_unlink(
|
||||||
*,
|
*,
|
||||||
job_id: str,
|
job_id: str,
|
||||||
qb_root: Path,
|
|
||||||
qbittorrent: QBittorrentReader,
|
qbittorrent: QBittorrentReader,
|
||||||
store: ClientStore,
|
store: ClientStore,
|
||||||
|
resource_root: Callable[[NormalizedResource], Path],
|
||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
completed = store.get_job_artifact(job_id, "eviction-unlinked")
|
completed = store.get_job_artifact(job_id, "eviction-unlinked")
|
||||||
if completed is not None:
|
if completed is not None:
|
||||||
@@ -101,11 +101,19 @@ def safe_unlink(
|
|||||||
if snapshot_row is None:
|
if snapshot_row is None:
|
||||||
raise EvictionError("eviction snapshot is missing")
|
raise EvictionError("eviction snapshot is missing")
|
||||||
snapshot = snapshot_row["value"]
|
snapshot = snapshot_row["value"]
|
||||||
|
if not isinstance(snapshot, dict):
|
||||||
|
raise EvictionError("eviction snapshot is invalid")
|
||||||
|
content_root_value = snapshot.get("content_root")
|
||||||
|
if not isinstance(content_root_value, str):
|
||||||
|
raise EvictionError("eviction snapshot content root is missing")
|
||||||
|
content_root = Path(content_root_value)
|
||||||
|
if not content_root.is_absolute() or content_root.is_symlink():
|
||||||
|
raise EvictionError("eviction snapshot content root is unsafe")
|
||||||
files = snapshot.get("files")
|
files = snapshot.get("files")
|
||||||
if not isinstance(files, list):
|
if not isinstance(files, list):
|
||||||
raise EvictionError("eviction snapshot is invalid")
|
raise EvictionError("eviction snapshot is invalid")
|
||||||
|
|
||||||
referenced = _remaining_paths(qbittorrent.list_resources())
|
referenced = _remaining_paths(qbittorrent.list_resources(), resource_root)
|
||||||
removed: list[str] = []
|
removed: list[str] = []
|
||||||
retained: list[dict[str, str]] = []
|
retained: list[dict[str, str]] = []
|
||||||
directories: set[Path] = set()
|
directories: set[Path] = set()
|
||||||
@@ -113,10 +121,10 @@ def safe_unlink(
|
|||||||
if not isinstance(record, dict) or not isinstance(record.get("path"), str):
|
if not isinstance(record, dict) or not isinstance(record.get("path"), str):
|
||||||
raise EvictionError("eviction file record is invalid")
|
raise EvictionError("eviction file record is invalid")
|
||||||
relative = _relative(record["path"])
|
relative = _relative(record["path"])
|
||||||
if relative.as_posix() in referenced:
|
path = content_root.joinpath(*relative.parts)
|
||||||
|
if path in referenced:
|
||||||
retained.append({"path": relative.as_posix(), "reason": "shared"})
|
retained.append({"path": relative.as_posix(), "reason": "shared"})
|
||||||
continue
|
continue
|
||||||
path = qb_root.joinpath(*relative.parts)
|
|
||||||
try:
|
try:
|
||||||
metadata = path.lstat()
|
metadata = path.lstat()
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
@@ -142,7 +150,7 @@ def safe_unlink(
|
|||||||
path.unlink()
|
path.unlink()
|
||||||
removed.append(relative.as_posix())
|
removed.append(relative.as_posix())
|
||||||
parent = path.parent
|
parent = path.parent
|
||||||
while parent != qb_root:
|
while parent != content_root:
|
||||||
directories.add(parent)
|
directories.add(parent)
|
||||||
parent = parent.parent
|
parent = parent.parent
|
||||||
|
|
||||||
@@ -153,7 +161,7 @@ def safe_unlink(
|
|||||||
try:
|
try:
|
||||||
directory.rmdir()
|
directory.rmdir()
|
||||||
removed_directories.append(
|
removed_directories.append(
|
||||||
directory.relative_to(qb_root).as_posix()
|
directory.relative_to(content_root).as_posix()
|
||||||
)
|
)
|
||||||
except OSError as exc:
|
except OSError as exc:
|
||||||
if exc.errno not in {errno.ENOTEMPTY, errno.ENOENT}:
|
if exc.errno not in {errno.ENOTEMPTY, errno.ENOENT}:
|
||||||
@@ -214,12 +222,24 @@ def compensate_materialized_files(
|
|||||||
return removed
|
return removed
|
||||||
|
|
||||||
|
|
||||||
def _remaining_paths(resources: Iterable[NormalizedResource]) -> set[str]:
|
def _remaining_paths(
|
||||||
return {
|
resources: Iterable[NormalizedResource],
|
||||||
item.canonical_path
|
resource_root: Callable[[NormalizedResource], Path],
|
||||||
for resource in resources
|
) -> set[Path]:
|
||||||
|
result: set[Path] = set()
|
||||||
|
for resource in resources:
|
||||||
|
try:
|
||||||
|
root = resource_root(resource)
|
||||||
|
except (OSError, RuntimeError, ValueError):
|
||||||
|
# A malformed unrelated qB entry must not stop an otherwise
|
||||||
|
# safe eviction. Its unresolvable path cannot be considered a
|
||||||
|
# shared path under the verified eviction root.
|
||||||
|
continue
|
||||||
|
result.update(
|
||||||
|
root.joinpath(*_relative(item.canonical_path).parts)
|
||||||
for item in resource.files
|
for item in resource.files
|
||||||
}
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def _relative(value: str) -> PurePosixPath:
|
def _relative(value: str) -> PurePosixPath:
|
||||||
|
|||||||
+78
-16
@@ -14,6 +14,7 @@ import uuid
|
|||||||
from pathlib import Path, PurePosixPath
|
from pathlib import Path, PurePosixPath
|
||||||
from typing import Callable, Iterable
|
from typing import Callable, Iterable
|
||||||
|
|
||||||
|
from archive_clients.config import ConfigError, RootMapping
|
||||||
from archive_clients.protocol import decode_message, encode_message
|
from archive_clients.protocol import decode_message, encode_message
|
||||||
from archive_clients.eviction import (
|
from archive_clients.eviction import (
|
||||||
EvictionError,
|
EvictionError,
|
||||||
@@ -67,6 +68,7 @@ class ClientJobExecutor:
|
|||||||
store: ClientStore,
|
store: ClientStore,
|
||||||
qb_root: Path,
|
qb_root: Path,
|
||||||
qb_api_root: PurePosixPath,
|
qb_api_root: PurePosixPath,
|
||||||
|
qb_roots: RootMapping | None = None,
|
||||||
route_path: Callable[[str], Path],
|
route_path: Callable[[str], Path],
|
||||||
syncthing_transport: object,
|
syncthing_transport: object,
|
||||||
sparse_supported: bool,
|
sparse_supported: bool,
|
||||||
@@ -78,7 +80,10 @@ class ClientJobExecutor:
|
|||||||
self.qbittorrent = qbittorrent
|
self.qbittorrent = qbittorrent
|
||||||
self.store = store
|
self.store = store
|
||||||
self.qb_root = qb_root
|
self.qb_root = qb_root
|
||||||
self.qb_api_root = qb_api_root
|
self.qb_api_root = PurePosixPath(qb_api_root)
|
||||||
|
self.qb_roots = qb_roots or RootMapping(
|
||||||
|
self.qb_api_root, self.qb_root
|
||||||
|
)
|
||||||
self.route_path = route_path
|
self.route_path = route_path
|
||||||
self.syncthing_transport = syncthing_transport
|
self.syncthing_transport = syncthing_transport
|
||||||
self.sparse_supported = sparse_supported
|
self.sparse_supported = sparse_supported
|
||||||
@@ -479,6 +484,7 @@ class ClientJobExecutor:
|
|||||||
"archive coverage no longer covers the cache selection"
|
"archive coverage no longer covers the cache selection"
|
||||||
)
|
)
|
||||||
resource = self._resource(definition)
|
resource = self._resource(definition)
|
||||||
|
resource_root = self._resource_root(resource)
|
||||||
current = _resource_fingerprint(resource, self.client_id)
|
current = _resource_fingerprint(resource, self.client_id)
|
||||||
if not _fingerprint_matches(
|
if not _fingerprint_matches(
|
||||||
current, definition.eviction.cache_fingerprint
|
current, definition.eviction.cache_fingerprint
|
||||||
@@ -490,7 +496,7 @@ class ClientJobExecutor:
|
|||||||
job_id=definition.job_id,
|
job_id=definition.job_id,
|
||||||
resource=resource,
|
resource=resource,
|
||||||
selected_indices=requested,
|
selected_indices=requested,
|
||||||
qb_root=self.qb_root,
|
content_root=resource_root,
|
||||||
store=self.store,
|
store=self.store,
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
@@ -505,9 +511,9 @@ class ClientJobExecutor:
|
|||||||
if step == job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK:
|
if step == job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK:
|
||||||
safe_unlink(
|
safe_unlink(
|
||||||
job_id=definition.job_id,
|
job_id=definition.job_id,
|
||||||
qb_root=self.qb_root,
|
|
||||||
qbittorrent=self.qbittorrent,
|
qbittorrent=self.qbittorrent,
|
||||||
store=self.store,
|
store=self.store,
|
||||||
|
resource_root=self._resource_root,
|
||||||
)
|
)
|
||||||
placement = resource_pb2.Placement(
|
placement = resource_pb2.Placement(
|
||||||
client_id=self.client_id,
|
client_id=self.client_id,
|
||||||
@@ -538,7 +544,8 @@ class ClientJobExecutor:
|
|||||||
raise JobExecutionError(
|
raise JobExecutionError(
|
||||||
"source resource changed after job confirmation"
|
"source resource changed after job confirmation"
|
||||||
)
|
)
|
||||||
self._reject_unsafe_partfile(definition, resource)
|
source_root = self._resource_root(resource)
|
||||||
|
self._reject_unsafe_partfile(definition, resource, source_root)
|
||||||
indices = _selection_indices(definition.transfer.transfer_delta_files)
|
indices = _selection_indices(definition.transfer.transfer_delta_files)
|
||||||
by_index = {item.file_index: item for item in resource.files}
|
by_index = {item.file_index: item for item in resource.files}
|
||||||
if not indices or any(index not in by_index for index in indices):
|
if not indices or any(index not in by_index for index in indices):
|
||||||
@@ -602,7 +609,7 @@ class ClientJobExecutor:
|
|||||||
self._require_space(
|
self._require_space(
|
||||||
route_root,
|
route_root,
|
||||||
self._copy_required_bytes(
|
self._copy_required_bytes(
|
||||||
self.qb_root,
|
source_root,
|
||||||
route_root,
|
route_root,
|
||||||
(
|
(
|
||||||
(entry.target_canonical_path, entry.logical_bytes)
|
(entry.target_canonical_path, entry.logical_bytes)
|
||||||
@@ -613,7 +620,7 @@ class ClientJobExecutor:
|
|||||||
)
|
)
|
||||||
stage_transfer(
|
stage_transfer(
|
||||||
manifest,
|
manifest,
|
||||||
source_root=self.qb_root,
|
source_root=source_root,
|
||||||
sync_root=route_root,
|
sync_root=route_root,
|
||||||
store=self.store,
|
store=self.store,
|
||||||
artifact_sources={"metainfo/source.torrent": metainfo_path},
|
artifact_sources={"metainfo/source.torrent": metainfo_path},
|
||||||
@@ -691,23 +698,27 @@ class ClientJobExecutor:
|
|||||||
"target materialization was sent to the wrong client"
|
"target materialization was sent to the wrong client"
|
||||||
)
|
)
|
||||||
published = load_published_transfer(self._job_directory(definition))
|
published = load_published_transfer(self._job_directory(definition))
|
||||||
|
info_hash = _info_hash(definition)
|
||||||
|
resource = self.qbittorrent.get_resource(info_hash)
|
||||||
|
target_root = (
|
||||||
|
self._resource_root(resource)
|
||||||
|
if resource is not None else self.qb_root
|
||||||
|
)
|
||||||
# The target can likewise hardlink an arrived Syncthing payload into
|
# The target can likewise hardlink an arrived Syncthing payload into
|
||||||
# qB's content root when those directories share a filesystem.
|
# qB's content root when those directories share a filesystem.
|
||||||
self._require_space(
|
self._require_space(
|
||||||
self.qb_root,
|
target_root,
|
||||||
self._copy_required_bytes(
|
self._copy_required_bytes(
|
||||||
published.job_directory,
|
published.job_directory,
|
||||||
self.qb_root,
|
target_root,
|
||||||
(
|
(
|
||||||
(entry.payload_relative_path, entry.logical_bytes)
|
(entry.payload_relative_path, entry.logical_bytes)
|
||||||
for entry in published.manifest.files
|
for entry in published.manifest.files
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
info_hash = _info_hash(definition)
|
|
||||||
resource = self.qbittorrent.get_resource(info_hash)
|
|
||||||
if resource is not None:
|
if resource is not None:
|
||||||
self._reject_unsafe_partfile(definition, resource)
|
self._reject_unsafe_partfile(definition, resource, target_root)
|
||||||
if definition.transfer.HasField("target_baseline_fingerprint"):
|
if definition.transfer.HasField("target_baseline_fingerprint"):
|
||||||
if resource is None or not _fingerprint_matches(
|
if resource is None or not _fingerprint_matches(
|
||||||
_resource_fingerprint(resource, self.client_id),
|
_resource_fingerprint(resource, self.client_id),
|
||||||
@@ -732,13 +743,14 @@ class ClientJobExecutor:
|
|||||||
== resource_pb2.TORRENT_RUNTIME_STATE_STOPPED
|
== resource_pb2.TORRENT_RUNTIME_STATE_STOPPED
|
||||||
),
|
),
|
||||||
"total_file_count": len(published.manifest.files),
|
"total_file_count": len(published.manifest.files),
|
||||||
|
"content_root": str(target_root),
|
||||||
}
|
}
|
||||||
self.store.put_job_artifact(
|
self.store.put_job_artifact(
|
||||||
definition.job_id, "target-baseline", baseline
|
definition.job_id, "target-baseline", baseline
|
||||||
)
|
)
|
||||||
materialize_transfer(
|
materialize_transfer(
|
||||||
published,
|
published,
|
||||||
target_root=self.qb_root,
|
target_root=target_root,
|
||||||
store=self.store,
|
store=self.store,
|
||||||
sparse_supported=self.sparse_supported,
|
sparse_supported=self.sparse_supported,
|
||||||
cancel_check=lambda: self._raise_if_cancelled(
|
cancel_check=lambda: self._raise_if_cancelled(
|
||||||
@@ -813,7 +825,9 @@ class ClientJobExecutor:
|
|||||||
raise JobExecutionError(
|
raise JobExecutionError(
|
||||||
"verified qBittorrent resource disappeared"
|
"verified qBittorrent resource disappeared"
|
||||||
)
|
)
|
||||||
_normalize_verified_resource_permissions(self.qb_root, verified)
|
_normalize_verified_resource_permissions(
|
||||||
|
self._resource_root(verified), verified
|
||||||
|
)
|
||||||
if should_start:
|
if should_start:
|
||||||
self.qbittorrent.start(qb_torrent_id)
|
self.qbittorrent.start(qb_torrent_id)
|
||||||
placement = resource_pb2.Placement(
|
placement = resource_pb2.Placement(
|
||||||
@@ -852,6 +866,7 @@ class ClientJobExecutor:
|
|||||||
)
|
)
|
||||||
return
|
return
|
||||||
baseline = baseline_row["value"]
|
baseline = baseline_row["value"]
|
||||||
|
content_root = self._artifact_content_root(baseline)
|
||||||
info_hash = _info_hash(definition)
|
info_hash = _info_hash(definition)
|
||||||
present = self.qbittorrent.get_resource(info_hash)
|
present = self.qbittorrent.get_resource(info_hash)
|
||||||
if present is not None:
|
if present is not None:
|
||||||
@@ -870,7 +885,7 @@ class ClientJobExecutor:
|
|||||||
self.qbittorrent.delete_entry(qb_torrent_id)
|
self.qbittorrent.delete_entry(qb_torrent_id)
|
||||||
compensate_materialized_files(
|
compensate_materialized_files(
|
||||||
job_id=definition.job_id,
|
job_id=definition.job_id,
|
||||||
qb_root=self.qb_root,
|
qb_root=content_root,
|
||||||
store=self.store,
|
store=self.store,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -906,10 +921,11 @@ class ClientJobExecutor:
|
|||||||
self,
|
self,
|
||||||
definition: job_pb2.JobDefinition,
|
definition: job_pb2.JobDefinition,
|
||||||
resource: NormalizedResource,
|
resource: NormalizedResource,
|
||||||
|
content_root: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
roots = {self.qb_root}
|
roots = {content_root}
|
||||||
for item in resource.files:
|
for item in resource.files:
|
||||||
candidate = self.qb_root / PurePosixPath(item.canonical_path).parts[0]
|
candidate = content_root / PurePosixPath(item.canonical_path).parts[0]
|
||||||
roots.add(candidate if candidate.is_dir() else candidate.parent)
|
roots.add(candidate if candidate.is_dir() else candidate.parent)
|
||||||
hashes = {
|
hashes = {
|
||||||
value.lower() for value in (
|
value.lower() for value in (
|
||||||
@@ -928,6 +944,52 @@ class ClientJobExecutor:
|
|||||||
"safely transferred by this client"
|
"safely transferred by this client"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _resource_root(self, resource: NormalizedResource) -> Path:
|
||||||
|
"""Resolve a qB resource's save path within this client's mounts.
|
||||||
|
|
||||||
|
qB returns paths in its own API/container namespace. The configured
|
||||||
|
root mapping translates those paths into the client namespace and
|
||||||
|
supports nested save paths and explicit prefix overrides. A missing
|
||||||
|
save path is retained only for older in-process test fixtures; every
|
||||||
|
normalized qB API resource has one.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if resource.save_path is None:
|
||||||
|
return self.qb_root
|
||||||
|
try:
|
||||||
|
root = self.qb_roots.api_to_local(resource.save_path.as_posix())
|
||||||
|
except ConfigError as exc:
|
||||||
|
raise JobExecutionError(
|
||||||
|
"qBittorrent resource save path is outside the configured "
|
||||||
|
"API root or has no local path mapping"
|
||||||
|
) from exc
|
||||||
|
try:
|
||||||
|
metadata = root.lstat()
|
||||||
|
except FileNotFoundError as exc:
|
||||||
|
raise JobExecutionError(
|
||||||
|
"qBittorrent resource save path is not visible in the "
|
||||||
|
"client container"
|
||||||
|
) from exc
|
||||||
|
if not stat.S_ISDIR(metadata.st_mode):
|
||||||
|
raise JobExecutionError(
|
||||||
|
"qBittorrent resource save path is not a real directory in "
|
||||||
|
"the client container"
|
||||||
|
)
|
||||||
|
return root
|
||||||
|
|
||||||
|
def _artifact_content_root(self, baseline: object) -> Path:
|
||||||
|
if not isinstance(baseline, dict):
|
||||||
|
raise JobExecutionError("target baseline is invalid")
|
||||||
|
value = baseline.get("content_root")
|
||||||
|
if not isinstance(value, str):
|
||||||
|
# A pre-existing durable baseline predates per-resource roots.
|
||||||
|
# It can only have materialized to the configured target root.
|
||||||
|
return self.qb_root
|
||||||
|
root = Path(value)
|
||||||
|
if not root.is_absolute() or root.is_symlink():
|
||||||
|
raise JobExecutionError("target baseline content root is unsafe")
|
||||||
|
return root
|
||||||
|
|
||||||
def _raise_if_cancelled(self, job_id: str) -> None:
|
def _raise_if_cancelled(self, job_id: str) -> None:
|
||||||
event = self._cancel_events.get(job_id)
|
event = self._cancel_events.get(job_id)
|
||||||
if event is not None and event.is_set():
|
if event is not None and event.is_set():
|
||||||
|
|||||||
@@ -19,10 +19,21 @@ class ResourceError(ValueError):
|
|||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class NormalizedResource:
|
class NormalizedResource:
|
||||||
|
"""A normalized qB observation for protocol data and local file work.
|
||||||
|
|
||||||
|
``save_path`` is qBittorrent's API-visible per-torrent content root. It
|
||||||
|
is deliberately local-only: clients resolve it through their qB path
|
||||||
|
mapping immediately before filesystem work, and it is never serialized in
|
||||||
|
inventory, placements, or control protocol messages.
|
||||||
|
"""
|
||||||
|
|
||||||
summary: resource_pb2.ResourceSummary
|
summary: resource_pb2.ResourceSummary
|
||||||
files: tuple[resource_pb2.TorrentFile, ...]
|
files: tuple[resource_pb2.TorrentFile, ...]
|
||||||
metainfo: Metainfo
|
metainfo: Metainfo
|
||||||
metainfo_bytes: bytes = b""
|
metainfo_bytes: bytes = b""
|
||||||
|
# qBittorrent's API-visible save path is intentionally local-only. It
|
||||||
|
# must never become part of inventory or placement protocol messages.
|
||||||
|
save_path: PurePosixPath | None = None
|
||||||
|
|
||||||
|
|
||||||
def build_content_tree(
|
def build_content_tree(
|
||||||
@@ -137,6 +148,7 @@ def normalize_resource(
|
|||||||
character not in "0123456789abcdef" for character in qb_torrent_id
|
character not in "0123456789abcdef" for character in qb_torrent_id
|
||||||
):
|
):
|
||||||
raise ResourceError("torrent hash is invalid")
|
raise ResourceError("torrent hash is invalid")
|
||||||
|
save_path = _save_path(torrent.get("save_path"))
|
||||||
summary = resource_pb2.ResourceSummary(
|
summary = resource_pb2.ResourceSummary(
|
||||||
qb_torrent_id=qb_torrent_id,
|
qb_torrent_id=qb_torrent_id,
|
||||||
display_name=_string(torrent.get("name"), "torrent name"),
|
display_name=_string(torrent.get("name"), "torrent name"),
|
||||||
@@ -172,7 +184,9 @@ def normalize_resource(
|
|||||||
revision_data, sort_keys=True, separators=(",", ":"),
|
revision_data, sort_keys=True, separators=(",", ":"),
|
||||||
).encode("utf-8")).hexdigest()
|
).encode("utf-8")).hexdigest()
|
||||||
summary.observed_at.FromDatetime(observed_at)
|
summary.observed_at.FromDatetime(observed_at)
|
||||||
return NormalizedResource(summary, tuple(files), metainfo, metainfo_bytes)
|
return NormalizedResource(
|
||||||
|
summary, tuple(files), metainfo, metainfo_bytes, save_path
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _set_selection(target: Any, indices: list[int]) -> None:
|
def _set_selection(target: Any, indices: list[int]) -> None:
|
||||||
@@ -217,6 +231,20 @@ def _path(value: Any) -> str:
|
|||||||
return candidate.as_posix()
|
return candidate.as_posix()
|
||||||
|
|
||||||
|
|
||||||
|
def _save_path(value: Any) -> PurePosixPath:
|
||||||
|
"""Validate qBittorrent's API-visible per-torrent content root."""
|
||||||
|
|
||||||
|
path = _string(value, "torrent save path")
|
||||||
|
candidate = PurePosixPath(path)
|
||||||
|
if (
|
||||||
|
not candidate.is_absolute()
|
||||||
|
or ".." in candidate.parts
|
||||||
|
or "." in candidate.parts
|
||||||
|
):
|
||||||
|
raise ResourceError("qBittorrent torrent save path is unsafe")
|
||||||
|
return candidate
|
||||||
|
|
||||||
|
|
||||||
def _integer(value: Any, name: str) -> int:
|
def _integer(value: Any, name: str) -> int:
|
||||||
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
||||||
raise ResourceError(f"{name} is invalid")
|
raise ResourceError(f"{name} is invalid")
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from contextlib import redirect_stdout
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from archive_clients.cli import main
|
||||||
|
|
||||||
|
|
||||||
|
class ClientCliTests(unittest.TestCase):
|
||||||
|
def test_check_config_probes_every_qb_override_root(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
for name in ("token", "qb-password", "syncthing-key"):
|
||||||
|
path = root / name
|
||||||
|
path.write_text(name, encoding="utf-8")
|
||||||
|
os.chmod(path, 0o600)
|
||||||
|
for name in ("qb", "qb-fast", "sync", "backups"):
|
||||||
|
(root / name).mkdir()
|
||||||
|
config = root / "client.toml"
|
||||||
|
config.write_text(
|
||||||
|
f'''client_id = "cache-1"
|
||||||
|
display_name = "Cache 1"
|
||||||
|
role = "cache"
|
||||||
|
control_endpoint = "ws://control/archive_control"
|
||||||
|
shared_token_file = "{root / "token"}"
|
||||||
|
state_db = "{root / "state.db"}"
|
||||||
|
backup_dir = "{root / "backups"}"
|
||||||
|
|
||||||
|
[qbittorrent]
|
||||||
|
endpoint = "http://qb"
|
||||||
|
username = "admin"
|
||||||
|
password_file = "{root / "qb-password"}"
|
||||||
|
api_root = "/downloads"
|
||||||
|
local_root = "{root / "qb"}"
|
||||||
|
local_path_overrides = {{ "/downloads/fast" = "{root / "qb-fast"}" }}
|
||||||
|
|
||||||
|
[syncthing]
|
||||||
|
endpoint = "http://syncthing"
|
||||||
|
api_key_file = "{root / "syncthing-key"}"
|
||||||
|
api_root = "/sync"
|
||||||
|
local_root = "{root / "sync"}"
|
||||||
|
''',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
output = io.StringIO()
|
||||||
|
with redirect_stdout(output):
|
||||||
|
self.assertEqual(main(["--config", str(config), "--check-config"]), 0)
|
||||||
|
reported = json.loads(output.getvalue())
|
||||||
|
self.assertEqual(
|
||||||
|
[item["root"] for item in reported["filesystems"]],
|
||||||
|
[str(root / "qb"), str(root / "qb-fast"), str(root / "sync")],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -78,6 +78,35 @@ class ConfigTests(unittest.TestCase):
|
|||||||
Path("/local/qb/Sync"),
|
Path("/local/qb/Sync"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_qbittorrent_can_override_a_nested_save_path(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
for name in ("token", "qb-password", "syncthing-key"):
|
||||||
|
path = root / name
|
||||||
|
path.write_text(name, encoding="utf-8")
|
||||||
|
os.chmod(path, 0o600)
|
||||||
|
(root / "qb").mkdir()
|
||||||
|
(root / "fast").mkdir()
|
||||||
|
(root / "sync").mkdir()
|
||||||
|
config_path = root / "client.toml"
|
||||||
|
config_path.write_text(
|
||||||
|
_config(root).replace(
|
||||||
|
f'local_root = "{root / "qb"}"',
|
||||||
|
f'local_root = "{root / "qb"}"\n'
|
||||||
|
"local_path_overrides = { \"/downloads/fast\" = "
|
||||||
|
f'"{root / "fast"}" }}',
|
||||||
|
1,
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
config = ClientConfig.load(config_path)
|
||||||
|
self.assertEqual(
|
||||||
|
config.qbittorrent.roots.api_to_local(
|
||||||
|
"/downloads/fast/resource/file.bin"
|
||||||
|
),
|
||||||
|
root / "fast/resource/file.bin",
|
||||||
|
)
|
||||||
|
|
||||||
def test_endpoint_scheme_and_job_keys_are_strict(self):
|
def test_endpoint_scheme_and_job_keys_are_strict(self):
|
||||||
with tempfile.TemporaryDirectory() as directory:
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
root = Path(directory)
|
root = Path(directory)
|
||||||
|
|||||||
@@ -50,6 +50,26 @@ class DeploymentPreflightTests(unittest.TestCase):
|
|||||||
"/data/qb/.archive-control-routes",
|
"/data/qb/.archive-control-routes",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_qb_override_must_map_to_the_same_host_path(self):
|
||||||
|
client = {"Mounts": [
|
||||||
|
{"Type": "bind", "Source": "/srv/fast", "Destination": "/data/fast"},
|
||||||
|
]}
|
||||||
|
qbittorrent = {"Mounts": [
|
||||||
|
{"Type": "bind", "Source": "/srv/fast", "Destination": "/downloads/fast"},
|
||||||
|
]}
|
||||||
|
preflight.require_qb_override_mappings(
|
||||||
|
client, qbittorrent,
|
||||||
|
{"local_path_overrides": {"/downloads/fast": "/data/fast"}},
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(preflight.CheckFailure, "host paths differ"):
|
||||||
|
preflight.require_qb_override_mappings(
|
||||||
|
client,
|
||||||
|
{"Mounts": [{
|
||||||
|
"Type": "bind", "Source": "/other", "Destination": "/downloads/fast",
|
||||||
|
}]},
|
||||||
|
{"local_path_overrides": {"/downloads/fast": "/data/fast"}},
|
||||||
|
)
|
||||||
|
|
||||||
def test_runs_hardlink_probe_with_qb_and_route_roots(self):
|
def test_runs_hardlink_probe_with_qb_and_route_roots(self):
|
||||||
completed = __import__("subprocess").CompletedProcess(
|
completed = __import__("subprocess").CompletedProcess(
|
||||||
args=[], returncode=0, stdout="hard-link staging probe passed\n", stderr=""
|
args=[], returncode=0, stdout="hard-link staging probe passed\n", stderr=""
|
||||||
|
|||||||
+47
-7
@@ -1,5 +1,7 @@
|
|||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
|
from dataclasses import replace
|
||||||
|
from pathlib import PurePosixPath
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from archive_clients.eviction import (
|
from archive_clients.eviction import (
|
||||||
@@ -86,7 +88,7 @@ class EvictionTests(unittest.TestCase):
|
|||||||
job_id="job-1",
|
job_id="job-1",
|
||||||
resource=evicted,
|
resource=evicted,
|
||||||
selected_indices=[0, 1],
|
selected_indices=[0, 1],
|
||||||
qb_root=self.root,
|
content_root=self.root,
|
||||||
store=self.store,
|
store=self.store,
|
||||||
)
|
)
|
||||||
remove_qb_entry(
|
remove_qb_entry(
|
||||||
@@ -97,9 +99,9 @@ class EvictionTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
result = safe_unlink(
|
result = safe_unlink(
|
||||||
job_id="job-1",
|
job_id="job-1",
|
||||||
qb_root=self.root,
|
|
||||||
qbittorrent=qb,
|
qbittorrent=qb,
|
||||||
store=self.store,
|
store=self.store,
|
||||||
|
resource_root=lambda _: self.root,
|
||||||
)
|
)
|
||||||
self.assertTrue((self.root / "tree/shared.bin").exists())
|
self.assertTrue((self.root / "tree/shared.bin").exists())
|
||||||
self.assertFalse((self.root / "tree/owned.bin").exists())
|
self.assertFalse((self.root / "tree/owned.bin").exists())
|
||||||
@@ -121,7 +123,7 @@ class EvictionTests(unittest.TestCase):
|
|||||||
job_id="job-1",
|
job_id="job-1",
|
||||||
resource=evicted,
|
resource=evicted,
|
||||||
selected_indices=[0],
|
selected_indices=[0],
|
||||||
qb_root=self.root,
|
content_root=self.root,
|
||||||
store=self.store,
|
store=self.store,
|
||||||
)
|
)
|
||||||
path.unlink()
|
path.unlink()
|
||||||
@@ -134,9 +136,9 @@ class EvictionTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
result = safe_unlink(
|
result = safe_unlink(
|
||||||
job_id="job-1",
|
job_id="job-1",
|
||||||
qb_root=self.root,
|
|
||||||
qbittorrent=qb,
|
qbittorrent=qb,
|
||||||
store=self.store,
|
store=self.store,
|
||||||
|
resource_root=lambda _: self.root,
|
||||||
)
|
)
|
||||||
self.assertTrue(path.exists())
|
self.assertTrue(path.exists())
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
@@ -152,7 +154,7 @@ class EvictionTests(unittest.TestCase):
|
|||||||
job_id="job-1",
|
job_id="job-1",
|
||||||
resource=evicted,
|
resource=evicted,
|
||||||
selected_indices=[0],
|
selected_indices=[0],
|
||||||
qb_root=self.root,
|
content_root=self.root,
|
||||||
store=self.store,
|
store=self.store,
|
||||||
)
|
)
|
||||||
remove_qb_entry(
|
remove_qb_entry(
|
||||||
@@ -163,19 +165,57 @@ class EvictionTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
first = safe_unlink(
|
first = safe_unlink(
|
||||||
job_id="job-1",
|
job_id="job-1",
|
||||||
qb_root=self.root,
|
|
||||||
qbittorrent=qb,
|
qbittorrent=qb,
|
||||||
store=self.store,
|
store=self.store,
|
||||||
|
resource_root=lambda _: self.root,
|
||||||
)
|
)
|
||||||
second = safe_unlink(
|
second = safe_unlink(
|
||||||
job_id="job-1",
|
job_id="job-1",
|
||||||
qb_root=self.root,
|
|
||||||
qbittorrent=qb,
|
qbittorrent=qb,
|
||||||
store=self.store,
|
store=self.store,
|
||||||
|
resource_root=lambda _: self.root,
|
||||||
)
|
)
|
||||||
self.assertEqual(first, second)
|
self.assertEqual(first, second)
|
||||||
self.assertEqual(qb.deleted, ["a" * 40])
|
self.assertEqual(qb.deleted, ["a" * 40])
|
||||||
|
|
||||||
|
def test_nested_save_path_uses_its_own_root_and_not_a_same_named_peer(self):
|
||||||
|
nested = self.root / "Downloading"
|
||||||
|
nested.mkdir()
|
||||||
|
evicted = replace(
|
||||||
|
normalized("a" * 40, ["resource/file.bin"]),
|
||||||
|
save_path=PurePosixPath("/downloads/Downloading"),
|
||||||
|
)
|
||||||
|
peer = replace(
|
||||||
|
normalized("b" * 40, ["resource/file.bin"]),
|
||||||
|
save_path=PurePosixPath("/downloads"),
|
||||||
|
)
|
||||||
|
nested_file = nested / "resource/file.bin"
|
||||||
|
nested_file.parent.mkdir()
|
||||||
|
nested_file.write_bytes(b"x" * evicted.files[0].logical_bytes)
|
||||||
|
root_file = self.root / "resource/file.bin"
|
||||||
|
root_file.parent.mkdir()
|
||||||
|
root_file.write_bytes(b"x" * peer.files[0].logical_bytes)
|
||||||
|
qb = FakeQB(evicted, [peer])
|
||||||
|
|
||||||
|
verify_and_snapshot(
|
||||||
|
job_id="job-1", resource=evicted, selected_indices=[0],
|
||||||
|
content_root=nested, store=self.store,
|
||||||
|
)
|
||||||
|
remove_qb_entry(
|
||||||
|
job_id="job-1", torrent_hash="a" * 40,
|
||||||
|
qbittorrent=qb, store=self.store,
|
||||||
|
)
|
||||||
|
safe_unlink(
|
||||||
|
job_id="job-1", qbittorrent=qb, store=self.store,
|
||||||
|
resource_root=lambda item: (
|
||||||
|
nested
|
||||||
|
if item.save_path == PurePosixPath("/downloads/Downloading")
|
||||||
|
else self.root
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertFalse(nested_file.exists())
|
||||||
|
self.assertTrue(root_file.exists())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
+64
-3
@@ -4,11 +4,13 @@ import stat
|
|||||||
import tempfile
|
import tempfile
|
||||||
import threading
|
import threading
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from dataclasses import replace
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
from unittest.mock import Mock, patch
|
from unittest.mock import Mock, patch
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from archive_clients.bencode import encode
|
from archive_clients.bencode import encode
|
||||||
|
from archive_clients.config import RootMapping
|
||||||
from archive_clients.jobs import (
|
from archive_clients.jobs import (
|
||||||
ClientJobExecutor,
|
ClientJobExecutor,
|
||||||
JobExecutionError,
|
JobExecutionError,
|
||||||
@@ -246,6 +248,55 @@ class ClientJobHappyPathTests(unittest.TestCase):
|
|||||||
content=b"x" * 4096,
|
content=b"x" * 4096,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_nested_qb_save_path_stages_from_mapped_subdirectory(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
self._run_transfer(
|
||||||
|
Path(directory),
|
||||||
|
job_pb2.JOB_OPERATION_UNARCHIVE,
|
||||||
|
source_save_path="/downloads/Downloading",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_qb_save_path_preflight_rejects_unmapped_path(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
executor = ClientJobExecutor(
|
||||||
|
client_id="cache-1", qbittorrent=Mock(),
|
||||||
|
store=ClientStore(root / "state.db"), qb_root=root,
|
||||||
|
qb_api_root=PurePosixPath("/downloads"),
|
||||||
|
route_path=lambda _: root, syncthing_transport=Mock(),
|
||||||
|
sparse_supported=True,
|
||||||
|
)
|
||||||
|
resource = NormalizedResource(
|
||||||
|
resource_pb2.ResourceSummary(), (), Mock(),
|
||||||
|
save_path=PurePosixPath("/outside"),
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(JobExecutionError, "outside"):
|
||||||
|
executor._resource_root(resource)
|
||||||
|
|
||||||
|
def test_qb_save_path_override_uses_its_dedicated_local_mount(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
primary = root / "primary"
|
||||||
|
override = root / "override"
|
||||||
|
primary.mkdir()
|
||||||
|
override.mkdir()
|
||||||
|
executor = ClientJobExecutor(
|
||||||
|
client_id="cache-1", qbittorrent=Mock(),
|
||||||
|
store=ClientStore(root / "state.db"), qb_root=primary,
|
||||||
|
qb_api_root=PurePosixPath("/downloads"),
|
||||||
|
qb_roots=RootMapping(
|
||||||
|
PurePosixPath("/downloads"), primary,
|
||||||
|
((PurePosixPath("/downloads/slow"), override),),
|
||||||
|
),
|
||||||
|
route_path=lambda _: root, syncthing_transport=Mock(),
|
||||||
|
sparse_supported=True,
|
||||||
|
)
|
||||||
|
resource = NormalizedResource(
|
||||||
|
resource_pb2.ResourceSummary(), (), Mock(),
|
||||||
|
save_path=PurePosixPath("/downloads/slow"),
|
||||||
|
)
|
||||||
|
self.assertEqual(executor._resource_root(resource), override)
|
||||||
|
|
||||||
def test_mount_boundary_requires_copy_space_even_with_same_device(self):
|
def test_mount_boundary_requires_copy_space_even_with_same_device(self):
|
||||||
with tempfile.TemporaryDirectory() as directory:
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
root = Path(directory)
|
root = Path(directory)
|
||||||
@@ -390,6 +441,7 @@ class ClientJobHappyPathTests(unittest.TestCase):
|
|||||||
source_stage_free_bytes: int | None = None,
|
source_stage_free_bytes: int | None = None,
|
||||||
content: bytes = b"archive-control-happy-path",
|
content: bytes = b"archive-control-happy-path",
|
||||||
syncthing: CompleteSyncthing | None = None,
|
syncthing: CompleteSyncthing | None = None,
|
||||||
|
source_save_path: str = "/downloads",
|
||||||
):
|
):
|
||||||
source_root = root / "source"
|
source_root = root / "source"
|
||||||
target_root = root / "target"
|
target_root = root / "target"
|
||||||
@@ -397,7 +449,12 @@ class ClientJobHappyPathTests(unittest.TestCase):
|
|||||||
source_root.mkdir()
|
source_root.mkdir()
|
||||||
target_root.mkdir()
|
target_root.mkdir()
|
||||||
route_root.mkdir()
|
route_root.mkdir()
|
||||||
(source_root / "fixture.bin").write_bytes(content)
|
save_relative = PurePosixPath(source_save_path).relative_to(
|
||||||
|
PurePosixPath("/downloads")
|
||||||
|
)
|
||||||
|
source_content_root = source_root.joinpath(*save_relative.parts)
|
||||||
|
source_content_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
(source_content_root / "fixture.bin").write_bytes(content)
|
||||||
info = {
|
info = {
|
||||||
b"length": len(content),
|
b"length": len(content),
|
||||||
b"name": b"fixture.bin",
|
b"name": b"fixture.bin",
|
||||||
@@ -411,6 +468,7 @@ class ClientJobHappyPathTests(unittest.TestCase):
|
|||||||
"hash": torrent_hash,
|
"hash": torrent_hash,
|
||||||
"name": "fixture.bin",
|
"name": "fixture.bin",
|
||||||
"state": "uploading",
|
"state": "uploading",
|
||||||
|
"save_path": source_save_path,
|
||||||
},
|
},
|
||||||
[{
|
[{
|
||||||
"index": 0,
|
"index": 0,
|
||||||
@@ -454,8 +512,11 @@ class ClientJobHappyPathTests(unittest.TestCase):
|
|||||||
source_qb = Mock()
|
source_qb = Mock()
|
||||||
source_qb.get_resource.return_value = resource
|
source_qb.get_resource.return_value = resource
|
||||||
target_qb = Mock()
|
target_qb = Mock()
|
||||||
|
target_resource = replace(
|
||||||
|
resource, save_path=PurePosixPath("/downloads")
|
||||||
|
)
|
||||||
target_qb.get_resource.side_effect = [
|
target_qb.get_resource.side_effect = [
|
||||||
None, None, resource, resource,
|
None, None, target_resource, target_resource,
|
||||||
]
|
]
|
||||||
syncthing = syncthing or CompleteSyncthing()
|
syncthing = syncthing or CompleteSyncthing()
|
||||||
source = ClientJobExecutor(
|
source = ClientJobExecutor(
|
||||||
|
|||||||
@@ -55,6 +55,7 @@ class QBittorrentReaderTests(unittest.TestCase):
|
|||||||
b"Ok.",
|
b"Ok.",
|
||||||
json.dumps([{
|
json.dumps([{
|
||||||
"hash": torrent_hash, "name": "a.txt", "state": "uploading",
|
"hash": torrent_hash, "name": "a.txt", "state": "uploading",
|
||||||
|
"save_path": "/downloads",
|
||||||
}]).encode(),
|
}]).encode(),
|
||||||
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
|
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
|
||||||
torrent_bytes,
|
torrent_bytes,
|
||||||
@@ -102,6 +103,7 @@ class QBittorrentReaderTests(unittest.TestCase):
|
|||||||
b"Ok.",
|
b"Ok.",
|
||||||
json.dumps([{
|
json.dumps([{
|
||||||
"hash": qb_hash, "name": "a.txt", "state": "uploading",
|
"hash": qb_hash, "name": "a.txt", "state": "uploading",
|
||||||
|
"save_path": "/downloads",
|
||||||
}]).encode(),
|
}]).encode(),
|
||||||
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
|
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
|
||||||
torrent_bytes,
|
torrent_bytes,
|
||||||
@@ -157,6 +159,7 @@ class QBittorrentReaderTests(unittest.TestCase):
|
|||||||
"infohash_v2": v2_hash,
|
"infohash_v2": v2_hash,
|
||||||
"name": "a.txt",
|
"name": "a.txt",
|
||||||
"state": "uploading",
|
"state": "uploading",
|
||||||
|
"save_path": "/downloads",
|
||||||
}]).encode(),
|
}]).encode(),
|
||||||
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
|
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
|
||||||
torrent_bytes,
|
torrent_bytes,
|
||||||
@@ -217,9 +220,9 @@ class QBittorrentReaderTests(unittest.TestCase):
|
|||||||
malformed_bytes, malformed_hash = torrent("broken.txt", b"bad")
|
malformed_bytes, malformed_hash = torrent("broken.txt", b"bad")
|
||||||
second_bytes, second_hash = torrent("second.txt", b"two")
|
second_bytes, second_hash = torrent("second.txt", b"two")
|
||||||
torrents = [
|
torrents = [
|
||||||
{"hash": first_hash, "name": "first.txt", "state": "uploading"},
|
{"hash": first_hash, "name": "first.txt", "state": "uploading", "save_path": "/downloads"},
|
||||||
{"hash": malformed_hash, "name": "broken.txt", "state": "stalledUP"},
|
{"hash": malformed_hash, "name": "broken.txt", "state": "stalledUP", "save_path": "/downloads"},
|
||||||
{"hash": second_hash, "name": "second.txt", "state": "uploading"},
|
{"hash": second_hash, "name": "second.txt", "state": "uploading", "save_path": "/downloads"},
|
||||||
]
|
]
|
||||||
responses = [
|
responses = [
|
||||||
b"Ok.", json.dumps(torrents).encode(),
|
b"Ok.", json.dumps(torrents).encode(),
|
||||||
|
|||||||
+22
-1
@@ -40,6 +40,7 @@ class ResourceTests(unittest.TestCase):
|
|||||||
"hash": decoded.info_hash_v2_hex,
|
"hash": decoded.info_hash_v2_hex,
|
||||||
"name": "v2.bin",
|
"name": "v2.bin",
|
||||||
"state": "stoppedUP",
|
"state": "stoppedUP",
|
||||||
|
"save_path": "/downloads",
|
||||||
},
|
},
|
||||||
[{
|
[{
|
||||||
"index": 0,
|
"index": 0,
|
||||||
@@ -91,6 +92,7 @@ class ResourceTests(unittest.TestCase):
|
|||||||
"hash": decoded.info_hash_v1_hex,
|
"hash": decoded.info_hash_v1_hex,
|
||||||
"name": "resource",
|
"name": "resource",
|
||||||
"state": "stoppedUP",
|
"state": "stoppedUP",
|
||||||
|
"save_path": "/downloads/nested",
|
||||||
},
|
},
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
@@ -106,6 +108,7 @@ class ResourceTests(unittest.TestCase):
|
|||||||
observed,
|
observed,
|
||||||
)
|
)
|
||||||
summary = normalized.summary
|
summary = normalized.summary
|
||||||
|
self.assertEqual(normalized.save_path.as_posix(), "/downloads/nested")
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
summary.runtime_state, resource_pb2.TORRENT_RUNTIME_STATE_STOPPED
|
summary.runtime_state, resource_pb2.TORRENT_RUNTIME_STATE_STOPPED
|
||||||
)
|
)
|
||||||
@@ -139,6 +142,7 @@ class ResourceTests(unittest.TestCase):
|
|||||||
"hash": torrent_hash,
|
"hash": torrent_hash,
|
||||||
"name": "with-padding",
|
"name": "with-padding",
|
||||||
"state": "stalledUP",
|
"state": "stalledUP",
|
||||||
|
"save_path": "/downloads",
|
||||||
},
|
},
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
@@ -169,7 +173,7 @@ class ResourceTests(unittest.TestCase):
|
|||||||
metainfo = encode({b"info": info})
|
metainfo = encode({b"info": info})
|
||||||
torrent = {
|
torrent = {
|
||||||
"hash": hashlib.sha1(encode(info)).hexdigest(),
|
"hash": hashlib.sha1(encode(info)).hexdigest(),
|
||||||
"name": "renamed", "state": "uploading",
|
"name": "renamed", "state": "uploading", "save_path": "/downloads",
|
||||||
}
|
}
|
||||||
renamed = normalize_resource(torrent, [{
|
renamed = normalize_resource(torrent, [{
|
||||||
"index": 0, "name": "renamed.txt", "size": 3,
|
"index": 0, "name": "renamed.txt", "size": 3,
|
||||||
@@ -182,6 +186,23 @@ class ResourceTests(unittest.TestCase):
|
|||||||
"progress": 1.0, "priority": 1,
|
"progress": 1.0, "priority": 1,
|
||||||
}], metainfo)
|
}], metainfo)
|
||||||
|
|
||||||
|
def test_missing_or_out_of_shape_save_path_is_rejected(self):
|
||||||
|
info = {
|
||||||
|
b"length": 3, b"name": b"a.txt", b"piece length": 16384,
|
||||||
|
b"pieces": b"x" * 20,
|
||||||
|
}
|
||||||
|
metainfo = encode({b"info": info})
|
||||||
|
torrent = {
|
||||||
|
"hash": hashlib.sha1(encode(info)).hexdigest(),
|
||||||
|
"name": "a.txt", "state": "uploading", "save_path": "relative",
|
||||||
|
}
|
||||||
|
with self.assertRaisesRegex(ResourceError, "save path is unsafe"):
|
||||||
|
normalize_resource(torrent, [{
|
||||||
|
"index": 0, "name": "a.txt", "size": 3,
|
||||||
|
"completed": 3, "priority": 1,
|
||||||
|
}], metainfo)
|
||||||
|
|
||||||
|
|
||||||
def test_noncanonical_bencode_is_rejected(self):
|
def test_noncanonical_bencode_is_rejected(self):
|
||||||
with self.assertRaisesRegex(BencodeError, "unsorted"):
|
with self.assertRaisesRegex(BencodeError, "unsorted"):
|
||||||
decode_metainfo(b"d4:infod1:b1:x1:a1:yee")
|
decode_metainfo(b"d4:infod1:b1:x1:a1:yee")
|
||||||
|
|||||||
Reference in New Issue
Block a user