cabbage/archive-clients
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: resolve per-torrent qb save paths

Browse Source
This commit is contained in:
cabbage
2026-08-14 11:56:06 +00:00
parent ae7175719b
commit a1bf3e4315
17 changed files with 453 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_cli.py
+59
View File
@@ -0,0 +1,59 @@
import io
import json
import os
import tempfile
import unittest
from contextlib import redirect_stdout
from pathlib import Path
from archive_clients.cli import main
class ClientCliTests(unittest.TestCase):
def test_check_config_probes_every_qb_override_root(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
for name in ("token", "qb-password", "syncthing-key"):
path = root / name
path.write_text(name, encoding="utf-8")
os.chmod(path, 0o600)
for name in ("qb", "qb-fast", "sync", "backups"):
(root / name).mkdir()
config = root / "client.toml"
config.write_text(
f'''client_id = "cache-1"
display_name = "Cache 1"
role = "cache"
control_endpoint = "ws://control/archive_control"
shared_token_file = "{root / "token"}"
state_db = "{root / "state.db"}"
backup_dir = "{root / "backups"}"
[qbittorrent]
endpoint = "http://qb"
username = "admin"
password_file = "{root / "qb-password"}"
api_root = "/downloads"
local_root = "{root / "qb"}"
local_path_overrides = {{ "/downloads/fast" = "{root / "qb-fast"}" }}
[syncthing]
endpoint = "http://syncthing"
api_key_file = "{root / "syncthing-key"}"
api_root = "/sync"
local_root = "{root / "sync"}"
''',
encoding="utf-8",
)
output = io.StringIO()
with redirect_stdout(output):
self.assertEqual(main(["--config", str(config), "--check-config"]), 0)
reported = json.loads(output.getvalue())
self.assertEqual(
[item["root"] for item in reported["filesystems"]],
[str(root / "qb"), str(root / "qb-fast"), str(root / "sync")],
)
if __name__ == "__main__":
unittest.main()
tests/test_config.py
+29
View File
@@ -78,6 +78,35 @@ class ConfigTests(unittest.TestCase):
Path("/local/qb/Sync"),
)
def test_qbittorrent_can_override_a_nested_save_path(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
for name in ("token", "qb-password", "syncthing-key"):
path = root / name
path.write_text(name, encoding="utf-8")
os.chmod(path, 0o600)
(root / "qb").mkdir()
(root / "fast").mkdir()
(root / "sync").mkdir()
config_path = root / "client.toml"
config_path.write_text(
_config(root).replace(
f'local_root = "{root / "qb"}"',
f'local_root = "{root / "qb"}"\n'
"local_path_overrides = { \"/downloads/fast\" = "
f'"{root / "fast"}" }}',
1,
),
encoding="utf-8",
)
config = ClientConfig.load(config_path)
self.assertEqual(
config.qbittorrent.roots.api_to_local(
"/downloads/fast/resource/file.bin"
),
root / "fast/resource/file.bin",
)
def test_endpoint_scheme_and_job_keys_are_strict(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
tests/test_deployment_preflight.py
+20
View File
@@ -50,6 +50,26 @@ class DeploymentPreflightTests(unittest.TestCase):
"/data/qb/.archive-control-routes",
)
def test_qb_override_must_map_to_the_same_host_path(self):
client = {"Mounts": [
{"Type": "bind", "Source": "/srv/fast", "Destination": "/data/fast"},
]}
qbittorrent = {"Mounts": [
{"Type": "bind", "Source": "/srv/fast", "Destination": "/downloads/fast"},
]}
preflight.require_qb_override_mappings(
client, qbittorrent,
{"local_path_overrides": {"/downloads/fast": "/data/fast"}},
)
with self.assertRaisesRegex(preflight.CheckFailure, "host paths differ"):
preflight.require_qb_override_mappings(
client,
{"Mounts": [{
"Type": "bind", "Source": "/other", "Destination": "/downloads/fast",
}]},
{"local_path_overrides": {"/downloads/fast": "/data/fast"}},
)
def test_runs_hardlink_probe_with_qb_and_route_roots(self):
completed = __import__("subprocess").CompletedProcess(
args=[], returncode=0, stdout="hard-link staging probe passed\n", stderr=""
tests/test_eviction.py
+47 -7
View File
@@ -1,5 +1,7 @@
import tempfile
import unittest
from dataclasses import replace
from pathlib import PurePosixPath
from pathlib import Path
from archive_clients.eviction import (
@@ -86,7 +88,7 @@ class EvictionTests(unittest.TestCase):
job_id="job-1",
resource=evicted,
selected_indices=[0, 1],
qb_root=self.root,
content_root=self.root,
store=self.store,
)
remove_qb_entry(
@@ -97,9 +99,9 @@ class EvictionTests(unittest.TestCase):
)
result = safe_unlink(
job_id="job-1",
qb_root=self.root,
qbittorrent=qb,
store=self.store,
resource_root=lambda _: self.root,
)
self.assertTrue((self.root / "tree/shared.bin").exists())
self.assertFalse((self.root / "tree/owned.bin").exists())
@@ -121,7 +123,7 @@ class EvictionTests(unittest.TestCase):
job_id="job-1",
resource=evicted,
selected_indices=[0],
qb_root=self.root,
content_root=self.root,
store=self.store,
)
path.unlink()
@@ -134,9 +136,9 @@ class EvictionTests(unittest.TestCase):
)
result = safe_unlink(
job_id="job-1",
qb_root=self.root,
qbittorrent=qb,
store=self.store,
resource_root=lambda _: self.root,
)
self.assertTrue(path.exists())
self.assertEqual(
@@ -152,7 +154,7 @@ class EvictionTests(unittest.TestCase):
job_id="job-1",
resource=evicted,
selected_indices=[0],
qb_root=self.root,
content_root=self.root,
store=self.store,
)
remove_qb_entry(
@@ -163,19 +165,57 @@ class EvictionTests(unittest.TestCase):
)
first = safe_unlink(
job_id="job-1",
qb_root=self.root,
qbittorrent=qb,
store=self.store,
resource_root=lambda _: self.root,
)
second = safe_unlink(
job_id="job-1",
qb_root=self.root,
qbittorrent=qb,
store=self.store,
resource_root=lambda _: self.root,
)
self.assertEqual(first, second)
self.assertEqual(qb.deleted, ["a" * 40])
def test_nested_save_path_uses_its_own_root_and_not_a_same_named_peer(self):
nested = self.root / "Downloading"
nested.mkdir()
evicted = replace(
normalized("a" * 40, ["resource/file.bin"]),
save_path=PurePosixPath("/downloads/Downloading"),
)
peer = replace(
normalized("b" * 40, ["resource/file.bin"]),
save_path=PurePosixPath("/downloads"),
)
nested_file = nested / "resource/file.bin"
nested_file.parent.mkdir()
nested_file.write_bytes(b"x" * evicted.files[0].logical_bytes)
root_file = self.root / "resource/file.bin"
root_file.parent.mkdir()
root_file.write_bytes(b"x" * peer.files[0].logical_bytes)
qb = FakeQB(evicted, [peer])
verify_and_snapshot(
job_id="job-1", resource=evicted, selected_indices=[0],
content_root=nested, store=self.store,
)
remove_qb_entry(
job_id="job-1", torrent_hash="a" * 40,
qbittorrent=qb, store=self.store,
)
safe_unlink(
job_id="job-1", qbittorrent=qb, store=self.store,
resource_root=lambda item: (
nested
if item.save_path == PurePosixPath("/downloads/Downloading")
else self.root
),
)
self.assertFalse(nested_file.exists())
self.assertTrue(root_file.exists())
if __name__ == "__main__":
unittest.main()
tests/test_jobs.py
+64 -3
View File
@@ -4,11 +4,13 @@ import stat
import tempfile
import threading
import unittest
from pathlib import Path
from dataclasses import replace
from pathlib import Path, PurePosixPath
from unittest.mock import Mock, patch
from uuid import uuid4
from archive_clients.bencode import encode
from archive_clients.config import RootMapping
from archive_clients.jobs import (
ClientJobExecutor,
JobExecutionError,
@@ -246,6 +248,55 @@ class ClientJobHappyPathTests(unittest.TestCase):
content=b"x" * 4096,
)
def test_nested_qb_save_path_stages_from_mapped_subdirectory(self):
with tempfile.TemporaryDirectory() as directory:
self._run_transfer(
Path(directory),
job_pb2.JOB_OPERATION_UNARCHIVE,
source_save_path="/downloads/Downloading",
)
def test_qb_save_path_preflight_rejects_unmapped_path(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
executor = ClientJobExecutor(
client_id="cache-1", qbittorrent=Mock(),
store=ClientStore(root / "state.db"), qb_root=root,
qb_api_root=PurePosixPath("/downloads"),
route_path=lambda _: root, syncthing_transport=Mock(),
sparse_supported=True,
)
resource = NormalizedResource(
resource_pb2.ResourceSummary(), (), Mock(),
save_path=PurePosixPath("/outside"),
)
with self.assertRaisesRegex(JobExecutionError, "outside"):
executor._resource_root(resource)
def test_qb_save_path_override_uses_its_dedicated_local_mount(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
primary = root / "primary"
override = root / "override"
primary.mkdir()
override.mkdir()
executor = ClientJobExecutor(
client_id="cache-1", qbittorrent=Mock(),
store=ClientStore(root / "state.db"), qb_root=primary,
qb_api_root=PurePosixPath("/downloads"),
qb_roots=RootMapping(
PurePosixPath("/downloads"), primary,
((PurePosixPath("/downloads/slow"), override),),
),
route_path=lambda _: root, syncthing_transport=Mock(),
sparse_supported=True,
)
resource = NormalizedResource(
resource_pb2.ResourceSummary(), (), Mock(),
save_path=PurePosixPath("/downloads/slow"),
)
self.assertEqual(executor._resource_root(resource), override)
def test_mount_boundary_requires_copy_space_even_with_same_device(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
@@ -390,6 +441,7 @@ class ClientJobHappyPathTests(unittest.TestCase):
source_stage_free_bytes: int | None = None,
content: bytes = b"archive-control-happy-path",
syncthing: CompleteSyncthing | None = None,
source_save_path: str = "/downloads",
):
source_root = root / "source"
target_root = root / "target"
@@ -397,7 +449,12 @@ class ClientJobHappyPathTests(unittest.TestCase):
source_root.mkdir()
target_root.mkdir()
route_root.mkdir()
(source_root / "fixture.bin").write_bytes(content)
save_relative = PurePosixPath(source_save_path).relative_to(
PurePosixPath("/downloads")
)
source_content_root = source_root.joinpath(*save_relative.parts)
source_content_root.mkdir(parents=True, exist_ok=True)
(source_content_root / "fixture.bin").write_bytes(content)
info = {
b"length": len(content),
b"name": b"fixture.bin",
@@ -411,6 +468,7 @@ class ClientJobHappyPathTests(unittest.TestCase):
"hash": torrent_hash,
"name": "fixture.bin",
"state": "uploading",
"save_path": source_save_path,
},
[{
"index": 0,
@@ -454,8 +512,11 @@ class ClientJobHappyPathTests(unittest.TestCase):
source_qb = Mock()
source_qb.get_resource.return_value = resource
target_qb = Mock()
target_resource = replace(
resource, save_path=PurePosixPath("/downloads")
)
target_qb.get_resource.side_effect = [
None, None, resource, resource,
None, None, target_resource, target_resource,
]
syncthing = syncthing or CompleteSyncthing()
source = ClientJobExecutor(
tests/test_qbittorrent.py
+6 -3
View File
@@ -55,6 +55,7 @@ class QBittorrentReaderTests(unittest.TestCase):
b"Ok.",
json.dumps([{
"hash": torrent_hash, "name": "a.txt", "state": "uploading",
"save_path": "/downloads",
}]).encode(),
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
torrent_bytes,
@@ -102,6 +103,7 @@ class QBittorrentReaderTests(unittest.TestCase):
b"Ok.",
json.dumps([{
"hash": qb_hash, "name": "a.txt", "state": "uploading",
"save_path": "/downloads",
}]).encode(),
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
torrent_bytes,
@@ -157,6 +159,7 @@ class QBittorrentReaderTests(unittest.TestCase):
"infohash_v2": v2_hash,
"name": "a.txt",
"state": "uploading",
"save_path": "/downloads",
}]).encode(),
b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]',
torrent_bytes,
@@ -217,9 +220,9 @@ class QBittorrentReaderTests(unittest.TestCase):
malformed_bytes, malformed_hash = torrent("broken.txt", b"bad")
second_bytes, second_hash = torrent("second.txt", b"two")
torrents = [
{"hash": first_hash, "name": "first.txt", "state": "uploading"},
{"hash": malformed_hash, "name": "broken.txt", "state": "stalledUP"},
{"hash": second_hash, "name": "second.txt", "state": "uploading"},
{"hash": first_hash, "name": "first.txt", "state": "uploading", "save_path": "/downloads"},
{"hash": malformed_hash, "name": "broken.txt", "state": "stalledUP", "save_path": "/downloads"},
{"hash": second_hash, "name": "second.txt", "state": "uploading", "save_path": "/downloads"},
]
responses = [
b"Ok.", json.dumps(torrents).encode(),
tests/test_resources.py
+22 -1
View File
@@ -40,6 +40,7 @@ class ResourceTests(unittest.TestCase):
"hash": decoded.info_hash_v2_hex,
"name": "v2.bin",
"state": "stoppedUP",
"save_path": "/downloads",
},
[{
"index": 0,
@@ -91,6 +92,7 @@ class ResourceTests(unittest.TestCase):
"hash": decoded.info_hash_v1_hex,
"name": "resource",
"state": "stoppedUP",
"save_path": "/downloads/nested",
},
[
{
@@ -106,6 +108,7 @@ class ResourceTests(unittest.TestCase):
observed,
)
summary = normalized.summary
self.assertEqual(normalized.save_path.as_posix(), "/downloads/nested")
self.assertEqual(
summary.runtime_state, resource_pb2.TORRENT_RUNTIME_STATE_STOPPED
)
@@ -139,6 +142,7 @@ class ResourceTests(unittest.TestCase):
"hash": torrent_hash,
"name": "with-padding",
"state": "stalledUP",
"save_path": "/downloads",
},
[
{
@@ -169,7 +173,7 @@ class ResourceTests(unittest.TestCase):
metainfo = encode({b"info": info})
torrent = {
"hash": hashlib.sha1(encode(info)).hexdigest(),
"name": "renamed", "state": "uploading",
"name": "renamed", "state": "uploading", "save_path": "/downloads",
}
renamed = normalize_resource(torrent, [{
"index": 0, "name": "renamed.txt", "size": 3,
@@ -182,6 +186,23 @@ class ResourceTests(unittest.TestCase):
"progress": 1.0, "priority": 1,
}], metainfo)
def test_missing_or_out_of_shape_save_path_is_rejected(self):
info = {
b"length": 3, b"name": b"a.txt", b"piece length": 16384,
b"pieces": b"x" * 20,
}
metainfo = encode({b"info": info})
torrent = {
"hash": hashlib.sha1(encode(info)).hexdigest(),
"name": "a.txt", "state": "uploading", "save_path": "relative",
}
with self.assertRaisesRegex(ResourceError, "save path is unsafe"):
normalize_resource(torrent, [{
"index": 0, "name": "a.txt", "size": 3,
"completed": 3, "priority": 1,
}], metainfo)
def test_noncanonical_bencode_is_rejected(self):
with self.assertRaisesRegex(BencodeError, "unsorted"):
decode_metainfo(b"d4:infod1:b1:x1:a1:yee")