diff --git a/e2e/README.md b/e2e/README.md index f3d8147..3c969b5 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -52,7 +52,7 @@ restores the target root's original mode even on an aborted run. The scheduler pause/resume endpoints exist only on the loopback test adapter; normal control and Telegram orchestration always use the automatic scheduler. -Set `E2E_RUN_ADVERSARIAL=1` to run the deferred Phase 8 v2/fault tranche in +Set `E2E_RUN_ADVERSARIAL=1` to run the Phase 8 v2/fault tranche in one pass. The scenario performs archive → eviction → unarchive round trips for a pure-v2 torrent and a hybrid torrent, checking that hybrid placements retain both identities. It then: @@ -69,8 +69,8 @@ both identities. It then: reordered convergence and duplicate idempotency over the real WebSocket. The adversarial script restores the normal archive-2 client configuration and -automatic scheduler through an exit trap. This tranche is intentionally -checked in before its first execution; run it later with: +automatic scheduler through an exit trap. It passed from a clean five-project +topology on 2026-07-23. Run it with: ```bash E2E_RUN_ADVERSARIAL=1 \ diff --git a/e2e/scenarios/adversarial_jobs.py b/e2e/scenarios/adversarial_jobs.py index 0c91f6e..3f8a64f 100755 --- a/e2e/scenarios/adversarial_jobs.py +++ b/e2e/scenarios/adversarial_jobs.py @@ -101,6 +101,12 @@ def command_mark_stalled(args: argparse.Namespace) -> None: deadline = time.monotonic() + args.timeout last = None while time.monotonic() < deadline: + current = get_json(f"{CONTROL}/jobs/{args.job_id}") + if current["state"] == "JOB_STATE_STALLED": + args.state = "JOB_STATE_STALLED" + args.minimum_sequence = 1 + command_wait_state(args) + return outcomes = post_json(f"{CONTROL}/scheduler/advance", {}) last = outcomes if any( diff --git a/e2e/scripts/phase8-adversarial.sh b/e2e/scripts/phase8-adversarial.sh index e638859..09ef5a3 100755 --- a/e2e/scripts/phase8-adversarial.sh +++ b/e2e/scripts/phase8-adversarial.sh @@ -3,12 +3,13 @@ set -euo pipefail source "$(dirname "$0")/lib.sh" -v2_hash=8650a1a074039673ce156d4fdede8e38051108ddc90936eea818a20be2de5bfc +v2_hash=3a22a96fa77fb2617b98ea276c8d720d2b82b97a1a463e843d38c9def4f814fa v2_qb_prefix=${v2_hash:0:40} -v2_torrent=ZDQ6aW5mb2Q5OmZpbGUgdHJlZWQwOmQ2Omxlbmd0aGkxOGUxMTpwaWVjZXMgcm9vdDMyOkzwzP0tijSmHJ/EFMYDoUaQHYtd6SNXTmo3yoKC8M+SZWUxMjptZXRhIHZlcnNpb25pMmU0Om5hbWU2OnYyLmJpbjEyOnBpZWNlIGxlbmd0aGkxNjM4NGVlZQ== -hybrid_v1=b77eea6da964b9661e082d567218ddf26609af56 -hybrid_v2=a7ca521665677a5bdace1bd4be32dbe0e43e25a3c578bb5e98efc76fb2bd9eeb -hybrid_torrent=ZDQ6aW5mb2Q5OmZpbGUgdHJlZWQwOmQ2Omxlbmd0aGkyMmUxMTpwaWVjZXMgcm9vdDMyOnAzSygNaQ3iIRKqDJWOXW4HIsea/KNFhE1SN+3/qvJSZWU2Omxlbmd0aGkyMmUxMjptZXRhIHZlcnNpb25pMmU0Om5hbWUxMDpoeWJyaWQuYmluMTI6cGllY2UgbGVuZ3RoaTE2Mzg0ZTY6cGllY2VzMjA6xQCrcqTOQ+LNUqZJC7p47gKZ/GhlZQ== +v2_torrent=ZDQ6aW5mb2Q5OmZpbGUgdHJlZWQ2OnYyLmJpbmQwOmQ2Omxlbmd0aGkxOGUxMTpwaWVjZXMgcm9vdDMyOryYO9v2kD7tZAErfryevFyLb0HWrdRyIk3CnxEgVd56ZWVlMTI6bWV0YSB2ZXJzaW9uaTJlNDpuYW1lNjp2Mi5iaW4xMjpwaWVjZSBsZW5ndGhpMTYzODRlZWU= +hybrid_v1=41ea01236373d80fc64f390a73ef4924ac8d2904 +hybrid_v2=d341089faab562d2c722821ec55a92b5a71578a94f2fa4cd5f849e640dc02b42 +hybrid_qb_prefix=${hybrid_v2:0:40} +hybrid_torrent=ZDQ6aW5mb2Q5OmZpbGUgdHJlZWQxMDpoeWJyaWQuYmluZDA6ZDY6bGVuZ3RoaTIyZTExOnBpZWNlcyByb290MzI6cQSAen8RmDzTs/9c24a6rb/7z0N8MjVuzXUvDbHywddlZWU2Omxlbmd0aGkyMmUxMjptZXRhIHZlcnNpb25pMmU0Om5hbWUxMDpoeWJyaWQuYmluMTI6cGllY2UgbGVuZ3RoaTE2Mzg0ZTY6cGllY2VzMjA6xQCrcqTOQ+LNUqZJC7p47gKZ/GhlZQ== stall_hash=330be0cb7c2201135a2de63b28e77993745ff688 stall_torrent=ZDQ6aW5mb2Q2Omxlbmd0aGkzMWU0Om5hbWUxMTpmaXh0dXJlLmJpbjEyOnBpZWNlIGxlbmd0aGkxNjM4NGU2OnBpZWNlczIwOlFlB05MtIUU4oem2MNz5LJW/GI6ZWU= @@ -46,23 +47,37 @@ install_torrent() { local name=$2 local encoded=$3 compose_node "$node" exec -T qbittorrent /bin/sh -c \ - "printf '%s' '$encoded' | base64 -d > /tmp/phase8.torrent" - compose_node "$node" exec -T qbittorrent curl -fsS \ - -X POST http://127.0.0.1:8080/api/v2/torrents/add \ - -F torrents=@/tmp/phase8.torrent \ - -F savepath=/downloads \ - -F stopped=true >/dev/null + "printf '%s' '$encoded' | base64 -d > /tmp/phase8.torrent" \ + || return + local attempt + for attempt in 1 2 3; do + if compose_node "$node" exec -T qbittorrent curl -fsS \ + -X POST http://127.0.0.1:8080/api/v2/torrents/add \ + -F torrents=@/tmp/phase8.torrent \ + -F savepath=/downloads \ + -F stopped=true >/dev/null; then + break + fi + if (( attempt == 3 )); then + return 1 + fi + sleep "$attempt" + done local qb_hash - qb_hash=$(control_adversarial qb-hash \ - --endpoint "$(qb_endpoint "$node")" \ - --name "$name") + qb_hash=$( + control_adversarial qb-hash \ + --endpoint "$(qb_endpoint "$node")" \ + --name "$name" + ) || return compose_node "$node" exec -T qbittorrent curl -fsS \ -X POST http://127.0.0.1:8080/api/v2/torrents/recheck \ - --data-urlencode "hashes=$qb_hash" >/dev/null + --data-urlencode "hashes=$qb_hash" >/dev/null \ + || return control_complex assert-qb \ --endpoint "$(qb_endpoint "$node")" \ --info-hash "$qb_hash" \ - --selected '0' >/dev/null + --selected '0' >/dev/null \ + || return printf '%s' "$qb_hash" } @@ -99,7 +114,8 @@ trap restore_test_environment EXIT for node in cache-1 cache-2 archive-1 archive-2; do for hash in \ - "$v2_hash" "$v2_qb_prefix" "$hybrid_v1" "$hybrid_v2" "$stall_hash"; do + "$v2_hash" "$v2_qb_prefix" "$hybrid_v1" "$hybrid_v2" \ + "$hybrid_qb_prefix" "$stall_hash"; do delete_hash "$node" "$hash" done compose_node "$node" exec -T qbittorrent \ @@ -192,6 +208,7 @@ stall_job=$(control_complex drive \ --info-hash "$stall_hash" \ --through source_stage) compose_node archive-2 stop client +restore_archive2=true control_adversarial wait-client \ --client-id archive-2 \ --no-connected @@ -204,6 +221,7 @@ control_adversarial mark-stalled \ --minimum-overall 0.2 compose_node archive-2 start client control_adversarial wait-client --client-id archive-2 +restore_archive2=false control_complex resume --job-id "$stall_job" assert_qb_by_name archive-2 fixture.bin diff --git a/src/archive_clients/bencode.py b/src/archive_clients/bencode.py index c00cbdf..5640d5e 100644 --- a/src/archive_clients/bencode.py +++ b/src/archive_clients/bencode.py @@ -174,7 +174,7 @@ def _v1_files(info: dict[bytes, Any]) -> list[MetaFile]: def _v2_files(info: dict[bytes, Any]) -> list[MetaFile]: - name = _component(info.get(b"name.utf-8", info.get(b"name"))) + _component(info.get(b"name.utf-8", info.get(b"name"))) tree = info.get(b"file tree") if not isinstance(tree, dict): raise BencodeError("v2 metainfo has no file tree") @@ -186,7 +186,7 @@ def _v2_files(info: dict[bytes, Any]) -> list[MetaFile]: if not isinstance(leaf, dict): raise BencodeError("invalid v2 file leaf") result.append(MetaFile( - "/".join([name] + components), + "/".join(components), _length(leaf.get(b"length")), _padding(leaf), )) diff --git a/src/archive_clients/eviction.py b/src/archive_clients/eviction.py index faeda58..e9ca22e 100644 --- a/src/archive_clients/eviction.py +++ b/src/archive_clients/eviction.py @@ -79,8 +79,9 @@ def remove_qb_entry( marker = store.get_job_artifact(job_id, "qb-entry-removed") if marker is not None: return - if qbittorrent.get_resource(torrent_hash) is not None: - qbittorrent.delete_entry(torrent_hash) + resource = qbittorrent.get_resource(torrent_hash) + if resource is not None: + qbittorrent.delete_entry(resource.summary.qb_torrent_id) store.put_job_artifact( job_id, "qb-entry-removed", {"torrent_hash": torrent_hash} ) diff --git a/src/archive_clients/jobs.py b/src/archive_clients/jobs.py index 40feec3..e5450d5 100644 --- a/src/archive_clients/jobs.py +++ b/src/archive_clients/jobs.py @@ -686,6 +686,12 @@ class ClientJobExecutor: self.qb_api_root.as_posix(), info_hash, ) + resource = self.qbittorrent.get_resource(info_hash) + if resource is None: + raise JobExecutionError( + "added qBittorrent resource did not become visible" + ) + qb_torrent_id = resource.summary.qb_torrent_id target_union = _union_selection( definition.transfer.target_baseline_files, definition.transfer.transfer_delta_files, @@ -694,9 +700,11 @@ class ClientJobExecutor: total_files = len( decode_message_metainfo(metainfo_path).files ) - self.qbittorrent.set_selection(info_hash, selected, total_files) + self.qbittorrent.set_selection( + qb_torrent_id, selected, total_files + ) self.qbittorrent.recheck_and_wait( - info_hash, + qb_torrent_id, selected, timeout=self.verification_timeout, poll_interval=self.poll_interval, @@ -708,7 +716,7 @@ class ClientJobExecutor: ), ) if should_start: - self.qbittorrent.start(info_hash) + self.qbittorrent.start(qb_torrent_id) verified = self.qbittorrent.get_resource(info_hash) if verified is None: raise JobExecutionError( @@ -753,18 +761,19 @@ class ClientJobExecutor: info_hash = _info_hash(definition) present = self.qbittorrent.get_resource(info_hash) if present is not None: + qb_torrent_id = present.summary.qb_torrent_id if baseline.get("present"): selected = baseline.get("selected_file_indices") if isinstance(selected, list) and selected: self.qbittorrent.set_selection( - info_hash, selected, len(present.files) + qb_torrent_id, selected, len(present.files) ) if baseline.get("stopped"): - self.qbittorrent.stop(info_hash) + self.qbittorrent.stop(qb_torrent_id) else: - self.qbittorrent.start(info_hash) + self.qbittorrent.start(qb_torrent_id) else: - self.qbittorrent.delete_entry(info_hash) + self.qbittorrent.delete_entry(qb_torrent_id) compensate_materialized_files( job_id=definition.job_id, qb_root=self.qb_root, diff --git a/src/archive_clients/qbittorrent.py b/src/archive_clients/qbittorrent.py index 84f7460..bf975b5 100644 --- a/src/archive_clients/qbittorrent.py +++ b/src/archive_clients/qbittorrent.py @@ -76,21 +76,11 @@ class QBittorrentReader: return result def get_resource(self, torrent_hash: str) -> NormalizedResource | None: - torrents = self._json( - "/api/v2/torrents/info", {"hashes": torrent_hash.lower()} - ) - if not isinstance(torrents, list): - raise QBittorrentError("qBittorrent lookup response is invalid") - exact = [ - torrent for torrent in torrents - if isinstance(torrent, dict) - and str(torrent.get("hash", "")).lower() == torrent_hash.lower() - ] - if not exact: + torrent = self._lookup_torrent(torrent_hash) + if torrent is None: return None - if len(exact) != 1: - raise QBittorrentError("qBittorrent lookup is ambiguous") - return self._normalize(exact[0], torrent_hash) + qb_torrent_id = str(torrent["hash"]).lower() + return self._normalize(torrent, qb_torrent_id) def add_stopped( self, @@ -164,6 +154,7 @@ class QBittorrentReader: selected_file_indices: list[int] | tuple[int, ...], total_file_count: int, ) -> None: + torrent_hash = self._require_torrent_hash(torrent_hash) selected = sorted(set(selected_file_indices)) if total_file_count < 1: raise QBittorrentError("torrent file count must be positive") @@ -189,6 +180,7 @@ class QBittorrentReader: ) def stop(self, torrent_hash: str) -> None: + torrent_hash = self._require_torrent_hash(torrent_hash) try: self._post_form( "/api/v2/torrents/stop", {"hashes": torrent_hash} @@ -201,6 +193,7 @@ class QBittorrentReader: ) def start(self, torrent_hash: str) -> None: + torrent_hash = self._require_torrent_hash(torrent_hash) try: self._post_form( "/api/v2/torrents/start", {"hashes": torrent_hash} @@ -234,20 +227,10 @@ class QBittorrentReader: ) def _torrent_is_present(self, torrent_hash: str) -> bool: - torrents = self._json( - "/api/v2/torrents/info", {"hashes": torrent_hash} - ) - if not isinstance(torrents, list): - raise QBittorrentError( - "qBittorrent lookup response is invalid" - ) - return any( - isinstance(item, dict) - and str(item.get("hash", "")).lower() == torrent_hash.lower() - for item in torrents - ) + return self._lookup_torrent(torrent_hash) is not None def delete_entry(self, torrent_hash: str) -> None: + torrent_hash = self._require_torrent_hash(torrent_hash) self._post_form( "/api/v2/torrents/delete", {"hashes": torrent_hash, "deleteFiles": "false"}, @@ -263,6 +246,7 @@ class QBittorrentReader: cancel_check: Callable[[], None] | None = None, progress_callback: Callable[[float], None] | None = None, ) -> RecheckResult: + torrent_hash = self._require_torrent_hash(torrent_hash) selected = tuple(sorted(set(selected_file_indices))) if not selected: raise QBittorrentError("recheck selection cannot be empty") @@ -324,19 +308,55 @@ class QBittorrentReader: time.sleep(min(poll_interval, max(0, deadline - time.monotonic()))) def _torrent_record(self, torrent_hash: str) -> dict[str, Any]: - torrents = self._json( - "/api/v2/torrents/info", {"hashes": torrent_hash} - ) + torrent = self._lookup_torrent(torrent_hash) + if torrent is None: + raise QBittorrentError("qBittorrent target torrent is missing") + return torrent + + def _require_torrent_hash(self, torrent_hash: str) -> str: + requested = torrent_hash.lower() + if len(requested) != 64: + return requested + torrent = self._lookup_torrent(torrent_hash) + if torrent is None: + raise QBittorrentError("qBittorrent target torrent is missing") + return str(torrent["hash"]).lower() + + def _lookup_torrent(self, torrent_hash: str) -> dict[str, Any] | None: + requested = torrent_hash.lower() + # qBittorrent 5 uses the truncated v2 hash as `hash` for both pure-v2 + # and hybrid torrents. Its `hashes` filter does not match a hybrid's + # v1 alias, so identity lookup must inspect all advertised identities. + torrents = self._json("/api/v2/torrents/info") if not isinstance(torrents, list): raise QBittorrentError("qBittorrent lookup response is invalid") exact = [ item for item in torrents if isinstance(item, dict) - and str(item.get("hash", "")).lower() == torrent_hash.lower() + and requested in { + str(item.get("hash", "")).lower(), + str(item.get("infohash_v1", "")).lower(), + str(item.get("infohash_v2", "")).lower(), + } ] - if len(exact) != 1: - raise QBittorrentError("qBittorrent target torrent is missing") - return exact[0] + if len(exact) == 1: + return exact[0] + if len(exact) > 1: + raise QBittorrentError("qBittorrent lookup is ambiguous") + truncated = [ + item for item in torrents + if ( + len(requested) == 64 + and isinstance(item, dict) + and len(str(item.get("hash", ""))) == 40 + and requested.startswith(str(item["hash"]).lower()) + ) + ] + if len(truncated) == 1: + return truncated[0] + if len(truncated) > 1: + raise QBittorrentError("qBittorrent lookup is ambiguous") + return None def _downloaded_bytes(self, torrent_hash: str) -> int: properties = self._json( diff --git a/tests/test_jobs.py b/tests/test_jobs.py index cb51cb6..dbe385e 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -48,7 +48,6 @@ class ClientJobHappyPathTests(unittest.TestCase): sparse_supported=True, free_space_reserve_bytes=100, ) - before = {path.name for path in root.iterdir()} with patch( "archive_clients.jobs.shutil.disk_usage", return_value=Mock(free=109), @@ -57,9 +56,9 @@ class ClientJobHappyPathTests(unittest.TestCase): "109 bytes available, 110 bytes required including reserve", ): executor._require_space(root, 10) - self.assertEqual( + self.assertLessEqual( {path.name for path in root.iterdir()}, - before, + {"client.db", "client.db-wal", "client.db-shm"}, ) def test_archive_and_unarchive_five_step_execution(self): @@ -135,7 +134,9 @@ class ClientJobHappyPathTests(unittest.TestCase): source_qb = Mock() source_qb.get_resource.return_value = resource target_qb = Mock() - target_qb.get_resource.side_effect = [None, None, resource] + target_qb.get_resource.side_effect = [ + None, None, resource, resource, + ] syncthing = CompleteSyncthing() source = ClientJobExecutor( client_id=source_id, diff --git a/tests/test_qbittorrent.py b/tests/test_qbittorrent.py index 041d70a..f5848b1 100644 --- a/tests/test_qbittorrent.py +++ b/tests/test_qbittorrent.py @@ -76,7 +76,114 @@ class QBittorrentReaderTests(unittest.TestCase): resource = QBittorrentReader(config).get_resource(torrent_hash) self.assertIsNotNone(resource) self.assertEqual(resource.summary.resource_id.info_hash_v1_hex, torrent_hash) - self.assertIn("hashes=", opener.calls[1]) + lookup_url = getattr( + opener.calls[1], "full_url", opener.calls[1] + ) + self.assertTrue(lookup_url.endswith("/api/v2/torrents/info")) + + def test_full_v2_lookup_accepts_qbittorrent_truncated_hash(self): + info = { + b"file tree": { + b"a.txt": { + b"": { + b"length": 3, + b"pieces root": hashlib.sha256(b"abc").digest(), + } + } + }, + b"meta version": 2, + b"name": b"a.txt", + b"piece length": 16384, + } + torrent_bytes = encode({b"info": info}) + full_hash = hashlib.sha256(encode(info)).hexdigest() + qb_hash = full_hash[:40] + responses = [ + b"Ok.", + json.dumps([{ + "hash": qb_hash, "name": "a.txt", "state": "uploading", + }]).encode(), + b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]', + torrent_bytes, + ] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + password = root / "password" + password.write_text("secret", encoding="utf-8") + os.chmod(password, 0o600) + config = ServiceConfig( + "http://qb", PurePosixPath("/downloads"), root, + username="admin", password_file=password, + ) + opener = _Opener(responses) + with patch( + "archive_clients.qbittorrent.request.build_opener", + return_value=opener, + ): + resource = QBittorrentReader(config).get_resource(full_hash) + self.assertIsNotNone(resource) + self.assertEqual(resource.summary.qb_torrent_id, qb_hash) + self.assertEqual( + resource.summary.resource_id.info_hash_v2_hex, full_hash + ) + self.assertNotIn("hashes=", opener.calls[1]) + + def test_hybrid_lookup_accepts_v1_alias_of_v2_primary_hash(self): + info = { + b"file tree": { + b"a.txt": { + b"": { + b"length": 3, + b"pieces root": hashlib.sha256(b"abc").digest(), + } + } + }, + b"length": 3, + b"meta version": 2, + b"name": b"a.txt", + b"piece length": 16384, + b"pieces": hashlib.sha1(b"abc").digest(), + } + torrent_bytes = encode({b"info": info}) + encoded_info = encode(info) + v1_hash = hashlib.sha1(encoded_info).hexdigest() + v2_hash = hashlib.sha256(encoded_info).hexdigest() + qb_hash = v2_hash[:40] + responses = [ + b"Ok.", + json.dumps([{ + "hash": qb_hash, + "infohash_v1": v1_hash, + "infohash_v2": v2_hash, + "name": "a.txt", + "state": "uploading", + }]).encode(), + b'[{"index":0,"name":"a.txt","size":3,"progress":1,"priority":1}]', + torrent_bytes, + ] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + password = root / "password" + password.write_text("secret", encoding="utf-8") + os.chmod(password, 0o600) + config = ServiceConfig( + "http://qb", PurePosixPath("/downloads"), root, + username="admin", password_file=password, + ) + opener = _Opener(responses) + with patch( + "archive_clients.qbittorrent.request.build_opener", + return_value=opener, + ): + resource = QBittorrentReader(config).get_resource(v1_hash) + self.assertIsNotNone(resource) + self.assertEqual(resource.summary.qb_torrent_id, qb_hash) + self.assertEqual( + resource.summary.resource_id.info_hash_v1_hex, v1_hash + ) + self.assertEqual( + resource.summary.resource_id.info_hash_v2_hex, v2_hash + ) def test_current_empty_204_login_is_accepted(self): responses = [(b"", 204), b"[]"] diff --git a/tests/test_resources.py b/tests/test_resources.py index e46574f..03ec030 100644 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -15,9 +15,11 @@ class ResourceTests(unittest.TestCase): def test_pure_v2_identity_and_single_file_tree_normalization(self): info = { b"file tree": { - b"": { - b"length": 3, - b"pieces root": b"x" * 32, + b"v2.bin": { + b"": { + b"length": 3, + b"pieces root": b"x" * 32, + }, }, }, b"meta version": 2, @@ -57,8 +59,10 @@ class ResourceTests(unittest.TestCase): def test_hybrid_identity_and_selection_normalization(self): info = { b"file tree": { - b"a.txt": {b"": {b"length": 3}}, - b"b.bin": {b"": {b"attr": b"p", b"length": 5}}, + b"resource": { + b"a.txt": {b"": {b"length": 3}}, + b"b.bin": {b"": {b"attr": b"p", b"length": 5}}, + }, }, b"files": [ {b"length": 3, b"path": [b"a.txt"]},