cabbage/archive-clients
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: execute durable archive transfers

Browse Source
This commit is contained in:
cabbage
2026-07-23 07:21:19 +00:00
parent 68db3ec4c5
commit 884aed9921
14 changed files with 1618 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_daemon.py
+1 -1
View File
@@ -344,7 +344,7 @@ class DaemonTransportTests(unittest.IsolatedAsyncioTestCase):
observed["rejected"],
(
control_pb2.COMMAND_ACK_STATUS_REJECTED,
common_pb2.ERROR_CODE_UNSUPPORTED,
common_pb2.ERROR_CODE_INVALID_ARGUMENT,
),
)
self.assertEqual(
tests/test_jobs.py
+255
View File
@@ -0,0 +1,255 @@
import hashlib
import tempfile
import unittest
from pathlib import Path
from unittest.mock import Mock, patch
from uuid import uuid4
from archive_clients.bencode import encode
from archive_clients.jobs import ClientJobExecutor, JobExecutionError
from archive_clients.resources import normalize_resource
from archive_clients.state import ClientStore
from archive_control.v1 import control_pb2, job_pb2
class CompleteSyncthing:
def __init__(self):
self.posts = []
def get_json(self, path):
if path.startswith("/rest/db/completion?"):
return {"completion": 100}
if path.startswith("/rest/db/need?"):
return {"progress": [], "queued": [], "rest": []}
raise AssertionError(path)
def post(self, path):
self.posts.append(path)
class ClientJobHappyPathTests(unittest.TestCase):
def test_archive_and_unarchive_five_step_execution(self):
for operation in (
job_pb2.JOB_OPERATION_ARCHIVE,
job_pb2.JOB_OPERATION_UNARCHIVE,
):
with self.subTest(operation=operation), tempfile.TemporaryDirectory() as directory:
self._run_transfer(Path(directory), operation)
def _run_transfer(self, root: Path, operation: int):
source_root = root / "source"
target_root = root / "target"
route_root = root / "route"
source_root.mkdir()
target_root.mkdir()
route_root.mkdir()
content = b"archive-control-happy-path"
(source_root / "fixture.bin").write_bytes(content)
info = {
b"length": len(content),
b"name": b"fixture.bin",
b"piece length": 16384,
b"pieces": hashlib.sha1(content).digest(),
}
metainfo = encode({b"info": info})
torrent_hash = hashlib.sha1(encode(info)).hexdigest()
resource = normalize_resource(
{
"hash": torrent_hash,
"name": "fixture.bin",
"state": "uploading",
},
[{
"index": 0,
"name": "fixture.bin",
"size": len(content),
"completed": len(content),
"priority": 1,
}],
metainfo,
)
source_id, target_id = (
("cache-1", "archive-1")
if operation == job_pb2.JOB_OPERATION_ARCHIVE
else ("archive-1", "cache-1")
)
definition = job_pb2.JobDefinition(
job_id=str(uuid4()),
idempotency_key=str(uuid4()),
operation=operation,
resource_display_name="fixture.bin",
transfer={
"source_client_id": source_id,
"target_client_id": target_id,
"route_id": "route-1",
"requested_logical_bytes": len(content),
"transfer_delta_logical_bytes": len(content),
},
)
definition.resource_id.info_hash_v1_hex = torrent_hash
definition.created_at.GetCurrentTime()
definition.transfer.requested_files.ranges.add(first=0, last=0)
definition.transfer.transfer_delta_files.ranges.add(first=0, last=0)
source_store = ClientStore(root / "source.db")
target_store = ClientStore(root / "target.db")
source_store.initialize()
target_store.initialize()
source_qb = Mock()
source_qb.get_resource.return_value = resource
target_qb = Mock()
target_qb.get_resource.side_effect = [None, resource]
syncthing = CompleteSyncthing()
source = ClientJobExecutor(
client_id=source_id,
qbittorrent=source_qb,
store=source_store,
qb_root=source_root,
qb_api_root=Path("/downloads"),
route_path=lambda _: route_root,
syncthing_transport=syncthing,
sparse_supported=True,
poll_interval=0,
)
target = ClientJobExecutor(
client_id=target_id,
qbittorrent=target_qb,
store=target_store,
qb_root=target_root,
qb_api_root=Path("/downloads"),
route_path=lambda _: route_root,
syncthing_transport=syncthing,
sparse_supported=True,
poll_interval=0,
)
source_assigned = source.assign(control_pb2.AssignJobCommand(
job=definition,
expected_job_revision=1,
expected_last_event_sequence=0,
))
target_assigned = target.assign(control_pb2.AssignJobCommand(
job=definition,
expected_job_revision=1,
expected_last_event_sequence=1,
))
self.assertEqual(source_assigned[0].sequence, 1)
self.assertEqual(target_assigned[0].sequence, 2)
cursor_revision = 1
cursor_sequence = 2
pipeline = (
(source, job_pb2.JOB_STEP_KIND_SOURCE_STAGE),
(target, job_pb2.JOB_STEP_KIND_SYNCTHING_TRANSFER),
(target, job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE),
(target, job_pb2.JOB_STEP_KIND_QB_VERIFY),
(source, job_pb2.JOB_STEP_KIND_STAGING_CLEANUP),
)
final = None
for executor, step in pipeline:
events = executor.execute(control_pb2.ExecuteStepCommand(
job_id=definition.job_id,
expected_job_revision=cursor_revision,
expected_last_event_sequence=cursor_sequence,
step=step,
attempt=1,
))
self.assertEqual(
[event.sequence for event in events],
[cursor_sequence + 1, cursor_sequence + 2],
)
cursor_sequence = events[-1].sequence
cursor_revision = events[-1].job_revision
final = events[-1]
self.assertEqual((target_root / "fixture.bin").read_bytes(), content)
target_qb.add_stopped_with_retry.assert_called_once_with(
metainfo, "/downloads", torrent_hash
)
target_qb.set_selection.assert_called_once_with(torrent_hash, [0], 1)
target_qb.recheck_and_wait.assert_called_once()
target_qb.start.assert_called_once_with(torrent_hash)
self.assertIsNotNone(final)
self.assertTrue(final.committed)
self.assertEqual(final.type, control_pb2.JOB_EVENT_TYPE_SUCCEEDED)
self.assertFalse(
(route_root / ".archive-control/jobs" / definition.job_id).exists()
)
replay = source.execute(control_pb2.ExecuteStepCommand(
job_id=definition.job_id,
expected_job_revision=cursor_revision - 2,
expected_last_event_sequence=cursor_sequence - 2,
step=job_pb2.JOB_STEP_KIND_STAGING_CLEANUP,
attempt=1,
))
self.assertEqual(replay[-1].event_id, final.event_id)
def test_step_failure_is_durable_and_reports_clear_reason(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
store = ClientStore(root / "client.db")
store.initialize()
definition = job_pb2.JobDefinition(
job_id=str(uuid4()),
idempotency_key=str(uuid4()),
operation=job_pb2.JOB_OPERATION_ARCHIVE,
resource_display_name="fixture",
transfer={
"source_client_id": "cache-1",
"target_client_id": "archive-1",
"route_id": "route-1",
},
)
definition.resource_id.info_hash_v1_hex = "a" * 40
definition.created_at.GetCurrentTime()
executor = ClientJobExecutor(
client_id="cache-1",
qbittorrent=Mock(),
store=store,
qb_root=root,
qb_api_root=Path("/downloads"),
route_path=lambda _: root,
syncthing_transport=Mock(),
sparse_supported=True,
poll_interval=0,
)
executor.assign(control_pb2.AssignJobCommand(
job=definition,
expected_job_revision=1,
expected_last_event_sequence=0,
))
command = control_pb2.ExecuteStepCommand(
job_id=definition.job_id,
expected_job_revision=1,
expected_last_event_sequence=1,
step=job_pb2.JOB_STEP_KIND_SOURCE_STAGE,
attempt=1,
)
with patch.object(
executor,
"_execute_step",
side_effect=JobExecutionError(
"partfile cannot be handled safely"
),
):
events = executor.execute(command)
self.assertEqual(
[event.type for event in events],
[
control_pb2.JOB_EVENT_TYPE_STEP_STARTED,
control_pb2.JOB_EVENT_TYPE_FAILED,
],
)
self.assertEqual(events[-1].state, job_pb2.JOB_STATE_FAILED)
self.assertIn("partfile", events[-1].error.message)
self.assertEqual(
executor.execute(command)[-1].event_id,
events[-1].event_id,
)
self.assertEqual(store.list_active_job_cursors(), [])
if __name__ == "__main__":
unittest.main()
tests/test_qbittorrent.py
+91
View File
@@ -108,6 +108,11 @@ class QBittorrentReaderTests(unittest.TestCase):
b"", # stop before recheck
b'{"total_downloaded":0}',
b"", # recheck
json.dumps([{
"hash": torrent_hash, "state": "stoppedDL",
}]).encode(),
b'{"total_downloaded":0}',
b'[{"index":0,"progress":0},{"index":1,"progress":0}]',
json.dumps([{
"hash": torrent_hash, "state": "checkingUP",
}]).encode(),
@@ -118,6 +123,7 @@ class QBittorrentReaderTests(unittest.TestCase):
}]).encode(),
b'{"total_downloaded":0}',
b'[{"index":0,"progress":1},{"index":1,"progress":0}]',
b"", # start after successful recheck
b"", # entry-only delete
]
with tempfile.TemporaryDirectory() as directory:
@@ -140,6 +146,7 @@ class QBittorrentReaderTests(unittest.TestCase):
result = adapter.recheck_and_wait(
torrent_hash, [0], timeout=1, poll_interval=0
)
adapter.start(torrent_hash)
adapter.delete_entry(torrent_hash)
self.assertEqual(result.final_state, "stoppedUP")
@@ -155,6 +162,7 @@ class QBittorrentReaderTests(unittest.TestCase):
for call in calls[2:]
if getattr(call, "data", None)
]
self.assertIn("id=0%7C1", form_bodies[0])
self.assertIn("priority=0", form_bodies[0])
self.assertIn("priority=1", form_bodies[1])
self.assertIn("deleteFiles=false", form_bodies[-1])
@@ -182,6 +190,89 @@ class QBittorrentReaderTests(unittest.TestCase):
self.assertTrue(opener.calls[-1].full_url.endswith("/torrents/pause"))
def test_start_falls_back_to_qbittorrent_4_resume_endpoint(self):
missing = error.HTTPError(
"http://qb/api/v2/torrents/start", 404, "not found", {}, None
)
responses = [b"Ok.", missing, b""]
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
password = root / "password"
password.write_text("secret", encoding="utf-8")
os.chmod(password, 0o600)
config = ServiceConfig(
"http://qb", PurePosixPath("/downloads"), root,
username="admin", password_file=password,
)
opener = _Opener(responses)
with patch(
"archive_clients.qbittorrent.request.build_opener",
return_value=opener,
):
QBittorrentReader(config).start("a" * 40)
self.assertTrue(opener.calls[-1].full_url.endswith("/torrents/resume"))
def test_wait_until_present_polls_until_added_torrent_is_visible(self):
torrent_hash = "a" * 40
responses = [
b"Ok.",
b"[]",
json.dumps([{"hash": torrent_hash}]).encode(),
]
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
password = root / "password"
password.write_text("secret", encoding="utf-8")
os.chmod(password, 0o600)
config = ServiceConfig(
"http://qb", PurePosixPath("/downloads"), root,
username="admin", password_file=password,
)
opener = _Opener(responses)
with patch(
"archive_clients.qbittorrent.request.build_opener",
return_value=opener,
):
QBittorrentReader(config).wait_until_present(
torrent_hash, timeout=1, poll_interval=0
)
self.assertEqual(len(opener.calls), 3)
def test_stopped_add_retries_while_recently_deleted_hash_is_busy(self):
torrent_hash = "a" * 40
responses = [
b"Ok.",
b"Fails.",
b"[]",
b"Ok.",
json.dumps([{"hash": torrent_hash}]).encode(),
]
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
password = root / "password"
password.write_text("secret", encoding="utf-8")
os.chmod(password, 0o600)
config = ServiceConfig(
"http://qb", PurePosixPath("/downloads"), root,
username="admin", password_file=password,
)
opener = _Opener(responses)
with patch(
"archive_clients.qbittorrent.request.build_opener",
return_value=opener,
):
QBittorrentReader(config).add_stopped_with_retry(
b"torrent",
"/downloads",
torrent_hash,
max_attempts=2,
initial_delay=0,
)
self.assertEqual(len(opener.calls), 5)
if __name__ == "__main__":
unittest.main()