256 lines
9.3 KiB
Python
256 lines
9.3 KiB
Python
import hashlib
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import Mock, patch
|
|
from uuid import uuid4
|
|
|
|
from archive_clients.bencode import encode
|
|
from archive_clients.jobs import ClientJobExecutor, JobExecutionError
|
|
from archive_clients.resources import normalize_resource
|
|
from archive_clients.state import ClientStore
|
|
from archive_control.v1 import control_pb2, job_pb2
|
|
|
|
|
|
class CompleteSyncthing:
|
|
def __init__(self):
|
|
self.posts = []
|
|
|
|
def get_json(self, path):
|
|
if path.startswith("/rest/db/completion?"):
|
|
return {"completion": 100}
|
|
if path.startswith("/rest/db/need?"):
|
|
return {"progress": [], "queued": [], "rest": []}
|
|
raise AssertionError(path)
|
|
|
|
def post(self, path):
|
|
self.posts.append(path)
|
|
|
|
|
|
class ClientJobHappyPathTests(unittest.TestCase):
|
|
def test_archive_and_unarchive_five_step_execution(self):
|
|
for operation in (
|
|
job_pb2.JOB_OPERATION_ARCHIVE,
|
|
job_pb2.JOB_OPERATION_UNARCHIVE,
|
|
):
|
|
with self.subTest(operation=operation), tempfile.TemporaryDirectory() as directory:
|
|
self._run_transfer(Path(directory), operation)
|
|
|
|
def _run_transfer(self, root: Path, operation: int):
|
|
source_root = root / "source"
|
|
target_root = root / "target"
|
|
route_root = root / "route"
|
|
source_root.mkdir()
|
|
target_root.mkdir()
|
|
route_root.mkdir()
|
|
content = b"archive-control-happy-path"
|
|
(source_root / "fixture.bin").write_bytes(content)
|
|
info = {
|
|
b"length": len(content),
|
|
b"name": b"fixture.bin",
|
|
b"piece length": 16384,
|
|
b"pieces": hashlib.sha1(content).digest(),
|
|
}
|
|
metainfo = encode({b"info": info})
|
|
torrent_hash = hashlib.sha1(encode(info)).hexdigest()
|
|
resource = normalize_resource(
|
|
{
|
|
"hash": torrent_hash,
|
|
"name": "fixture.bin",
|
|
"state": "uploading",
|
|
},
|
|
[{
|
|
"index": 0,
|
|
"name": "fixture.bin",
|
|
"size": len(content),
|
|
"completed": len(content),
|
|
"priority": 1,
|
|
}],
|
|
metainfo,
|
|
)
|
|
source_id, target_id = (
|
|
("cache-1", "archive-1")
|
|
if operation == job_pb2.JOB_OPERATION_ARCHIVE
|
|
else ("archive-1", "cache-1")
|
|
)
|
|
definition = job_pb2.JobDefinition(
|
|
job_id=str(uuid4()),
|
|
idempotency_key=str(uuid4()),
|
|
operation=operation,
|
|
resource_display_name="fixture.bin",
|
|
transfer={
|
|
"source_client_id": source_id,
|
|
"target_client_id": target_id,
|
|
"route_id": "route-1",
|
|
"requested_logical_bytes": len(content),
|
|
"transfer_delta_logical_bytes": len(content),
|
|
},
|
|
)
|
|
definition.resource_id.info_hash_v1_hex = torrent_hash
|
|
definition.created_at.GetCurrentTime()
|
|
definition.transfer.requested_files.ranges.add(first=0, last=0)
|
|
definition.transfer.transfer_delta_files.ranges.add(first=0, last=0)
|
|
|
|
source_store = ClientStore(root / "source.db")
|
|
target_store = ClientStore(root / "target.db")
|
|
source_store.initialize()
|
|
target_store.initialize()
|
|
source_qb = Mock()
|
|
source_qb.get_resource.return_value = resource
|
|
target_qb = Mock()
|
|
target_qb.get_resource.side_effect = [None, resource]
|
|
syncthing = CompleteSyncthing()
|
|
source = ClientJobExecutor(
|
|
client_id=source_id,
|
|
qbittorrent=source_qb,
|
|
store=source_store,
|
|
qb_root=source_root,
|
|
qb_api_root=Path("/downloads"),
|
|
route_path=lambda _: route_root,
|
|
syncthing_transport=syncthing,
|
|
sparse_supported=True,
|
|
poll_interval=0,
|
|
)
|
|
target = ClientJobExecutor(
|
|
client_id=target_id,
|
|
qbittorrent=target_qb,
|
|
store=target_store,
|
|
qb_root=target_root,
|
|
qb_api_root=Path("/downloads"),
|
|
route_path=lambda _: route_root,
|
|
syncthing_transport=syncthing,
|
|
sparse_supported=True,
|
|
poll_interval=0,
|
|
)
|
|
|
|
source_assigned = source.assign(control_pb2.AssignJobCommand(
|
|
job=definition,
|
|
expected_job_revision=1,
|
|
expected_last_event_sequence=0,
|
|
))
|
|
target_assigned = target.assign(control_pb2.AssignJobCommand(
|
|
job=definition,
|
|
expected_job_revision=1,
|
|
expected_last_event_sequence=1,
|
|
))
|
|
self.assertEqual(source_assigned[0].sequence, 1)
|
|
self.assertEqual(target_assigned[0].sequence, 2)
|
|
|
|
cursor_revision = 1
|
|
cursor_sequence = 2
|
|
pipeline = (
|
|
(source, job_pb2.JOB_STEP_KIND_SOURCE_STAGE),
|
|
(target, job_pb2.JOB_STEP_KIND_SYNCTHING_TRANSFER),
|
|
(target, job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE),
|
|
(target, job_pb2.JOB_STEP_KIND_QB_VERIFY),
|
|
(source, job_pb2.JOB_STEP_KIND_STAGING_CLEANUP),
|
|
)
|
|
final = None
|
|
for executor, step in pipeline:
|
|
events = executor.execute(control_pb2.ExecuteStepCommand(
|
|
job_id=definition.job_id,
|
|
expected_job_revision=cursor_revision,
|
|
expected_last_event_sequence=cursor_sequence,
|
|
step=step,
|
|
attempt=1,
|
|
))
|
|
self.assertEqual(
|
|
[event.sequence for event in events],
|
|
[cursor_sequence + 1, cursor_sequence + 2],
|
|
)
|
|
cursor_sequence = events[-1].sequence
|
|
cursor_revision = events[-1].job_revision
|
|
final = events[-1]
|
|
|
|
self.assertEqual((target_root / "fixture.bin").read_bytes(), content)
|
|
target_qb.add_stopped_with_retry.assert_called_once_with(
|
|
metainfo, "/downloads", torrent_hash
|
|
)
|
|
target_qb.set_selection.assert_called_once_with(torrent_hash, [0], 1)
|
|
target_qb.recheck_and_wait.assert_called_once()
|
|
target_qb.start.assert_called_once_with(torrent_hash)
|
|
self.assertIsNotNone(final)
|
|
self.assertTrue(final.committed)
|
|
self.assertEqual(final.type, control_pb2.JOB_EVENT_TYPE_SUCCEEDED)
|
|
self.assertFalse(
|
|
(route_root / ".archive-control/jobs" / definition.job_id).exists()
|
|
)
|
|
|
|
replay = source.execute(control_pb2.ExecuteStepCommand(
|
|
job_id=definition.job_id,
|
|
expected_job_revision=cursor_revision - 2,
|
|
expected_last_event_sequence=cursor_sequence - 2,
|
|
step=job_pb2.JOB_STEP_KIND_STAGING_CLEANUP,
|
|
attempt=1,
|
|
))
|
|
self.assertEqual(replay[-1].event_id, final.event_id)
|
|
|
|
def test_step_failure_is_durable_and_reports_clear_reason(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
store = ClientStore(root / "client.db")
|
|
store.initialize()
|
|
definition = job_pb2.JobDefinition(
|
|
job_id=str(uuid4()),
|
|
idempotency_key=str(uuid4()),
|
|
operation=job_pb2.JOB_OPERATION_ARCHIVE,
|
|
resource_display_name="fixture",
|
|
transfer={
|
|
"source_client_id": "cache-1",
|
|
"target_client_id": "archive-1",
|
|
"route_id": "route-1",
|
|
},
|
|
)
|
|
definition.resource_id.info_hash_v1_hex = "a" * 40
|
|
definition.created_at.GetCurrentTime()
|
|
executor = ClientJobExecutor(
|
|
client_id="cache-1",
|
|
qbittorrent=Mock(),
|
|
store=store,
|
|
qb_root=root,
|
|
qb_api_root=Path("/downloads"),
|
|
route_path=lambda _: root,
|
|
syncthing_transport=Mock(),
|
|
sparse_supported=True,
|
|
poll_interval=0,
|
|
)
|
|
executor.assign(control_pb2.AssignJobCommand(
|
|
job=definition,
|
|
expected_job_revision=1,
|
|
expected_last_event_sequence=0,
|
|
))
|
|
command = control_pb2.ExecuteStepCommand(
|
|
job_id=definition.job_id,
|
|
expected_job_revision=1,
|
|
expected_last_event_sequence=1,
|
|
step=job_pb2.JOB_STEP_KIND_SOURCE_STAGE,
|
|
attempt=1,
|
|
)
|
|
with patch.object(
|
|
executor,
|
|
"_execute_step",
|
|
side_effect=JobExecutionError(
|
|
"partfile cannot be handled safely"
|
|
),
|
|
):
|
|
events = executor.execute(command)
|
|
|
|
self.assertEqual(
|
|
[event.type for event in events],
|
|
[
|
|
control_pb2.JOB_EVENT_TYPE_STEP_STARTED,
|
|
control_pb2.JOB_EVENT_TYPE_FAILED,
|
|
],
|
|
)
|
|
self.assertEqual(events[-1].state, job_pb2.JOB_STATE_FAILED)
|
|
self.assertIn("partfile", events[-1].error.message)
|
|
self.assertEqual(
|
|
executor.execute(command)[-1].event_id,
|
|
events[-1].event_id,
|
|
)
|
|
self.assertEqual(store.list_active_job_cursors(), [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|