cabbage/archive-clients
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete transfer and eviction execution

Browse Source
This commit is contained in:
cabbage
2026-07-23 09:14:40 +00:00
parent 884aed9921
commit b6de490b7e
17 changed files with 1445 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_daemon.py
+45
View File
@@ -22,6 +22,51 @@ from archive_control.v1 import (
class DaemonTransportTests(unittest.IsolatedAsyncioTestCase):
async def test_eviction_assignment_and_steps_are_admitted(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
token = root / "token"
token.write_text("shared-secret", encoding="utf-8")
os.chmod(token, 0o600)
service = ServiceConfig(
"http://local", PurePosixPath("/api"), root
)
config = ClientConfig(
"cache-1", "Cache 1", "cache", "ws://control", token,
root / "state.db", root / "backups", service, service,
)
probe = FilesystemProbe(root, True, True, True, True, True)
daemon = ArchiveClientDaemon(config, [probe, probe], [])
daemon.jobs = Mock()
assign = control_pb2.Command(command_id=str(uuid4()))
definition = assign.assign_job.job
definition.job_id = str(uuid4())
definition.eviction.cache_client_id = "cache-1"
acknowledgement = daemon._initial_acknowledgement(
assign, set(), False
)
self.assertEqual(
acknowledgement.status,
control_pb2.COMMAND_ACK_STATUS_ACCEPTED,
)
for step_kind in (
job_pb2.JOB_STEP_KIND_VERIFY_ARCHIVE_COVERAGE,
job_pb2.JOB_STEP_KIND_QB_REMOVE_ENTRY,
job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK,
):
execute = control_pb2.Command(command_id=str(uuid4()))
execute.execute_step.job_id = definition.job_id
execute.execute_step.step = step_kind
acknowledgement = daemon._initial_acknowledgement(
execute, set(), False
)
self.assertEqual(
acknowledgement.status,
control_pb2.COMMAND_ACK_STATUS_ACCEPTED,
)
async def test_ensure_route_is_durable_and_duplicate_replays_updates(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
tests/test_eviction.py
+181
View File
@@ -0,0 +1,181 @@
import tempfile
import unittest
from pathlib import Path
from archive_clients.eviction import (
remove_qb_entry,
safe_unlink,
verify_and_snapshot,
)
from archive_clients.resources import NormalizedResource
from archive_clients.state import ClientStore
from archive_control.v1 import resource_pb2
def normalized(torrent_hash, paths):
summary = resource_pb2.ResourceSummary(
qb_torrent_id=torrent_hash,
display_name="resource",
total_file_count=len(paths),
canonical_paths=True,
)
summary.resource_id.info_hash_v1_hex = torrent_hash
files = []
for index, path in enumerate(paths):
size = len(path) + 10
files.append(resource_pb2.TorrentFile(
file_index=index,
canonical_path=path,
logical_bytes=size,
completed_bytes=size,
selected=True,
))
summary.selected_files.ranges.add(first=index, last=index)
summary.selected_complete_files.ranges.add(first=index, last=index)
return NormalizedResource(summary, tuple(files), None)
class FakeQB:
def __init__(self, evicted, remaining):
self.evicted = evicted
self.remaining = remaining
self.deleted = []
def get_resource(self, torrent_hash):
return (
self.evicted
if self.evicted
and self.evicted.summary.qb_torrent_id == torrent_hash
else None
)
def delete_entry(self, torrent_hash):
self.deleted.append(torrent_hash)
self.evicted = None
def list_resources(self):
return list(self.remaining)
class EvictionTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.root = Path(self.temp.name) / "qb"
self.root.mkdir()
self.store = ClientStore(Path(self.temp.name) / "state.db")
self.store.initialize()
self.store.save_job(
"job-1", '{"jobId":"job-1"}', "JOB_STATE_RUNNING", 1, 0, False
)
def tearDown(self):
self.temp.cleanup()
def test_shared_paths_and_unknown_files_survive(self):
evicted = normalized("a" * 40, ["tree/shared.bin", "tree/owned.bin"])
other = normalized("b" * 40, ["tree/shared.bin"])
for item in evicted.files:
path = self.root / item.canonical_path
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b"x" * item.logical_bytes)
unknown = self.root / "tree" / "from-another-app.txt"
unknown.write_text("keep")
qb = FakeQB(evicted, [other])
verify_and_snapshot(
job_id="job-1",
resource=evicted,
selected_indices=[0, 1],
qb_root=self.root,
store=self.store,
)
remove_qb_entry(
job_id="job-1",
torrent_hash="a" * 40,
qbittorrent=qb,
store=self.store,
)
result = safe_unlink(
job_id="job-1",
qb_root=self.root,
qbittorrent=qb,
store=self.store,
)
self.assertTrue((self.root / "tree/shared.bin").exists())
self.assertFalse((self.root / "tree/owned.bin").exists())
self.assertTrue(unknown.exists())
self.assertTrue((self.root / "tree").is_dir())
self.assertEqual(qb.deleted, ["a" * 40])
self.assertEqual(
result["retained_files"],
[{"path": "tree/shared.bin", "reason": "shared"}],
)
def test_changed_inode_is_not_unlinked(self):
evicted = normalized("a" * 40, ["tree/owned.bin"])
path = self.root / "tree/owned.bin"
path.parent.mkdir(parents=True)
path.write_bytes(b"x" * evicted.files[0].logical_bytes)
qb = FakeQB(evicted, [])
verify_and_snapshot(
job_id="job-1",
resource=evicted,
selected_indices=[0],
qb_root=self.root,
store=self.store,
)
path.unlink()
path.write_bytes(b"y" * evicted.files[0].logical_bytes)
remove_qb_entry(
job_id="job-1",
torrent_hash="a" * 40,
qbittorrent=qb,
store=self.store,
)
result = safe_unlink(
job_id="job-1",
qb_root=self.root,
qbittorrent=qb,
store=self.store,
)
self.assertTrue(path.exists())
self.assertEqual(
result["retained_files"][0]["reason"], "identity-changed"
)
def test_retry_is_idempotent(self):
evicted = normalized("a" * 40, ["owned.bin"])
path = self.root / "owned.bin"
path.write_bytes(b"x" * evicted.files[0].logical_bytes)
qb = FakeQB(evicted, [])
verify_and_snapshot(
job_id="job-1",
resource=evicted,
selected_indices=[0],
qb_root=self.root,
store=self.store,
)
remove_qb_entry(
job_id="job-1",
torrent_hash="a" * 40,
qbittorrent=qb,
store=self.store,
)
first = safe_unlink(
job_id="job-1",
qb_root=self.root,
qbittorrent=qb,
store=self.store,
)
second = safe_unlink(
job_id="job-1",
qb_root=self.root,
qbittorrent=qb,
store=self.store,
)
self.assertEqual(first, second)
self.assertEqual(qb.deleted, ["a" * 40])
if __name__ == "__main__":
unittest.main()
tests/test_jobs.py
+61 -3
View File
@@ -6,7 +6,11 @@ from unittest.mock import Mock, patch
from uuid import uuid4
from archive_clients.bencode import encode
from archive_clients.jobs import ClientJobExecutor, JobExecutionError
from archive_clients.jobs import (
ClientJobExecutor,
JobExecutionError,
_resource_fingerprint,
)
from archive_clients.resources import normalize_resource
from archive_clients.state import ClientStore
from archive_control.v1 import control_pb2, job_pb2
@@ -90,6 +94,9 @@ class ClientJobHappyPathTests(unittest.TestCase):
definition.created_at.GetCurrentTime()
definition.transfer.requested_files.ranges.add(first=0, last=0)
definition.transfer.transfer_delta_files.ranges.add(first=0, last=0)
definition.transfer.source_fingerprint.CopyFrom(
_resource_fingerprint(resource, source_id)
)
source_store = ClientStore(root / "source.db")
target_store = ClientStore(root / "target.db")
@@ -98,7 +105,7 @@ class ClientJobHappyPathTests(unittest.TestCase):
source_qb = Mock()
source_qb.get_resource.return_value = resource
target_qb = Mock()
target_qb.get_resource.side_effect = [None, resource]
target_qb.get_resource.side_effect = [None, None, resource]
syncthing = CompleteSyncthing()
source = ClientJobExecutor(
client_id=source_id,
@@ -156,7 +163,10 @@ class ClientJobHappyPathTests(unittest.TestCase):
))
self.assertEqual(
[event.sequence for event in events],
[cursor_sequence + 1, cursor_sequence + 2],
list(range(
cursor_sequence + 1,
cursor_sequence + len(events) + 1,
)),
)
cursor_sequence = events[-1].sequence
cursor_revision = events[-1].job_revision
@@ -250,6 +260,54 @@ class ClientJobHappyPathTests(unittest.TestCase):
)
self.assertEqual(store.list_active_job_cursors(), [])
def test_active_source_cancellation_reports_durable_rollback(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
store = ClientStore(root / "client.db")
store.initialize()
definition = job_pb2.JobDefinition(
job_id=str(uuid4()),
idempotency_key=str(uuid4()),
operation=job_pb2.JOB_OPERATION_ARCHIVE,
resource_display_name="fixture",
transfer={
"source_client_id": "cache-1",
"target_client_id": "archive-1",
"route_id": "route-1",
},
)
definition.resource_id.info_hash_v1_hex = "a" * 40
definition.created_at.GetCurrentTime()
executor = ClientJobExecutor(
client_id="cache-1",
qbittorrent=Mock(),
store=store,
qb_root=root,
qb_api_root=Path("/downloads"),
route_path=lambda _: root / "route",
syncthing_transport=Mock(),
sparse_supported=True,
poll_interval=0,
)
executor.assign(control_pb2.AssignJobCommand(
job=definition,
expected_job_revision=1,
expected_last_event_sequence=0,
))
events = executor.cancel(control_pb2.CancelJobCommand(
job_id=definition.job_id,
expected_job_revision=2,
expected_last_event_sequence=1,
reason="test",
))
self.assertEqual(
events[-1].type,
control_pb2.JOB_EVENT_TYPE_ROLLBACK_SUCCEEDED,
)
self.assertEqual(
events[-1].state, job_pb2.JOB_STATE_ROLLING_BACK
)
if __name__ == "__main__":
unittest.main()
tests/test_state.py
+22
View File
@@ -120,6 +120,28 @@ class ClientStoreTests(unittest.TestCase):
"operation-1", '{"method":2}'
)
def test_job_artifacts_are_immutable_and_survive_restart(self):
with tempfile.TemporaryDirectory() as directory:
database = Path(directory) / "state.db"
store = ClientStore(database)
store.initialize()
store.save_job(
"job-1", '{"jobId":"job-1"}', "JOB_STATE_RUNNING",
1, 0, False,
)
store.put_job_artifact(
"job-1", "baseline", {"selected": [1, 3]}
)
reopened = ClientStore(database)
self.assertEqual(
reopened.get_job_artifact("job-1", "baseline")["value"],
{"selected": [1, 3]},
)
with self.assertRaises(JobConflict):
reopened.put_job_artifact(
"job-1", "baseline", {"selected": [2]}
)
if __name__ == "__main__":
unittest.main()
tests/test_transfer.py
+17
View File
@@ -9,6 +9,7 @@ from uuid import uuid4
from archive_clients.state import ClientStore
from archive_clients.transfer import (
FileMaterializer,
TransferIntegrityError,
canonical_message_json,
load_published_transfer,
materialize_transfer,
@@ -212,6 +213,22 @@ class TransferHappyPathTests(unittest.TestCase):
self.assertEqual(first, second)
self.assertEqual(first, canonical_message_json(manifest))
def test_manifest_rejects_case_colliding_target_paths(self):
manifest = self._manifest()
manifest.files[1].target_canonical_path = "album/ONE.bin"
with self.assertRaisesRegex(
TransferIntegrityError, "case-colliding target"
):
stage_transfer(
manifest,
source_root=self.source,
sync_root=self.sync,
store=self.store,
artifact_sources={
"metainfo/source.torrent": self.metainfo
},
)
def _manifest(self, job_id=None):
manifest = transfer_pb2.TransferManifest(
manifest_version=1,