feat: complete transfer and eviction execution

This commit is contained in:
2026-07-23 09:14:40 +00:00
parent 884aed9921
commit b6de490b7e
17 changed files with 1445 additions and 81 deletions
+6 -5
View File
@@ -27,11 +27,12 @@ deadline. `down.sh` removes only the five exact Compose projects and the
labelled E2E network; it intentionally retains all bind-mounted runtime state. labelled E2E network; it intentionally retains all bind-mounted runtime state.
Set `E2E_RUN_TRANSFER=1` to follow route verification with a real Set `E2E_RUN_TRANSFER=1` to follow route verification with a real
cache-1 → archive-1 transfer followed by archive-1 → cache-2 unarchive. The cache-1 → archive-1 transfer, covered cache-1 eviction, and
scenario creates a deterministic one-file torrent in the isolated cache archive-1 → cache-2 unarchive. The scenario creates a deterministic one-file
qBittorrent, submits both jobs through the test HTTP adapter, waits for all five torrent in the isolated cache qBittorrent, uses fresh preview revisions for all
durable steps in each direction, and verifies target qBittorrent selections, jobs, waits for the durable five/three/five-step flows, and verifies target
retained source entries, and target file digests. qBittorrent selections, archive retention, safe cache removal, and target file
digests.
The Syncthing 2.1.2 and LinuxServer qBittorrent multi-platform image indexes are The Syncthing 2.1.2 and LinuxServer qBittorrent multi-platform image indexes are
digest-pinned. Runtime secrets are generated with mode 0600 and ignored by digest-pinned. Runtime secrets are generated with mode 0600 and ignored by
+12 -26
View File
@@ -9,8 +9,6 @@ import sys
import time import time
import urllib.parse import urllib.parse
import urllib.request import urllib.request
import uuid
from datetime import datetime, timezone
CONTROL = "http://127.0.0.1:18081/test/v1" CONTROL = "http://127.0.0.1:18081/test/v1"
@@ -103,30 +101,18 @@ def main() -> int:
60, 60,
f"{args.cache_client}/{args.archive_client} route", f"{args.cache_client}/{args.archive_client} route",
) )
job_id = str(uuid.uuid4()) preview = post_json(f"{CONTROL}/jobs/preview", {
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") "operation": args.operation,
definition = { "source_client_id": args.source_client,
"jobId": job_id, "target_client_id": args.target_client,
"idempotencyKey": str(uuid.uuid4()), "resource_id": {"info_hash_v1_hex": INFO_HASH},
"operation": ( })
"JOB_OPERATION_ARCHIVE" definition = preview["definition"]
if args.operation == "archive" job_id = definition["job_id"]
else "JOB_OPERATION_UNARCHIVE" created = post_json(f"{CONTROL}/jobs", {
), "preview_revision": preview["preview_revision"],
"resourceId": {"infoHashV1Hex": INFO_HASH}, "definition": definition,
"resourceDisplayName": "fixture.bin", })
"createdAt": now,
"transfer": {
"sourceClientId": args.source_client,
"targetClientId": args.target_client,
"routeId": route["route_id"],
"requestedFiles": {"ranges": [{"first": 0, "last": 0}]},
"transferDeltaFiles": {"ranges": [{"first": 0, "last": 0}]},
"requestedLogicalBytes": str(LOGICAL_BYTES),
"transferDeltaLogicalBytes": str(LOGICAL_BYTES),
},
}
created = post_json(f"{CONTROL}/jobs", definition)
if created["job_id"] != job_id: if created["job_id"] != job_id:
raise RuntimeError("control returned the wrong job") raise RuntimeError("control returned the wrong job")
+65
View File
@@ -0,0 +1,65 @@
#!/usr/bin/env python3
"""Drive a covered cache eviction through the bot-free HTTP adapter."""
from __future__ import annotations
import json
import sys
import time
import urllib.request
CONTROL = "http://127.0.0.1:18081/test/v1"
INFO_HASH = "330be0cb7c2201135a2de63b28e77993745ff688"
def get_json(url: str):
with urllib.request.urlopen(url, timeout=35) as response:
return json.load(response)
def post_json(url: str, value: object):
request = urllib.request.Request(
url,
data=json.dumps(value, separators=(",", ":")).encode(),
method="POST",
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(request, timeout=35) as response:
return json.load(response)
def main() -> int:
preview = post_json(f"{CONTROL}/jobs/preview", {
"operation": "evict_cache",
"cache_client_id": "cache-1",
"resource_id": {"info_hash_v1_hex": INFO_HASH},
})
definition = preview["definition"]
job_id = definition["job_id"]
post_json(f"{CONTROL}/jobs", {
"preview_revision": preview["preview_revision"],
"definition": definition,
})
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
job = get_json(f"{CONTROL}/jobs/{job_id}")
if job["state"] == "JOB_STATE_SUCCEEDED" and job["committed"]:
print(json.dumps({
"job_id": job_id,
"state": job["state"],
"committed": job["committed"],
}, sort_keys=True))
return 0
if job["state"] in {"JOB_STATE_FAILED", "JOB_STATE_CANCELLED"}:
raise RuntimeError(f"eviction did not succeed: {job}")
time.sleep(0.5)
raise RuntimeError("timed out waiting for cache eviction")
if __name__ == "__main__":
try:
raise SystemExit(main())
except Exception as exc:
print(f"eviction happy-path failed: {exc}", file=sys.stderr)
raise
+10
View File
@@ -63,6 +63,16 @@ if [[ "$target_digest" != \
fi fi
printf 'archive filesystem content verified: sha256=%s\n' "$target_digest" printf 'archive filesystem content verified: sha256=%s\n' "$target_digest"
compose_control exec -T control \
python /e2e/scenarios/evict_happy.py
wait_qb_absent cache-1
if compose_node cache-1 exec -T qbittorrent \
test -e /downloads/fixture.bin; then
printf 'eviction left the unshared cache file behind\n' >&2
exit 1
fi
printf 'covered cache eviction verified\n'
compose_node cache-2 exec -T qbittorrent curl -fsS \ compose_node cache-2 exec -T qbittorrent curl -fsS \
-X POST http://127.0.0.1:8080/api/v2/torrents/delete \ -X POST http://127.0.0.1:8080/api/v2/torrents/delete \
--data-urlencode "hashes=$info_hash" \ --data-urlencode "hashes=$info_hash" \
+10
View File
@@ -13,6 +13,16 @@ reconnect_max = "60s"
reconnect_reset_after = "60s" reconnect_reset_after = "60s"
reconnect_jitter = true reconnect_jitter = true
[jobs]
# Mark a job stalled after it has made no progress for this long.
stall_after = "30m"
# Upper bound for qBittorrent's stopped full recheck.
verification_timeout = "30m"
# Poll qBittorrent and Syncthing at this interval while a step is active.
poll_interval = "1s"
# Space that must remain free after a worst-case copy fallback.
free_space_reserve_bytes = 1073741824
[backup] [backup]
interval = "6h" interval = "6h"
recent = 12 recent = 12
+24 -2
View File
@@ -80,6 +80,9 @@ class ConnectionConfig:
@dataclass(frozen=True) @dataclass(frozen=True)
class JobsConfig: class JobsConfig:
stall_after: float = 30 * 60 stall_after: float = 30 * 60
verification_timeout: float = 30 * 60
poll_interval: float = 1
free_space_reserve_bytes: int = 1024 * 1024 * 1024
@dataclass(frozen=True) @dataclass(frozen=True)
@@ -217,8 +220,27 @@ def _connection(value: Any) -> ConnectionConfig:
def _jobs(value: Any) -> JobsConfig: def _jobs(value: Any) -> JobsConfig:
if not isinstance(value, dict): if not isinstance(value, dict):
raise ConfigError("jobs must be a table") raise ConfigError("jobs must be a table")
_keys(value, {"stall_after"}, "jobs") _keys(
return JobsConfig(stall_after=_duration(value.get("stall_after", "30m"))) value,
{
"stall_after",
"verification_timeout",
"poll_interval",
"free_space_reserve_bytes",
},
"jobs",
)
return JobsConfig(
stall_after=_duration(value.get("stall_after", "30m")),
verification_timeout=_duration(
value.get("verification_timeout", "30m")
),
poll_interval=_duration(value.get("poll_interval", "1s")),
free_space_reserve_bytes=_positive_int(
value.get("free_space_reserve_bytes", 1024 * 1024 * 1024),
"jobs.free_space_reserve_bytes",
),
)
def _backup(value: Any) -> BackupConfig: def _backup(value: Any) -> BackupConfig:
+55 -9
View File
@@ -105,6 +105,11 @@ class ArchiveClientDaemon:
route_path=self._route_path, route_path=self._route_path,
syncthing_transport=self.routes.transport, syncthing_transport=self.routes.transport,
sparse_supported=all(probe.sparse_files for probe in probes), sparse_supported=all(probe.sparse_files for probe in probes),
poll_interval=config.jobs.poll_interval,
verification_timeout=config.jobs.verification_timeout,
free_space_reserve_bytes=(
config.jobs.free_space_reserve_bytes
),
) )
if resource_reader is not None and self.routes is not None if resource_reader is not None and self.routes is not None
else None else None
@@ -415,9 +420,13 @@ class ArchiveClientDaemon:
elif ( elif (
accepted is not None accepted is not None
and accepted_for_execution and accepted_for_execution
and command.WhichOneof("payload") in {"assign_job", "execute_step"} and command.WhichOneof("payload") in {
"assign_job", "execute_step", "cancel_job"
}
and self.jobs is not None and self.jobs is not None
): ):
if command.WhichOneof("payload") == "cancel_job":
self.jobs.request_cancel(command.cancel_job.job_id)
self._schedule_job_command( self._schedule_job_command(
command, command,
envelope.message_id, envelope.message_id,
@@ -445,9 +454,13 @@ class ArchiveClientDaemon:
command, "", outbound, command_tasks command, "", outbound, command_tasks
) )
elif ( elif (
command.WhichOneof("payload") in {"assign_job", "execute_step"} command.WhichOneof("payload") in {
"assign_job", "execute_step", "cancel_job"
}
and self.jobs is not None and self.jobs is not None
): ):
if command.WhichOneof("payload") == "cancel_job":
self.jobs.request_cancel(command.cancel_job.job_id)
job_commands.append(command) job_commands.append(command)
if job_commands: if job_commands:
task = asyncio.create_task( task = asyncio.create_task(
@@ -543,17 +556,39 @@ class ArchiveClientDaemon:
) -> None: ) -> None:
assert self.jobs is not None assert self.jobs is not None
payload = command.WhichOneof("payload") payload = command.WhichOneof("payload")
streamed = False
if payload == "assign_job": if payload == "assign_job":
events = await asyncio.to_thread( events = await asyncio.to_thread(
self.jobs.assign, command.assign_job self.jobs.assign, command.assign_job
) )
elif payload == "execute_step": elif payload == "execute_step":
loop = asyncio.get_running_loop()
def emit(event):
response = new_envelope()
response.correlation_id = correlation_id
response.job_event.CopyFrom(event)
future = asyncio.run_coroutine_threadsafe(
outbound.put(encode(response)), loop
)
try:
future.result(timeout=30)
except Exception:
# The event is already durable in the client DB and will
# be replayed after reconnect.
pass
events = await asyncio.to_thread( events = await asyncio.to_thread(
self.jobs.execute, command.execute_step self.jobs.execute, command.execute_step, emit
)
streamed = True
elif payload == "cancel_job":
events = await asyncio.to_thread(
self.jobs.cancel, command.cancel_job
) )
else: else:
raise JobExecutionError("job command payload is unsupported") raise JobExecutionError("job command payload is unsupported")
for event in events: for event in (() if streamed else events):
response = new_envelope() response = new_envelope()
response.correlation_id = correlation_id response.correlation_id = correlation_id
response.job_event.CopyFrom(event) response.job_event.CopyFrom(event)
@@ -880,17 +915,25 @@ class ArchiveClientDaemon:
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED
elif command.WhichOneof("payload") == "assign_job": elif command.WhichOneof("payload") == "assign_job":
definition = command.assign_job.job definition = command.assign_job.job
specification = definition.WhichOneof("spec")
assigned_to_client = (
specification == "transfer"
and self.config.client_id
in {
definition.transfer.source_client_id,
definition.transfer.target_client_id,
}
) or (
specification == "eviction"
and self.config.client_id == definition.eviction.cache_client_id
)
if self.jobs is None: if self.jobs is None:
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
acknowledgement.error.code = common_pb2.ERROR_CODE_UNAVAILABLE acknowledgement.error.code = common_pb2.ERROR_CODE_UNAVAILABLE
acknowledgement.error.message = "job executor is unavailable" acknowledgement.error.message = "job executor is unavailable"
elif ( elif (
not definition.job_id not definition.job_id
or definition.WhichOneof("spec") != "transfer" or not assigned_to_client
or self.config.client_id not in {
definition.transfer.source_client_id,
definition.transfer.target_client_id,
}
): ):
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
acknowledgement.error.code = common_pb2.ERROR_CODE_INVALID_ARGUMENT acknowledgement.error.code = common_pb2.ERROR_CODE_INVALID_ARGUMENT
@@ -911,6 +954,9 @@ class ArchiveClientDaemon:
job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE, job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE,
job_pb2.JOB_STEP_KIND_QB_VERIFY, job_pb2.JOB_STEP_KIND_QB_VERIFY,
job_pb2.JOB_STEP_KIND_STAGING_CLEANUP, job_pb2.JOB_STEP_KIND_STAGING_CLEANUP,
job_pb2.JOB_STEP_KIND_VERIFY_ARCHIVE_COVERAGE,
job_pb2.JOB_STEP_KIND_QB_REMOVE_ENTRY,
job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK,
} }
): ):
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
+242
View File
@@ -0,0 +1,242 @@
"""Idempotent cache eviction without recursive or qB data deletion."""
from __future__ import annotations
import errno
import hashlib
import os
import stat
from pathlib import Path, PurePosixPath
from typing import Iterable
from archive_clients.qbittorrent import QBittorrentReader
from archive_clients.resources import NormalizedResource
from archive_clients.state import ClientStore
class EvictionError(RuntimeError):
pass
def verify_and_snapshot(
*,
job_id: str,
resource: NormalizedResource,
selected_indices: Iterable[int],
qb_root: Path,
store: ClientStore,
) -> dict[str, object]:
selected = set(selected_indices)
by_index = {item.file_index: item for item in resource.files}
if not selected or any(index not in by_index for index in selected):
raise EvictionError("eviction file selection is invalid")
files = []
for index in sorted(selected):
item = by_index[index]
if not item.selected or item.completed_bytes != item.logical_bytes:
raise EvictionError(
f"cache file {index} is not selected and complete"
)
relative = _relative(item.canonical_path)
path = qb_root.joinpath(*relative.parts)
try:
metadata = path.lstat()
except FileNotFoundError as exc:
raise EvictionError(
f"cache file disappeared before eviction: {relative}"
) from exc
if not stat.S_ISREG(metadata.st_mode):
raise EvictionError(
f"cache path is not a regular file: {relative}"
)
if metadata.st_size != item.logical_bytes:
raise EvictionError(
f"cache file size changed before eviction: {relative}"
)
files.append(
{
"file_index": index,
"path": relative.as_posix(),
"logical_bytes": item.logical_bytes,
"device": metadata.st_dev,
"inode": metadata.st_ino,
"mtime_ns": metadata.st_mtime_ns,
"ctime_ns": metadata.st_ctime_ns,
"sha256": _sha256(path),
}
)
snapshot = {"files": files}
return store.put_job_artifact(job_id, "eviction-snapshot", snapshot)["value"]
def remove_qb_entry(
*,
job_id: str,
torrent_hash: str,
qbittorrent: QBittorrentReader,
store: ClientStore,
) -> None:
marker = store.get_job_artifact(job_id, "qb-entry-removed")
if marker is not None:
return
if qbittorrent.get_resource(torrent_hash) is not None:
qbittorrent.delete_entry(torrent_hash)
store.put_job_artifact(
job_id, "qb-entry-removed", {"torrent_hash": torrent_hash}
)
def safe_unlink(
*,
job_id: str,
qb_root: Path,
qbittorrent: QBittorrentReader,
store: ClientStore,
) -> dict[str, object]:
completed = store.get_job_artifact(job_id, "eviction-unlinked")
if completed is not None:
return completed["value"]
snapshot_row = store.get_job_artifact(job_id, "eviction-snapshot")
if snapshot_row is None:
raise EvictionError("eviction snapshot is missing")
snapshot = snapshot_row["value"]
files = snapshot.get("files")
if not isinstance(files, list):
raise EvictionError("eviction snapshot is invalid")
referenced = _remaining_paths(qbittorrent.list_resources())
removed: list[str] = []
retained: list[dict[str, str]] = []
directories: set[Path] = set()
for record in files:
if not isinstance(record, dict) or not isinstance(record.get("path"), str):
raise EvictionError("eviction file record is invalid")
relative = _relative(record["path"])
if relative.as_posix() in referenced:
retained.append({"path": relative.as_posix(), "reason": "shared"})
continue
path = qb_root.joinpath(*relative.parts)
try:
metadata = path.lstat()
except FileNotFoundError:
removed.append(relative.as_posix())
continue
if not stat.S_ISREG(metadata.st_mode):
retained.append(
{"path": relative.as_posix(), "reason": "type-changed"}
)
continue
if (
metadata.st_dev != record.get("device")
or metadata.st_ino != record.get("inode")
or metadata.st_size != record.get("logical_bytes")
or metadata.st_mtime_ns != record.get("mtime_ns")
or metadata.st_ctime_ns != record.get("ctime_ns")
or _sha256(path) != record.get("sha256")
):
retained.append(
{"path": relative.as_posix(), "reason": "identity-changed"}
)
continue
path.unlink()
removed.append(relative.as_posix())
parent = path.parent
while parent != qb_root:
directories.add(parent)
parent = parent.parent
removed_directories: list[str] = []
for directory in sorted(
directories, key=lambda item: len(item.parts), reverse=True
):
try:
directory.rmdir()
removed_directories.append(
directory.relative_to(qb_root).as_posix()
)
except OSError as exc:
if exc.errno not in {errno.ENOTEMPTY, errno.ENOENT}:
raise
result = {
"removed_files": removed,
"retained_files": retained,
"removed_directories": removed_directories,
}
return store.put_job_artifact(job_id, "eviction-unlinked", result)["value"]
def compensate_materialized_files(
*, job_id: str, qb_root: Path, store: ClientStore
) -> list[str]:
"""Remove only target files this job created and whose inode still matches."""
removed: list[str] = []
directories: set[Path] = set()
for row in store.file_operation_rows(job_id):
if row["state"] != "completed" or row["result_json"] is None:
continue
import json
intent = json.loads(str(row["intent_json"]))
result = json.loads(str(row["result_json"]))
if result.get("destination_preexisted"):
continue
relative_value = intent.get("destination")
if not isinstance(relative_value, str):
continue
relative = _relative(relative_value)
path = qb_root.joinpath(*relative.parts)
try:
metadata = path.lstat()
except FileNotFoundError:
continue
if (
not stat.S_ISREG(metadata.st_mode)
or metadata.st_dev != result.get("destination_device")
or metadata.st_ino != result.get("destination_inode")
):
continue
path.unlink()
removed.append(relative.as_posix())
parent = path.parent
while parent != qb_root:
directories.add(parent)
parent = parent.parent
for directory in sorted(
directories, key=lambda item: len(item.parts), reverse=True
):
try:
directory.rmdir()
except OSError as exc:
if exc.errno not in {errno.ENOTEMPTY, errno.ENOENT}:
raise
return removed
def _remaining_paths(resources: Iterable[NormalizedResource]) -> set[str]:
return {
item.canonical_path
for resource in resources
for item in resource.files
}
def _relative(value: str) -> PurePosixPath:
candidate = PurePosixPath(value)
if (
not value
or candidate.is_absolute()
or ".." in candidate.parts
or "." in candidate.parts
or "\\" in value
):
raise EvictionError("eviction path is unsafe")
return candidate
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as source:
while chunk := source.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
+561 -29
View File
@@ -3,12 +3,22 @@
from __future__ import annotations from __future__ import annotations
import hashlib import hashlib
import json
import shutil
import threading
import time import time
import uuid import uuid
from pathlib import Path, PurePosixPath from pathlib import Path, PurePosixPath
from typing import Callable from typing import Callable
from archive_clients.protocol import decode_message, encode_message from archive_clients.protocol import decode_message, encode_message
from archive_clients.eviction import (
EvictionError,
compensate_materialized_files,
remove_qb_entry,
safe_unlink,
verify_and_snapshot,
)
from archive_clients.qbittorrent import ( from archive_clients.qbittorrent import (
QBittorrentDownloadAttempt, QBittorrentDownloadAttempt,
QBittorrentReader, QBittorrentReader,
@@ -37,6 +47,10 @@ class JobExecutionError(RuntimeError):
pass pass
class JobCancelled(JobExecutionError):
pass
class ClientJobExecutor: class ClientJobExecutor:
def __init__( def __init__(
self, self,
@@ -50,6 +64,8 @@ class ClientJobExecutor:
syncthing_transport: object, syncthing_transport: object,
sparse_supported: bool, sparse_supported: bool,
poll_interval: float = 1, poll_interval: float = 1,
verification_timeout: float = 30 * 60,
free_space_reserve_bytes: int = 1024 * 1024 * 1024,
): ):
self.client_id = client_id self.client_id = client_id
self.qbittorrent = qbittorrent self.qbittorrent = qbittorrent
@@ -60,6 +76,12 @@ class ClientJobExecutor:
self.syncthing_transport = syncthing_transport self.syncthing_transport = syncthing_transport
self.sparse_supported = sparse_supported self.sparse_supported = sparse_supported
self.poll_interval = poll_interval self.poll_interval = poll_interval
self.verification_timeout = verification_timeout
self.free_space_reserve_bytes = free_space_reserve_bytes
self._cancel_events: dict[str, threading.Event] = {}
def request_cancel(self, job_id: str) -> None:
self._cancel_events.setdefault(job_id, threading.Event()).set()
def assign( def assign(
self, command: control_pb2.AssignJobCommand self, command: control_pb2.AssignJobCommand
@@ -83,7 +105,9 @@ class ClientJobExecutor:
return [event] return [event]
def execute( def execute(
self, command: control_pb2.ExecuteStepCommand self,
command: control_pb2.ExecuteStepCommand,
event_callback: Callable[[control_pb2.JobEvent], None] | None = None,
) -> list[control_pb2.JobEvent]: ) -> list[control_pb2.JobEvent]:
definition = self._definition(command.job_id) definition = self._definition(command.job_id)
replay = self._replay( replay = self._replay(
@@ -97,6 +121,9 @@ class ClientJobExecutor:
control_pb2.JOB_EVENT_TYPE_CLEANUP_REQUIRED, control_pb2.JOB_EVENT_TYPE_CLEANUP_REQUIRED,
control_pb2.JOB_EVENT_TYPE_CANCELLED, control_pb2.JOB_EVENT_TYPE_CANCELLED,
}: }:
if event_callback is not None:
for event in replay:
event_callback(event)
return replay return replay
started = replay[0] if replay else self._event( started = replay[0] if replay else self._event(
definition, definition,
@@ -107,23 +134,124 @@ class ClientJobExecutor:
committed=( committed=(
self._committed(command.job_id) self._committed(command.job_id)
or command.step == job_pb2.JOB_STEP_KIND_STAGING_CLEANUP or command.step == job_pb2.JOB_STEP_KIND_STAGING_CLEANUP
or (
command.step == job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK
and self.store.get_job_artifact(
command.job_id, "qb-entry-removed"
) is not None
)
), ),
step=command.step, step=command.step,
step_state=job_pb2.STEP_STATE_RUNNING, step_state=job_pb2.STEP_STATE_RUNNING,
) )
if not replay: if not replay:
self._record(definition, started) self._record(definition, started)
if event_callback is not None:
event_callback(started)
emitted = [started]
cursor = started
speed_sample = [time.monotonic(), 0]
def progress(
fraction: float,
bytes_complete: int = 0,
bytes_total: int = 0,
detail: str = "",
) -> None:
nonlocal cursor
now = time.monotonic()
elapsed = now - float(speed_sample[0])
if fraction < 1 and elapsed < 1:
return
speed = (
max(0, bytes_complete - int(speed_sample[1])) / elapsed
if elapsed > 0 else 0
)
event = self._event(
definition,
sequence=cursor.sequence + 1,
revision=cursor.job_revision + 1,
event_type=control_pb2.JOB_EVENT_TYPE_PROGRESS,
state=job_pb2.JOB_STATE_RUNNING,
committed=cursor.committed,
step=command.step,
step_state=job_pb2.STEP_STATE_RUNNING,
)
event.progress.fraction_complete = max(
0.0, min(float(fraction), 1.0)
)
event.progress.bytes_complete = max(0, bytes_complete)
event.progress.bytes_total = max(0, bytes_total)
event.progress.approximate_bytes_per_second = speed
event.progress.detail = detail
completed_steps = max(0, _step_number(command.step) - 1)
event.progress.overall_fraction_complete = (
completed_steps + event.progress.fraction_complete
) / event.progress.display_step_total
self._record(definition, event)
emitted.append(event)
cursor = event
speed_sample[:] = [now, bytes_complete]
if event_callback is not None:
event_callback(event)
try: try:
result = self._execute_step(definition, command.step) result = self._execute_step(definition, command.step, progress)
except JobCancelled as error:
cancelling = self._event(
definition,
sequence=cursor.sequence + 1,
revision=cursor.job_revision + 1,
event_type=control_pb2.JOB_EVENT_TYPE_CANCELLING,
state=job_pb2.JOB_STATE_CANCELLING,
committed=started.committed,
step=command.step,
step_state=job_pb2.STEP_STATE_CANCELLED,
)
cancelling.error.code = common_pb2.ERROR_CODE_CANCELLED
cancelling.error.message = str(error)
self._record(definition, cancelling)
emitted.append(cancelling)
if event_callback is not None:
event_callback(cancelling)
return emitted
except Exception as error: except Exception as error:
rollback_error: Exception | None = None
if (
definition.WhichOneof("spec") == "transfer"
and not started.committed
):
try:
if (
self.client_id == definition.transfer.target_client_id
and command.step in {
job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE,
job_pb2.JOB_STEP_KIND_QB_VERIFY,
}
):
self._rollback_target(definition)
elif (
self.client_id == definition.transfer.source_client_id
and command.step == job_pb2.JOB_STEP_KIND_SOURCE_STAGE
):
self._cleanup_staging(definition)
except Exception as compensation_error:
rollback_error = compensation_error
cleanup = ( cleanup = (
(
command.step == job_pb2.JOB_STEP_KIND_STAGING_CLEANUP command.step == job_pb2.JOB_STEP_KIND_STAGING_CLEANUP
and started.committed and started.committed
) )
or (
command.step == job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK
and self.store.get_job_artifact(
definition.job_id, "qb-entry-removed"
) is not None
)
)
failed = self._event( failed = self._event(
definition, definition,
sequence=started.sequence + 1, sequence=cursor.sequence + 1,
revision=started.job_revision + 1, revision=cursor.job_revision + 1,
event_type=( event_type=(
control_pb2.JOB_EVENT_TYPE_CLEANUP_REQUIRED control_pb2.JOB_EVENT_TYPE_CLEANUP_REQUIRED
if cleanup else control_pb2.JOB_EVENT_TYPE_FAILED if cleanup else control_pb2.JOB_EVENT_TYPE_FAILED
@@ -139,11 +267,23 @@ class ClientJobExecutor:
failed.error.code = _job_error_code(error) failed.error.code = _job_error_code(error)
failed.error.message = str(error) or type(error).__name__ failed.error.message = str(error) or type(error).__name__
failed.error.retryable = cleanup failed.error.retryable = cleanup
if rollback_error is not None:
failed.error.code = (
common_pb2.ERROR_CODE_MANUAL_INTERVENTION_REQUIRED
)
failed.error.message = (
f"{failed.error.message}; compensation also failed: "
f"{rollback_error}"
)
failed.error.retryable = False
self._record(definition, failed) self._record(definition, failed)
return [started, failed] emitted.append(failed)
sequence = started.sequence + 1 if event_callback is not None:
revision = started.job_revision + 1 event_callback(failed)
committed = started.committed return emitted
sequence = cursor.sequence + 1
revision = cursor.job_revision + 1
committed = cursor.committed
event_type = control_pb2.JOB_EVENT_TYPE_STEP_SUCCEEDED event_type = control_pb2.JOB_EVENT_TYPE_STEP_SUCCEEDED
state = job_pb2.JOB_STATE_RUNNING state = job_pb2.JOB_STATE_RUNNING
if command.step == job_pb2.JOB_STEP_KIND_QB_VERIFY: if command.step == job_pb2.JOB_STEP_KIND_QB_VERIFY:
@@ -152,6 +292,10 @@ class ClientJobExecutor:
elif command.step == job_pb2.JOB_STEP_KIND_STAGING_CLEANUP: elif command.step == job_pb2.JOB_STEP_KIND_STAGING_CLEANUP:
event_type = control_pb2.JOB_EVENT_TYPE_SUCCEEDED event_type = control_pb2.JOB_EVENT_TYPE_SUCCEEDED
state = job_pb2.JOB_STATE_SUCCEEDED state = job_pb2.JOB_STATE_SUCCEEDED
elif command.step == job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK:
committed = True
event_type = control_pb2.JOB_EVENT_TYPE_SUCCEEDED
state = job_pb2.JOB_STATE_SUCCEEDED
succeeded = self._event( succeeded = self._event(
definition, definition,
sequence=sequence, sequence=sequence,
@@ -165,24 +309,91 @@ class ClientJobExecutor:
if result is not None: if result is not None:
succeeded.observed_placement.CopyFrom(result) succeeded.observed_placement.CopyFrom(result)
self._record(definition, succeeded) self._record(definition, succeeded)
return [started, succeeded] emitted.append(succeeded)
if event_callback is not None:
event_callback(succeeded)
return emitted
def cancel(
self, command: control_pb2.CancelJobCommand
) -> list[control_pb2.JobEvent]:
definition = self._definition(command.job_id)
self.request_cancel(command.job_id)
replay = self._replay(
command.job_id, command.expected_last_event_sequence
)
if replay:
return replay
committed = self._committed(command.job_id)
if definition.WhichOneof("spec") == "eviction":
removed = self.store.get_job_artifact(
definition.job_id, "qb-entry-removed"
)
placement = None
if removed is not None:
placement = self._execute_eviction_step(
definition, job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK
)
committed = True
event_type = control_pb2.JOB_EVENT_TYPE_CANCELLED
state = job_pb2.JOB_STATE_CANCELLED
elif committed:
if definition.transfer.source_client_id == self.client_id:
self._cleanup_staging(definition)
event_type = control_pb2.JOB_EVENT_TYPE_CANCELLED
state = job_pb2.JOB_STATE_CANCELLED
placement = None
elif definition.transfer.source_client_id == self.client_id:
self._cleanup_staging(definition)
temporary = self._metainfo_path(definition.job_id)
if temporary.exists():
temporary.unlink()
event_type = control_pb2.JOB_EVENT_TYPE_ROLLBACK_SUCCEEDED
state = job_pb2.JOB_STATE_ROLLING_BACK
placement = None
else:
self._rollback_target(definition)
self._cleanup_staging(definition)
event_type = control_pb2.JOB_EVENT_TYPE_CANCELLED
state = job_pb2.JOB_STATE_CANCELLED
placement = None
event = self._event(
definition,
sequence=command.expected_last_event_sequence + 1,
revision=command.expected_job_revision + 1,
event_type=event_type,
state=state,
committed=committed,
step=job_pb2.JOB_STEP_KIND_ROLLBACK,
step_state=job_pb2.STEP_STATE_SUCCEEDED,
)
if placement is not None:
event.observed_placement.CopyFrom(placement)
self._record(definition, event)
return [event]
def _execute_step( def _execute_step(
self, definition: job_pb2.JobDefinition, step: int self,
definition: job_pb2.JobDefinition,
step: int,
progress: Callable[[float, int, int, str], None],
): ):
self._raise_if_cancelled(definition.job_id)
if definition.WhichOneof("spec") == "eviction":
return self._execute_eviction_step(definition, step)
if step == job_pb2.JOB_STEP_KIND_SOURCE_STAGE: if step == job_pb2.JOB_STEP_KIND_SOURCE_STAGE:
self._source_stage(definition) self._source_stage(definition, progress)
return None return None
if step == job_pb2.JOB_STEP_KIND_SYNCTHING_TRANSFER: if step == job_pb2.JOB_STEP_KIND_SYNCTHING_TRANSFER:
self._wait_for_syncthing(definition) self._wait_for_syncthing(definition, progress)
return None return None
if step == job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE: if step == job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE:
self._target_materialize(definition) self._target_materialize(definition, progress)
return None return None
if step == job_pb2.JOB_STEP_KIND_QB_VERIFY: if step == job_pb2.JOB_STEP_KIND_QB_VERIFY:
return self._qb_verify(definition) return self._qb_verify(definition, progress)
if step == job_pb2.JOB_STEP_KIND_STAGING_CLEANUP: if step == job_pb2.JOB_STEP_KIND_STAGING_CLEANUP:
cleanup_transfer(self._job_directory(definition)) self._cleanup_staging(definition)
temporary_metainfo = self._metainfo_path(definition.job_id) temporary_metainfo = self._metainfo_path(definition.job_id)
if temporary_metainfo.exists(): if temporary_metainfo.exists():
temporary_metainfo.unlink() temporary_metainfo.unlink()
@@ -197,10 +408,86 @@ class ClientJobExecutor:
return None return None
raise JobExecutionError("job step is unsupported") raise JobExecutionError("job step is unsupported")
def _source_stage(self, definition: job_pb2.JobDefinition) -> None: def _execute_eviction_step(
self, definition: job_pb2.JobDefinition, step: int
):
if definition.eviction.cache_client_id != self.client_id:
raise JobExecutionError("eviction step was sent to the wrong client")
info_hash = _info_hash(definition)
if step == job_pb2.JOB_STEP_KIND_VERIFY_ARCHIVE_COVERAGE:
covered = {
index
for proof in definition.eviction.archive_coverage
for index in _selection_indices(proof.covered_files)
}
requested = set(
_selection_indices(definition.eviction.files_to_evict)
)
if not requested or requested - covered:
raise JobExecutionError(
"archive coverage no longer covers the cache selection"
)
resource = self._resource(definition)
current = _resource_fingerprint(resource, self.client_id)
if not _fingerprint_matches(
current, definition.eviction.cache_fingerprint
):
raise JobExecutionError(
"cache resource changed after eviction confirmation"
)
verify_and_snapshot(
job_id=definition.job_id,
resource=resource,
selected_indices=requested,
qb_root=self.qb_root,
store=self.store,
)
return None
if step == job_pb2.JOB_STEP_KIND_QB_REMOVE_ENTRY:
remove_qb_entry(
job_id=definition.job_id,
torrent_hash=info_hash,
qbittorrent=self.qbittorrent,
store=self.store,
)
return None
if step == job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK:
safe_unlink(
job_id=definition.job_id,
qb_root=self.qb_root,
qbittorrent=self.qbittorrent,
store=self.store,
)
placement = resource_pb2.Placement(
client_id=self.client_id,
state=resource_pb2.PLACEMENT_STATE_ABSENT,
generation=definition.eviction.cache_placement_generation + 1,
created_by_job_id=definition.job_id,
)
placement.resource_id.CopyFrom(definition.resource_id)
placement.fingerprint.CopyFrom(
definition.eviction.cache_fingerprint
)
placement.verified_at.GetCurrentTime()
return placement
raise JobExecutionError("eviction job step is unsupported")
def _source_stage(
self,
definition: job_pb2.JobDefinition,
progress: Callable[[float, int, int, str], None],
) -> None:
if definition.transfer.source_client_id != self.client_id: if definition.transfer.source_client_id != self.client_id:
raise JobExecutionError("source stage was sent to the wrong client") raise JobExecutionError("source stage was sent to the wrong client")
resource = self._resource(definition) resource = self._resource(definition)
if not _fingerprint_matches(
_resource_fingerprint(resource, self.client_id),
definition.transfer.source_fingerprint,
):
raise JobExecutionError(
"source resource changed after job confirmation"
)
self._reject_unsafe_partfile(definition, resource)
indices = _selection_indices(definition.transfer.transfer_delta_files) indices = _selection_indices(definition.transfer.transfer_delta_files)
by_index = {item.file_index: item for item in resource.files} by_index = {item.file_index: item for item in resource.files}
if not indices or any(index not in by_index for index in indices): if not indices or any(index not in by_index for index in indices):
@@ -254,6 +541,11 @@ class ClientJobExecutor:
sha256_hex=hashlib.sha256(resource.metainfo_bytes).hexdigest(), sha256_hex=hashlib.sha256(resource.metainfo_bytes).hexdigest(),
) )
del artifact del artifact
self._require_space(
self.route_path(definition.transfer.route_id),
definition.transfer.transfer_delta_logical_bytes
+ len(resource.metainfo_bytes),
)
stage_transfer( stage_transfer(
manifest, manifest,
source_root=self.qb_root, source_root=self.qb_root,
@@ -261,18 +553,43 @@ class ClientJobExecutor:
store=self.store, store=self.store,
artifact_sources={"metainfo/source.torrent": metainfo_path}, artifact_sources={"metainfo/source.torrent": metainfo_path},
sparse_supported=self.sparse_supported, sparse_supported=self.sparse_supported,
cancel_check=lambda: self._raise_if_cancelled(
definition.job_id
),
progress=lambda completed, total: progress(
completed / total if total else 1,
completed,
total,
f"staged {completed} of {total} bytes",
),
) )
self._observer(definition).rescan() self._observer(definition).rescan()
def _wait_for_syncthing(self, definition: job_pb2.JobDefinition) -> None: def _wait_for_syncthing(
self,
definition: job_pb2.JobDefinition,
progress: Callable[[float, int, int, str], None],
) -> None:
if definition.transfer.target_client_id != self.client_id: if definition.transfer.target_client_id != self.client_id:
raise JobExecutionError( raise JobExecutionError(
"Syncthing completion was sent to the wrong client" "Syncthing completion was sent to the wrong client"
) )
observer = self._observer(definition) observer = self._observer(definition)
last_reported: tuple[int, int] | None = None
while True: while True:
self._raise_if_cancelled(definition.job_id)
try: try:
if observer.status().complete: status = observer.status()
sample = (status.bytes_complete, status.bytes_total)
if sample != last_reported:
progress(
status.fraction_complete,
status.bytes_complete,
status.bytes_total,
f"{status.needed_items} Syncthing items still needed",
)
last_reported = sample
if status.complete:
return return
except (FileNotFoundError, TransferIntegrityError): except (FileNotFoundError, TransferIntegrityError):
# Syncthing can expose the per-job directory before the # Syncthing can expose the per-job directory before the
@@ -280,20 +597,73 @@ class ClientJobExecutor:
pass pass
time.sleep(self.poll_interval) time.sleep(self.poll_interval)
def _target_materialize(self, definition: job_pb2.JobDefinition) -> None: def _target_materialize(
self,
definition: job_pb2.JobDefinition,
progress: Callable[[float, int, int, str], None],
) -> None:
if definition.transfer.target_client_id != self.client_id: if definition.transfer.target_client_id != self.client_id:
raise JobExecutionError( raise JobExecutionError(
"target materialization was sent to the wrong client" "target materialization was sent to the wrong client"
) )
published = load_published_transfer(self._job_directory(definition)) published = load_published_transfer(self._job_directory(definition))
self._require_space(
self.qb_root,
definition.transfer.transfer_delta_logical_bytes,
)
info_hash = _info_hash(definition)
resource = self.qbittorrent.get_resource(info_hash)
if resource is not None:
self._reject_unsafe_partfile(definition, resource)
if definition.transfer.HasField("target_baseline_fingerprint"):
if resource is None or not _fingerprint_matches(
_resource_fingerprint(resource, self.client_id),
definition.transfer.target_baseline_fingerprint,
):
raise JobExecutionError(
"target resource changed after job confirmation"
)
elif resource is not None:
raise JobExecutionError(
"target resource appeared after job confirmation"
)
baseline = {
"present": resource is not None,
"selected_file_indices": (
_selection_indices(resource.summary.selected_files)
if resource is not None else []
),
"stopped": (
resource is not None
and resource.summary.runtime_state
== resource_pb2.TORRENT_RUNTIME_STATE_STOPPED
),
"total_file_count": len(published.manifest.files),
}
self.store.put_job_artifact(
definition.job_id, "target-baseline", baseline
)
materialize_transfer( materialize_transfer(
published, published,
target_root=self.qb_root, target_root=self.qb_root,
store=self.store, store=self.store,
sparse_supported=self.sparse_supported, sparse_supported=self.sparse_supported,
cancel_check=lambda: self._raise_if_cancelled(
definition.job_id
),
progress=lambda completed, total: progress(
completed / total if total else 1,
completed,
total,
f"materialized {completed} of {total} bytes",
),
) )
def _qb_verify(self, definition: job_pb2.JobDefinition): def _qb_verify(
self,
definition: job_pb2.JobDefinition,
progress: Callable[[float, int, int, str], None],
):
if definition.transfer.target_client_id != self.client_id: if definition.transfer.target_client_id != self.client_id:
raise JobExecutionError("qB verification was sent to the wrong client") raise JobExecutionError("qB verification was sent to the wrong client")
published = load_published_transfer(self._job_directory(definition)) published = load_published_transfer(self._job_directory(definition))
@@ -326,7 +696,16 @@ class ClientJobExecutor:
) )
self.qbittorrent.set_selection(info_hash, selected, total_files) self.qbittorrent.set_selection(info_hash, selected, total_files)
self.qbittorrent.recheck_and_wait( self.qbittorrent.recheck_and_wait(
info_hash, selected, timeout=30 * 60, poll_interval=self.poll_interval info_hash,
selected,
timeout=self.verification_timeout,
poll_interval=self.poll_interval,
cancel_check=lambda: self._raise_if_cancelled(
definition.job_id
),
progress_callback=lambda fraction: progress(
fraction, 0, 0, "qBittorrent stopped recheck"
),
) )
if should_start: if should_start:
self.qbittorrent.start(info_hash) self.qbittorrent.start(info_hash)
@@ -359,6 +738,102 @@ class ClientJobExecutor:
placement.verified_at.GetCurrentTime() placement.verified_at.GetCurrentTime()
return placement return placement
def _rollback_target(self, definition: job_pb2.JobDefinition) -> None:
baseline_row = self.store.get_job_artifact(
definition.job_id, "target-baseline"
)
if baseline_row is None:
compensate_materialized_files(
job_id=definition.job_id,
qb_root=self.qb_root,
store=self.store,
)
return
baseline = baseline_row["value"]
info_hash = _info_hash(definition)
present = self.qbittorrent.get_resource(info_hash)
if present is not None:
if baseline.get("present"):
selected = baseline.get("selected_file_indices")
if isinstance(selected, list) and selected:
self.qbittorrent.set_selection(
info_hash, selected, len(present.files)
)
if baseline.get("stopped"):
self.qbittorrent.stop(info_hash)
else:
self.qbittorrent.start(info_hash)
else:
self.qbittorrent.delete_entry(info_hash)
compensate_materialized_files(
job_id=definition.job_id,
qb_root=self.qb_root,
store=self.store,
)
def _cleanup_staging(self, definition: job_pb2.JobDefinition) -> None:
job_directory = self._job_directory(definition)
try:
if cleanup_transfer(job_directory):
return
except FileNotFoundError:
pass
if not job_directory.is_dir():
return
compensate_materialized_files(
job_id=definition.job_id,
qb_root=job_directory,
store=self.store,
)
for name in ("ready.json", "manifest.json"):
candidate = job_directory / name
if candidate.is_file() and not candidate.is_symlink():
candidate.unlink()
try:
job_directory.rmdir()
except OSError:
pass
def _reject_unsafe_partfile(
self,
definition: job_pb2.JobDefinition,
resource: NormalizedResource,
) -> None:
roots = {self.qb_root}
for item in resource.files:
candidate = self.qb_root / PurePosixPath(item.canonical_path).parts[0]
roots.add(candidate if candidate.is_dir() else candidate.parent)
hashes = {
value.lower() for value in (
definition.resource_id.info_hash_v1_hex,
definition.resource_id.info_hash_v2_hex,
) if value
}
for root in roots:
if not root.is_dir():
continue
for candidate in root.glob("*.parts"):
lowered = candidate.name.lower()
if lowered == ".parts" or any(value in lowered for value in hashes):
raise JobExecutionError(
"resource uses a qBittorrent partfile that cannot be "
"safely transferred by this client"
)
def _raise_if_cancelled(self, job_id: str) -> None:
event = self._cancel_events.get(job_id)
if event is not None and event.is_set():
raise JobCancelled("job cancellation requested")
def _require_space(self, root: Path, required_bytes: int) -> None:
available = shutil.disk_usage(root).free
required = required_bytes + self.free_space_reserve_bytes
if available < required:
raise JobExecutionError(
f"insufficient free space: {available} bytes available, "
f"{required} bytes required including reserve"
)
def _observer( def _observer(
self, definition: job_pb2.JobDefinition self, definition: job_pb2.JobDefinition
) -> SyncthingTransferObserver: ) -> SyncthingTransferObserver:
@@ -455,21 +930,29 @@ class ClientJobExecutor:
) )
event.progress.overall_fraction_complete = _overall(step, step_state) event.progress.overall_fraction_complete = _overall(step, step_state)
event.progress.display_step_number = _step_number(step) event.progress.display_step_number = _step_number(step)
event.progress.display_step_total = 5 event.progress.display_step_total = (
3
if definition.operation == job_pb2.JOB_OPERATION_EVICT_CACHE
else 5
)
event.progress.last_progress_at.CopyFrom(event.occurred_at) event.progress.last_progress_at.CopyFrom(event.occurred_at)
return event return event
@staticmethod @staticmethod
def _validate_definition(definition: job_pb2.JobDefinition) -> None: def _validate_definition(definition: job_pb2.JobDefinition) -> None:
if ( if not definition.job_id:
not definition.job_id raise JobExecutionError("job definition has no ID")
or definition.operation not in { if definition.operation in {
job_pb2.JOB_OPERATION_ARCHIVE, job_pb2.JOB_OPERATION_ARCHIVE,
job_pb2.JOB_OPERATION_UNARCHIVE, job_pb2.JOB_OPERATION_UNARCHIVE,
} } and definition.WhichOneof("spec") == "transfer":
or definition.WhichOneof("spec") != "transfer" return
if (
definition.operation == job_pb2.JOB_OPERATION_EVICT_CACHE
and definition.WhichOneof("spec") == "eviction"
): ):
raise JobExecutionError("transfer job definition is invalid") return
raise JobExecutionError("job definition operation/spec is invalid")
def _selection_indices(selection) -> list[int]: def _selection_indices(selection) -> list[int]:
@@ -510,12 +993,27 @@ def _step_number(step: int) -> int:
job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE: 3, job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE: 3,
job_pb2.JOB_STEP_KIND_QB_VERIFY: 4, job_pb2.JOB_STEP_KIND_QB_VERIFY: 4,
job_pb2.JOB_STEP_KIND_STAGING_CLEANUP: 5, job_pb2.JOB_STEP_KIND_STAGING_CLEANUP: 5,
job_pb2.JOB_STEP_KIND_VERIFY_ARCHIVE_COVERAGE: 1,
job_pb2.JOB_STEP_KIND_QB_REMOVE_ENTRY: 2,
job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK: 3,
job_pb2.JOB_STEP_KIND_ROLLBACK: 1,
}.get(step, 0) }.get(step, 0)
def _overall(step: int, state: int) -> float: def _overall(step: int, state: int) -> float:
number = _step_number(step) number = _step_number(step)
return max(0, number - (0 if state == job_pb2.STEP_STATE_SUCCEEDED else 1)) / 5 total = (
3
if step in {
job_pb2.JOB_STEP_KIND_VERIFY_ARCHIVE_COVERAGE,
job_pb2.JOB_STEP_KIND_QB_REMOVE_ENTRY,
job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK,
}
else 5
)
return max(
0, number - (0 if state == job_pb2.STEP_STATE_SUCCEEDED else 1)
) / total
def decode_message_metainfo(path: Path): def decode_message_metainfo(path: Path):
@@ -543,6 +1041,38 @@ def _resource_fingerprint(
return fingerprint return fingerprint
def _info_hash(definition: job_pb2.JobDefinition) -> str:
value = (
definition.resource_id.info_hash_v1_hex
or definition.resource_id.info_hash_v2_hex
)
if not value:
raise JobExecutionError("job resource identity has no info hash")
return value
def _fingerprint_matches(
current: resource_pb2.ResourceStateFingerprint,
expected: resource_pb2.ResourceStateFingerprint,
) -> bool:
left = resource_pb2.ResourceStateFingerprint()
right = resource_pb2.ResourceStateFingerprint()
left.CopyFrom(current)
right.CopyFrom(expected)
left.ClearField("observed_at")
right.ClearField("observed_at")
for value in (left, right):
if value.runtime_state in {
resource_pb2.TORRENT_RUNTIME_STATE_QUEUED,
resource_pb2.TORRENT_RUNTIME_STATE_SEEDING,
resource_pb2.TORRENT_RUNTIME_STATE_STALLED,
}:
value.runtime_state = (
resource_pb2.TORRENT_RUNTIME_STATE_SEEDING
)
return left == right
def _job_error_code(error: Exception) -> int: def _job_error_code(error: Exception) -> int:
if isinstance(error, QBittorrentDownloadAttempt): if isinstance(error, QBittorrentDownloadAttempt):
return common_pb2.ERROR_CODE_MANUAL_INTERVENTION_REQUIRED return common_pb2.ERROR_CODE_MANUAL_INTERVENTION_REQUIRED
@@ -554,4 +1084,6 @@ def _job_error_code(error: Exception) -> int:
return common_pb2.ERROR_CODE_PRECONDITION_FAILED return common_pb2.ERROR_CODE_PRECONDITION_FAILED
if isinstance(error, TransferError): if isinstance(error, TransferError):
return common_pb2.ERROR_CODE_PATH_CONFLICT return common_pb2.ERROR_CODE_PATH_CONFLICT
if isinstance(error, EvictionError):
return common_pb2.ERROR_CODE_PRECONDITION_FAILED
return common_pb2.ERROR_CODE_INTERNAL return common_pb2.ERROR_CODE_INTERNAL
+12
View File
@@ -10,6 +10,7 @@ from dataclasses import dataclass
from http.cookiejar import CookieJar from http.cookiejar import CookieJar
from pathlib import PurePosixPath from pathlib import PurePosixPath
from typing import Any from typing import Any
from typing import Callable
from urllib import error, parse, request from urllib import error, parse, request
from archive_clients.config import ServiceConfig from archive_clients.config import ServiceConfig
@@ -259,6 +260,8 @@ class QBittorrentReader:
*, *,
timeout: float, timeout: float,
poll_interval: float = 1, poll_interval: float = 1,
cancel_check: Callable[[], None] | None = None,
progress_callback: Callable[[float], None] | None = None,
) -> RecheckResult: ) -> RecheckResult:
selected = tuple(sorted(set(selected_file_indices))) selected = tuple(sorted(set(selected_file_indices)))
if not selected: if not selected:
@@ -271,7 +274,9 @@ class QBittorrentReader:
"/api/v2/torrents/recheck", {"hashes": torrent_hash} "/api/v2/torrents/recheck", {"hashes": torrent_hash}
) )
deadline = time.monotonic() + timeout deadline = time.monotonic() + timeout
cancel_check = cancel_check or (lambda: None)
while True: while True:
cancel_check()
record = self._torrent_record(torrent_hash) record = self._torrent_record(torrent_hash)
state = record.get("state") state = record.get("state")
if not isinstance(state, str): if not isinstance(state, str):
@@ -295,6 +300,13 @@ class QBittorrentReader:
item.get("index"): item.get("progress") item.get("index"): item.get("progress")
for item in files for item in files
} }
if progress_callback is not None:
progress_callback(
sum(
max(0.0, min(float(progress.get(index, 0)), 1.0))
for index in selected
) / len(selected)
)
if all( if all(
isinstance(progress.get(index), (int, float)) isinstance(progress.get(index), (int, float))
and float(progress[index]) >= 1 and float(progress[index]) >= 1
+66 -1
View File
@@ -11,7 +11,7 @@ from dataclasses import dataclass
from pathlib import Path from pathlib import Path
SCHEMA_VERSION = 2 SCHEMA_VERSION = 3
class CommandConflict(RuntimeError): class CommandConflict(RuntimeError):
@@ -58,6 +58,10 @@ class ClientStore:
if version == 1: if version == 1:
connection.executescript(_SCHEMA_V2) connection.executescript(_SCHEMA_V2)
connection.execute("PRAGMA user_version = 2") connection.execute("PRAGMA user_version = 2")
version = 2
if version == 2:
connection.executescript(_SCHEMA_V3)
connection.execute("PRAGMA user_version = 3")
if connection.execute("PRAGMA foreign_key_check").fetchall(): if connection.execute("PRAGMA foreign_key_check").fetchall():
raise RuntimeError("client database foreign-key check failed") raise RuntimeError("client database foreign-key check failed")
if not existed: if not existed:
@@ -268,6 +272,57 @@ class ClientStore:
).fetchall() ).fetchall()
return [dict(row) for row in rows] return [dict(row) for row in rows]
def put_job_artifact(
self, job_id: str, kind: str, value: dict[str, object]
) -> dict[str, object]:
encoded = _canonical(value)
with self._connect() as connection:
connection.execute("BEGIN IMMEDIATE")
existing = connection.execute(
"""
SELECT value_json FROM job_artifacts
WHERE job_id = ? AND kind = ?
""",
(job_id, kind),
).fetchone()
if existing is not None and existing["value_json"] != encoded:
raise JobConflict(f"job artifact {kind} is immutable")
connection.execute(
"""
INSERT INTO job_artifacts (job_id, kind, value_json)
VALUES (?, ?, ?)
ON CONFLICT(job_id, kind) DO NOTHING
""",
(job_id, kind, encoded),
)
row = connection.execute(
"""
SELECT job_id, kind, value_json FROM job_artifacts
WHERE job_id = ? AND kind = ?
""",
(job_id, kind),
).fetchone()
result = dict(row)
result["value"] = json.loads(str(result.pop("value_json")))
return result
def get_job_artifact(
self, job_id: str, kind: str
) -> dict[str, object] | None:
with self._connect() as connection:
row = connection.execute(
"""
SELECT job_id, kind, value_json FROM job_artifacts
WHERE job_id = ? AND kind = ?
""",
(job_id, kind),
).fetchone()
if row is None:
return None
result = dict(row)
result["value"] = json.loads(str(result.pop("value_json")))
return result
def record_job_event( def record_job_event(
self, self,
*, *,
@@ -567,3 +622,13 @@ CREATE TABLE route_attempt_updates (
PRIMARY KEY(command_id, sequence) PRIMARY KEY(command_id, sequence)
); );
""" """
_SCHEMA_V3 = """
CREATE TABLE job_artifacts (
job_id TEXT NOT NULL REFERENCES jobs(job_id),
kind TEXT NOT NULL,
value_json TEXT NOT NULL,
PRIMARY KEY(job_id, kind)
);
"""
+53 -3
View File
@@ -11,7 +11,7 @@ import stat
import uuid import uuid
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path, PurePosixPath from pathlib import Path, PurePosixPath
from typing import Mapping from typing import Callable, Mapping
from google.protobuf import json_format from google.protobuf import json_format
@@ -79,6 +79,7 @@ class FileMaterializer:
allow_hardlink: bool = True, allow_hardlink: bool = True,
allow_reflink: bool = True, allow_reflink: bool = True,
copy_chunk_bytes: int = _COPY_CHUNK_BYTES, copy_chunk_bytes: int = _COPY_CHUNK_BYTES,
cancel_check: Callable[[], None] | None = None,
): ):
if copy_chunk_bytes < 1: if copy_chunk_bytes < 1:
raise ValueError("copy_chunk_bytes must be positive") raise ValueError("copy_chunk_bytes must be positive")
@@ -86,6 +87,7 @@ class FileMaterializer:
self.allow_hardlink = allow_hardlink self.allow_hardlink = allow_hardlink
self.allow_reflink = allow_reflink self.allow_reflink = allow_reflink
self.copy_chunk_bytes = copy_chunk_bytes self.copy_chunk_bytes = copy_chunk_bytes
self.cancel_check = cancel_check or (lambda: None)
def materialize( def materialize(
self, self,
@@ -100,6 +102,7 @@ class FileMaterializer:
allow_preexisting_reuse: bool, allow_preexisting_reuse: bool,
sparse_supported: bool, sparse_supported: bool,
) -> MaterializedFile: ) -> MaterializedFile:
self.cancel_check()
source_relative = _relative_path(source_relative_path) source_relative = _relative_path(source_relative_path)
destination_relative = _relative_path(destination_relative_path) destination_relative = _relative_path(destination_relative_path)
source = _existing_regular_file(source_root, source_relative) source = _existing_regular_file(source_root, source_relative)
@@ -184,6 +187,7 @@ class FileMaterializer:
destination, destination,
operation_id, operation_id,
self.copy_chunk_bytes, self.copy_chunk_bytes,
self.cancel_check,
) )
method = transfer_pb2.MATERIALIZATION_METHOD_COPY method = transfer_pb2.MATERIALIZATION_METHOD_COPY
@@ -233,6 +237,8 @@ def stage_transfer(
allow_hardlink: bool = True, allow_hardlink: bool = True,
allow_reflink: bool = True, allow_reflink: bool = True,
sparse_supported: bool = True, sparse_supported: bool = True,
cancel_check: Callable[[], None] | None = None,
progress: Callable[[int, int], None] | None = None,
) -> PublishedTransfer: ) -> PublishedTransfer:
"""Publish a complete isolated transfer namespace and ready marker.""" """Publish a complete isolated transfer namespace and ready marker."""
@@ -244,7 +250,12 @@ def stage_transfer(
store, store,
allow_hardlink=allow_hardlink, allow_hardlink=allow_hardlink,
allow_reflink=allow_reflink, allow_reflink=allow_reflink,
cancel_check=cancel_check,
) )
total_bytes = sum(entry.logical_bytes for entry in staged.files) + sum(
artifact.logical_bytes for artifact in staged.artifacts
)
completed_bytes = 0
for entry in staged.files: for entry in staged.files:
result = materializer.materialize( result = materializer.materialize(
job_id=staged.job_id, job_id=staged.job_id,
@@ -263,6 +274,9 @@ def stage_transfer(
entry.allocated_bytes = result.allocated_bytes entry.allocated_bytes = result.allocated_bytes
entry.sparse = result.sparse entry.sparse = result.sparse
entry.source_staging_method = result.method entry.source_staging_method = result.method
completed_bytes += entry.logical_bytes
if progress is not None:
progress(completed_bytes, total_bytes)
artifact_sources = artifact_sources or {} artifact_sources = artifact_sources or {}
for artifact in staged.artifacts: for artifact in staged.artifacts:
@@ -287,6 +301,9 @@ def stage_transfer(
) )
artifact.allocated_bytes = result.allocated_bytes artifact.allocated_bytes = result.allocated_bytes
artifact.sparse = result.sparse artifact.sparse = result.sparse
completed_bytes += artifact.logical_bytes
if progress is not None:
progress(completed_bytes, total_bytes)
digest = _sha256_file( digest = _sha256_file(
_existing_regular_file( _existing_regular_file(
job_directory, _relative_path(artifact.payload_relative_path) job_directory, _relative_path(artifact.payload_relative_path)
@@ -356,6 +373,8 @@ def materialize_transfer(
allow_hardlink: bool = True, allow_hardlink: bool = True,
allow_reflink: bool = True, allow_reflink: bool = True,
sparse_supported: bool = True, sparse_supported: bool = True,
cancel_check: Callable[[], None] | None = None,
progress: Callable[[int, int], None] | None = None,
) -> transfer_pb2.TransferManifest: ) -> transfer_pb2.TransferManifest:
"""Materialize a verified published payload into a target content root.""" """Materialize a verified published payload into a target content root."""
@@ -366,7 +385,10 @@ def materialize_transfer(
store, store,
allow_hardlink=allow_hardlink, allow_hardlink=allow_hardlink,
allow_reflink=allow_reflink, allow_reflink=allow_reflink,
cancel_check=cancel_check,
) )
total_bytes = sum(entry.logical_bytes for entry in result_manifest.files)
completed_bytes = 0
for entry in result_manifest.files: for entry in result_manifest.files:
result = materializer.materialize( result = materializer.materialize(
job_id=result_manifest.job_id, job_id=result_manifest.job_id,
@@ -384,6 +406,9 @@ def materialize_transfer(
) )
entry.target_method = result.method entry.target_method = result.method
entry.target_preexisted = result.destination_preexisted entry.target_preexisted = result.destination_preexisted
completed_bytes += entry.logical_bytes
if progress is not None:
progress(completed_bytes, total_bytes)
return result_manifest return result_manifest
@@ -455,20 +480,35 @@ def _validate_manifest(manifest: transfer_pb2.TransferManifest) -> None:
raise TransferIntegrityError("manifest creation timestamp is required") raise TransferIntegrityError("manifest creation timestamp is required")
indices: set[int] = set() indices: set[int] = set()
payload_paths: set[str] = set() payload_paths: set[str] = set()
folded_payload_paths: set[str] = set()
target_paths: set[str] = set()
folded_target_paths: set[str] = set()
for entry in manifest.files: for entry in manifest.files:
if entry.file_index in indices: if entry.file_index in indices:
raise TransferIntegrityError("manifest contains duplicate file indices") raise TransferIntegrityError("manifest contains duplicate file indices")
indices.add(entry.file_index) indices.add(entry.file_index)
payload = _relative_path(entry.payload_relative_path).as_posix() payload = _relative_path(entry.payload_relative_path).as_posix()
_relative_path(entry.target_canonical_path) target = _relative_path(entry.target_canonical_path).as_posix()
if payload in payload_paths: if payload in payload_paths:
raise TransferIntegrityError("manifest contains duplicate payload paths") raise TransferIntegrityError("manifest contains duplicate payload paths")
if payload.casefold() in folded_payload_paths:
raise TransferIntegrityError("manifest contains case-colliding payload paths")
if target in target_paths:
raise TransferIntegrityError("manifest contains duplicate target paths")
if target.casefold() in folded_target_paths:
raise TransferIntegrityError("manifest contains case-colliding target paths")
payload_paths.add(payload) payload_paths.add(payload)
folded_payload_paths.add(payload.casefold())
target_paths.add(target)
folded_target_paths.add(target.casefold())
for artifact in manifest.artifacts: for artifact in manifest.artifacts:
payload = _relative_path(artifact.payload_relative_path).as_posix() payload = _relative_path(artifact.payload_relative_path).as_posix()
if payload in payload_paths: if payload in payload_paths:
raise TransferIntegrityError("manifest contains duplicate payload paths") raise TransferIntegrityError("manifest contains duplicate payload paths")
if payload.casefold() in folded_payload_paths:
raise TransferIntegrityError("manifest contains case-colliding payload paths")
payload_paths.add(payload) payload_paths.add(payload)
folded_payload_paths.add(payload.casefold())
def _relative_path(value: str) -> PurePosixPath: def _relative_path(value: str) -> PurePosixPath:
@@ -569,6 +609,7 @@ def _sparse_copy(
destination: Path, destination: Path,
operation_id: str, operation_id: str,
chunk_bytes: int, chunk_bytes: int,
cancel_check: Callable[[], None],
) -> None: ) -> None:
temporary = _temporary_path(destination, operation_id) temporary = _temporary_path(destination, operation_id)
source_fd = os.open(source, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) source_fd = os.open(source, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
@@ -580,7 +621,13 @@ def _sparse_copy(
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
stat.S_IMODE(source_stat.st_mode), stat.S_IMODE(source_stat.st_mode),
) )
_copy_extents(source_fd, destination_fd, source_stat.st_size, chunk_bytes) _copy_extents(
source_fd,
destination_fd,
source_stat.st_size,
chunk_bytes,
cancel_check,
)
os.ftruncate(destination_fd, source_stat.st_size) os.ftruncate(destination_fd, source_stat.st_size)
os.fsync(destination_fd) os.fsync(destination_fd)
_publish_temporary(temporary, destination) _publish_temporary(temporary, destination)
@@ -596,6 +643,7 @@ def _copy_extents(
destination_fd: int, destination_fd: int,
size: int, size: int,
chunk_bytes: int, chunk_bytes: int,
cancel_check: Callable[[], None],
) -> None: ) -> None:
if size == 0: if size == 0:
return return
@@ -613,6 +661,7 @@ def _copy_extents(
os.lseek(destination_fd, data_offset, os.SEEK_SET) os.lseek(destination_fd, data_offset, os.SEEK_SET)
remaining = min(hole_offset, size) - data_offset remaining = min(hole_offset, size) - data_offset
while remaining: while remaining:
cancel_check()
data = os.read(source_fd, min(chunk_bytes, remaining)) data = os.read(source_fd, min(chunk_bytes, remaining))
if not data: if not data:
raise TransferIntegrityError("source ended during sparse copy") raise TransferIntegrityError("source ended during sparse copy")
@@ -626,6 +675,7 @@ def _copy_extents(
os.lseek(destination_fd, 0, os.SEEK_SET) os.lseek(destination_fd, 0, os.SEEK_SET)
remaining = size remaining = size
while remaining: while remaining:
cancel_check()
data = os.read(source_fd, min(chunk_bytes, remaining)) data = os.read(source_fd, min(chunk_bytes, remaining))
if not data: if not data:
raise TransferIntegrityError("source ended during copy") raise TransferIntegrityError("source ended during copy")
+45
View File
@@ -22,6 +22,51 @@ from archive_control.v1 import (
class DaemonTransportTests(unittest.IsolatedAsyncioTestCase): class DaemonTransportTests(unittest.IsolatedAsyncioTestCase):
async def test_eviction_assignment_and_steps_are_admitted(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
token = root / "token"
token.write_text("shared-secret", encoding="utf-8")
os.chmod(token, 0o600)
service = ServiceConfig(
"http://local", PurePosixPath("/api"), root
)
config = ClientConfig(
"cache-1", "Cache 1", "cache", "ws://control", token,
root / "state.db", root / "backups", service, service,
)
probe = FilesystemProbe(root, True, True, True, True, True)
daemon = ArchiveClientDaemon(config, [probe, probe], [])
daemon.jobs = Mock()
assign = control_pb2.Command(command_id=str(uuid4()))
definition = assign.assign_job.job
definition.job_id = str(uuid4())
definition.eviction.cache_client_id = "cache-1"
acknowledgement = daemon._initial_acknowledgement(
assign, set(), False
)
self.assertEqual(
acknowledgement.status,
control_pb2.COMMAND_ACK_STATUS_ACCEPTED,
)
for step_kind in (
job_pb2.JOB_STEP_KIND_VERIFY_ARCHIVE_COVERAGE,
job_pb2.JOB_STEP_KIND_QB_REMOVE_ENTRY,
job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK,
):
execute = control_pb2.Command(command_id=str(uuid4()))
execute.execute_step.job_id = definition.job_id
execute.execute_step.step = step_kind
acknowledgement = daemon._initial_acknowledgement(
execute, set(), False
)
self.assertEqual(
acknowledgement.status,
control_pb2.COMMAND_ACK_STATUS_ACCEPTED,
)
async def test_ensure_route_is_durable_and_duplicate_replays_updates(self): async def test_ensure_route_is_durable_and_duplicate_replays_updates(self):
with tempfile.TemporaryDirectory() as directory: with tempfile.TemporaryDirectory() as directory:
root = Path(directory) root = Path(directory)
+181
View File
@@ -0,0 +1,181 @@
import tempfile
import unittest
from pathlib import Path
from archive_clients.eviction import (
remove_qb_entry,
safe_unlink,
verify_and_snapshot,
)
from archive_clients.resources import NormalizedResource
from archive_clients.state import ClientStore
from archive_control.v1 import resource_pb2
def normalized(torrent_hash, paths):
summary = resource_pb2.ResourceSummary(
qb_torrent_id=torrent_hash,
display_name="resource",
total_file_count=len(paths),
canonical_paths=True,
)
summary.resource_id.info_hash_v1_hex = torrent_hash
files = []
for index, path in enumerate(paths):
size = len(path) + 10
files.append(resource_pb2.TorrentFile(
file_index=index,
canonical_path=path,
logical_bytes=size,
completed_bytes=size,
selected=True,
))
summary.selected_files.ranges.add(first=index, last=index)
summary.selected_complete_files.ranges.add(first=index, last=index)
return NormalizedResource(summary, tuple(files), None)
class FakeQB:
def __init__(self, evicted, remaining):
self.evicted = evicted
self.remaining = remaining
self.deleted = []
def get_resource(self, torrent_hash):
return (
self.evicted
if self.evicted
and self.evicted.summary.qb_torrent_id == torrent_hash
else None
)
def delete_entry(self, torrent_hash):
self.deleted.append(torrent_hash)
self.evicted = None
def list_resources(self):
return list(self.remaining)
class EvictionTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.root = Path(self.temp.name) / "qb"
self.root.mkdir()
self.store = ClientStore(Path(self.temp.name) / "state.db")
self.store.initialize()
self.store.save_job(
"job-1", '{"jobId":"job-1"}', "JOB_STATE_RUNNING", 1, 0, False
)
def tearDown(self):
self.temp.cleanup()
def test_shared_paths_and_unknown_files_survive(self):
evicted = normalized("a" * 40, ["tree/shared.bin", "tree/owned.bin"])
other = normalized("b" * 40, ["tree/shared.bin"])
for item in evicted.files:
path = self.root / item.canonical_path
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b"x" * item.logical_bytes)
unknown = self.root / "tree" / "from-another-app.txt"
unknown.write_text("keep")
qb = FakeQB(evicted, [other])
verify_and_snapshot(
job_id="job-1",
resource=evicted,
selected_indices=[0, 1],
qb_root=self.root,
store=self.store,
)
remove_qb_entry(
job_id="job-1",
torrent_hash="a" * 40,
qbittorrent=qb,
store=self.store,
)
result = safe_unlink(
job_id="job-1",
qb_root=self.root,
qbittorrent=qb,
store=self.store,
)
self.assertTrue((self.root / "tree/shared.bin").exists())
self.assertFalse((self.root / "tree/owned.bin").exists())
self.assertTrue(unknown.exists())
self.assertTrue((self.root / "tree").is_dir())
self.assertEqual(qb.deleted, ["a" * 40])
self.assertEqual(
result["retained_files"],
[{"path": "tree/shared.bin", "reason": "shared"}],
)
def test_changed_inode_is_not_unlinked(self):
evicted = normalized("a" * 40, ["tree/owned.bin"])
path = self.root / "tree/owned.bin"
path.parent.mkdir(parents=True)
path.write_bytes(b"x" * evicted.files[0].logical_bytes)
qb = FakeQB(evicted, [])
verify_and_snapshot(
job_id="job-1",
resource=evicted,
selected_indices=[0],
qb_root=self.root,
store=self.store,
)
path.unlink()
path.write_bytes(b"y" * evicted.files[0].logical_bytes)
remove_qb_entry(
job_id="job-1",
torrent_hash="a" * 40,
qbittorrent=qb,
store=self.store,
)
result = safe_unlink(
job_id="job-1",
qb_root=self.root,
qbittorrent=qb,
store=self.store,
)
self.assertTrue(path.exists())
self.assertEqual(
result["retained_files"][0]["reason"], "identity-changed"
)
def test_retry_is_idempotent(self):
evicted = normalized("a" * 40, ["owned.bin"])
path = self.root / "owned.bin"
path.write_bytes(b"x" * evicted.files[0].logical_bytes)
qb = FakeQB(evicted, [])
verify_and_snapshot(
job_id="job-1",
resource=evicted,
selected_indices=[0],
qb_root=self.root,
store=self.store,
)
remove_qb_entry(
job_id="job-1",
torrent_hash="a" * 40,
qbittorrent=qb,
store=self.store,
)
first = safe_unlink(
job_id="job-1",
qb_root=self.root,
qbittorrent=qb,
store=self.store,
)
second = safe_unlink(
job_id="job-1",
qb_root=self.root,
qbittorrent=qb,
store=self.store,
)
self.assertEqual(first, second)
self.assertEqual(qb.deleted, ["a" * 40])
if __name__ == "__main__":
unittest.main()
+61 -3
View File
@@ -6,7 +6,11 @@ from unittest.mock import Mock, patch
from uuid import uuid4 from uuid import uuid4
from archive_clients.bencode import encode from archive_clients.bencode import encode
from archive_clients.jobs import ClientJobExecutor, JobExecutionError from archive_clients.jobs import (
ClientJobExecutor,
JobExecutionError,
_resource_fingerprint,
)
from archive_clients.resources import normalize_resource from archive_clients.resources import normalize_resource
from archive_clients.state import ClientStore from archive_clients.state import ClientStore
from archive_control.v1 import control_pb2, job_pb2 from archive_control.v1 import control_pb2, job_pb2
@@ -90,6 +94,9 @@ class ClientJobHappyPathTests(unittest.TestCase):
definition.created_at.GetCurrentTime() definition.created_at.GetCurrentTime()
definition.transfer.requested_files.ranges.add(first=0, last=0) definition.transfer.requested_files.ranges.add(first=0, last=0)
definition.transfer.transfer_delta_files.ranges.add(first=0, last=0) definition.transfer.transfer_delta_files.ranges.add(first=0, last=0)
definition.transfer.source_fingerprint.CopyFrom(
_resource_fingerprint(resource, source_id)
)
source_store = ClientStore(root / "source.db") source_store = ClientStore(root / "source.db")
target_store = ClientStore(root / "target.db") target_store = ClientStore(root / "target.db")
@@ -98,7 +105,7 @@ class ClientJobHappyPathTests(unittest.TestCase):
source_qb = Mock() source_qb = Mock()
source_qb.get_resource.return_value = resource source_qb.get_resource.return_value = resource
target_qb = Mock() target_qb = Mock()
target_qb.get_resource.side_effect = [None, resource] target_qb.get_resource.side_effect = [None, None, resource]
syncthing = CompleteSyncthing() syncthing = CompleteSyncthing()
source = ClientJobExecutor( source = ClientJobExecutor(
client_id=source_id, client_id=source_id,
@@ -156,7 +163,10 @@ class ClientJobHappyPathTests(unittest.TestCase):
)) ))
self.assertEqual( self.assertEqual(
[event.sequence for event in events], [event.sequence for event in events],
[cursor_sequence + 1, cursor_sequence + 2], list(range(
cursor_sequence + 1,
cursor_sequence + len(events) + 1,
)),
) )
cursor_sequence = events[-1].sequence cursor_sequence = events[-1].sequence
cursor_revision = events[-1].job_revision cursor_revision = events[-1].job_revision
@@ -250,6 +260,54 @@ class ClientJobHappyPathTests(unittest.TestCase):
) )
self.assertEqual(store.list_active_job_cursors(), []) self.assertEqual(store.list_active_job_cursors(), [])
def test_active_source_cancellation_reports_durable_rollback(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
store = ClientStore(root / "client.db")
store.initialize()
definition = job_pb2.JobDefinition(
job_id=str(uuid4()),
idempotency_key=str(uuid4()),
operation=job_pb2.JOB_OPERATION_ARCHIVE,
resource_display_name="fixture",
transfer={
"source_client_id": "cache-1",
"target_client_id": "archive-1",
"route_id": "route-1",
},
)
definition.resource_id.info_hash_v1_hex = "a" * 40
definition.created_at.GetCurrentTime()
executor = ClientJobExecutor(
client_id="cache-1",
qbittorrent=Mock(),
store=store,
qb_root=root,
qb_api_root=Path("/downloads"),
route_path=lambda _: root / "route",
syncthing_transport=Mock(),
sparse_supported=True,
poll_interval=0,
)
executor.assign(control_pb2.AssignJobCommand(
job=definition,
expected_job_revision=1,
expected_last_event_sequence=0,
))
events = executor.cancel(control_pb2.CancelJobCommand(
job_id=definition.job_id,
expected_job_revision=2,
expected_last_event_sequence=1,
reason="test",
))
self.assertEqual(
events[-1].type,
control_pb2.JOB_EVENT_TYPE_ROLLBACK_SUCCEEDED,
)
self.assertEqual(
events[-1].state, job_pb2.JOB_STATE_ROLLING_BACK
)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+22
View File
@@ -120,6 +120,28 @@ class ClientStoreTests(unittest.TestCase):
"operation-1", '{"method":2}' "operation-1", '{"method":2}'
) )
def test_job_artifacts_are_immutable_and_survive_restart(self):
with tempfile.TemporaryDirectory() as directory:
database = Path(directory) / "state.db"
store = ClientStore(database)
store.initialize()
store.save_job(
"job-1", '{"jobId":"job-1"}', "JOB_STATE_RUNNING",
1, 0, False,
)
store.put_job_artifact(
"job-1", "baseline", {"selected": [1, 3]}
)
reopened = ClientStore(database)
self.assertEqual(
reopened.get_job_artifact("job-1", "baseline")["value"],
{"selected": [1, 3]},
)
with self.assertRaises(JobConflict):
reopened.put_job_artifact(
"job-1", "baseline", {"selected": [2]}
)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+17
View File
@@ -9,6 +9,7 @@ from uuid import uuid4
from archive_clients.state import ClientStore from archive_clients.state import ClientStore
from archive_clients.transfer import ( from archive_clients.transfer import (
FileMaterializer, FileMaterializer,
TransferIntegrityError,
canonical_message_json, canonical_message_json,
load_published_transfer, load_published_transfer,
materialize_transfer, materialize_transfer,
@@ -212,6 +213,22 @@ class TransferHappyPathTests(unittest.TestCase):
self.assertEqual(first, second) self.assertEqual(first, second)
self.assertEqual(first, canonical_message_json(manifest)) self.assertEqual(first, canonical_message_json(manifest))
def test_manifest_rejects_case_colliding_target_paths(self):
manifest = self._manifest()
manifest.files[1].target_canonical_path = "album/ONE.bin"
with self.assertRaisesRegex(
TransferIntegrityError, "case-colliding target"
):
stage_transfer(
manifest,
source_root=self.source,
sync_root=self.sync,
store=self.store,
artifact_sources={
"metainfo/source.torrent": self.metainfo
},
)
def _manifest(self, job_id=None): def _manifest(self, job_id=None):
manifest = transfer_pb2.TransferManifest( manifest = transfer_pb2.TransferManifest(
manifest_version=1, manifest_version=1,