cabbage/archive-clients
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete transfer and eviction execution

Browse Source
This commit is contained in:
cabbage
2026-07-23 09:14:40 +00:00
parent 884aed9921
commit b6de490b7e
17 changed files with 1445 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/archive_clients/config.py
+24 -2
View File
@@ -80,6 +80,9 @@ class ConnectionConfig:
@dataclass(frozen=True)
class JobsConfig:
stall_after: float = 30 * 60
verification_timeout: float = 30 * 60
poll_interval: float = 1
free_space_reserve_bytes: int = 1024 * 1024 * 1024
@dataclass(frozen=True)
@@ -217,8 +220,27 @@ def _connection(value: Any) -> ConnectionConfig:
def _jobs(value: Any) -> JobsConfig:
if not isinstance(value, dict):
raise ConfigError("jobs must be a table")
_keys(value, {"stall_after"}, "jobs")
return JobsConfig(stall_after=_duration(value.get("stall_after", "30m")))
_keys(
value,
{
"stall_after",
"verification_timeout",
"poll_interval",
"free_space_reserve_bytes",
},
"jobs",
)
return JobsConfig(
stall_after=_duration(value.get("stall_after", "30m")),
verification_timeout=_duration(
value.get("verification_timeout", "30m")
),
poll_interval=_duration(value.get("poll_interval", "1s")),
free_space_reserve_bytes=_positive_int(
value.get("free_space_reserve_bytes", 1024 * 1024 * 1024),
"jobs.free_space_reserve_bytes",
),
)
def _backup(value: Any) -> BackupConfig:
src/archive_clients/daemon.py
+55 -9
View File
@@ -105,6 +105,11 @@ class ArchiveClientDaemon:
route_path=self._route_path,
syncthing_transport=self.routes.transport,
sparse_supported=all(probe.sparse_files for probe in probes),
poll_interval=config.jobs.poll_interval,
verification_timeout=config.jobs.verification_timeout,
free_space_reserve_bytes=(
config.jobs.free_space_reserve_bytes
),
)
if resource_reader is not None and self.routes is not None
else None
@@ -415,9 +420,13 @@ class ArchiveClientDaemon:
elif (
accepted is not None
and accepted_for_execution
and command.WhichOneof("payload") in {"assign_job", "execute_step"}
and command.WhichOneof("payload") in {
"assign_job", "execute_step", "cancel_job"
}
and self.jobs is not None
):
if command.WhichOneof("payload") == "cancel_job":
self.jobs.request_cancel(command.cancel_job.job_id)
self._schedule_job_command(
command,
envelope.message_id,
@@ -445,9 +454,13 @@ class ArchiveClientDaemon:
command, "", outbound, command_tasks
)
elif (
command.WhichOneof("payload") in {"assign_job", "execute_step"}
command.WhichOneof("payload") in {
"assign_job", "execute_step", "cancel_job"
}
and self.jobs is not None
):
if command.WhichOneof("payload") == "cancel_job":
self.jobs.request_cancel(command.cancel_job.job_id)
job_commands.append(command)
if job_commands:
task = asyncio.create_task(
@@ -543,17 +556,39 @@ class ArchiveClientDaemon:
) -> None:
assert self.jobs is not None
payload = command.WhichOneof("payload")
streamed = False
if payload == "assign_job":
events = await asyncio.to_thread(
self.jobs.assign, command.assign_job
)
elif payload == "execute_step":
loop = asyncio.get_running_loop()
def emit(event):
response = new_envelope()
response.correlation_id = correlation_id
response.job_event.CopyFrom(event)
future = asyncio.run_coroutine_threadsafe(
outbound.put(encode(response)), loop
)
try:
future.result(timeout=30)
except Exception:
# The event is already durable in the client DB and will
# be replayed after reconnect.
pass
events = await asyncio.to_thread(
self.jobs.execute, command.execute_step
self.jobs.execute, command.execute_step, emit
)
streamed = True
elif payload == "cancel_job":
events = await asyncio.to_thread(
self.jobs.cancel, command.cancel_job
)
else:
raise JobExecutionError("job command payload is unsupported")
for event in events:
for event in (() if streamed else events):
response = new_envelope()
response.correlation_id = correlation_id
response.job_event.CopyFrom(event)
@@ -880,17 +915,25 @@ class ArchiveClientDaemon:
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_ACCEPTED
elif command.WhichOneof("payload") == "assign_job":
definition = command.assign_job.job
specification = definition.WhichOneof("spec")
assigned_to_client = (
specification == "transfer"
and self.config.client_id
in {
definition.transfer.source_client_id,
definition.transfer.target_client_id,
}
) or (
specification == "eviction"
and self.config.client_id == definition.eviction.cache_client_id
)
if self.jobs is None:
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
acknowledgement.error.code = common_pb2.ERROR_CODE_UNAVAILABLE
acknowledgement.error.message = "job executor is unavailable"
elif (
not definition.job_id
or definition.WhichOneof("spec") != "transfer"
or self.config.client_id not in {
definition.transfer.source_client_id,
definition.transfer.target_client_id,
}
or not assigned_to_client
):
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
acknowledgement.error.code = common_pb2.ERROR_CODE_INVALID_ARGUMENT
@@ -911,6 +954,9 @@ class ArchiveClientDaemon:
job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE,
job_pb2.JOB_STEP_KIND_QB_VERIFY,
job_pb2.JOB_STEP_KIND_STAGING_CLEANUP,
job_pb2.JOB_STEP_KIND_VERIFY_ARCHIVE_COVERAGE,
job_pb2.JOB_STEP_KIND_QB_REMOVE_ENTRY,
job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK,
}
):
acknowledgement.status = control_pb2.COMMAND_ACK_STATUS_REJECTED
src/archive_clients/eviction.py
+242
View File
@@ -0,0 +1,242 @@
"""Idempotent cache eviction without recursive or qB data deletion."""
from __future__ import annotations
import errno
import hashlib
import os
import stat
from pathlib import Path, PurePosixPath
from typing import Iterable
from archive_clients.qbittorrent import QBittorrentReader
from archive_clients.resources import NormalizedResource
from archive_clients.state import ClientStore
class EvictionError(RuntimeError):
pass
def verify_and_snapshot(
*,
job_id: str,
resource: NormalizedResource,
selected_indices: Iterable[int],
qb_root: Path,
store: ClientStore,
) -> dict[str, object]:
selected = set(selected_indices)
by_index = {item.file_index: item for item in resource.files}
if not selected or any(index not in by_index for index in selected):
raise EvictionError("eviction file selection is invalid")
files = []
for index in sorted(selected):
item = by_index[index]
if not item.selected or item.completed_bytes != item.logical_bytes:
raise EvictionError(
f"cache file {index} is not selected and complete"
)
relative = _relative(item.canonical_path)
path = qb_root.joinpath(*relative.parts)
try:
metadata = path.lstat()
except FileNotFoundError as exc:
raise EvictionError(
f"cache file disappeared before eviction: {relative}"
) from exc
if not stat.S_ISREG(metadata.st_mode):
raise EvictionError(
f"cache path is not a regular file: {relative}"
)
if metadata.st_size != item.logical_bytes:
raise EvictionError(
f"cache file size changed before eviction: {relative}"
)
files.append(
{
"file_index": index,
"path": relative.as_posix(),
"logical_bytes": item.logical_bytes,
"device": metadata.st_dev,
"inode": metadata.st_ino,
"mtime_ns": metadata.st_mtime_ns,
"ctime_ns": metadata.st_ctime_ns,
"sha256": _sha256(path),
}
)
snapshot = {"files": files}
return store.put_job_artifact(job_id, "eviction-snapshot", snapshot)["value"]
def remove_qb_entry(
*,
job_id: str,
torrent_hash: str,
qbittorrent: QBittorrentReader,
store: ClientStore,
) -> None:
marker = store.get_job_artifact(job_id, "qb-entry-removed")
if marker is not None:
return
if qbittorrent.get_resource(torrent_hash) is not None:
qbittorrent.delete_entry(torrent_hash)
store.put_job_artifact(
job_id, "qb-entry-removed", {"torrent_hash": torrent_hash}
)
def safe_unlink(
*,
job_id: str,
qb_root: Path,
qbittorrent: QBittorrentReader,
store: ClientStore,
) -> dict[str, object]:
completed = store.get_job_artifact(job_id, "eviction-unlinked")
if completed is not None:
return completed["value"]
snapshot_row = store.get_job_artifact(job_id, "eviction-snapshot")
if snapshot_row is None:
raise EvictionError("eviction snapshot is missing")
snapshot = snapshot_row["value"]
files = snapshot.get("files")
if not isinstance(files, list):
raise EvictionError("eviction snapshot is invalid")
referenced = _remaining_paths(qbittorrent.list_resources())
removed: list[str] = []
retained: list[dict[str, str]] = []
directories: set[Path] = set()
for record in files:
if not isinstance(record, dict) or not isinstance(record.get("path"), str):
raise EvictionError("eviction file record is invalid")
relative = _relative(record["path"])
if relative.as_posix() in referenced:
retained.append({"path": relative.as_posix(), "reason": "shared"})
continue
path = qb_root.joinpath(*relative.parts)
try:
metadata = path.lstat()
except FileNotFoundError:
removed.append(relative.as_posix())
continue
if not stat.S_ISREG(metadata.st_mode):
retained.append(
{"path": relative.as_posix(), "reason": "type-changed"}
)
continue
if (
metadata.st_dev != record.get("device")
or metadata.st_ino != record.get("inode")
or metadata.st_size != record.get("logical_bytes")
or metadata.st_mtime_ns != record.get("mtime_ns")
or metadata.st_ctime_ns != record.get("ctime_ns")
or _sha256(path) != record.get("sha256")
):
retained.append(
{"path": relative.as_posix(), "reason": "identity-changed"}
)
continue
path.unlink()
removed.append(relative.as_posix())
parent = path.parent
while parent != qb_root:
directories.add(parent)
parent = parent.parent
removed_directories: list[str] = []
for directory in sorted(
directories, key=lambda item: len(item.parts), reverse=True
):
try:
directory.rmdir()
removed_directories.append(
directory.relative_to(qb_root).as_posix()
)
except OSError as exc:
if exc.errno not in {errno.ENOTEMPTY, errno.ENOENT}:
raise
result = {
"removed_files": removed,
"retained_files": retained,
"removed_directories": removed_directories,
}
return store.put_job_artifact(job_id, "eviction-unlinked", result)["value"]
def compensate_materialized_files(
*, job_id: str, qb_root: Path, store: ClientStore
) -> list[str]:
"""Remove only target files this job created and whose inode still matches."""
removed: list[str] = []
directories: set[Path] = set()
for row in store.file_operation_rows(job_id):
if row["state"] != "completed" or row["result_json"] is None:
continue
import json
intent = json.loads(str(row["intent_json"]))
result = json.loads(str(row["result_json"]))
if result.get("destination_preexisted"):
continue
relative_value = intent.get("destination")
if not isinstance(relative_value, str):
continue
relative = _relative(relative_value)
path = qb_root.joinpath(*relative.parts)
try:
metadata = path.lstat()
except FileNotFoundError:
continue
if (
not stat.S_ISREG(metadata.st_mode)
or metadata.st_dev != result.get("destination_device")
or metadata.st_ino != result.get("destination_inode")
):
continue
path.unlink()
removed.append(relative.as_posix())
parent = path.parent
while parent != qb_root:
directories.add(parent)
parent = parent.parent
for directory in sorted(
directories, key=lambda item: len(item.parts), reverse=True
):
try:
directory.rmdir()
except OSError as exc:
if exc.errno not in {errno.ENOTEMPTY, errno.ENOENT}:
raise
return removed
def _remaining_paths(resources: Iterable[NormalizedResource]) -> set[str]:
return {
item.canonical_path
for resource in resources
for item in resource.files
}
def _relative(value: str) -> PurePosixPath:
candidate = PurePosixPath(value)
if (
not value
or candidate.is_absolute()
or ".." in candidate.parts
or "." in candidate.parts
or "\\" in value
):
raise EvictionError("eviction path is unsafe")
return candidate
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as source:
while chunk := source.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
src/archive_clients/jobs.py
+564 -32
View File
@@ -3,12 +3,22 @@
from __future__ import annotations
import hashlib
import json
import shutil
import threading
import time
import uuid
from pathlib import Path, PurePosixPath
from typing import Callable
from archive_clients.protocol import decode_message, encode_message
from archive_clients.eviction import (
EvictionError,
compensate_materialized_files,
remove_qb_entry,
safe_unlink,
verify_and_snapshot,
)
from archive_clients.qbittorrent import (
QBittorrentDownloadAttempt,
QBittorrentReader,
@@ -37,6 +47,10 @@ class JobExecutionError(RuntimeError):
pass
class JobCancelled(JobExecutionError):
pass
class ClientJobExecutor:
def __init__(
self,
@@ -50,6 +64,8 @@ class ClientJobExecutor:
syncthing_transport: object,
sparse_supported: bool,
poll_interval: float = 1,
verification_timeout: float = 30 * 60,
free_space_reserve_bytes: int = 1024 * 1024 * 1024,
):
self.client_id = client_id
self.qbittorrent = qbittorrent
@@ -60,6 +76,12 @@ class ClientJobExecutor:
self.syncthing_transport = syncthing_transport
self.sparse_supported = sparse_supported
self.poll_interval = poll_interval
self.verification_timeout = verification_timeout
self.free_space_reserve_bytes = free_space_reserve_bytes
self._cancel_events: dict[str, threading.Event] = {}
def request_cancel(self, job_id: str) -> None:
self._cancel_events.setdefault(job_id, threading.Event()).set()
def assign(
self, command: control_pb2.AssignJobCommand
@@ -83,7 +105,9 @@ class ClientJobExecutor:
return [event]
def execute(
self, command: control_pb2.ExecuteStepCommand
self,
command: control_pb2.ExecuteStepCommand,
event_callback: Callable[[control_pb2.JobEvent], None] | None = None,
) -> list[control_pb2.JobEvent]:
definition = self._definition(command.job_id)
replay = self._replay(
@@ -97,6 +121,9 @@ class ClientJobExecutor:
control_pb2.JOB_EVENT_TYPE_CLEANUP_REQUIRED,
control_pb2.JOB_EVENT_TYPE_CANCELLED,
}:
if event_callback is not None:
for event in replay:
event_callback(event)
return replay
started = replay[0] if replay else self._event(
definition,
@@ -107,23 +134,124 @@ class ClientJobExecutor:
committed=(
self._committed(command.job_id)
or command.step == job_pb2.JOB_STEP_KIND_STAGING_CLEANUP
or (
command.step == job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK
and self.store.get_job_artifact(
command.job_id, "qb-entry-removed"
) is not None
)
),
step=command.step,
step_state=job_pb2.STEP_STATE_RUNNING,
)
if not replay:
self._record(definition, started)
if event_callback is not None:
event_callback(started)
emitted = [started]
cursor = started
speed_sample = [time.monotonic(), 0]
def progress(
fraction: float,
bytes_complete: int = 0,
bytes_total: int = 0,
detail: str = "",
) -> None:
nonlocal cursor
now = time.monotonic()
elapsed = now - float(speed_sample[0])
if fraction < 1 and elapsed < 1:
return
speed = (
max(0, bytes_complete - int(speed_sample[1])) / elapsed
if elapsed > 0 else 0
)
event = self._event(
definition,
sequence=cursor.sequence + 1,
revision=cursor.job_revision + 1,
event_type=control_pb2.JOB_EVENT_TYPE_PROGRESS,
state=job_pb2.JOB_STATE_RUNNING,
committed=cursor.committed,
step=command.step,
step_state=job_pb2.STEP_STATE_RUNNING,
)
event.progress.fraction_complete = max(
0.0, min(float(fraction), 1.0)
)
event.progress.bytes_complete = max(0, bytes_complete)
event.progress.bytes_total = max(0, bytes_total)
event.progress.approximate_bytes_per_second = speed
event.progress.detail = detail
completed_steps = max(0, _step_number(command.step) - 1)
event.progress.overall_fraction_complete = (
completed_steps + event.progress.fraction_complete
) / event.progress.display_step_total
self._record(definition, event)
emitted.append(event)
cursor = event
speed_sample[:] = [now, bytes_complete]
if event_callback is not None:
event_callback(event)
try:
result = self._execute_step(definition, command.step)
result = self._execute_step(definition, command.step, progress)
except JobCancelled as error:
cancelling = self._event(
definition,
sequence=cursor.sequence + 1,
revision=cursor.job_revision + 1,
event_type=control_pb2.JOB_EVENT_TYPE_CANCELLING,
state=job_pb2.JOB_STATE_CANCELLING,
committed=started.committed,
step=command.step,
step_state=job_pb2.STEP_STATE_CANCELLED,
)
cancelling.error.code = common_pb2.ERROR_CODE_CANCELLED
cancelling.error.message = str(error)
self._record(definition, cancelling)
emitted.append(cancelling)
if event_callback is not None:
event_callback(cancelling)
return emitted
except Exception as error:
rollback_error: Exception | None = None
if (
definition.WhichOneof("spec") == "transfer"
and not started.committed
):
try:
if (
self.client_id == definition.transfer.target_client_id
and command.step in {
job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE,
job_pb2.JOB_STEP_KIND_QB_VERIFY,
}
):
self._rollback_target(definition)
elif (
self.client_id == definition.transfer.source_client_id
and command.step == job_pb2.JOB_STEP_KIND_SOURCE_STAGE
):
self._cleanup_staging(definition)
except Exception as compensation_error:
rollback_error = compensation_error
cleanup = (
command.step == job_pb2.JOB_STEP_KIND_STAGING_CLEANUP
and started.committed
(
command.step == job_pb2.JOB_STEP_KIND_STAGING_CLEANUP
and started.committed
)
or (
command.step == job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK
and self.store.get_job_artifact(
definition.job_id, "qb-entry-removed"
) is not None
)
)
failed = self._event(
definition,
sequence=started.sequence + 1,
revision=started.job_revision + 1,
sequence=cursor.sequence + 1,
revision=cursor.job_revision + 1,
event_type=(
control_pb2.JOB_EVENT_TYPE_CLEANUP_REQUIRED
if cleanup else control_pb2.JOB_EVENT_TYPE_FAILED
@@ -139,11 +267,23 @@ class ClientJobExecutor:
failed.error.code = _job_error_code(error)
failed.error.message = str(error) or type(error).__name__
failed.error.retryable = cleanup
if rollback_error is not None:
failed.error.code = (
common_pb2.ERROR_CODE_MANUAL_INTERVENTION_REQUIRED
)
failed.error.message = (
f"{failed.error.message}; compensation also failed: "
f"{rollback_error}"
)
failed.error.retryable = False
self._record(definition, failed)
return [started, failed]
sequence = started.sequence + 1
revision = started.job_revision + 1
committed = started.committed
emitted.append(failed)
if event_callback is not None:
event_callback(failed)
return emitted
sequence = cursor.sequence + 1
revision = cursor.job_revision + 1
committed = cursor.committed
event_type = control_pb2.JOB_EVENT_TYPE_STEP_SUCCEEDED
state = job_pb2.JOB_STATE_RUNNING
if command.step == job_pb2.JOB_STEP_KIND_QB_VERIFY:
@@ -152,6 +292,10 @@ class ClientJobExecutor:
elif command.step == job_pb2.JOB_STEP_KIND_STAGING_CLEANUP:
event_type = control_pb2.JOB_EVENT_TYPE_SUCCEEDED
state = job_pb2.JOB_STATE_SUCCEEDED
elif command.step == job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK:
committed = True
event_type = control_pb2.JOB_EVENT_TYPE_SUCCEEDED
state = job_pb2.JOB_STATE_SUCCEEDED
succeeded = self._event(
definition,
sequence=sequence,
@@ -165,24 +309,91 @@ class ClientJobExecutor:
if result is not None:
succeeded.observed_placement.CopyFrom(result)
self._record(definition, succeeded)
return [started, succeeded]
emitted.append(succeeded)
if event_callback is not None:
event_callback(succeeded)
return emitted
def cancel(
self, command: control_pb2.CancelJobCommand
) -> list[control_pb2.JobEvent]:
definition = self._definition(command.job_id)
self.request_cancel(command.job_id)
replay = self._replay(
command.job_id, command.expected_last_event_sequence
)
if replay:
return replay
committed = self._committed(command.job_id)
if definition.WhichOneof("spec") == "eviction":
removed = self.store.get_job_artifact(
definition.job_id, "qb-entry-removed"
)
placement = None
if removed is not None:
placement = self._execute_eviction_step(
definition, job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK
)
committed = True
event_type = control_pb2.JOB_EVENT_TYPE_CANCELLED
state = job_pb2.JOB_STATE_CANCELLED
elif committed:
if definition.transfer.source_client_id == self.client_id:
self._cleanup_staging(definition)
event_type = control_pb2.JOB_EVENT_TYPE_CANCELLED
state = job_pb2.JOB_STATE_CANCELLED
placement = None
elif definition.transfer.source_client_id == self.client_id:
self._cleanup_staging(definition)
temporary = self._metainfo_path(definition.job_id)
if temporary.exists():
temporary.unlink()
event_type = control_pb2.JOB_EVENT_TYPE_ROLLBACK_SUCCEEDED
state = job_pb2.JOB_STATE_ROLLING_BACK
placement = None
else:
self._rollback_target(definition)
self._cleanup_staging(definition)
event_type = control_pb2.JOB_EVENT_TYPE_CANCELLED
state = job_pb2.JOB_STATE_CANCELLED
placement = None
event = self._event(
definition,
sequence=command.expected_last_event_sequence + 1,
revision=command.expected_job_revision + 1,
event_type=event_type,
state=state,
committed=committed,
step=job_pb2.JOB_STEP_KIND_ROLLBACK,
step_state=job_pb2.STEP_STATE_SUCCEEDED,
)
if placement is not None:
event.observed_placement.CopyFrom(placement)
self._record(definition, event)
return [event]
def _execute_step(
self, definition: job_pb2.JobDefinition, step: int
self,
definition: job_pb2.JobDefinition,
step: int,
progress: Callable[[float, int, int, str], None],
):
self._raise_if_cancelled(definition.job_id)
if definition.WhichOneof("spec") == "eviction":
return self._execute_eviction_step(definition, step)
if step == job_pb2.JOB_STEP_KIND_SOURCE_STAGE:
self._source_stage(definition)
self._source_stage(definition, progress)
return None
if step == job_pb2.JOB_STEP_KIND_SYNCTHING_TRANSFER:
self._wait_for_syncthing(definition)
self._wait_for_syncthing(definition, progress)
return None
if step == job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE:
self._target_materialize(definition)
self._target_materialize(definition, progress)
return None
if step == job_pb2.JOB_STEP_KIND_QB_VERIFY:
return self._qb_verify(definition)
return self._qb_verify(definition, progress)
if step == job_pb2.JOB_STEP_KIND_STAGING_CLEANUP:
cleanup_transfer(self._job_directory(definition))
self._cleanup_staging(definition)
temporary_metainfo = self._metainfo_path(definition.job_id)
if temporary_metainfo.exists():
temporary_metainfo.unlink()
@@ -197,10 +408,86 @@ class ClientJobExecutor:
return None
raise JobExecutionError("job step is unsupported")
def _source_stage(self, definition: job_pb2.JobDefinition) -> None:
def _execute_eviction_step(
self, definition: job_pb2.JobDefinition, step: int
):
if definition.eviction.cache_client_id != self.client_id:
raise JobExecutionError("eviction step was sent to the wrong client")
info_hash = _info_hash(definition)
if step == job_pb2.JOB_STEP_KIND_VERIFY_ARCHIVE_COVERAGE:
covered = {
index
for proof in definition.eviction.archive_coverage
for index in _selection_indices(proof.covered_files)
}
requested = set(
_selection_indices(definition.eviction.files_to_evict)
)
if not requested or requested - covered:
raise JobExecutionError(
"archive coverage no longer covers the cache selection"
)
resource = self._resource(definition)
current = _resource_fingerprint(resource, self.client_id)
if not _fingerprint_matches(
current, definition.eviction.cache_fingerprint
):
raise JobExecutionError(
"cache resource changed after eviction confirmation"
)
verify_and_snapshot(
job_id=definition.job_id,
resource=resource,
selected_indices=requested,
qb_root=self.qb_root,
store=self.store,
)
return None
if step == job_pb2.JOB_STEP_KIND_QB_REMOVE_ENTRY:
remove_qb_entry(
job_id=definition.job_id,
torrent_hash=info_hash,
qbittorrent=self.qbittorrent,
store=self.store,
)
return None
if step == job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK:
safe_unlink(
job_id=definition.job_id,
qb_root=self.qb_root,
qbittorrent=self.qbittorrent,
store=self.store,
)
placement = resource_pb2.Placement(
client_id=self.client_id,
state=resource_pb2.PLACEMENT_STATE_ABSENT,
generation=definition.eviction.cache_placement_generation + 1,
created_by_job_id=definition.job_id,
)
placement.resource_id.CopyFrom(definition.resource_id)
placement.fingerprint.CopyFrom(
definition.eviction.cache_fingerprint
)
placement.verified_at.GetCurrentTime()
return placement
raise JobExecutionError("eviction job step is unsupported")
def _source_stage(
self,
definition: job_pb2.JobDefinition,
progress: Callable[[float, int, int, str], None],
) -> None:
if definition.transfer.source_client_id != self.client_id:
raise JobExecutionError("source stage was sent to the wrong client")
resource = self._resource(definition)
if not _fingerprint_matches(
_resource_fingerprint(resource, self.client_id),
definition.transfer.source_fingerprint,
):
raise JobExecutionError(
"source resource changed after job confirmation"
)
self._reject_unsafe_partfile(definition, resource)
indices = _selection_indices(definition.transfer.transfer_delta_files)
by_index = {item.file_index: item for item in resource.files}
if not indices or any(index not in by_index for index in indices):
@@ -254,6 +541,11 @@ class ClientJobExecutor:
sha256_hex=hashlib.sha256(resource.metainfo_bytes).hexdigest(),
)
del artifact
self._require_space(
self.route_path(definition.transfer.route_id),
definition.transfer.transfer_delta_logical_bytes
+ len(resource.metainfo_bytes),
)
stage_transfer(
manifest,
source_root=self.qb_root,
@@ -261,18 +553,43 @@ class ClientJobExecutor:
store=self.store,
artifact_sources={"metainfo/source.torrent": metainfo_path},
sparse_supported=self.sparse_supported,
cancel_check=lambda: self._raise_if_cancelled(
definition.job_id
),
progress=lambda completed, total: progress(
completed / total if total else 1,
completed,
total,
f"staged {completed} of {total} bytes",
),
)
self._observer(definition).rescan()
def _wait_for_syncthing(self, definition: job_pb2.JobDefinition) -> None:
def _wait_for_syncthing(
self,
definition: job_pb2.JobDefinition,
progress: Callable[[float, int, int, str], None],
) -> None:
if definition.transfer.target_client_id != self.client_id:
raise JobExecutionError(
"Syncthing completion was sent to the wrong client"
)
observer = self._observer(definition)
last_reported: tuple[int, int] | None = None
while True:
self._raise_if_cancelled(definition.job_id)
try:
if observer.status().complete:
status = observer.status()
sample = (status.bytes_complete, status.bytes_total)
if sample != last_reported:
progress(
status.fraction_complete,
status.bytes_complete,
status.bytes_total,
f"{status.needed_items} Syncthing items still needed",
)
last_reported = sample
if status.complete:
return
except (FileNotFoundError, TransferIntegrityError):
# Syncthing can expose the per-job directory before the
@@ -280,20 +597,73 @@ class ClientJobExecutor:
pass
time.sleep(self.poll_interval)
def _target_materialize(self, definition: job_pb2.JobDefinition) -> None:
def _target_materialize(
self,
definition: job_pb2.JobDefinition,
progress: Callable[[float, int, int, str], None],
) -> None:
if definition.transfer.target_client_id != self.client_id:
raise JobExecutionError(
"target materialization was sent to the wrong client"
)
published = load_published_transfer(self._job_directory(definition))
self._require_space(
self.qb_root,
definition.transfer.transfer_delta_logical_bytes,
)
info_hash = _info_hash(definition)
resource = self.qbittorrent.get_resource(info_hash)
if resource is not None:
self._reject_unsafe_partfile(definition, resource)
if definition.transfer.HasField("target_baseline_fingerprint"):
if resource is None or not _fingerprint_matches(
_resource_fingerprint(resource, self.client_id),
definition.transfer.target_baseline_fingerprint,
):
raise JobExecutionError(
"target resource changed after job confirmation"
)
elif resource is not None:
raise JobExecutionError(
"target resource appeared after job confirmation"
)
baseline = {
"present": resource is not None,
"selected_file_indices": (
_selection_indices(resource.summary.selected_files)
if resource is not None else []
),
"stopped": (
resource is not None
and resource.summary.runtime_state
== resource_pb2.TORRENT_RUNTIME_STATE_STOPPED
),
"total_file_count": len(published.manifest.files),
}
self.store.put_job_artifact(
definition.job_id, "target-baseline", baseline
)
materialize_transfer(
published,
target_root=self.qb_root,
store=self.store,
sparse_supported=self.sparse_supported,
cancel_check=lambda: self._raise_if_cancelled(
definition.job_id
),
progress=lambda completed, total: progress(
completed / total if total else 1,
completed,
total,
f"materialized {completed} of {total} bytes",
),
)
def _qb_verify(self, definition: job_pb2.JobDefinition):
def _qb_verify(
self,
definition: job_pb2.JobDefinition,
progress: Callable[[float, int, int, str], None],
):
if definition.transfer.target_client_id != self.client_id:
raise JobExecutionError("qB verification was sent to the wrong client")
published = load_published_transfer(self._job_directory(definition))
@@ -326,7 +696,16 @@ class ClientJobExecutor:
)
self.qbittorrent.set_selection(info_hash, selected, total_files)
self.qbittorrent.recheck_and_wait(
info_hash, selected, timeout=30 * 60, poll_interval=self.poll_interval
info_hash,
selected,
timeout=self.verification_timeout,
poll_interval=self.poll_interval,
cancel_check=lambda: self._raise_if_cancelled(
definition.job_id
),
progress_callback=lambda fraction: progress(
fraction, 0, 0, "qBittorrent stopped recheck"
),
)
if should_start:
self.qbittorrent.start(info_hash)
@@ -359,6 +738,102 @@ class ClientJobExecutor:
placement.verified_at.GetCurrentTime()
return placement
def _rollback_target(self, definition: job_pb2.JobDefinition) -> None:
baseline_row = self.store.get_job_artifact(
definition.job_id, "target-baseline"
)
if baseline_row is None:
compensate_materialized_files(
job_id=definition.job_id,
qb_root=self.qb_root,
store=self.store,
)
return
baseline = baseline_row["value"]
info_hash = _info_hash(definition)
present = self.qbittorrent.get_resource(info_hash)
if present is not None:
if baseline.get("present"):
selected = baseline.get("selected_file_indices")
if isinstance(selected, list) and selected:
self.qbittorrent.set_selection(
info_hash, selected, len(present.files)
)
if baseline.get("stopped"):
self.qbittorrent.stop(info_hash)
else:
self.qbittorrent.start(info_hash)
else:
self.qbittorrent.delete_entry(info_hash)
compensate_materialized_files(
job_id=definition.job_id,
qb_root=self.qb_root,
store=self.store,
)
def _cleanup_staging(self, definition: job_pb2.JobDefinition) -> None:
job_directory = self._job_directory(definition)
try:
if cleanup_transfer(job_directory):
return
except FileNotFoundError:
pass
if not job_directory.is_dir():
return
compensate_materialized_files(
job_id=definition.job_id,
qb_root=job_directory,
store=self.store,
)
for name in ("ready.json", "manifest.json"):
candidate = job_directory / name
if candidate.is_file() and not candidate.is_symlink():
candidate.unlink()
try:
job_directory.rmdir()
except OSError:
pass
def _reject_unsafe_partfile(
self,
definition: job_pb2.JobDefinition,
resource: NormalizedResource,
) -> None:
roots = {self.qb_root}
for item in resource.files:
candidate = self.qb_root / PurePosixPath(item.canonical_path).parts[0]
roots.add(candidate if candidate.is_dir() else candidate.parent)
hashes = {
value.lower() for value in (
definition.resource_id.info_hash_v1_hex,
definition.resource_id.info_hash_v2_hex,
) if value
}
for root in roots:
if not root.is_dir():
continue
for candidate in root.glob("*.parts"):
lowered = candidate.name.lower()
if lowered == ".parts" or any(value in lowered for value in hashes):
raise JobExecutionError(
"resource uses a qBittorrent partfile that cannot be "
"safely transferred by this client"
)
def _raise_if_cancelled(self, job_id: str) -> None:
event = self._cancel_events.get(job_id)
if event is not None and event.is_set():
raise JobCancelled("job cancellation requested")
def _require_space(self, root: Path, required_bytes: int) -> None:
available = shutil.disk_usage(root).free
required = required_bytes + self.free_space_reserve_bytes
if available < required:
raise JobExecutionError(
f"insufficient free space: {available} bytes available, "
f"{required} bytes required including reserve"
)
def _observer(
self, definition: job_pb2.JobDefinition
) -> SyncthingTransferObserver:
@@ -455,21 +930,29 @@ class ClientJobExecutor:
)
event.progress.overall_fraction_complete = _overall(step, step_state)
event.progress.display_step_number = _step_number(step)
event.progress.display_step_total = 5
event.progress.display_step_total = (
3
if definition.operation == job_pb2.JOB_OPERATION_EVICT_CACHE
else 5
)
event.progress.last_progress_at.CopyFrom(event.occurred_at)
return event
@staticmethod
def _validate_definition(definition: job_pb2.JobDefinition) -> None:
if not definition.job_id:
raise JobExecutionError("job definition has no ID")
if definition.operation in {
job_pb2.JOB_OPERATION_ARCHIVE,
job_pb2.JOB_OPERATION_UNARCHIVE,
} and definition.WhichOneof("spec") == "transfer":
return
if (
not definition.job_id
or definition.operation not in {
job_pb2.JOB_OPERATION_ARCHIVE,
job_pb2.JOB_OPERATION_UNARCHIVE,
}
or definition.WhichOneof("spec") != "transfer"
definition.operation == job_pb2.JOB_OPERATION_EVICT_CACHE
and definition.WhichOneof("spec") == "eviction"
):
raise JobExecutionError("transfer job definition is invalid")
return
raise JobExecutionError("job definition operation/spec is invalid")
def _selection_indices(selection) -> list[int]:
@@ -510,12 +993,27 @@ def _step_number(step: int) -> int:
job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE: 3,
job_pb2.JOB_STEP_KIND_QB_VERIFY: 4,
job_pb2.JOB_STEP_KIND_STAGING_CLEANUP: 5,
job_pb2.JOB_STEP_KIND_VERIFY_ARCHIVE_COVERAGE: 1,
job_pb2.JOB_STEP_KIND_QB_REMOVE_ENTRY: 2,
job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK: 3,
job_pb2.JOB_STEP_KIND_ROLLBACK: 1,
}.get(step, 0)
def _overall(step: int, state: int) -> float:
number = _step_number(step)
return max(0, number - (0 if state == job_pb2.STEP_STATE_SUCCEEDED else 1)) / 5
total = (
3
if step in {
job_pb2.JOB_STEP_KIND_VERIFY_ARCHIVE_COVERAGE,
job_pb2.JOB_STEP_KIND_QB_REMOVE_ENTRY,
job_pb2.JOB_STEP_KIND_SAFE_FILE_UNLINK,
}
else 5
)
return max(
0, number - (0 if state == job_pb2.STEP_STATE_SUCCEEDED else 1)
) / total
def decode_message_metainfo(path: Path):
@@ -543,6 +1041,38 @@ def _resource_fingerprint(
return fingerprint
def _info_hash(definition: job_pb2.JobDefinition) -> str:
value = (
definition.resource_id.info_hash_v1_hex
or definition.resource_id.info_hash_v2_hex
)
if not value:
raise JobExecutionError("job resource identity has no info hash")
return value
def _fingerprint_matches(
current: resource_pb2.ResourceStateFingerprint,
expected: resource_pb2.ResourceStateFingerprint,
) -> bool:
left = resource_pb2.ResourceStateFingerprint()
right = resource_pb2.ResourceStateFingerprint()
left.CopyFrom(current)
right.CopyFrom(expected)
left.ClearField("observed_at")
right.ClearField("observed_at")
for value in (left, right):
if value.runtime_state in {
resource_pb2.TORRENT_RUNTIME_STATE_QUEUED,
resource_pb2.TORRENT_RUNTIME_STATE_SEEDING,
resource_pb2.TORRENT_RUNTIME_STATE_STALLED,
}:
value.runtime_state = (
resource_pb2.TORRENT_RUNTIME_STATE_SEEDING
)
return left == right
def _job_error_code(error: Exception) -> int:
if isinstance(error, QBittorrentDownloadAttempt):
return common_pb2.ERROR_CODE_MANUAL_INTERVENTION_REQUIRED
@@ -554,4 +1084,6 @@ def _job_error_code(error: Exception) -> int:
return common_pb2.ERROR_CODE_PRECONDITION_FAILED
if isinstance(error, TransferError):
return common_pb2.ERROR_CODE_PATH_CONFLICT
if isinstance(error, EvictionError):
return common_pb2.ERROR_CODE_PRECONDITION_FAILED
return common_pb2.ERROR_CODE_INTERNAL
src/archive_clients/qbittorrent.py
+12
View File
@@ -10,6 +10,7 @@ from dataclasses import dataclass
from http.cookiejar import CookieJar
from pathlib import PurePosixPath
from typing import Any
from typing import Callable
from urllib import error, parse, request
from archive_clients.config import ServiceConfig
@@ -259,6 +260,8 @@ class QBittorrentReader:
*,
timeout: float,
poll_interval: float = 1,
cancel_check: Callable[[], None] | None = None,
progress_callback: Callable[[float], None] | None = None,
) -> RecheckResult:
selected = tuple(sorted(set(selected_file_indices)))
if not selected:
@@ -271,7 +274,9 @@ class QBittorrentReader:
"/api/v2/torrents/recheck", {"hashes": torrent_hash}
)
deadline = time.monotonic() + timeout
cancel_check = cancel_check or (lambda: None)
while True:
cancel_check()
record = self._torrent_record(torrent_hash)
state = record.get("state")
if not isinstance(state, str):
@@ -295,6 +300,13 @@ class QBittorrentReader:
item.get("index"): item.get("progress")
for item in files
}
if progress_callback is not None:
progress_callback(
sum(
max(0.0, min(float(progress.get(index, 0)), 1.0))
for index in selected
) / len(selected)
)
if all(
isinstance(progress.get(index), (int, float))
and float(progress[index]) >= 1
src/archive_clients/state.py
+66 -1
View File
@@ -11,7 +11,7 @@ from dataclasses import dataclass
from pathlib import Path
SCHEMA_VERSION = 2
SCHEMA_VERSION = 3
class CommandConflict(RuntimeError):
@@ -58,6 +58,10 @@ class ClientStore:
if version == 1:
connection.executescript(_SCHEMA_V2)
connection.execute("PRAGMA user_version = 2")
version = 2
if version == 2:
connection.executescript(_SCHEMA_V3)
connection.execute("PRAGMA user_version = 3")
if connection.execute("PRAGMA foreign_key_check").fetchall():
raise RuntimeError("client database foreign-key check failed")
if not existed:
@@ -268,6 +272,57 @@ class ClientStore:
).fetchall()
return [dict(row) for row in rows]
def put_job_artifact(
self, job_id: str, kind: str, value: dict[str, object]
) -> dict[str, object]:
encoded = _canonical(value)
with self._connect() as connection:
connection.execute("BEGIN IMMEDIATE")
existing = connection.execute(
"""
SELECT value_json FROM job_artifacts
WHERE job_id = ? AND kind = ?
""",
(job_id, kind),
).fetchone()
if existing is not None and existing["value_json"] != encoded:
raise JobConflict(f"job artifact {kind} is immutable")
connection.execute(
"""
INSERT INTO job_artifacts (job_id, kind, value_json)
VALUES (?, ?, ?)
ON CONFLICT(job_id, kind) DO NOTHING
""",
(job_id, kind, encoded),
)
row = connection.execute(
"""
SELECT job_id, kind, value_json FROM job_artifacts
WHERE job_id = ? AND kind = ?
""",
(job_id, kind),
).fetchone()
result = dict(row)
result["value"] = json.loads(str(result.pop("value_json")))
return result
def get_job_artifact(
self, job_id: str, kind: str
) -> dict[str, object] | None:
with self._connect() as connection:
row = connection.execute(
"""
SELECT job_id, kind, value_json FROM job_artifacts
WHERE job_id = ? AND kind = ?
""",
(job_id, kind),
).fetchone()
if row is None:
return None
result = dict(row)
result["value"] = json.loads(str(result.pop("value_json")))
return result
def record_job_event(
self,
*,
@@ -567,3 +622,13 @@ CREATE TABLE route_attempt_updates (
PRIMARY KEY(command_id, sequence)
);
"""
_SCHEMA_V3 = """
CREATE TABLE job_artifacts (
job_id TEXT NOT NULL REFERENCES jobs(job_id),
kind TEXT NOT NULL,
value_json TEXT NOT NULL,
PRIMARY KEY(job_id, kind)
);
"""
src/archive_clients/transfer.py
+53 -3
View File
@@ -11,7 +11,7 @@ import stat
import uuid
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import Mapping
from typing import Callable, Mapping
from google.protobuf import json_format
@@ -79,6 +79,7 @@ class FileMaterializer:
allow_hardlink: bool = True,
allow_reflink: bool = True,
copy_chunk_bytes: int = _COPY_CHUNK_BYTES,
cancel_check: Callable[[], None] | None = None,
):
if copy_chunk_bytes < 1:
raise ValueError("copy_chunk_bytes must be positive")
@@ -86,6 +87,7 @@ class FileMaterializer:
self.allow_hardlink = allow_hardlink
self.allow_reflink = allow_reflink
self.copy_chunk_bytes = copy_chunk_bytes
self.cancel_check = cancel_check or (lambda: None)
def materialize(
self,
@@ -100,6 +102,7 @@ class FileMaterializer:
allow_preexisting_reuse: bool,
sparse_supported: bool,
) -> MaterializedFile:
self.cancel_check()
source_relative = _relative_path(source_relative_path)
destination_relative = _relative_path(destination_relative_path)
source = _existing_regular_file(source_root, source_relative)
@@ -184,6 +187,7 @@ class FileMaterializer:
destination,
operation_id,
self.copy_chunk_bytes,
self.cancel_check,
)
method = transfer_pb2.MATERIALIZATION_METHOD_COPY
@@ -233,6 +237,8 @@ def stage_transfer(
allow_hardlink: bool = True,
allow_reflink: bool = True,
sparse_supported: bool = True,
cancel_check: Callable[[], None] | None = None,
progress: Callable[[int, int], None] | None = None,
) -> PublishedTransfer:
"""Publish a complete isolated transfer namespace and ready marker."""
@@ -244,7 +250,12 @@ def stage_transfer(
store,
allow_hardlink=allow_hardlink,
allow_reflink=allow_reflink,
cancel_check=cancel_check,
)
total_bytes = sum(entry.logical_bytes for entry in staged.files) + sum(
artifact.logical_bytes for artifact in staged.artifacts
)
completed_bytes = 0
for entry in staged.files:
result = materializer.materialize(
job_id=staged.job_id,
@@ -263,6 +274,9 @@ def stage_transfer(
entry.allocated_bytes = result.allocated_bytes
entry.sparse = result.sparse
entry.source_staging_method = result.method
completed_bytes += entry.logical_bytes
if progress is not None:
progress(completed_bytes, total_bytes)
artifact_sources = artifact_sources or {}
for artifact in staged.artifacts:
@@ -287,6 +301,9 @@ def stage_transfer(
)
artifact.allocated_bytes = result.allocated_bytes
artifact.sparse = result.sparse
completed_bytes += artifact.logical_bytes
if progress is not None:
progress(completed_bytes, total_bytes)
digest = _sha256_file(
_existing_regular_file(
job_directory, _relative_path(artifact.payload_relative_path)
@@ -356,6 +373,8 @@ def materialize_transfer(
allow_hardlink: bool = True,
allow_reflink: bool = True,
sparse_supported: bool = True,
cancel_check: Callable[[], None] | None = None,
progress: Callable[[int, int], None] | None = None,
) -> transfer_pb2.TransferManifest:
"""Materialize a verified published payload into a target content root."""
@@ -366,7 +385,10 @@ def materialize_transfer(
store,
allow_hardlink=allow_hardlink,
allow_reflink=allow_reflink,
cancel_check=cancel_check,
)
total_bytes = sum(entry.logical_bytes for entry in result_manifest.files)
completed_bytes = 0
for entry in result_manifest.files:
result = materializer.materialize(
job_id=result_manifest.job_id,
@@ -384,6 +406,9 @@ def materialize_transfer(
)
entry.target_method = result.method
entry.target_preexisted = result.destination_preexisted
completed_bytes += entry.logical_bytes
if progress is not None:
progress(completed_bytes, total_bytes)
return result_manifest
@@ -455,20 +480,35 @@ def _validate_manifest(manifest: transfer_pb2.TransferManifest) -> None:
raise TransferIntegrityError("manifest creation timestamp is required")
indices: set[int] = set()
payload_paths: set[str] = set()
folded_payload_paths: set[str] = set()
target_paths: set[str] = set()
folded_target_paths: set[str] = set()
for entry in manifest.files:
if entry.file_index in indices:
raise TransferIntegrityError("manifest contains duplicate file indices")
indices.add(entry.file_index)
payload = _relative_path(entry.payload_relative_path).as_posix()
_relative_path(entry.target_canonical_path)
target = _relative_path(entry.target_canonical_path).as_posix()
if payload in payload_paths:
raise TransferIntegrityError("manifest contains duplicate payload paths")
if payload.casefold() in folded_payload_paths:
raise TransferIntegrityError("manifest contains case-colliding payload paths")
if target in target_paths:
raise TransferIntegrityError("manifest contains duplicate target paths")
if target.casefold() in folded_target_paths:
raise TransferIntegrityError("manifest contains case-colliding target paths")
payload_paths.add(payload)
folded_payload_paths.add(payload.casefold())
target_paths.add(target)
folded_target_paths.add(target.casefold())
for artifact in manifest.artifacts:
payload = _relative_path(artifact.payload_relative_path).as_posix()
if payload in payload_paths:
raise TransferIntegrityError("manifest contains duplicate payload paths")
if payload.casefold() in folded_payload_paths:
raise TransferIntegrityError("manifest contains case-colliding payload paths")
payload_paths.add(payload)
folded_payload_paths.add(payload.casefold())
def _relative_path(value: str) -> PurePosixPath:
@@ -569,6 +609,7 @@ def _sparse_copy(
destination: Path,
operation_id: str,
chunk_bytes: int,
cancel_check: Callable[[], None],
) -> None:
temporary = _temporary_path(destination, operation_id)
source_fd = os.open(source, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
@@ -580,7 +621,13 @@ def _sparse_copy(
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
stat.S_IMODE(source_stat.st_mode),
)
_copy_extents(source_fd, destination_fd, source_stat.st_size, chunk_bytes)
_copy_extents(
source_fd,
destination_fd,
source_stat.st_size,
chunk_bytes,
cancel_check,
)
os.ftruncate(destination_fd, source_stat.st_size)
os.fsync(destination_fd)
_publish_temporary(temporary, destination)
@@ -596,6 +643,7 @@ def _copy_extents(
destination_fd: int,
size: int,
chunk_bytes: int,
cancel_check: Callable[[], None],
) -> None:
if size == 0:
return
@@ -613,6 +661,7 @@ def _copy_extents(
os.lseek(destination_fd, data_offset, os.SEEK_SET)
remaining = min(hole_offset, size) - data_offset
while remaining:
cancel_check()
data = os.read(source_fd, min(chunk_bytes, remaining))
if not data:
raise TransferIntegrityError("source ended during sparse copy")
@@ -626,6 +675,7 @@ def _copy_extents(
os.lseek(destination_fd, 0, os.SEEK_SET)
remaining = size
while remaining:
cancel_check()
data = os.read(source_fd, min(chunk_bytes, remaining))
if not data:
raise TransferIntegrityError("source ended during copy")