758 lines
26 KiB
Python
758 lines
26 KiB
Python
"""Safe, journaled transfer staging and target materialization."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import errno
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import stat
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Mapping
|
|
|
|
from google.protobuf import json_format
|
|
|
|
from archive_clients.state import ClientStore
|
|
from archive_control.v1 import transfer_pb2
|
|
|
|
|
|
_FICLONE = 0x40049409
|
|
_COPY_CHUNK_BYTES = 1024 * 1024
|
|
_FALLBACK_ERRNOS = {
|
|
errno.EXDEV,
|
|
errno.EPERM,
|
|
errno.EACCES,
|
|
errno.EMLINK,
|
|
errno.ENOSYS,
|
|
errno.EINVAL,
|
|
errno.ENOTTY,
|
|
errno.EOPNOTSUPP,
|
|
}
|
|
|
|
|
|
class TransferError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class UnsafeTransferPath(TransferError):
|
|
pass
|
|
|
|
|
|
class TransferCollision(TransferError):
|
|
pass
|
|
|
|
|
|
class TransferIntegrityError(TransferError):
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class MaterializedFile:
|
|
method: int
|
|
logical_bytes: int
|
|
allocated_bytes: int
|
|
sparse: bool
|
|
destination_preexisted: bool
|
|
destination_device: int
|
|
destination_inode: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PublishedTransfer:
|
|
job_directory: Path
|
|
manifest_path: Path
|
|
ready_path: Path
|
|
manifest_sha256_hex: str
|
|
manifest: transfer_pb2.TransferManifest
|
|
|
|
|
|
class FileMaterializer:
|
|
"""Materialize one regular file without overwriting an existing path."""
|
|
|
|
def __init__(
|
|
self,
|
|
store: ClientStore,
|
|
*,
|
|
allow_hardlink: bool = True,
|
|
allow_reflink: bool = True,
|
|
copy_chunk_bytes: int = _COPY_CHUNK_BYTES,
|
|
):
|
|
if copy_chunk_bytes < 1:
|
|
raise ValueError("copy_chunk_bytes must be positive")
|
|
self.store = store
|
|
self.allow_hardlink = allow_hardlink
|
|
self.allow_reflink = allow_reflink
|
|
self.copy_chunk_bytes = copy_chunk_bytes
|
|
|
|
def materialize(
|
|
self,
|
|
*,
|
|
job_id: str,
|
|
operation_id: str,
|
|
source_root: Path,
|
|
source_relative_path: str,
|
|
destination_root: Path,
|
|
destination_relative_path: str,
|
|
expected_bytes: int,
|
|
allow_preexisting_reuse: bool,
|
|
sparse_supported: bool,
|
|
) -> MaterializedFile:
|
|
source_relative = _relative_path(source_relative_path)
|
|
destination_relative = _relative_path(destination_relative_path)
|
|
source = _existing_regular_file(source_root, source_relative)
|
|
source_stat = source.stat(follow_symlinks=False)
|
|
if source_stat.st_size != expected_bytes:
|
|
raise TransferIntegrityError(
|
|
f"source size is {source_stat.st_size}, expected {expected_bytes}"
|
|
)
|
|
source_sparse = _is_sparse(source_stat)
|
|
if source_sparse and not sparse_supported:
|
|
raise TransferError(
|
|
"source is sparse but the destination filesystem lacks sparse support"
|
|
)
|
|
destination = _prepare_destination(
|
|
destination_root, destination_relative
|
|
)
|
|
intent = {
|
|
"destination": destination_relative.as_posix(),
|
|
"destination_root_device": destination_root.stat().st_dev,
|
|
"destination_root_inode": destination_root.stat().st_ino,
|
|
"expected_bytes": expected_bytes,
|
|
"source": source_relative.as_posix(),
|
|
"source_device": source_stat.st_dev,
|
|
"source_inode": source_stat.st_ino,
|
|
"source_mtime_ns": source_stat.st_mtime_ns,
|
|
}
|
|
row = self.store.begin_file_operation(
|
|
operation_id, job_id, _canonical_json(intent).decode()
|
|
)
|
|
if row["state"] == "completed":
|
|
result = json.loads(str(row["result_json"]))
|
|
replayed = _result_from_json(result)
|
|
_verify_completed_destination(destination, replayed)
|
|
return replayed
|
|
|
|
if destination.exists() or destination.is_symlink():
|
|
if not allow_preexisting_reuse:
|
|
raise TransferCollision("destination already exists")
|
|
existing = destination.lstat()
|
|
if not stat.S_ISREG(existing.st_mode):
|
|
raise TransferCollision(
|
|
"pre-existing destination is not a regular file"
|
|
)
|
|
if existing.st_size != expected_bytes:
|
|
raise TransferCollision(
|
|
"pre-existing destination size does not match"
|
|
)
|
|
result = MaterializedFile(
|
|
transfer_pb2.MATERIALIZATION_METHOD_PREEXISTING_REUSED,
|
|
existing.st_size,
|
|
_allocated_bytes(existing),
|
|
_is_sparse(existing),
|
|
True,
|
|
existing.st_dev,
|
|
existing.st_ino,
|
|
)
|
|
self._complete(operation_id, result)
|
|
return result
|
|
|
|
method: int | None = None
|
|
if self.allow_hardlink:
|
|
try:
|
|
os.link(source, destination, follow_symlinks=False)
|
|
method = transfer_pb2.MATERIALIZATION_METHOD_HARD_LINK
|
|
except OSError as exc:
|
|
if exc.errno == errno.EEXIST:
|
|
raise TransferCollision("destination appeared concurrently") from exc
|
|
if exc.errno not in _FALLBACK_ERRNOS:
|
|
raise
|
|
if method is None and self.allow_reflink:
|
|
try:
|
|
_reflink(source, destination, operation_id)
|
|
method = transfer_pb2.MATERIALIZATION_METHOD_REFLINK
|
|
except OSError as exc:
|
|
if exc.errno == errno.EEXIST:
|
|
raise TransferCollision("destination appeared concurrently") from exc
|
|
if exc.errno not in _FALLBACK_ERRNOS:
|
|
raise
|
|
if method is None:
|
|
_sparse_copy(
|
|
source,
|
|
destination,
|
|
operation_id,
|
|
self.copy_chunk_bytes,
|
|
)
|
|
method = transfer_pb2.MATERIALIZATION_METHOD_COPY
|
|
|
|
created = destination.stat(follow_symlinks=False)
|
|
if created.st_size != expected_bytes:
|
|
raise TransferIntegrityError(
|
|
"materialized destination has an unexpected size"
|
|
)
|
|
result = MaterializedFile(
|
|
method,
|
|
created.st_size,
|
|
_allocated_bytes(created),
|
|
_is_sparse(created),
|
|
False,
|
|
created.st_dev,
|
|
created.st_ino,
|
|
)
|
|
self._complete(operation_id, result)
|
|
return result
|
|
|
|
def _complete(
|
|
self, operation_id: str, result: MaterializedFile
|
|
) -> None:
|
|
self.store.complete_file_operation(
|
|
operation_id,
|
|
_canonical_json(
|
|
{
|
|
"allocated_bytes": result.allocated_bytes,
|
|
"destination_preexisted": result.destination_preexisted,
|
|
"destination_device": result.destination_device,
|
|
"destination_inode": result.destination_inode,
|
|
"logical_bytes": result.logical_bytes,
|
|
"method": result.method,
|
|
"sparse": result.sparse,
|
|
}
|
|
).decode(),
|
|
)
|
|
|
|
|
|
def stage_transfer(
|
|
manifest: transfer_pb2.TransferManifest,
|
|
*,
|
|
source_root: Path,
|
|
sync_root: Path,
|
|
store: ClientStore,
|
|
artifact_sources: Mapping[str, Path] | None = None,
|
|
allow_hardlink: bool = True,
|
|
allow_reflink: bool = True,
|
|
sparse_supported: bool = True,
|
|
) -> PublishedTransfer:
|
|
"""Publish a complete isolated transfer namespace and ready marker."""
|
|
|
|
_validate_manifest(manifest)
|
|
staged = transfer_pb2.TransferManifest()
|
|
staged.CopyFrom(manifest)
|
|
job_directory = _job_directory(sync_root, staged.job_id)
|
|
materializer = FileMaterializer(
|
|
store,
|
|
allow_hardlink=allow_hardlink,
|
|
allow_reflink=allow_reflink,
|
|
)
|
|
for entry in staged.files:
|
|
result = materializer.materialize(
|
|
job_id=staged.job_id,
|
|
operation_id=_operation_id(
|
|
staged.job_id, "source-stage", entry.file_index,
|
|
entry.payload_relative_path,
|
|
),
|
|
source_root=source_root,
|
|
source_relative_path=entry.target_canonical_path,
|
|
destination_root=job_directory,
|
|
destination_relative_path=entry.payload_relative_path,
|
|
expected_bytes=entry.logical_bytes,
|
|
allow_preexisting_reuse=False,
|
|
sparse_supported=sparse_supported,
|
|
)
|
|
entry.allocated_bytes = result.allocated_bytes
|
|
entry.sparse = result.sparse
|
|
entry.source_staging_method = result.method
|
|
|
|
artifact_sources = artifact_sources or {}
|
|
for artifact in staged.artifacts:
|
|
source = artifact_sources.get(artifact.payload_relative_path)
|
|
if source is None:
|
|
raise TransferIntegrityError(
|
|
f"artifact source is missing: {artifact.payload_relative_path}"
|
|
)
|
|
result = materializer.materialize(
|
|
job_id=staged.job_id,
|
|
operation_id=_operation_id(
|
|
staged.job_id, "artifact-stage", artifact.kind,
|
|
artifact.payload_relative_path,
|
|
),
|
|
source_root=source.parent,
|
|
source_relative_path=source.name,
|
|
destination_root=job_directory,
|
|
destination_relative_path=artifact.payload_relative_path,
|
|
expected_bytes=artifact.logical_bytes,
|
|
allow_preexisting_reuse=False,
|
|
sparse_supported=sparse_supported,
|
|
)
|
|
artifact.allocated_bytes = result.allocated_bytes
|
|
artifact.sparse = result.sparse
|
|
digest = _sha256_file(
|
|
_existing_regular_file(
|
|
job_directory, _relative_path(artifact.payload_relative_path)
|
|
)
|
|
)
|
|
if artifact.sha256_hex and artifact.sha256_hex != digest:
|
|
raise TransferIntegrityError("artifact SHA-256 does not match")
|
|
artifact.sha256_hex = digest
|
|
|
|
manifest_bytes = canonical_message_json(staged)
|
|
manifest_path = job_directory / "manifest.json"
|
|
_atomic_create_or_verify(manifest_path, manifest_bytes)
|
|
digest = hashlib.sha256(manifest_bytes).hexdigest()
|
|
ready = transfer_pb2.ReadyMarker(
|
|
job_id=staged.job_id,
|
|
manifest_sha256_hex=digest,
|
|
)
|
|
ready.ready_at.GetCurrentTime()
|
|
ready_path = job_directory / "ready.json"
|
|
if ready_path.exists():
|
|
existing = load_published_transfer(job_directory)
|
|
if existing.manifest_sha256_hex != digest:
|
|
raise TransferCollision("existing ready marker describes another manifest")
|
|
return existing
|
|
_atomic_create_or_verify(ready_path, canonical_message_json(ready))
|
|
return PublishedTransfer(
|
|
job_directory,
|
|
manifest_path,
|
|
ready_path,
|
|
digest,
|
|
staged,
|
|
)
|
|
|
|
|
|
def load_published_transfer(job_directory: Path) -> PublishedTransfer:
|
|
manifest_path = job_directory / "manifest.json"
|
|
ready_path = job_directory / "ready.json"
|
|
manifest_bytes = _read_regular_file(manifest_path)
|
|
ready_bytes = _read_regular_file(ready_path)
|
|
ready = transfer_pb2.ReadyMarker()
|
|
manifest = transfer_pb2.TransferManifest()
|
|
try:
|
|
json_format.Parse(ready_bytes.decode(), ready)
|
|
json_format.Parse(manifest_bytes.decode(), manifest)
|
|
except (UnicodeDecodeError, json_format.ParseError) as exc:
|
|
raise TransferIntegrityError("published transfer JSON is invalid") from exc
|
|
digest = hashlib.sha256(manifest_bytes).hexdigest()
|
|
if ready.job_id != manifest.job_id:
|
|
raise TransferIntegrityError("ready marker job ID does not match manifest")
|
|
if ready.manifest_sha256_hex != digest:
|
|
raise TransferIntegrityError("ready marker manifest digest does not match")
|
|
_validate_manifest(manifest)
|
|
return PublishedTransfer(
|
|
job_directory,
|
|
manifest_path,
|
|
ready_path,
|
|
digest,
|
|
manifest,
|
|
)
|
|
|
|
|
|
def materialize_transfer(
|
|
published: PublishedTransfer,
|
|
*,
|
|
target_root: Path,
|
|
store: ClientStore,
|
|
allow_hardlink: bool = True,
|
|
allow_reflink: bool = True,
|
|
sparse_supported: bool = True,
|
|
) -> transfer_pb2.TransferManifest:
|
|
"""Materialize a verified published payload into a target content root."""
|
|
|
|
verified = load_published_transfer(published.job_directory)
|
|
result_manifest = transfer_pb2.TransferManifest()
|
|
result_manifest.CopyFrom(verified.manifest)
|
|
materializer = FileMaterializer(
|
|
store,
|
|
allow_hardlink=allow_hardlink,
|
|
allow_reflink=allow_reflink,
|
|
)
|
|
for entry in result_manifest.files:
|
|
result = materializer.materialize(
|
|
job_id=result_manifest.job_id,
|
|
operation_id=_operation_id(
|
|
result_manifest.job_id, "target-materialize",
|
|
entry.file_index, entry.target_canonical_path,
|
|
),
|
|
source_root=verified.job_directory,
|
|
source_relative_path=entry.payload_relative_path,
|
|
destination_root=target_root,
|
|
destination_relative_path=entry.target_canonical_path,
|
|
expected_bytes=entry.logical_bytes,
|
|
allow_preexisting_reuse=True,
|
|
sparse_supported=sparse_supported,
|
|
)
|
|
entry.target_method = result.method
|
|
entry.target_preexisted = result.destination_preexisted
|
|
return result_manifest
|
|
|
|
|
|
def cleanup_transfer(job_directory: Path) -> bool:
|
|
"""Remove only manifest-declared files and known job metadata."""
|
|
|
|
try:
|
|
published = load_published_transfer(job_directory)
|
|
except FileNotFoundError:
|
|
return False
|
|
paths = [
|
|
job_directory / _relative_path(entry.payload_relative_path)
|
|
for entry in published.manifest.files
|
|
]
|
|
paths.extend(
|
|
job_directory / _relative_path(artifact.payload_relative_path)
|
|
for artifact in published.manifest.artifacts
|
|
)
|
|
paths.extend((published.ready_path, published.manifest_path))
|
|
directories: set[Path] = set()
|
|
for path in paths:
|
|
try:
|
|
metadata = path.lstat()
|
|
except FileNotFoundError:
|
|
continue
|
|
if not stat.S_ISREG(metadata.st_mode):
|
|
raise TransferIntegrityError(
|
|
"job-owned cleanup path is not a regular file"
|
|
)
|
|
path.unlink()
|
|
parent = path.parent
|
|
while parent != job_directory.parent:
|
|
directories.add(parent)
|
|
if parent == job_directory:
|
|
break
|
|
parent = parent.parent
|
|
for directory in sorted(
|
|
directories, key=lambda item: len(item.parts), reverse=True
|
|
):
|
|
try:
|
|
directory.rmdir()
|
|
except OSError as exc:
|
|
if exc.errno not in {errno.ENOTEMPTY, errno.ENOENT}:
|
|
raise
|
|
return True
|
|
|
|
|
|
def canonical_message_json(message: object) -> bytes:
|
|
value = json_format.MessageToDict(
|
|
message,
|
|
preserving_proto_field_name=False,
|
|
always_print_fields_with_no_presence=True,
|
|
)
|
|
return _canonical_json(value)
|
|
|
|
|
|
def _validate_manifest(manifest: transfer_pb2.TransferManifest) -> None:
|
|
if manifest.manifest_version != 1:
|
|
raise TransferIntegrityError("unsupported transfer manifest version")
|
|
try:
|
|
uuid.UUID(manifest.job_id)
|
|
except (ValueError, AttributeError) as exc:
|
|
raise TransferIntegrityError("manifest job ID is not a UUID") from exc
|
|
if not manifest.source_client_id or not manifest.target_client_id:
|
|
raise TransferIntegrityError("manifest source and target are required")
|
|
if not manifest.route_id:
|
|
raise TransferIntegrityError("manifest route is required")
|
|
if not manifest.HasField("created_at"):
|
|
raise TransferIntegrityError("manifest creation timestamp is required")
|
|
indices: set[int] = set()
|
|
payload_paths: set[str] = set()
|
|
for entry in manifest.files:
|
|
if entry.file_index in indices:
|
|
raise TransferIntegrityError("manifest contains duplicate file indices")
|
|
indices.add(entry.file_index)
|
|
payload = _relative_path(entry.payload_relative_path).as_posix()
|
|
_relative_path(entry.target_canonical_path)
|
|
if payload in payload_paths:
|
|
raise TransferIntegrityError("manifest contains duplicate payload paths")
|
|
payload_paths.add(payload)
|
|
for artifact in manifest.artifacts:
|
|
payload = _relative_path(artifact.payload_relative_path).as_posix()
|
|
if payload in payload_paths:
|
|
raise TransferIntegrityError("manifest contains duplicate payload paths")
|
|
payload_paths.add(payload)
|
|
|
|
|
|
def _relative_path(value: str) -> PurePosixPath:
|
|
if (
|
|
not value
|
|
or "\x00" in value
|
|
or "\\" in value
|
|
or value.startswith("/")
|
|
or any(part in {"", ".", ".."} for part in value.split("/"))
|
|
):
|
|
raise UnsafeTransferPath("transfer path is not a safe relative POSIX path")
|
|
path = PurePosixPath(value)
|
|
if path.is_absolute():
|
|
raise UnsafeTransferPath("transfer path must be relative")
|
|
return path
|
|
|
|
|
|
def _job_directory(sync_root: Path, job_id: str) -> Path:
|
|
try:
|
|
uuid.UUID(job_id)
|
|
except ValueError as exc:
|
|
raise TransferIntegrityError("job ID is not a UUID") from exc
|
|
return _prepare_directory(
|
|
sync_root, PurePosixPath(".archive-control", "jobs", job_id)
|
|
)
|
|
|
|
|
|
def _prepare_destination(root: Path, relative: PurePosixPath) -> Path:
|
|
root_stat = root.stat(follow_symlinks=False)
|
|
if not stat.S_ISDIR(root_stat.st_mode):
|
|
raise UnsafeTransferPath("configured root is not a directory")
|
|
parent = _prepare_directory(root, PurePosixPath(*relative.parts[:-1]))
|
|
return parent / relative.name
|
|
|
|
|
|
def _prepare_directory(root: Path, relative: PurePosixPath) -> Path:
|
|
current = root
|
|
root_stat = current.stat(follow_symlinks=False)
|
|
if not stat.S_ISDIR(root_stat.st_mode):
|
|
raise UnsafeTransferPath("configured root is not a directory")
|
|
for component in relative.parts:
|
|
current = current / component
|
|
try:
|
|
metadata = current.lstat()
|
|
except FileNotFoundError:
|
|
current.mkdir(mode=0o700)
|
|
metadata = current.lstat()
|
|
if not stat.S_ISDIR(metadata.st_mode):
|
|
raise UnsafeTransferPath("transfer parent is not a real directory")
|
|
return current
|
|
|
|
|
|
def _existing_regular_file(root: Path, relative: PurePosixPath) -> Path:
|
|
current = root
|
|
if not stat.S_ISDIR(current.stat(follow_symlinks=False).st_mode):
|
|
raise UnsafeTransferPath("configured root is not a directory")
|
|
for component in relative.parts[:-1]:
|
|
current = current / component
|
|
metadata = current.lstat()
|
|
if not stat.S_ISDIR(metadata.st_mode):
|
|
raise UnsafeTransferPath("source parent is not a real directory")
|
|
result = current / relative.name
|
|
metadata = result.lstat()
|
|
if not stat.S_ISREG(metadata.st_mode):
|
|
raise UnsafeTransferPath("source is not a regular file")
|
|
return result
|
|
|
|
|
|
def _read_regular_file(path: Path) -> bytes:
|
|
metadata = path.lstat()
|
|
if not stat.S_ISREG(metadata.st_mode):
|
|
raise TransferIntegrityError("transfer metadata is not a regular file")
|
|
return path.read_bytes()
|
|
|
|
|
|
def _reflink(source: Path, destination: Path, operation_id: str) -> None:
|
|
temporary = _temporary_path(destination, operation_id)
|
|
source_fd = os.open(source, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
|
destination_fd = -1
|
|
try:
|
|
destination_fd = os.open(
|
|
temporary,
|
|
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
|
|
stat.S_IMODE(source.stat(follow_symlinks=False).st_mode),
|
|
)
|
|
fcntl.ioctl(destination_fd, _FICLONE, source_fd)
|
|
os.fsync(destination_fd)
|
|
_publish_temporary(temporary, destination)
|
|
finally:
|
|
os.close(source_fd)
|
|
if destination_fd >= 0:
|
|
os.close(destination_fd)
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def _sparse_copy(
|
|
source: Path,
|
|
destination: Path,
|
|
operation_id: str,
|
|
chunk_bytes: int,
|
|
) -> None:
|
|
temporary = _temporary_path(destination, operation_id)
|
|
source_fd = os.open(source, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
|
destination_fd = -1
|
|
try:
|
|
source_stat = os.fstat(source_fd)
|
|
destination_fd = os.open(
|
|
temporary,
|
|
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
|
|
stat.S_IMODE(source_stat.st_mode),
|
|
)
|
|
_copy_extents(source_fd, destination_fd, source_stat.st_size, chunk_bytes)
|
|
os.ftruncate(destination_fd, source_stat.st_size)
|
|
os.fsync(destination_fd)
|
|
_publish_temporary(temporary, destination)
|
|
finally:
|
|
os.close(source_fd)
|
|
if destination_fd >= 0:
|
|
os.close(destination_fd)
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def _copy_extents(
|
|
source_fd: int,
|
|
destination_fd: int,
|
|
size: int,
|
|
chunk_bytes: int,
|
|
) -> None:
|
|
if size == 0:
|
|
return
|
|
try:
|
|
position = 0
|
|
while position < size:
|
|
try:
|
|
data_offset = os.lseek(source_fd, position, os.SEEK_DATA)
|
|
except OSError as exc:
|
|
if exc.errno == errno.ENXIO:
|
|
break
|
|
raise
|
|
hole_offset = os.lseek(source_fd, data_offset, os.SEEK_HOLE)
|
|
os.lseek(source_fd, data_offset, os.SEEK_SET)
|
|
os.lseek(destination_fd, data_offset, os.SEEK_SET)
|
|
remaining = min(hole_offset, size) - data_offset
|
|
while remaining:
|
|
data = os.read(source_fd, min(chunk_bytes, remaining))
|
|
if not data:
|
|
raise TransferIntegrityError("source ended during sparse copy")
|
|
_write_all(destination_fd, data)
|
|
remaining -= len(data)
|
|
position = hole_offset
|
|
except OSError as exc:
|
|
if exc.errno not in {errno.EINVAL, errno.ENOTSUP, errno.ENOSYS}:
|
|
raise
|
|
os.lseek(source_fd, 0, os.SEEK_SET)
|
|
os.lseek(destination_fd, 0, os.SEEK_SET)
|
|
remaining = size
|
|
while remaining:
|
|
data = os.read(source_fd, min(chunk_bytes, remaining))
|
|
if not data:
|
|
raise TransferIntegrityError("source ended during copy")
|
|
_write_all(destination_fd, data)
|
|
remaining -= len(data)
|
|
|
|
|
|
def _write_all(file_descriptor: int, data: bytes) -> None:
|
|
view = memoryview(data)
|
|
while view:
|
|
written = os.write(file_descriptor, view)
|
|
view = view[written:]
|
|
|
|
|
|
def _publish_temporary(temporary: Path, destination: Path) -> None:
|
|
try:
|
|
os.link(temporary, destination, follow_symlinks=False)
|
|
except FileExistsError as exc:
|
|
raise TransferCollision("destination appeared concurrently") from exc
|
|
_fsync_directory(destination.parent)
|
|
|
|
|
|
def _temporary_path(destination: Path, operation_id: str) -> Path:
|
|
digest = hashlib.sha256(operation_id.encode()).hexdigest()[:16]
|
|
return destination.with_name(f".{destination.name}.archive-control-{digest}.tmp")
|
|
|
|
|
|
def _atomic_create_or_verify(path: Path, content: bytes) -> None:
|
|
if path.exists() or path.is_symlink():
|
|
if _read_regular_file(path) != content:
|
|
raise TransferCollision(f"{path.name} already exists with other content")
|
|
return
|
|
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
|
|
file_descriptor = os.open(
|
|
temporary,
|
|
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
|
|
0o600,
|
|
)
|
|
try:
|
|
_write_all(file_descriptor, content)
|
|
os.fsync(file_descriptor)
|
|
finally:
|
|
os.close(file_descriptor)
|
|
try:
|
|
_publish_temporary(temporary, path)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def _verify_completed_destination(
|
|
destination: Path, expected: MaterializedFile
|
|
) -> None:
|
|
try:
|
|
metadata = destination.lstat()
|
|
except FileNotFoundError as exc:
|
|
raise TransferIntegrityError(
|
|
"completed journal entry has no destination"
|
|
) from exc
|
|
if (
|
|
not stat.S_ISREG(metadata.st_mode)
|
|
or metadata.st_size != expected.logical_bytes
|
|
or metadata.st_dev != expected.destination_device
|
|
or metadata.st_ino != expected.destination_inode
|
|
):
|
|
raise TransferIntegrityError(
|
|
"completed journal destination no longer matches"
|
|
)
|
|
|
|
|
|
def _result_from_json(value: object) -> MaterializedFile:
|
|
if not isinstance(value, dict):
|
|
raise TransferIntegrityError("file journal result is invalid")
|
|
try:
|
|
return MaterializedFile(
|
|
int(value["method"]),
|
|
int(value["logical_bytes"]),
|
|
int(value["allocated_bytes"]),
|
|
bool(value["sparse"]),
|
|
bool(value["destination_preexisted"]),
|
|
int(value["destination_device"]),
|
|
int(value["destination_inode"]),
|
|
)
|
|
except (KeyError, TypeError, ValueError) as exc:
|
|
raise TransferIntegrityError("file journal result is invalid") from exc
|
|
|
|
|
|
def _operation_id(
|
|
job_id: str, phase: str, index: int, path: str
|
|
) -> str:
|
|
digest = hashlib.sha256(
|
|
f"{job_id}\0{phase}\0{index}\0{path}".encode()
|
|
).hexdigest()
|
|
return f"{phase}:{digest}"
|
|
|
|
|
|
def _allocated_bytes(metadata: os.stat_result) -> int:
|
|
return int(getattr(metadata, "st_blocks", 0)) * 512
|
|
|
|
|
|
def _sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
|
try:
|
|
while data := os.read(descriptor, _COPY_CHUNK_BYTES):
|
|
digest.update(data)
|
|
finally:
|
|
os.close(descriptor)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _is_sparse(metadata: os.stat_result) -> bool:
|
|
return metadata.st_size > 0 and _allocated_bytes(metadata) < metadata.st_size
|
|
|
|
|
|
def _canonical_json(value: object) -> bytes:
|
|
return json.dumps(
|
|
value,
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
sort_keys=True,
|
|
).encode()
|
|
|
|
|
|
def _fsync_directory(path: Path) -> None:
|
|
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
|
try:
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|