feat: add journaled transfer materialization
This commit is contained in:
@@ -0,0 +1,713 @@
|
||||
"""Safe, journaled transfer staging and target materialization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Mapping
|
||||
|
||||
from google.protobuf import json_format
|
||||
|
||||
from archive_clients.state import ClientStore
|
||||
from archive_control.v1 import transfer_pb2
|
||||
|
||||
|
||||
_FICLONE = 0x40049409
|
||||
_COPY_CHUNK_BYTES = 1024 * 1024
|
||||
_FALLBACK_ERRNOS = {
|
||||
errno.EXDEV,
|
||||
errno.EPERM,
|
||||
errno.EACCES,
|
||||
errno.EMLINK,
|
||||
errno.ENOSYS,
|
||||
errno.EINVAL,
|
||||
errno.ENOTTY,
|
||||
errno.EOPNOTSUPP,
|
||||
}
|
||||
|
||||
|
||||
class TransferError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class UnsafeTransferPath(TransferError):
|
||||
pass
|
||||
|
||||
|
||||
class TransferCollision(TransferError):
|
||||
pass
|
||||
|
||||
|
||||
class TransferIntegrityError(TransferError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MaterializedFile:
|
||||
method: int
|
||||
logical_bytes: int
|
||||
allocated_bytes: int
|
||||
sparse: bool
|
||||
destination_preexisted: bool
|
||||
destination_device: int
|
||||
destination_inode: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PublishedTransfer:
|
||||
job_directory: Path
|
||||
manifest_path: Path
|
||||
ready_path: Path
|
||||
manifest_sha256_hex: str
|
||||
manifest: transfer_pb2.TransferManifest
|
||||
|
||||
|
||||
class FileMaterializer:
|
||||
"""Materialize one regular file without overwriting an existing path."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: ClientStore,
|
||||
*,
|
||||
allow_hardlink: bool = True,
|
||||
allow_reflink: bool = True,
|
||||
copy_chunk_bytes: int = _COPY_CHUNK_BYTES,
|
||||
):
|
||||
if copy_chunk_bytes < 1:
|
||||
raise ValueError("copy_chunk_bytes must be positive")
|
||||
self.store = store
|
||||
self.allow_hardlink = allow_hardlink
|
||||
self.allow_reflink = allow_reflink
|
||||
self.copy_chunk_bytes = copy_chunk_bytes
|
||||
|
||||
def materialize(
|
||||
self,
|
||||
*,
|
||||
job_id: str,
|
||||
operation_id: str,
|
||||
source_root: Path,
|
||||
source_relative_path: str,
|
||||
destination_root: Path,
|
||||
destination_relative_path: str,
|
||||
expected_bytes: int,
|
||||
allow_preexisting_reuse: bool,
|
||||
sparse_supported: bool,
|
||||
) -> MaterializedFile:
|
||||
source_relative = _relative_path(source_relative_path)
|
||||
destination_relative = _relative_path(destination_relative_path)
|
||||
source = _existing_regular_file(source_root, source_relative)
|
||||
source_stat = source.stat(follow_symlinks=False)
|
||||
if source_stat.st_size != expected_bytes:
|
||||
raise TransferIntegrityError(
|
||||
f"source size is {source_stat.st_size}, expected {expected_bytes}"
|
||||
)
|
||||
source_sparse = _is_sparse(source_stat)
|
||||
if source_sparse and not sparse_supported:
|
||||
raise TransferError(
|
||||
"source is sparse but the destination filesystem lacks sparse support"
|
||||
)
|
||||
destination = _prepare_destination(
|
||||
destination_root, destination_relative
|
||||
)
|
||||
intent = {
|
||||
"destination": destination_relative.as_posix(),
|
||||
"destination_root_device": destination_root.stat().st_dev,
|
||||
"destination_root_inode": destination_root.stat().st_ino,
|
||||
"expected_bytes": expected_bytes,
|
||||
"source": source_relative.as_posix(),
|
||||
"source_device": source_stat.st_dev,
|
||||
"source_inode": source_stat.st_ino,
|
||||
"source_mtime_ns": source_stat.st_mtime_ns,
|
||||
}
|
||||
row = self.store.begin_file_operation(
|
||||
operation_id, job_id, _canonical_json(intent).decode()
|
||||
)
|
||||
if row["state"] == "completed":
|
||||
result = json.loads(str(row["result_json"]))
|
||||
replayed = _result_from_json(result)
|
||||
_verify_completed_destination(destination, replayed)
|
||||
return replayed
|
||||
|
||||
if destination.exists() or destination.is_symlink():
|
||||
if not allow_preexisting_reuse:
|
||||
raise TransferCollision("destination already exists")
|
||||
existing = destination.lstat()
|
||||
if not stat.S_ISREG(existing.st_mode):
|
||||
raise TransferCollision(
|
||||
"pre-existing destination is not a regular file"
|
||||
)
|
||||
if existing.st_size != expected_bytes:
|
||||
raise TransferCollision(
|
||||
"pre-existing destination size does not match"
|
||||
)
|
||||
result = MaterializedFile(
|
||||
transfer_pb2.MATERIALIZATION_METHOD_PREEXISTING_REUSED,
|
||||
existing.st_size,
|
||||
_allocated_bytes(existing),
|
||||
_is_sparse(existing),
|
||||
True,
|
||||
existing.st_dev,
|
||||
existing.st_ino,
|
||||
)
|
||||
self._complete(operation_id, result)
|
||||
return result
|
||||
|
||||
method: int | None = None
|
||||
if self.allow_hardlink:
|
||||
try:
|
||||
os.link(source, destination, follow_symlinks=False)
|
||||
method = transfer_pb2.MATERIALIZATION_METHOD_HARD_LINK
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.EEXIST:
|
||||
raise TransferCollision("destination appeared concurrently") from exc
|
||||
if exc.errno not in _FALLBACK_ERRNOS:
|
||||
raise
|
||||
if method is None and self.allow_reflink:
|
||||
try:
|
||||
_reflink(source, destination, operation_id)
|
||||
method = transfer_pb2.MATERIALIZATION_METHOD_REFLINK
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.EEXIST:
|
||||
raise TransferCollision("destination appeared concurrently") from exc
|
||||
if exc.errno not in _FALLBACK_ERRNOS:
|
||||
raise
|
||||
if method is None:
|
||||
_sparse_copy(
|
||||
source,
|
||||
destination,
|
||||
operation_id,
|
||||
self.copy_chunk_bytes,
|
||||
)
|
||||
method = transfer_pb2.MATERIALIZATION_METHOD_COPY
|
||||
|
||||
created = destination.stat(follow_symlinks=False)
|
||||
if created.st_size != expected_bytes:
|
||||
raise TransferIntegrityError(
|
||||
"materialized destination has an unexpected size"
|
||||
)
|
||||
result = MaterializedFile(
|
||||
method,
|
||||
created.st_size,
|
||||
_allocated_bytes(created),
|
||||
_is_sparse(created),
|
||||
False,
|
||||
created.st_dev,
|
||||
created.st_ino,
|
||||
)
|
||||
self._complete(operation_id, result)
|
||||
return result
|
||||
|
||||
def _complete(
|
||||
self, operation_id: str, result: MaterializedFile
|
||||
) -> None:
|
||||
self.store.complete_file_operation(
|
||||
operation_id,
|
||||
_canonical_json(
|
||||
{
|
||||
"allocated_bytes": result.allocated_bytes,
|
||||
"destination_preexisted": result.destination_preexisted,
|
||||
"destination_device": result.destination_device,
|
||||
"destination_inode": result.destination_inode,
|
||||
"logical_bytes": result.logical_bytes,
|
||||
"method": result.method,
|
||||
"sparse": result.sparse,
|
||||
}
|
||||
).decode(),
|
||||
)
|
||||
|
||||
|
||||
def stage_transfer(
|
||||
manifest: transfer_pb2.TransferManifest,
|
||||
*,
|
||||
source_root: Path,
|
||||
sync_root: Path,
|
||||
store: ClientStore,
|
||||
artifact_sources: Mapping[str, Path] | None = None,
|
||||
allow_hardlink: bool = True,
|
||||
allow_reflink: bool = True,
|
||||
sparse_supported: bool = True,
|
||||
) -> PublishedTransfer:
|
||||
"""Publish a complete isolated transfer namespace and ready marker."""
|
||||
|
||||
_validate_manifest(manifest)
|
||||
staged = transfer_pb2.TransferManifest()
|
||||
staged.CopyFrom(manifest)
|
||||
job_directory = _job_directory(sync_root, staged.job_id)
|
||||
materializer = FileMaterializer(
|
||||
store,
|
||||
allow_hardlink=allow_hardlink,
|
||||
allow_reflink=allow_reflink,
|
||||
)
|
||||
for entry in staged.files:
|
||||
result = materializer.materialize(
|
||||
job_id=staged.job_id,
|
||||
operation_id=_operation_id(
|
||||
staged.job_id, "source-stage", entry.file_index,
|
||||
entry.payload_relative_path,
|
||||
),
|
||||
source_root=source_root,
|
||||
source_relative_path=entry.target_canonical_path,
|
||||
destination_root=job_directory,
|
||||
destination_relative_path=entry.payload_relative_path,
|
||||
expected_bytes=entry.logical_bytes,
|
||||
allow_preexisting_reuse=False,
|
||||
sparse_supported=sparse_supported,
|
||||
)
|
||||
entry.allocated_bytes = result.allocated_bytes
|
||||
entry.sparse = result.sparse
|
||||
entry.source_staging_method = result.method
|
||||
|
||||
artifact_sources = artifact_sources or {}
|
||||
for artifact in staged.artifacts:
|
||||
source = artifact_sources.get(artifact.payload_relative_path)
|
||||
if source is None:
|
||||
raise TransferIntegrityError(
|
||||
f"artifact source is missing: {artifact.payload_relative_path}"
|
||||
)
|
||||
result = materializer.materialize(
|
||||
job_id=staged.job_id,
|
||||
operation_id=_operation_id(
|
||||
staged.job_id, "artifact-stage", artifact.kind,
|
||||
artifact.payload_relative_path,
|
||||
),
|
||||
source_root=source.parent,
|
||||
source_relative_path=source.name,
|
||||
destination_root=job_directory,
|
||||
destination_relative_path=artifact.payload_relative_path,
|
||||
expected_bytes=artifact.logical_bytes,
|
||||
allow_preexisting_reuse=False,
|
||||
sparse_supported=sparse_supported,
|
||||
)
|
||||
artifact.allocated_bytes = result.allocated_bytes
|
||||
artifact.sparse = result.sparse
|
||||
digest = _sha256_file(
|
||||
_existing_regular_file(
|
||||
job_directory, _relative_path(artifact.payload_relative_path)
|
||||
)
|
||||
)
|
||||
if artifact.sha256_hex and artifact.sha256_hex != digest:
|
||||
raise TransferIntegrityError("artifact SHA-256 does not match")
|
||||
artifact.sha256_hex = digest
|
||||
|
||||
manifest_bytes = canonical_message_json(staged)
|
||||
manifest_path = job_directory / "manifest.json"
|
||||
_atomic_create_or_verify(manifest_path, manifest_bytes)
|
||||
digest = hashlib.sha256(manifest_bytes).hexdigest()
|
||||
ready = transfer_pb2.ReadyMarker(
|
||||
job_id=staged.job_id,
|
||||
manifest_sha256_hex=digest,
|
||||
)
|
||||
ready.ready_at.GetCurrentTime()
|
||||
ready_path = job_directory / "ready.json"
|
||||
if ready_path.exists():
|
||||
existing = load_published_transfer(job_directory)
|
||||
if existing.manifest_sha256_hex != digest:
|
||||
raise TransferCollision("existing ready marker describes another manifest")
|
||||
return existing
|
||||
_atomic_create_or_verify(ready_path, canonical_message_json(ready))
|
||||
return PublishedTransfer(
|
||||
job_directory,
|
||||
manifest_path,
|
||||
ready_path,
|
||||
digest,
|
||||
staged,
|
||||
)
|
||||
|
||||
|
||||
def load_published_transfer(job_directory: Path) -> PublishedTransfer:
|
||||
manifest_path = job_directory / "manifest.json"
|
||||
ready_path = job_directory / "ready.json"
|
||||
manifest_bytes = _read_regular_file(manifest_path)
|
||||
ready_bytes = _read_regular_file(ready_path)
|
||||
ready = transfer_pb2.ReadyMarker()
|
||||
manifest = transfer_pb2.TransferManifest()
|
||||
try:
|
||||
json_format.Parse(ready_bytes.decode(), ready)
|
||||
json_format.Parse(manifest_bytes.decode(), manifest)
|
||||
except (UnicodeDecodeError, json_format.ParseError) as exc:
|
||||
raise TransferIntegrityError("published transfer JSON is invalid") from exc
|
||||
digest = hashlib.sha256(manifest_bytes).hexdigest()
|
||||
if ready.job_id != manifest.job_id:
|
||||
raise TransferIntegrityError("ready marker job ID does not match manifest")
|
||||
if ready.manifest_sha256_hex != digest:
|
||||
raise TransferIntegrityError("ready marker manifest digest does not match")
|
||||
_validate_manifest(manifest)
|
||||
return PublishedTransfer(
|
||||
job_directory,
|
||||
manifest_path,
|
||||
ready_path,
|
||||
digest,
|
||||
manifest,
|
||||
)
|
||||
|
||||
|
||||
def materialize_transfer(
|
||||
published: PublishedTransfer,
|
||||
*,
|
||||
target_root: Path,
|
||||
store: ClientStore,
|
||||
allow_hardlink: bool = True,
|
||||
allow_reflink: bool = True,
|
||||
sparse_supported: bool = True,
|
||||
) -> transfer_pb2.TransferManifest:
|
||||
"""Materialize a verified published payload into a target content root."""
|
||||
|
||||
verified = load_published_transfer(published.job_directory)
|
||||
result_manifest = transfer_pb2.TransferManifest()
|
||||
result_manifest.CopyFrom(verified.manifest)
|
||||
materializer = FileMaterializer(
|
||||
store,
|
||||
allow_hardlink=allow_hardlink,
|
||||
allow_reflink=allow_reflink,
|
||||
)
|
||||
for entry in result_manifest.files:
|
||||
result = materializer.materialize(
|
||||
job_id=result_manifest.job_id,
|
||||
operation_id=_operation_id(
|
||||
result_manifest.job_id, "target-materialize",
|
||||
entry.file_index, entry.target_canonical_path,
|
||||
),
|
||||
source_root=verified.job_directory,
|
||||
source_relative_path=entry.payload_relative_path,
|
||||
destination_root=target_root,
|
||||
destination_relative_path=entry.target_canonical_path,
|
||||
expected_bytes=entry.logical_bytes,
|
||||
allow_preexisting_reuse=True,
|
||||
sparse_supported=sparse_supported,
|
||||
)
|
||||
entry.target_method = result.method
|
||||
entry.target_preexisted = result.destination_preexisted
|
||||
return result_manifest
|
||||
|
||||
|
||||
def canonical_message_json(message: object) -> bytes:
|
||||
value = json_format.MessageToDict(
|
||||
message,
|
||||
preserving_proto_field_name=False,
|
||||
always_print_fields_with_no_presence=True,
|
||||
)
|
||||
return _canonical_json(value)
|
||||
|
||||
|
||||
def _validate_manifest(manifest: transfer_pb2.TransferManifest) -> None:
|
||||
if manifest.manifest_version != 1:
|
||||
raise TransferIntegrityError("unsupported transfer manifest version")
|
||||
try:
|
||||
uuid.UUID(manifest.job_id)
|
||||
except (ValueError, AttributeError) as exc:
|
||||
raise TransferIntegrityError("manifest job ID is not a UUID") from exc
|
||||
if not manifest.source_client_id or not manifest.target_client_id:
|
||||
raise TransferIntegrityError("manifest source and target are required")
|
||||
if not manifest.route_id:
|
||||
raise TransferIntegrityError("manifest route is required")
|
||||
if not manifest.HasField("created_at"):
|
||||
raise TransferIntegrityError("manifest creation timestamp is required")
|
||||
indices: set[int] = set()
|
||||
payload_paths: set[str] = set()
|
||||
for entry in manifest.files:
|
||||
if entry.file_index in indices:
|
||||
raise TransferIntegrityError("manifest contains duplicate file indices")
|
||||
indices.add(entry.file_index)
|
||||
payload = _relative_path(entry.payload_relative_path).as_posix()
|
||||
_relative_path(entry.target_canonical_path)
|
||||
if payload in payload_paths:
|
||||
raise TransferIntegrityError("manifest contains duplicate payload paths")
|
||||
payload_paths.add(payload)
|
||||
for artifact in manifest.artifacts:
|
||||
payload = _relative_path(artifact.payload_relative_path).as_posix()
|
||||
if payload in payload_paths:
|
||||
raise TransferIntegrityError("manifest contains duplicate payload paths")
|
||||
payload_paths.add(payload)
|
||||
|
||||
|
||||
def _relative_path(value: str) -> PurePosixPath:
|
||||
if (
|
||||
not value
|
||||
or "\x00" in value
|
||||
or "\\" in value
|
||||
or value.startswith("/")
|
||||
or any(part in {"", ".", ".."} for part in value.split("/"))
|
||||
):
|
||||
raise UnsafeTransferPath("transfer path is not a safe relative POSIX path")
|
||||
path = PurePosixPath(value)
|
||||
if path.is_absolute():
|
||||
raise UnsafeTransferPath("transfer path must be relative")
|
||||
return path
|
||||
|
||||
|
||||
def _job_directory(sync_root: Path, job_id: str) -> Path:
|
||||
try:
|
||||
uuid.UUID(job_id)
|
||||
except ValueError as exc:
|
||||
raise TransferIntegrityError("job ID is not a UUID") from exc
|
||||
return _prepare_directory(
|
||||
sync_root, PurePosixPath(".archive-control", "jobs", job_id)
|
||||
)
|
||||
|
||||
|
||||
def _prepare_destination(root: Path, relative: PurePosixPath) -> Path:
|
||||
root_stat = root.stat(follow_symlinks=False)
|
||||
if not stat.S_ISDIR(root_stat.st_mode):
|
||||
raise UnsafeTransferPath("configured root is not a directory")
|
||||
parent = _prepare_directory(root, PurePosixPath(*relative.parts[:-1]))
|
||||
return parent / relative.name
|
||||
|
||||
|
||||
def _prepare_directory(root: Path, relative: PurePosixPath) -> Path:
|
||||
current = root
|
||||
root_stat = current.stat(follow_symlinks=False)
|
||||
if not stat.S_ISDIR(root_stat.st_mode):
|
||||
raise UnsafeTransferPath("configured root is not a directory")
|
||||
for component in relative.parts:
|
||||
current = current / component
|
||||
try:
|
||||
metadata = current.lstat()
|
||||
except FileNotFoundError:
|
||||
current.mkdir(mode=0o700)
|
||||
metadata = current.lstat()
|
||||
if not stat.S_ISDIR(metadata.st_mode):
|
||||
raise UnsafeTransferPath("transfer parent is not a real directory")
|
||||
return current
|
||||
|
||||
|
||||
def _existing_regular_file(root: Path, relative: PurePosixPath) -> Path:
|
||||
current = root
|
||||
if not stat.S_ISDIR(current.stat(follow_symlinks=False).st_mode):
|
||||
raise UnsafeTransferPath("configured root is not a directory")
|
||||
for component in relative.parts[:-1]:
|
||||
current = current / component
|
||||
metadata = current.lstat()
|
||||
if not stat.S_ISDIR(metadata.st_mode):
|
||||
raise UnsafeTransferPath("source parent is not a real directory")
|
||||
result = current / relative.name
|
||||
metadata = result.lstat()
|
||||
if not stat.S_ISREG(metadata.st_mode):
|
||||
raise UnsafeTransferPath("source is not a regular file")
|
||||
return result
|
||||
|
||||
|
||||
def _read_regular_file(path: Path) -> bytes:
|
||||
metadata = path.lstat()
|
||||
if not stat.S_ISREG(metadata.st_mode):
|
||||
raise TransferIntegrityError("transfer metadata is not a regular file")
|
||||
return path.read_bytes()
|
||||
|
||||
|
||||
def _reflink(source: Path, destination: Path, operation_id: str) -> None:
|
||||
temporary = _temporary_path(destination, operation_id)
|
||||
source_fd = os.open(source, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
||||
destination_fd = -1
|
||||
try:
|
||||
destination_fd = os.open(
|
||||
temporary,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
|
||||
stat.S_IMODE(source.stat(follow_symlinks=False).st_mode),
|
||||
)
|
||||
fcntl.ioctl(destination_fd, _FICLONE, source_fd)
|
||||
os.fsync(destination_fd)
|
||||
_publish_temporary(temporary, destination)
|
||||
finally:
|
||||
os.close(source_fd)
|
||||
if destination_fd >= 0:
|
||||
os.close(destination_fd)
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _sparse_copy(
|
||||
source: Path,
|
||||
destination: Path,
|
||||
operation_id: str,
|
||||
chunk_bytes: int,
|
||||
) -> None:
|
||||
temporary = _temporary_path(destination, operation_id)
|
||||
source_fd = os.open(source, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
||||
destination_fd = -1
|
||||
try:
|
||||
source_stat = os.fstat(source_fd)
|
||||
destination_fd = os.open(
|
||||
temporary,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
|
||||
stat.S_IMODE(source_stat.st_mode),
|
||||
)
|
||||
_copy_extents(source_fd, destination_fd, source_stat.st_size, chunk_bytes)
|
||||
os.ftruncate(destination_fd, source_stat.st_size)
|
||||
os.fsync(destination_fd)
|
||||
_publish_temporary(temporary, destination)
|
||||
finally:
|
||||
os.close(source_fd)
|
||||
if destination_fd >= 0:
|
||||
os.close(destination_fd)
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _copy_extents(
|
||||
source_fd: int,
|
||||
destination_fd: int,
|
||||
size: int,
|
||||
chunk_bytes: int,
|
||||
) -> None:
|
||||
if size == 0:
|
||||
return
|
||||
try:
|
||||
position = 0
|
||||
while position < size:
|
||||
try:
|
||||
data_offset = os.lseek(source_fd, position, os.SEEK_DATA)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.ENXIO:
|
||||
break
|
||||
raise
|
||||
hole_offset = os.lseek(source_fd, data_offset, os.SEEK_HOLE)
|
||||
os.lseek(source_fd, data_offset, os.SEEK_SET)
|
||||
os.lseek(destination_fd, data_offset, os.SEEK_SET)
|
||||
remaining = min(hole_offset, size) - data_offset
|
||||
while remaining:
|
||||
data = os.read(source_fd, min(chunk_bytes, remaining))
|
||||
if not data:
|
||||
raise TransferIntegrityError("source ended during sparse copy")
|
||||
_write_all(destination_fd, data)
|
||||
remaining -= len(data)
|
||||
position = hole_offset
|
||||
except OSError as exc:
|
||||
if exc.errno not in {errno.EINVAL, errno.ENOTSUP, errno.ENOSYS}:
|
||||
raise
|
||||
os.lseek(source_fd, 0, os.SEEK_SET)
|
||||
os.lseek(destination_fd, 0, os.SEEK_SET)
|
||||
remaining = size
|
||||
while remaining:
|
||||
data = os.read(source_fd, min(chunk_bytes, remaining))
|
||||
if not data:
|
||||
raise TransferIntegrityError("source ended during copy")
|
||||
_write_all(destination_fd, data)
|
||||
remaining -= len(data)
|
||||
|
||||
|
||||
def _write_all(file_descriptor: int, data: bytes) -> None:
|
||||
view = memoryview(data)
|
||||
while view:
|
||||
written = os.write(file_descriptor, view)
|
||||
view = view[written:]
|
||||
|
||||
|
||||
def _publish_temporary(temporary: Path, destination: Path) -> None:
|
||||
try:
|
||||
os.link(temporary, destination, follow_symlinks=False)
|
||||
except FileExistsError as exc:
|
||||
raise TransferCollision("destination appeared concurrently") from exc
|
||||
_fsync_directory(destination.parent)
|
||||
|
||||
|
||||
def _temporary_path(destination: Path, operation_id: str) -> Path:
|
||||
digest = hashlib.sha256(operation_id.encode()).hexdigest()[:16]
|
||||
return destination.with_name(f".{destination.name}.archive-control-{digest}.tmp")
|
||||
|
||||
|
||||
def _atomic_create_or_verify(path: Path, content: bytes) -> None:
|
||||
if path.exists() or path.is_symlink():
|
||||
if _read_regular_file(path) != content:
|
||||
raise TransferCollision(f"{path.name} already exists with other content")
|
||||
return
|
||||
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
|
||||
file_descriptor = os.open(
|
||||
temporary,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
|
||||
0o600,
|
||||
)
|
||||
try:
|
||||
_write_all(file_descriptor, content)
|
||||
os.fsync(file_descriptor)
|
||||
finally:
|
||||
os.close(file_descriptor)
|
||||
try:
|
||||
_publish_temporary(temporary, path)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _verify_completed_destination(
|
||||
destination: Path, expected: MaterializedFile
|
||||
) -> None:
|
||||
try:
|
||||
metadata = destination.lstat()
|
||||
except FileNotFoundError as exc:
|
||||
raise TransferIntegrityError(
|
||||
"completed journal entry has no destination"
|
||||
) from exc
|
||||
if (
|
||||
not stat.S_ISREG(metadata.st_mode)
|
||||
or metadata.st_size != expected.logical_bytes
|
||||
or metadata.st_dev != expected.destination_device
|
||||
or metadata.st_ino != expected.destination_inode
|
||||
):
|
||||
raise TransferIntegrityError(
|
||||
"completed journal destination no longer matches"
|
||||
)
|
||||
|
||||
|
||||
def _result_from_json(value: object) -> MaterializedFile:
|
||||
if not isinstance(value, dict):
|
||||
raise TransferIntegrityError("file journal result is invalid")
|
||||
try:
|
||||
return MaterializedFile(
|
||||
int(value["method"]),
|
||||
int(value["logical_bytes"]),
|
||||
int(value["allocated_bytes"]),
|
||||
bool(value["sparse"]),
|
||||
bool(value["destination_preexisted"]),
|
||||
int(value["destination_device"]),
|
||||
int(value["destination_inode"]),
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise TransferIntegrityError("file journal result is invalid") from exc
|
||||
|
||||
|
||||
def _operation_id(
|
||||
job_id: str, phase: str, index: int, path: str
|
||||
) -> str:
|
||||
digest = hashlib.sha256(
|
||||
f"{job_id}\0{phase}\0{index}\0{path}".encode()
|
||||
).hexdigest()
|
||||
return f"{phase}:{digest}"
|
||||
|
||||
|
||||
def _allocated_bytes(metadata: os.stat_result) -> int:
|
||||
return int(getattr(metadata, "st_blocks", 0)) * 512
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
||||
try:
|
||||
while data := os.read(descriptor, _COPY_CHUNK_BYTES):
|
||||
digest.update(data)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _is_sparse(metadata: os.stat_result) -> bool:
|
||||
return metadata.st_size > 0 and _allocated_bytes(metadata) < metadata.st_size
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
).encode()
|
||||
|
||||
|
||||
def _fsync_directory(path: Path) -> None:
|
||||
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
Reference in New Issue
Block a user