From d2fa69a1d6a231160ef2e156e7e7004b9de5d444 Mon Sep 17 00:00:00 2001 From: Cabbagec Date: Sat, 25 Jul 2026 02:37:30 +0000 Subject: [PATCH] fix: report partial Syncthing transfer progress --- deploy/production/lithium/compose.yaml | 2 +- deploy/production/x1/compose.yaml | 2 +- deploy/production/x2/compose.yaml | 2 +- docs/deployment-and-usage.md | 2 +- pyproject.toml | 2 +- src/archive_clients/syncthing.py | 88 +++++++++++++++++++++----- 6 files changed, 77 insertions(+), 21 deletions(-) diff --git a/deploy/production/lithium/compose.yaml b/deploy/production/lithium/compose.yaml index 83fe98c..06cd807 100644 --- a/deploy/production/lithium/compose.yaml +++ b/deploy/production/lithium/compose.yaml @@ -2,7 +2,7 @@ name: archive-control-archive services: archive-client: - image: sodium/archive-clients:v0.1.9 + image: sodium/archive-clients:v0.1.10 user: "1000:1000" restart: unless-stopped command: ["--config", "/etc/archive-control/client.toml"] diff --git a/deploy/production/x1/compose.yaml b/deploy/production/x1/compose.yaml index 757d9ea..5233777 100644 --- a/deploy/production/x1/compose.yaml +++ b/deploy/production/x1/compose.yaml @@ -2,7 +2,7 @@ name: archive-control-cache services: archive-client: - image: sodium/archive-clients:v0.1.9 + image: sodium/archive-clients:v0.1.10 user: "1001:1001" restart: unless-stopped network_mode: host diff --git a/deploy/production/x2/compose.yaml b/deploy/production/x2/compose.yaml index f48195e..e4ae4b9 100644 --- a/deploy/production/x2/compose.yaml +++ b/deploy/production/x2/compose.yaml @@ -2,7 +2,7 @@ name: archive-control-cache services: archive-client: - image: sodium/archive-clients:v0.1.9 + image: sodium/archive-clients:v0.1.10 user: "1001:1001" restart: unless-stopped network_mode: host diff --git a/docs/deployment-and-usage.md b/docs/deployment-and-usage.md index f07fc81..ea68f3a 100644 --- a/docs/deployment-and-usage.md +++ b/docs/deployment-and-usage.md @@ -218,7 +218,7 @@ cache/archive routes according to policy. ```yaml services: archive-client: - image: sodium/archive-clients:v0.1.9 + image: sodium/archive-clients:v0.1.10 user: "1001:1001" restart: unless-stopped command: ["archive-client", "--config", "/etc/archive-control/client.toml"] diff --git a/pyproject.toml b/pyproject.toml index 0501f4d..71b5d53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "archive-clients" -version = "0.1.9" +version = "0.1.10" requires-python = ">=3.11" dependencies = ["protobuf==7.35.1", "websockets==16.0"] diff --git a/src/archive_clients/syncthing.py b/src/archive_clients/syncthing.py index 4896b15..563a89d 100644 --- a/src/archive_clients/syncthing.py +++ b/src/archive_clients/syncthing.py @@ -139,19 +139,14 @@ class SyncthingTransferObserver: def status(self) -> SyncthingTransferStatus: published = load_published_transfer(self.local_job_directory) - total = 0 - for entry in published.manifest.files: - total += _verified_job_file_size( - self.local_job_directory, - entry.payload_relative_path, - entry.logical_bytes, - ) - for artifact in published.manifest.artifacts: - total += _verified_job_file_size( - self.local_job_directory, - artifact.payload_relative_path, - artifact.logical_bytes, - ) + declared_files = [ + (entry.payload_relative_path, entry.logical_bytes) + for entry in published.manifest.files + ] + [ + (artifact.payload_relative_path, artifact.logical_bytes) + for artifact in published.manifest.artifacts + ] + total = sum(size for _, size in declared_files) completion = self.transport.get_json( "/rest/db/completion?" @@ -178,11 +173,36 @@ class SyncthingTransferObserver: name for name in needed_names if name == self.job_relative_path or name.startswith(prefix) } - fraction = float(raw_fraction) / 100 - complete = fraction == 1 and not relevant + complete = not relevant + if complete: + for relative_path, expected_bytes in declared_files: + _verified_job_file_size( + self.local_job_directory, + relative_path, + expected_bytes, + ) + completed_bytes = total + else: + observed_bytes = sum( + _received_job_file_bytes( + self.local_job_directory, + relative_path, + expected_bytes, + ) + for relative_path, expected_bytes in declared_files + ) + observed_fraction = observed_bytes / total if total else 1.0 + # Folder completion remains useful when Syncthing has already + # atomically published a file but still reports it in its need + # queue; the allocated-byte estimate is needed for sparse temp + # files that are pre-sized before their blocks arrive. + fraction = min(observed_fraction, float(raw_fraction) / 100) + completed_bytes = int(total * fraction) + if complete: + fraction = 1.0 return SyncthingTransferStatus( fraction, - total if complete else int(total * fraction), + completed_bytes, total, complete, len(relevant), @@ -472,6 +492,42 @@ def _verified_job_file_size( return metadata.st_size +def _received_job_file_bytes( + job_directory: Path, + relative_path: str, + expected_bytes: int, +) -> int: + """Return a conservative receive estimate for one job-owned file. + + Syncthing writes incomplete files as ``.syncthing..tmp`` and may + pre-size that sparse temporary to its final logical length. Allocated + blocks, rather than ``st_size``, therefore provide the useful progress + signal until the final atomic rename occurs. + """ + + relative = PurePosixPath(relative_path) + current = job_directory + for component in relative.parts[:-1]: + current = current / component + final = current / relative.name + try: + metadata = final.lstat() + except FileNotFoundError: + metadata = None + if metadata is not None: + if stat.S_ISREG(metadata.st_mode) and metadata.st_size == expected_bytes: + return expected_bytes + return 0 + temporary = current / f".syncthing.{relative.name}.tmp" + try: + temporary_metadata = temporary.lstat() + except FileNotFoundError: + return 0 + if not stat.S_ISREG(temporary_metadata.st_mode): + return 0 + return min(expected_bytes, temporary_metadata.st_blocks * 512) + + def _needed_names(value: dict[str, Any]) -> set[str]: result: set[str] = set() for key in ("progress", "queued", "rest"):