diff --git a/docs/deployment-preflight.md b/docs/deployment-preflight.md index aeacc3a..6a116b9 100644 --- a/docs/deployment-preflight.md +++ b/docs/deployment-preflight.md @@ -89,6 +89,10 @@ python3 scripts/preflight-deployment.py ... \ - A real `link(2)` operation between a unique zero-byte file in the qB root and one in the future automatic-route root. The probe verifies that both names refer to the same inode and removes them unconditionally. +- Every existing `archive-control:*` Syncthing folder beneath `routes/` still + has its local directory. This catches a route-root bind-mount migration that + would otherwise hide an already configured folder and later fail a job with + `No such file or directory`. - The token, qB password, and Syncthing API-key files are non-empty regular files with no group/world permissions. - The client image can read its configuration and reports usable permissions, @@ -102,6 +106,12 @@ root and add a `local_path_overrides` mapping for that exact API path. This prevents automatic route folders being created on a small configuration filesystem while the client expects to hardlink from the qB data mount. +When converting an existing node, stop its client and Syncthing containers, +move each existing `routes/` directory from the old Syncthing config +tree into the new qB-backed route-root directory, then recreate Syncthing and +run this preflight. Do not discard existing route directories: they can contain +route handshake state or an in-progress transfer namespace. + ## Failure handling Treat a nonzero exit status as a deployment blocker. Correct the compose diff --git a/scripts/preflight-deployment.py b/scripts/preflight-deployment.py index df14b15..c1f5bf4 100755 --- a/scripts/preflight-deployment.py +++ b/scripts/preflight-deployment.py @@ -66,6 +66,11 @@ def map_path(container: dict[str, Any], path: str) -> Mapping: return max(matches, key=lambda item: item[0])[1] +def container_user(container: dict[str, Any]) -> str | None: + value = container.get("Config", {}).get("User") + return value if isinstance(value, str) and value else None + + def require_same_path(label: str, left: Mapping, right: Mapping) -> None: if left.source.resolve(strict=False) != right.source.resolve(strict=False): raise CheckFailure( @@ -169,7 +174,9 @@ raise SystemExit(0 if all(p.state == 1 for p in probes) else 1)''' print(result.stdout.strip()) -def run_hardlink_probe(container: str, qb_root: str, route_root: str) -> None: +def run_hardlink_probe( + container: str, qb_root: str, route_root: str, user: str | None = None, +) -> None: """Prove that the daemon can hard-link from qB data into automatic routes.""" program = r'''import os,sys,tempfile @@ -196,8 +203,12 @@ finally: except FileNotFoundError: pass ''' + command = ["docker", "exec"] + if user: + command.extend(["--user", user]) + command.extend([container, "python", "-c", program, qb_root, route_root]) result = subprocess.run( - ["docker", "exec", container, "python", "-c", program, qb_root, route_root], + command, check=False, text=True, capture_output=True, ) if result.returncode: @@ -206,6 +217,51 @@ finally: print(result.stdout.strip()) +def run_existing_route_directory_check(container: str, config: str) -> None: + """Reject a mount migration that hides an already configured route folder.""" + + program = r'''import json,os,sys +from pathlib import Path,PurePosixPath +from archive_clients.config import ClientConfig +from archive_clients.syncthing import SyncthingHttp +config=ClientConfig.load(Path(sys.argv[1])) +transport=SyncthingHttp(config.syncthing) +route_root=config.syncthing.api_root / "routes" +missing=[] +checked=[] +for folder in transport.get_json("/rest/config").get("folders",[]): + if not isinstance(folder,dict) or not str(folder.get("label","")).startswith("archive-control:"): + continue + raw=folder.get("path") + if not isinstance(raw,str) or not raw: + continue + api=PurePosixPath(raw) + if api.parts and api.parts[0] == "~": + api=config.syncthing.api_root.joinpath(*api.parts[1:]) + try: + api.relative_to(route_root) + except ValueError: + continue + local=config.syncthing.roots.api_to_local(api.as_posix()) + checked.append(str(local)) + if not local.is_dir(): + missing.append({"folder_id":folder.get("id",""),"path":str(local)}) +print(json.dumps({"checked":checked,"missing":missing},sort_keys=True)) +raise SystemExit(1 if missing else 0) +''' + result = subprocess.run( + ["docker", "exec", container, "python", "-c", program, config], + check=False, text=True, capture_output=True, + ) + if result.returncode: + detail = result.stderr.strip() or result.stdout.strip() or "failed" + raise CheckFailure( + "an existing Archive Control Syncthing route directory is missing; " + f"complete the route-directory migration before starting jobs: {detail}" + ) + print(result.stdout.strip()) + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description="Read-only Archive Control Docker deployment preflight" @@ -248,7 +304,11 @@ def main(argv: list[str] | None = None) -> int: "mounts; hard-link staging would be unavailable" ) run_hardlink_probe( - args.client_container, qb["local_root"], route_local_path(sync) + args.client_container, qb["local_root"], route_local_path(sync), + container_user(client), + ) + run_existing_route_directory_check( + args.client_container, args.container_config ) for key in ("shared_token_file",): require_regular_secret(map_path(client, config[key]).source) diff --git a/src/archive_clients/daemon.py b/src/archive_clients/daemon.py index 0a6f2fd..6367132 100644 --- a/src/archive_clients/daemon.py +++ b/src/archive_clients/daemon.py @@ -697,7 +697,22 @@ class ArchiveClientDaemon: decode_message(str(row["update_json"]), control_pb2.RouteUpdate()) ) await outbound.put(encode(response)) - if attempt["state"] in {"ready", "failed"}: + if attempt["state"] == "ready": + # Route readiness is durable, but this process-local lookup is + # not. Reconfigure (which validates the existing Syncthing folder + # and recreates its local directory if necessary) before replaying + # the stored READY updates. This is essential after a daemon or + # mount restart: a previously ready route may otherwise point at a + # path that is no longer present, and a later source-stage command + # would fail with a bare ENOENT. + configured = await asyncio.to_thread( + self.routes.configure, + spec, + time.monotonic() + spec.setup_timeout_seconds, + ) + self._known_route_paths[spec.route_id] = configured.local_path + return + if attempt["state"] == "failed": return deadline = time.monotonic() + spec.setup_timeout_seconds diff --git a/tests/test_daemon.py b/tests/test_daemon.py index bd00917..408d2ee 100644 --- a/tests/test_daemon.py +++ b/tests/test_daemon.py @@ -280,6 +280,10 @@ class DaemonTransportTests(unittest.IsolatedAsyncioTestCase): for _ in range(3) ] self.assertEqual([item.sequence for item in resumed], [1, 2, 3]) + self.assertEqual(manager.configure.call_count, 2) + self.assertEqual( + restarted._route_path("route-1"), root / "routes/route-1" + ) async def test_slow_inventory_does_not_block_heartbeat(self): with tempfile.TemporaryDirectory() as directory: diff --git a/tests/test_deployment_preflight.py b/tests/test_deployment_preflight.py index f0e596c..55d3698 100644 --- a/tests/test_deployment_preflight.py +++ b/tests/test_deployment_preflight.py @@ -55,9 +55,11 @@ class DeploymentPreflightTests(unittest.TestCase): args=[], returncode=0, stdout="hard-link staging probe passed\n", stderr="" ) with mock.patch.object(preflight.subprocess, "run", return_value=completed) as run: - preflight.run_hardlink_probe("client", "/data/qb", "/data/qb/routes") + preflight.run_hardlink_probe( + "client", "/data/qb", "/data/qb/routes", "1001:1001" + ) args = run.call_args.args[0] - self.assertEqual(args[:5], ["docker", "exec", "client", "python", "-c"]) + self.assertEqual(args[:6], ["docker", "exec", "--user", "1001:1001", "client", "python"]) self.assertEqual(args[-2:], ["/data/qb", "/data/qb/routes"]) def test_hardlink_probe_failure_is_actionable(self): @@ -68,6 +70,14 @@ class DeploymentPreflightTests(unittest.TestCase): with self.assertRaisesRegex(preflight.CheckFailure, "hard-link staging probe failed"): preflight.run_hardlink_probe("client", "/data/qb", "/data/qb/routes") + def test_existing_route_directory_failure_is_actionable(self): + completed = __import__("subprocess").CompletedProcess( + args=[], returncode=1, stdout='{"missing":[{"folder_id":"route-1"}]}', stderr="" + ) + with mock.patch.object(preflight.subprocess, "run", return_value=completed): + with self.assertRaisesRegex(preflight.CheckFailure, "route directory is missing"): + preflight.run_existing_route_directory_check("client", "/config.toml") + if __name__ == "__main__": unittest.main()