fix: recover durable route paths after restart

docs/deployment-preflight.md

@@ -89,6 +89,10 @@ python3 scripts/preflight-deployment.py ... \
 - A real `link(2)` operation between a unique zero-byte file in the qB root
   and one in the future automatic-route root. The probe verifies that both
   names refer to the same inode and removes them unconditionally.
+- Every existing `archive-control:*` Syncthing folder beneath `routes/` still
+  has its local directory. This catches a route-root bind-mount migration that
+  would otherwise hide an already configured folder and later fail a job with
+  `No such file or directory`.
 - The token, qB password, and Syncthing API-key files are non-empty regular
   files with no group/world permissions.
 - The client image can read its configuration and reports usable permissions,
@@ -102,6 +106,12 @@ root and add a `local_path_overrides` mapping for that exact API path. This
 prevents automatic route folders being created on a small configuration
 filesystem while the client expects to hardlink from the qB data mount.
 
+When converting an existing node, stop its client and Syncthing containers,
+move each existing `routes/<route-id>` directory from the old Syncthing config
+tree into the new qB-backed route-root directory, then recreate Syncthing and
+run this preflight. Do not discard existing route directories: they can contain
+route handshake state or an in-progress transfer namespace.
+
 ## Failure handling
 
 Treat a nonzero exit status as a deployment blocker. Correct the compose

scripts/preflight-deployment.py

@@ -66,6 +66,11 @@ def map_path(container: dict[str, Any], path: str) -> Mapping:
     return max(matches, key=lambda item: item[0])[1]
 
 
+def container_user(container: dict[str, Any]) -> str | None:
+    value = container.get("Config", {}).get("User")
+    return value if isinstance(value, str) and value else None
+
+
 def require_same_path(label: str, left: Mapping, right: Mapping) -> None:
     if left.source.resolve(strict=False) != right.source.resolve(strict=False):
         raise CheckFailure(
@@ -169,7 +174,9 @@ raise SystemExit(0 if all(p.state == 1 for p in probes) else 1)'''
     print(result.stdout.strip())
 
 
-def run_hardlink_probe(container: str, qb_root: str, route_root: str) -> None:
+def run_hardlink_probe(
+    container: str, qb_root: str, route_root: str, user: str | None = None,
+) -> None:
     """Prove that the daemon can hard-link from qB data into automatic routes."""
 
     program = r'''import os,sys,tempfile
@@ -196,8 +203,12 @@ finally:
             except FileNotFoundError:
                 pass
 '''
+    command = ["docker", "exec"]
+    if user:
+        command.extend(["--user", user])
+    command.extend([container, "python", "-c", program, qb_root, route_root])
     result = subprocess.run(
-        ["docker", "exec", container, "python", "-c", program, qb_root, route_root],
+        command,
         check=False, text=True, capture_output=True,
     )
     if result.returncode:
@@ -206,6 +217,51 @@ finally:
     print(result.stdout.strip())
 
 
+def run_existing_route_directory_check(container: str, config: str) -> None:
+    """Reject a mount migration that hides an already configured route folder."""
+
+    program = r'''import json,os,sys
+from pathlib import Path,PurePosixPath
+from archive_clients.config import ClientConfig
+from archive_clients.syncthing import SyncthingHttp
+config=ClientConfig.load(Path(sys.argv[1]))
+transport=SyncthingHttp(config.syncthing)
+route_root=config.syncthing.api_root / "routes"
+missing=[]
+checked=[]
+for folder in transport.get_json("/rest/config").get("folders",[]):
+    if not isinstance(folder,dict) or not str(folder.get("label","")).startswith("archive-control:"):
+        continue
+    raw=folder.get("path")
+    if not isinstance(raw,str) or not raw:
+        continue
+    api=PurePosixPath(raw)
+    if api.parts and api.parts[0] == "~":
+        api=config.syncthing.api_root.joinpath(*api.parts[1:])
+    try:
+        api.relative_to(route_root)
+    except ValueError:
+        continue
+    local=config.syncthing.roots.api_to_local(api.as_posix())
+    checked.append(str(local))
+    if not local.is_dir():
+        missing.append({"folder_id":folder.get("id",""),"path":str(local)})
+print(json.dumps({"checked":checked,"missing":missing},sort_keys=True))
+raise SystemExit(1 if missing else 0)
+'''
+    result = subprocess.run(
+        ["docker", "exec", container, "python", "-c", program, config],
+        check=False, text=True, capture_output=True,
+    )
+    if result.returncode:
+        detail = result.stderr.strip() or result.stdout.strip() or "failed"
+        raise CheckFailure(
+            "an existing Archive Control Syncthing route directory is missing; "
+            f"complete the route-directory migration before starting jobs: {detail}"
+        )
+    print(result.stdout.strip())
+
+
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(
         description="Read-only Archive Control Docker deployment preflight"
@@ -248,7 +304,11 @@ def main(argv: list[str] | None = None) -> int:
                 "mounts; hard-link staging would be unavailable"
             )
         run_hardlink_probe(
-            args.client_container, qb["local_root"], route_local_path(sync)
+            args.client_container, qb["local_root"], route_local_path(sync),
+            container_user(client),
+        )
+        run_existing_route_directory_check(
+            args.client_container, args.container_config
         )
         for key in ("shared_token_file",):
             require_regular_secret(map_path(client, config[key]).source)

src/archive_clients/daemon.py

@@ -697,7 +697,22 @@ class ArchiveClientDaemon:
                 decode_message(str(row["update_json"]), control_pb2.RouteUpdate())
             )
             await outbound.put(encode(response))
-        if attempt["state"] in {"ready", "failed"}:
+        if attempt["state"] == "ready":
+            # Route readiness is durable, but this process-local lookup is
+            # not. Reconfigure (which validates the existing Syncthing folder
+            # and recreates its local directory if necessary) before replaying
+            # the stored READY updates. This is essential after a daemon or
+            # mount restart: a previously ready route may otherwise point at a
+            # path that is no longer present, and a later source-stage command
+            # would fail with a bare ENOENT.
+            configured = await asyncio.to_thread(
+                self.routes.configure,
+                spec,
+                time.monotonic() + spec.setup_timeout_seconds,
+            )
+            self._known_route_paths[spec.route_id] = configured.local_path
+            return
+        if attempt["state"] == "failed":
             return
 
         deadline = time.monotonic() + spec.setup_timeout_seconds

tests/test_daemon.py

@@ -280,6 +280,10 @@ class DaemonTransportTests(unittest.IsolatedAsyncioTestCase):
                 for _ in range(3)
             ]
             self.assertEqual([item.sequence for item in resumed], [1, 2, 3])
+            self.assertEqual(manager.configure.call_count, 2)
+            self.assertEqual(
+                restarted._route_path("route-1"), root / "routes/route-1"
+            )
 
     async def test_slow_inventory_does_not_block_heartbeat(self):
         with tempfile.TemporaryDirectory() as directory:

tests/test_deployment_preflight.py

@@ -55,9 +55,11 @@ class DeploymentPreflightTests(unittest.TestCase):
             args=[], returncode=0, stdout="hard-link staging probe passed\n", stderr=""
         )
         with mock.patch.object(preflight.subprocess, "run", return_value=completed) as run:
-            preflight.run_hardlink_probe("client", "/data/qb", "/data/qb/routes")
+            preflight.run_hardlink_probe(
+                "client", "/data/qb", "/data/qb/routes", "1001:1001"
+            )
         args = run.call_args.args[0]
-        self.assertEqual(args[:5], ["docker", "exec", "client", "python", "-c"])
+        self.assertEqual(args[:6], ["docker", "exec", "--user", "1001:1001", "client", "python"])
         self.assertEqual(args[-2:], ["/data/qb", "/data/qb/routes"])
 
     def test_hardlink_probe_failure_is_actionable(self):
@@ -68,6 +70,14 @@ class DeploymentPreflightTests(unittest.TestCase):
             with self.assertRaisesRegex(preflight.CheckFailure, "hard-link staging probe failed"):
                 preflight.run_hardlink_probe("client", "/data/qb", "/data/qb/routes")
 
+    def test_existing_route_directory_failure_is_actionable(self):
+        completed = __import__("subprocess").CompletedProcess(
+            args=[], returncode=1, stdout='{"missing":[{"folder_id":"route-1"}]}', stderr=""
+        )
+        with mock.patch.object(preflight.subprocess, "run", return_value=completed):
+            with self.assertRaisesRegex(preflight.CheckFailure, "route directory is missing"):
+                preflight.run_existing_route_directory_check("client", "/config.toml")
+
 
 if __name__ == "__main__":
     unittest.main()