release: prepare production client deployment

2026-07-23 15:35:07 +00:00
parent 7cd7a747f0
commit fbb7eb5d9c
14 changed files with 279 additions and 7 deletions
@@ -0,0 +1,30 @@
+# Production deployment
+
+These files are the non-secret, host-specific deployment manifests for the
+initial x1/x2/lithium topology.
+
+- Install the x1 and x2 files as
+  `~/compose/ArchiveControl-cache/{compose.yaml,client.toml}`.
+- Install the lithium files as
+  `~/compose/ArchiveControl-archive/{compose.yaml,client.toml}`.
+- Create sibling `state`, `backups`, and `secrets` directories owned by the
+  configured container UID/GID.
+- Secret files are never committed. Each `secrets` directory contains
+  `archive_control_token`, `qb_password`, and `syncthing_api_key`, each a
+  regular non-empty file with mode `0600`.
+
+The Syncthing mounts intentionally reproduce each instance's `/var/syncthing`
+layout, including nested data binds. This lets route discovery and route
+provisioning use one safe API-to-local path mapping without altering an
+existing Syncthing configuration.
+
+Before starting a stack, validate it with:
+
+```sh
+docker compose config
+docker compose run --rm archive-client --check-config
+```
+
+The check is fail-fast and performs local permission, filesystem, sparse-file,
+hard-link, and reflink probes. Normal startup additionally probes the local
+qBittorrent and Syncthing APIs before registration.
@@ -0,0 +1,42 @@
+client_id = "lithium-archive"
+display_name = "lithium archive"
+role = "archive"
+control_endpoint = "ws://bot.everdream.xyz:8766/archive_control"
+shared_token_file = "/run/secrets/archive_control_token"
+state_db = "/var/lib/archive-control/client.db"
+backup_dir = "/var/backups/archive-control"
+
+[connection]
+registration_timeout = "10s"
+heartbeat_interval = "15s"
+offline_timeout = "45s"
+reconnect_initial = "1s"
+reconnect_max = "60s" # Retry forever, capped at this delay.
+reconnect_reset_after = "60s"
+reconnect_jitter = true
+
+[jobs]
+stall_after = "30m" # Status warning only; it does not fail the job.
+verification_timeout = "30m"
+poll_interval = "1s"
+free_space_reserve_bytes = 1073741824 # Rechecked immediately before work.
+
+[backup]
+interval = "6h"
+recent = 12
+daily = 14
+weekly = 8
+
+[qbittorrent]
+endpoint = "http://qb:18081"
+username = "${QB_USER}"
+password_file = "/run/secrets/qb_password"
+api_root = "/media/Data"
+local_root = "/data/qb"
+
+[syncthing]
+endpoint = "http://syncthing:8384"
+api_key_file = "/run/secrets/syncthing_api_key"
+api_root = "/var/syncthing"
+local_root = "/data/sync"
+advertised_addresses = ["dynamic"]
@@ -0,0 +1,26 @@
+name: archive-control-archive
+
+services:
+  archive-client:
+    image: sodium/archive-clients:v0.1.0
+    user: "1000:1000"
+    restart: unless-stopped
+    command: ["--config", "/etc/archive-control/client.toml"]
+    environment:
+      QB_USER: admin
+    networks:
+      - archive-services
+    volumes:
+      - ./client.toml:/etc/archive-control/client.toml:ro
+      - ./secrets:/run/secrets:ro
+      - ./state:/var/lib/archive-control
+      - ./backups:/var/backups/archive-control
+      - /media/Data:/data/qb
+      - /home/cyz/Repositories/syncthing/st_home:/data/sync
+      - /home/cyz/Sync:/data/sync/Sync
+      - /home/cyz/cache_pool:/data/sync/cache_pool
+
+networks:
+  archive-services:
+    external: true
+    name: qb-frpc-v2ray_default
@@ -0,0 +1,42 @@
+client_id = "x1-cache"
+display_name = "x1 cache"
+role = "cache"
+control_endpoint = "ws://bot.everdream.xyz:8766/archive_control"
+shared_token_file = "/run/secrets/archive_control_token"
+state_db = "/var/lib/archive-control/client.db"
+backup_dir = "/var/backups/archive-control"
+
+[connection]
+registration_timeout = "10s"
+heartbeat_interval = "15s"
+offline_timeout = "45s"
+reconnect_initial = "1s"
+reconnect_max = "60s" # Retry forever, capped at this delay.
+reconnect_reset_after = "60s"
+reconnect_jitter = true
+
+[jobs]
+stall_after = "30m" # Status warning only; it does not fail the job.
+verification_timeout = "30m"
+poll_interval = "1s"
+free_space_reserve_bytes = 1073741824 # Rechecked immediately before work.
+
+[backup]
+interval = "6h"
+recent = 12
+daily = 14
+weekly = 8
+
+[qbittorrent]
+endpoint = "http://127.0.0.1:8081"
+username = "${QB_USER}"
+password_file = "/run/secrets/qb_password"
+api_root = "/downloads"
+local_root = "/data/qb"
+
+[syncthing]
+endpoint = "http://127.0.0.1:8384"
+api_key_file = "/run/secrets/syncthing_api_key"
+api_root = "/var/syncthing"
+local_root = "/data/sync"
+advertised_addresses = ["dynamic"]
@@ -0,0 +1,19 @@
+name: archive-control-cache
+
+services:
+  archive-client:
+    image: sodium/archive-clients:v0.1.0
+    user: "1001:1001"
+    restart: unless-stopped
+    network_mode: host
+    command: ["--config", "/etc/archive-control/client.toml"]
+    environment:
+      QB_USER: admin
+    volumes:
+      - ./client.toml:/etc/archive-control/client.toml:ro
+      - ./secrets:/run/secrets:ro
+      - ./state:/var/lib/archive-control
+      - ./backups:/var/backups/archive-control
+      - /home/ubuntu/Downloads:/data/qb
+      - /home/ubuntu/compose/syncthing/st_home:/data/sync
+      - /home/ubuntu/Downloads/Sync:/data/sync/DownloadsSync
@@ -0,0 +1,42 @@
+client_id = "x2-cache"
+display_name = "x2 cache"
+role = "cache"
+control_endpoint = "ws://bot.everdream.xyz:8766/archive_control"
+shared_token_file = "/run/secrets/archive_control_token"
+state_db = "/var/lib/archive-control/client.db"
+backup_dir = "/var/backups/archive-control"
+
+[connection]
+registration_timeout = "10s"
+heartbeat_interval = "15s"
+offline_timeout = "45s"
+reconnect_initial = "1s"
+reconnect_max = "60s" # Retry forever, capped at this delay.
+reconnect_reset_after = "60s"
+reconnect_jitter = true
+
+[jobs]
+stall_after = "30m" # Status warning only; it does not fail the job.
+verification_timeout = "30m"
+poll_interval = "1s"
+free_space_reserve_bytes = 1073741824 # Rechecked immediately before work.
+
+[backup]
+interval = "6h"
+recent = 12
+daily = 14
+weekly = 8
+
+[qbittorrent]
+endpoint = "http://127.0.0.1:8081"
+username = "${QB_USER}"
+password_file = "/run/secrets/qb_password"
+api_root = "/downloads"
+local_root = "/data/qb"
+
+[syncthing]
+endpoint = "http://127.0.0.1:8384"
+api_key_file = "/run/secrets/syncthing_api_key"
+api_root = "/var/syncthing"
+local_root = "/data/sync"
+advertised_addresses = ["dynamic"]
@@ -0,0 +1,19 @@
+name: archive-control-cache
+
+services:
+  archive-client:
+    image: sodium/archive-clients:v0.1.0
+    user: "1001:1001"
+    restart: unless-stopped
+    network_mode: host
+    command: ["--config", "/etc/archive-control/client.toml"]
+    environment:
+      QB_USER: admin
+    volumes:
+      - ./client.toml:/etc/archive-control/client.toml:ro
+      - ./secrets:/run/secrets:ro
+      - ./state:/var/lib/archive-control
+      - ./backups:/var/backups/archive-control
+      - /home/ubuntu/Downloads:/data/qb
+      - /home/ubuntu/compose/syncthing/st_home:/data/sync
+      - /home/ubuntu/Downloads/Sync:/data/sync/Downloads/Sync
@@ -105,7 +105,7 @@ follow-up task.
 ### Actions

 1. Implement qBittorrent cookie authentication and capability adapters for
-   supported 4.5–4.6 and 5.x APIs.
+   supported 4.4–4.6 and 5.x APIs.
 2. Implement on-demand torrent summaries, scoped hash lookup, lazy complete
   content trees, v1/v2 identity validation, torrent export, stopped add,
   selection application, recheck monitoring, and entry-only deletion.
@@ -6,7 +6,7 @@ shapes.

 ## qBittorrent Web API

-The implementation targets supported qBittorrent 4.5–4.6 and 5.x releases and
+The implementation targets supported qBittorrent 4.4–4.6 and 5.x releases and
 detects the application and Web API versions at startup. The authoritative
 references are the official [5.0 WebUI API](https://github.com/qbittorrent/qBittorrent/wiki/WebUI-API-%28qBittorrent-5.0%29)
 and [4.1-compatible WebUI API](https://github.com/qbittorrent/qBittorrent/wiki/WebUI-API-%28qBittorrent-4.1%29).
@@ -83,6 +83,12 @@ The folder's native ID becomes `route_id`. Multi-device, send-only,
 receive-only, out-of-root, or ambiguous folders are reported as unsupported and
 left unchanged.

+Syncthing may serialize a folder path relative to its home as `~/...`. The
+client normalizes that notation beneath the configured API-visible sync root
+before applying the API-to-local root mapping. Deployments must mirror
+Syncthing's nested bind mounts into the client so the normalized API path and
+the client filesystem path refer to the same bytes.
+
 Provisioning uses idempotent device and folder configuration updates. Each
 client receives its peer device ID and optional advertised addresses
 (`dynamic` by default). It changes only the specific device/folder objects
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

 [project]
 name = "archive-clients"
-version = "0.1.0.dev0"
+version = "0.1.0"
 requires-python = ">=3.11"
 dependencies = ["protobuf==7.35.1", "websockets==16.0"]

@@ -32,8 +32,9 @@ def discover_routes(
        if not isinstance(api_path, str):
            continue
        try:
-            local_path = roots.api_to_local(api_path)
-            relative = PurePosixPath(api_path).relative_to(roots.api_root)
+            normalized_api_path = _normalize_api_path(api_path, roots)
+            local_path = roots.api_to_local(normalized_api_path)
+            relative = normalized_api_path.relative_to(roots.api_root)
        except (ConfigError, ValueError):
            continue
        if not _lexically_within(local_path, roots.local_root):
@@ -93,3 +94,14 @@ def _lexically_within(path: Path, root: Path) -> bool:
        return True
    except ValueError:
        return False
+
+
+def _normalize_api_path(
+    api_path: str, roots: RootMapping
+) -> PurePosixPath:
+    """Resolve Syncthing's home-relative folder notation under api_root."""
+
+    candidate = PurePosixPath(api_path)
+    if candidate.parts and candidate.parts[0] == "~":
+        candidate = roots.api_root.joinpath(*candidate.parts[1:])
+    return candidate
@@ -159,7 +159,7 @@ def _supported_qb_version(version: str) -> bool:
    if match is None:
        return False
    major, minor = int(match.group(1)), int(match.group(2))
-    return major == 5 or (major == 4 and minor in {5, 6})
+    return major == 5 or (major == 4 and minor in {4, 5, 6})


 def _text_get(
@@ -40,6 +40,28 @@ class RouteDiscoveryTests(unittest.TestCase):
        self.assertEqual(routes[1].local_relative_path, "routes/b")
        self.assertFalse(routes[1].archive_control_created)

+    def test_home_relative_paths_are_normalized_under_api_root(self):
+        with tempfile.TemporaryDirectory() as directory:
+            roots = RootMapping(PurePosixPath("/var/syncthing"), Path(directory))
+            routes = discover_routes(
+                {"folders": [{
+                    "id": "DownloadsSync",
+                    "path": "~/DownloadsSync",
+                    "type": "sendreceive",
+                    "devices": [
+                        {"deviceID": "LOCAL"},
+                        {"deviceID": "ARCHIVE"},
+                    ],
+                }]},
+                "LOCAL",
+                roots,
+                True,
+            )
+        self.assertEqual(len(routes), 1)
+        self.assertEqual(routes[0].route_id, "DownloadsSync")
+        self.assertEqual(routes[0].local_relative_path, "DownloadsSync")
+        self.assertEqual(routes[0].state, route_pb2.ROUTE_STATE_DISCOVERED)
+

 if __name__ == "__main__":
    unittest.main()
@@ -6,7 +6,11 @@ from pathlib import Path, PurePosixPath
 from unittest.mock import patch

 from archive_clients.config import ServiceConfig
-from archive_clients.services import probe_qbittorrent, probe_syncthing
+from archive_clients.services import (
+    _supported_qb_version,
+    probe_qbittorrent,
+    probe_syncthing,
+)
 from archive_control.v1 import common_pb2


@@ -39,6 +43,14 @@ class _Opener:


 class ServiceProbeTests(unittest.TestCase):
+    def test_supported_qbittorrent_versions_include_live_4_4_api(self):
+        for version in ("v4.4.5", "v4.5.5", "v4.6.7", "v5.2.3"):
+            with self.subTest(version=version):
+                self.assertTrue(_supported_qb_version(version))
+        for version in ("v4.3.9", "v6.0.0", "invalid"):
+            with self.subTest(version=version):
+                self.assertFalse(_supported_qb_version(version))
+
    def test_qbittorrent_versions_are_normalized(self):
        with tempfile.TemporaryDirectory() as directory:
            root = Path(directory)