diff --git a/deploy/production/README.md b/deploy/production/README.md new file mode 100644 index 0000000..e981ed0 --- /dev/null +++ b/deploy/production/README.md @@ -0,0 +1,30 @@ +# Production deployment + +These files are the non-secret, host-specific deployment manifests for the +initial x1/x2/lithium topology. + +- Install the x1 and x2 files as + `~/compose/ArchiveControl-cache/{compose.yaml,client.toml}`. +- Install the lithium files as + `~/compose/ArchiveControl-archive/{compose.yaml,client.toml}`. +- Create sibling `state`, `backups`, and `secrets` directories owned by the + configured container UID/GID. +- Secret files are never committed. Each `secrets` directory contains + `archive_control_token`, `qb_password`, and `syncthing_api_key`, each a + regular non-empty file with mode `0600`. + +The Syncthing mounts intentionally reproduce each instance's `/var/syncthing` +layout, including nested data binds. This lets route discovery and route +provisioning use one safe API-to-local path mapping without altering an +existing Syncthing configuration. + +Before starting a stack, validate it with: + +```sh +docker compose config +docker compose run --rm archive-client --check-config +``` + +The check is fail-fast and performs local permission, filesystem, sparse-file, +hard-link, and reflink probes. Normal startup additionally probes the local +qBittorrent and Syncthing APIs before registration. diff --git a/deploy/production/lithium/client.toml b/deploy/production/lithium/client.toml new file mode 100644 index 0000000..f01dcab --- /dev/null +++ b/deploy/production/lithium/client.toml @@ -0,0 +1,42 @@ +client_id = "lithium-archive" +display_name = "lithium archive" +role = "archive" +control_endpoint = "ws://bot.everdream.xyz:8766/archive_control" +shared_token_file = "/run/secrets/archive_control_token" +state_db = "/var/lib/archive-control/client.db" +backup_dir = "/var/backups/archive-control" + +[connection] +registration_timeout = "10s" +heartbeat_interval = "15s" +offline_timeout = "45s" +reconnect_initial = "1s" +reconnect_max = "60s" # Retry forever, capped at this delay. +reconnect_reset_after = "60s" +reconnect_jitter = true + +[jobs] +stall_after = "30m" # Status warning only; it does not fail the job. +verification_timeout = "30m" +poll_interval = "1s" +free_space_reserve_bytes = 1073741824 # Rechecked immediately before work. + +[backup] +interval = "6h" +recent = 12 +daily = 14 +weekly = 8 + +[qbittorrent] +endpoint = "http://qb:18081" +username = "${QB_USER}" +password_file = "/run/secrets/qb_password" +api_root = "/media/Data" +local_root = "/data/qb" + +[syncthing] +endpoint = "http://syncthing:8384" +api_key_file = "/run/secrets/syncthing_api_key" +api_root = "/var/syncthing" +local_root = "/data/sync" +advertised_addresses = ["dynamic"] diff --git a/deploy/production/lithium/compose.yaml b/deploy/production/lithium/compose.yaml new file mode 100644 index 0000000..93239b9 --- /dev/null +++ b/deploy/production/lithium/compose.yaml @@ -0,0 +1,26 @@ +name: archive-control-archive + +services: + archive-client: + image: sodium/archive-clients:v0.1.0 + user: "1000:1000" + restart: unless-stopped + command: ["--config", "/etc/archive-control/client.toml"] + environment: + QB_USER: admin + networks: + - archive-services + volumes: + - ./client.toml:/etc/archive-control/client.toml:ro + - ./secrets:/run/secrets:ro + - ./state:/var/lib/archive-control + - ./backups:/var/backups/archive-control + - /media/Data:/data/qb + - /home/cyz/Repositories/syncthing/st_home:/data/sync + - /home/cyz/Sync:/data/sync/Sync + - /home/cyz/cache_pool:/data/sync/cache_pool + +networks: + archive-services: + external: true + name: qb-frpc-v2ray_default diff --git a/deploy/production/x1/client.toml b/deploy/production/x1/client.toml new file mode 100644 index 0000000..0ffd995 --- /dev/null +++ b/deploy/production/x1/client.toml @@ -0,0 +1,42 @@ +client_id = "x1-cache" +display_name = "x1 cache" +role = "cache" +control_endpoint = "ws://bot.everdream.xyz:8766/archive_control" +shared_token_file = "/run/secrets/archive_control_token" +state_db = "/var/lib/archive-control/client.db" +backup_dir = "/var/backups/archive-control" + +[connection] +registration_timeout = "10s" +heartbeat_interval = "15s" +offline_timeout = "45s" +reconnect_initial = "1s" +reconnect_max = "60s" # Retry forever, capped at this delay. +reconnect_reset_after = "60s" +reconnect_jitter = true + +[jobs] +stall_after = "30m" # Status warning only; it does not fail the job. +verification_timeout = "30m" +poll_interval = "1s" +free_space_reserve_bytes = 1073741824 # Rechecked immediately before work. + +[backup] +interval = "6h" +recent = 12 +daily = 14 +weekly = 8 + +[qbittorrent] +endpoint = "http://127.0.0.1:8081" +username = "${QB_USER}" +password_file = "/run/secrets/qb_password" +api_root = "/downloads" +local_root = "/data/qb" + +[syncthing] +endpoint = "http://127.0.0.1:8384" +api_key_file = "/run/secrets/syncthing_api_key" +api_root = "/var/syncthing" +local_root = "/data/sync" +advertised_addresses = ["dynamic"] diff --git a/deploy/production/x1/compose.yaml b/deploy/production/x1/compose.yaml new file mode 100644 index 0000000..6ca27c3 --- /dev/null +++ b/deploy/production/x1/compose.yaml @@ -0,0 +1,19 @@ +name: archive-control-cache + +services: + archive-client: + image: sodium/archive-clients:v0.1.0 + user: "1001:1001" + restart: unless-stopped + network_mode: host + command: ["--config", "/etc/archive-control/client.toml"] + environment: + QB_USER: admin + volumes: + - ./client.toml:/etc/archive-control/client.toml:ro + - ./secrets:/run/secrets:ro + - ./state:/var/lib/archive-control + - ./backups:/var/backups/archive-control + - /home/ubuntu/Downloads:/data/qb + - /home/ubuntu/compose/syncthing/st_home:/data/sync + - /home/ubuntu/Downloads/Sync:/data/sync/DownloadsSync diff --git a/deploy/production/x2/client.toml b/deploy/production/x2/client.toml new file mode 100644 index 0000000..9f474e9 --- /dev/null +++ b/deploy/production/x2/client.toml @@ -0,0 +1,42 @@ +client_id = "x2-cache" +display_name = "x2 cache" +role = "cache" +control_endpoint = "ws://bot.everdream.xyz:8766/archive_control" +shared_token_file = "/run/secrets/archive_control_token" +state_db = "/var/lib/archive-control/client.db" +backup_dir = "/var/backups/archive-control" + +[connection] +registration_timeout = "10s" +heartbeat_interval = "15s" +offline_timeout = "45s" +reconnect_initial = "1s" +reconnect_max = "60s" # Retry forever, capped at this delay. +reconnect_reset_after = "60s" +reconnect_jitter = true + +[jobs] +stall_after = "30m" # Status warning only; it does not fail the job. +verification_timeout = "30m" +poll_interval = "1s" +free_space_reserve_bytes = 1073741824 # Rechecked immediately before work. + +[backup] +interval = "6h" +recent = 12 +daily = 14 +weekly = 8 + +[qbittorrent] +endpoint = "http://127.0.0.1:8081" +username = "${QB_USER}" +password_file = "/run/secrets/qb_password" +api_root = "/downloads" +local_root = "/data/qb" + +[syncthing] +endpoint = "http://127.0.0.1:8384" +api_key_file = "/run/secrets/syncthing_api_key" +api_root = "/var/syncthing" +local_root = "/data/sync" +advertised_addresses = ["dynamic"] diff --git a/deploy/production/x2/compose.yaml b/deploy/production/x2/compose.yaml new file mode 100644 index 0000000..167589c --- /dev/null +++ b/deploy/production/x2/compose.yaml @@ -0,0 +1,19 @@ +name: archive-control-cache + +services: + archive-client: + image: sodium/archive-clients:v0.1.0 + user: "1001:1001" + restart: unless-stopped + network_mode: host + command: ["--config", "/etc/archive-control/client.toml"] + environment: + QB_USER: admin + volumes: + - ./client.toml:/etc/archive-control/client.toml:ro + - ./secrets:/run/secrets:ro + - ./state:/var/lib/archive-control + - ./backups:/var/backups/archive-control + - /home/ubuntu/Downloads:/data/qb + - /home/ubuntu/compose/syncthing/st_home:/data/sync + - /home/ubuntu/Downloads/Sync:/data/sync/Downloads/Sync diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index d22db8b..614f29f 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -105,7 +105,7 @@ follow-up task. ### Actions 1. Implement qBittorrent cookie authentication and capability adapters for - supported 4.5–4.6 and 5.x APIs. + supported 4.4–4.6 and 5.x APIs. 2. Implement on-demand torrent summaries, scoped hash lookup, lazy complete content trees, v1/v2 identity validation, torrent export, stopped add, selection application, recheck monitoring, and entry-only deletion. diff --git a/docs/service-apis.md b/docs/service-apis.md index ab06086..c6b5367 100644 --- a/docs/service-apis.md +++ b/docs/service-apis.md @@ -6,7 +6,7 @@ shapes. ## qBittorrent Web API -The implementation targets supported qBittorrent 4.5–4.6 and 5.x releases and +The implementation targets supported qBittorrent 4.4–4.6 and 5.x releases and detects the application and Web API versions at startup. The authoritative references are the official [5.0 WebUI API](https://github.com/qbittorrent/qBittorrent/wiki/WebUI-API-%28qBittorrent-5.0%29) and [4.1-compatible WebUI API](https://github.com/qbittorrent/qBittorrent/wiki/WebUI-API-%28qBittorrent-4.1%29). @@ -83,6 +83,12 @@ The folder's native ID becomes `route_id`. Multi-device, send-only, receive-only, out-of-root, or ambiguous folders are reported as unsupported and left unchanged. +Syncthing may serialize a folder path relative to its home as `~/...`. The +client normalizes that notation beneath the configured API-visible sync root +before applying the API-to-local root mapping. Deployments must mirror +Syncthing's nested bind mounts into the client so the normalized API path and +the client filesystem path refer to the same bytes. + Provisioning uses idempotent device and folder configuration updates. Each client receives its peer device ID and optional advertised addresses (`dynamic` by default). It changes only the specific device/folder objects diff --git a/pyproject.toml b/pyproject.toml index 2ee37ee..3d99a2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "archive-clients" -version = "0.1.0.dev0" +version = "0.1.0" requires-python = ">=3.11" dependencies = ["protobuf==7.35.1", "websockets==16.0"] diff --git a/src/archive_clients/routes.py b/src/archive_clients/routes.py index 984eebc..825e088 100644 --- a/src/archive_clients/routes.py +++ b/src/archive_clients/routes.py @@ -32,8 +32,9 @@ def discover_routes( if not isinstance(api_path, str): continue try: - local_path = roots.api_to_local(api_path) - relative = PurePosixPath(api_path).relative_to(roots.api_root) + normalized_api_path = _normalize_api_path(api_path, roots) + local_path = roots.api_to_local(normalized_api_path) + relative = normalized_api_path.relative_to(roots.api_root) except (ConfigError, ValueError): continue if not _lexically_within(local_path, roots.local_root): @@ -93,3 +94,14 @@ def _lexically_within(path: Path, root: Path) -> bool: return True except ValueError: return False + + +def _normalize_api_path( + api_path: str, roots: RootMapping +) -> PurePosixPath: + """Resolve Syncthing's home-relative folder notation under api_root.""" + + candidate = PurePosixPath(api_path) + if candidate.parts and candidate.parts[0] == "~": + candidate = roots.api_root.joinpath(*candidate.parts[1:]) + return candidate diff --git a/src/archive_clients/services.py b/src/archive_clients/services.py index ed853c0..7f54552 100644 --- a/src/archive_clients/services.py +++ b/src/archive_clients/services.py @@ -159,7 +159,7 @@ def _supported_qb_version(version: str) -> bool: if match is None: return False major, minor = int(match.group(1)), int(match.group(2)) - return major == 5 or (major == 4 and minor in {5, 6}) + return major == 5 or (major == 4 and minor in {4, 5, 6}) def _text_get( diff --git a/tests/test_routes.py b/tests/test_routes.py index 7e86e07..4fa28be 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -40,6 +40,28 @@ class RouteDiscoveryTests(unittest.TestCase): self.assertEqual(routes[1].local_relative_path, "routes/b") self.assertFalse(routes[1].archive_control_created) + def test_home_relative_paths_are_normalized_under_api_root(self): + with tempfile.TemporaryDirectory() as directory: + roots = RootMapping(PurePosixPath("/var/syncthing"), Path(directory)) + routes = discover_routes( + {"folders": [{ + "id": "DownloadsSync", + "path": "~/DownloadsSync", + "type": "sendreceive", + "devices": [ + {"deviceID": "LOCAL"}, + {"deviceID": "ARCHIVE"}, + ], + }]}, + "LOCAL", + roots, + True, + ) + self.assertEqual(len(routes), 1) + self.assertEqual(routes[0].route_id, "DownloadsSync") + self.assertEqual(routes[0].local_relative_path, "DownloadsSync") + self.assertEqual(routes[0].state, route_pb2.ROUTE_STATE_DISCOVERED) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_services.py b/tests/test_services.py index e1699e8..84a9414 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -6,7 +6,11 @@ from pathlib import Path, PurePosixPath from unittest.mock import patch from archive_clients.config import ServiceConfig -from archive_clients.services import probe_qbittorrent, probe_syncthing +from archive_clients.services import ( + _supported_qb_version, + probe_qbittorrent, + probe_syncthing, +) from archive_control.v1 import common_pb2 @@ -39,6 +43,14 @@ class _Opener: class ServiceProbeTests(unittest.TestCase): + def test_supported_qbittorrent_versions_include_live_4_4_api(self): + for version in ("v4.4.5", "v4.5.5", "v4.6.7", "v5.2.3"): + with self.subTest(version=version): + self.assertTrue(_supported_qb_version(version)) + for version in ("v4.3.9", "v6.0.0", "invalid"): + with self.subTest(version=version): + self.assertFalse(_supported_qb_version(version)) + def test_qbittorrent_versions_are_normalized(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory)