feat: add archive client foundation

2026-07-22 14:55:02 +00:00
commit c11a7b5b5b
39 changed files with 2985 additions and 0 deletions
@@ -0,0 +1,101 @@
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+from archive_clients.config import ClientConfig, ConfigError, RootMapping
+
+
+class ConfigTests(unittest.TestCase):
+    def test_strict_config_mode_override_secrets_and_mapping(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            for name in ("token", "qb-password", "syncthing-key"):
+                path = root / name
+                path.write_text(name, encoding="utf-8")
+                os.chmod(path, 0o600)
+            (root / "qb").mkdir()
+            (root / "sync").mkdir()
+            config_path = root / "client.toml"
+            config_path.write_text(
+                _config(root, role="cache"), encoding="utf-8"
+            )
+            config = ClientConfig.load(config_path, "archive")
+            self.assertEqual(config.role, "archive")
+            self.assertEqual(config.read_shared_token(), "token")
+            self.assertEqual(
+                config.qbittorrent.roots.api_to_local("/downloads/a/b"),
+                root / "qb/a/b",
+            )
+            self.assertEqual(config.jobs.stall_after, 30 * 60)
+            self.assertEqual(
+                config.syncthing.advertised_addresses, ("dynamic",)
+            )
+
+    def test_unknown_key_and_unsafe_secret_fail(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            for name in ("token", "qb-password", "syncthing-key"):
+                path = root / name
+                path.write_text(name, encoding="utf-8")
+                os.chmod(path, 0o600)
+            config_path = root / "client.toml"
+            invalid = _config(root).replace(
+                "[qbittorrent]", "typo = true\n[qbittorrent]"
+            )
+            config_path.write_text(invalid, encoding="utf-8")
+            with self.assertRaisesRegex(ConfigError, "unknown root"):
+                ClientConfig.load(config_path)
+            os.chmod(root / "token", 0o640)
+            config_path.write_text(_config(root), encoding="utf-8")
+            with self.assertRaisesRegex(ConfigError, "permissions"):
+                ClientConfig.load(config_path).read_shared_token()
+
+    def test_mapping_rejects_escape(self):
+        mapping = RootMapping(Path("/api"), Path("/local"))
+        with self.assertRaises(ConfigError):
+            mapping.api_to_local("/elsewhere/file")
+
+    def test_endpoint_scheme_and_job_keys_are_strict(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            invalid_endpoint = _config(root).replace(
+                'control_endpoint = "ws://control/archive_control"',
+                'control_endpoint = "http://control/archive_control"',
+            )
+            path = root / "client.toml"
+            path.write_text(invalid_endpoint, encoding="utf-8")
+            with self.assertRaisesRegex(ConfigError, "ws/wss"):
+                ClientConfig.load(path)
+            path.write_text(
+                _config(root) + "\n[jobs]\nunknown = true\n", encoding="utf-8"
+            )
+            with self.assertRaisesRegex(ConfigError, "unknown jobs"):
+                ClientConfig.load(path)
+
+
+def _config(root: Path, role: str = "cache") -> str:
+    return f'''client_id = "cache-1"
+display_name = "Cache 1"
+role = "{role}"
+control_endpoint = "ws://control/archive_control"
+shared_token_file = "{root / 'token'}"
+state_db = "{root / 'state.db'}"
+backup_dir = "{root / 'backups'}"
+[qbittorrent]
+endpoint = "http://qb"
+username = "admin"
+password_file = "{root / 'qb-password'}"
+api_root = "/downloads"
+local_root = "{root / 'qb'}"
+[syncthing]
+endpoint = "http://syncthing"
+api_key_file = "{root / 'syncthing-key'}"
+api_root = "/sync"
+local_root = "{root / 'sync'}"
+advertised_addresses = ["dynamic"]
+'''
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,112 @@
+import asyncio
+import os
+import tempfile
+import unittest
+from pathlib import Path, PurePosixPath
+from uuid import uuid4
+
+from websockets.asyncio.server import serve
+
+from archive_clients.config import ClientConfig, ConnectionConfig, ServiceConfig
+from archive_clients.daemon import ArchiveClientDaemon
+from archive_clients.probes import FilesystemProbe
+from archive_clients.protocol import decode, encode, new_envelope
+from archive_control.v1 import client_pb2, common_pb2, control_pb2
+
+
+class DaemonTransportTests(unittest.IsolatedAsyncioTestCase):
+    async def test_registration_heartbeat_and_duplicate_command(self):
+        observed = {}
+
+        async def control(websocket):
+            registration = decode(await websocket.recv())
+            observed["token"] = registration.register_request.shared_token
+            observed["root_names"] = [
+                item.root_name
+                for item in registration.register_request.capabilities.filesystems
+            ]
+            observed["addresses"] = list(
+                registration.register_request.capabilities
+                .syncthing_advertised_addresses
+            )
+            response = new_envelope()
+            response.correlation_id = registration.message_id
+            response.register_response.status = client_pb2.REGISTRATION_STATUS_ACCEPTED
+            response.register_response.negotiated_version.major = 1
+            await websocket.send(encode(response))
+            heartbeat = new_envelope()
+            heartbeat.heartbeat.sequence = 7
+            await websocket.send(encode(heartbeat))
+            observed["heartbeat"] = decode(await websocket.recv()).heartbeat_ack.sequence
+            command_id = str(uuid4())
+            command = new_envelope()
+            command.command.command_id = command_id
+            command.command.created_at.CopyFrom(command.sent_at)
+            command.command.request_job_snapshot.job_ids.append(str(uuid4()))
+            await websocket.send(encode(command))
+            observed["first"] = decode(await websocket.recv()).command_ack.status
+            observed["snapshot"] = (
+                decode(await websocket.recv()).WhichOneof("payload")
+            )
+            duplicate = new_envelope()
+            duplicate.command.CopyFrom(command.command)
+            await websocket.send(encode(duplicate))
+            observed["second"] = decode(await websocket.recv()).command_ack.status
+            unsupported = new_envelope()
+            unsupported.command.command_id = str(uuid4())
+            unsupported.command.created_at.CopyFrom(unsupported.sent_at)
+            unsupported.command.assign_job.SetInParent()
+            await websocket.send(encode(unsupported))
+            rejected = decode(await websocket.recv()).command_ack
+            observed["rejected"] = (rejected.status, rejected.error.code)
+            unsupported_duplicate = new_envelope()
+            unsupported_duplicate.command.CopyFrom(unsupported.command)
+            await websocket.send(encode(unsupported_duplicate))
+            observed["rejected_duplicate"] = (
+                decode(await websocket.recv()).command_ack.status
+            )
+
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            token = root / "token"
+            token.write_text("shared-secret", encoding="utf-8")
+            os.chmod(token, 0o600)
+            async with serve(control, "127.0.0.1", 0, ping_interval=None) as server:
+                port = server.sockets[0].getsockname()[1]
+                service = ServiceConfig(
+                    "http://local", PurePosixPath("/api"), root,
+                    advertised_addresses=("dynamic",),
+                )
+                config = ClientConfig(
+                    "cache-1", "Cache 1", "cache",
+                    f"ws://127.0.0.1:{port}", token,
+                    root / "state.db", root / "backups", service, service,
+                    ConnectionConfig(registration_timeout=2),
+                )
+                probe = FilesystemProbe(root, True, True, True, True)
+                daemon = ArchiveClientDaemon(config, [probe, probe])
+                await asyncio.to_thread(daemon.store.initialize)
+                await daemon._connection()
+
+        self.assertEqual(observed["token"], "shared-secret")
+        self.assertEqual(observed["root_names"], ["qbittorrent", "syncthing"])
+        self.assertEqual(observed["addresses"], ["dynamic"])
+        self.assertEqual(observed["heartbeat"], 7)
+        self.assertEqual(observed["first"], control_pb2.COMMAND_ACK_STATUS_ACCEPTED)
+        self.assertEqual(observed["second"], control_pb2.COMMAND_ACK_STATUS_DUPLICATE)
+        self.assertEqual(observed["snapshot"], "client_state_snapshot")
+        self.assertEqual(
+            observed["rejected"],
+            (
+                control_pb2.COMMAND_ACK_STATUS_REJECTED,
+                common_pb2.ERROR_CODE_UNSUPPORTED,
+            ),
+        )
+        self.assertEqual(
+            observed["rejected_duplicate"],
+            control_pb2.COMMAND_ACK_STATUS_REJECTED,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,17 @@
+import unittest
+from pathlib import Path
+
+from archive_clients import PROTO_COMMIT
+
+
+class GeneratedContractTests(unittest.TestCase):
+    def test_all_generated_files_record_proto_commit(self):
+        files = list(Path("src/archive_control/v1").glob("*_pb2.py"))
+        files += list(Path("src/archive_control/v1").glob("*_pb2.pyi"))
+        self.assertEqual(len(files), 18)
+        for path in files:
+            self.assertIn(PROTO_COMMIT, path.read_text(encoding="utf-8")[:256])
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,33 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from archive_clients.probes import probe_root
+from archive_clients.protocol import ProtocolError, decode, encode, new_envelope
+
+
+class ProtocolAndProbeTests(unittest.TestCase):
+    def test_protocol_round_trip_and_duplicate_key_rejection(self):
+        envelope = new_envelope()
+        envelope.heartbeat.sequence = 1
+        self.assertEqual(decode(encode(envelope)).heartbeat.sequence, 1)
+        raw = json.loads(encode(envelope))
+        duplicate = json.dumps(raw).replace("{", '{"messageId":"duplicate",', 1)
+        with self.assertRaises(ProtocolError):
+            decode(duplicate)
+        raw["messageId"] = "not-a-uuid"
+        with self.assertRaisesRegex(ProtocolError, "canonical UUID"):
+            decode(json.dumps(raw))
+
+    def test_filesystem_probe_cleans_up(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            result = probe_root(root)
+            self.assertTrue(result.readable)
+            self.assertTrue(result.writable)
+            self.assertEqual(list(root.iterdir()), [])
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,28 @@
+import tempfile
+import unittest
+from pathlib import Path
+from uuid import uuid4
+
+from archive_clients.state import ClientStore, CommandConflict
+
+
+class ClientStoreTests(unittest.TestCase):
+    def test_command_acceptance_is_durable_and_content_addressed(self):
+        with tempfile.TemporaryDirectory() as directory:
+            database = Path(directory) / "state.db"
+            store = ClientStore(database)
+            store.initialize()
+            command_id = str(uuid4())
+            first = store.accept_command(command_id, '{"b":2,"a":1}', '{"ok":true}')
+            duplicate = ClientStore(database).accept_command(
+                command_id, '{"a":1,"b":2}', '{"ok":false}'
+            )
+            self.assertFalse(first.duplicate)
+            self.assertTrue(duplicate.duplicate)
+            self.assertEqual(duplicate.acknowledgement_json, '{"ok":true}')
+            with self.assertRaises(CommandConflict):
+                store.accept_command(command_id, '{"a":2}', '{"ok":true}')
+
+
+if __name__ == "__main__":
+    unittest.main()