cabbage/archive-clients
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete client runtime foundation

Browse Source
This commit is contained in:
cabbage
2026-07-22 16:25:38 +00:00
parent c11a7b5b5b
commit 1219117403
20 changed files with 1201 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_backup.py
+57
View File
@@ -0,0 +1,57 @@
import os
import tempfile
import unittest
from pathlib import Path
from archive_clients.backup import BackupError, SQLiteBackupManager
from archive_clients.config import BackupConfig
from archive_clients.locking import DatabaseLease, DatabaseLockedError
from archive_clients.state import ClientStore
class BackupTests(unittest.TestCase):
def test_create_verify_detect_corruption_and_restore(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
database = root / "state" / "client.db"
backups = root / "backups"
store = ClientStore(database)
store.initialize()
manager = SQLiteBackupManager(database, backups, BackupConfig())
record = manager.create("test")
self.assertEqual(manager.verify(record.database).sha256, record.sha256)
database.write_bytes(b"broken")
preserved = manager.restore(record.database)
self.assertIsNotNone(preserved)
self.assertEqual(ClientStore(database).list_active_job_cursors(), [])
with record.database.open("ab") as target:
target.write(b"corruption")
with self.assertRaisesRegex(BackupError, "checksum"):
manager.verify(record.database)
def test_restore_refuses_a_live_database_lease(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
database = root / "client.db"
ClientStore(database).initialize()
manager = SQLiteBackupManager(database, root / "backups", BackupConfig())
record = manager.create("test")
with DatabaseLease(database):
with self.assertRaises(DatabaseLockedError):
manager.restore(record.database)
def test_backup_database_has_private_permissions(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
database = root / "client.db"
ClientStore(database).initialize()
record = SQLiteBackupManager(
database, root / "backups", BackupConfig()
).create("test")
self.assertEqual(os.stat(record.database).st_mode & 0o777, 0o600)
if __name__ == "__main__":
unittest.main()
tests/test_config.py
+12
View File
@@ -73,6 +73,18 @@ class ConfigTests(unittest.TestCase):
with self.assertRaisesRegex(ConfigError, "unknown jobs"):
ClientConfig.load(path)
def test_state_and_backups_cannot_live_under_data_roots(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
path = root / "client.toml"
nested = _config(root).replace(
f'state_db = "{root / "state.db"}"',
f'state_db = "{root / "qb/state.db"}"',
)
path.write_text(nested, encoding="utf-8")
with self.assertRaisesRegex(ConfigError, "outside data roots"):
ClientConfig.load(path)
def _config(root: Path, role: str = "cache") -> str:
return f'''client_id = "cache-1"
tests/test_daemon.py
+47 -8
View File
@@ -2,6 +2,7 @@ import asyncio
import os
import tempfile
import unittest
from datetime import datetime, timezone
from pathlib import Path, PurePosixPath
from uuid import uuid4
@@ -10,13 +11,15 @@ from websockets.asyncio.server import serve
from archive_clients.config import ClientConfig, ConnectionConfig, ServiceConfig
from archive_clients.daemon import ArchiveClientDaemon
from archive_clients.probes import FilesystemProbe
from archive_clients.protocol import decode, encode, new_envelope
from archive_control.v1 import client_pb2, common_pb2, control_pb2
from archive_clients.services import ServiceProbe
from archive_clients.protocol import decode, encode, encode_message, new_envelope
from archive_control.v1 import client_pb2, common_pb2, control_pb2, job_pb2
class DaemonTransportTests(unittest.IsolatedAsyncioTestCase):
async def test_registration_heartbeat_and_duplicate_command(self):
observed = {}
job_id = str(uuid4())
async def control(websocket):
registration = decode(await websocket.recv())
@@ -29,6 +32,13 @@ class DaemonTransportTests(unittest.IsolatedAsyncioTestCase):
registration.register_request.capabilities
.syncthing_advertised_addresses
)
observed["device_id"] = (
registration.register_request.capabilities.syncthing_device_id
)
observed["services"] = [
item.service
for item in registration.register_request.capabilities.services
]
response = new_envelope()
response.correlation_id = registration.message_id
response.register_response.status = client_pb2.REGISTRATION_STATUS_ACCEPTED
@@ -42,16 +52,21 @@ class DaemonTransportTests(unittest.IsolatedAsyncioTestCase):
command = new_envelope()
command.command.command_id = command_id
command.command.created_at.CopyFrom(command.sent_at)
command.command.request_job_snapshot.job_ids.append(str(uuid4()))
command.command.request_job_snapshot.job_ids.append(job_id)
await websocket.send(encode(command))
observed["first"] = decode(await websocket.recv()).command_ack.status
observed["snapshot"] = (
decode(await websocket.recv()).WhichOneof("payload")
snapshot = decode(await websocket.recv())
observed["snapshot"] = snapshot.WhichOneof("payload")
observed["snapshot_job_id"] = (
snapshot.job_snapshot.job.definition.job_id
)
duplicate = new_envelope()
duplicate.command.CopyFrom(command.command)
await websocket.send(encode(duplicate))
observed["second"] = decode(await websocket.recv()).command_ack.status
observed["duplicate_snapshot"] = (
decode(await websocket.recv()).WhichOneof("payload")
)
unsupported = new_envelope()
unsupported.command.command_id = str(uuid4())
unsupported.command.created_at.CopyFrom(unsupported.sent_at)
@@ -83,18 +98,42 @@ class DaemonTransportTests(unittest.IsolatedAsyncioTestCase):
root / "state.db", root / "backups", service, service,
ConnectionConfig(registration_timeout=2),
)
probe = FilesystemProbe(root, True, True, True, True)
daemon = ArchiveClientDaemon(config, [probe, probe])
probe = FilesystemProbe(root, True, True, True, True, True)
service_probe = ServiceProbe(
"syncthing", common_pb2.HEALTH_STATE_HEALTHY,
datetime.now(timezone.utc), version="v2", device_id="DEVICE",
)
daemon = ArchiveClientDaemon(
config, [probe, probe], [service_probe]
)
await asyncio.to_thread(daemon.store.initialize)
definition = job_pb2.JobDefinition(
job_id=job_id,
operation=job_pb2.JOB_OPERATION_ARCHIVE,
)
definition.created_at.GetCurrentTime()
await asyncio.to_thread(
daemon.store.save_job,
job_id,
encode_message(definition),
"JOB_STATE_WAITING",
2,
3,
False,
)
await daemon._connection()
self.assertEqual(observed["token"], "shared-secret")
self.assertEqual(observed["root_names"], ["qbittorrent", "syncthing"])
self.assertEqual(observed["addresses"], ["dynamic"])
self.assertEqual(observed["device_id"], "DEVICE")
self.assertEqual(observed["services"], ["syncthing"])
self.assertEqual(observed["heartbeat"], 7)
self.assertEqual(observed["first"], control_pb2.COMMAND_ACK_STATUS_ACCEPTED)
self.assertEqual(observed["second"], control_pb2.COMMAND_ACK_STATUS_DUPLICATE)
self.assertEqual(observed["snapshot"], "client_state_snapshot")
self.assertEqual(observed["snapshot"], "job_snapshot")
self.assertEqual(observed["snapshot_job_id"], job_id)
self.assertEqual(observed["duplicate_snapshot"], "job_snapshot")
self.assertEqual(
observed["rejected"],
(
tests/test_logging.py
+22
View File
@@ -0,0 +1,22 @@
import json
import logging
import unittest
from archive_clients.logging_config import RedactingJsonFormatter
class LoggingTests(unittest.TestCase):
def test_secret_is_redacted_from_message_and_fields(self):
formatter = RedactingJsonFormatter(["shared-secret"])
record = logging.LogRecord(
"test", logging.INFO, __file__, 1,
"token=%s", ("shared-secret",), None,
)
record.error_type = "shared-secret"
rendered = formatter.format(record)
self.assertNotIn("shared-secret", rendered)
self.assertEqual(json.loads(rendered)["error_type"], "[REDACTED]")
if __name__ == "__main__":
unittest.main()
tests/test_protocol_and_probes.py
+4 -1
View File
@@ -3,7 +3,7 @@ import tempfile
import unittest
from pathlib import Path
from archive_clients.probes import probe_root
from archive_clients.probes import probe_root, probe_writable_directory
from archive_clients.protocol import ProtocolError, decode, encode, new_envelope
@@ -26,6 +26,9 @@ class ProtocolAndProbeTests(unittest.TestCase):
result = probe_root(root)
self.assertTrue(result.readable)
self.assertTrue(result.writable)
self.assertIsInstance(result.reflink, bool)
self.assertEqual(list(root.iterdir()), [])
probe_writable_directory(root)
self.assertEqual(list(root.iterdir()), [])
tests/test_services.py
+92
View File
@@ -0,0 +1,92 @@
import io
import os
import tempfile
import unittest
from pathlib import Path, PurePosixPath
from unittest.mock import patch
from archive_clients.config import ServiceConfig
from archive_clients.services import probe_qbittorrent, probe_syncthing
from archive_control.v1 import common_pb2
class _Response:
def __init__(self, value: str):
self._source = io.BytesIO(value.encode("utf-8"))
def read(self, size: int = -1) -> bytes:
return self._source.read(size)
def __enter__(self):
return self
def __exit__(self, *_):
return None
class _Opener:
def __init__(self, responses: list[str]):
self.responses = iter(responses)
self.requests = []
def open(self, call, timeout):
self.requests.append((call, timeout))
return _Response(next(self.responses))
class ServiceProbeTests(unittest.TestCase):
def test_qbittorrent_versions_are_normalized(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
password = self._secret(root, "password", "private")
config = ServiceConfig(
"http://qb:8080", PurePosixPath("/downloads"), root,
username="admin", password_file=password,
)
opener = _Opener([
"Ok.", "v5.0.4", "2.11.4", '{"libtorrent":"2.0.11"}',
])
with patch(
"archive_clients.services.request.build_opener",
return_value=opener,
):
result = probe_qbittorrent(config)
self.assertEqual(result.state, common_pb2.HEALTH_STATE_HEALTHY)
self.assertEqual(result.version, "v5.0.4")
self.assertEqual(result.api_version, "2.11.4")
self.assertEqual(result.libtorrent_version, "2.0.11")
self.assertIn(b"password=private", opener.requests[0][0].data)
def test_syncthing_identity_is_normalized(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
api_key = self._secret(root, "api-key", "private")
config = ServiceConfig(
"http://syncthing:8384", PurePosixPath("/sync"), root,
api_key_file=api_key,
)
opener = _Opener([
'{"version":"v2.0.1","longVersion":"syncthing v2.0.1"}',
'{"myID":"DEVICE-ID"}',
])
with patch(
"archive_clients.services.request.build_opener",
return_value=opener,
):
result = probe_syncthing(config)
self.assertEqual(result.state, common_pb2.HEALTH_STATE_HEALTHY)
self.assertEqual(result.device_id, "DEVICE-ID")
self.assertEqual(
opener.requests[0][0].headers["X-api-key"], "private"
)
@staticmethod
def _secret(root: Path, name: str, value: str) -> Path:
path = root / name
path.write_text(value, encoding="utf-8")
os.chmod(path, 0o600)
return path
if __name__ == "__main__":
unittest.main()
tests/test_state.py
+29 -1
View File
@@ -1,9 +1,10 @@
import tempfile
import unittest
import os
from pathlib import Path
from uuid import uuid4
from archive_clients.state import ClientStore, CommandConflict
from archive_clients.state import ClientStore, CommandConflict, JobConflict
class ClientStoreTests(unittest.TestCase):
@@ -12,6 +13,7 @@ class ClientStoreTests(unittest.TestCase):
database = Path(directory) / "state.db"
store = ClientStore(database)
store.initialize()
self.assertEqual(os.stat(database).st_mode & 0o777, 0o600)
command_id = str(uuid4())
first = store.accept_command(command_id, '{"b":2,"a":1}', '{"ok":true}')
duplicate = ClientStore(database).accept_command(
@@ -23,6 +25,32 @@ class ClientStoreTests(unittest.TestCase):
with self.assertRaises(CommandConflict):
store.accept_command(command_id, '{"a":2}', '{"ok":true}')
def test_job_definition_is_immutable_while_cursor_advances(self):
with tempfile.TemporaryDirectory() as directory:
store = ClientStore(Path(directory) / "state.db")
store.initialize()
store.save_job(
"job-1", '{"jobId":"job-1"}', "JOB_STATE_WAITING",
1, 2, False,
)
store.save_job(
"job-1", '{"jobId":"job-1"}', "JOB_STATE_RUNNING",
2, 3, False,
)
self.assertEqual(
store.job_snapshot_rows(["job-1"])[0]["revision"], 2
)
with self.assertRaisesRegex(JobConflict, "backwards"):
store.save_job(
"job-1", '{"jobId":"job-1"}', "JOB_STATE_WAITING",
1, 2, False,
)
with self.assertRaises(JobConflict):
store.save_job(
"job-1", '{"jobId":"other"}', "JOB_STATE_RUNNING",
2, 3, False,
)
if __name__ == "__main__":
unittest.main()