cabbage/archive-clients
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: execute durable syncthing route setup

Browse Source
This commit is contained in:
cabbage
2026-07-23 02:27:17 +00:00
parent 4058b6d3c8
commit 10e803eaee
8 changed files with 1189 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_daemon.py
+122 -1
View File
@@ -14,13 +14,130 @@ from archive_clients.config import ClientConfig, ConnectionConfig, ServiceConfig
from archive_clients.daemon import ArchiveClientDaemon
from archive_clients.probes import FilesystemProbe
from archive_clients.services import ServiceProbe
from archive_clients.syncthing import ConfiguredRoute
from archive_clients.protocol import decode, encode, encode_message, new_envelope
from archive_control.v1 import (
client_pb2, common_pb2, control_pb2, inventory_pb2, job_pb2,
client_pb2, common_pb2, control_pb2, inventory_pb2, job_pb2, route_pb2,
)
class DaemonTransportTests(unittest.IsolatedAsyncioTestCase):
async def test_ensure_route_is_durable_and_duplicate_replays_updates(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
token = root / "token"
token.write_text("shared-secret", encoding="utf-8")
os.chmod(token, 0o600)
service = ServiceConfig(
"http://local", PurePosixPath("/sync"), root
)
config = ClientConfig(
"cache-1", "Cache 1", "cache", "ws://control", token,
root / "state.db", root / "backups", service, service,
)
local_route = route_pb2.LocalRoute(
route_id="route-1",
local_relative_path="routes/route-1",
folder_type=route_pb2.SYNCTHING_FOLDER_TYPE_SEND_RECEIVE,
local_syncthing_device_id="LOCAL",
peer_syncthing_device_ids=["PEER"],
state=route_pb2.ROUTE_STATE_PROVISIONING,
writable=True,
)
local_route.observed_at.GetCurrentTime()
manager = Mock()
manager.configure.return_value = ConfiguredRoute(
"LOCAL", root / "routes/route-1", local_route, True, True
)
manager.verify.return_value = (True, True)
probe = FilesystemProbe(root, True, True, True, True, True)
syncthing_probe = ServiceProbe(
"syncthing",
common_pb2.HEALTH_STATE_HEALTHY,
datetime.now(timezone.utc),
device_id="LOCAL",
)
daemon = ArchiveClientDaemon(
config,
[probe, probe],
[syncthing_probe],
route_manager=manager,
)
await asyncio.to_thread(daemon.store.initialize)
command = new_envelope()
command.command.command_id = str(uuid4())
command.command.created_at.CopyFrom(command.sent_at)
spec = command.command.ensure_route.route
spec.route_id = "route-1"
spec.peer_client_id = "archive-1"
spec.peer_syncthing_device_id = "PEER"
spec.peer_addresses.append("dynamic")
spec.local_relative_path = "routes/route-1"
spec.setup_timeout_seconds = 1800
outbound = asyncio.Queue()
tasks = set()
await daemon._handle(command, outbound, tasks)
self.assertEqual(
decode(await outbound.get()).command_ack.status,
control_pb2.COMMAND_ACK_STATUS_ACCEPTED,
)
await next(iter(tasks))
first_updates = [decode(await outbound.get()).route_update for _ in range(3)]
self.assertEqual(
[update.sequence for update in first_updates], [1, 2, 3]
)
self.assertEqual(
first_updates[-1].verification.state,
route_pb2.ROUTE_STATE_READY,
)
self.assertTrue(first_updates[-1].route.archive_control_created)
duplicate = new_envelope()
duplicate.command.CopyFrom(command.command)
tasks = set()
await daemon._handle(duplicate, outbound, tasks)
self.assertEqual(
decode(await outbound.get()).command_ack.status,
control_pb2.COMMAND_ACK_STATUS_DUPLICATE,
)
await next(iter(tasks))
replay = [decode(await outbound.get()).route_update for _ in range(3)]
self.assertEqual(
[update.update_id for update in replay],
[update.update_id for update in first_updates],
)
self.assertEqual(manager.configure.call_count, 1)
accepted_before_crash = new_envelope()
accepted_before_crash.command.CopyFrom(command.command)
accepted_before_crash.command.command_id = str(uuid4())
acknowledgement = daemon._initial_acknowledgement(
accepted_before_crash.command, set(), False
)
await asyncio.to_thread(
daemon.store.accept_command,
accepted_before_crash.command.command_id,
encode_message(accepted_before_crash.command),
encode_message(acknowledgement),
)
restarted = ArchiveClientDaemon(
config,
[probe, probe],
[syncthing_probe],
route_manager=manager,
)
resumed_outbound = asyncio.Queue()
resumed_tasks = set()
await restarted._resume_route_commands(
resumed_outbound, resumed_tasks
)
await next(iter(resumed_tasks))
resumed = [
decode(await resumed_outbound.get()).route_update
for _ in range(3)
]
self.assertEqual([item.sequence for item in resumed], [1, 2, 3])
async def test_slow_inventory_does_not_block_heartbeat(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
@@ -213,6 +330,10 @@ class DaemonTransportTests(unittest.IsolatedAsyncioTestCase):
client_pb2.CLIENT_FEATURE_INVENTORY_CHUNKS,
observed["features"],
)
self.assertIn(
client_pb2.CLIENT_FEATURE_ROUTE_PROVISIONING,
observed["features"],
)
self.assertEqual(observed["heartbeat"], 7)
self.assertEqual(observed["first"], control_pb2.COMMAND_ACK_STATUS_ACCEPTED)
self.assertEqual(observed["second"], control_pb2.COMMAND_ACK_STATUS_DUPLICATE)
tests/test_state.py
+35
View File
@@ -51,6 +51,41 @@ class ClientStoreTests(unittest.TestCase):
2, 3, False,
)
def test_route_attempt_nonce_and_updates_survive_duplicate_commands(self):
with tempfile.TemporaryDirectory() as directory:
store = ClientStore(Path(directory) / "state.db")
store.initialize()
command_id = str(uuid4())
store.accept_command(command_id, '{"route":"one"}', '{"ok":true}')
first = store.begin_route_attempt(
command_id, "route-1", '{"routeId":"route-1"}', "nonce-1"
)
duplicate = store.begin_route_attempt(
command_id, "route-1", '{"routeId":"route-1"}', "new-nonce"
)
self.assertEqual(first["nonce"], "nonce-1")
self.assertEqual(duplicate["nonce"], "nonce-1")
store.record_route_update(
command_id, 1, "provisioning", '{"sequence":1}'
)
store.record_route_ownership(command_id, True, True)
store.record_route_update(
command_id, 1, "provisioning", '{"sequence":1}'
)
store.record_route_update(
command_id, 2, "ready", '{"sequence":2}'
)
self.assertEqual(store.get_route_attempt(command_id)["state"], "ready")
self.assertEqual(store.get_route_attempt(command_id)["created_folder"], 1)
self.assertEqual(
[row["sequence"] for row in store.route_update_rows(command_id)],
[1, 2],
)
with self.assertRaises(CommandConflict):
store.record_route_update(
command_id, 2, "ready", '{"sequence":2,"changed":true}'
)
if __name__ == "__main__":
unittest.main()
tests/test_syncthing.py
+151
View File
@@ -0,0 +1,151 @@
import json
import tempfile
import time
import unittest
from pathlib import Path, PurePosixPath
from archive_clients.config import ServiceConfig
from archive_clients.syncthing import (
RoutePathConflict,
RouteSetupTimeout,
SyncthingRouteManager,
)
from archive_control.v1 import route_pb2
class FakeTransport:
def __init__(self):
self.status = {"myID": "LOCAL"}
self.config = {"devices": [], "folders": []}
self.puts = []
self.posts = []
def get_json(self, path):
if path == "/rest/system/status":
return self.status
if path == "/rest/config":
return self.config
raise AssertionError(path)
def put_json(self, path, payload):
self.puts.append((path, payload))
if path.startswith("/rest/config/devices/"):
self.config["devices"].append(payload)
elif path.startswith("/rest/config/folders/"):
self.config["folders"].append(payload)
else:
raise AssertionError(path)
def post(self, path):
self.posts.append(path)
class SyncthingRouteManagerTests(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.root = Path(self.temp_dir.name) / "sync"
self.root.mkdir()
self.transport = FakeTransport()
self.manager = SyncthingRouteManager(
ServiceConfig(
"http://syncthing",
PurePosixPath("/sync"),
self.root,
),
sparse_supported=True,
transport=self.transport,
poll_interval=0,
)
self.spec = route_pb2.EnsureRouteSpec(
route_id="route-1",
peer_client_id="archive-1",
peer_syncthing_device_id="PEER",
peer_addresses=["tcp://archive:22000"],
local_relative_path="routes/route-1",
setup_timeout_seconds=1800,
)
def tearDown(self):
self.temp_dir.cleanup()
def test_configure_adds_only_peer_and_pairwise_folder(self):
configured = self.manager.configure(
self.spec, time.monotonic() + 1
)
self.assertEqual(configured.local_device_id, "LOCAL")
self.assertTrue(configured.local_path.is_dir())
self.assertEqual(len(self.transport.puts), 2)
folder = self.transport.config["folders"][0]
self.assertEqual(folder["path"], "/sync/routes/route-1")
self.assertEqual(folder["type"], "sendreceive")
self.assertEqual(
{item["deviceID"] for item in folder["devices"]},
{"LOCAL", "PEER"},
)
self.assertTrue(configured.local_route.archive_control_created)
repeated = self.manager.configure(self.spec, time.monotonic() + 1)
self.assertEqual(len(self.transport.puts), 2)
self.assertFalse(repeated.local_route.archive_control_created)
def test_existing_folder_conflicts_are_never_overwritten(self):
self.transport.config["folders"].append(
{
"id": "route-1",
"path": "/somewhere-else",
"type": "sendreceive",
"devices": [{"deviceID": "LOCAL"}, {"deviceID": "PEER"}],
}
)
with self.assertRaisesRegex(RoutePathConflict, "different path"):
self.manager.configure(self.spec, time.monotonic() + 1)
self.assertEqual(self.transport.puts, [])
def test_bidirectional_nonce_and_ack_are_required(self):
configured = self.manager.configure(self.spec, time.monotonic() + 1)
peer_nonce = configured.local_path / (
".archive-control-route-nonce.archive-1"
)
peer_nonce.write_text(
json.dumps({
"route_id": "route-1",
"client_id": "archive-1",
"nonce": "peer-nonce",
}),
encoding="utf-8",
)
acknowledgement = configured.local_path / (
".archive-control-route-ack.cache-1.archive-1"
)
acknowledgement.write_text(
json.dumps({"route_id": "route-1", "nonce": "local-nonce"}),
encoding="utf-8",
)
verified = self.manager.verify(
configured,
self.spec,
"cache-1",
"local-nonce",
time.monotonic() + 1,
)
self.assertEqual(verified, (True, True))
peer_ack = configured.local_path / (
".archive-control-route-ack.archive-1.cache-1"
)
self.assertEqual(json.loads(peer_ack.read_text())["nonce"], "peer-nonce")
def test_verification_times_out_without_peer_evidence(self):
configured = self.manager.configure(self.spec, time.monotonic() + 1)
with self.assertRaises(RouteSetupTimeout):
self.manager.verify(
configured,
self.spec,
"cache-1",
"local-nonce",
time.monotonic() + 0.01,
)
self.assertTrue(self.transport.posts)
if __name__ == "__main__":
unittest.main()