feat: execute durable syncthing route setup
This commit is contained in:
+122
-1
@@ -14,13 +14,130 @@ from archive_clients.config import ClientConfig, ConnectionConfig, ServiceConfig
|
||||
from archive_clients.daemon import ArchiveClientDaemon
|
||||
from archive_clients.probes import FilesystemProbe
|
||||
from archive_clients.services import ServiceProbe
|
||||
from archive_clients.syncthing import ConfiguredRoute
|
||||
from archive_clients.protocol import decode, encode, encode_message, new_envelope
|
||||
from archive_control.v1 import (
|
||||
client_pb2, common_pb2, control_pb2, inventory_pb2, job_pb2,
|
||||
client_pb2, common_pb2, control_pb2, inventory_pb2, job_pb2, route_pb2,
|
||||
)
|
||||
|
||||
|
||||
class DaemonTransportTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_ensure_route_is_durable_and_duplicate_replays_updates(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
token = root / "token"
|
||||
token.write_text("shared-secret", encoding="utf-8")
|
||||
os.chmod(token, 0o600)
|
||||
service = ServiceConfig(
|
||||
"http://local", PurePosixPath("/sync"), root
|
||||
)
|
||||
config = ClientConfig(
|
||||
"cache-1", "Cache 1", "cache", "ws://control", token,
|
||||
root / "state.db", root / "backups", service, service,
|
||||
)
|
||||
local_route = route_pb2.LocalRoute(
|
||||
route_id="route-1",
|
||||
local_relative_path="routes/route-1",
|
||||
folder_type=route_pb2.SYNCTHING_FOLDER_TYPE_SEND_RECEIVE,
|
||||
local_syncthing_device_id="LOCAL",
|
||||
peer_syncthing_device_ids=["PEER"],
|
||||
state=route_pb2.ROUTE_STATE_PROVISIONING,
|
||||
writable=True,
|
||||
)
|
||||
local_route.observed_at.GetCurrentTime()
|
||||
manager = Mock()
|
||||
manager.configure.return_value = ConfiguredRoute(
|
||||
"LOCAL", root / "routes/route-1", local_route, True, True
|
||||
)
|
||||
manager.verify.return_value = (True, True)
|
||||
probe = FilesystemProbe(root, True, True, True, True, True)
|
||||
syncthing_probe = ServiceProbe(
|
||||
"syncthing",
|
||||
common_pb2.HEALTH_STATE_HEALTHY,
|
||||
datetime.now(timezone.utc),
|
||||
device_id="LOCAL",
|
||||
)
|
||||
daemon = ArchiveClientDaemon(
|
||||
config,
|
||||
[probe, probe],
|
||||
[syncthing_probe],
|
||||
route_manager=manager,
|
||||
)
|
||||
await asyncio.to_thread(daemon.store.initialize)
|
||||
command = new_envelope()
|
||||
command.command.command_id = str(uuid4())
|
||||
command.command.created_at.CopyFrom(command.sent_at)
|
||||
spec = command.command.ensure_route.route
|
||||
spec.route_id = "route-1"
|
||||
spec.peer_client_id = "archive-1"
|
||||
spec.peer_syncthing_device_id = "PEER"
|
||||
spec.peer_addresses.append("dynamic")
|
||||
spec.local_relative_path = "routes/route-1"
|
||||
spec.setup_timeout_seconds = 1800
|
||||
outbound = asyncio.Queue()
|
||||
tasks = set()
|
||||
await daemon._handle(command, outbound, tasks)
|
||||
self.assertEqual(
|
||||
decode(await outbound.get()).command_ack.status,
|
||||
control_pb2.COMMAND_ACK_STATUS_ACCEPTED,
|
||||
)
|
||||
await next(iter(tasks))
|
||||
first_updates = [decode(await outbound.get()).route_update for _ in range(3)]
|
||||
self.assertEqual(
|
||||
[update.sequence for update in first_updates], [1, 2, 3]
|
||||
)
|
||||
self.assertEqual(
|
||||
first_updates[-1].verification.state,
|
||||
route_pb2.ROUTE_STATE_READY,
|
||||
)
|
||||
self.assertTrue(first_updates[-1].route.archive_control_created)
|
||||
|
||||
duplicate = new_envelope()
|
||||
duplicate.command.CopyFrom(command.command)
|
||||
tasks = set()
|
||||
await daemon._handle(duplicate, outbound, tasks)
|
||||
self.assertEqual(
|
||||
decode(await outbound.get()).command_ack.status,
|
||||
control_pb2.COMMAND_ACK_STATUS_DUPLICATE,
|
||||
)
|
||||
await next(iter(tasks))
|
||||
replay = [decode(await outbound.get()).route_update for _ in range(3)]
|
||||
self.assertEqual(
|
||||
[update.update_id for update in replay],
|
||||
[update.update_id for update in first_updates],
|
||||
)
|
||||
self.assertEqual(manager.configure.call_count, 1)
|
||||
|
||||
accepted_before_crash = new_envelope()
|
||||
accepted_before_crash.command.CopyFrom(command.command)
|
||||
accepted_before_crash.command.command_id = str(uuid4())
|
||||
acknowledgement = daemon._initial_acknowledgement(
|
||||
accepted_before_crash.command, set(), False
|
||||
)
|
||||
await asyncio.to_thread(
|
||||
daemon.store.accept_command,
|
||||
accepted_before_crash.command.command_id,
|
||||
encode_message(accepted_before_crash.command),
|
||||
encode_message(acknowledgement),
|
||||
)
|
||||
restarted = ArchiveClientDaemon(
|
||||
config,
|
||||
[probe, probe],
|
||||
[syncthing_probe],
|
||||
route_manager=manager,
|
||||
)
|
||||
resumed_outbound = asyncio.Queue()
|
||||
resumed_tasks = set()
|
||||
await restarted._resume_route_commands(
|
||||
resumed_outbound, resumed_tasks
|
||||
)
|
||||
await next(iter(resumed_tasks))
|
||||
resumed = [
|
||||
decode(await resumed_outbound.get()).route_update
|
||||
for _ in range(3)
|
||||
]
|
||||
self.assertEqual([item.sequence for item in resumed], [1, 2, 3])
|
||||
|
||||
async def test_slow_inventory_does_not_block_heartbeat(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
@@ -213,6 +330,10 @@ class DaemonTransportTests(unittest.IsolatedAsyncioTestCase):
|
||||
client_pb2.CLIENT_FEATURE_INVENTORY_CHUNKS,
|
||||
observed["features"],
|
||||
)
|
||||
self.assertIn(
|
||||
client_pb2.CLIENT_FEATURE_ROUTE_PROVISIONING,
|
||||
observed["features"],
|
||||
)
|
||||
self.assertEqual(observed["heartbeat"], 7)
|
||||
self.assertEqual(observed["first"], control_pb2.COMMAND_ACK_STATUS_ACCEPTED)
|
||||
self.assertEqual(observed["second"], control_pb2.COMMAND_ACK_STATUS_DUPLICATE)
|
||||
|
||||
Reference in New Issue
Block a user