test: add deferred phase 8 adversarial matrix
This commit is contained in:
Executable
+213
@@ -0,0 +1,213 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Assertions used by the Phase 8 v2, stall, and capacity scenarios."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
|
||||
from complex_jobs import (
|
||||
CONTROL,
|
||||
get_json,
|
||||
post_json,
|
||||
qb_json,
|
||||
selection_indices,
|
||||
)
|
||||
|
||||
|
||||
def command_qb_hash(args: argparse.Namespace) -> None:
|
||||
deadline = time.monotonic() + args.timeout
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
records = qb_json(args.endpoint, "/torrents/info")
|
||||
matches = [
|
||||
item for item in records
|
||||
if item.get("name") == args.name
|
||||
]
|
||||
last = matches
|
||||
if len(matches) == 1:
|
||||
value = str(matches[0].get("hash", ""))
|
||||
if len(value) not in {40, 64}:
|
||||
raise RuntimeError(f"qBittorrent exposed an invalid hash: {value}")
|
||||
print(value)
|
||||
return
|
||||
if len(matches) > 1:
|
||||
raise RuntimeError(f"qBittorrent name lookup is ambiguous: {matches}")
|
||||
time.sleep(0.25)
|
||||
raise RuntimeError(f"qBittorrent torrent did not appear: {last}")
|
||||
|
||||
|
||||
def command_wait_client(args: argparse.Namespace) -> None:
|
||||
deadline = time.monotonic() + args.timeout
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
clients = get_json(f"{CONTROL}/clients")
|
||||
matches = [
|
||||
item for item in clients
|
||||
if item.get("client_id") == args.client_id
|
||||
]
|
||||
last = matches
|
||||
if (
|
||||
len(matches) == 1
|
||||
and bool(matches[0].get("connected")) == args.connected
|
||||
):
|
||||
print(json.dumps({
|
||||
"client_id": args.client_id,
|
||||
"connected": args.connected,
|
||||
"generation": matches[0].get("generation"),
|
||||
}, sort_keys=True))
|
||||
return
|
||||
time.sleep(0.25)
|
||||
raise RuntimeError(f"client connection state did not converge: {last}")
|
||||
|
||||
|
||||
def command_wait_state(args: argparse.Namespace) -> None:
|
||||
deadline = time.monotonic() + args.timeout
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
last = get_json(f"{CONTROL}/jobs/{args.job_id}")
|
||||
if last["state"] == args.state:
|
||||
sequence = int(last.get("last_event_sequence", 0))
|
||||
if sequence < args.minimum_sequence:
|
||||
time.sleep(0.1)
|
||||
continue
|
||||
progress = (last.get("latest_event") or {}).get("progress") or {}
|
||||
overall = float(progress.get("overallFractionComplete", 0))
|
||||
if overall < args.minimum_overall:
|
||||
raise RuntimeError(
|
||||
f"job lost overall progress while {args.state}: {last}"
|
||||
)
|
||||
print(json.dumps({
|
||||
"job_id": args.job_id,
|
||||
"last_event_sequence": sequence,
|
||||
"overall_fraction_complete": overall,
|
||||
"state": last["state"],
|
||||
}, sort_keys=True))
|
||||
return
|
||||
if last["state"] in {
|
||||
"JOB_STATE_CANCELLED",
|
||||
"JOB_STATE_FAILED",
|
||||
"JOB_STATE_SUCCEEDED",
|
||||
}:
|
||||
raise RuntimeError(
|
||||
f"job reached {last['state']} while waiting for {args.state}"
|
||||
)
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError(f"job state did not converge: {last}")
|
||||
|
||||
|
||||
def command_mark_stalled(args: argparse.Namespace) -> None:
|
||||
deadline = time.monotonic() + args.timeout
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
outcomes = post_json(f"{CONTROL}/scheduler/advance", {})
|
||||
last = outcomes
|
||||
if any(
|
||||
item.get("job_id") == args.job_id
|
||||
and item.get("action") == "marked_stalled"
|
||||
for item in outcomes
|
||||
):
|
||||
args.state = "JOB_STATE_STALLED"
|
||||
args.minimum_sequence = 1
|
||||
command_wait_state(args)
|
||||
return
|
||||
time.sleep(0.25)
|
||||
raise RuntimeError(f"scheduler did not mark the job stalled: {last}")
|
||||
|
||||
|
||||
def command_assert_placement(args: argparse.Namespace) -> None:
|
||||
expected_files = [
|
||||
int(item) for item in args.files.split(",") if item
|
||||
]
|
||||
placements = get_json(f"{CONTROL}/placements")
|
||||
matches = [
|
||||
item for item in placements
|
||||
if item.get("client_id") == args.client_id
|
||||
and item.get("state") == "PLACEMENT_STATE_PRESENT"
|
||||
and (
|
||||
(args.v1 and item.get("info_hash_v1_hex") == args.v1)
|
||||
or (args.v2 and item.get("info_hash_v2_hex") == args.v2)
|
||||
)
|
||||
]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError(f"placement identity is missing or ambiguous: {matches}")
|
||||
placement = matches[0]
|
||||
actual_files = selection_indices(placement["verified_files"])
|
||||
if (
|
||||
actual_files != expected_files
|
||||
or (args.v1 and placement.get("info_hash_v1_hex") != args.v1)
|
||||
or (args.v2 and placement.get("info_hash_v2_hex") != args.v2)
|
||||
):
|
||||
raise RuntimeError(f"placement identity/selection mismatch: {placement}")
|
||||
print(json.dumps({
|
||||
"client_id": args.client_id,
|
||||
"files": actual_files,
|
||||
"info_hash_v1_hex": placement.get("info_hash_v1_hex", ""),
|
||||
"info_hash_v2_hex": placement.get("info_hash_v2_hex", ""),
|
||||
}, sort_keys=True))
|
||||
|
||||
|
||||
def command_scheduler(args: argparse.Namespace) -> None:
|
||||
result = post_json(f"{CONTROL}/scheduler/{args.action}", {})
|
||||
expected = args.action == "resume"
|
||||
if result.get("automatic_scheduler") is not expected:
|
||||
raise RuntimeError(f"scheduler gate did not {args.action}: {result}")
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser()
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
qb_hash = commands.add_parser("qb-hash")
|
||||
qb_hash.add_argument("--endpoint", required=True)
|
||||
qb_hash.add_argument("--name", required=True)
|
||||
qb_hash.add_argument("--timeout", type=float, default=30)
|
||||
qb_hash.set_defaults(run=command_qb_hash)
|
||||
|
||||
client = commands.add_parser("wait-client")
|
||||
client.add_argument("--client-id", required=True)
|
||||
client.add_argument(
|
||||
"--connected", action=argparse.BooleanOptionalAction, default=True
|
||||
)
|
||||
client.add_argument("--timeout", type=float, default=30)
|
||||
client.set_defaults(run=command_wait_client)
|
||||
|
||||
state = commands.add_parser("wait-state")
|
||||
state.add_argument("--job-id", required=True)
|
||||
state.add_argument("--state", required=True)
|
||||
state.add_argument("--minimum-sequence", type=int, default=0)
|
||||
state.add_argument("--minimum-overall", type=float, default=0)
|
||||
state.add_argument("--timeout", type=float, default=30)
|
||||
state.set_defaults(run=command_wait_state)
|
||||
|
||||
stalled = commands.add_parser("mark-stalled")
|
||||
stalled.add_argument("--job-id", required=True)
|
||||
stalled.add_argument("--minimum-overall", type=float, default=0.01)
|
||||
stalled.add_argument("--timeout", type=float, default=15)
|
||||
stalled.set_defaults(run=command_mark_stalled)
|
||||
|
||||
placement = commands.add_parser("assert-placement")
|
||||
placement.add_argument("--client-id", required=True)
|
||||
placement.add_argument("--v1", default="")
|
||||
placement.add_argument("--v2", default="")
|
||||
placement.add_argument("--files", required=True)
|
||||
placement.set_defaults(run=command_assert_placement)
|
||||
|
||||
scheduler = commands.add_parser("scheduler")
|
||||
scheduler.add_argument(
|
||||
"--action", choices=("pause", "resume"), required=True
|
||||
)
|
||||
scheduler.set_defaults(run=command_scheduler)
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
args.run(args)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+220
@@ -0,0 +1,220 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise duplicate and reordered job events across a real WebSocket."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from websockets.asyncio.client import connect
|
||||
|
||||
from archive_clients.protocol import decode, encode, new_envelope
|
||||
from archive_control.v1 import client_pb2, common_pb2, control_pb2, job_pb2
|
||||
|
||||
|
||||
CONTROL_HTTP = "http://127.0.0.1:18081/test/v1"
|
||||
CONTROL_WS = "ws://127.0.0.1:8765/archive_control"
|
||||
CLIENT_ID = "protocol-probe"
|
||||
|
||||
|
||||
def http_json(method: str, path: str, value: object | None = None):
|
||||
encoded = (
|
||||
json.dumps(value, separators=(",", ":")).encode()
|
||||
if value is not None
|
||||
else None
|
||||
)
|
||||
request = urllib.request.Request(
|
||||
f"{CONTROL_HTTP}{path}",
|
||||
data=encoded,
|
||||
method=method,
|
||||
headers={"Content-Type": "application/json"} if encoded else {},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def registration():
|
||||
envelope = new_envelope()
|
||||
request = envelope.register_request
|
||||
request.protocol_version.CopyFrom(envelope.protocol_version)
|
||||
request.client.client_id = CLIENT_ID
|
||||
request.client.display_name = "E2E protocol ordering probe"
|
||||
request.client.role = common_pb2.CLIENT_ROLE_CACHE
|
||||
request.connection_instance_id = str(uuid4())
|
||||
request.shared_token = Path(
|
||||
"/run/secrets/archive_control_token"
|
||||
).read_text(encoding="utf-8").strip()
|
||||
request.capabilities.max_envelope_bytes = 1024 * 1024
|
||||
return envelope
|
||||
|
||||
|
||||
def acknowledgement(command_id: str):
|
||||
envelope = new_envelope()
|
||||
envelope.command_ack.command_id = command_id
|
||||
envelope.command_ack.status = (
|
||||
control_pb2.COMMAND_ACK_STATUS_ACCEPTED
|
||||
)
|
||||
return envelope
|
||||
|
||||
|
||||
def heartbeat_ack(sequence: int):
|
||||
envelope = new_envelope()
|
||||
envelope.heartbeat_ack.sequence = sequence
|
||||
return envelope
|
||||
|
||||
|
||||
def job_event(
|
||||
job_id: str,
|
||||
*,
|
||||
event_id: str,
|
||||
sequence: int,
|
||||
revision: int,
|
||||
event_type: int,
|
||||
state: int,
|
||||
):
|
||||
envelope = new_envelope()
|
||||
event = envelope.job_event
|
||||
event.event_id = event_id
|
||||
event.job_id = job_id
|
||||
event.sequence = sequence
|
||||
event.job_revision = revision
|
||||
event.type = event_type
|
||||
event.state = state
|
||||
event.occurred_at.GetCurrentTime()
|
||||
if state == job_pb2.JOB_STATE_FAILED:
|
||||
event.error.code = common_pb2.ERROR_CODE_PRECONDITION_FAILED
|
||||
event.error.message = "protocol ordering probe terminal event"
|
||||
return envelope
|
||||
|
||||
|
||||
async def receive_payload(websocket, expected: set[str], timeout: float = 15):
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise RuntimeError(f"timed out waiting for {sorted(expected)}")
|
||||
envelope = decode(await asyncio.wait_for(websocket.recv(), remaining))
|
||||
payload = envelope.WhichOneof("payload")
|
||||
if payload == "heartbeat":
|
||||
await websocket.send(
|
||||
encode(heartbeat_ack(envelope.heartbeat.sequence))
|
||||
)
|
||||
continue
|
||||
if payload in expected:
|
||||
return envelope
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
async with connect(
|
||||
CONTROL_WS,
|
||||
ping_interval=None,
|
||||
proxy=None,
|
||||
max_size=1024 * 1024,
|
||||
) as websocket:
|
||||
await websocket.send(encode(registration()))
|
||||
response = await receive_payload(websocket, {"register_response"})
|
||||
if (
|
||||
response.register_response.status
|
||||
!= client_pb2.REGISTRATION_STATUS_ACCEPTED
|
||||
):
|
||||
raise RuntimeError("protocol probe registration was rejected")
|
||||
|
||||
lease = await asyncio.to_thread(
|
||||
http_json,
|
||||
"POST",
|
||||
"/protocol/order-probe",
|
||||
{"client_id": CLIENT_ID},
|
||||
)
|
||||
if not lease.get("delivered"):
|
||||
raise RuntimeError("protocol probe command was not delivered")
|
||||
assigned = await receive_payload(websocket, {"command"})
|
||||
if assigned.command.WhichOneof("payload") != "assign_job":
|
||||
raise RuntimeError("protocol probe received the wrong command")
|
||||
if assigned.command.command_id != lease["command_id"]:
|
||||
raise RuntimeError("protocol probe received the wrong command ID")
|
||||
await websocket.send(
|
||||
encode(acknowledgement(assigned.command.command_id))
|
||||
)
|
||||
|
||||
job_id = lease["job_id"]
|
||||
first = job_event(
|
||||
job_id,
|
||||
event_id=str(uuid4()),
|
||||
sequence=1,
|
||||
revision=0,
|
||||
event_type=control_pb2.JOB_EVENT_TYPE_ASSIGNED,
|
||||
state=job_pb2.JOB_STATE_PREPARING,
|
||||
)
|
||||
second = job_event(
|
||||
job_id,
|
||||
event_id=str(uuid4()),
|
||||
sequence=2,
|
||||
revision=1,
|
||||
event_type=control_pb2.JOB_EVENT_TYPE_FAILED,
|
||||
state=job_pb2.JOB_STATE_FAILED,
|
||||
)
|
||||
|
||||
second_frame = encode(second)
|
||||
await websocket.send(second_frame)
|
||||
observed: set[str] = set()
|
||||
while observed != {"snapshot", "error"}:
|
||||
envelope = await receive_payload(
|
||||
websocket, {"command", "protocol_error"}
|
||||
)
|
||||
if envelope.WhichOneof("payload") == "protocol_error":
|
||||
error = envelope.protocol_error
|
||||
if error.offending_message_id != second.message_id:
|
||||
raise RuntimeError("gap error identified the wrong envelope")
|
||||
if (
|
||||
"expected event sequence 1, received 2"
|
||||
not in error.error.message
|
||||
):
|
||||
raise RuntimeError("gap error did not explain the ordering fault")
|
||||
observed.add("error")
|
||||
continue
|
||||
if (
|
||||
envelope.command.WhichOneof("payload")
|
||||
!= "request_job_snapshot"
|
||||
or list(envelope.command.request_job_snapshot.job_ids)
|
||||
!= [job_id]
|
||||
):
|
||||
raise RuntimeError("gap did not request the expected snapshot")
|
||||
await websocket.send(
|
||||
encode(acknowledgement(envelope.command.command_id))
|
||||
)
|
||||
observed.add("snapshot")
|
||||
|
||||
first_frame = encode(first)
|
||||
await websocket.send(first_frame)
|
||||
await websocket.send(first_frame)
|
||||
await websocket.send(second_frame)
|
||||
|
||||
deadline = time.monotonic() + 15
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
last = await asyncio.to_thread(
|
||||
http_json, "GET", f"/jobs/{job_id}"
|
||||
)
|
||||
if (
|
||||
last["state"] == "JOB_STATE_FAILED"
|
||||
and int(last["last_event_sequence"]) == 2
|
||||
and int(last["revision"]) == 1
|
||||
):
|
||||
print(json.dumps({
|
||||
"duplicate_sequence": 1,
|
||||
"gap_sequence": 2,
|
||||
"job_id": job_id,
|
||||
"recovered_terminal_state": last["state"],
|
||||
"snapshot_requested": True,
|
||||
}, sort_keys=True))
|
||||
return
|
||||
await asyncio.sleep(0.1)
|
||||
raise RuntimeError(f"protocol ordering probe did not converge: {last}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user