diff --git a/README.md b/README.md index ba9566b..b4a04da 100644 --- a/README.md +++ b/README.md @@ -57,5 +57,9 @@ docker run --rm archive-clients-test docker build -t sodium/archive-clients:dev . ``` +The local cross-project route test is under `e2e/`. It brings up isolated +2×2 cache/archive node stacks plus the standalone control core and verifies all +four eager-mesh routes without Telegram. See `e2e/README.md`. + Generated bindings are pinned to archive-control-proto commit `4ec852014dad74606d4078b3ae1aa208c814b033`. diff --git a/e2e/.gitignore b/e2e/.gitignore new file mode 100644 index 0000000..e1a8be8 --- /dev/null +++ b/e2e/.gitignore @@ -0,0 +1,3 @@ +*/runtime/ +*/secrets/ + diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 0000000..03f5df3 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,32 @@ +# Local 2×2 route E2E + +This harness runs one bot-free control stack plus four isolated data-node +Compose projects. Every data node contains its own qBittorrent, Syncthing, and +archive-client containers and its own bind-mounted config, secrets, state, +backup, qB data, and sync roots. The stacks share only the explicitly named +`archive-control-e2e` network. + +The initial scenario starts with no route folders. With the control policy set +to `eager_mesh`, two cache and two archive registrations must produce four +independent, bidirectionally verified Syncthing routes. The assertion talks to +the loopback-only test adapter through a curl sidecar sharing the control +container's network namespace; Telegram is not initialized. + +Run from the archive-clients checkout: + +```bash +./e2e/scripts/up.sh +./e2e/scripts/logs.sh +./e2e/scripts/down.sh +``` + +`up.sh` defaults `ARCHIVE_CONTROL_SOURCE` to the sibling playground +`mogic-bot/mogic-repo` checkout. Set that environment variable to test another +control worktree. `E2E_WAIT_SECONDS` overrides the 360-second assertion +deadline. `down.sh` removes only the five exact Compose projects and the +labelled E2E network; it intentionally retains all bind-mounted runtime state. + +The Syncthing 2.1.2 and LinuxServer qBittorrent multi-platform image indexes are +digest-pinned. Runtime secrets are generated with mode 0600 and ignored by +Git. The qBittorrent test config limits its authentication bypass to the +isolated E2E network and must never be reused for deployment. diff --git a/e2e/archive-1/compose.yaml b/e2e/archive-1/compose.yaml new file mode 100644 index 0000000..297b165 --- /dev/null +++ b/e2e/archive-1/compose.yaml @@ -0,0 +1,71 @@ +services: + syncthing: + image: syncthing/syncthing:2.1.2@sha256:4464f4161dd0251e20d46bb3aec83363db75d80cef1abdd5d5fd4054b04a004d + user: "1001:1001" + entrypoint: ["/e2e/start-syncthing.sh"] + hostname: syncthing-archive-1 + volumes: + - ../common/start-syncthing.sh:/e2e/start-syncthing.sh:ro + - ./secrets:/run/secrets:ro + - ./runtime/syncthing:/var/syncthing + - ./runtime/sync:/sync + networks: + archive-control-e2e: + aliases: [syncthing-archive-1] + healthcheck: + test: ["CMD-SHELL", "curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -q OK"] + interval: 2s + timeout: 2s + retries: 30 + restart: unless-stopped + + qbittorrent: + image: lscr.io/linuxserver/qbittorrent:latest@sha256:b024436f8ca665d16d9a997d26fd27fdf867ee5566ba09f32764e7b2976d3e02 + environment: + PUID: "1001" + PGID: "1001" + TZ: Etc/UTC + WEBUI_PORT: "8080" + TORRENTING_PORT: "6881" + volumes: + - ./runtime/qb-config:/config + - ./runtime/qb-data:/downloads + networks: + archive-control-e2e: + aliases: [qb-archive-1] + healthcheck: + test: ["CMD-SHELL", "curl -fsS -m 2 http://127.0.0.1:8080/api/v2/app/version >/dev/null"] + interval: 2s + timeout: 2s + retries: 30 + restart: unless-stopped + + client: + image: archive-clients:e2e + build: + context: ../.. + dockerfile: Dockerfile + user: "1001:1001" + command: ["--config", "/etc/archive-control/client.toml"] + depends_on: + syncthing: + condition: service_healthy + qbittorrent: + condition: service_healthy + volumes: + - ./config/client.toml:/etc/archive-control/client.toml:ro + - ./secrets:/run/secrets:ro + - ./runtime/client-state:/var/lib/archive-control + - ./runtime/client-backups:/var/backups/archive-control + - ./runtime/qb-data:/data/qb + - ./runtime/sync:/data/sync + networks: + archive-control-e2e: + aliases: [client-archive-1] + restart: unless-stopped + +networks: + archive-control-e2e: + external: true + name: archive-control-e2e + diff --git a/e2e/archive-1/config/client.toml b/e2e/archive-1/config/client.toml new file mode 100644 index 0000000..aee0153 --- /dev/null +++ b/e2e/archive-1/config/client.toml @@ -0,0 +1,31 @@ +client_id = "archive-1" +display_name = "E2E Archive 1" +role = "archive" +control_endpoint = "ws://control:8765/archive_control" +shared_token_file = "/run/secrets/archive_control_token" +state_db = "/var/lib/archive-control/client.db" +backup_dir = "/var/backups/archive-control" + +[connection] +registration_timeout = "5s" +heartbeat_interval = "2s" +offline_timeout = "10s" +reconnect_initial = "1s" +reconnect_max = "5s" +reconnect_reset_after = "10s" +reconnect_jitter = false + +[qbittorrent] +endpoint = "http://qb-archive-1:8080" +username = "admin" +password_file = "/run/secrets/qb_password" +api_root = "/downloads" +local_root = "/data/qb" + +[syncthing] +endpoint = "http://syncthing-archive-1:8384" +api_key_file = "/run/secrets/syncthing_api_key" +api_root = "/sync" +local_root = "/data/sync" +advertised_addresses = ["tcp://syncthing-archive-1:22000"] + diff --git a/e2e/archive-2/compose.yaml b/e2e/archive-2/compose.yaml new file mode 100644 index 0000000..b3c5a12 --- /dev/null +++ b/e2e/archive-2/compose.yaml @@ -0,0 +1,71 @@ +services: + syncthing: + image: syncthing/syncthing:2.1.2@sha256:4464f4161dd0251e20d46bb3aec83363db75d80cef1abdd5d5fd4054b04a004d + user: "1001:1001" + entrypoint: ["/e2e/start-syncthing.sh"] + hostname: syncthing-archive-2 + volumes: + - ../common/start-syncthing.sh:/e2e/start-syncthing.sh:ro + - ./secrets:/run/secrets:ro + - ./runtime/syncthing:/var/syncthing + - ./runtime/sync:/sync + networks: + archive-control-e2e: + aliases: [syncthing-archive-2] + healthcheck: + test: ["CMD-SHELL", "curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -q OK"] + interval: 2s + timeout: 2s + retries: 30 + restart: unless-stopped + + qbittorrent: + image: lscr.io/linuxserver/qbittorrent:latest@sha256:b024436f8ca665d16d9a997d26fd27fdf867ee5566ba09f32764e7b2976d3e02 + environment: + PUID: "1001" + PGID: "1001" + TZ: Etc/UTC + WEBUI_PORT: "8080" + TORRENTING_PORT: "6881" + volumes: + - ./runtime/qb-config:/config + - ./runtime/qb-data:/downloads + networks: + archive-control-e2e: + aliases: [qb-archive-2] + healthcheck: + test: ["CMD-SHELL", "curl -fsS -m 2 http://127.0.0.1:8080/api/v2/app/version >/dev/null"] + interval: 2s + timeout: 2s + retries: 30 + restart: unless-stopped + + client: + image: archive-clients:e2e + build: + context: ../.. + dockerfile: Dockerfile + user: "1001:1001" + command: ["--config", "/etc/archive-control/client.toml"] + depends_on: + syncthing: + condition: service_healthy + qbittorrent: + condition: service_healthy + volumes: + - ./config/client.toml:/etc/archive-control/client.toml:ro + - ./secrets:/run/secrets:ro + - ./runtime/client-state:/var/lib/archive-control + - ./runtime/client-backups:/var/backups/archive-control + - ./runtime/qb-data:/data/qb + - ./runtime/sync:/data/sync + networks: + archive-control-e2e: + aliases: [client-archive-2] + restart: unless-stopped + +networks: + archive-control-e2e: + external: true + name: archive-control-e2e + diff --git a/e2e/archive-2/config/client.toml b/e2e/archive-2/config/client.toml new file mode 100644 index 0000000..f7cf2f9 --- /dev/null +++ b/e2e/archive-2/config/client.toml @@ -0,0 +1,31 @@ +client_id = "archive-2" +display_name = "E2E Archive 2" +role = "archive" +control_endpoint = "ws://control:8765/archive_control" +shared_token_file = "/run/secrets/archive_control_token" +state_db = "/var/lib/archive-control/client.db" +backup_dir = "/var/backups/archive-control" + +[connection] +registration_timeout = "5s" +heartbeat_interval = "2s" +offline_timeout = "10s" +reconnect_initial = "1s" +reconnect_max = "5s" +reconnect_reset_after = "10s" +reconnect_jitter = false + +[qbittorrent] +endpoint = "http://qb-archive-2:8080" +username = "admin" +password_file = "/run/secrets/qb_password" +api_root = "/downloads" +local_root = "/data/qb" + +[syncthing] +endpoint = "http://syncthing-archive-2:8384" +api_key_file = "/run/secrets/syncthing_api_key" +api_root = "/sync" +local_root = "/data/sync" +advertised_addresses = ["tcp://syncthing-archive-2:22000"] + diff --git a/e2e/cache-1/compose.yaml b/e2e/cache-1/compose.yaml new file mode 100644 index 0000000..0ac61ec --- /dev/null +++ b/e2e/cache-1/compose.yaml @@ -0,0 +1,71 @@ +services: + syncthing: + image: syncthing/syncthing:2.1.2@sha256:4464f4161dd0251e20d46bb3aec83363db75d80cef1abdd5d5fd4054b04a004d + user: "1001:1001" + entrypoint: ["/e2e/start-syncthing.sh"] + hostname: syncthing-cache-1 + volumes: + - ../common/start-syncthing.sh:/e2e/start-syncthing.sh:ro + - ./secrets:/run/secrets:ro + - ./runtime/syncthing:/var/syncthing + - ./runtime/sync:/sync + networks: + archive-control-e2e: + aliases: [syncthing-cache-1] + healthcheck: + test: ["CMD-SHELL", "curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -q OK"] + interval: 2s + timeout: 2s + retries: 30 + restart: unless-stopped + + qbittorrent: + image: lscr.io/linuxserver/qbittorrent:latest@sha256:b024436f8ca665d16d9a997d26fd27fdf867ee5566ba09f32764e7b2976d3e02 + environment: + PUID: "1001" + PGID: "1001" + TZ: Etc/UTC + WEBUI_PORT: "8080" + TORRENTING_PORT: "6881" + volumes: + - ./runtime/qb-config:/config + - ./runtime/qb-data:/downloads + networks: + archive-control-e2e: + aliases: [qb-cache-1] + healthcheck: + test: ["CMD-SHELL", "curl -fsS -m 2 http://127.0.0.1:8080/api/v2/app/version >/dev/null"] + interval: 2s + timeout: 2s + retries: 30 + restart: unless-stopped + + client: + image: archive-clients:e2e + build: + context: ../.. + dockerfile: Dockerfile + user: "1001:1001" + command: ["--config", "/etc/archive-control/client.toml"] + depends_on: + syncthing: + condition: service_healthy + qbittorrent: + condition: service_healthy + volumes: + - ./config/client.toml:/etc/archive-control/client.toml:ro + - ./secrets:/run/secrets:ro + - ./runtime/client-state:/var/lib/archive-control + - ./runtime/client-backups:/var/backups/archive-control + - ./runtime/qb-data:/data/qb + - ./runtime/sync:/data/sync + networks: + archive-control-e2e: + aliases: [client-cache-1] + restart: unless-stopped + +networks: + archive-control-e2e: + external: true + name: archive-control-e2e + diff --git a/e2e/cache-1/config/client.toml b/e2e/cache-1/config/client.toml new file mode 100644 index 0000000..38b7e6c --- /dev/null +++ b/e2e/cache-1/config/client.toml @@ -0,0 +1,31 @@ +client_id = "cache-1" +display_name = "E2E Cache 1" +role = "cache" +control_endpoint = "ws://control:8765/archive_control" +shared_token_file = "/run/secrets/archive_control_token" +state_db = "/var/lib/archive-control/client.db" +backup_dir = "/var/backups/archive-control" + +[connection] +registration_timeout = "5s" +heartbeat_interval = "2s" +offline_timeout = "10s" +reconnect_initial = "1s" +reconnect_max = "5s" +reconnect_reset_after = "10s" +reconnect_jitter = false + +[qbittorrent] +endpoint = "http://qb-cache-1:8080" +username = "admin" +password_file = "/run/secrets/qb_password" +api_root = "/downloads" +local_root = "/data/qb" + +[syncthing] +endpoint = "http://syncthing-cache-1:8384" +api_key_file = "/run/secrets/syncthing_api_key" +api_root = "/sync" +local_root = "/data/sync" +advertised_addresses = ["tcp://syncthing-cache-1:22000"] + diff --git a/e2e/cache-2/compose.yaml b/e2e/cache-2/compose.yaml new file mode 100644 index 0000000..69020ef --- /dev/null +++ b/e2e/cache-2/compose.yaml @@ -0,0 +1,71 @@ +services: + syncthing: + image: syncthing/syncthing:2.1.2@sha256:4464f4161dd0251e20d46bb3aec83363db75d80cef1abdd5d5fd4054b04a004d + user: "1001:1001" + entrypoint: ["/e2e/start-syncthing.sh"] + hostname: syncthing-cache-2 + volumes: + - ../common/start-syncthing.sh:/e2e/start-syncthing.sh:ro + - ./secrets:/run/secrets:ro + - ./runtime/syncthing:/var/syncthing + - ./runtime/sync:/sync + networks: + archive-control-e2e: + aliases: [syncthing-cache-2] + healthcheck: + test: ["CMD-SHELL", "curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -q OK"] + interval: 2s + timeout: 2s + retries: 30 + restart: unless-stopped + + qbittorrent: + image: lscr.io/linuxserver/qbittorrent:latest@sha256:b024436f8ca665d16d9a997d26fd27fdf867ee5566ba09f32764e7b2976d3e02 + environment: + PUID: "1001" + PGID: "1001" + TZ: Etc/UTC + WEBUI_PORT: "8080" + TORRENTING_PORT: "6881" + volumes: + - ./runtime/qb-config:/config + - ./runtime/qb-data:/downloads + networks: + archive-control-e2e: + aliases: [qb-cache-2] + healthcheck: + test: ["CMD-SHELL", "curl -fsS -m 2 http://127.0.0.1:8080/api/v2/app/version >/dev/null"] + interval: 2s + timeout: 2s + retries: 30 + restart: unless-stopped + + client: + image: archive-clients:e2e + build: + context: ../.. + dockerfile: Dockerfile + user: "1001:1001" + command: ["--config", "/etc/archive-control/client.toml"] + depends_on: + syncthing: + condition: service_healthy + qbittorrent: + condition: service_healthy + volumes: + - ./config/client.toml:/etc/archive-control/client.toml:ro + - ./secrets:/run/secrets:ro + - ./runtime/client-state:/var/lib/archive-control + - ./runtime/client-backups:/var/backups/archive-control + - ./runtime/qb-data:/data/qb + - ./runtime/sync:/data/sync + networks: + archive-control-e2e: + aliases: [client-cache-2] + restart: unless-stopped + +networks: + archive-control-e2e: + external: true + name: archive-control-e2e + diff --git a/e2e/cache-2/config/client.toml b/e2e/cache-2/config/client.toml new file mode 100644 index 0000000..ecaca20 --- /dev/null +++ b/e2e/cache-2/config/client.toml @@ -0,0 +1,31 @@ +client_id = "cache-2" +display_name = "E2E Cache 2" +role = "cache" +control_endpoint = "ws://control:8765/archive_control" +shared_token_file = "/run/secrets/archive_control_token" +state_db = "/var/lib/archive-control/client.db" +backup_dir = "/var/backups/archive-control" + +[connection] +registration_timeout = "5s" +heartbeat_interval = "2s" +offline_timeout = "10s" +reconnect_initial = "1s" +reconnect_max = "5s" +reconnect_reset_after = "10s" +reconnect_jitter = false + +[qbittorrent] +endpoint = "http://qb-cache-2:8080" +username = "admin" +password_file = "/run/secrets/qb_password" +api_root = "/downloads" +local_root = "/data/qb" + +[syncthing] +endpoint = "http://syncthing-cache-2:8384" +api_key_file = "/run/secrets/syncthing_api_key" +api_root = "/sync" +local_root = "/data/sync" +advertised_addresses = ["tcp://syncthing-cache-2:22000"] + diff --git a/e2e/common/qBittorrent.conf b/e2e/common/qBittorrent.conf new file mode 100644 index 0000000..71686a0 --- /dev/null +++ b/e2e/common/qBittorrent.conf @@ -0,0 +1,14 @@ +[LegalNotice] +Accepted=true + +[Preferences] +Downloads\SavePath=/downloads/ +WebUI\Address=* +WebUI\AlternativeUIEnabled=false +WebUI\AuthSubnetWhitelist=0.0.0.0/0 +WebUI\AuthSubnetWhitelistEnabled=true +WebUI\CSRFProtection=false +WebUI\HostHeaderValidation=false +WebUI\Password_PBKDF2="@ByteArray(ARQ77eY1NUZaQsuDHbIMCA==:0WMRkYTUWVT9wVvdDtHAjU9b3b7uB8NR1Gur2hmQCvCDpm39Q+PsJRJPaCU51dEiz+dTzh8qbPsL8WkFljQYFQ==)" +WebUI\Port=8080 +WebUI\Username=admin diff --git a/e2e/common/start-syncthing.sh b/e2e/common/start-syncthing.sh new file mode 100755 index 0000000..4de30ee --- /dev/null +++ b/e2e/common/start-syncthing.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu + +home=/var/syncthing +api_key=$(cat /run/secrets/syncthing_api_key) + +if [ ! -f "$home/config.xml" ]; then + syncthing generate --home="$home" --no-port-probing +fi + +exec syncthing serve \ + --home="$home" \ + --gui-address=http://0.0.0.0:8384 \ + --gui-apikey="$api_key" \ + --no-browser \ + --no-port-probing \ + --no-restart \ + --no-upgrade + diff --git a/e2e/control/compose.yaml b/e2e/control/compose.yaml new file mode 100644 index 0000000..54c02bb --- /dev/null +++ b/e2e/control/compose.yaml @@ -0,0 +1,29 @@ +services: + control: + image: archive-control-standalone:e2e + build: + context: ${ARCHIVE_CONTROL_SOURCE:?set ARCHIVE_CONTROL_SOURCE to the mogic-bot checkout} + dockerfile: Dockerfile.archive-control + command: ["--config", "/etc/archive-control/config.json"] + user: "1001:1001" + volumes: + - ./config/config.json:/etc/archive-control/config.json:ro + - ./secrets:/run/secrets:ro + - ./runtime/state:/var/lib/archive-control + - ./runtime/backups:/var/backups/archive-control + networks: + archive-control-e2e: + aliases: [control] + restart: unless-stopped + + curl: + image: curlimages/curl:8.16.0 + network_mode: service:control + profiles: [tools] + depends_on: [control] + +networks: + archive-control-e2e: + external: true + name: archive-control-e2e + diff --git a/e2e/control/config/config.json b/e2e/control/config/config.json new file mode 100644 index 0000000..96d4bb7 --- /dev/null +++ b/e2e/control/config/config.json @@ -0,0 +1,22 @@ +{ + "archive_control": { + "enabled": true, + "listen": "0.0.0.0:8765", + "websocket_path": "/archive_control", + "shared_token_file": "/run/secrets/archive_control_token", + "database": "/var/lib/archive-control/control.db", + "backup_dir": "/var/backups/archive-control", + "registration_timeout": "5s", + "heartbeat_interval": "2s", + "offline_timeout": "10s", + "command_ack_timeout": "2s", + "command_max_attempts": 3, + "route_policy": "eager_mesh", + "route_setup_timeout": "5m", + "test_http": { + "enabled": true, + "listen": "127.0.0.1:18081", + "max_body_bytes": 1048576 + } + } +} diff --git a/e2e/scripts/down.sh b/e2e/scripts/down.sh new file mode 100755 index 0000000..fdf3ba3 --- /dev/null +++ b/e2e/scripts/down.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +source "$(dirname "$0")/lib.sh" + +for ((index=${#E2E_NODES[@]} - 1; index >= 0; index--)); do + compose_node "${E2E_NODES[index]}" down --remove-orphans +done +compose_control down --remove-orphans + +label=$(docker network inspect \ + -f '{{ index .Labels "archive-control.e2e" }}' \ + "$E2E_NETWORK" 2>/dev/null || true) +if [[ "$label" == "true" ]]; then + docker network rm "$E2E_NETWORK" >/dev/null +fi + +printf 'E2E containers stopped; bind-mounted runtime state was retained.\n' + diff --git a/e2e/scripts/lib.sh b/e2e/scripts/lib.sh new file mode 100755 index 0000000..50414aa --- /dev/null +++ b/e2e/scripts/lib.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +E2E_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +E2E_NETWORK=archive-control-e2e +E2E_NODES=(cache-1 cache-2 archive-1 archive-2) +if [[ -z "${ARCHIVE_CONTROL_SOURCE:-}" ]]; then + control_candidate="$E2E_ROOT/../../../mogic-bot/mogic-repo" + if [[ -d "$control_candidate" ]]; then + ARCHIVE_CONTROL_SOURCE=$(cd "$control_candidate" && pwd) + else + ARCHIVE_CONTROL_SOURCE=$control_candidate + fi +fi +export ARCHIVE_CONTROL_SOURCE + +compose_control() { + docker compose \ + -p archive-e2e-control \ + -f "$E2E_ROOT/control/compose.yaml" "$@" +} + +compose_node() { + local node=$1 + shift + docker compose \ + -p "archive-e2e-${node}" \ + -f "$E2E_ROOT/${node}/compose.yaml" "$@" +} diff --git a/e2e/scripts/logs.sh b/e2e/scripts/logs.sh new file mode 100755 index 0000000..472c4bf --- /dev/null +++ b/e2e/scripts/logs.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +source "$(dirname "$0")/lib.sh" + +compose_control logs --no-color --tail=200 || true +for node in "${E2E_NODES[@]}"; do + compose_node "$node" logs --no-color --tail=200 || true +done + diff --git a/e2e/scripts/prepare.sh b/e2e/scripts/prepare.sh new file mode 100755 index 0000000..42dbeee --- /dev/null +++ b/e2e/scripts/prepare.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +source "$(dirname "$0")/lib.sh" +umask 077 + +prepare_secret() { + local path=$1 + local value=$2 + if [[ ! -f "$path" ]]; then + printf '%s\n' "$value" >"$path" + fi + chmod 600 "$path" +} + +mkdir -p \ + "$E2E_ROOT/control/runtime/state" \ + "$E2E_ROOT/control/runtime/backups" \ + "$E2E_ROOT/control/secrets" +prepare_secret \ + "$E2E_ROOT/control/secrets/archive_control_token" \ + "archive-control-e2e-token" + +for node in "${E2E_NODES[@]}"; do + node_root="$E2E_ROOT/$node" + mkdir -p \ + "$node_root/secrets" \ + "$node_root/runtime/syncthing" \ + "$node_root/runtime/sync" \ + "$node_root/runtime/qb-config/qBittorrent" \ + "$node_root/runtime/qb-data" \ + "$node_root/runtime/client-state" \ + "$node_root/runtime/client-backups" + prepare_secret \ + "$node_root/secrets/archive_control_token" \ + "archive-control-e2e-token" + prepare_secret \ + "$node_root/secrets/syncthing_api_key" \ + "archive-control-e2e-${node}-syncthing-key" + prepare_secret "$node_root/secrets/qb_password" "adminadmin" + qb_config="$node_root/runtime/qb-config/qBittorrent/qBittorrent.conf" + if [[ ! -f "$qb_config" ]]; then + cp "$E2E_ROOT/common/qBittorrent.conf" "$qb_config" + fi +done diff --git a/e2e/scripts/up.sh b/e2e/scripts/up.sh new file mode 100755 index 0000000..5a0cfcc --- /dev/null +++ b/e2e/scripts/up.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +source "$(dirname "$0")/lib.sh" +"$E2E_ROOT/scripts/prepare.sh" + +if ! docker network inspect "$E2E_NETWORK" >/dev/null 2>&1; then + docker network create \ + --label archive-control.e2e=true \ + "$E2E_NETWORK" >/dev/null +fi + +if [[ ! -f "$ARCHIVE_CONTROL_SOURCE/Dockerfile.archive-control" ]]; then + printf 'control source is missing Dockerfile.archive-control: %s\n' \ + "$ARCHIVE_CONTROL_SOURCE" >&2 + exit 2 +fi + +compose_control up -d --build control +for node in "${E2E_NODES[@]}"; do + compose_node "$node" up -d --build +done +"$E2E_ROOT/scripts/wait-routes.sh" diff --git a/e2e/scripts/wait-routes.sh b/e2e/scripts/wait-routes.sh new file mode 100755 index 0000000..e9932c5 --- /dev/null +++ b/e2e/scripts/wait-routes.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +source "$(dirname "$0")/lib.sh" +timeout_seconds=${E2E_WAIT_SECONDS:-360} +deadline=$((SECONDS + timeout_seconds)) + +while (( SECONDS < deadline )); do + payload=$(compose_control run --rm --no-deps \ + --entrypoint /bin/sh curl -c \ + 'curl --max-time 5 -fsS http://127.0.0.1:18081/test/v1/clients; printf "\n"; curl --max-time 5 -fsS http://127.0.0.1:18081/test/v1/routes' \ + 2>/dev/null || true) + clients=${payload%%$'\n'*} + routes=${payload#*$'\n'} + client_count=$(grep -o '"client_id"' <<<"$clients" | wc -l || true) + ready_count=$(grep -o '"state":"ready"' <<<"$routes" | wc -l || true) + if [[ "$client_count" -eq 4 && "$ready_count" -eq 4 ]]; then + printf '2x2 route mesh ready: clients=%s ready_routes=%s\n' \ + "$client_count" "$ready_count" + printf '%s\n' "$routes" + exit 0 + fi + sleep 2 +done + +printf 'timed out waiting for 2x2 route mesh after %ss\n' \ + "$timeout_seconds" >&2 +"$E2E_ROOT/scripts/logs.sh" >&2 +exit 1 diff --git a/src/archive_clients/qbittorrent.py b/src/archive_clients/qbittorrent.py index 0b7ba5c..0b0ef50 100644 --- a/src/archive_clients/qbittorrent.py +++ b/src/archive_clients/qbittorrent.py @@ -125,10 +125,14 @@ class QBittorrentReader: headers={"Content-Type": "application/x-www-form-urlencoded"}, ) try: - response = self._read(self._opener.open(call, timeout=self.timeout), 64) + login_response = self._opener.open(call, timeout=self.timeout) + login_status = getattr(login_response, "status", 200) + response = self._read(login_response, 64) except (error.URLError, OSError) as exc: raise QBittorrentError("qBittorrent authentication failed") from exc - if response.strip() != b"Ok.": + if response.strip() != b"Ok." and not ( + login_status == 204 and not response.strip() + ): raise QBittorrentError("qBittorrent authentication failed") self._authenticated = True diff --git a/src/archive_clients/services.py b/src/archive_clients/services.py index 1fa6629..ed853c0 100644 --- a/src/archive_clients/services.py +++ b/src/archive_clients/services.py @@ -55,7 +55,12 @@ def probe_qbittorrent( method="POST", headers={"Content-Type": "application/x-www-form-urlencoded"}, ) - if _read(opener.open(login, timeout=timeout)).strip() != "Ok.": + login_response = opener.open(login, timeout=timeout) + login_status = getattr(login_response, "status", 200) + login_body = _read(login_response).strip() + if login_body != "Ok." and not ( + login_status == 204 and not login_body + ): raise PermissionError("qBittorrent authentication failed") version = _text_get(opener, config.endpoint, "/api/v2/app/version", timeout) api_version = _text_get( diff --git a/src/archive_clients/syncthing.py b/src/archive_clients/syncthing.py index d922739..90014cf 100644 --- a/src/archive_clients/syncthing.py +++ b/src/archive_clients/syncthing.py @@ -325,6 +325,13 @@ def _ack_name(nonce_owner: str, observer: str) -> str: def _atomic_json(path: Path, value: dict[str, Any]) -> None: encoded = json.dumps(value, sort_keys=True, separators=(",", ":")) + try: + if path.read_text(encoding="utf-8") == encoded: + return + except FileNotFoundError: + pass + except (OSError, UnicodeError) as exc: + raise RouteSetupError("route verification file is unreadable") from exc temporary = path.with_name( f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp" ) diff --git a/tests/test_qbittorrent.py b/tests/test_qbittorrent.py index ed46c17..c25e48d 100644 --- a/tests/test_qbittorrent.py +++ b/tests/test_qbittorrent.py @@ -13,8 +13,9 @@ from archive_clients.qbittorrent import QBittorrentReader class _Response: - def __init__(self, value: bytes): + def __init__(self, value: bytes, status: int = 200): self.value = io.BytesIO(value) + self.status = status def read(self, size=-1): return self.value.read(size) @@ -33,7 +34,10 @@ class _Opener: def open(self, call, timeout): self.calls.append(call) - return _Response(next(self.values)) + response = next(self.values) + if isinstance(response, tuple): + return _Response(*response) + return _Response(response) class QBittorrentReaderTests(unittest.TestCase): @@ -71,6 +75,26 @@ class QBittorrentReaderTests(unittest.TestCase): self.assertEqual(resource.summary.resource_id.info_hash_v1_hex, torrent_hash) self.assertIn("hashes=", opener.calls[1]) + def test_current_empty_204_login_is_accepted(self): + responses = [(b"", 204), b"[]"] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + password = root / "password" + password.write_text("secret", encoding="utf-8") + os.chmod(password, 0o600) + config = ServiceConfig( + "http://qb", PurePosixPath("/downloads"), root, + username="admin", password_file=password, + ) + opener = _Opener(responses) + with patch( + "archive_clients.qbittorrent.request.build_opener", + return_value=opener, + ): + resources = QBittorrentReader(config).list_resources() + self.assertEqual(resources, []) + self.assertEqual(len(opener.calls), 2) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_services.py b/tests/test_services.py index 09bbd44..e1699e8 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -11,8 +11,9 @@ from archive_control.v1 import common_pb2 class _Response: - def __init__(self, value: str): + def __init__(self, value: str, status: int = 200): self._source = io.BytesIO(value.encode("utf-8")) + self.status = status def read(self, size: int = -1) -> bytes: return self._source.read(size) @@ -31,7 +32,10 @@ class _Opener: def open(self, call, timeout): self.requests.append((call, timeout)) - return _Response(next(self.responses)) + response = next(self.responses) + if isinstance(response, tuple): + return _Response(*response) + return _Response(response) class ServiceProbeTests(unittest.TestCase): @@ -57,6 +61,26 @@ class ServiceProbeTests(unittest.TestCase): self.assertEqual(result.libtorrent_version, "2.0.11") self.assertIn(b"password=private", opener.requests[0][0].data) + def test_qbittorrent_accepts_current_empty_204_login(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + password = self._secret(root, "password", "private") + config = ServiceConfig( + "http://qb:8080", PurePosixPath("/downloads"), root, + username="admin", password_file=password, + ) + opener = _Opener([ + ("", 204), "v5.2.3", "2.11.4", + '{"libtorrent":"2.0.13"}', + ]) + with patch( + "archive_clients.services.request.build_opener", + return_value=opener, + ): + result = probe_qbittorrent(config) + self.assertEqual(result.state, common_pb2.HEALTH_STATE_HEALTHY) + self.assertEqual(result.version, "v5.2.3") + def test_syncthing_identity_is_normalized(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/tests/test_syncthing.py b/tests/test_syncthing.py index db15619..e569ac7 100644 --- a/tests/test_syncthing.py +++ b/tests/test_syncthing.py @@ -134,6 +134,24 @@ class SyncthingRouteManagerTests(unittest.TestCase): ) self.assertEqual(json.loads(peer_ack.read_text())["nonce"], "peer-nonce") + local_nonce = configured.local_path / ( + ".archive-control-route-nonce.cache-1" + ) + before = (local_nonce.stat(), peer_ack.stat()) + repeated = self.manager.verify( + configured, + self.spec, + "cache-1", + "local-nonce", + time.monotonic() + 1, + ) + after = (local_nonce.stat(), peer_ack.stat()) + self.assertEqual(repeated, (True, True)) + self.assertEqual( + [(item.st_ino, item.st_mtime_ns) for item in before], + [(item.st_ino, item.st_mtime_ns) for item in after], + ) + def test_verification_times_out_without_peer_evidence(self): configured = self.manager.configure(self.spec, time.monotonic() + 1) with self.assertRaises(RouteSetupTimeout):