test: add local 2x2 route topology
This commit is contained in:
@@ -57,5 +57,9 @@ docker run --rm archive-clients-test
|
|||||||
docker build -t sodium/archive-clients:dev .
|
docker build -t sodium/archive-clients:dev .
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The local cross-project route test is under `e2e/`. It brings up isolated
|
||||||
|
2×2 cache/archive node stacks plus the standalone control core and verifies all
|
||||||
|
four eager-mesh routes without Telegram. See `e2e/README.md`.
|
||||||
|
|
||||||
Generated bindings are pinned to archive-control-proto commit
|
Generated bindings are pinned to archive-control-proto commit
|
||||||
`4ec852014dad74606d4078b3ae1aa208c814b033`.
|
`4ec852014dad74606d4078b3ae1aa208c814b033`.
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
*/runtime/
|
||||||
|
*/secrets/
|
||||||
|
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
# Local 2×2 route E2E
|
||||||
|
|
||||||
|
This harness runs one bot-free control stack plus four isolated data-node
|
||||||
|
Compose projects. Every data node contains its own qBittorrent, Syncthing, and
|
||||||
|
archive-client containers and its own bind-mounted config, secrets, state,
|
||||||
|
backup, qB data, and sync roots. The stacks share only the explicitly named
|
||||||
|
`archive-control-e2e` network.
|
||||||
|
|
||||||
|
The initial scenario starts with no route folders. With the control policy set
|
||||||
|
to `eager_mesh`, two cache and two archive registrations must produce four
|
||||||
|
independent, bidirectionally verified Syncthing routes. The assertion talks to
|
||||||
|
the loopback-only test adapter through a curl sidecar sharing the control
|
||||||
|
container's network namespace; Telegram is not initialized.
|
||||||
|
|
||||||
|
Run from the archive-clients checkout:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./e2e/scripts/up.sh
|
||||||
|
./e2e/scripts/logs.sh
|
||||||
|
./e2e/scripts/down.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
`up.sh` defaults `ARCHIVE_CONTROL_SOURCE` to the sibling playground
|
||||||
|
`mogic-bot/mogic-repo` checkout. Set that environment variable to test another
|
||||||
|
control worktree. `E2E_WAIT_SECONDS` overrides the 360-second assertion
|
||||||
|
deadline. `down.sh` removes only the five exact Compose projects and the
|
||||||
|
labelled E2E network; it intentionally retains all bind-mounted runtime state.
|
||||||
|
|
||||||
|
The Syncthing 2.1.2 and LinuxServer qBittorrent multi-platform image indexes are
|
||||||
|
digest-pinned. Runtime secrets are generated with mode 0600 and ignored by
|
||||||
|
Git. The qBittorrent test config limits its authentication bypass to the
|
||||||
|
isolated E2E network and must never be reused for deployment.
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
services:
|
||||||
|
syncthing:
|
||||||
|
image: syncthing/syncthing:2.1.2@sha256:4464f4161dd0251e20d46bb3aec83363db75d80cef1abdd5d5fd4054b04a004d
|
||||||
|
user: "1001:1001"
|
||||||
|
entrypoint: ["/e2e/start-syncthing.sh"]
|
||||||
|
hostname: syncthing-archive-1
|
||||||
|
volumes:
|
||||||
|
- ../common/start-syncthing.sh:/e2e/start-syncthing.sh:ro
|
||||||
|
- ./secrets:/run/secrets:ro
|
||||||
|
- ./runtime/syncthing:/var/syncthing
|
||||||
|
- ./runtime/sync:/sync
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
aliases: [syncthing-archive-1]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -q OK"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
qbittorrent:
|
||||||
|
image: lscr.io/linuxserver/qbittorrent:latest@sha256:b024436f8ca665d16d9a997d26fd27fdf867ee5566ba09f32764e7b2976d3e02
|
||||||
|
environment:
|
||||||
|
PUID: "1001"
|
||||||
|
PGID: "1001"
|
||||||
|
TZ: Etc/UTC
|
||||||
|
WEBUI_PORT: "8080"
|
||||||
|
TORRENTING_PORT: "6881"
|
||||||
|
volumes:
|
||||||
|
- ./runtime/qb-config:/config
|
||||||
|
- ./runtime/qb-data:/downloads
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
aliases: [qb-archive-1]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "curl -fsS -m 2 http://127.0.0.1:8080/api/v2/app/version >/dev/null"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
client:
|
||||||
|
image: archive-clients:e2e
|
||||||
|
build:
|
||||||
|
context: ../..
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
user: "1001:1001"
|
||||||
|
command: ["--config", "/etc/archive-control/client.toml"]
|
||||||
|
depends_on:
|
||||||
|
syncthing:
|
||||||
|
condition: service_healthy
|
||||||
|
qbittorrent:
|
||||||
|
condition: service_healthy
|
||||||
|
volumes:
|
||||||
|
- ./config/client.toml:/etc/archive-control/client.toml:ro
|
||||||
|
- ./secrets:/run/secrets:ro
|
||||||
|
- ./runtime/client-state:/var/lib/archive-control
|
||||||
|
- ./runtime/client-backups:/var/backups/archive-control
|
||||||
|
- ./runtime/qb-data:/data/qb
|
||||||
|
- ./runtime/sync:/data/sync
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
aliases: [client-archive-1]
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
external: true
|
||||||
|
name: archive-control-e2e
|
||||||
|
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
client_id = "archive-1"
|
||||||
|
display_name = "E2E Archive 1"
|
||||||
|
role = "archive"
|
||||||
|
control_endpoint = "ws://control:8765/archive_control"
|
||||||
|
shared_token_file = "/run/secrets/archive_control_token"
|
||||||
|
state_db = "/var/lib/archive-control/client.db"
|
||||||
|
backup_dir = "/var/backups/archive-control"
|
||||||
|
|
||||||
|
[connection]
|
||||||
|
registration_timeout = "5s"
|
||||||
|
heartbeat_interval = "2s"
|
||||||
|
offline_timeout = "10s"
|
||||||
|
reconnect_initial = "1s"
|
||||||
|
reconnect_max = "5s"
|
||||||
|
reconnect_reset_after = "10s"
|
||||||
|
reconnect_jitter = false
|
||||||
|
|
||||||
|
[qbittorrent]
|
||||||
|
endpoint = "http://qb-archive-1:8080"
|
||||||
|
username = "admin"
|
||||||
|
password_file = "/run/secrets/qb_password"
|
||||||
|
api_root = "/downloads"
|
||||||
|
local_root = "/data/qb"
|
||||||
|
|
||||||
|
[syncthing]
|
||||||
|
endpoint = "http://syncthing-archive-1:8384"
|
||||||
|
api_key_file = "/run/secrets/syncthing_api_key"
|
||||||
|
api_root = "/sync"
|
||||||
|
local_root = "/data/sync"
|
||||||
|
advertised_addresses = ["tcp://syncthing-archive-1:22000"]
|
||||||
|
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
services:
|
||||||
|
syncthing:
|
||||||
|
image: syncthing/syncthing:2.1.2@sha256:4464f4161dd0251e20d46bb3aec83363db75d80cef1abdd5d5fd4054b04a004d
|
||||||
|
user: "1001:1001"
|
||||||
|
entrypoint: ["/e2e/start-syncthing.sh"]
|
||||||
|
hostname: syncthing-archive-2
|
||||||
|
volumes:
|
||||||
|
- ../common/start-syncthing.sh:/e2e/start-syncthing.sh:ro
|
||||||
|
- ./secrets:/run/secrets:ro
|
||||||
|
- ./runtime/syncthing:/var/syncthing
|
||||||
|
- ./runtime/sync:/sync
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
aliases: [syncthing-archive-2]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -q OK"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
qbittorrent:
|
||||||
|
image: lscr.io/linuxserver/qbittorrent:latest@sha256:b024436f8ca665d16d9a997d26fd27fdf867ee5566ba09f32764e7b2976d3e02
|
||||||
|
environment:
|
||||||
|
PUID: "1001"
|
||||||
|
PGID: "1001"
|
||||||
|
TZ: Etc/UTC
|
||||||
|
WEBUI_PORT: "8080"
|
||||||
|
TORRENTING_PORT: "6881"
|
||||||
|
volumes:
|
||||||
|
- ./runtime/qb-config:/config
|
||||||
|
- ./runtime/qb-data:/downloads
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
aliases: [qb-archive-2]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "curl -fsS -m 2 http://127.0.0.1:8080/api/v2/app/version >/dev/null"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
client:
|
||||||
|
image: archive-clients:e2e
|
||||||
|
build:
|
||||||
|
context: ../..
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
user: "1001:1001"
|
||||||
|
command: ["--config", "/etc/archive-control/client.toml"]
|
||||||
|
depends_on:
|
||||||
|
syncthing:
|
||||||
|
condition: service_healthy
|
||||||
|
qbittorrent:
|
||||||
|
condition: service_healthy
|
||||||
|
volumes:
|
||||||
|
- ./config/client.toml:/etc/archive-control/client.toml:ro
|
||||||
|
- ./secrets:/run/secrets:ro
|
||||||
|
- ./runtime/client-state:/var/lib/archive-control
|
||||||
|
- ./runtime/client-backups:/var/backups/archive-control
|
||||||
|
- ./runtime/qb-data:/data/qb
|
||||||
|
- ./runtime/sync:/data/sync
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
aliases: [client-archive-2]
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
external: true
|
||||||
|
name: archive-control-e2e
|
||||||
|
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
client_id = "archive-2"
|
||||||
|
display_name = "E2E Archive 2"
|
||||||
|
role = "archive"
|
||||||
|
control_endpoint = "ws://control:8765/archive_control"
|
||||||
|
shared_token_file = "/run/secrets/archive_control_token"
|
||||||
|
state_db = "/var/lib/archive-control/client.db"
|
||||||
|
backup_dir = "/var/backups/archive-control"
|
||||||
|
|
||||||
|
[connection]
|
||||||
|
registration_timeout = "5s"
|
||||||
|
heartbeat_interval = "2s"
|
||||||
|
offline_timeout = "10s"
|
||||||
|
reconnect_initial = "1s"
|
||||||
|
reconnect_max = "5s"
|
||||||
|
reconnect_reset_after = "10s"
|
||||||
|
reconnect_jitter = false
|
||||||
|
|
||||||
|
[qbittorrent]
|
||||||
|
endpoint = "http://qb-archive-2:8080"
|
||||||
|
username = "admin"
|
||||||
|
password_file = "/run/secrets/qb_password"
|
||||||
|
api_root = "/downloads"
|
||||||
|
local_root = "/data/qb"
|
||||||
|
|
||||||
|
[syncthing]
|
||||||
|
endpoint = "http://syncthing-archive-2:8384"
|
||||||
|
api_key_file = "/run/secrets/syncthing_api_key"
|
||||||
|
api_root = "/sync"
|
||||||
|
local_root = "/data/sync"
|
||||||
|
advertised_addresses = ["tcp://syncthing-archive-2:22000"]
|
||||||
|
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
services:
|
||||||
|
syncthing:
|
||||||
|
image: syncthing/syncthing:2.1.2@sha256:4464f4161dd0251e20d46bb3aec83363db75d80cef1abdd5d5fd4054b04a004d
|
||||||
|
user: "1001:1001"
|
||||||
|
entrypoint: ["/e2e/start-syncthing.sh"]
|
||||||
|
hostname: syncthing-cache-1
|
||||||
|
volumes:
|
||||||
|
- ../common/start-syncthing.sh:/e2e/start-syncthing.sh:ro
|
||||||
|
- ./secrets:/run/secrets:ro
|
||||||
|
- ./runtime/syncthing:/var/syncthing
|
||||||
|
- ./runtime/sync:/sync
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
aliases: [syncthing-cache-1]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -q OK"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
qbittorrent:
|
||||||
|
image: lscr.io/linuxserver/qbittorrent:latest@sha256:b024436f8ca665d16d9a997d26fd27fdf867ee5566ba09f32764e7b2976d3e02
|
||||||
|
environment:
|
||||||
|
PUID: "1001"
|
||||||
|
PGID: "1001"
|
||||||
|
TZ: Etc/UTC
|
||||||
|
WEBUI_PORT: "8080"
|
||||||
|
TORRENTING_PORT: "6881"
|
||||||
|
volumes:
|
||||||
|
- ./runtime/qb-config:/config
|
||||||
|
- ./runtime/qb-data:/downloads
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
aliases: [qb-cache-1]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "curl -fsS -m 2 http://127.0.0.1:8080/api/v2/app/version >/dev/null"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
client:
|
||||||
|
image: archive-clients:e2e
|
||||||
|
build:
|
||||||
|
context: ../..
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
user: "1001:1001"
|
||||||
|
command: ["--config", "/etc/archive-control/client.toml"]
|
||||||
|
depends_on:
|
||||||
|
syncthing:
|
||||||
|
condition: service_healthy
|
||||||
|
qbittorrent:
|
||||||
|
condition: service_healthy
|
||||||
|
volumes:
|
||||||
|
- ./config/client.toml:/etc/archive-control/client.toml:ro
|
||||||
|
- ./secrets:/run/secrets:ro
|
||||||
|
- ./runtime/client-state:/var/lib/archive-control
|
||||||
|
- ./runtime/client-backups:/var/backups/archive-control
|
||||||
|
- ./runtime/qb-data:/data/qb
|
||||||
|
- ./runtime/sync:/data/sync
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
aliases: [client-cache-1]
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
external: true
|
||||||
|
name: archive-control-e2e
|
||||||
|
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
client_id = "cache-1"
|
||||||
|
display_name = "E2E Cache 1"
|
||||||
|
role = "cache"
|
||||||
|
control_endpoint = "ws://control:8765/archive_control"
|
||||||
|
shared_token_file = "/run/secrets/archive_control_token"
|
||||||
|
state_db = "/var/lib/archive-control/client.db"
|
||||||
|
backup_dir = "/var/backups/archive-control"
|
||||||
|
|
||||||
|
[connection]
|
||||||
|
registration_timeout = "5s"
|
||||||
|
heartbeat_interval = "2s"
|
||||||
|
offline_timeout = "10s"
|
||||||
|
reconnect_initial = "1s"
|
||||||
|
reconnect_max = "5s"
|
||||||
|
reconnect_reset_after = "10s"
|
||||||
|
reconnect_jitter = false
|
||||||
|
|
||||||
|
[qbittorrent]
|
||||||
|
endpoint = "http://qb-cache-1:8080"
|
||||||
|
username = "admin"
|
||||||
|
password_file = "/run/secrets/qb_password"
|
||||||
|
api_root = "/downloads"
|
||||||
|
local_root = "/data/qb"
|
||||||
|
|
||||||
|
[syncthing]
|
||||||
|
endpoint = "http://syncthing-cache-1:8384"
|
||||||
|
api_key_file = "/run/secrets/syncthing_api_key"
|
||||||
|
api_root = "/sync"
|
||||||
|
local_root = "/data/sync"
|
||||||
|
advertised_addresses = ["tcp://syncthing-cache-1:22000"]
|
||||||
|
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
services:
|
||||||
|
syncthing:
|
||||||
|
image: syncthing/syncthing:2.1.2@sha256:4464f4161dd0251e20d46bb3aec83363db75d80cef1abdd5d5fd4054b04a004d
|
||||||
|
user: "1001:1001"
|
||||||
|
entrypoint: ["/e2e/start-syncthing.sh"]
|
||||||
|
hostname: syncthing-cache-2
|
||||||
|
volumes:
|
||||||
|
- ../common/start-syncthing.sh:/e2e/start-syncthing.sh:ro
|
||||||
|
- ./secrets:/run/secrets:ro
|
||||||
|
- ./runtime/syncthing:/var/syncthing
|
||||||
|
- ./runtime/sync:/sync
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
aliases: [syncthing-cache-2]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -q OK"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
qbittorrent:
|
||||||
|
image: lscr.io/linuxserver/qbittorrent:latest@sha256:b024436f8ca665d16d9a997d26fd27fdf867ee5566ba09f32764e7b2976d3e02
|
||||||
|
environment:
|
||||||
|
PUID: "1001"
|
||||||
|
PGID: "1001"
|
||||||
|
TZ: Etc/UTC
|
||||||
|
WEBUI_PORT: "8080"
|
||||||
|
TORRENTING_PORT: "6881"
|
||||||
|
volumes:
|
||||||
|
- ./runtime/qb-config:/config
|
||||||
|
- ./runtime/qb-data:/downloads
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
aliases: [qb-cache-2]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "curl -fsS -m 2 http://127.0.0.1:8080/api/v2/app/version >/dev/null"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
client:
|
||||||
|
image: archive-clients:e2e
|
||||||
|
build:
|
||||||
|
context: ../..
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
user: "1001:1001"
|
||||||
|
command: ["--config", "/etc/archive-control/client.toml"]
|
||||||
|
depends_on:
|
||||||
|
syncthing:
|
||||||
|
condition: service_healthy
|
||||||
|
qbittorrent:
|
||||||
|
condition: service_healthy
|
||||||
|
volumes:
|
||||||
|
- ./config/client.toml:/etc/archive-control/client.toml:ro
|
||||||
|
- ./secrets:/run/secrets:ro
|
||||||
|
- ./runtime/client-state:/var/lib/archive-control
|
||||||
|
- ./runtime/client-backups:/var/backups/archive-control
|
||||||
|
- ./runtime/qb-data:/data/qb
|
||||||
|
- ./runtime/sync:/data/sync
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
aliases: [client-cache-2]
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
external: true
|
||||||
|
name: archive-control-e2e
|
||||||
|
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
client_id = "cache-2"
|
||||||
|
display_name = "E2E Cache 2"
|
||||||
|
role = "cache"
|
||||||
|
control_endpoint = "ws://control:8765/archive_control"
|
||||||
|
shared_token_file = "/run/secrets/archive_control_token"
|
||||||
|
state_db = "/var/lib/archive-control/client.db"
|
||||||
|
backup_dir = "/var/backups/archive-control"
|
||||||
|
|
||||||
|
[connection]
|
||||||
|
registration_timeout = "5s"
|
||||||
|
heartbeat_interval = "2s"
|
||||||
|
offline_timeout = "10s"
|
||||||
|
reconnect_initial = "1s"
|
||||||
|
reconnect_max = "5s"
|
||||||
|
reconnect_reset_after = "10s"
|
||||||
|
reconnect_jitter = false
|
||||||
|
|
||||||
|
[qbittorrent]
|
||||||
|
endpoint = "http://qb-cache-2:8080"
|
||||||
|
username = "admin"
|
||||||
|
password_file = "/run/secrets/qb_password"
|
||||||
|
api_root = "/downloads"
|
||||||
|
local_root = "/data/qb"
|
||||||
|
|
||||||
|
[syncthing]
|
||||||
|
endpoint = "http://syncthing-cache-2:8384"
|
||||||
|
api_key_file = "/run/secrets/syncthing_api_key"
|
||||||
|
api_root = "/sync"
|
||||||
|
local_root = "/data/sync"
|
||||||
|
advertised_addresses = ["tcp://syncthing-cache-2:22000"]
|
||||||
|
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
[LegalNotice]
|
||||||
|
Accepted=true
|
||||||
|
|
||||||
|
[Preferences]
|
||||||
|
Downloads\SavePath=/downloads/
|
||||||
|
WebUI\Address=*
|
||||||
|
WebUI\AlternativeUIEnabled=false
|
||||||
|
WebUI\AuthSubnetWhitelist=0.0.0.0/0
|
||||||
|
WebUI\AuthSubnetWhitelistEnabled=true
|
||||||
|
WebUI\CSRFProtection=false
|
||||||
|
WebUI\HostHeaderValidation=false
|
||||||
|
WebUI\Password_PBKDF2="@ByteArray(ARQ77eY1NUZaQsuDHbIMCA==:0WMRkYTUWVT9wVvdDtHAjU9b3b7uB8NR1Gur2hmQCvCDpm39Q+PsJRJPaCU51dEiz+dTzh8qbPsL8WkFljQYFQ==)"
|
||||||
|
WebUI\Port=8080
|
||||||
|
WebUI\Username=admin
|
||||||
Executable
+19
@@ -0,0 +1,19 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
home=/var/syncthing
|
||||||
|
api_key=$(cat /run/secrets/syncthing_api_key)
|
||||||
|
|
||||||
|
if [ ! -f "$home/config.xml" ]; then
|
||||||
|
syncthing generate --home="$home" --no-port-probing
|
||||||
|
fi
|
||||||
|
|
||||||
|
exec syncthing serve \
|
||||||
|
--home="$home" \
|
||||||
|
--gui-address=http://0.0.0.0:8384 \
|
||||||
|
--gui-apikey="$api_key" \
|
||||||
|
--no-browser \
|
||||||
|
--no-port-probing \
|
||||||
|
--no-restart \
|
||||||
|
--no-upgrade
|
||||||
|
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
services:
|
||||||
|
control:
|
||||||
|
image: archive-control-standalone:e2e
|
||||||
|
build:
|
||||||
|
context: ${ARCHIVE_CONTROL_SOURCE:?set ARCHIVE_CONTROL_SOURCE to the mogic-bot checkout}
|
||||||
|
dockerfile: Dockerfile.archive-control
|
||||||
|
command: ["--config", "/etc/archive-control/config.json"]
|
||||||
|
user: "1001:1001"
|
||||||
|
volumes:
|
||||||
|
- ./config/config.json:/etc/archive-control/config.json:ro
|
||||||
|
- ./secrets:/run/secrets:ro
|
||||||
|
- ./runtime/state:/var/lib/archive-control
|
||||||
|
- ./runtime/backups:/var/backups/archive-control
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
aliases: [control]
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
curl:
|
||||||
|
image: curlimages/curl:8.16.0
|
||||||
|
network_mode: service:control
|
||||||
|
profiles: [tools]
|
||||||
|
depends_on: [control]
|
||||||
|
|
||||||
|
networks:
|
||||||
|
archive-control-e2e:
|
||||||
|
external: true
|
||||||
|
name: archive-control-e2e
|
||||||
|
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
{
|
||||||
|
"archive_control": {
|
||||||
|
"enabled": true,
|
||||||
|
"listen": "0.0.0.0:8765",
|
||||||
|
"websocket_path": "/archive_control",
|
||||||
|
"shared_token_file": "/run/secrets/archive_control_token",
|
||||||
|
"database": "/var/lib/archive-control/control.db",
|
||||||
|
"backup_dir": "/var/backups/archive-control",
|
||||||
|
"registration_timeout": "5s",
|
||||||
|
"heartbeat_interval": "2s",
|
||||||
|
"offline_timeout": "10s",
|
||||||
|
"command_ack_timeout": "2s",
|
||||||
|
"command_max_attempts": 3,
|
||||||
|
"route_policy": "eager_mesh",
|
||||||
|
"route_setup_timeout": "5m",
|
||||||
|
"test_http": {
|
||||||
|
"enabled": true,
|
||||||
|
"listen": "127.0.0.1:18081",
|
||||||
|
"max_body_bytes": 1048576
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Executable
+19
@@ -0,0 +1,19 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
source "$(dirname "$0")/lib.sh"
|
||||||
|
|
||||||
|
for ((index=${#E2E_NODES[@]} - 1; index >= 0; index--)); do
|
||||||
|
compose_node "${E2E_NODES[index]}" down --remove-orphans
|
||||||
|
done
|
||||||
|
compose_control down --remove-orphans
|
||||||
|
|
||||||
|
label=$(docker network inspect \
|
||||||
|
-f '{{ index .Labels "archive-control.e2e" }}' \
|
||||||
|
"$E2E_NETWORK" 2>/dev/null || true)
|
||||||
|
if [[ "$label" == "true" ]]; then
|
||||||
|
docker network rm "$E2E_NETWORK" >/dev/null
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf 'E2E containers stopped; bind-mounted runtime state was retained.\n'
|
||||||
|
|
||||||
Executable
+29
@@ -0,0 +1,29 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
E2E_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||||
|
E2E_NETWORK=archive-control-e2e
|
||||||
|
E2E_NODES=(cache-1 cache-2 archive-1 archive-2)
|
||||||
|
if [[ -z "${ARCHIVE_CONTROL_SOURCE:-}" ]]; then
|
||||||
|
control_candidate="$E2E_ROOT/../../../mogic-bot/mogic-repo"
|
||||||
|
if [[ -d "$control_candidate" ]]; then
|
||||||
|
ARCHIVE_CONTROL_SOURCE=$(cd "$control_candidate" && pwd)
|
||||||
|
else
|
||||||
|
ARCHIVE_CONTROL_SOURCE=$control_candidate
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
export ARCHIVE_CONTROL_SOURCE
|
||||||
|
|
||||||
|
compose_control() {
|
||||||
|
docker compose \
|
||||||
|
-p archive-e2e-control \
|
||||||
|
-f "$E2E_ROOT/control/compose.yaml" "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
compose_node() {
|
||||||
|
local node=$1
|
||||||
|
shift
|
||||||
|
docker compose \
|
||||||
|
-p "archive-e2e-${node}" \
|
||||||
|
-f "$E2E_ROOT/${node}/compose.yaml" "$@"
|
||||||
|
}
|
||||||
Executable
+10
@@ -0,0 +1,10 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
source "$(dirname "$0")/lib.sh"
|
||||||
|
|
||||||
|
compose_control logs --no-color --tail=200 || true
|
||||||
|
for node in "${E2E_NODES[@]}"; do
|
||||||
|
compose_node "$node" logs --no-color --tail=200 || true
|
||||||
|
done
|
||||||
|
|
||||||
Executable
+45
@@ -0,0 +1,45 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
source "$(dirname "$0")/lib.sh"
|
||||||
|
umask 077
|
||||||
|
|
||||||
|
prepare_secret() {
|
||||||
|
local path=$1
|
||||||
|
local value=$2
|
||||||
|
if [[ ! -f "$path" ]]; then
|
||||||
|
printf '%s\n' "$value" >"$path"
|
||||||
|
fi
|
||||||
|
chmod 600 "$path"
|
||||||
|
}
|
||||||
|
|
||||||
|
mkdir -p \
|
||||||
|
"$E2E_ROOT/control/runtime/state" \
|
||||||
|
"$E2E_ROOT/control/runtime/backups" \
|
||||||
|
"$E2E_ROOT/control/secrets"
|
||||||
|
prepare_secret \
|
||||||
|
"$E2E_ROOT/control/secrets/archive_control_token" \
|
||||||
|
"archive-control-e2e-token"
|
||||||
|
|
||||||
|
for node in "${E2E_NODES[@]}"; do
|
||||||
|
node_root="$E2E_ROOT/$node"
|
||||||
|
mkdir -p \
|
||||||
|
"$node_root/secrets" \
|
||||||
|
"$node_root/runtime/syncthing" \
|
||||||
|
"$node_root/runtime/sync" \
|
||||||
|
"$node_root/runtime/qb-config/qBittorrent" \
|
||||||
|
"$node_root/runtime/qb-data" \
|
||||||
|
"$node_root/runtime/client-state" \
|
||||||
|
"$node_root/runtime/client-backups"
|
||||||
|
prepare_secret \
|
||||||
|
"$node_root/secrets/archive_control_token" \
|
||||||
|
"archive-control-e2e-token"
|
||||||
|
prepare_secret \
|
||||||
|
"$node_root/secrets/syncthing_api_key" \
|
||||||
|
"archive-control-e2e-${node}-syncthing-key"
|
||||||
|
prepare_secret "$node_root/secrets/qb_password" "adminadmin"
|
||||||
|
qb_config="$node_root/runtime/qb-config/qBittorrent/qBittorrent.conf"
|
||||||
|
if [[ ! -f "$qb_config" ]]; then
|
||||||
|
cp "$E2E_ROOT/common/qBittorrent.conf" "$qb_config"
|
||||||
|
fi
|
||||||
|
done
|
||||||
Executable
+23
@@ -0,0 +1,23 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
source "$(dirname "$0")/lib.sh"
|
||||||
|
"$E2E_ROOT/scripts/prepare.sh"
|
||||||
|
|
||||||
|
if ! docker network inspect "$E2E_NETWORK" >/dev/null 2>&1; then
|
||||||
|
docker network create \
|
||||||
|
--label archive-control.e2e=true \
|
||||||
|
"$E2E_NETWORK" >/dev/null
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! -f "$ARCHIVE_CONTROL_SOURCE/Dockerfile.archive-control" ]]; then
|
||||||
|
printf 'control source is missing Dockerfile.archive-control: %s\n' \
|
||||||
|
"$ARCHIVE_CONTROL_SOURCE" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
compose_control up -d --build control
|
||||||
|
for node in "${E2E_NODES[@]}"; do
|
||||||
|
compose_node "$node" up -d --build
|
||||||
|
done
|
||||||
|
"$E2E_ROOT/scripts/wait-routes.sh"
|
||||||
Executable
+29
@@ -0,0 +1,29 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
source "$(dirname "$0")/lib.sh"
|
||||||
|
timeout_seconds=${E2E_WAIT_SECONDS:-360}
|
||||||
|
deadline=$((SECONDS + timeout_seconds))
|
||||||
|
|
||||||
|
while (( SECONDS < deadline )); do
|
||||||
|
payload=$(compose_control run --rm --no-deps \
|
||||||
|
--entrypoint /bin/sh curl -c \
|
||||||
|
'curl --max-time 5 -fsS http://127.0.0.1:18081/test/v1/clients; printf "\n"; curl --max-time 5 -fsS http://127.0.0.1:18081/test/v1/routes' \
|
||||||
|
2>/dev/null || true)
|
||||||
|
clients=${payload%%$'\n'*}
|
||||||
|
routes=${payload#*$'\n'}
|
||||||
|
client_count=$(grep -o '"client_id"' <<<"$clients" | wc -l || true)
|
||||||
|
ready_count=$(grep -o '"state":"ready"' <<<"$routes" | wc -l || true)
|
||||||
|
if [[ "$client_count" -eq 4 && "$ready_count" -eq 4 ]]; then
|
||||||
|
printf '2x2 route mesh ready: clients=%s ready_routes=%s\n' \
|
||||||
|
"$client_count" "$ready_count"
|
||||||
|
printf '%s\n' "$routes"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
|
||||||
|
printf 'timed out waiting for 2x2 route mesh after %ss\n' \
|
||||||
|
"$timeout_seconds" >&2
|
||||||
|
"$E2E_ROOT/scripts/logs.sh" >&2
|
||||||
|
exit 1
|
||||||
@@ -125,10 +125,14 @@ class QBittorrentReader:
|
|||||||
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
response = self._read(self._opener.open(call, timeout=self.timeout), 64)
|
login_response = self._opener.open(call, timeout=self.timeout)
|
||||||
|
login_status = getattr(login_response, "status", 200)
|
||||||
|
response = self._read(login_response, 64)
|
||||||
except (error.URLError, OSError) as exc:
|
except (error.URLError, OSError) as exc:
|
||||||
raise QBittorrentError("qBittorrent authentication failed") from exc
|
raise QBittorrentError("qBittorrent authentication failed") from exc
|
||||||
if response.strip() != b"Ok.":
|
if response.strip() != b"Ok." and not (
|
||||||
|
login_status == 204 and not response.strip()
|
||||||
|
):
|
||||||
raise QBittorrentError("qBittorrent authentication failed")
|
raise QBittorrentError("qBittorrent authentication failed")
|
||||||
self._authenticated = True
|
self._authenticated = True
|
||||||
|
|
||||||
|
|||||||
@@ -55,7 +55,12 @@ def probe_qbittorrent(
|
|||||||
method="POST",
|
method="POST",
|
||||||
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||||
)
|
)
|
||||||
if _read(opener.open(login, timeout=timeout)).strip() != "Ok.":
|
login_response = opener.open(login, timeout=timeout)
|
||||||
|
login_status = getattr(login_response, "status", 200)
|
||||||
|
login_body = _read(login_response).strip()
|
||||||
|
if login_body != "Ok." and not (
|
||||||
|
login_status == 204 and not login_body
|
||||||
|
):
|
||||||
raise PermissionError("qBittorrent authentication failed")
|
raise PermissionError("qBittorrent authentication failed")
|
||||||
version = _text_get(opener, config.endpoint, "/api/v2/app/version", timeout)
|
version = _text_get(opener, config.endpoint, "/api/v2/app/version", timeout)
|
||||||
api_version = _text_get(
|
api_version = _text_get(
|
||||||
|
|||||||
@@ -325,6 +325,13 @@ def _ack_name(nonce_owner: str, observer: str) -> str:
|
|||||||
|
|
||||||
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
|
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
|
||||||
encoded = json.dumps(value, sort_keys=True, separators=(",", ":"))
|
encoded = json.dumps(value, sort_keys=True, separators=(",", ":"))
|
||||||
|
try:
|
||||||
|
if path.read_text(encoding="utf-8") == encoded:
|
||||||
|
return
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass
|
||||||
|
except (OSError, UnicodeError) as exc:
|
||||||
|
raise RouteSetupError("route verification file is unreadable") from exc
|
||||||
temporary = path.with_name(
|
temporary = path.with_name(
|
||||||
f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp"
|
f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -13,8 +13,9 @@ from archive_clients.qbittorrent import QBittorrentReader
|
|||||||
|
|
||||||
|
|
||||||
class _Response:
|
class _Response:
|
||||||
def __init__(self, value: bytes):
|
def __init__(self, value: bytes, status: int = 200):
|
||||||
self.value = io.BytesIO(value)
|
self.value = io.BytesIO(value)
|
||||||
|
self.status = status
|
||||||
|
|
||||||
def read(self, size=-1):
|
def read(self, size=-1):
|
||||||
return self.value.read(size)
|
return self.value.read(size)
|
||||||
@@ -33,7 +34,10 @@ class _Opener:
|
|||||||
|
|
||||||
def open(self, call, timeout):
|
def open(self, call, timeout):
|
||||||
self.calls.append(call)
|
self.calls.append(call)
|
||||||
return _Response(next(self.values))
|
response = next(self.values)
|
||||||
|
if isinstance(response, tuple):
|
||||||
|
return _Response(*response)
|
||||||
|
return _Response(response)
|
||||||
|
|
||||||
|
|
||||||
class QBittorrentReaderTests(unittest.TestCase):
|
class QBittorrentReaderTests(unittest.TestCase):
|
||||||
@@ -71,6 +75,26 @@ class QBittorrentReaderTests(unittest.TestCase):
|
|||||||
self.assertEqual(resource.summary.resource_id.info_hash_v1_hex, torrent_hash)
|
self.assertEqual(resource.summary.resource_id.info_hash_v1_hex, torrent_hash)
|
||||||
self.assertIn("hashes=", opener.calls[1])
|
self.assertIn("hashes=", opener.calls[1])
|
||||||
|
|
||||||
|
def test_current_empty_204_login_is_accepted(self):
|
||||||
|
responses = [(b"", 204), b"[]"]
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
password = root / "password"
|
||||||
|
password.write_text("secret", encoding="utf-8")
|
||||||
|
os.chmod(password, 0o600)
|
||||||
|
config = ServiceConfig(
|
||||||
|
"http://qb", PurePosixPath("/downloads"), root,
|
||||||
|
username="admin", password_file=password,
|
||||||
|
)
|
||||||
|
opener = _Opener(responses)
|
||||||
|
with patch(
|
||||||
|
"archive_clients.qbittorrent.request.build_opener",
|
||||||
|
return_value=opener,
|
||||||
|
):
|
||||||
|
resources = QBittorrentReader(config).list_resources()
|
||||||
|
self.assertEqual(resources, [])
|
||||||
|
self.assertEqual(len(opener.calls), 2)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
+26
-2
@@ -11,8 +11,9 @@ from archive_control.v1 import common_pb2
|
|||||||
|
|
||||||
|
|
||||||
class _Response:
|
class _Response:
|
||||||
def __init__(self, value: str):
|
def __init__(self, value: str, status: int = 200):
|
||||||
self._source = io.BytesIO(value.encode("utf-8"))
|
self._source = io.BytesIO(value.encode("utf-8"))
|
||||||
|
self.status = status
|
||||||
|
|
||||||
def read(self, size: int = -1) -> bytes:
|
def read(self, size: int = -1) -> bytes:
|
||||||
return self._source.read(size)
|
return self._source.read(size)
|
||||||
@@ -31,7 +32,10 @@ class _Opener:
|
|||||||
|
|
||||||
def open(self, call, timeout):
|
def open(self, call, timeout):
|
||||||
self.requests.append((call, timeout))
|
self.requests.append((call, timeout))
|
||||||
return _Response(next(self.responses))
|
response = next(self.responses)
|
||||||
|
if isinstance(response, tuple):
|
||||||
|
return _Response(*response)
|
||||||
|
return _Response(response)
|
||||||
|
|
||||||
|
|
||||||
class ServiceProbeTests(unittest.TestCase):
|
class ServiceProbeTests(unittest.TestCase):
|
||||||
@@ -57,6 +61,26 @@ class ServiceProbeTests(unittest.TestCase):
|
|||||||
self.assertEqual(result.libtorrent_version, "2.0.11")
|
self.assertEqual(result.libtorrent_version, "2.0.11")
|
||||||
self.assertIn(b"password=private", opener.requests[0][0].data)
|
self.assertIn(b"password=private", opener.requests[0][0].data)
|
||||||
|
|
||||||
|
def test_qbittorrent_accepts_current_empty_204_login(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
password = self._secret(root, "password", "private")
|
||||||
|
config = ServiceConfig(
|
||||||
|
"http://qb:8080", PurePosixPath("/downloads"), root,
|
||||||
|
username="admin", password_file=password,
|
||||||
|
)
|
||||||
|
opener = _Opener([
|
||||||
|
("", 204), "v5.2.3", "2.11.4",
|
||||||
|
'{"libtorrent":"2.0.13"}',
|
||||||
|
])
|
||||||
|
with patch(
|
||||||
|
"archive_clients.services.request.build_opener",
|
||||||
|
return_value=opener,
|
||||||
|
):
|
||||||
|
result = probe_qbittorrent(config)
|
||||||
|
self.assertEqual(result.state, common_pb2.HEALTH_STATE_HEALTHY)
|
||||||
|
self.assertEqual(result.version, "v5.2.3")
|
||||||
|
|
||||||
def test_syncthing_identity_is_normalized(self):
|
def test_syncthing_identity_is_normalized(self):
|
||||||
with tempfile.TemporaryDirectory() as directory:
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
root = Path(directory)
|
root = Path(directory)
|
||||||
|
|||||||
@@ -134,6 +134,24 @@ class SyncthingRouteManagerTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
self.assertEqual(json.loads(peer_ack.read_text())["nonce"], "peer-nonce")
|
self.assertEqual(json.loads(peer_ack.read_text())["nonce"], "peer-nonce")
|
||||||
|
|
||||||
|
local_nonce = configured.local_path / (
|
||||||
|
".archive-control-route-nonce.cache-1"
|
||||||
|
)
|
||||||
|
before = (local_nonce.stat(), peer_ack.stat())
|
||||||
|
repeated = self.manager.verify(
|
||||||
|
configured,
|
||||||
|
self.spec,
|
||||||
|
"cache-1",
|
||||||
|
"local-nonce",
|
||||||
|
time.monotonic() + 1,
|
||||||
|
)
|
||||||
|
after = (local_nonce.stat(), peer_ack.stat())
|
||||||
|
self.assertEqual(repeated, (True, True))
|
||||||
|
self.assertEqual(
|
||||||
|
[(item.st_ino, item.st_mtime_ns) for item in before],
|
||||||
|
[(item.st_ino, item.st_mtime_ns) for item in after],
|
||||||
|
)
|
||||||
|
|
||||||
def test_verification_times_out_without_peer_evidence(self):
|
def test_verification_times_out_without_peer_evidence(self):
|
||||||
configured = self.manager.configure(self.spec, time.monotonic() + 1)
|
configured = self.manager.configure(self.spec, time.monotonic() + 1)
|
||||||
with self.assertRaises(RouteSetupTimeout):
|
with self.assertRaises(RouteSetupTimeout):
|
||||||
|
|||||||
Reference in New Issue
Block a user