Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ea35ba2758 | ||
|
|
5e5e2f9095 | ||
|
|
0b11e2a3a2 | ||
|
|
1009defc46 | ||
|
|
8ed8e75447 |
@@ -2,7 +2,7 @@ name: archive-control-archive
|
|||||||
|
|
||||||
services:
|
services:
|
||||||
archive-client:
|
archive-client:
|
||||||
image: sodium/archive-clients:v0.1.13
|
image: sodium/archive-clients:v0.1.14
|
||||||
user: "1000:1000"
|
user: "1000:1000"
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
command: ["--config", "/etc/archive-control/client.toml"]
|
command: ["--config", "/etc/archive-control/client.toml"]
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ name: archive-control-cache
|
|||||||
|
|
||||||
services:
|
services:
|
||||||
archive-client:
|
archive-client:
|
||||||
image: sodium/archive-clients:v0.1.13
|
image: sodium/archive-clients:v0.1.14
|
||||||
user: "1001:1001"
|
user: "1001:1001"
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
network_mode: host
|
network_mode: host
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ name: archive-control-cache
|
|||||||
|
|
||||||
services:
|
services:
|
||||||
archive-client:
|
archive-client:
|
||||||
image: sodium/archive-clients:v0.1.13
|
image: sodium/archive-clients:v0.1.14
|
||||||
user: "1001:1001"
|
user: "1001:1001"
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
network_mode: host
|
network_mode: host
|
||||||
|
|||||||
+5
-2
@@ -35,8 +35,11 @@ expand these rules but must not contradict them.
|
|||||||
manifests, and observed external state.
|
manifests, and observed external state.
|
||||||
- SQLite backups use the online backup API. Defaults are every six hours, 12
|
- SQLite backups use the online backup API. Defaults are every six hours, 12
|
||||||
recent, 14 daily, and eight weekly copies, plus pre/post-migration backups.
|
recent, 14 daily, and eight weekly copies, plus pre/post-migration backups.
|
||||||
- Default concurrency is one active data-moving job per client and per route.
|
- Default concurrency is unrestricted: every eligible queued job is admitted
|
||||||
Disjoint node pairs may run concurrently. Queueing is durable FIFO.
|
in one scheduler pass and independent jobs execute concurrently on a client.
|
||||||
|
Physical qBittorrent, Syncthing, disk, and network capacity are therefore
|
||||||
|
the natural limit. `enforce_concurrency_limits=true` restores the optional
|
||||||
|
legacy per-client/per-route gates. Commands for one job remain serialized.
|
||||||
- A queued job owns a per-resource reservation. Cancelling it removes only the
|
- A queued job owns a per-resource reservation. Cancelling it removes only the
|
||||||
queued record/reservation and never sends cleanup commands.
|
queued record/reservation and never sends cleanup commands.
|
||||||
- Connectivity or transfer stalls wait indefinitely. A configurable 30-minute
|
- Connectivity or transfer stalls wait indefinitely. A configurable 30-minute
|
||||||
|
|||||||
@@ -44,6 +44,9 @@ command_max_attempts = 3 # Total sends, including the initial attempt.
|
|||||||
stall_after = "30m" # Warning state only; jobs continue waiting.
|
stall_after = "30m" # Warning state only; jobs continue waiting.
|
||||||
route_policy = "on_demand" # Alternative: eager_mesh.
|
route_policy = "on_demand" # Alternative: eager_mesh.
|
||||||
route_setup_timeout = "30m"
|
route_setup_timeout = "30m"
|
||||||
|
# Default: allow all eligible jobs; disk/network/service capacity is the limit.
|
||||||
|
enforce_concurrency_limits = false
|
||||||
|
# Used only when enforce_concurrency_limits is true.
|
||||||
max_active_per_client = 1
|
max_active_per_client = 1
|
||||||
max_active_per_route = 1
|
max_active_per_route = 1
|
||||||
max_envelope_bytes = 1048576
|
max_envelope_bytes = 1048576
|
||||||
@@ -223,7 +226,7 @@ cache/archive routes according to policy.
|
|||||||
```yaml
|
```yaml
|
||||||
services:
|
services:
|
||||||
archive-client:
|
archive-client:
|
||||||
image: sodium/archive-clients:v0.1.13
|
image: sodium/archive-clients:v0.1.14
|
||||||
user: "1001:1001"
|
user: "1001:1001"
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
command: ["archive-client", "--config", "/etc/archive-control/client.toml"]
|
command: ["archive-client", "--config", "/etc/archive-control/client.toml"]
|
||||||
@@ -256,6 +259,28 @@ Generated protobuf Python bindings are committed in each consumer with the
|
|||||||
exact `archive-control-proto` tag/commit recorded. Release order is proto,
|
exact `archive-control-proto` tag/commit recorded. Release order is proto,
|
||||||
control consumer, client consumer, E2E, then image publication.
|
control consumer, client consumer, E2E, then image publication.
|
||||||
|
|
||||||
|
### Reproducible multi-platform Buildx lifecycle
|
||||||
|
|
||||||
|
The named Buildx builders are local acceleration/cache only; they are not a
|
||||||
|
deployment dependency and can be removed after publication. To create a fresh
|
||||||
|
builder, verify its platforms, publish a release, and remove it afterwards:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker buildx create --name archive-control-release --driver docker-container --use
|
||||||
|
docker buildx inspect --bootstrap
|
||||||
|
docker buildx build --platform linux/amd64,linux/arm64 \
|
||||||
|
--tag sodium/archive-clients:vX.Y.Z --push .
|
||||||
|
docker buildx rm archive-control-release
|
||||||
|
```
|
||||||
|
|
||||||
|
`docker buildx inspect` must list both `linux/amd64` and `linux/arm64` before
|
||||||
|
publishing. If the host has no arm64 emulation, install/configure it according
|
||||||
|
to the host Docker distribution before the build; do not publish a partial
|
||||||
|
single-platform tag. Retain the pushed manifest digest in the release notes
|
||||||
|
and deploy the immutable tag or digest. The optional `archive-control-qemu`
|
||||||
|
builder follows the same lifecycle when it is used for an emulation smoke
|
||||||
|
build.
|
||||||
|
|
||||||
## Operator usage
|
## Operator usage
|
||||||
|
|
||||||
1. Prepare local qB, sync, state, backup, config, and secret mounts with the
|
1. Prepare local qB, sync, state, backup, config, and secret mounts with the
|
||||||
|
|||||||
+13
-4
@@ -133,10 +133,13 @@ a resource may be active or queued at a time. This prevents incompatible
|
|||||||
baselines even when jobs would use different nodes or observe different sides
|
baselines even when jobs would use different nodes or observe different sides
|
||||||
of a hybrid identity.
|
of a hybrid identity.
|
||||||
|
|
||||||
Defaults allow one active data-moving job per client and one per route. A job
|
By default every eligible queued job is claimed in the same scheduler pass and
|
||||||
must acquire its source client, target client, route, and resource reservation
|
independent jobs run concurrently on each client; physical service and storage
|
||||||
atomically. Disjoint node pairs may run concurrently. Route setup is a
|
capacity are the limit. Set `enforce_concurrency_limits=true` on control to
|
||||||
preflight activity and does not permit a data step to bypass these leases.
|
enable the optional one-per-client/one-per-route gates. A job always retains
|
||||||
|
its resource reservation, and commands for the same job remain serialized.
|
||||||
|
Route setup is a preflight activity and does not permit a data step to bypass
|
||||||
|
the resource reservation.
|
||||||
|
|
||||||
Offline nodes do not prevent unrelated jobs from being listed or run. A job
|
Offline nodes do not prevent unrelated jobs from being listed or run. A job
|
||||||
requiring an offline node remains waiting indefinitely; it does not consume an
|
requiring an offline node remains waiting indefinitely; it does not consume an
|
||||||
@@ -154,6 +157,12 @@ advances to the next step. On reconnect:
|
|||||||
4. Control observes relevant qBittorrent, Syncthing, staging, and manifest
|
4. Control observes relevant qBittorrent, Syncthing, staging, and manifest
|
||||||
state before selecting retry, resume, compensation, cleanup, or manual
|
state before selecting retry, resume, compensation, cleanup, or manual
|
||||||
intervention.
|
intervention.
|
||||||
|
|
||||||
|
If a connection disappears while a synchronous job operation is in progress,
|
||||||
|
the client journals events locally, reconnects indefinitely, and serializes a
|
||||||
|
replayed command behind the in-flight operation for that job. The replay reads
|
||||||
|
the journal rather than repeating the data operation. This applies to every
|
||||||
|
transfer step, including post-commit staging cleanup.
|
||||||
5. A command is reissued with its original ID when the acceptance result is
|
5. A command is reissued with its original ID when the acceptance result is
|
||||||
uncertain.
|
uncertain.
|
||||||
|
|
||||||
|
|||||||
+4
-2
@@ -25,8 +25,10 @@ What do you want to do?
|
|||||||
[ Evict Cache ] [ Job Status ]
|
[ Evict Cache ] [ Job Status ]
|
||||||
```
|
```
|
||||||
|
|
||||||
All subsequent pages edit this message. `Cancel` closes the active selection
|
All subsequent pages edit this message. Leaving a resource/tree selection
|
||||||
flow. `Back` returns one level while retaining validated filters/selections.
|
returns to the Archive Control operation menu without closing the shared
|
||||||
|
conversation. Confirmation `Cancel` returns to its immediate prior selection;
|
||||||
|
the main menu alone can close Archive Control.
|
||||||
Callback payloads contain opaque session/action IDs, not resource names or
|
Callback payloads contain opaque session/action IDs, not resource names or
|
||||||
paths, and are validated against persisted session revision and expiry.
|
paths, and are validated against persisted session revision and expiry.
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -225,7 +225,7 @@ Fixtures include small deterministic v1, v2, and hybrid torrents with:
|
|||||||
| Recovery | restart every step on source/target/control, lost DB with proof, ambiguous loss fails closed |
|
| Recovery | restart every step on source/target/control, lost DB with proof, ambiguous loss fails closed |
|
||||||
| Messaging | duplicate command/event, lost ack, sequence gap, stale revision, duplicate client ID |
|
| Messaging | duplicate command/event, lost ack, sequence gap, stale revision, duplicate client ID |
|
||||||
| Safety | attempted download, corrupt same-size file, symlink/path escape, special file, partfile mismatch |
|
| Safety | attempted download, corrupt same-size file, symlink/path escape, special file, partfile mismatch |
|
||||||
| Operations | stall retains percent, offline wait, cancellation each phase, postcommit cleanup retry |
|
| Operations | stall retains percent, offline wait, cancellation each phase, postcommit cleanup retry, disconnect/reconnect during every transfer step with exactly-once replay |
|
||||||
| Database | online backup under load, pre/post migration, retention, corrupt backup rejection, offline restore |
|
| Database | online backup under load, pre/post migration, retention, corrupt backup rejection, offline restore |
|
||||||
| UI | pagination/filter chain, stale/double callback, persisted session, clear vs evict separation |
|
| UI | pagination/filter chain, stale/double callback, persisted session, clear vs evict separation |
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "archive-clients"
|
name = "archive-clients"
|
||||||
version = "0.1.13"
|
version = "0.1.16"
|
||||||
requires-python = ">=3.11"
|
requires-python = ">=3.11"
|
||||||
dependencies = ["protobuf==7.35.1", "websockets==16.0"]
|
dependencies = ["protobuf==7.35.1", "websockets==16.0"]
|
||||||
|
|
||||||
|
|||||||
@@ -43,6 +43,19 @@ from archive_control.v1 import (
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _job_id_for_command(command: control_pb2.Command) -> str:
|
||||||
|
"""Return the durable job key for commands whose execution is serialized."""
|
||||||
|
|
||||||
|
payload = command.WhichOneof("payload")
|
||||||
|
if payload == "assign_job":
|
||||||
|
return command.assign_job.job.job_id
|
||||||
|
if payload == "execute_step":
|
||||||
|
return command.execute_step.job_id
|
||||||
|
if payload == "cancel_job":
|
||||||
|
return command.cancel_job.job_id
|
||||||
|
raise ValueError(f"command {command.command_id} does not execute a job")
|
||||||
|
|
||||||
|
|
||||||
class ArchiveClientDaemon:
|
class ArchiveClientDaemon:
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -95,7 +108,10 @@ class ArchiveClientDaemon:
|
|||||||
self._lease = DatabaseLease(config.state_db)
|
self._lease = DatabaseLease(config.state_db)
|
||||||
self._active_route_commands: set[str] = set()
|
self._active_route_commands: set[str] = set()
|
||||||
self._active_job_commands: set[str] = set()
|
self._active_job_commands: set[str] = set()
|
||||||
self._job_execution_lock = asyncio.Lock()
|
# Commands for one job remain ordered locally, while unrelated jobs
|
||||||
|
# may use the node's available qB/Syncthing/filesystem capacity in
|
||||||
|
# parallel. The control daemon owns admission policy.
|
||||||
|
self._job_execution_locks: dict[str, asyncio.Lock] = {}
|
||||||
self.jobs = (
|
self.jobs = (
|
||||||
ClientJobExecutor(
|
ClientJobExecutor(
|
||||||
client_id=config.client_id,
|
client_id=config.client_id,
|
||||||
@@ -560,7 +576,9 @@ class ArchiveClientDaemon:
|
|||||||
correlation_id: str,
|
correlation_id: str,
|
||||||
outbound: asyncio.Queue[str],
|
outbound: asyncio.Queue[str],
|
||||||
) -> None:
|
) -> None:
|
||||||
async with self._job_execution_lock:
|
job_id = _job_id_for_command(command)
|
||||||
|
lock = self._job_execution_locks.setdefault(job_id, asyncio.Lock())
|
||||||
|
async with lock:
|
||||||
await self._execute_job_command_locked(
|
await self._execute_job_command_locked(
|
||||||
command, correlation_id, outbound
|
command, correlation_id, outbound
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -86,6 +86,8 @@ class ClientJobExecutor:
|
|||||||
self.verification_timeout = verification_timeout
|
self.verification_timeout = verification_timeout
|
||||||
self.free_space_reserve_bytes = free_space_reserve_bytes
|
self.free_space_reserve_bytes = free_space_reserve_bytes
|
||||||
self._cancel_events: dict[str, threading.Event] = {}
|
self._cancel_events: dict[str, threading.Event] = {}
|
||||||
|
self._execution_locks: dict[str, threading.Lock] = {}
|
||||||
|
self._execution_locks_guard = threading.Lock()
|
||||||
|
|
||||||
def request_cancel(self, job_id: str) -> None:
|
def request_cancel(self, job_id: str) -> None:
|
||||||
self._cancel_events.setdefault(job_id, threading.Event()).set()
|
self._cancel_events.setdefault(job_id, threading.Event()).set()
|
||||||
@@ -115,6 +117,22 @@ class ClientJobExecutor:
|
|||||||
self,
|
self,
|
||||||
command: control_pb2.ExecuteStepCommand,
|
command: control_pb2.ExecuteStepCommand,
|
||||||
event_callback: Callable[[control_pb2.JobEvent], None] | None = None,
|
event_callback: Callable[[control_pb2.JobEvent], None] | None = None,
|
||||||
|
) -> list[control_pb2.JobEvent]:
|
||||||
|
# asyncio cancellation of a connection-bound task cannot stop the
|
||||||
|
# synchronous filesystem/qB operation already running in its worker
|
||||||
|
# thread. A replay after reconnect therefore waits for that operation
|
||||||
|
# and then reads its durable event journal instead of executing twice.
|
||||||
|
with self._execution_lock(command.job_id):
|
||||||
|
return self._execute_locked(command, event_callback)
|
||||||
|
|
||||||
|
def _execution_lock(self, job_id: str) -> threading.Lock:
|
||||||
|
with self._execution_locks_guard:
|
||||||
|
return self._execution_locks.setdefault(job_id, threading.Lock())
|
||||||
|
|
||||||
|
def _execute_locked(
|
||||||
|
self,
|
||||||
|
command: control_pb2.ExecuteStepCommand,
|
||||||
|
event_callback: Callable[[control_pb2.JobEvent], None] | None = None,
|
||||||
) -> list[control_pb2.JobEvent]:
|
) -> list[control_pb2.JobEvent]:
|
||||||
definition = self._definition(command.job_id)
|
definition = self._definition(command.job_id)
|
||||||
replay = self._replay(
|
replay = self._replay(
|
||||||
@@ -653,6 +671,15 @@ class ClientJobExecutor:
|
|||||||
# Syncthing can expose the per-job directory before the
|
# Syncthing can expose the per-job directory before the
|
||||||
# ready marker and manifest have arrived atomically as a set.
|
# ready marker and manifest have arrived atomically as a set.
|
||||||
pass
|
pass
|
||||||
|
except RouteSetupError as exc:
|
||||||
|
# A network interruption can temporarily make the local
|
||||||
|
# Syncthing REST API unavailable. The staged payload is
|
||||||
|
# durable and the job must remain recoverable; keep polling
|
||||||
|
# instead of turning a transient outage into a failed job.
|
||||||
|
logger.warning("syncthing_status_deferred", extra={
|
||||||
|
"job_id": definition.job_id,
|
||||||
|
"error_type": type(exc).__name__,
|
||||||
|
})
|
||||||
time.sleep(self.poll_interval)
|
time.sleep(self.poll_interval)
|
||||||
|
|
||||||
def _target_materialize(
|
def _target_materialize(
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
@@ -16,6 +17,8 @@ from urllib import error, parse, request
|
|||||||
from archive_clients.config import ServiceConfig
|
from archive_clients.config import ServiceConfig
|
||||||
from archive_clients.resources import NormalizedResource, normalize_resource
|
from archive_clients.resources import NormalizedResource, normalize_resource
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class QBittorrentError(RuntimeError):
|
class QBittorrentError(RuntimeError):
|
||||||
pass
|
pass
|
||||||
@@ -72,7 +75,17 @@ class QBittorrentReader:
|
|||||||
torrent_hash = torrent.get("hash")
|
torrent_hash = torrent.get("hash")
|
||||||
if not isinstance(torrent_hash, str):
|
if not isinstance(torrent_hash, str):
|
||||||
raise QBittorrentError("qBittorrent torrent hash is invalid")
|
raise QBittorrentError("qBittorrent torrent hash is invalid")
|
||||||
result.append(self._normalize(torrent, torrent_hash))
|
try:
|
||||||
|
result.append(self._normalize(torrent, torrent_hash))
|
||||||
|
except ValueError as exc:
|
||||||
|
# A stale or malformed qBittorrent entry must not hide every
|
||||||
|
# otherwise valid resource from Archive/Evict inventory.
|
||||||
|
# Keep it ineligible and leave an operator-visible diagnosis.
|
||||||
|
logger.warning(
|
||||||
|
"Skipping qBittorrent resource with inconsistent "
|
||||||
|
"metadata: hash=%s name=%r reason=%s",
|
||||||
|
torrent_hash, torrent.get("name"), exc,
|
||||||
|
)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def get_resource(self, torrent_hash: str) -> NormalizedResource | None:
|
def get_resource(self, torrent_hash: str) -> NormalizedResource | None:
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ import json
|
|||||||
import os
|
import os
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import stat
|
import stat
|
||||||
|
import threading
|
||||||
|
from contextlib import contextmanager
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -14,6 +16,12 @@ from pathlib import Path
|
|||||||
SCHEMA_VERSION = 3
|
SCHEMA_VERSION = 3
|
||||||
|
|
||||||
|
|
||||||
|
# A client daemon executes unrelated jobs concurrently. SQLite still permits
|
||||||
|
# only one writer, so serialize this process's short state transactions rather
|
||||||
|
# than allowing an otherwise healthy job to fail after its busy timeout.
|
||||||
|
_DATABASE_LOCK = threading.RLock()
|
||||||
|
|
||||||
|
|
||||||
class CommandConflict(RuntimeError):
|
class CommandConflict(RuntimeError):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@@ -557,14 +565,30 @@ class ClientStore:
|
|||||||
).fetchall()
|
).fetchall()
|
||||||
return [dict(row) for row in rows]
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
def _connect(self) -> sqlite3.Connection:
|
@contextmanager
|
||||||
connection = sqlite3.connect(self.database, isolation_level=None, timeout=5)
|
def _connect(self):
|
||||||
connection.row_factory = sqlite3.Row
|
"""Yield one connection while serializing local SQLite writers.
|
||||||
connection.execute("PRAGMA foreign_keys = ON")
|
|
||||||
connection.execute("PRAGMA journal_mode = WAL")
|
The longer SQLite timeout also covers a short lock held by a separate
|
||||||
connection.execute("PRAGMA synchronous = FULL")
|
maintenance process such as a backup. The lock is deliberately held
|
||||||
connection.execute("PRAGMA busy_timeout = 5000")
|
for the full transaction, including ``BEGIN IMMEDIATE``.
|
||||||
return connection
|
"""
|
||||||
|
with _DATABASE_LOCK:
|
||||||
|
connection = sqlite3.connect(
|
||||||
|
self.database, isolation_level=None, timeout=30
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
connection.row_factory = sqlite3.Row
|
||||||
|
connection.execute("PRAGMA foreign_keys = ON")
|
||||||
|
connection.execute("PRAGMA journal_mode = WAL")
|
||||||
|
connection.execute("PRAGMA synchronous = FULL")
|
||||||
|
connection.execute("PRAGMA busy_timeout = 30000")
|
||||||
|
# Preserve the original ``with connection`` commit/rollback
|
||||||
|
# behavior used by every store operation.
|
||||||
|
with connection:
|
||||||
|
yield connection
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
def _canonical(value: object) -> str:
|
def _canonical(value: object) -> str:
|
||||||
|
|||||||
@@ -22,6 +22,47 @@ from archive_control.v1 import (
|
|||||||
|
|
||||||
|
|
||||||
class DaemonTransportTests(unittest.IsolatedAsyncioTestCase):
|
class DaemonTransportTests(unittest.IsolatedAsyncioTestCase):
|
||||||
|
async def test_unrelated_job_commands_execute_concurrently(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
token = root / "token"
|
||||||
|
token.write_text("shared-secret", encoding="utf-8")
|
||||||
|
os.chmod(token, 0o600)
|
||||||
|
service = ServiceConfig("http://local", PurePosixPath("/api"), root)
|
||||||
|
config = ClientConfig(
|
||||||
|
"cache-1", "Cache 1", "cache", "ws://control", token,
|
||||||
|
root / "state.db", root / "backups", service, service,
|
||||||
|
)
|
||||||
|
probe = FilesystemProbe(root, True, True, True, True, True)
|
||||||
|
daemon = ArchiveClientDaemon(config, [probe, probe], [])
|
||||||
|
started: set[str] = set()
|
||||||
|
release = asyncio.Event()
|
||||||
|
|
||||||
|
async def execute(command, correlation_id, outbound):
|
||||||
|
started.add(command.assign_job.job.job_id)
|
||||||
|
await release.wait()
|
||||||
|
|
||||||
|
daemon._execute_job_command_locked = execute
|
||||||
|
commands = []
|
||||||
|
for _ in range(2):
|
||||||
|
command = control_pb2.Command(command_id=str(uuid4()))
|
||||||
|
command.assign_job.job.job_id = str(uuid4())
|
||||||
|
commands.append(command)
|
||||||
|
outbound = asyncio.Queue()
|
||||||
|
tasks = [
|
||||||
|
asyncio.create_task(
|
||||||
|
daemon._execute_job_command(command, "", outbound)
|
||||||
|
)
|
||||||
|
for command in commands
|
||||||
|
]
|
||||||
|
for _ in range(100):
|
||||||
|
if len(started) == 2:
|
||||||
|
break
|
||||||
|
await asyncio.sleep(0.01)
|
||||||
|
self.assertEqual(len(started), 2)
|
||||||
|
release.set()
|
||||||
|
await asyncio.gather(*tasks)
|
||||||
|
|
||||||
async def test_silent_control_connection_ends_for_reconnect(self):
|
async def test_silent_control_connection_ends_for_reconnect(self):
|
||||||
"""A lost server heartbeat must not leave durable commands stranded."""
|
"""A lost server heartbeat must not leave durable commands stranded."""
|
||||||
|
|
||||||
|
|||||||
+116
-1
@@ -1,5 +1,6 @@
|
|||||||
import hashlib
|
import hashlib
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import threading
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import Mock, patch
|
from unittest.mock import Mock, patch
|
||||||
@@ -11,7 +12,7 @@ from archive_clients.jobs import (
|
|||||||
JobExecutionError,
|
JobExecutionError,
|
||||||
_resource_fingerprint,
|
_resource_fingerprint,
|
||||||
)
|
)
|
||||||
from archive_clients.syncthing import RouteSetupError
|
from archive_clients.syncthing import RouteSetupError, SyncthingTransferStatus
|
||||||
from archive_clients.resources import normalize_resource
|
from archive_clients.resources import normalize_resource
|
||||||
from archive_clients.state import ClientStore
|
from archive_clients.state import ClientStore
|
||||||
from archive_control.v1 import control_pb2, job_pb2
|
from archive_control.v1 import control_pb2, job_pb2
|
||||||
@@ -38,6 +39,120 @@ class SlowRescanSyncthing(CompleteSyncthing):
|
|||||||
|
|
||||||
|
|
||||||
class ClientJobHappyPathTests(unittest.TestCase):
|
class ClientJobHappyPathTests(unittest.TestCase):
|
||||||
|
def test_syncthing_api_outage_during_transfer_is_retried(self):
|
||||||
|
"""A transient local REST outage must not terminally fail the job."""
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
store = ClientStore(root / "client.db")
|
||||||
|
store.initialize()
|
||||||
|
definition = job_pb2.JobDefinition(
|
||||||
|
job_id=str(uuid4()),
|
||||||
|
transfer={
|
||||||
|
"source_client_id": "cache-1",
|
||||||
|
"target_client_id": "archive-1",
|
||||||
|
"route_id": "route-1",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
observer = Mock()
|
||||||
|
observer.status.side_effect = [
|
||||||
|
RouteSetupError("Syncthing API is unavailable"),
|
||||||
|
SyncthingTransferStatus(1, 42, 42, True, 0),
|
||||||
|
]
|
||||||
|
executor = ClientJobExecutor(
|
||||||
|
client_id="archive-1",
|
||||||
|
qbittorrent=Mock(),
|
||||||
|
store=store,
|
||||||
|
qb_root=root,
|
||||||
|
qb_api_root=Path("/downloads"),
|
||||||
|
route_path=lambda _: root,
|
||||||
|
syncthing_transport=Mock(),
|
||||||
|
sparse_supported=True,
|
||||||
|
poll_interval=0,
|
||||||
|
)
|
||||||
|
executor._observer = Mock(return_value=observer)
|
||||||
|
progress = Mock()
|
||||||
|
|
||||||
|
executor._wait_for_syncthing(definition, progress)
|
||||||
|
|
||||||
|
self.assertEqual(observer.status.call_count, 2)
|
||||||
|
progress.assert_called_once_with(1, 42, 42, "0 Syncthing items still needed")
|
||||||
|
|
||||||
|
def test_reconnect_replay_never_duplicates_any_transfer_step(self):
|
||||||
|
for step in (
|
||||||
|
job_pb2.JOB_STEP_KIND_SOURCE_STAGE,
|
||||||
|
job_pb2.JOB_STEP_KIND_SYNCTHING_TRANSFER,
|
||||||
|
job_pb2.JOB_STEP_KIND_TARGET_MATERIALIZE,
|
||||||
|
job_pb2.JOB_STEP_KIND_QB_VERIFY,
|
||||||
|
job_pb2.JOB_STEP_KIND_STAGING_CLEANUP,
|
||||||
|
):
|
||||||
|
with self.subTest(step=step), tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
store = ClientStore(root / "client.db")
|
||||||
|
store.initialize()
|
||||||
|
definition = job_pb2.JobDefinition(
|
||||||
|
job_id=str(uuid4()),
|
||||||
|
idempotency_key=str(uuid4()),
|
||||||
|
operation=job_pb2.JOB_OPERATION_ARCHIVE,
|
||||||
|
resource_display_name="reconnect fixture",
|
||||||
|
transfer={
|
||||||
|
"source_client_id": "cache-1",
|
||||||
|
"target_client_id": "archive-1",
|
||||||
|
"route_id": "route-1",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
definition.resource_id.info_hash_v1_hex = "a" * 40
|
||||||
|
definition.created_at.GetCurrentTime()
|
||||||
|
executor = ClientJobExecutor(
|
||||||
|
client_id="cache-1",
|
||||||
|
qbittorrent=Mock(),
|
||||||
|
store=store,
|
||||||
|
qb_root=root,
|
||||||
|
qb_api_root=Path("/downloads"),
|
||||||
|
route_path=lambda _: root,
|
||||||
|
syncthing_transport=Mock(),
|
||||||
|
sparse_supported=True,
|
||||||
|
)
|
||||||
|
executor.assign(control_pb2.AssignJobCommand(
|
||||||
|
job=definition,
|
||||||
|
expected_job_revision=1,
|
||||||
|
expected_last_event_sequence=0,
|
||||||
|
))
|
||||||
|
entered = threading.Event()
|
||||||
|
release = threading.Event()
|
||||||
|
|
||||||
|
def execute_step(*_args):
|
||||||
|
entered.set()
|
||||||
|
release.wait(1)
|
||||||
|
return None
|
||||||
|
|
||||||
|
executor._execute_step = Mock(side_effect=execute_step)
|
||||||
|
command = control_pb2.ExecuteStepCommand(
|
||||||
|
job_id=definition.job_id,
|
||||||
|
expected_job_revision=1,
|
||||||
|
expected_last_event_sequence=1,
|
||||||
|
step=step,
|
||||||
|
attempt=1,
|
||||||
|
)
|
||||||
|
results: list[list[control_pb2.JobEvent]] = []
|
||||||
|
first = threading.Thread(
|
||||||
|
target=lambda: results.append(executor.execute(command))
|
||||||
|
)
|
||||||
|
second = threading.Thread(
|
||||||
|
target=lambda: results.append(executor.execute(command))
|
||||||
|
)
|
||||||
|
first.start()
|
||||||
|
self.assertTrue(entered.wait(1))
|
||||||
|
second.start()
|
||||||
|
release.set()
|
||||||
|
first.join(1)
|
||||||
|
second.join(1)
|
||||||
|
|
||||||
|
self.assertFalse(first.is_alive())
|
||||||
|
self.assertFalse(second.is_alive())
|
||||||
|
self.assertEqual(executor._execute_step.call_count, 1)
|
||||||
|
self.assertEqual(len(results), 2)
|
||||||
|
self.assertEqual(results[0][-1].event_id, results[1][-1].event_id)
|
||||||
|
|
||||||
def test_capacity_guard_fails_before_data_movement(self):
|
def test_capacity_guard_fails_before_data_movement(self):
|
||||||
with tempfile.TemporaryDirectory() as directory:
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
root = Path(directory)
|
root = Path(directory)
|
||||||
|
|||||||
@@ -205,6 +205,54 @@ class QBittorrentReaderTests(unittest.TestCase):
|
|||||||
self.assertEqual(resources, [])
|
self.assertEqual(resources, [])
|
||||||
self.assertEqual(len(opener.calls), 2)
|
self.assertEqual(len(opener.calls), 2)
|
||||||
|
|
||||||
|
def test_listing_skips_malformed_torrent_and_keeps_valid_resources(self):
|
||||||
|
def torrent(name, content):
|
||||||
|
info = {
|
||||||
|
b"length": len(content), b"name": name.encode(),
|
||||||
|
b"piece length": 16384, b"pieces": b"x" * 20,
|
||||||
|
}
|
||||||
|
return encode({b"info": info}), hashlib.sha1(encode(info)).hexdigest()
|
||||||
|
|
||||||
|
first_bytes, first_hash = torrent("first.txt", b"one")
|
||||||
|
malformed_bytes, malformed_hash = torrent("broken.txt", b"bad")
|
||||||
|
second_bytes, second_hash = torrent("second.txt", b"two")
|
||||||
|
torrents = [
|
||||||
|
{"hash": first_hash, "name": "first.txt", "state": "uploading"},
|
||||||
|
{"hash": malformed_hash, "name": "broken.txt", "state": "stalledUP"},
|
||||||
|
{"hash": second_hash, "name": "second.txt", "state": "uploading"},
|
||||||
|
]
|
||||||
|
responses = [
|
||||||
|
b"Ok.", json.dumps(torrents).encode(),
|
||||||
|
b'[{"index":0,"name":"first.txt","size":3,"progress":1,"priority":1}]',
|
||||||
|
first_bytes,
|
||||||
|
b'[{"index":0,"name":"broken.txt","size":3,"progress":1,"priority":1},'
|
||||||
|
b'{"index":1,"name":"unexpected.txt","size":1,"progress":1,"priority":1}]',
|
||||||
|
malformed_bytes,
|
||||||
|
b'[{"index":0,"name":"second.txt","size":3,"progress":1,"priority":1}]',
|
||||||
|
second_bytes,
|
||||||
|
]
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
password = root / "password"
|
||||||
|
password.write_text("secret", encoding="utf-8")
|
||||||
|
os.chmod(password, 0o600)
|
||||||
|
config = ServiceConfig(
|
||||||
|
"http://qb", PurePosixPath("/downloads"), root,
|
||||||
|
username="admin", password_file=password,
|
||||||
|
)
|
||||||
|
opener = _Opener(responses)
|
||||||
|
with patch(
|
||||||
|
"archive_clients.qbittorrent.request.build_opener",
|
||||||
|
return_value=opener,
|
||||||
|
), self.assertLogs("archive_clients.qbittorrent", "WARNING") as logs:
|
||||||
|
resources = QBittorrentReader(config).list_resources()
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
[resource.summary.display_name for resource in resources],
|
||||||
|
["first.txt", "second.txt"],
|
||||||
|
)
|
||||||
|
self.assertIn(malformed_hash, "\n".join(logs.output))
|
||||||
|
|
||||||
def test_stopped_add_selection_recheck_and_entry_only_delete(self):
|
def test_stopped_add_selection_recheck_and_entry_only_delete(self):
|
||||||
torrent_hash = "a" * 40
|
torrent_hash = "a" * 40
|
||||||
responses = [
|
responses = [
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
import os
|
import os
|
||||||
|
import threading
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
@@ -13,6 +14,40 @@ from archive_clients.state import (
|
|||||||
|
|
||||||
|
|
||||||
class ClientStoreTests(unittest.TestCase):
|
class ClientStoreTests(unittest.TestCase):
|
||||||
|
def test_database_connections_serialize_local_writers(self):
|
||||||
|
"""Concurrent jobs share one daemon DB without SQLite lock failures."""
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
database = Path(directory) / "state.db"
|
||||||
|
first = ClientStore(database)
|
||||||
|
second = ClientStore(database)
|
||||||
|
first.initialize()
|
||||||
|
barrier = threading.Barrier(2)
|
||||||
|
failures: list[Exception] = []
|
||||||
|
|
||||||
|
def write(store, command_id):
|
||||||
|
try:
|
||||||
|
barrier.wait()
|
||||||
|
store.accept_command(command_id, '{"kind":"job"}', '{"ok":true}')
|
||||||
|
except Exception as exc: # pragma: no cover - assertion below
|
||||||
|
failures.append(exc)
|
||||||
|
|
||||||
|
left = threading.Thread(target=write, args=(first, "command-1"))
|
||||||
|
right = threading.Thread(target=write, args=(second, "command-2"))
|
||||||
|
left.start()
|
||||||
|
right.start()
|
||||||
|
left.join(1)
|
||||||
|
right.join(1)
|
||||||
|
|
||||||
|
self.assertFalse(left.is_alive())
|
||||||
|
self.assertFalse(right.is_alive())
|
||||||
|
self.assertEqual(failures, [])
|
||||||
|
self.assertEqual(len(first.list_accepted_commands()), 2)
|
||||||
|
with first._connect() as connection:
|
||||||
|
self.assertEqual(
|
||||||
|
connection.execute("PRAGMA busy_timeout").fetchone()[0],
|
||||||
|
30000,
|
||||||
|
)
|
||||||
|
|
||||||
def test_command_acceptance_is_durable_and_content_addressed(self):
|
def test_command_acceptance_is_durable_and_content_addressed(self):
|
||||||
with tempfile.TemporaryDirectory() as directory:
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
database = Path(directory) / "state.db"
|
database = Path(directory) / "state.db"
|
||||||
|
|||||||
Reference in New Issue
Block a user