cabbage/archive-control-proto
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Define archive control v1 protocol

Browse Source
This commit is contained in:
cabbage
2026-07-22 11:59:57 +00:00
commit 7652262b4c
25 changed files with 1333 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+6
View File
@@ -0,0 +1,6 @@
/gen/
/descriptor.bin
/.idea/
/.vscode/
*.pyc
__pycache__/
Makefile
+29
View File
@@ -0,0 +1,29 @@
BUF_IMAGE ?= bufbuild/buf:1.47.2
.PHONY: format lint build validate-examples generate breaking
format:
docker run --rm --user "$$(id -u):$$(id -g)" -e XDG_CACHE_HOME=/tmp \
-v "$$(pwd):/workspace" -w /workspace $(BUF_IMAGE) format -w
lint:
docker run --rm --user "$$(id -u):$$(id -g)" -e XDG_CACHE_HOME=/tmp \
-v "$$(pwd):/workspace" -w /workspace $(BUF_IMAGE) lint
build:
docker run --rm --user "$$(id -u):$$(id -g)" -e XDG_CACHE_HOME=/tmp \
-v "$$(pwd):/workspace" -w /workspace $(BUF_IMAGE) build \
-o descriptor.bin
validate-examples:
./scripts/validate-examples.sh
generate:
docker run --rm --user "$$(id -u):$$(id -g)" -e XDG_CACHE_HOME=/tmp \
-v "$$(pwd):/workspace" -w /workspace $(BUF_IMAGE) generate
breaking:
@test -n "$(AGAINST)" || (echo "usage: make breaking AGAINST=.git#tag=v0.1.0"; exit 2)
docker run --rm --user "$$(id -u):$$(id -g)" -e XDG_CACHE_HOME=/tmp \
-v "$$(pwd):/workspace" -w /workspace $(BUF_IMAGE) breaking \
--against "$(AGAINST)"
README.md
+76
View File
@@ -0,0 +1,76 @@
# archive-control-proto
Source-of-truth protobuf contract for Archive Control clients and the Titan
control daemon. Implementations exchange canonical protobuf JSON as one text
message per WebSocket frame.
Upstream: `cabbage/archive-control-proto`
## Contract overview
- `common.proto`: protocol versioning, roles, health, and stable error codes
- `resource.proto`: v1/v2 torrent identity, selected-file sets, trees, and
placements
- `route.proto`: discovered and automatically provisioned pairwise Syncthing
routes
- `transfer.proto`: the protobuf-JSON on-disk transfer manifest and ready marker
- `job.proto`: immutable job requests, runtime states, steps, progress, archive
coverage, and eviction
- `inventory.proto`: scoped, chunked, on-demand inventory/tree responses
- `client.proto`: capabilities, registration, active-job cursors, and heartbeat
- `control.proto`: durable commands, acknowledgements, job events, snapshots,
and route updates
- `envelope.proto`: the only top-level WebSocket application message
## Wire rules
1. A new connection's first application message is `RegisterRequest`.
2. The pre-shared token appears only in that request. It must not be logged.
3. A server command is delivered at least once. Clients durably deduplicate
`command_id` and return `DUPLICATE` for a replayed accepted command.
4. `CommandAck` means durable acceptance, not operation completion.
5. Job events have monotonically increasing global per-job sequences. A command
carries the accepted sequence base and grants one client the event-writer
lease; a gap requires `JobSnapshot` reconciliation before further
destructive transitions.
6. UUIDs use lowercase canonical text. Info hashes use validated lowercase hex.
7. `SelectionSet.ranges` are inclusive, sorted, non-overlapping, and coalesced.
8. Receivers ignore unknown JSON fields for compatible minor versions. They
reject an unknown required command/job operation.
9. Published field numbers and enum values are never reused. Removed values are
reserved in the change that removes them.
10. Implementations cap an envelope at the negotiated maximum and keep streamed
inventory chunks below that value.
## Dockerized tooling
The host remains a thin editor. Buf runs in Docker and writes as the calling
UID/GID:
```bash
./scripts/buf.sh format
./scripts/buf.sh lint
./scripts/buf.sh build
./scripts/validate-examples.sh
./scripts/buf.sh generate
```
An equivalent `Makefile` is provided for environments where `make` is already
available; it is not required on the thin development host.
Generated Python bindings are copied into and committed by consumer
repositories with the exact proto release/commit recorded in their generated
header. `gen/` is intentionally ignored here.
Before releasing a compatible update:
```bash
./scripts/buf.sh breaking '.git#tag=v0.1.0'
```
## JSON examples
`examples/v1/` contains representative canonical protobuf JSON for
registration, inventory, route setup, assignment, progress, commit,
cancellation, eviction, and reconnect reconciliation. They are protocol
fixtures, not hand-maintained alternative schemas.
buf.gen.yaml
+7
View File
@@ -0,0 +1,7 @@
version: v2
clean: true
plugins:
- remote: buf.build/protocolbuffers/python
out: gen/python
- remote: buf.build/protocolbuffers/pyi
out: gen/python
buf.yaml
+9
View File
@@ -0,0 +1,9 @@
version: v2
modules:
- path: proto
lint:
use:
- STANDARD
breaking:
use:
- FILE
examples/v1/assign-archive-job.json
+57
View File
@@ -0,0 +1,57 @@
{
"protocolVersion": {
"major": 1,
"minor": 0
},
"messageId": "2fc6a524-84cd-47ab-bd73-30d463b7c43f",
"sentAt": "2026-07-22T12:03:00Z",
"command": {
"commandId": "6bfccbf4-866f-46fe-bf67-d8943c1ece91",
"createdAt": "2026-07-22T12:03:00Z",
"assignJob": {
"job": {
"jobId": "fd6844d0-ed76-475e-8375-2f98d8eea470",
"idempotencyKey": "telegram-confirmation-615e42",
"operation": "JOB_OPERATION_ARCHIVE",
"resourceId": {
"infoHashV1Hex": "0123456789abcdef0123456789abcdef01234567"
},
"resourceDisplayName": "Resource A",
"createdAt": "2026-07-22T12:03:00Z",
"transfer": {
"sourceClientId": "x1",
"targetClientId": "lithium",
"routeId": "archive-control-x1-lithium-f8b629",
"requestedFiles": {
"ranges": [
{
"first": 2,
"last": 4
}
]
},
"expectedTargetPlacementGeneration": "3",
"targetBaselineFiles": {
"ranges": [
{
"first": 2,
"last": 2
}
]
},
"transferDeltaFiles": {
"ranges": [
{
"first": 3,
"last": 4
}
]
},
"requestedLogicalBytes": "10735000000",
"transferDeltaLogicalBytes": "7340000000"
}
},
"expectedLastEventSequence": "0"
}
}
}
examples/v1/assign-eviction-job.json
+63
View File
@@ -0,0 +1,63 @@
{
"protocolVersion": {
"major": 1,
"minor": 0
},
"messageId": "9c62fcda-5f83-4286-80cf-a628001e44d3",
"sentAt": "2026-07-22T12:05:00Z",
"command": {
"commandId": "daf7ebaf-f3fa-4ff9-979e-1c0c95e08376",
"createdAt": "2026-07-22T12:05:00Z",
"assignJob": {
"job": {
"jobId": "5e76f086-79ed-41e5-9306-c4a0312e4d26",
"idempotencyKey": "evict-confirmation-c9c04a",
"operation": "JOB_OPERATION_EVICT_CACHE",
"resourceId": {
"infoHashV1Hex": "0123456789abcdef0123456789abcdef01234567"
},
"resourceDisplayName": "Resource A",
"createdAt": "2026-07-22T12:05:00Z",
"eviction": {
"cacheClientId": "x1",
"filesToEvict": {
"ranges": [
{
"first": 0,
"last": 6
}
]
},
"cachePlacementGeneration": "8",
"archiveCoverage": [
{
"archiveClientId": "lithium",
"coveredFiles": {
"ranges": [
{
"first": 0,
"last": 4
}
]
},
"placementGeneration": "4"
},
{
"archiveClientId": "cold2",
"coveredFiles": {
"ranges": [
{
"first": 5,
"last": 6
}
]
},
"placementGeneration": "2"
}
]
}
},
"expectedLastEventSequence": "0"
}
}
}
examples/v1/cancel-job-command.json
+18
View File
@@ -0,0 +1,18 @@
{
"protocolVersion": {
"major": 1,
"minor": 0
},
"messageId": "7c4a3534-c754-4c4d-a12e-1a1047e3f96e",
"sentAt": "2026-07-22T12:06:00Z",
"command": {
"commandId": "d9721ad2-f98d-485f-9370-476e49f7897a",
"createdAt": "2026-07-22T12:06:00Z",
"cancelJob": {
"jobId": "fd6844d0-ed76-475e-8375-2f98d8eea470",
"expectedJobRevision": "6",
"reason": "administrator requested removal",
"expectedLastEventSequence": "17"
}
}
}
examples/v1/client-state-snapshot.json
+78
View File
@@ -0,0 +1,78 @@
{
"protocolVersion": {
"major": 1,
"minor": 0
},
"messageId": "56aae470-d1c9-402a-bf14-d7a28ff8998f",
"sentAt": "2026-07-22T12:10:00Z",
"clientStateSnapshot": {
"snapshotId": "6067263d-548c-4548-974f-9f73ca9fb7ec",
"activeJobs": [
{
"job": {
"definition": {
"jobId": "fd6844d0-ed76-475e-8375-2f98d8eea470",
"idempotencyKey": "telegram-confirmation-615e42",
"operation": "JOB_OPERATION_ARCHIVE",
"resourceId": {
"infoHashV1Hex": "0123456789abcdef0123456789abcdef01234567"
},
"resourceDisplayName": "Resource A",
"createdAt": "2026-07-22T12:03:00Z",
"transfer": {
"sourceClientId": "x1",
"targetClientId": "lithium",
"routeId": "archive-control-x1-lithium-f8b629",
"requestedFiles": {
"ranges": [
{
"first": 2,
"last": 4
}
]
},
"expectedTargetPlacementGeneration": "3",
"targetBaselineFiles": {
"ranges": [
{
"first": 2,
"last": 2
}
]
},
"transferDeltaFiles": {
"ranges": [
{
"first": 3,
"last": 4
}
]
},
"requestedLogicalBytes": "10735000000",
"transferDeltaLogicalBytes": "7340000000"
}
},
"state": "JOB_STATE_RUNNING",
"revision": "6",
"progress": {
"step": "JOB_STEP_KIND_SYNCTHING_TRANSFER",
"displayStepNumber": 2,
"displayStepTotal": 5,
"state": "STEP_STATE_RUNNING",
"fractionComplete": 0.4231,
"bytesComplete": "4543000000",
"bytesTotal": "10735000000",
"lastProgressAt": "2026-07-22T12:04:00Z",
"overallFractionComplete": 0.2846
},
"updatedAt": "2026-07-22T12:04:00Z"
},
"lastEventSequence": "17",
"inFlightCommandIds": [
"f253f9fc-2a43-491d-8317-24795bb7638a"
]
}
],
"observedAt": "2026-07-22T12:10:00Z"
}
}
examples/v1/ensure-route-command.json
+24
View File
@@ -0,0 +1,24 @@
{
"protocolVersion": {
"major": 1,
"minor": 0
},
"messageId": "830626f3-dc76-46f0-a9b1-a6fb61b3ba73",
"sentAt": "2026-07-22T12:02:00Z",
"command": {
"commandId": "75425c3b-1ceb-4e9d-9af6-402a06e5bf8d",
"createdAt": "2026-07-22T12:02:00Z",
"ensureRoute": {
"route": {
"routeId": "archive-control-x1-lithium-f8b629",
"peerClientId": "lithium",
"peerSyncthingDeviceId": "LITHIUM-DEVICE-ID",
"peerAddresses": [
"tcp://lithium-syncthing:22000"
],
"localRelativePath": ".archive-control/routes/archive-control-x1-lithium-f8b629",
"setupTimeoutSeconds": 1800
}
}
}
}
examples/v1/inventory-query.json
+20
View File
@@ -0,0 +1,20 @@
{
"protocolVersion": {
"major": 1,
"minor": 0
},
"messageId": "d00def27-cde1-43b7-a6af-ee3582d0c7c1",
"sentAt": "2026-07-22T12:01:00Z",
"command": {
"commandId": "cc791b88-cbca-418f-9732-c605c8144c90",
"createdAt": "2026-07-22T12:01:00Z",
"inventoryQuery": {
"queryId": "1944768a-843c-460a-8ea2-27ea0f682aa6",
"scope": "INVENTORY_SCOPE_RESOURCE_SUMMARIES",
"page": {
"pageSize": 200
},
"selectedCompleteOnly": true
}
}
}
examples/v1/job-committed-event.json
+50
View File
@@ -0,0 +1,50 @@
{
"protocolVersion": {
"major": 1,
"minor": 0
},
"messageId": "f337dc9c-84b8-42c6-a2ae-52f8f91258a8",
"correlationId": "6bfccbf4-866f-46fe-bf67-d8943c1ece91",
"sentAt": "2026-07-22T12:08:00Z",
"jobEvent": {
"eventId": "ecfbd79e-e0fe-41da-ab77-012cde0ed53f",
"jobId": "fd6844d0-ed76-475e-8375-2f98d8eea470",
"sequence": "23",
"jobRevision": "9",
"type": "JOB_EVENT_TYPE_COMMITTED",
"state": "JOB_STATE_RUNNING",
"committed": true,
"progress": {
"step": "JOB_STEP_KIND_QB_VERIFY",
"displayStepNumber": 4,
"displayStepTotal": 5,
"state": "STEP_STATE_SUCCEEDED",
"fractionComplete": 1,
"bytesComplete": "10735000000",
"bytesTotal": "10735000000",
"detail": "target selection passed a stopped full recheck",
"lastProgressAt": "2026-07-22T12:08:00Z",
"overallFractionComplete": 0.95
},
"observedPlacement": {
"resourceId": {
"infoHashV1Hex": "0123456789abcdef0123456789abcdef01234567"
},
"clientId": "lithium",
"state": "PLACEMENT_STATE_PRESENT",
"verifiedFiles": {
"ranges": [
{
"first": 2,
"last": 4
}
]
},
"verifiedLogicalBytes": "10735000000",
"generation": "4",
"verifiedAt": "2026-07-22T12:08:00Z",
"createdByJobId": "fd6844d0-ed76-475e-8375-2f98d8eea470"
},
"occurredAt": "2026-07-22T12:08:00Z"
}
}
examples/v1/job-progress-event.json
+31
View File
@@ -0,0 +1,31 @@
{
"protocolVersion": {
"major": 1,
"minor": 0
},
"messageId": "44b64096-cd58-4f49-bae4-d829f50f787c",
"correlationId": "6bfccbf4-866f-46fe-bf67-d8943c1ece91",
"sentAt": "2026-07-22T12:04:00Z",
"jobEvent": {
"eventId": "18cf6a34-b0a5-429e-b56a-9228b2dca640",
"jobId": "fd6844d0-ed76-475e-8375-2f98d8eea470",
"sequence": "17",
"jobRevision": "6",
"type": "JOB_EVENT_TYPE_PROGRESS",
"state": "JOB_STATE_RUNNING",
"progress": {
"step": "JOB_STEP_KIND_SYNCTHING_TRANSFER",
"displayStepNumber": 2,
"displayStepTotal": 5,
"state": "STEP_STATE_RUNNING",
"fractionComplete": 0.4231,
"bytesComplete": "4543000000",
"bytesTotal": "10735000000",
"approximateBytesPerSecond": 12450000,
"detail": "x1 to lithium",
"lastProgressAt": "2026-07-22T12:04:00Z",
"overallFractionComplete": 0.2846
},
"occurredAt": "2026-07-22T12:04:00Z"
}
}
examples/v1/register-request.json
+38
View File
@@ -0,0 +1,38 @@
{
"protocolVersion": {
"major": 1,
"minor": 0
},
"messageId": "0d6d31f8-1242-4a40-a261-433897102fe4",
"sentAt": "2026-07-22T12:00:00Z",
"registerRequest": {
"protocolVersion": {
"major": 1,
"minor": 0
},
"client": {
"clientId": "x1",
"displayName": "Cache x1",
"role": "CLIENT_ROLE_CACHE"
},
"connectionInstanceId": "fd4f7c2a-836a-4699-8078-f0ec8601febe",
"sharedToken": "redacted-example-token",
"capabilities": {
"features": [
"CLIENT_FEATURE_INVENTORY_CHUNKS",
"CLIENT_FEATURE_CONTENT_TREE",
"CLIENT_FEATURE_ROUTE_PROVISIONING",
"CLIENT_FEATURE_EVICTION"
],
"syncthingDeviceId": "CACHE1-DEVICE-ID",
"syncthingAdvertisedAddresses": [
"dynamic"
],
"qbittorrentVersion": "v5.1.2",
"qbittorrentWebApiVersion": "2.11.3",
"libtorrentVersion": "2.0.11.0",
"syncthingVersion": "v2.0.0",
"maxEnvelopeBytes": 1048576
}
}
}
proto/archive_control/v1/client.proto
+90
View File
@@ -0,0 +1,90 @@
syntax = "proto3";
package archive_control.v1;
import "archive_control/v1/common.proto";
import "archive_control/v1/job.proto";
import "archive_control/v1/route.proto";
import "google/protobuf/duration.proto";
enum ClientFeature {
CLIENT_FEATURE_UNSPECIFIED = 0;
CLIENT_FEATURE_INVENTORY_CHUNKS = 1;
CLIENT_FEATURE_CONTENT_TREE = 2;
CLIENT_FEATURE_ROUTE_PROVISIONING = 3;
CLIENT_FEATURE_HARD_LINK = 4;
CLIENT_FEATURE_REFLINK = 5;
CLIENT_FEATURE_SPARSE_FILES = 6;
CLIENT_FEATURE_PARTFILE_MIGRATION = 7;
CLIENT_FEATURE_PARTFILE_MERGE = 8;
CLIENT_FEATURE_QB_STOPPED_ADD = 9;
CLIENT_FEATURE_EVICTION = 10;
CLIENT_FEATURE_DB_BACKUP = 11;
}
message FilesystemCapabilities {
string root_name = 1;
bool readable = 2;
bool writable = 3;
bool hard_link = 4;
bool reflink = 5;
bool sparse_files = 6;
string detail = 7;
}
message ClientCapabilities {
repeated ClientFeature features = 1;
repeated ServiceHealth services = 2;
repeated FilesystemCapabilities filesystems = 3;
repeated LocalRoute routes = 4;
string syncthing_device_id = 5;
repeated string syncthing_advertised_addresses = 6;
string qbittorrent_version = 7;
string qbittorrent_web_api_version = 8;
string libtorrent_version = 9;
string syncthing_version = 10;
repeated string supported_partfile_formats = 11;
uint32 max_envelope_bytes = 12;
}
message ActiveJobCursor {
string job_id = 1;
uint64 job_revision = 2;
uint64 last_event_sequence = 3;
JobState state = 4;
bool committed = 5;
}
// RegisterRequest must be the first application message on each WebSocket.
// shared_token is forbidden in every other message and must never be logged.
message RegisterRequest {
ProtocolVersion protocol_version = 1;
ClientIdentity client = 2;
string connection_instance_id = 3;
string shared_token = 4;
ClientCapabilities capabilities = 5;
repeated ActiveJobCursor active_jobs = 6;
}
enum RegistrationStatus {
REGISTRATION_STATUS_UNSPECIFIED = 0;
REGISTRATION_STATUS_ACCEPTED = 1;
REGISTRATION_STATUS_REJECTED = 2;
}
message RegisterResponse {
RegistrationStatus status = 1;
ProtocolVersion negotiated_version = 2;
string connection_id = 3;
google.protobuf.Duration heartbeat_interval = 4;
google.protobuf.Duration heartbeat_timeout = 5;
Error error = 6;
}
message Heartbeat {
uint64 sequence = 1;
}
message HeartbeatAck {
uint64 sequence = 1;
}
proto/archive_control/v1/common.proto
+75
View File
@@ -0,0 +1,75 @@
syntax = "proto3";
package archive_control.v1;
import "google/protobuf/timestamp.proto";
// ProtocolVersion is negotiated during connection registration. A different
// major version is incompatible; minor versions are feature-negotiated.
message ProtocolVersion {
uint32 major = 1;
uint32 minor = 2;
}
enum ClientRole {
CLIENT_ROLE_UNSPECIFIED = 0;
CLIENT_ROLE_ARCHIVE = 1;
CLIENT_ROLE_CACHE = 2;
}
enum HealthState {
HEALTH_STATE_UNSPECIFIED = 0;
HEALTH_STATE_HEALTHY = 1;
HEALTH_STATE_DEGRADED = 2;
HEALTH_STATE_UNHEALTHY = 3;
HEALTH_STATE_UNKNOWN = 4;
}
enum ErrorCode {
ERROR_CODE_UNSPECIFIED = 0;
ERROR_CODE_INVALID_ARGUMENT = 1;
ERROR_CODE_UNAUTHENTICATED = 2;
ERROR_CODE_PROTOCOL_VERSION_MISMATCH = 3;
ERROR_CODE_CLIENT_ID_CONFLICT = 4;
ERROR_CODE_NOT_FOUND = 5;
ERROR_CODE_CONFLICT = 6;
ERROR_CODE_STALE_STATE = 7;
ERROR_CODE_UNSUPPORTED = 8;
ERROR_CODE_PRECONDITION_FAILED = 9;
ERROR_CODE_RESOURCE_EXHAUSTED = 10;
ERROR_CODE_UNAVAILABLE = 11;
ERROR_CODE_TIMEOUT = 12;
ERROR_CODE_CANCELLED = 13;
ERROR_CODE_INTEGRITY_CHECK_FAILED = 14;
ERROR_CODE_PATH_CONFLICT = 15;
ERROR_CODE_PERMISSION_DENIED = 16;
ERROR_CODE_MANUAL_INTERVENTION_REQUIRED = 17;
ERROR_CODE_INTERNAL = 18;
}
message Error {
ErrorCode code = 1;
string message = 2;
bool retryable = 3;
map<string, string> details = 4;
}
message ServiceHealth {
string service = 1;
HealthState state = 2;
string version = 3;
string api_version = 4;
string detail = 5;
google.protobuf.Timestamp checked_at = 6;
}
message ClientIdentity {
string client_id = 1;
string display_name = 2;
ClientRole role = 3;
}
message PageRequest {
uint32 page_size = 1;
string page_token = 2;
}
proto/archive_control/v1/control.proto
+130
View File
@@ -0,0 +1,130 @@
syntax = "proto3";
package archive_control.v1;
import "archive_control/v1/common.proto";
import "archive_control/v1/inventory.proto";
import "archive_control/v1/job.proto";
import "archive_control/v1/resource.proto";
import "archive_control/v1/route.proto";
import "google/protobuf/timestamp.proto";
message AssignJobCommand {
JobDefinition job = 1;
uint64 expected_job_revision = 2;
uint64 expected_last_event_sequence = 3;
}
message ExecuteStepCommand {
string job_id = 1;
uint64 expected_job_revision = 2;
JobStepKind step = 3;
uint32 attempt = 4;
uint64 expected_last_event_sequence = 5;
}
message CancelJobCommand {
string job_id = 1;
uint64 expected_job_revision = 2;
string reason = 3;
uint64 expected_last_event_sequence = 4;
}
message EnsureRouteCommand {
EnsureRouteSpec route = 1;
}
message RequestJobSnapshotCommand {
repeated string job_ids = 1;
}
message Command {
string command_id = 1;
google.protobuf.Timestamp created_at = 2;
oneof payload {
AssignJobCommand assign_job = 10;
ExecuteStepCommand execute_step = 11;
CancelJobCommand cancel_job = 12;
EnsureRouteCommand ensure_route = 13;
InventoryQuery inventory_query = 14;
RequestJobSnapshotCommand request_job_snapshot = 15;
}
}
enum CommandAckStatus {
COMMAND_ACK_STATUS_UNSPECIFIED = 0;
COMMAND_ACK_STATUS_ACCEPTED = 1;
COMMAND_ACK_STATUS_DUPLICATE = 2;
COMMAND_ACK_STATUS_REJECTED = 3;
}
// CommandAck confirms durable acceptance/rejection, not completion. Completion
// is reported by JobEvent, InventoryChunk, RouteUpdate, or JobSnapshot.
message CommandAck {
string command_id = 1;
CommandAckStatus status = 2;
Error error = 3;
}
enum JobEventType {
JOB_EVENT_TYPE_UNSPECIFIED = 0;
JOB_EVENT_TYPE_ASSIGNED = 1;
JOB_EVENT_TYPE_STEP_STARTED = 2;
JOB_EVENT_TYPE_PROGRESS = 3;
JOB_EVENT_TYPE_WAITING = 4;
JOB_EVENT_TYPE_STALLED = 5;
JOB_EVENT_TYPE_STEP_SUCCEEDED = 6;
JOB_EVENT_TYPE_COMMITTED = 7;
JOB_EVENT_TYPE_CANCELLING = 8;
JOB_EVENT_TYPE_ROLLBACK_STARTED = 9;
JOB_EVENT_TYPE_ROLLBACK_SUCCEEDED = 10;
JOB_EVENT_TYPE_CLEANUP_REQUIRED = 11;
JOB_EVENT_TYPE_SUCCEEDED = 12;
JOB_EVENT_TYPE_FAILED = 13;
JOB_EVENT_TYPE_CANCELLED = 14;
}
message JobEvent {
string event_id = 1;
string job_id = 2;
uint64 sequence = 3;
uint64 job_revision = 4;
JobEventType type = 5;
JobState state = 6;
bool committed = 7;
JobProgress progress = 8;
Error error = 9;
ResourceStateFingerprint observed_resource = 10;
Placement observed_placement = 11;
google.protobuf.Timestamp occurred_at = 12;
}
message JobSnapshot {
JobRecord job = 1;
uint64 last_event_sequence = 2;
repeated string in_flight_command_ids = 3;
}
message ClientStateSnapshot {
string snapshot_id = 1;
repeated JobSnapshot active_jobs = 2;
repeated LocalRoute routes = 3;
repeated ServiceHealth services = 4;
google.protobuf.Timestamp observed_at = 5;
}
message RouteUpdate {
string command_id = 1;
LocalRoute route = 2;
RouteVerification verification = 3;
Error error = 4;
string update_id = 5;
// Monotonic for this command_id; duplicate sequences must have identical
// content and gaps are reconciled through ClientStateSnapshot.
uint64 sequence = 6;
}
message ProtocolError {
Error error = 1;
string offending_message_id = 2;
}
proto/archive_control/v1/envelope.proto
+33
View File
@@ -0,0 +1,33 @@
syntax = "proto3";
package archive_control.v1;
import "archive_control/v1/client.proto";
import "archive_control/v1/common.proto";
import "archive_control/v1/control.proto";
import "archive_control/v1/inventory.proto";
import "google/protobuf/timestamp.proto";
// Envelope is the only top-level WebSocket application message. UUID fields
// use lowercase canonical UUID text. correlation_id links a response/event to
// the initiating envelope when applicable.
message Envelope {
ProtocolVersion protocol_version = 1;
string message_id = 2;
string correlation_id = 3;
google.protobuf.Timestamp sent_at = 4;
oneof payload {
RegisterRequest register_request = 10;
RegisterResponse register_response = 11;
Heartbeat heartbeat = 12;
HeartbeatAck heartbeat_ack = 13;
Command command = 14;
CommandAck command_ack = 15;
InventoryChunk inventory_chunk = 16;
JobEvent job_event = 17;
JobSnapshot job_snapshot = 18;
ClientStateSnapshot client_state_snapshot = 19;
RouteUpdate route_update = 20;
ProtocolError protocol_error = 21;
}
}
proto/archive_control/v1/inventory.proto
+55
View File
@@ -0,0 +1,55 @@
syntax = "proto3";
package archive_control.v1;
import "archive_control/v1/common.proto";
import "archive_control/v1/resource.proto";
enum InventoryScope {
INVENTORY_SCOPE_UNSPECIFIED = 0;
INVENTORY_SCOPE_RESOURCE_SUMMARIES = 1;
INVENTORY_SCOPE_RESOURCE_LOOKUP = 2;
INVENTORY_SCOPE_CONTENT_TREE = 3;
INVENTORY_SCOPE_PLACEMENTS = 4;
}
message InventoryQuery {
string query_id = 1;
InventoryScope scope = 2;
repeated ResourceId resource_ids = 3;
PageRequest page = 4;
string page_filter = 5;
bool selected_complete_only = 6;
string expected_revision = 7;
}
message ResourceSummaryChunk {
repeated ResourceSummary resources = 1;
}
message ContentTreeChunk {
ResourceId resource_id = 1;
repeated ContentTreeEntry entries = 2;
string resource_content_revision = 3;
}
message PlacementChunk {
repeated Placement placements = 1;
}
// InventoryChunk is an atomic-snapshot stream. Receivers commit results only
// after a last_chunk=true chunk arrives for the same snapshot and revision.
message InventoryChunk {
string query_id = 1;
string snapshot_id = 2;
string revision = 3;
uint32 chunk_index = 4;
bool last_chunk = 5;
string next_page_token = 6;
Error error = 7;
oneof payload {
ResourceSummaryChunk resource_summaries = 10;
ContentTreeChunk content_tree = 11;
PlacementChunk placements = 12;
}
}
proto/archive_control/v1/job.proto
+130
View File
@@ -0,0 +1,130 @@
syntax = "proto3";
package archive_control.v1;
import "archive_control/v1/common.proto";
import "archive_control/v1/resource.proto";
import "google/protobuf/timestamp.proto";
enum JobOperation {
JOB_OPERATION_UNSPECIFIED = 0;
JOB_OPERATION_ARCHIVE = 1;
JOB_OPERATION_UNARCHIVE = 2;
JOB_OPERATION_EVICT_CACHE = 3;
}
enum JobState {
JOB_STATE_UNSPECIFIED = 0;
JOB_STATE_QUEUED = 1;
JOB_STATE_PREPARING = 2;
JOB_STATE_RUNNING = 3;
JOB_STATE_WAITING = 4;
JOB_STATE_STALLED = 5;
JOB_STATE_CANCELLING = 6;
JOB_STATE_ROLLING_BACK = 7;
JOB_STATE_CLEANUP_REQUIRED = 8;
JOB_STATE_SUCCEEDED = 9;
JOB_STATE_FAILED = 10;
JOB_STATE_CANCELLED = 11;
}
enum JobStepKind {
JOB_STEP_KIND_UNSPECIFIED = 0;
JOB_STEP_KIND_ROUTE_SETUP = 1;
JOB_STEP_KIND_PREFLIGHT = 2;
JOB_STEP_KIND_SOURCE_STAGE = 3;
JOB_STEP_KIND_SYNCTHING_TRANSFER = 4;
JOB_STEP_KIND_TARGET_MATERIALIZE = 5;
JOB_STEP_KIND_QB_VERIFY = 6;
JOB_STEP_KIND_STAGING_CLEANUP = 7;
JOB_STEP_KIND_VERIFY_ARCHIVE_COVERAGE = 8;
JOB_STEP_KIND_QB_REMOVE_ENTRY = 9;
JOB_STEP_KIND_SAFE_FILE_UNLINK = 10;
JOB_STEP_KIND_ROLLBACK = 11;
}
enum StepState {
STEP_STATE_UNSPECIFIED = 0;
STEP_STATE_NOT_STARTED = 1;
STEP_STATE_QUEUED = 2;
STEP_STATE_RUNNING = 3;
STEP_STATE_WAITING = 4;
STEP_STATE_STALLED = 5;
STEP_STATE_SUCCEEDED = 6;
STEP_STATE_FAILED = 7;
STEP_STATE_CANCELLED = 8;
}
message TransferJobSpec {
string source_client_id = 1;
string target_client_id = 2;
string route_id = 3;
SelectionSet requested_files = 4;
ResourceStateFingerprint source_fingerprint = 5;
ResourceStateFingerprint target_baseline_fingerprint = 6;
// Absent for a new target placement. Presence distinguishes an absent
// placement from an existing placement whose generation is invalidly zero.
optional uint64 expected_target_placement_generation = 7;
SelectionSet target_baseline_files = 8;
SelectionSet transfer_delta_files = 9;
uint64 requested_logical_bytes = 10;
uint64 transfer_delta_logical_bytes = 11;
}
message ArchiveCoverage {
string archive_client_id = 1;
SelectionSet covered_files = 2;
uint64 placement_generation = 3;
ResourceStateFingerprint fingerprint = 4;
}
message EvictionJobSpec {
string cache_client_id = 1;
SelectionSet files_to_evict = 2;
uint64 cache_placement_generation = 3;
ResourceStateFingerprint cache_fingerprint = 4;
repeated ArchiveCoverage archive_coverage = 5;
}
// JobDefinition is immutable after acceptance. Runtime state and calculated
// transfer deltas belong in JobRecord and TransferManifest.
message JobDefinition {
string job_id = 1;
string idempotency_key = 2;
JobOperation operation = 3;
ResourceId resource_id = 4;
string resource_display_name = 5;
google.protobuf.Timestamp created_at = 6;
oneof spec {
TransferJobSpec transfer = 10;
EvictionJobSpec eviction = 11;
}
}
message JobProgress {
JobStepKind step = 1;
uint32 display_step_number = 2;
uint32 display_step_total = 3;
StepState state = 4;
double fraction_complete = 5;
uint64 bytes_complete = 6;
uint64 bytes_total = 7;
double approximate_bytes_per_second = 8;
string detail = 9;
google.protobuf.Timestamp last_progress_at = 10;
// Weighted progress across all visible steps. This remains available while
// a job is waiting or stalled; fraction_complete is for the current step.
double overall_fraction_complete = 11;
}
message JobRecord {
JobDefinition definition = 1;
JobState state = 2;
bool committed = 3;
uint64 revision = 4;
JobProgress progress = 5;
Error error = 6;
google.protobuf.Timestamp updated_at = 7;
google.protobuf.Timestamp committed_at = 8;
google.protobuf.Timestamp finished_at = 9;
}
proto/archive_control/v1/resource.proto
+121
View File
@@ -0,0 +1,121 @@
syntax = "proto3";
package archive_control.v1;
import "google/protobuf/timestamp.proto";
// ResourceId models BitTorrent v1, v2, and hybrid torrent identity. Hex values
// are lowercase and have exact SHA-1/SHA-256 lengths. At least one is required.
message ResourceId {
string info_hash_v1_hex = 1;
string info_hash_v2_hex = 2;
}
// FileIndexRange is inclusive at both ends. Ranges in a SelectionSet must be
// sorted, non-overlapping, and coalesced.
message FileIndexRange {
uint32 first = 1;
uint32 last = 2;
}
message SelectionSet {
repeated FileIndexRange ranges = 1;
}
enum TorrentRuntimeState {
TORRENT_RUNTIME_STATE_UNSPECIFIED = 0;
TORRENT_RUNTIME_STATE_STOPPED = 1;
TORRENT_RUNTIME_STATE_QUEUED = 2;
TORRENT_RUNTIME_STATE_CHECKING = 3;
TORRENT_RUNTIME_STATE_DOWNLOADING = 4;
TORRENT_RUNTIME_STATE_SEEDING = 5;
TORRENT_RUNTIME_STATE_STALLED = 6;
TORRENT_RUNTIME_STATE_ERROR = 7;
TORRENT_RUNTIME_STATE_MISSING = 8;
}
enum ContentEntryType {
CONTENT_ENTRY_TYPE_UNSPECIFIED = 0;
CONTENT_ENTRY_TYPE_FILE = 1;
CONTENT_ENTRY_TYPE_DIRECTORY = 2;
CONTENT_ENTRY_TYPE_PADDING_FILE = 3;
}
message TorrentFile {
uint32 file_index = 1;
string canonical_path = 2;
uint64 logical_bytes = 3;
uint64 allocated_bytes = 4;
uint64 completed_bytes = 5;
bool selected = 6;
bool sparse = 7;
bool padding = 8;
}
// ContentTreeEntry is a flat tree record. Directory selections are expanded by
// the source into available_file_indices before a job is created.
message ContentTreeEntry {
string canonical_path = 1;
string parent_path = 2;
string display_name = 3;
ContentEntryType type = 4;
optional uint32 file_index = 5;
SelectionSet available_file_indices = 6;
uint64 available_logical_bytes = 7;
uint32 available_file_count = 8;
// Present for FILE and PADDING_FILE entries. The duplicated canonical_path
// must match this tree entry; receivers reject inconsistent records.
TorrentFile file = 9;
}
message ResourceSummary {
ResourceId resource_id = 1;
// qb_torrent_id is meaningful only to the reporting qBittorrent instance.
string qb_torrent_id = 2;
string display_name = 3;
TorrentRuntimeState runtime_state = 4;
SelectionSet selected_files = 5;
SelectionSet selected_complete_files = 6;
uint64 selected_logical_bytes = 7;
uint64 selected_complete_bytes = 8;
uint64 total_logical_bytes = 9;
uint32 total_file_count = 10;
string content_revision = 11;
bool canonical_paths = 12;
google.protobuf.Timestamp observed_at = 13;
}
// ResourceStateFingerprint is an optimistic concurrency token for active jobs.
message ResourceStateFingerprint {
ResourceId resource_id = 1;
string client_id = 2;
string qb_torrent_id = 3;
string content_revision = 4;
SelectionSet selected_files = 5;
SelectionSet selected_complete_files = 6;
TorrentRuntimeState runtime_state = 7;
string save_path_fingerprint = 8;
google.protobuf.Timestamp observed_at = 9;
}
enum PlacementState {
PLACEMENT_STATE_UNSPECIFIED = 0;
PLACEMENT_STATE_PRESENT = 1;
PLACEMENT_STATE_MERGING = 2;
PLACEMENT_STATE_EVICTING = 3;
PLACEMENT_STATE_ABSENT = 4;
PLACEMENT_STATE_ERROR = 5;
PLACEMENT_STATE_UNKNOWN = 6;
}
message Placement {
ResourceId resource_id = 1;
string client_id = 2;
PlacementState state = 3;
SelectionSet verified_files = 4;
uint64 verified_logical_bytes = 5;
uint64 generation = 6;
ResourceStateFingerprint fingerprint = 7;
google.protobuf.Timestamp verified_at = 8;
string created_by_job_id = 9;
}
proto/archive_control/v1/route.proto
+70
View File
@@ -0,0 +1,70 @@
syntax = "proto3";
package archive_control.v1;
import "google/protobuf/timestamp.proto";
enum SyncthingFolderType {
SYNCTHING_FOLDER_TYPE_UNSPECIFIED = 0;
SYNCTHING_FOLDER_TYPE_SEND_RECEIVE = 1;
SYNCTHING_FOLDER_TYPE_SEND_ONLY = 2;
SYNCTHING_FOLDER_TYPE_RECEIVE_ONLY = 3;
SYNCTHING_FOLDER_TYPE_OTHER = 4;
}
enum RouteState {
ROUTE_STATE_UNSPECIFIED = 0;
ROUTE_STATE_DISCOVERED = 1;
ROUTE_STATE_PROVISIONING = 2;
ROUTE_STATE_VERIFYING = 3;
ROUTE_STATE_READY = 4;
ROUTE_STATE_PAUSED = 5;
ROUTE_STATE_UNHEALTHY = 6;
ROUTE_STATE_UNSUPPORTED = 7;
}
// LocalRoute is a client's non-secret view of one Syncthing folder. route_id is
// the Syncthing folder ID. Archive Control routes must have exactly two devices.
message LocalRoute {
string route_id = 1;
string local_relative_path = 2;
SyncthingFolderType folder_type = 3;
string local_syncthing_device_id = 4;
repeated string peer_syncthing_device_ids = 5;
RouteState state = 6;
bool writable = 7;
bool sparse_supported = 8;
bool archive_control_created = 9;
string detail = 10;
google.protobuf.Timestamp observed_at = 11;
}
message RouteDescriptor {
string route_id = 1;
string archive_client_id = 2;
string cache_client_id = 3;
string archive_syncthing_device_id = 4;
string cache_syncthing_device_id = 5;
RouteState state = 6;
google.protobuf.Timestamp verified_at = 7;
string detail = 8;
}
message EnsureRouteSpec {
string route_id = 1;
string peer_client_id = 2;
string peer_syncthing_device_id = 3;
repeated string peer_addresses = 4;
string local_relative_path = 5;
uint32 setup_timeout_seconds = 6;
}
message RouteVerification {
string route_id = 1;
string nonce = 2;
bool local_nonce_seen_by_peer = 3;
bool peer_nonce_seen_locally = 4;
RouteState state = 5;
string detail = 6;
google.protobuf.Timestamp observed_at = 7;
}
proto/archive_control/v1/transfer.proto
+70
View File
@@ -0,0 +1,70 @@
syntax = "proto3";
package archive_control.v1;
import "archive_control/v1/resource.proto";
import "google/protobuf/timestamp.proto";
enum MaterializationMethod {
MATERIALIZATION_METHOD_UNSPECIFIED = 0;
MATERIALIZATION_METHOD_HARD_LINK = 1;
MATERIALIZATION_METHOD_REFLINK = 2;
MATERIALIZATION_METHOD_COPY = 3;
MATERIALIZATION_METHOD_PREEXISTING_REUSED = 4;
}
enum ArtifactKind {
ARTIFACT_KIND_UNSPECIFIED = 0;
ARTIFACT_KIND_TORRENT_FILE = 1;
ARTIFACT_KIND_LIBTORRENT_PARTFILE = 2;
ARTIFACT_KIND_OTHER_AUXILIARY = 3;
}
message ManifestFile {
uint32 file_index = 1;
string payload_relative_path = 2;
string target_canonical_path = 3;
uint64 logical_bytes = 4;
uint64 allocated_bytes = 5;
bool sparse = 6;
MaterializationMethod source_staging_method = 7;
MaterializationMethod target_method = 8;
bool target_preexisted = 9;
}
message ManifestArtifact {
ArtifactKind kind = 1;
string payload_relative_path = 2;
uint64 logical_bytes = 3;
uint64 allocated_bytes = 4;
bool sparse = 5;
string format = 6;
string sha256_hex = 7;
}
// TransferManifest is written as canonical protobuf JSON into the isolated
// Syncthing job namespace. Paths are always relative to configured safe roots.
message TransferManifest {
uint32 manifest_version = 1;
string job_id = 2;
ResourceId resource_id = 3;
string source_client_id = 4;
string target_client_id = 5;
string route_id = 6;
SelectionSet requested_files = 7;
SelectionSet target_baseline_files = 8;
SelectionSet transfer_delta_files = 9;
repeated ManifestFile files = 10;
repeated ManifestArtifact artifacts = 11;
ResourceStateFingerprint source_fingerprint = 12;
ResourceStateFingerprint target_baseline_fingerprint = 13;
google.protobuf.Timestamp created_at = 14;
}
// ReadyMarker is written last. A destination trusts no manifest until the
// marker exists and its digest matches the manifest bytes.
message ReadyMarker {
string job_id = 1;
string manifest_sha256_hex = 2;
google.protobuf.Timestamp ready_at = 3;
}
scripts/buf.sh
Executable
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env bash
set -euo pipefail
BUF_IMAGE="${BUF_IMAGE:-bufbuild/buf:1.47.2}"
action="${1:-}"
case "$action" in
format)
shift
set -- format -w "$@"
;;
lint)
shift
set -- lint "$@"
;;
build)
shift
set -- build -o descriptor.bin "$@"
;;
generate)
shift
set -- generate "$@"
;;
breaking)
shift
if [[ $# -ne 1 ]]; then
echo "usage: $0 breaking <against>" >&2
exit 2
fi
set -- breaking --against "$1"
;;
*)
echo "usage: $0 {format|lint|build|generate|breaking <against>}" >&2
exit 2
;;
esac
exec docker run --rm --user "$(id -u):$(id -g)" \
-e XDG_CACHE_HOME=/tmp \
-v "$(pwd):/workspace" -w /workspace "$BUF_IMAGE" "$@"
scripts/validate-examples.sh
Executable
+13
View File
@@ -0,0 +1,13 @@
#!/usr/bin/env bash
set -euo pipefail
BUF_IMAGE="${BUF_IMAGE:-bufbuild/buf:1.47.2}"
for example in examples/v1/*.json; do
docker run --rm --user "$(id -u):$(id -g)" -e XDG_CACHE_HOME=/tmp \
-v "$(pwd):/workspace" -w /workspace "$BUF_IMAGE" \
convert proto \
--from "$example#format=json" \
--to "/tmp/example.binpb#format=binpb" \
--type archive_control.v1.Envelope
done