lab5: RabbitMQ, relay и worker в стенде; reconciler lab4 убран

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

broker: RabbitMQ 4.3.7 (тег+digest), quorum queue + DLX/DLQ из definitions, пользователи
relay (только publish в exchange) и worker (только consume), без management UI и портов.
relay: outbox -> publish с publisher confirm, reaper истёкших lease, страховка от потерянных
сообщений. worker: claim + heartbeat lease, варианты, публикация с fencing, ack после COMMIT,
graceful stop, уборка staging и мусора попыток. S3: два bucket и четыре identity
(admin, api, worker, auditor) с проверкой модели доступа в s3-init. Отдельная сеть queue.
Reconciler lab4 (трек прошлой лабы) удалён: его роль выполняет уборка по журналу попыток.
This commit is contained in:
Alexey Bodanyuk
2026-10-31 16:12:52 +03:00
parent 655f264104
commit bc32a50967
15 changed files with 1058 additions and 141 deletions
+34 -2
View File
@@ -24,6 +24,8 @@ S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
# Lab 5: private staging bucket for originals accepted before 202 (API: Write; worker: Read+Write).
S3_STAGING_BUCKET=messenger-staging
# Contract limits (may be lowered, never raised above 10485760 / 40000000).
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
@@ -38,8 +40,33 @@ S3_TIMEOUT_SECONDS=3
# Janitor: 'pending' intents older than this are failed uploads (object + row removed).
PENDING_GRACE_SECONDS=900
JANITOR_INTERVAL_SECONDS=60
# Level 4 failure experiments only (after_s3_put, after_ready_commit); keep empty.
FAULT_POINTS=
# Lab 5: broker credentials are files too (relay and worker get their OWN RabbitMQ user/URL;
# secrets/broker_url_relay|worker are mounted as /run/secrets/broker_url). The API has none.
BROKER_URL_FILE=/run/secrets/broker_url
# Jobs processed at once by one worker = RabbitMQ prefetch; image child processes at once.
WORKER_CONCURRENCY=2
WORKER_REPLICAS=1
# Managed attempts (claims) per job, then failed/retry_exhausted; backoff = full jitter.
JOB_MAX_ATTEMPTS=5
JOB_RETRY_BASE_SECONDS=2
JOB_RETRY_MAX_SECONDS=60
# One attempt: hard time limit (child process killed) and the lease renewed by its heartbeat.
JOB_TIMEOUT_SECONDS=60
JOB_LEASE_SECONDS=30
# Address-space limit of the image child process (RLIMIT_AS).
IMAGE_MAX_MEMORY_MB=768
# Staging copy of a FAILED attachment is kept this long for a replay (ready: deleted at once).
STAGING_RETENTION_SECONDS=86400
# Objects of attempts that never published are deleted this long after the attempt ended.
ORPHAN_GRACE_SECONDS=600
# Queued job without a claim for this long gets a new outbox row (lost-message guard).
QUEUED_REPUBLISH_SECONDS=300
# Pipeline version of new uploads (track 2: versions 1, 2, 3 in src/messenger/pipeline.py).
PIPELINE_VERSION=1
# Failure experiments only, per service (see src/messenger/config.py FAULT_POINTS); keep empty.
API_FAULT_POINTS=
RELAY_FAULT_POINTS=
WORKER_FAULT_POINTS=
# Additional settings of this implementation (defaults shown; all optional):
LOG_LEVEL=info
DB_POOL_MIN=1
@@ -60,6 +87,11 @@ PG_CPUS=2.0
PG_MEMORY=1g
VALKEY_CPUS=0.5
LB_CPUS=1.0
BROKER_CPUS=1.0
BROKER_MEMORY=512m
WORKER_CPUS=2.0
WORKER_MEMORY=1536m
RELAY_CPUS=0.5
PG_MAX_CONNECTIONS=100
# Host ports of the load balancer (published on 127.0.0.1 only; API replicas publish nothing)
APP_PUBLISH_PORT=8080
+3 -2
View File
@@ -1,6 +1,7 @@
# syntax=docker/dockerfile:1
# Messenger API, lab 4 (replicas behind HAProxy, PostgreSQL + Valkey + S3). Built by compose.yaml
# (services api, migrate, s3-init, reconciler: one image, different commands).
# Messenger, lab 5 (API replicas behind HAProxy, PostgreSQL + Valkey + S3 + RabbitMQ). Built by
# compose.yaml (services api, migrate, s3-init, relay, worker, auditor, jobctl: one image, different
# commands; only the worker imports Pillow).
#
# Python is interpreted, so there is no separate compile stage; the image
# contains only the interpreter, locked dependencies and the source code.
+29 -18
View File
@@ -29,23 +29,23 @@ test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
# ---------------------------------------------------------------------------
# Student solution (lab 4, Python). On Windows: PowerShell/Git Bash with GNU make, public
# Student solution (lab 5, Python). On Windows: PowerShell/Git Bash with GNU make, public
# tests need PYTHONUTF8=1, and the hooks are started through scripts/*.cmd (Git Bash).
.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series reconcile reconcile-apply bench-memory bench-reconcile
.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series bench-memory worker-stop worker-start jobs failed audit track2
secrets:
$(PYTHON) scripts/init_secrets.py
# Build, apply migrations ONCE (service migrate), start API replicas and the LB, wait for healthy.
# Build, apply migrations ONCE (service migrate), start broker, API replicas, LB, relay and worker.
up: secrets
docker compose up -d --build --wait
# Safe stop: containers stop, named volumes (data) stay.
# Safe stop: containers stop, named volumes (data, queue) stay.
down:
docker compose down
logs:
docker compose logs --tail 50 api lb
docker compose logs --tail 50 api lb relay worker
ps:
docker compose ps -a
@@ -67,10 +67,32 @@ restore:
rolling-update:
sh scripts/rolling-update.sh
# Level 4: stop each replica in turn under load, measure RPS/errors/recovery (evidence/failover/).
# Lab 3: stop each replica in turn under load, measure RPS/errors/recovery (evidence/failover/).
failover-series:
$(PYTHON) bench/failover_series.py --output evidence/failover
# Lab 5 demo: stop the worker (uploads still get 202, content answers 409), start it again.
worker-stop:
docker compose stop worker
worker-start:
docker compose start worker
# Lab 5 operator view: jobs by state + outbox backlog; failed registry (replay: see REPORT.md).
jobs:
docker compose run --rm --no-deps jobctl stats
failed:
docker compose run --rm --no-deps jobctl list-failed
# Track 2: read-only check of the pipeline invariants over PostgreSQL and both buckets.
audit:
docker compose run --rm --no-deps auditor
# Track 2: automatic fault-injection matrix (duplicate, reorder, crashes, stale versions).
track2:
$(PYTHON) bench/pipeline_faults.py --output evidence/track2
# Own tests: unit + black-box against the running stand (needs requirements*.txt in a venv).
student-test:
$(PYTHON) -m pytest student-tests -v
@@ -81,17 +103,6 @@ unit-test:
explain:
$(PYTHON) scripts/explain_history.py --output evidence/explain-history.txt
# Track 2: reconciler of S3 objects vs attachment rows (dry-run by default, read-only).
reconcile:
docker compose run --rm reconciler
reconcile-apply:
docker compose run --rm reconciler --apply
# Level 4: peak RAM of the replicas under 16 simultaneous uploads (evidence/memory/).
# Lab 4: peak RAM of the replicas under 16 simultaneous uploads (evidence/memory/).
bench-memory:
$(PYTHON) bench/upload_memory.py --output evidence/memory
# Track 2: cost of one reconciler pass for 1k/10k/50k attachments (evidence/reconcile/).
bench-reconcile:
$(PYTHON) bench/reconcile_cost.py --output evidence/reconcile
+225 -42
View File
@@ -1,15 +1,20 @@
# Lab 4 stand: client -> HAProxy (lb) -> API x N (one scalable service `api`)
# -> PostgreSQL (metadata, source of truth) + Valkey (sessions) + SeaweedFS S3 (bytes).
# Lab 5 stand: client -> HAProxy (lb) -> API x N (scalable service `api`)
# -> PostgreSQL (metadata, jobs, outbox: source of truth) + Valkey (sessions)
# + SeaweedFS S3 (staging bucket and final bucket, both private)
# relay : outbox in PostgreSQL -> RabbitMQ (publisher confirms) [service `relay`]
# broker : RabbitMQ 4.3, durable quorum queue + dead-letter queue [service `broker`]
# worker : RabbitMQ -> variants -> final bucket + PostgreSQL, then ack [service `worker`]
# From a clean clone (details and Windows notes: REPORT.md, docs/WINDOWS.md):
# python scripts/init_secrets.py # once: random local credentials into ./secrets (git-ignored)
# cp .env.example .env # optional overrides (API_REPLICAS, limits, ...)
# docker compose --env-file .env up -d --build --wait
# Order: postgres/valkey/s3 healthy -> migrate + s3-init (ONE run each, exit 0) -> api healthy -> lb.
# Order: postgres/valkey/s3/broker healthy -> migrate + s3-init (ONE run each, exit 0)
# -> api healthy -> lb; relay and worker start after migrate/s3-init.
# Images are pinned by version AND digest. Passwords/keys are passed only as secret files, and
# every service gets only the secrets it needs. Only the LB publishes ports (loopback only).
# Three networks: `edge` (lb <-> api), `data` (api/migrate/reconciler <-> postgres/valkey) and
# `storage` (api/s3-init/reconciler <-> s3); data and storage are internal (no route outside).
name: mtusi-lab4
# Networks: `edge` (lb <-> api); internal (no route outside): `data` (postgres/valkey),
# `storage` (s3), `queue` (broker). The API is NOT on `queue`: it never talks to the broker.
name: mtusi-lab5
x-api-env: &api-env
APP_ENV: ${APP_ENV:-lab}
@@ -29,10 +34,11 @@ x-api-env: &api-env
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-8}
DOCS_ENABLED: ${DOCS_ENABLED:-true}
MESSAGE_WRITE_PATH: ${MESSAGE_WRITE_PATH:-function}
# Lab 4: S3 (the API's own Read+Write key pair, see infra/seaweedfs/start.sh) and upload limits.
# S3: the API's own key pair (Write staging + Read final, see infra/seaweedfs/start.sh).
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3}
@@ -46,8 +52,33 @@ x-api-env: &api-env
JPEG_MAX_MEMORY_MB: ${JPEG_MAX_MEMORY_MB:-256}
PENDING_GRACE_SECONDS: ${PENDING_GRACE_SECONDS:-900}
JANITOR_INTERVAL_SECONDS: ${JANITOR_INTERVAL_SECONDS:-60}
# Level 4 failure experiments only (compose.faults.yaml); empty = no injected failures.
FAULT_POINTS: ${FAULT_POINTS:-}
# Lab 5: pipeline version and attempt budget of the jobs created by uploads.
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
# Failure experiments only (student-tests, bench/); empty = no injected failures.
FAULT_POINTS: ${API_FAULT_POINTS:-}
# Settings of the pipeline shared by relay and worker (docs/PIPELINE.md).
x-pipeline-env: &pipeline-env
APP_ENV: ${APP_ENV:-lab}
LOG_LEVEL: ${LOG_LEVEL:-info}
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
DB_POOL_MAX: ${PIPELINE_DB_POOL_MAX:-6}
DB_TIMEOUT_SECONDS: ${DB_TIMEOUT_SECONDS:-3}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging}
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-30}
JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-2}
JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:-60}
QUEUED_REPUBLISH_SECONDS: ${QUEUED_REPUBLISH_SECONDS:-300}
services:
postgres:
@@ -102,15 +133,15 @@ services:
resources:
limits: {cpus: "${VALKEY_CPUS:-0.5}", memory: 256m}
# Lab 4: S3-compatible object storage (SeaweedFS: master + volume + filer + S3 gateway in one
# process). Bytes of the attachments live ONLY here, on the named volume s3data. Identities
# and exposure: infra/seaweedfs/start.sh. Not published on the host; its web UIs listen on
# 127.0.0.1 inside the container only.
# S3-compatible object storage (SeaweedFS: master + volume + filer + S3 gateway in one process).
# Bytes live ONLY here, on the named volume s3data: staging bucket (originals accepted before
# 202) and final bucket (published originals and variants). Identities: infra/seaweedfs/start.sh.
s3:
image: chrislusf/seaweedfs:4.46@sha256:08d516132314207d10c8e37cbffc1f32b147d870169688734cc61c6231625b62
entrypoint: ["/bin/sh", "/etc/seaweedfs/start.sh"]
environment:
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging}
# Go heap target below the container limit: the GC returns memory before the cgroup OOM.
GOMEMLIMIT: ${S3_GOMEMLIMIT:-600MiB}
secrets:
@@ -118,8 +149,10 @@ services:
- s3_admin_secret_access_key
- s3_api_access_key_id
- s3_api_secret_access_key
- s3_reconciler_access_key_id
- s3_reconciler_secret_access_key
- s3_worker_access_key_id
- s3_worker_secret_access_key
- s3_auditor_access_key_id
- s3_auditor_secret_access_key
volumes:
- ./infra/seaweedfs/start.sh:/etc/seaweedfs/start.sh:ro
- s3data:/data
@@ -137,6 +170,34 @@ services:
resources:
limits: {cpus: "${S3_CPUS:-1.0}", memory: "${S3_MEMORY:-1g}"}
# Lab 5: RabbitMQ. Topology (exchange, quorum queue, DLX/DLQ) and users come from definition
# files imported at boot (infra/rabbitmq, secrets/rabbitmq_users.json). Messages of the quorum
# queue live on the named volume rabbitmqdata. No management UI, no host port.
broker:
image: rabbitmq:4.3.7-alpine@sha256:b1f997f74103d08fc1f20a13890dc44fe6cfa51d528cf521b480ada988d3ffa5
hostname: broker # node name rabbit@broker: stable across container re-creation
secrets:
- source: rabbitmq_users
target: /etc/rabbitmq/definitions.d/users.json
volumes:
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/conf.d/20-lab5.conf:ro
- ./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro
- ./infra/rabbitmq/topology.json:/etc/rabbitmq/definitions.d/topology.json:ro
- rabbitmqdata:/var/lib/rabbitmq
healthcheck:
# The node runs and the AMQP listener accepts connections.
test: ["CMD-SHELL", "rabbitmq-diagnostics -q ping && rabbitmq-diagnostics -q check_port_connectivity"]
interval: 5s
timeout: 10s
retries: 30
start_period: 20s
stop_grace_period: 30s
restart: unless-stopped
networks: [queue]
deploy:
resources:
limits: {cpus: "${BROKER_CPUS:-1.0}", memory: "${BROKER_MEMORY:-512m}"}
# One-shot schema migration: runs ONCE per `up`, before any API replica starts
# (api depends_on service_completed_successfully). Replicas never migrate themselves.
migrate:
@@ -144,7 +205,7 @@ services:
context: .
args:
BUILD_REVISION: ${BUILD_REVISION:-dev}
image: ${API_IMAGE:-mtusi-messenger:lab4}
image: ${API_IMAGE:-mtusi-messenger:lab5}
command: ["python", "-m", "messenger.migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
@@ -160,28 +221,33 @@ services:
resources:
limits: {cpus: "0.5", memory: 256m}
# One-shot, idempotent S3 bootstrap: bucket, sentinel object, verification of the access model
# (private bucket, least-privilege keys). The ONLY holder of the admin key pair besides s3.
# One-shot, idempotent S3 bootstrap: both buckets, sentinel object, verification of the access
# model (private buckets, least-privilege keys). The ONLY holder of the admin key pair besides s3.
s3-init:
image: ${API_IMAGE:-mtusi-messenger:lab4}
image: ${API_IMAGE:-mtusi-messenger:lab5}
command: ["python", "-m", "messenger.s3_bootstrap"]
environment:
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging}
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_access_key
S3_RECONCILER_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id
S3_RECONCILER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key
S3_WORKER_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key_id
S3_WORKER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_access_key
S3_AUDITOR_ACCESS_KEY_ID_FILE: /run/secrets/s3_auditor_access_key_id
S3_AUDITOR_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_auditor_secret_access_key
secrets:
- s3_admin_access_key_id
- s3_admin_secret_access_key
- s3_api_access_key_id
- s3_api_secret_access_key
- s3_reconciler_access_key_id
- s3_reconciler_secret_access_key
- s3_worker_access_key_id
- s3_worker_secret_access_key
- s3_auditor_access_key_id
- s3_auditor_secret_access_key
depends_on:
s3:
condition: service_healthy
@@ -200,7 +266,7 @@ services:
context: .
args:
BUILD_REVISION: ${BUILD_REVISION:-dev}
image: ${API_IMAGE:-mtusi-messenger:lab4}
image: ${API_IMAGE:-mtusi-messenger:lab5}
environment: *api-env
secrets:
- database_url
@@ -236,7 +302,7 @@ services:
replicas: ${API_REPLICAS:-2}
resources:
# 512m: UPLOAD_CONCURRENCY (4) x 10 MiB bodies + one djpeg (<= JPEG_MAX_MEMORY_MB) + runtime;
# measured in evidence/memory/ (bench/upload_memory.py).
# measured in lab 4 (bench/upload_memory.py). No resizing happens in the API.
limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-512m}"}
# No swap on top of the limit (Docker's default would allow as much swap again): the RAM bound
# of the experiments is a real bound, and running out of it is visible, not hidden in swap.
@@ -268,29 +334,112 @@ services:
resources:
limits: {cpus: "${LB_CPUS:-1.0}", memory: 128m}
# Track 2 (lab 4): reconciler of S3 objects vs PostgreSQL metadata. Not a daemon: started on
# demand, dry-run by default (read-only on both stores):
# docker compose run --rm reconciler # report only
# docker compose run --rm reconciler --apply # fix: orphans, stale intents, quarantine
reconciler:
# Lab 5: outbox relay + lease reaper (src/messenger/relay.py). One process; a second one would
# be safe (SKIP LOCKED) but is not needed. Database + broker only, no S3 keys.
relay:
image: ${API_IMAGE:-mtusi-messenger:lab5}
command: ["python", "-m", "messenger.relay"]
environment:
<<: *pipeline-env
RELAY_POLL_SECONDS: ${RELAY_POLL_SECONDS:-0.5}
FAULT_POINTS: ${RELAY_FAULT_POINTS:-}
secrets:
- database_url
- source: broker_url_relay
target: broker_url
healthcheck:
test: ["CMD", "python", "-m", "messenger.healthcheck", "--file", "/tmp/heartbeat", "--max-age", "30"]
interval: 5s
timeout: 3s
retries: 3
start_period: 10s
depends_on:
postgres:
condition: service_healthy
broker:
condition: service_healthy
migrate:
condition: service_completed_successfully
stop_grace_period: 10s
restart: unless-stopped
networks: [data, queue]
deploy:
resources:
limits: {cpus: "${RELAY_CPUS:-0.5}", memory: 256m}
# Lab 5: image worker (src/messenger/worker.py). Scalable (`--scale worker=2`): competing
# consumers of one queue; the database lease decides who works on a job.
worker:
image: ${API_IMAGE:-mtusi-messenger:lab5}
command: ["python", "-m", "messenger.worker"]
environment:
<<: *pipeline-env
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
WORKER_SHUTDOWN_SECONDS: ${WORKER_SHUTDOWN_SECONDS:-20}
IMAGE_MAX_MEMORY_MB: ${IMAGE_MAX_MEMORY_MB:-768}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-600}
CLEANUP_INTERVAL_SECONDS: ${CLEANUP_INTERVAL_SECONDS:-30}
FAULT_POINTS: ${WORKER_FAULT_POINTS:-}
secrets:
- database_url
- source: broker_url_worker
target: broker_url
- source: s3_worker_access_key_id
target: s3_access_key_id
- source: s3_worker_secret_access_key
target: s3_secret_access_key
healthcheck:
test: ["CMD", "python", "-m", "messenger.healthcheck", "--file", "/tmp/heartbeat", "--max-age", "30"]
interval: 5s
timeout: 3s
retries: 3
start_period: 10s
depends_on:
postgres:
condition: service_healthy
broker:
condition: service_healthy
s3:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
stop_grace_period: 30s # > WORKER_SHUTDOWN_SECONDS (20 s) + release
restart: unless-stopped
networks: [data, storage, queue]
deploy:
replicas: ${WORKER_REPLICAS:-1}
resources:
# WORKER_CONCURRENCY (2) child processes x IMAGE_MAX_MEMORY_MB (768) address space + worker.
limits: {cpus: "${WORKER_CPUS:-2.0}", memory: "${WORKER_MEMORY:-1536m}"}
memswap_limit: ${WORKER_MEMORY:-1536m}
# Track 2: read-only checker of the pipeline invariants (database in read-only transactions,
# S3 auditor key: Read + List, no Write). On demand: docker compose run --rm auditor
auditor:
profiles: ["tools"]
image: ${API_IMAGE:-mtusi-messenger:lab4}
entrypoint: ["python", "-m", "messenger.reconcile"]
image: ${API_IMAGE:-mtusi-messenger:lab5}
entrypoint: ["python", "-m", "messenger.audit"]
command: []
environment:
DATABASE_URL_FILE: /run/secrets/database_url
DB_TIMEOUT_SECONDS: ${RECONCILE_DB_TIMEOUT_SECONDS:-10}
DB_TIMEOUT_SECONDS: ${AUDIT_DB_TIMEOUT_SECONDS:-30}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3}
secrets:
- database_url
- source: s3_reconciler_access_key_id
- source: s3_auditor_access_key_id
target: s3_access_key_id
- source: s3_reconciler_secret_access_key
- source: s3_auditor_secret_access_key
target: s3_secret_access_key
depends_on:
postgres:
@@ -301,19 +450,43 @@ services:
networks: [data, storage]
deploy:
resources:
limits: {cpus: "${RECONCILER_CPUS:-1.0}", memory: 256m}
limits: {cpus: "1.0", memory: 256m}
# Operator CLI (replay of failed jobs, reprocess with a newer pipeline, failed registry):
# docker compose run --rm jobctl list-failed
# docker compose run --rm jobctl replay <attachment_id> --operator alice --reason "..."
jobctl:
profiles: ["tools"]
image: ${API_IMAGE:-mtusi-messenger:lab5}
entrypoint: ["python", "-m", "messenger.jobctl"]
command: []
environment:
DATABASE_URL_FILE: /run/secrets/database_url
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
secrets: [database_url]
depends_on:
postgres:
condition: service_healthy
restart: "no"
networks: [data]
deploy:
resources:
limits: {cpus: "0.5", memory: 128m}
networks:
edge:
data:
internal: true # no route outside: stores talk only to api/migrate
internal: true # no route outside: stores talk only to the services above
storage:
internal: true # S3 API reachable only by api, s3-init, reconciler
internal: true # S3 API reachable only by api, s3-init, worker, auditor
queue:
internal: true # broker reachable only by relay and worker
volumes:
pgdata:
valkeydata:
s3data:
rabbitmqdata:
secrets:
postgres_password:
@@ -332,7 +505,17 @@ secrets:
file: ./secrets/s3_api_access_key_id
s3_api_secret_access_key:
file: ./secrets/s3_api_secret_access_key
s3_reconciler_access_key_id:
file: ./secrets/s3_reconciler_access_key_id
s3_reconciler_secret_access_key:
file: ./secrets/s3_reconciler_secret_access_key
s3_worker_access_key_id:
file: ./secrets/s3_worker_access_key_id
s3_worker_secret_access_key:
file: ./secrets/s3_worker_secret_access_key
s3_auditor_access_key_id:
file: ./secrets/s3_auditor_access_key_id
s3_auditor_secret_access_key:
file: ./secrets/s3_auditor_secret_access_key
rabbitmq_users:
file: ./secrets/rabbitmq_users.json
broker_url_relay:
file: ./secrets/broker_url_relay
broker_url_worker:
file: ./secrets/broker_url_worker
+1
View File
@@ -0,0 +1 @@
[].
+24
View File
@@ -0,0 +1,24 @@
# RabbitMQ 4.3 of the lab 5 stand (compose service `broker`), mounted as conf.d/20-lab5.conf.
# Only AMQP 0-9-1 on 5672 inside the internal network `queue`: no management UI, no Prometheus
# listener (enabled_plugins is empty), nothing published on the host.
listeners.tcp.default = 5672
# Users, permissions and topology are imported from files at every boot (idempotent):
# /etc/rabbitmq/definitions.d/topology.json exchanges, quorum queues, bindings (this repo)
# /etc/rabbitmq/definitions.d/users.json users relay/worker + permissions (Docker secret,
# generated by scripts/init_secrets.py)
# With imported definitions no default `guest` user is created.
definitions.import_backend = local_filesystem
definitions.local.path = /etc/rabbitmq/definitions.d
definitions.skip_if_unchanged = false
# A delivery that stays unacked longer than this closes the consumer's channel and the message
# is returned to the queue (a worker that hangs while connected). Far above JOB_TIMEOUT_SECONDS.
consumer_timeout = 300000
# The container has a 512 MiB limit; flow control starts well before the cgroup OOM.
vm_memory_high_watermark.relative = 0.5
disk_free_limit.absolute = 200MB
log.console = true
log.console.level = info
+21
View File
@@ -0,0 +1,21 @@
{
"vhosts": [{"name": "/"}],
"exchanges": [
{"name": "messenger.jobs", "vhost": "/", "type": "direct", "durable": true, "auto_delete": false,
"internal": false, "arguments": {}},
{"name": "messenger.jobs.dlx", "vhost": "/", "type": "fanout", "durable": true, "auto_delete": false,
"internal": false, "arguments": {}}
],
"queues": [
{"name": "messenger.attachment-jobs", "vhost": "/", "durable": true, "auto_delete": false,
"arguments": {"x-queue-type": "quorum", "x-delivery-limit": 20, "x-dead-letter-exchange": "messenger.jobs.dlx"}},
{"name": "messenger.attachment-jobs.dlq", "vhost": "/", "durable": true, "auto_delete": false,
"arguments": {"x-queue-type": "quorum"}}
],
"bindings": [
{"source": "messenger.jobs", "vhost": "/", "destination": "messenger.attachment-jobs", "destination_type": "queue",
"routing_key": "attachment.process.v1", "arguments": {}},
{"source": "messenger.jobs.dlx", "vhost": "/", "destination": "messenger.attachment-jobs.dlq",
"destination_type": "queue", "routing_key": "", "arguments": {}}
]
}
+19 -12
View File
@@ -2,11 +2,14 @@
# Entry point of the compose service `s3` (SeaweedFS 4.46, one container: master + volume +
# filer + S3 gateway). Renders the S3 identities from Docker secrets, then execs `weed`.
#
# Identities (SeaweedFS s3.json, actions scoped to the bucket):
# bootstrap-admin Admin (create bucket) -> only s3-init has these keys
# messenger-api Read + Write on S3_BUCKET -> API replicas (no List, no Admin)
# messenger-reconciler Read + Write + List on S3_BUCKET -> reconciler CLI (needs the listing)
# There is NO anonymous identity, so every unsigned request is refused: the bucket is private.
# Two private buckets (lab 5): S3_STAGING_BUCKET holds originals accepted by the API before 202,
# S3_BUCKET holds the published originals and variants written by the worker.
# Identities (SeaweedFS s3.json, actions scoped to a bucket):
# bootstrap-admin Admin (create buckets) -> only s3-init has these keys
# messenger-api Write staging, Read final -> API replicas (no List, no Admin)
# messenger-worker Read + Write staging, Read + Write final -> worker (no List, no Admin)
# messenger-auditor Read + List both buckets (no Write) -> invariant checker (tools profile)
# There is NO anonymous identity, so every unsigned request is refused: the buckets are private.
#
# Exposure: master (9333), volume (8080), filer (8888) with their web UIs and gRPC listen on
# 127.0.0.1 INSIDE the container only (-ip.bind); only the S3 API port 8333 listens on the
@@ -15,6 +18,7 @@ set -eu
umask 077
secret() { tr -d '\r\n' < "/run/secrets/$1"; }
bucket="${S3_BUCKET:?S3_BUCKET is required}"
staging="${S3_STAGING_BUCKET:?S3_STAGING_BUCKET is required}"
cat > /tmp/s3.json <<EOF
{
"identities": [
@@ -23,17 +27,20 @@ cat > /tmp/s3.json <<EOF
"actions": ["Admin", "Read", "Write", "List", "Tagging"]},
{"name": "messenger-api",
"credentials": [{"accessKey": "$(secret s3_api_access_key_id)", "secretKey": "$(secret s3_api_secret_access_key)"}],
"actions": ["Read:${bucket}", "Write:${bucket}"]},
{"name": "messenger-reconciler",
"credentials": [{"accessKey": "$(secret s3_reconciler_access_key_id)", "secretKey": "$(secret s3_reconciler_secret_access_key)"}],
"actions": ["Read:${bucket}", "Write:${bucket}", "List:${bucket}"]}
"actions": ["Write:${staging}", "Read:${bucket}"]},
{"name": "messenger-worker",
"credentials": [{"accessKey": "$(secret s3_worker_access_key_id)", "secretKey": "$(secret s3_worker_secret_access_key)"}],
"actions": ["Read:${staging}", "Write:${staging}", "Read:${bucket}", "Write:${bucket}"]},
{"name": "messenger-auditor",
"credentials": [{"accessKey": "$(secret s3_auditor_access_key_id)", "secretKey": "$(secret s3_auditor_secret_access_key)"}],
"actions": ["Read:${staging}", "List:${staging}", "Read:${bucket}", "List:${bucket}"]}
]
}
EOF
# Memory: by default SeaweedFS buffers every in-flight upload without a limit and the Go runtime
# keeps freed heap; measured ~1 GB under 16 parallel 9.4 MiB PUTs (evidence/memory). The upload
# limits make excess PUTs wait inside SeaweedFS instead of growing the heap, and GOMEMLIMIT (set
# in compose.yaml below the container limit) makes the Go GC return memory before the cgroup OOM.
# keeps freed heap; measured ~1 GB under 16 parallel 9.4 MiB PUTs (lab 4). The upload limits make
# excess PUTs wait inside SeaweedFS instead of growing the heap, and GOMEMLIMIT (set in
# compose.yaml below the container limit) makes the Go GC return memory before the cgroup OOM.
exec weed server \
-dir=/data \
-ip=127.0.0.1 -ip.bind=127.0.0.1 \
+32 -5
View File
@@ -9,6 +9,9 @@ Delete ./secrets together with the volumes (`docker compose down -v`) to start o
"""
from __future__ import annotations
import base64
import hashlib
import json
import os
import secrets
import sys
@@ -24,8 +27,8 @@ def write(name: str, value: str) -> None:
print(f"keep secrets/{name}")
return
path.write_text(value + "\n", encoding="utf-8", newline="\n")
# Readable by the container users (postgres, valkey, seaweedfs, app uid 10001) through the
# compose bind mount. Local synthetic lab credentials only.
# Readable by the container users (postgres, valkey, seaweedfs, rabbitmq, app uid 10001)
# through the compose bind mount. Local synthetic lab credentials only.
os.chmod(path, 0o644)
print(f"created secrets/{name}")
@@ -34,17 +37,41 @@ def read(name: str) -> str:
return (SECRETS / name).read_text(encoding="utf-8").strip()
def rabbit_hash(password: str) -> str:
"""RabbitMQ `rabbit_password_hashing_sha256`: base64(salt + sha256(salt + password)), 4-byte salt.
The broker stores and compares only this hash; the plain password is in the client's URL."""
salt = secrets.token_bytes(4)
return base64.b64encode(salt + hashlib.sha256(salt + password.encode("utf-8")).digest()).decode("ascii")
def main() -> int:
SECRETS.mkdir(exist_ok=True)
write("postgres_password", secrets.token_urlsafe(24)) # [A-Za-z0-9_-]: safe inside URLs
write("valkey_password", secrets.token_urlsafe(24))
write("database_url", f"postgresql://messenger:{read('postgres_password')}@postgres:5432/messenger")
write("cache_url", f"redis://:{read('valkey_password')}@valkey:6379/0")
# Lab 4: three separate S3 key pairs (SeaweedFS identities, infra/seaweedfs/start.sh):
# admin = bucket bootstrap only; api = Read+Write on the bucket; reconciler = + List.
for who in ("admin", "api", "reconciler"):
# S3 key pairs (SeaweedFS identities, infra/seaweedfs/start.sh): admin = bucket bootstrap only;
# api = Write staging + Read final; worker = Read/Write both; auditor = Read + List (no Write).
for who in ("admin", "api", "worker", "auditor"):
write(f"s3_{who}_access_key_id", "AK" + secrets.token_hex(9).upper())
write(f"s3_{who}_secret_access_key", secrets.token_hex(20))
# Lab 5: RabbitMQ users. relay may only publish to the jobs exchange, worker may only consume
# the jobs queue; neither may declare/delete anything (configure "^$").
write("rabbitmq_relay_password", secrets.token_urlsafe(24))
write("rabbitmq_worker_password", secrets.token_urlsafe(24))
users = {
"users": [{"name": who, "password_hash": rabbit_hash(read(f"rabbitmq_{who}_password")),
"hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []} for who in ("relay", "worker")],
"permissions": [
{"user": "relay", "vhost": "/", "configure": "^$", "write": "^messenger\\.jobs$", "read": "^$"},
{"user": "worker", "vhost": "/", "configure": "^$", "write": "^$", "read": "^messenger\\.attachment-jobs$"},
],
}
write("rabbitmq_users.json", json.dumps(users, indent=1))
for who in ("relay", "worker"):
# AMQP heartbeat 10 s: a frozen client loses its connection after ~2 missed heartbeats, and
# the broker returns its unacked messages to the queue for the other consumers.
write(f"broker_url_{who}", f"amqp://{who}:{read(f'rabbitmq_{who}_password')}@broker:5672/?heartbeat=10")
return 0
+12 -9
View File
@@ -1,18 +1,21 @@
#!/bin/sh
# Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`.
# Restarts EVERY process of the stand: LB, all API replicas, PostgreSQL, Valkey and S3 (SeaweedFS).
# Volumes, schema and data are preserved: no `down -v`, no FLUSH, no re-seed.
# Restarts EVERY process of the stand: LB, all API replicas, relay, worker(s), PostgreSQL, Valkey,
# S3 (SeaweedFS) and the broker (RabbitMQ). Volumes, schema, data and queued messages are
# preserved: no `down -v`, no FLUSH, no purge, no re-seed.
# Output is suppressed by the test; run it by hand to see the steps.
set -eu
cd "$(dirname "$0")/.."
# 1. Entry point and API first: no request is served while the stores restart underneath.
docker compose stop lb api
# 1. Entry point, API and the pipeline processes first: nothing is served or processed while the
# stores restart underneath.
docker compose stop lb api relay worker
# 2. Stores: clean shutdown (PostgreSQL checkpoint, Valkey AOF fsync, SeaweedFS volume/filer
# flush), then start again. The S3 bytes live on the named volume s3data.
docker compose restart postgres valkey s3
# 3. API replicas (wait for healthy stores via depends_on), then the LB. HAProxy resolves
# the new replica addresses through Docker DNS by itself - no stale backend list.
# flush, RabbitMQ quorum log), then start again. Bytes live on s3data, messages on rabbitmqdata.
docker compose restart postgres valkey s3 broker
# 3. API replicas (wait for healthy stores via depends_on), the LB, then relay and worker. HAProxy
# resolves the new replica addresses through Docker DNS by itself - no stale backend list.
docker compose start api
docker compose start lb
echo "restart.sh: lb, api (all replicas), postgres, valkey and s3 restarted with volumes preserved"
docker compose start relay worker
echo "restart.sh: lb, api (all replicas), relay, worker, postgres, valkey, s3 and broker restarted with volumes preserved"
+15 -3
View File
@@ -1,12 +1,24 @@
"""Docker HEALTHCHECK probe without curl/wget in the image: `python -m messenger.healthcheck`."""
"""Docker HEALTHCHECK probe without curl/wget in the image: `python -m messenger.healthcheck`.
(no arguments) API: GET /health/ready on the local port
--file /tmp/heartbeat --max-age 30 relay/worker: the main loop wrote its heartbeat recently
(it writes it only while its database/broker work)
"""
from __future__ import annotations
import os
import sys
import time
import urllib.request
def main() -> int:
def main(argv: list[str]) -> int:
if len(argv) == 4 and argv[0] == "--file" and argv[2] == "--max-age":
try:
with open(argv[1], encoding="ascii") as f:
return 0 if time.time() - float(f.read().strip()) <= float(argv[3]) else 1
except (OSError, ValueError):
return 1
port = os.environ.get("APP_PORT", "8080").strip() or "8080"
try:
with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/ready", timeout=2) as response:
@@ -16,4 +28,4 @@ def main() -> int:
if __name__ == "__main__":
sys.exit(main())
sys.exit(main(sys.argv[1:]))
+163
View File
@@ -0,0 +1,163 @@
"""Outbox relay: `python -m messenger.relay` (compose service `relay`, one process).
loop: SELECT due outbox rows FOR UPDATE SKIP LOCKED
-> publish each job message to RabbitMQ, wait for the publisher CONFIRM
-> UPDATE outbox SET published_at -> COMMIT
every few seconds: reaper (expired leases -> queued/failed), lost-message guard.
Why an outbox: the API commits the job and the outbox row in ONE PostgreSQL transaction and never
talks to the broker. A crash between COMMIT and publish leaves the row unpublished; the relay
publishes it when it (or the broker) is back - nobody re-creates a job by hand. While the broker
is down, uploads still get 202 and the rows wait in the outbox.
A crash after the confirm but before the COMMIT of published_at publishes the row once more after
the restart: the queue then holds a duplicate. That is at-least-once delivery; the worker turns
the duplicate into a no-op (jobs.claim). The relay never deletes or rewrites jobs.
"""
from __future__ import annotations
import asyncio
import logging
import sys
import time
import aio_pika
from aio_pika.abc import AbstractChannel, AbstractConnection, AbstractExchange
from . import broker, jobs, logging_setup, service
from .config import ConfigError, Settings
from .db import Database
from .errors import Unavailable
log = logging.getLogger("messenger.relay")
ACTOR = "relay"
MAINTENANCE_SECONDS = 5.0
PURGE_SECONDS = 600.0
class Relay:
def __init__(self, settings: Settings):
self.settings = settings
self.db = Database(settings, name="messenger-relay")
self.connection: AbstractConnection | None = None
self.channel: AbstractChannel | None = None
self.exchange: AbstractExchange | None = None
async def connect(self) -> None:
self.connection = await aio_pika.connect(
self.settings.broker_url, timeout=5,
client_properties={"connection_name": "relay@" + self.settings.instance_id})
# Confirm mode: every publish() waits for basic.ack of the broker (message in the quorum log).
self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True)
# No declare: the topology belongs to the broker definitions; the relay may only write.
self.exchange = await self.channel.get_exchange(broker.EXCHANGE, ensure=False)
log.info("broker_connected", extra={"fields": {"exchange": broker.EXCHANGE}})
async def disconnect(self) -> None:
if self.connection is not None:
try:
await self.connection.close()
except Exception: # noqa: BLE001 - already broken
pass
self.connection = self.channel = self.exchange = None
async def publish_due(self) -> int:
settings = self.settings
exchange = self.exchange
async def work(conn) -> list[jobs.Due]:
rows = await jobs.due_outbox(conn, settings.relay_batch)
if "reverse_order" in settings.fault_points:
rows.reverse()
# Rows of jobs that are no longer queued (finished meanwhile) need no message.
sendable = [r for r in rows if r.job_state == "queued"]
copies = 2 if "duplicate_publish" in settings.fault_points else 1
for row in sendable:
message = aio_pika.Message(
broker.encode(row.job_id, row.attachment_id, row.pipeline_version, row.attempt, row.created_at),
delivery_mode=aio_pika.DeliveryMode.PERSISTENT, content_type="application/json",
message_id=f"{row.job_id}:{row.outbox_id}", type=broker.JOB_TYPE)
for _ in range(copies):
await exchange.publish(message, routing_key=broker.ROUTING_KEY, mandatory=True, timeout=5)
for row in sendable:
# Confirmed by the broker, outbox not marked yet: the window "after publish, before
# the confirm is recorded". The crash rolls this transaction back.
await service.crash_once(settings, self.db, "crash_after_publish", row.job_id, ACTOR,
outbox_id=row.outbox_id)
await jobs.mark_published(conn, [r.outbox_id for r in rows])
return sendable
sent = await self.db.run(work, deadline=10 + 5 * settings.relay_batch)
for row in sent:
log.info("job_published", extra={"fields": {"job_id": row.job_id, "attempt": row.attempt,
"outbox_id": row.outbox_id}})
return len(sent)
async def maintenance(self) -> None:
expired = await self.db.run(lambda c: jobs.expire_leases(c, ACTOR))
republished = await self.db.run(lambda c: jobs.republish_stale(c, self.settings.queued_republish_seconds, ACTOR))
if expired or republished:
log.info("maintenance", extra={"fields": {"leases_expired": expired, "republished": republished}})
async def run(self) -> int:
stop = service.stop_event()
await self.db.open()
next_maintenance = next_purge = 0.0
failures = 0
try:
while not stop.is_set():
healthy = True
now = time.monotonic()
try:
if now >= next_maintenance:
await self.maintenance()
next_maintenance = now + MAINTENANCE_SECONDS
if now >= next_purge:
purged = await self.db.run(jobs.purge_outbox)
next_purge = now + PURGE_SECONDS
if purged:
log.info("outbox_purged", extra={"fields": {"rows": purged}})
except Unavailable:
healthy = False
try:
if self.exchange is None or self.connection is None or self.connection.is_closed:
await self.disconnect()
await self.connect()
sent = await self.publish_due()
failures = 0
except Unavailable:
healthy, sent = False, 0
except (aio_pika.exceptions.AMQPError, ConnectionError, OSError, asyncio.TimeoutError) as exc:
# Broker down/unreachable or a message returned/nacked: nothing marked, the rows
# stay in the outbox (transaction rolled back) and are published later.
failures += 1
healthy, sent = False, 0
log.warning("publish_failed", extra={"fields": {"error": type(exc).__name__, "failures": failures}})
await self.disconnect()
if healthy:
service.beat()
if not sent:
await service.sleep_or_stop(stop, min(5.0, self.settings.relay_poll_seconds * (2 ** min(failures, 3))))
finally:
await self.disconnect()
await self.db.close()
log.info("relay_stopped")
return 0
def main() -> int:
try:
settings = Settings.from_env(s3_credentials=False)
if not settings.broker_url:
raise ConfigError("BROKER_URL_FILE (or BROKER_URL) must be set")
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
logging_setup.configure(settings.log_level)
log.info("relay_starting", extra={"fields": {"instance_id": settings.instance_id,
"fault_points": sorted(settings.fault_points)}})
return asyncio.run(Relay(settings).run())
if __name__ == "__main__":
sys.exit(main())
+79 -48
View File
@@ -3,14 +3,15 @@
Idempotent, safe to run on every `docker compose up`:
1. waits until the S3 endpoint answers;
2. creates the bucket with the ADMIN key pair if it does not exist yet (an existing bucket and
its objects are left untouched);
3. writes the sentinel object `health/sentinel` that the API's readiness probe HEADs;
2. creates the final bucket (S3_BUCKET) and the staging bucket (S3_STAGING_BUCKET) with the ADMIN
key pair if they do not exist yet (existing buckets and their objects are left untouched);
3. writes the sentinel object `health/sentinel` into the final bucket (the API readiness HEADs it);
4. proves the access model and FAILS the stand (exit 1) if it is wrong:
* anonymous (unsigned) GET of an object and anonymous LIST of the bucket are refused;
* the API key pair can read/write objects of ITS bucket, but can neither list it, nor create
a bucket, nor see or read another bucket (a canary bucket created here for this check);
* the reconciler key pair can list the bucket but cannot create one.
* anonymous (unsigned) GET of an object and anonymous LIST are refused on both buckets;
* api: writes staging, reads final; cannot read staging, write final, list or create buckets,
nor read another bucket (a canary bucket created here for this check);
* worker: reads and writes both buckets; cannot list or create buckets;
* auditor: reads and lists both buckets; cannot write.
Only file names and verdicts are printed, never key values.
"""
from __future__ import annotations
@@ -57,14 +58,33 @@ def settings_with(keys: tuple[str, str]) -> Settings:
return Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024, **base)
def pair(prefix: str) -> tuple[str, str]:
return _secret(f"{prefix}ACCESS_KEY_ID", True), _secret(f"{prefix}SECRET_ACCESS_KEY", True)
async def ensure_bucket(admin_s3: Storage, bucket: str) -> bool:
code = await error_code(admin_s3.client.head_bucket(Bucket=bucket))
if code == "":
say(f"bucket {bucket}: exists, kept as is")
return True
if code in ("404", "NoSuchBucket", "NotFound"):
code = await error_code(admin_s3.client.create_bucket(Bucket=bucket))
if code not in ("", "BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
say(f"bucket {bucket}: create failed ({code})")
return False
say(f"bucket {bucket}: created")
return True
say(f"bucket {bucket}: head failed ({code})")
return False
async def run() -> int:
admin = (_secret("S3_ADMIN_ACCESS_KEY_ID", True), _secret("S3_ADMIN_SECRET_ACCESS_KEY", True))
api = (_secret("S3_ACCESS_KEY_ID", True), _secret("S3_SECRET_ACCESS_KEY", True))
reconciler = (_secret("S3_RECONCILER_ACCESS_KEY_ID", True), _secret("S3_RECONCILER_SECRET_ACCESS_KEY", True))
settings = settings_with(admin)
bucket = settings.s3_bucket
admin_s3, api_s3, rec_s3 = Storage(settings), Storage(settings_with(api)), Storage(settings_with(reconciler))
for s in (admin_s3, api_s3, rec_s3):
settings = settings_with(pair("S3_ADMIN_"))
final, staging = settings.s3_bucket, settings.s3_staging_bucket
admin_s3 = Storage(settings)
clients = {"api": Storage(settings_with(pair("S3_"))), "worker": Storage(settings_with(pair("S3_WORKER_"))),
"auditor": Storage(settings_with(pair("S3_AUDITOR_")))}
for s in (admin_s3, *clients.values()):
await s.open()
try:
deadline = time.monotonic() + 60
@@ -77,64 +97,75 @@ async def run() -> int:
say("S3 endpoint does not answer within 60 s")
return 1
await asyncio.sleep(1)
code = await error_code(admin_s3.client.head_bucket(Bucket=bucket))
if code == "":
say(f"bucket {bucket}: exists, kept as is")
elif code in ("404", "NoSuchBucket", "NotFound"):
code = await error_code(admin_s3.client.create_bucket(Bucket=bucket))
if code not in ("", "BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
say(f"bucket {bucket}: create failed ({code})")
for bucket in (final, staging):
if not await ensure_bucket(admin_s3, bucket):
return 1
say(f"bucket {bucket}: created")
else:
say(f"bucket {bucket}: head failed ({code})")
return 1
await admin_s3.put(SENTINEL_KEY, b"ok\n", "text/plain", {})
await admin_s3.client.put_object(Bucket=staging, Key=SENTINEL_KEY, Body=b"ok\n")
say(f"sentinel {SENTINEL_KEY}: written")
canary = bucket + "-canary"
canary = final + "-canary"
if await error_code(admin_s3.client.head_bucket(Bucket=canary)):
await admin_s3.client.create_bucket(Bucket=canary)
await admin_s3.client.put_object(Bucket=canary, Key="canary", Body=b"secret of another bucket")
failures = []
base = settings.s3_endpoint.rstrip("/")
for what, url in (("anonymous GET object", f"{base}/{bucket}/{SENTINEL_KEY}"),
("anonymous LIST bucket", f"{base}/{bucket}?list-type=2")):
status = anonymous_status(url)
verdict = "refused" if status in (401, 403) else "ALLOWED"
say(f"{what}: HTTP {status} -> {verdict}")
if verdict != "refused":
failures.append(what)
for bucket in (final, staging):
for what, url in ((f"anonymous GET object of {bucket}", f"{base}/{bucket}/{SENTINEL_KEY}"),
(f"anonymous LIST of {bucket}", f"{base}/{bucket}?list-type=2")):
status = anonymous_status(url)
verdict = "refused" if status in (401, 403) else "ALLOWED"
say(f"{what}: HTTP {status} -> {verdict}")
if verdict != "refused":
failures.append(what)
probe = "health/bootstrap-probe"
def c(who: str):
return clients[who].client
# (what, coroutine, allowed?) - every refusal must be a real AccessDenied, not an outage.
checks = [
("api: read object", await error_code(api_s3.client.head_object(Bucket=bucket, Key=SENTINEL_KEY)), True),
("api: write object", await error_code(api_s3.client.put_object(Bucket=bucket, Key=probe, Body=b"p")), True),
("api: delete object", await error_code(api_s3.client.delete_object(Bucket=bucket, Key=probe)), True),
("api: list bucket", await error_code(api_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), False),
("api: create bucket", await error_code(api_s3.client.create_bucket(Bucket=bucket + "-x")), False),
("api: read object of another bucket",
await error_code(api_s3.client.head_object(Bucket=canary, Key="canary")), False),
("reconciler: list bucket", await error_code(rec_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), True),
("reconciler: create bucket", await error_code(rec_s3.client.create_bucket(Bucket=bucket + "-x")), False),
("api: write staging", c("api").put_object(Bucket=staging, Key=probe, Body=b"p"), True),
("api: delete staging (janitor)", c("api").delete_object(Bucket=staging, Key=probe), True),
("api: read staging", c("api").head_object(Bucket=staging, Key=SENTINEL_KEY), False),
("api: read final", c("api").head_object(Bucket=final, Key=SENTINEL_KEY), True),
("api: write final", c("api").put_object(Bucket=final, Key=probe, Body=b"p"), False),
("api: list final", c("api").list_objects_v2(Bucket=final, MaxKeys=1), False),
("api: list staging", c("api").list_objects_v2(Bucket=staging, MaxKeys=1), False),
("api: create bucket", c("api").create_bucket(Bucket=final + "-x"), False),
("api: read another bucket", c("api").head_object(Bucket=canary, Key="canary"), False),
("worker: read staging", c("worker").head_object(Bucket=staging, Key=SENTINEL_KEY), True),
("worker: write final", c("worker").put_object(Bucket=final, Key=probe, Body=b"p"), True),
("worker: delete final", c("worker").delete_object(Bucket=final, Key=probe), True),
("worker: write staging", c("worker").put_object(Bucket=staging, Key=probe, Body=b"p"), True),
("worker: delete staging", c("worker").delete_object(Bucket=staging, Key=probe), True),
("worker: list final", c("worker").list_objects_v2(Bucket=final, MaxKeys=1), False),
("worker: create bucket", c("worker").create_bucket(Bucket=final + "-x"), False),
("worker: read another bucket", c("worker").head_object(Bucket=canary, Key="canary"), False),
("auditor: list final", c("auditor").list_objects_v2(Bucket=final, MaxKeys=1), True),
("auditor: list staging", c("auditor").list_objects_v2(Bucket=staging, MaxKeys=1), True),
("auditor: read final", c("auditor").head_object(Bucket=final, Key=SENTINEL_KEY), True),
("auditor: write final", c("auditor").put_object(Bucket=final, Key=probe, Body=b"p"), False),
("auditor: delete staging", c("auditor").delete_object(Bucket=staging, Key=SENTINEL_KEY), False),
]
for what, code, allowed in checks:
ok = (code == "") == allowed
for what, coro, allowed in checks:
code = await error_code(coro)
ok = (code == "") == allowed and code in ("", "AccessDenied", "403")
say(f"{what}: {'allowed' if code == '' else 'refused (' + code + ')'} -> {'OK' if ok else 'WRONG'}")
if not ok:
failures.append(what)
visible = [b["Name"] for b in (await api_s3.client.list_buckets()).get("Buckets", [])]
visible = [b["Name"] for b in (await c("api").list_buckets()).get("Buckets", [])]
say(f"api: buckets visible in ListBuckets: {visible} -> {'OK' if canary not in visible else 'WRONG'}")
if canary in visible:
failures.append("api sees another bucket")
if failures:
say("access model is WRONG: " + ", ".join(failures))
return 1
say("done: private bucket, least-privilege keys verified")
say("done: private buckets, least-privilege keys verified")
return 0
finally:
for s in (admin_s3, api_s3, rec_s3):
for s in (admin_s3, *clients.values()):
await s.close()
+67
View File
@@ -0,0 +1,67 @@
"""Common runtime of the background processes (relay, worker): signals, heartbeat file for the
Docker healthcheck, failure injection that fires once per job and leaves an audit row."""
from __future__ import annotations
import asyncio
import logging
import os
import signal
import time
from pathlib import Path
from . import jobs
from .config import Settings
from .db import Database
log = logging.getLogger("messenger.service")
HEARTBEAT_FILE = Path(os.environ.get("HEARTBEAT_FILE", "/tmp/heartbeat"))
CRASH_EXIT_CODE = 70 # os._exit: no cleanup, no ack, no COMMIT - like SIGKILL in the middle of work
def beat() -> None:
"""Healthcheck input: `python -m messenger.healthcheck --file /tmp/heartbeat --max-age 30`."""
try:
HEARTBEAT_FILE.write_text(str(time.time()), encoding="ascii")
except OSError:
pass
def stop_event() -> asyncio.Event:
stop = asyncio.Event()
loop = asyncio.get_running_loop()
for sig in (signal.SIGTERM, signal.SIGINT):
try:
loop.add_signal_handler(sig, stop.set)
except (NotImplementedError, RuntimeError):
pass # Windows unit tests: no signal handlers
return stop
async def sleep_or_stop(stop: asyncio.Event, seconds: float) -> None:
try:
await asyncio.wait_for(stop.wait(), seconds)
except asyncio.TimeoutError:
pass
async def crash_once(settings: Settings, db: Database, point: str, job_id: str, actor: str, **detail) -> None:
"""Experiments only (FAULT_POINTS): kill THIS process at `point`, once per job (and per `detail`),
across restarts and replicas. The decision is an audit row committed before the crash, so the
restarted process (or another replica) continues normally and the evidence stays in job_events."""
if point not in settings.fault_points:
return
async def record(conn) -> bool:
cur = await conn.execute(
"SELECT 1 FROM job_events WHERE job_id = %s AND event = 'fault_injected' AND detail->>'point' = %s "
"AND detail @> %s::jsonb", (job_id, point, jobs.json.dumps({k: str(v) for k, v in detail.items()})))
if await cur.fetchone():
return False
await jobs.event(conn, job_id, "fault_injected", actor, point=point, **{k: str(v) for k, v in detail.items()})
return True
if await db.run(record):
log.warning("fault_injected_crash", extra={"fields": {"point": point, "job_id": job_id, **detail}})
logging.shutdown()
os._exit(CRASH_EXIT_CODE)
+334
View File
@@ -0,0 +1,334 @@
"""Image worker: `python -m messenger.worker` (compose service `worker`, separate from the API).
One delivery of a job message:
decode + validate (schema v1; invalid -> reject -> dead-letter queue)
-> CLAIM in PostgreSQL (jobs.claim): finished/leased/not due/stale -> no-op, ack
-> lease heartbeat every JOB_LEASE_SECONDS/3 while the attempt runs
-> read the source (staging copy or published original), check size + SHA-256 against the DB
-> variants in a child process (imaging.py: pixel, memory and time limits)
-> PUT the original (first publication) and the variants under keys of THIS attempt
-> ONE transaction: variants + attachment ready, fenced by the lease token (jobs.complete)
-> ack -> delete the staging copy (marked in the DB first, then S3).
The ack comes only after the durable effect (the COMMIT). A crash anywhere before it makes
RabbitMQ deliver the message again; the job row then says what is left to do. Failures: a
permanent one (invalid_image, image_too_large, source_missing/corrupt) fails the job at once, a
transient one (S3 down, timeout, killed decoder) is retried with backoff via the outbox, at most
JOB_MAX_ATTEMPTS claims in total.
Concurrency: RabbitMQ prefetch = WORKER_CONCURRENCY unacked messages per worker, and at most
WORKER_CONCURRENCY image child processes. Graceful stop (SIGTERM): stop consuming, let running
attempts finish for WORKER_SHUTDOWN_SECONDS, give the rest back (jobs.release).
The worker also runs the cleanup of the pipeline objects (staging copies that are no longer
needed, objects of attempts that never published) - always decided under a row lock.
"""
from __future__ import annotations
import asyncio
import hashlib
import logging
import sys
import time
import aio_pika
from aio_pika.abc import AbstractChannel, AbstractConnection, AbstractIncomingMessage, AbstractQueue
from . import broker, imaging, jobs, logging_setup, service
from .config import ConfigError, Settings
from .db import Database
from .errors import Unavailable
from .imaging import ProcessingError
from .storage import ObjectMissing, Storage
log = logging.getLogger("messenger.worker")
class Worker:
def __init__(self, settings: Settings):
self.settings = settings
self.name = settings.instance_id
self.actor = "worker:" + self.name
self.db = Database(settings, name="messenger-worker")
self.staging = Storage(settings, bucket=settings.s3_staging_bucket)
self.final = Storage(settings)
self.slots = asyncio.Semaphore(settings.worker_concurrency)
self.tasks: set[asyncio.Task] = set()
self.connection: AbstractConnection | None = None
self.channel: AbstractChannel | None = None
self.queue: AbstractQueue | None = None
self.consumer_tag: str | None = None
# ---------------------------------------------------------------- broker
async def connect(self) -> None:
self.connection = await aio_pika.connect(
self.settings.broker_url, timeout=5,
client_properties={"connection_name": "worker@" + self.name})
self.channel = await self.connection.channel()
# Prefetch: the broker keeps at most this many unacked messages in flight to this worker.
await self.channel.set_qos(prefetch_count=self.settings.worker_concurrency)
self.queue = await self.channel.get_queue(broker.QUEUE, ensure=False)
self.consumer_tag = await self.queue.consume(self.on_message, no_ack=False)
log.info("consuming", extra={"fields": {"queue": broker.QUEUE, "prefetch": self.settings.worker_concurrency}})
async def disconnect(self) -> None:
if self.connection is not None:
try:
await self.connection.close()
except Exception: # noqa: BLE001 - already broken
pass
self.connection = self.channel = self.queue = None
self.consumer_tag = None
async def on_message(self, message: AbstractIncomingMessage) -> None:
task = asyncio.create_task(self.deliver(message))
self.tasks.add(task)
task.add_done_callback(self.tasks.discard)
@staticmethod
async def settle(action) -> None:
"""ack/nack/reject; the channel may be gone (broker restart): then the broker requeues the
message by itself and the redelivery is a no-op."""
try:
await action
except Exception as exc: # noqa: BLE001
log.warning("settle_failed", extra={"fields": {"error": type(exc).__name__}})
async def deliver(self, message: AbstractIncomingMessage) -> None:
try:
body = broker.decode(message.body)
except broker.BadMessage as exc:
log.error("bad_message", extra={"fields": {"reason": str(exc), "message_id": message.message_id}})
await self.settle(message.reject(requeue=False)) # -> messenger.attachment-jobs.dlq
return
try:
async with self.slots:
outcome = await self.process(body["job_id"])
except Unavailable:
# PostgreSQL (or S3 during a DB step) unavailable: nothing decided. Back to the queue
# after a pause; a claim that did happen is protected by its lease.
log.warning("job_deferred", extra={"fields": {"job_id": body["job_id"]}})
await asyncio.sleep(2)
await self.settle(message.nack(requeue=True))
return
except asyncio.CancelledError:
# Shutdown: back to the queue. If the job was claimed it has been released (new outbox
# row), and one of the two deliveries becomes a no-op.
await self.settle(asyncio.shield(message.nack(requeue=True)))
raise
log.info("job_done", extra={"fields": {"job_id": body["job_id"], "outcome": outcome,
"message_attempt": body["attempt"], "redelivered": message.redelivered}})
await self.settle(message.ack())
# ---------------------------------------------------------------- one attempt
async def process(self, job_id: str) -> str:
settings = self.settings
claim = await self.db.run(lambda c: jobs.claim(c, job_id, self.name, settings.job_lease_seconds))
if isinstance(claim, jobs.Noop):
return "noop_" + claim.reason
await service.crash_once(settings, self.db, "crash_after_claim", job_id, self.actor)
lost = asyncio.Event()
work = asyncio.create_task(self.attempt(claim))
beat = asyncio.create_task(self.heartbeat(claim, work, lost))
try:
variants = await work
except asyncio.CancelledError:
if lost.is_set() and not asyncio.current_task().cancelling():
return "fenced" # lease taken over: this attempt may write nothing more
outcome = await asyncio.shield(self.db.run(lambda c: jobs.release(c, claim, self.name)))
log.info("job_released", extra={"fields": {"job_id": job_id, "outcome": outcome}})
raise
except ProcessingError as exc:
return await self.failed(claim, exc.code, exc.permanent)
except Unavailable:
return await self.failed(claim, "storage_unavailable", False)
finally:
beat.cancel()
await service.crash_once(settings, self.db, "crash_after_s3_write", job_id, self.actor)
outcome = await self.db.run(lambda c: jobs.complete(c, claim, variants, self.name))
await service.crash_once(settings, self.db, "crash_after_commit", job_id, self.actor)
if outcome == "succeeded":
await self.delete_staging(claim.attachment_id)
return outcome
async def failed(self, claim: jobs.Claim, code: str, permanent: bool) -> str:
s = self.settings
outcome, delay = await self.db.run(
lambda c: jobs.retry_or_fail(c, claim, code, permanent, self.name, s.job_retry_base_seconds,
s.job_retry_max_seconds))
log.info("attempt_failed", extra={"fields": {"job_id": claim.job_id, "attempt": claim.attempt, "code": code,
"permanent": permanent, "outcome": outcome,
"delay_s": round(delay, 3)}})
return outcome
async def heartbeat(self, claim: jobs.Claim, work: asyncio.Task, lost: asyncio.Event) -> None:
"""Renew the lease while the attempt is alive and within JOB_TIMEOUT_SECONDS. A hung attempt
stops being renewed, so the reaper can hand the job to another worker; if the lease turns out
to be lost, the attempt is cancelled (its result would be fenced anyway)."""
s = self.settings
if "hang" in s.fault_points:
return # experiment: a hung worker that no longer heartbeats
started = time.monotonic()
while True:
await asyncio.sleep(s.job_lease_seconds / 3)
if time.monotonic() - started > s.job_timeout_seconds:
return
try:
alive = await self.db.run(lambda c: jobs.heartbeat(c, claim, s.job_lease_seconds))
except Unavailable:
continue # the lease may expire meanwhile; the fence protects the result
if not alive:
log.warning("lease_lost", extra={"fields": {"job_id": claim.job_id, "attempt": claim.attempt}})
lost.set()
work.cancel()
return
async def attempt(self, claim: jobs.Claim) -> list[imaging.Rendered]:
s = self.settings
if "hang" in s.fault_points:
await asyncio.sleep(s.fault_value("hang", 3600)) # stuck outside of the time limit
try:
return await asyncio.wait_for(self.build(claim), s.job_timeout_seconds)
except asyncio.TimeoutError:
raise ProcessingError("processing_timeout", permanent=False) from None
async def build(self, claim: jobs.Claim) -> list[imaging.Rendered]:
s = self.settings
started = time.monotonic()
if claim.size_bytes > s.max_upload_bytes:
raise ProcessingError("image_too_large", permanent=True)
source = self.staging if claim.source == "staging" else self.final
try:
data = await source.read_all(claim.source_key, limit=s.max_upload_bytes)
except ObjectMissing:
raise ProcessingError("source_missing", permanent=True) from None
if len(data) != claim.size_bytes or hashlib.sha256(data).digest() != claim.sha256:
raise ProcessingError("source_corrupt", permanent=True)
self.injected(claim)
if "slow" in s.fault_points and s.fault_value("slow_pipeline", claim.pipeline_version) == claim.pipeline_version:
await asyncio.sleep(s.fault_value("slow", 10))
remaining = max(1.0, s.job_timeout_seconds - (time.monotonic() - started))
variants = await imaging.render(data, claim.content_type, claim.pipeline_version,
max_pixels=s.max_image_pixels, timeout=remaining,
memory_mb=s.image_max_memory_mb)
meta = {"attachment-id": claim.attachment_id, "pipeline-version": str(claim.pipeline_version)}
puts = [self.final.put(claim.key(v.name), v.data, v.content_type, meta) for v in variants]
if claim.write_original:
puts.append(self.final.put(claim.key("original"), data, claim.content_type,
{**meta, "sha256": claim.sha256.hex()}))
await asyncio.gather(*puts)
return variants
def injected(self, claim: jobs.Claim) -> None:
s = self.settings
if "permanent_error" in s.fault_points:
raise ProcessingError("invalid_image", permanent=True)
if "transient_error" in s.fault_points and claim.attempt <= s.fault_value("transient_error", 1000):
raise ProcessingError("injected_transient", permanent=False)
# ---------------------------------------------------------------- cleanup
async def delete_staging(self, attachment_id: str) -> None:
"""Mark `deleting` under the row lock (re-checked there), then S3 DELETE, then `deleted`."""
try:
key = await self.db.run(
lambda c: jobs.mark_staging_deleting(c, attachment_id, self.settings.staging_retention_seconds))
if key is None:
return
await self.staging.delete(key)
await self.db.run(lambda c: jobs.mark_staging_deleted(c, attachment_id))
log.info("staging_deleted", extra={"fields": {"attachment_id": attachment_id}})
except Unavailable:
pass # stays present/deleting; the next cleanup pass repeats (DELETE is idempotent)
async def cleanup(self) -> None:
s = self.settings
ids = await self.db.run(lambda c: jobs.staging_to_delete(c, s.staging_retention_seconds))
tokens = await self.db.run(lambda c: jobs.attempts_to_clean(c, s.orphan_grace_seconds))
if "pause_cleanup" in s.fault_points and (ids or tokens):
# Experiment: the race "cleanup read its candidates -> replay commits -> cleanup goes on".
log.warning("cleanup_paused", extra={"fields": {"attachments": ids, "attempts": tokens}})
await asyncio.sleep(s.fault_value("pause_cleanup", 10))
for attachment_id in ids:
await self.delete_staging(attachment_id)
for token in tokens:
keys = await self.db.run(lambda c: jobs.lock_attempt_for_cleanup(c, token))
if keys is None:
continue
await asyncio.gather(*(self.final.delete(k) for k in keys))
await self.db.run(lambda c: jobs.mark_attempt_cleaned(c, token))
log.info("attempt_objects_deleted", extra={"fields": {"token": token, "objects": len(keys)}})
async def cleanup_loop(self, stop: asyncio.Event) -> None:
while not stop.is_set() and self.settings.cleanup_interval_seconds > 0:
await service.sleep_or_stop(stop, self.settings.cleanup_interval_seconds)
try:
await self.cleanup()
except Unavailable:
pass
# ---------------------------------------------------------------- lifecycle
async def run(self) -> int:
stop = service.stop_event()
await self.db.open()
await self.staging.open()
await self.final.open()
cleaner = asyncio.create_task(self.cleanup_loop(stop))
try:
while not stop.is_set():
if self.connection is None or self.connection.is_closed:
await self.disconnect()
try:
await self.connect()
except (aio_pika.exceptions.AMQPError, ConnectionError, OSError, asyncio.TimeoutError) as exc:
log.warning("broker_unavailable", extra={"fields": {"error": type(exc).__name__}})
await self.disconnect()
if self.connection is not None and not self.connection.is_closed:
service.beat()
await service.sleep_or_stop(stop, 2.0)
await self.shutdown()
finally:
cleaner.cancel()
await self.disconnect()
await self.final.close()
await self.staging.close()
await self.db.close()
log.info("worker_stopped")
return 0
async def shutdown(self) -> None:
"""Stop consuming; running attempts get WORKER_SHUTDOWN_SECONDS, the rest is released."""
if self.queue is not None and self.consumer_tag is not None:
try:
await self.queue.cancel(self.consumer_tag)
except Exception: # noqa: BLE001
pass
pending = set(self.tasks)
log.info("worker_draining", extra={"fields": {"in_flight": len(pending)}})
if pending:
_, pending = await asyncio.wait(pending, timeout=self.settings.worker_shutdown_seconds)
for task in pending:
task.cancel()
if pending:
await asyncio.wait(pending, timeout=10)
def main() -> int:
try:
settings = Settings.from_env()
if not settings.broker_url:
raise ConfigError("BROKER_URL_FILE (or BROKER_URL) must be set")
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
logging_setup.configure(settings.log_level)
log.info("worker_starting", extra={"fields": {
"instance_id": settings.instance_id, "concurrency": settings.worker_concurrency,
"max_attempts": settings.job_max_attempts, "timeout_s": settings.job_timeout_seconds,
"lease_s": settings.job_lease_seconds, "memory_mb": settings.image_max_memory_mb,
"fault_points": sorted(settings.fault_points)}})
return asyncio.run(Worker(settings).run())
if __name__ == "__main__":
sys.exit(main())