From bc32a509674e739308cf6da1dde7f57792798914 Mon Sep 17 00:00:00 2001 From: Alexey Bodanyuk <144894236+wfa22@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:12:52 +0300 Subject: [PATCH] =?UTF-8?q?lab5:=20RabbitMQ,=20relay=20=D0=B8=20worker=20?= =?UTF-8?q?=D0=B2=20=D1=81=D1=82=D0=B5=D0=BD=D0=B4=D0=B5;=20reconciler=20l?= =?UTF-8?q?ab4=20=D1=83=D0=B1=D1=80=D0=B0=D0=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 broker: RabbitMQ 4.3.7 (тег+digest), quorum queue + DLX/DLQ из definitions, пользователи relay (только publish в exchange) и worker (только consume), без management UI и портов. relay: outbox -> publish с publisher confirm, reaper истёкших lease, страховка от потерянных сообщений. worker: claim + heartbeat lease, варианты, публикация с fencing, ack после COMMIT, graceful stop, уборка staging и мусора попыток. S3: два bucket и четыре identity (admin, api, worker, auditor) с проверкой модели доступа в s3-init. Отдельная сеть queue. Reconciler lab4 (трек прошлой лабы) удалён: его роль выполняет уборка по журналу попыток. --- .env.example | 36 +++- Dockerfile | 5 +- Makefile | 47 +++-- compose.yaml | 267 +++++++++++++++++++++----- infra/rabbitmq/enabled_plugins | 1 + infra/rabbitmq/rabbitmq.conf | 24 +++ infra/rabbitmq/topology.json | 21 +++ infra/seaweedfs/start.sh | 31 +-- scripts/init_secrets.py | 37 +++- scripts/restart.sh | 21 ++- src/messenger/healthcheck.py | 18 +- src/messenger/relay.py | 163 ++++++++++++++++ src/messenger/s3_bootstrap.py | 127 ++++++++----- src/messenger/service.py | 67 +++++++ src/messenger/worker.py | 334 +++++++++++++++++++++++++++++++++ 15 files changed, 1058 insertions(+), 141 deletions(-) create mode 100644 infra/rabbitmq/enabled_plugins create mode 100644 infra/rabbitmq/rabbitmq.conf create mode 100644 infra/rabbitmq/topology.json create mode 100644 src/messenger/relay.py create mode 100644 src/messenger/service.py create mode 100644 src/messenger/worker.py diff --git a/.env.example b/.env.example index f6143d6..e8c4d1a 100644 --- a/.env.example +++ b/.env.example @@ -24,6 +24,8 @@ S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +# Lab 5: private staging bucket for originals accepted before 202 (API: Write; worker: Read+Write). +S3_STAGING_BUCKET=messenger-staging # Contract limits (may be lowered, never raised above 10485760 / 40000000). MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 @@ -38,8 +40,33 @@ S3_TIMEOUT_SECONDS=3 # Janitor: 'pending' intents older than this are failed uploads (object + row removed). PENDING_GRACE_SECONDS=900 JANITOR_INTERVAL_SECONDS=60 -# Level 4 failure experiments only (after_s3_put, after_ready_commit); keep empty. -FAULT_POINTS= +# Lab 5: broker credentials are files too (relay and worker get their OWN RabbitMQ user/URL; +# secrets/broker_url_relay|worker are mounted as /run/secrets/broker_url). The API has none. +BROKER_URL_FILE=/run/secrets/broker_url +# Jobs processed at once by one worker = RabbitMQ prefetch; image child processes at once. +WORKER_CONCURRENCY=2 +WORKER_REPLICAS=1 +# Managed attempts (claims) per job, then failed/retry_exhausted; backoff = full jitter. +JOB_MAX_ATTEMPTS=5 +JOB_RETRY_BASE_SECONDS=2 +JOB_RETRY_MAX_SECONDS=60 +# One attempt: hard time limit (child process killed) and the lease renewed by its heartbeat. +JOB_TIMEOUT_SECONDS=60 +JOB_LEASE_SECONDS=30 +# Address-space limit of the image child process (RLIMIT_AS). +IMAGE_MAX_MEMORY_MB=768 +# Staging copy of a FAILED attachment is kept this long for a replay (ready: deleted at once). +STAGING_RETENTION_SECONDS=86400 +# Objects of attempts that never published are deleted this long after the attempt ended. +ORPHAN_GRACE_SECONDS=600 +# Queued job without a claim for this long gets a new outbox row (lost-message guard). +QUEUED_REPUBLISH_SECONDS=300 +# Pipeline version of new uploads (track 2: versions 1, 2, 3 in src/messenger/pipeline.py). +PIPELINE_VERSION=1 +# Failure experiments only, per service (see src/messenger/config.py FAULT_POINTS); keep empty. +API_FAULT_POINTS= +RELAY_FAULT_POINTS= +WORKER_FAULT_POINTS= # Additional settings of this implementation (defaults shown; all optional): LOG_LEVEL=info DB_POOL_MIN=1 @@ -60,6 +87,11 @@ PG_CPUS=2.0 PG_MEMORY=1g VALKEY_CPUS=0.5 LB_CPUS=1.0 +BROKER_CPUS=1.0 +BROKER_MEMORY=512m +WORKER_CPUS=2.0 +WORKER_MEMORY=1536m +RELAY_CPUS=0.5 PG_MAX_CONNECTIONS=100 # Host ports of the load balancer (published on 127.0.0.1 only; API replicas publish nothing) APP_PUBLISH_PORT=8080 diff --git a/Dockerfile b/Dockerfile index 9da4841..74f4f55 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ # syntax=docker/dockerfile:1 -# Messenger API, lab 4 (replicas behind HAProxy, PostgreSQL + Valkey + S3). Built by compose.yaml -# (services api, migrate, s3-init, reconciler: one image, different commands). +# Messenger, lab 5 (API replicas behind HAProxy, PostgreSQL + Valkey + S3 + RabbitMQ). Built by +# compose.yaml (services api, migrate, s3-init, relay, worker, auditor, jobctl: one image, different +# commands; only the worker imports Pillow). # # Python is interpreted, so there is no separate compile stage; the image # contains only the interpreter, locked dependencies and the source code. diff --git a/Makefile b/Makefile index 3e68a6d..7b7d879 100644 --- a/Makefile +++ b/Makefile @@ -29,23 +29,23 @@ test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) # --------------------------------------------------------------------------- -# Student solution (lab 4, Python). On Windows: PowerShell/Git Bash with GNU make, public +# Student solution (lab 5, Python). On Windows: PowerShell/Git Bash with GNU make, public # tests need PYTHONUTF8=1, and the hooks are started through scripts/*.cmd (Git Bash). -.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series reconcile reconcile-apply bench-memory bench-reconcile +.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series bench-memory worker-stop worker-start jobs failed audit track2 secrets: $(PYTHON) scripts/init_secrets.py -# Build, apply migrations ONCE (service migrate), start API replicas and the LB, wait for healthy. +# Build, apply migrations ONCE (service migrate), start broker, API replicas, LB, relay and worker. up: secrets docker compose up -d --build --wait -# Safe stop: containers stop, named volumes (data) stay. +# Safe stop: containers stop, named volumes (data, queue) stay. down: docker compose down logs: - docker compose logs --tail 50 api lb + docker compose logs --tail 50 api lb relay worker ps: docker compose ps -a @@ -67,10 +67,32 @@ restore: rolling-update: sh scripts/rolling-update.sh -# Level 4: stop each replica in turn under load, measure RPS/errors/recovery (evidence/failover/). +# Lab 3: stop each replica in turn under load, measure RPS/errors/recovery (evidence/failover/). failover-series: $(PYTHON) bench/failover_series.py --output evidence/failover +# Lab 5 demo: stop the worker (uploads still get 202, content answers 409), start it again. +worker-stop: + docker compose stop worker + +worker-start: + docker compose start worker + +# Lab 5 operator view: jobs by state + outbox backlog; failed registry (replay: see REPORT.md). +jobs: + docker compose run --rm --no-deps jobctl stats + +failed: + docker compose run --rm --no-deps jobctl list-failed + +# Track 2: read-only check of the pipeline invariants over PostgreSQL and both buckets. +audit: + docker compose run --rm --no-deps auditor + +# Track 2: automatic fault-injection matrix (duplicate, reorder, crashes, stale versions). +track2: + $(PYTHON) bench/pipeline_faults.py --output evidence/track2 + # Own tests: unit + black-box against the running stand (needs requirements*.txt in a venv). student-test: $(PYTHON) -m pytest student-tests -v @@ -81,17 +103,6 @@ unit-test: explain: $(PYTHON) scripts/explain_history.py --output evidence/explain-history.txt -# Track 2: reconciler of S3 objects vs attachment rows (dry-run by default, read-only). -reconcile: - docker compose run --rm reconciler - -reconcile-apply: - docker compose run --rm reconciler --apply - -# Level 4: peak RAM of the replicas under 16 simultaneous uploads (evidence/memory/). +# Lab 4: peak RAM of the replicas under 16 simultaneous uploads (evidence/memory/). bench-memory: $(PYTHON) bench/upload_memory.py --output evidence/memory - -# Track 2: cost of one reconciler pass for 1k/10k/50k attachments (evidence/reconcile/). -bench-reconcile: - $(PYTHON) bench/reconcile_cost.py --output evidence/reconcile diff --git a/compose.yaml b/compose.yaml index 6899f04..e14bb02 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,15 +1,20 @@ -# Lab 4 stand: client -> HAProxy (lb) -> API x N (one scalable service `api`) -# -> PostgreSQL (metadata, source of truth) + Valkey (sessions) + SeaweedFS S3 (bytes). +# Lab 5 stand: client -> HAProxy (lb) -> API x N (scalable service `api`) +# -> PostgreSQL (metadata, jobs, outbox: source of truth) + Valkey (sessions) +# + SeaweedFS S3 (staging bucket and final bucket, both private) +# relay : outbox in PostgreSQL -> RabbitMQ (publisher confirms) [service `relay`] +# broker : RabbitMQ 4.3, durable quorum queue + dead-letter queue [service `broker`] +# worker : RabbitMQ -> variants -> final bucket + PostgreSQL, then ack [service `worker`] # From a clean clone (details and Windows notes: REPORT.md, docs/WINDOWS.md): # python scripts/init_secrets.py # once: random local credentials into ./secrets (git-ignored) # cp .env.example .env # optional overrides (API_REPLICAS, limits, ...) # docker compose --env-file .env up -d --build --wait -# Order: postgres/valkey/s3 healthy -> migrate + s3-init (ONE run each, exit 0) -> api healthy -> lb. +# Order: postgres/valkey/s3/broker healthy -> migrate + s3-init (ONE run each, exit 0) +# -> api healthy -> lb; relay and worker start after migrate/s3-init. # Images are pinned by version AND digest. Passwords/keys are passed only as secret files, and # every service gets only the secrets it needs. Only the LB publishes ports (loopback only). -# Three networks: `edge` (lb <-> api), `data` (api/migrate/reconciler <-> postgres/valkey) and -# `storage` (api/s3-init/reconciler <-> s3); data and storage are internal (no route outside). -name: mtusi-lab4 +# Networks: `edge` (lb <-> api); internal (no route outside): `data` (postgres/valkey), +# `storage` (s3), `queue` (broker). The API is NOT on `queue`: it never talks to the broker. +name: mtusi-lab5 x-api-env: &api-env APP_ENV: ${APP_ENV:-lab} @@ -29,10 +34,11 @@ x-api-env: &api-env SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-8} DOCS_ENABLED: ${DOCS_ENABLED:-true} MESSAGE_WRITE_PATH: ${MESSAGE_WRITE_PATH:-function} - # Lab 4: S3 (the API's own Read+Write key pair, see infra/seaweedfs/start.sh) and upload limits. + # S3: the API's own key pair (Write staging + Read final, see infra/seaweedfs/start.sh). S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} S3_REGION: ${S3_REGION:-us-east-1} S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging} S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3} @@ -46,8 +52,33 @@ x-api-env: &api-env JPEG_MAX_MEMORY_MB: ${JPEG_MAX_MEMORY_MB:-256} PENDING_GRACE_SECONDS: ${PENDING_GRACE_SECONDS:-900} JANITOR_INTERVAL_SECONDS: ${JANITOR_INTERVAL_SECONDS:-60} - # Level 4 failure experiments only (compose.faults.yaml); empty = no injected failures. - FAULT_POINTS: ${FAULT_POINTS:-} + # Lab 5: pipeline version and attempt budget of the jobs created by uploads. + PIPELINE_VERSION: ${PIPELINE_VERSION:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + # Failure experiments only (student-tests, bench/); empty = no injected failures. + FAULT_POINTS: ${API_FAULT_POINTS:-} + +# Settings of the pipeline shared by relay and worker (docs/PIPELINE.md). +x-pipeline-env: &pipeline-env + APP_ENV: ${APP_ENV:-lab} + LOG_LEVEL: ${LOG_LEVEL:-info} + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + DB_POOL_MAX: ${PIPELINE_DB_POOL_MAX:-6} + DB_TIMEOUT_SECONDS: ${DB_TIMEOUT_SECONDS:-3} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging} + S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-30} + JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-2} + JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:-60} + QUEUED_REPUBLISH_SECONDS: ${QUEUED_REPUBLISH_SECONDS:-300} services: postgres: @@ -102,15 +133,15 @@ services: resources: limits: {cpus: "${VALKEY_CPUS:-0.5}", memory: 256m} - # Lab 4: S3-compatible object storage (SeaweedFS: master + volume + filer + S3 gateway in one - # process). Bytes of the attachments live ONLY here, on the named volume s3data. Identities - # and exposure: infra/seaweedfs/start.sh. Not published on the host; its web UIs listen on - # 127.0.0.1 inside the container only. + # S3-compatible object storage (SeaweedFS: master + volume + filer + S3 gateway in one process). + # Bytes live ONLY here, on the named volume s3data: staging bucket (originals accepted before + # 202) and final bucket (published originals and variants). Identities: infra/seaweedfs/start.sh. s3: image: chrislusf/seaweedfs:4.46@sha256:08d516132314207d10c8e37cbffc1f32b147d870169688734cc61c6231625b62 entrypoint: ["/bin/sh", "/etc/seaweedfs/start.sh"] environment: S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging} # Go heap target below the container limit: the GC returns memory before the cgroup OOM. GOMEMLIMIT: ${S3_GOMEMLIMIT:-600MiB} secrets: @@ -118,8 +149,10 @@ services: - s3_admin_secret_access_key - s3_api_access_key_id - s3_api_secret_access_key - - s3_reconciler_access_key_id - - s3_reconciler_secret_access_key + - s3_worker_access_key_id + - s3_worker_secret_access_key + - s3_auditor_access_key_id + - s3_auditor_secret_access_key volumes: - ./infra/seaweedfs/start.sh:/etc/seaweedfs/start.sh:ro - s3data:/data @@ -137,6 +170,34 @@ services: resources: limits: {cpus: "${S3_CPUS:-1.0}", memory: "${S3_MEMORY:-1g}"} + # Lab 5: RabbitMQ. Topology (exchange, quorum queue, DLX/DLQ) and users come from definition + # files imported at boot (infra/rabbitmq, secrets/rabbitmq_users.json). Messages of the quorum + # queue live on the named volume rabbitmqdata. No management UI, no host port. + broker: + image: rabbitmq:4.3.7-alpine@sha256:b1f997f74103d08fc1f20a13890dc44fe6cfa51d528cf521b480ada988d3ffa5 + hostname: broker # node name rabbit@broker: stable across container re-creation + secrets: + - source: rabbitmq_users + target: /etc/rabbitmq/definitions.d/users.json + volumes: + - ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/conf.d/20-lab5.conf:ro + - ./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro + - ./infra/rabbitmq/topology.json:/etc/rabbitmq/definitions.d/topology.json:ro + - rabbitmqdata:/var/lib/rabbitmq + healthcheck: + # The node runs and the AMQP listener accepts connections. + test: ["CMD-SHELL", "rabbitmq-diagnostics -q ping && rabbitmq-diagnostics -q check_port_connectivity"] + interval: 5s + timeout: 10s + retries: 30 + start_period: 20s + stop_grace_period: 30s + restart: unless-stopped + networks: [queue] + deploy: + resources: + limits: {cpus: "${BROKER_CPUS:-1.0}", memory: "${BROKER_MEMORY:-512m}"} + # One-shot schema migration: runs ONCE per `up`, before any API replica starts # (api depends_on service_completed_successfully). Replicas never migrate themselves. migrate: @@ -144,7 +205,7 @@ services: context: . args: BUILD_REVISION: ${BUILD_REVISION:-dev} - image: ${API_IMAGE:-mtusi-messenger:lab4} + image: ${API_IMAGE:-mtusi-messenger:lab5} command: ["python", "-m", "messenger.migrate"] environment: DATABASE_URL_FILE: /run/secrets/database_url @@ -160,28 +221,33 @@ services: resources: limits: {cpus: "0.5", memory: 256m} - # One-shot, idempotent S3 bootstrap: bucket, sentinel object, verification of the access model - # (private bucket, least-privilege keys). The ONLY holder of the admin key pair besides s3. + # One-shot, idempotent S3 bootstrap: both buckets, sentinel object, verification of the access + # model (private buckets, least-privilege keys). The ONLY holder of the admin key pair besides s3. s3-init: - image: ${API_IMAGE:-mtusi-messenger:lab4} + image: ${API_IMAGE:-mtusi-messenger:lab5} command: ["python", "-m", "messenger.s3_bootstrap"] environment: S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} S3_REGION: ${S3_REGION:-us-east-1} S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging} S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key_id S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_access_key - S3_RECONCILER_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id - S3_RECONCILER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key + S3_WORKER_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key_id + S3_WORKER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_access_key + S3_AUDITOR_ACCESS_KEY_ID_FILE: /run/secrets/s3_auditor_access_key_id + S3_AUDITOR_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_auditor_secret_access_key secrets: - s3_admin_access_key_id - s3_admin_secret_access_key - s3_api_access_key_id - s3_api_secret_access_key - - s3_reconciler_access_key_id - - s3_reconciler_secret_access_key + - s3_worker_access_key_id + - s3_worker_secret_access_key + - s3_auditor_access_key_id + - s3_auditor_secret_access_key depends_on: s3: condition: service_healthy @@ -200,7 +266,7 @@ services: context: . args: BUILD_REVISION: ${BUILD_REVISION:-dev} - image: ${API_IMAGE:-mtusi-messenger:lab4} + image: ${API_IMAGE:-mtusi-messenger:lab5} environment: *api-env secrets: - database_url @@ -236,7 +302,7 @@ services: replicas: ${API_REPLICAS:-2} resources: # 512m: UPLOAD_CONCURRENCY (4) x 10 MiB bodies + one djpeg (<= JPEG_MAX_MEMORY_MB) + runtime; - # measured in evidence/memory/ (bench/upload_memory.py). + # measured in lab 4 (bench/upload_memory.py). No resizing happens in the API. limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-512m}"} # No swap on top of the limit (Docker's default would allow as much swap again): the RAM bound # of the experiments is a real bound, and running out of it is visible, not hidden in swap. @@ -268,29 +334,112 @@ services: resources: limits: {cpus: "${LB_CPUS:-1.0}", memory: 128m} - # Track 2 (lab 4): reconciler of S3 objects vs PostgreSQL metadata. Not a daemon: started on - # demand, dry-run by default (read-only on both stores): - # docker compose run --rm reconciler # report only - # docker compose run --rm reconciler --apply # fix: orphans, stale intents, quarantine - reconciler: + # Lab 5: outbox relay + lease reaper (src/messenger/relay.py). One process; a second one would + # be safe (SKIP LOCKED) but is not needed. Database + broker only, no S3 keys. + relay: + image: ${API_IMAGE:-mtusi-messenger:lab5} + command: ["python", "-m", "messenger.relay"] + environment: + <<: *pipeline-env + RELAY_POLL_SECONDS: ${RELAY_POLL_SECONDS:-0.5} + FAULT_POINTS: ${RELAY_FAULT_POINTS:-} + secrets: + - database_url + - source: broker_url_relay + target: broker_url + healthcheck: + test: ["CMD", "python", "-m", "messenger.healthcheck", "--file", "/tmp/heartbeat", "--max-age", "30"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 10s + depends_on: + postgres: + condition: service_healthy + broker: + condition: service_healthy + migrate: + condition: service_completed_successfully + stop_grace_period: 10s + restart: unless-stopped + networks: [data, queue] + deploy: + resources: + limits: {cpus: "${RELAY_CPUS:-0.5}", memory: 256m} + + # Lab 5: image worker (src/messenger/worker.py). Scalable (`--scale worker=2`): competing + # consumers of one queue; the database lease decides who works on a job. + worker: + image: ${API_IMAGE:-mtusi-messenger:lab5} + command: ["python", "-m", "messenger.worker"] + environment: + <<: *pipeline-env + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_SHUTDOWN_SECONDS: ${WORKER_SHUTDOWN_SECONDS:-20} + IMAGE_MAX_MEMORY_MB: ${IMAGE_MAX_MEMORY_MB:-768} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-600} + CLEANUP_INTERVAL_SECONDS: ${CLEANUP_INTERVAL_SECONDS:-30} + FAULT_POINTS: ${WORKER_FAULT_POINTS:-} + secrets: + - database_url + - source: broker_url_worker + target: broker_url + - source: s3_worker_access_key_id + target: s3_access_key_id + - source: s3_worker_secret_access_key + target: s3_secret_access_key + healthcheck: + test: ["CMD", "python", "-m", "messenger.healthcheck", "--file", "/tmp/heartbeat", "--max-age", "30"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 10s + depends_on: + postgres: + condition: service_healthy + broker: + condition: service_healthy + s3: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + stop_grace_period: 30s # > WORKER_SHUTDOWN_SECONDS (20 s) + release + restart: unless-stopped + networks: [data, storage, queue] + deploy: + replicas: ${WORKER_REPLICAS:-1} + resources: + # WORKER_CONCURRENCY (2) child processes x IMAGE_MAX_MEMORY_MB (768) address space + worker. + limits: {cpus: "${WORKER_CPUS:-2.0}", memory: "${WORKER_MEMORY:-1536m}"} + memswap_limit: ${WORKER_MEMORY:-1536m} + + # Track 2: read-only checker of the pipeline invariants (database in read-only transactions, + # S3 auditor key: Read + List, no Write). On demand: docker compose run --rm auditor + auditor: profiles: ["tools"] - image: ${API_IMAGE:-mtusi-messenger:lab4} - entrypoint: ["python", "-m", "messenger.reconcile"] + image: ${API_IMAGE:-mtusi-messenger:lab5} + entrypoint: ["python", "-m", "messenger.audit"] command: [] environment: DATABASE_URL_FILE: /run/secrets/database_url - DB_TIMEOUT_SECONDS: ${RECONCILE_DB_TIMEOUT_SECONDS:-10} + DB_TIMEOUT_SECONDS: ${AUDIT_DB_TIMEOUT_SECONDS:-30} S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} S3_REGION: ${S3_REGION:-us-east-1} S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging} S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3} secrets: - database_url - - source: s3_reconciler_access_key_id + - source: s3_auditor_access_key_id target: s3_access_key_id - - source: s3_reconciler_secret_access_key + - source: s3_auditor_secret_access_key target: s3_secret_access_key depends_on: postgres: @@ -301,19 +450,43 @@ services: networks: [data, storage] deploy: resources: - limits: {cpus: "${RECONCILER_CPUS:-1.0}", memory: 256m} + limits: {cpus: "1.0", memory: 256m} + + # Operator CLI (replay of failed jobs, reprocess with a newer pipeline, failed registry): + # docker compose run --rm jobctl list-failed + # docker compose run --rm jobctl replay --operator alice --reason "..." + jobctl: + profiles: ["tools"] + image: ${API_IMAGE:-mtusi-messenger:lab5} + entrypoint: ["python", "-m", "messenger.jobctl"] + command: [] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + secrets: [database_url] + depends_on: + postgres: + condition: service_healthy + restart: "no" + networks: [data] + deploy: + resources: + limits: {cpus: "0.5", memory: 128m} networks: edge: data: - internal: true # no route outside: stores talk only to api/migrate + internal: true # no route outside: stores talk only to the services above storage: - internal: true # S3 API reachable only by api, s3-init, reconciler + internal: true # S3 API reachable only by api, s3-init, worker, auditor + queue: + internal: true # broker reachable only by relay and worker volumes: pgdata: valkeydata: s3data: + rabbitmqdata: secrets: postgres_password: @@ -332,7 +505,17 @@ secrets: file: ./secrets/s3_api_access_key_id s3_api_secret_access_key: file: ./secrets/s3_api_secret_access_key - s3_reconciler_access_key_id: - file: ./secrets/s3_reconciler_access_key_id - s3_reconciler_secret_access_key: - file: ./secrets/s3_reconciler_secret_access_key + s3_worker_access_key_id: + file: ./secrets/s3_worker_access_key_id + s3_worker_secret_access_key: + file: ./secrets/s3_worker_secret_access_key + s3_auditor_access_key_id: + file: ./secrets/s3_auditor_access_key_id + s3_auditor_secret_access_key: + file: ./secrets/s3_auditor_secret_access_key + rabbitmq_users: + file: ./secrets/rabbitmq_users.json + broker_url_relay: + file: ./secrets/broker_url_relay + broker_url_worker: + file: ./secrets/broker_url_worker diff --git a/infra/rabbitmq/enabled_plugins b/infra/rabbitmq/enabled_plugins new file mode 100644 index 0000000..57afcca --- /dev/null +++ b/infra/rabbitmq/enabled_plugins @@ -0,0 +1 @@ +[]. diff --git a/infra/rabbitmq/rabbitmq.conf b/infra/rabbitmq/rabbitmq.conf new file mode 100644 index 0000000..05165e0 --- /dev/null +++ b/infra/rabbitmq/rabbitmq.conf @@ -0,0 +1,24 @@ +# RabbitMQ 4.3 of the lab 5 stand (compose service `broker`), mounted as conf.d/20-lab5.conf. +# Only AMQP 0-9-1 on 5672 inside the internal network `queue`: no management UI, no Prometheus +# listener (enabled_plugins is empty), nothing published on the host. +listeners.tcp.default = 5672 + +# Users, permissions and topology are imported from files at every boot (idempotent): +# /etc/rabbitmq/definitions.d/topology.json exchanges, quorum queues, bindings (this repo) +# /etc/rabbitmq/definitions.d/users.json users relay/worker + permissions (Docker secret, +# generated by scripts/init_secrets.py) +# With imported definitions no default `guest` user is created. +definitions.import_backend = local_filesystem +definitions.local.path = /etc/rabbitmq/definitions.d +definitions.skip_if_unchanged = false + +# A delivery that stays unacked longer than this closes the consumer's channel and the message +# is returned to the queue (a worker that hangs while connected). Far above JOB_TIMEOUT_SECONDS. +consumer_timeout = 300000 + +# The container has a 512 MiB limit; flow control starts well before the cgroup OOM. +vm_memory_high_watermark.relative = 0.5 +disk_free_limit.absolute = 200MB + +log.console = true +log.console.level = info diff --git a/infra/rabbitmq/topology.json b/infra/rabbitmq/topology.json new file mode 100644 index 0000000..301ef1c --- /dev/null +++ b/infra/rabbitmq/topology.json @@ -0,0 +1,21 @@ +{ + "vhosts": [{"name": "/"}], + "exchanges": [ + {"name": "messenger.jobs", "vhost": "/", "type": "direct", "durable": true, "auto_delete": false, + "internal": false, "arguments": {}}, + {"name": "messenger.jobs.dlx", "vhost": "/", "type": "fanout", "durable": true, "auto_delete": false, + "internal": false, "arguments": {}} + ], + "queues": [ + {"name": "messenger.attachment-jobs", "vhost": "/", "durable": true, "auto_delete": false, + "arguments": {"x-queue-type": "quorum", "x-delivery-limit": 20, "x-dead-letter-exchange": "messenger.jobs.dlx"}}, + {"name": "messenger.attachment-jobs.dlq", "vhost": "/", "durable": true, "auto_delete": false, + "arguments": {"x-queue-type": "quorum"}} + ], + "bindings": [ + {"source": "messenger.jobs", "vhost": "/", "destination": "messenger.attachment-jobs", "destination_type": "queue", + "routing_key": "attachment.process.v1", "arguments": {}}, + {"source": "messenger.jobs.dlx", "vhost": "/", "destination": "messenger.attachment-jobs.dlq", + "destination_type": "queue", "routing_key": "", "arguments": {}} + ] +} diff --git a/infra/seaweedfs/start.sh b/infra/seaweedfs/start.sh index ad4f09c..f496154 100755 --- a/infra/seaweedfs/start.sh +++ b/infra/seaweedfs/start.sh @@ -2,11 +2,14 @@ # Entry point of the compose service `s3` (SeaweedFS 4.46, one container: master + volume + # filer + S3 gateway). Renders the S3 identities from Docker secrets, then execs `weed`. # -# Identities (SeaweedFS s3.json, actions scoped to the bucket): -# bootstrap-admin Admin (create bucket) -> only s3-init has these keys -# messenger-api Read + Write on S3_BUCKET -> API replicas (no List, no Admin) -# messenger-reconciler Read + Write + List on S3_BUCKET -> reconciler CLI (needs the listing) -# There is NO anonymous identity, so every unsigned request is refused: the bucket is private. +# Two private buckets (lab 5): S3_STAGING_BUCKET holds originals accepted by the API before 202, +# S3_BUCKET holds the published originals and variants written by the worker. +# Identities (SeaweedFS s3.json, actions scoped to a bucket): +# bootstrap-admin Admin (create buckets) -> only s3-init has these keys +# messenger-api Write staging, Read final -> API replicas (no List, no Admin) +# messenger-worker Read + Write staging, Read + Write final -> worker (no List, no Admin) +# messenger-auditor Read + List both buckets (no Write) -> invariant checker (tools profile) +# There is NO anonymous identity, so every unsigned request is refused: the buckets are private. # # Exposure: master (9333), volume (8080), filer (8888) with their web UIs and gRPC listen on # 127.0.0.1 INSIDE the container only (-ip.bind); only the S3 API port 8333 listens on the @@ -15,6 +18,7 @@ set -eu umask 077 secret() { tr -d '\r\n' < "/run/secrets/$1"; } bucket="${S3_BUCKET:?S3_BUCKET is required}" +staging="${S3_STAGING_BUCKET:?S3_STAGING_BUCKET is required}" cat > /tmp/s3.json < /tmp/s3.json < None: print(f"keep secrets/{name}") return path.write_text(value + "\n", encoding="utf-8", newline="\n") - # Readable by the container users (postgres, valkey, seaweedfs, app uid 10001) through the - # compose bind mount. Local synthetic lab credentials only. + # Readable by the container users (postgres, valkey, seaweedfs, rabbitmq, app uid 10001) + # through the compose bind mount. Local synthetic lab credentials only. os.chmod(path, 0o644) print(f"created secrets/{name}") @@ -34,17 +37,41 @@ def read(name: str) -> str: return (SECRETS / name).read_text(encoding="utf-8").strip() +def rabbit_hash(password: str) -> str: + """RabbitMQ `rabbit_password_hashing_sha256`: base64(salt + sha256(salt + password)), 4-byte salt. + The broker stores and compares only this hash; the plain password is in the client's URL.""" + salt = secrets.token_bytes(4) + return base64.b64encode(salt + hashlib.sha256(salt + password.encode("utf-8")).digest()).decode("ascii") + + def main() -> int: SECRETS.mkdir(exist_ok=True) write("postgres_password", secrets.token_urlsafe(24)) # [A-Za-z0-9_-]: safe inside URLs write("valkey_password", secrets.token_urlsafe(24)) write("database_url", f"postgresql://messenger:{read('postgres_password')}@postgres:5432/messenger") write("cache_url", f"redis://:{read('valkey_password')}@valkey:6379/0") - # Lab 4: three separate S3 key pairs (SeaweedFS identities, infra/seaweedfs/start.sh): - # admin = bucket bootstrap only; api = Read+Write on the bucket; reconciler = + List. - for who in ("admin", "api", "reconciler"): + # S3 key pairs (SeaweedFS identities, infra/seaweedfs/start.sh): admin = bucket bootstrap only; + # api = Write staging + Read final; worker = Read/Write both; auditor = Read + List (no Write). + for who in ("admin", "api", "worker", "auditor"): write(f"s3_{who}_access_key_id", "AK" + secrets.token_hex(9).upper()) write(f"s3_{who}_secret_access_key", secrets.token_hex(20)) + # Lab 5: RabbitMQ users. relay may only publish to the jobs exchange, worker may only consume + # the jobs queue; neither may declare/delete anything (configure "^$"). + write("rabbitmq_relay_password", secrets.token_urlsafe(24)) + write("rabbitmq_worker_password", secrets.token_urlsafe(24)) + users = { + "users": [{"name": who, "password_hash": rabbit_hash(read(f"rabbitmq_{who}_password")), + "hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []} for who in ("relay", "worker")], + "permissions": [ + {"user": "relay", "vhost": "/", "configure": "^$", "write": "^messenger\\.jobs$", "read": "^$"}, + {"user": "worker", "vhost": "/", "configure": "^$", "write": "^$", "read": "^messenger\\.attachment-jobs$"}, + ], + } + write("rabbitmq_users.json", json.dumps(users, indent=1)) + for who in ("relay", "worker"): + # AMQP heartbeat 10 s: a frozen client loses its connection after ~2 missed heartbeats, and + # the broker returns its unacked messages to the queue for the other consumers. + write(f"broker_url_{who}", f"amqp://{who}:{read(f'rabbitmq_{who}_password')}@broker:5672/?heartbeat=10") return 0 diff --git a/scripts/restart.sh b/scripts/restart.sh index f00dcb4..a16c322 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,18 +1,21 @@ #!/bin/sh # Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`. -# Restarts EVERY process of the stand: LB, all API replicas, PostgreSQL, Valkey and S3 (SeaweedFS). -# Volumes, schema and data are preserved: no `down -v`, no FLUSH, no re-seed. +# Restarts EVERY process of the stand: LB, all API replicas, relay, worker(s), PostgreSQL, Valkey, +# S3 (SeaweedFS) and the broker (RabbitMQ). Volumes, schema, data and queued messages are +# preserved: no `down -v`, no FLUSH, no purge, no re-seed. # Output is suppressed by the test; run it by hand to see the steps. set -eu cd "$(dirname "$0")/.." -# 1. Entry point and API first: no request is served while the stores restart underneath. -docker compose stop lb api +# 1. Entry point, API and the pipeline processes first: nothing is served or processed while the +# stores restart underneath. +docker compose stop lb api relay worker # 2. Stores: clean shutdown (PostgreSQL checkpoint, Valkey AOF fsync, SeaweedFS volume/filer -# flush), then start again. The S3 bytes live on the named volume s3data. -docker compose restart postgres valkey s3 -# 3. API replicas (wait for healthy stores via depends_on), then the LB. HAProxy resolves -# the new replica addresses through Docker DNS by itself - no stale backend list. +# flush, RabbitMQ quorum log), then start again. Bytes live on s3data, messages on rabbitmqdata. +docker compose restart postgres valkey s3 broker +# 3. API replicas (wait for healthy stores via depends_on), the LB, then relay and worker. HAProxy +# resolves the new replica addresses through Docker DNS by itself - no stale backend list. docker compose start api docker compose start lb -echo "restart.sh: lb, api (all replicas), postgres, valkey and s3 restarted with volumes preserved" +docker compose start relay worker +echo "restart.sh: lb, api (all replicas), relay, worker, postgres, valkey, s3 and broker restarted with volumes preserved" diff --git a/src/messenger/healthcheck.py b/src/messenger/healthcheck.py index 8411514..37b0040 100644 --- a/src/messenger/healthcheck.py +++ b/src/messenger/healthcheck.py @@ -1,12 +1,24 @@ -"""Docker HEALTHCHECK probe without curl/wget in the image: `python -m messenger.healthcheck`.""" +"""Docker HEALTHCHECK probe without curl/wget in the image: `python -m messenger.healthcheck`. + + (no arguments) API: GET /health/ready on the local port + --file /tmp/heartbeat --max-age 30 relay/worker: the main loop wrote its heartbeat recently + (it writes it only while its database/broker work) +""" from __future__ import annotations import os import sys +import time import urllib.request -def main() -> int: +def main(argv: list[str]) -> int: + if len(argv) == 4 and argv[0] == "--file" and argv[2] == "--max-age": + try: + with open(argv[1], encoding="ascii") as f: + return 0 if time.time() - float(f.read().strip()) <= float(argv[3]) else 1 + except (OSError, ValueError): + return 1 port = os.environ.get("APP_PORT", "8080").strip() or "8080" try: with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/ready", timeout=2) as response: @@ -16,4 +28,4 @@ def main() -> int: if __name__ == "__main__": - sys.exit(main()) + sys.exit(main(sys.argv[1:])) diff --git a/src/messenger/relay.py b/src/messenger/relay.py new file mode 100644 index 0000000..4158e2c --- /dev/null +++ b/src/messenger/relay.py @@ -0,0 +1,163 @@ +"""Outbox relay: `python -m messenger.relay` (compose service `relay`, one process). + + loop: SELECT due outbox rows FOR UPDATE SKIP LOCKED + -> publish each job message to RabbitMQ, wait for the publisher CONFIRM + -> UPDATE outbox SET published_at -> COMMIT + every few seconds: reaper (expired leases -> queued/failed), lost-message guard. + +Why an outbox: the API commits the job and the outbox row in ONE PostgreSQL transaction and never +talks to the broker. A crash between COMMIT and publish leaves the row unpublished; the relay +publishes it when it (or the broker) is back - nobody re-creates a job by hand. While the broker +is down, uploads still get 202 and the rows wait in the outbox. + +A crash after the confirm but before the COMMIT of published_at publishes the row once more after +the restart: the queue then holds a duplicate. That is at-least-once delivery; the worker turns +the duplicate into a no-op (jobs.claim). The relay never deletes or rewrites jobs. +""" +from __future__ import annotations + +import asyncio +import logging +import sys +import time + +import aio_pika +from aio_pika.abc import AbstractChannel, AbstractConnection, AbstractExchange + +from . import broker, jobs, logging_setup, service +from .config import ConfigError, Settings +from .db import Database +from .errors import Unavailable + +log = logging.getLogger("messenger.relay") +ACTOR = "relay" +MAINTENANCE_SECONDS = 5.0 +PURGE_SECONDS = 600.0 + + +class Relay: + def __init__(self, settings: Settings): + self.settings = settings + self.db = Database(settings, name="messenger-relay") + self.connection: AbstractConnection | None = None + self.channel: AbstractChannel | None = None + self.exchange: AbstractExchange | None = None + + async def connect(self) -> None: + self.connection = await aio_pika.connect( + self.settings.broker_url, timeout=5, + client_properties={"connection_name": "relay@" + self.settings.instance_id}) + # Confirm mode: every publish() waits for basic.ack of the broker (message in the quorum log). + self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True) + # No declare: the topology belongs to the broker definitions; the relay may only write. + self.exchange = await self.channel.get_exchange(broker.EXCHANGE, ensure=False) + log.info("broker_connected", extra={"fields": {"exchange": broker.EXCHANGE}}) + + async def disconnect(self) -> None: + if self.connection is not None: + try: + await self.connection.close() + except Exception: # noqa: BLE001 - already broken + pass + self.connection = self.channel = self.exchange = None + + async def publish_due(self) -> int: + settings = self.settings + exchange = self.exchange + + async def work(conn) -> list[jobs.Due]: + rows = await jobs.due_outbox(conn, settings.relay_batch) + if "reverse_order" in settings.fault_points: + rows.reverse() + # Rows of jobs that are no longer queued (finished meanwhile) need no message. + sendable = [r for r in rows if r.job_state == "queued"] + copies = 2 if "duplicate_publish" in settings.fault_points else 1 + for row in sendable: + message = aio_pika.Message( + broker.encode(row.job_id, row.attachment_id, row.pipeline_version, row.attempt, row.created_at), + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, content_type="application/json", + message_id=f"{row.job_id}:{row.outbox_id}", type=broker.JOB_TYPE) + for _ in range(copies): + await exchange.publish(message, routing_key=broker.ROUTING_KEY, mandatory=True, timeout=5) + for row in sendable: + # Confirmed by the broker, outbox not marked yet: the window "after publish, before + # the confirm is recorded". The crash rolls this transaction back. + await service.crash_once(settings, self.db, "crash_after_publish", row.job_id, ACTOR, + outbox_id=row.outbox_id) + await jobs.mark_published(conn, [r.outbox_id for r in rows]) + return sendable + + sent = await self.db.run(work, deadline=10 + 5 * settings.relay_batch) + for row in sent: + log.info("job_published", extra={"fields": {"job_id": row.job_id, "attempt": row.attempt, + "outbox_id": row.outbox_id}}) + return len(sent) + + async def maintenance(self) -> None: + expired = await self.db.run(lambda c: jobs.expire_leases(c, ACTOR)) + republished = await self.db.run(lambda c: jobs.republish_stale(c, self.settings.queued_republish_seconds, ACTOR)) + if expired or republished: + log.info("maintenance", extra={"fields": {"leases_expired": expired, "republished": republished}}) + + async def run(self) -> int: + stop = service.stop_event() + await self.db.open() + next_maintenance = next_purge = 0.0 + failures = 0 + try: + while not stop.is_set(): + healthy = True + now = time.monotonic() + try: + if now >= next_maintenance: + await self.maintenance() + next_maintenance = now + MAINTENANCE_SECONDS + if now >= next_purge: + purged = await self.db.run(jobs.purge_outbox) + next_purge = now + PURGE_SECONDS + if purged: + log.info("outbox_purged", extra={"fields": {"rows": purged}}) + except Unavailable: + healthy = False + try: + if self.exchange is None or self.connection is None or self.connection.is_closed: + await self.disconnect() + await self.connect() + sent = await self.publish_due() + failures = 0 + except Unavailable: + healthy, sent = False, 0 + except (aio_pika.exceptions.AMQPError, ConnectionError, OSError, asyncio.TimeoutError) as exc: + # Broker down/unreachable or a message returned/nacked: nothing marked, the rows + # stay in the outbox (transaction rolled back) and are published later. + failures += 1 + healthy, sent = False, 0 + log.warning("publish_failed", extra={"fields": {"error": type(exc).__name__, "failures": failures}}) + await self.disconnect() + if healthy: + service.beat() + if not sent: + await service.sleep_or_stop(stop, min(5.0, self.settings.relay_poll_seconds * (2 ** min(failures, 3)))) + finally: + await self.disconnect() + await self.db.close() + log.info("relay_stopped") + return 0 + + +def main() -> int: + try: + settings = Settings.from_env(s3_credentials=False) + if not settings.broker_url: + raise ConfigError("BROKER_URL_FILE (or BROKER_URL) must be set") + except ConfigError as exc: + print(f"configuration error: {exc}", file=sys.stderr) + return 2 + logging_setup.configure(settings.log_level) + log.info("relay_starting", extra={"fields": {"instance_id": settings.instance_id, + "fault_points": sorted(settings.fault_points)}}) + return asyncio.run(Relay(settings).run()) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/messenger/s3_bootstrap.py b/src/messenger/s3_bootstrap.py index 91a1b84..d7f2e65 100644 --- a/src/messenger/s3_bootstrap.py +++ b/src/messenger/s3_bootstrap.py @@ -3,14 +3,15 @@ Idempotent, safe to run on every `docker compose up`: 1. waits until the S3 endpoint answers; -2. creates the bucket with the ADMIN key pair if it does not exist yet (an existing bucket and - its objects are left untouched); -3. writes the sentinel object `health/sentinel` that the API's readiness probe HEADs; +2. creates the final bucket (S3_BUCKET) and the staging bucket (S3_STAGING_BUCKET) with the ADMIN + key pair if they do not exist yet (existing buckets and their objects are left untouched); +3. writes the sentinel object `health/sentinel` into the final bucket (the API readiness HEADs it); 4. proves the access model and FAILS the stand (exit 1) if it is wrong: - * anonymous (unsigned) GET of an object and anonymous LIST of the bucket are refused; - * the API key pair can read/write objects of ITS bucket, but can neither list it, nor create - a bucket, nor see or read another bucket (a canary bucket created here for this check); - * the reconciler key pair can list the bucket but cannot create one. + * anonymous (unsigned) GET of an object and anonymous LIST are refused on both buckets; + * api: writes staging, reads final; cannot read staging, write final, list or create buckets, + nor read another bucket (a canary bucket created here for this check); + * worker: reads and writes both buckets; cannot list or create buckets; + * auditor: reads and lists both buckets; cannot write. Only file names and verdicts are printed, never key values. """ from __future__ import annotations @@ -57,14 +58,33 @@ def settings_with(keys: tuple[str, str]) -> Settings: return Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024, **base) +def pair(prefix: str) -> tuple[str, str]: + return _secret(f"{prefix}ACCESS_KEY_ID", True), _secret(f"{prefix}SECRET_ACCESS_KEY", True) + + +async def ensure_bucket(admin_s3: Storage, bucket: str) -> bool: + code = await error_code(admin_s3.client.head_bucket(Bucket=bucket)) + if code == "": + say(f"bucket {bucket}: exists, kept as is") + return True + if code in ("404", "NoSuchBucket", "NotFound"): + code = await error_code(admin_s3.client.create_bucket(Bucket=bucket)) + if code not in ("", "BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + say(f"bucket {bucket}: create failed ({code})") + return False + say(f"bucket {bucket}: created") + return True + say(f"bucket {bucket}: head failed ({code})") + return False + + async def run() -> int: - admin = (_secret("S3_ADMIN_ACCESS_KEY_ID", True), _secret("S3_ADMIN_SECRET_ACCESS_KEY", True)) - api = (_secret("S3_ACCESS_KEY_ID", True), _secret("S3_SECRET_ACCESS_KEY", True)) - reconciler = (_secret("S3_RECONCILER_ACCESS_KEY_ID", True), _secret("S3_RECONCILER_SECRET_ACCESS_KEY", True)) - settings = settings_with(admin) - bucket = settings.s3_bucket - admin_s3, api_s3, rec_s3 = Storage(settings), Storage(settings_with(api)), Storage(settings_with(reconciler)) - for s in (admin_s3, api_s3, rec_s3): + settings = settings_with(pair("S3_ADMIN_")) + final, staging = settings.s3_bucket, settings.s3_staging_bucket + admin_s3 = Storage(settings) + clients = {"api": Storage(settings_with(pair("S3_"))), "worker": Storage(settings_with(pair("S3_WORKER_"))), + "auditor": Storage(settings_with(pair("S3_AUDITOR_")))} + for s in (admin_s3, *clients.values()): await s.open() try: deadline = time.monotonic() + 60 @@ -77,64 +97,75 @@ async def run() -> int: say("S3 endpoint does not answer within 60 s") return 1 await asyncio.sleep(1) - - code = await error_code(admin_s3.client.head_bucket(Bucket=bucket)) - if code == "": - say(f"bucket {bucket}: exists, kept as is") - elif code in ("404", "NoSuchBucket", "NotFound"): - code = await error_code(admin_s3.client.create_bucket(Bucket=bucket)) - if code not in ("", "BucketAlreadyOwnedByYou", "BucketAlreadyExists"): - say(f"bucket {bucket}: create failed ({code})") + for bucket in (final, staging): + if not await ensure_bucket(admin_s3, bucket): return 1 - say(f"bucket {bucket}: created") - else: - say(f"bucket {bucket}: head failed ({code})") - return 1 await admin_s3.put(SENTINEL_KEY, b"ok\n", "text/plain", {}) + await admin_s3.client.put_object(Bucket=staging, Key=SENTINEL_KEY, Body=b"ok\n") say(f"sentinel {SENTINEL_KEY}: written") - canary = bucket + "-canary" + canary = final + "-canary" if await error_code(admin_s3.client.head_bucket(Bucket=canary)): await admin_s3.client.create_bucket(Bucket=canary) await admin_s3.client.put_object(Bucket=canary, Key="canary", Body=b"secret of another bucket") failures = [] base = settings.s3_endpoint.rstrip("/") - for what, url in (("anonymous GET object", f"{base}/{bucket}/{SENTINEL_KEY}"), - ("anonymous LIST bucket", f"{base}/{bucket}?list-type=2")): - status = anonymous_status(url) - verdict = "refused" if status in (401, 403) else "ALLOWED" - say(f"{what}: HTTP {status} -> {verdict}") - if verdict != "refused": - failures.append(what) + for bucket in (final, staging): + for what, url in ((f"anonymous GET object of {bucket}", f"{base}/{bucket}/{SENTINEL_KEY}"), + (f"anonymous LIST of {bucket}", f"{base}/{bucket}?list-type=2")): + status = anonymous_status(url) + verdict = "refused" if status in (401, 403) else "ALLOWED" + say(f"{what}: HTTP {status} -> {verdict}") + if verdict != "refused": + failures.append(what) probe = "health/bootstrap-probe" + + def c(who: str): + return clients[who].client + + # (what, coroutine, allowed?) - every refusal must be a real AccessDenied, not an outage. checks = [ - ("api: read object", await error_code(api_s3.client.head_object(Bucket=bucket, Key=SENTINEL_KEY)), True), - ("api: write object", await error_code(api_s3.client.put_object(Bucket=bucket, Key=probe, Body=b"p")), True), - ("api: delete object", await error_code(api_s3.client.delete_object(Bucket=bucket, Key=probe)), True), - ("api: list bucket", await error_code(api_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), False), - ("api: create bucket", await error_code(api_s3.client.create_bucket(Bucket=bucket + "-x")), False), - ("api: read object of another bucket", - await error_code(api_s3.client.head_object(Bucket=canary, Key="canary")), False), - ("reconciler: list bucket", await error_code(rec_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), True), - ("reconciler: create bucket", await error_code(rec_s3.client.create_bucket(Bucket=bucket + "-x")), False), + ("api: write staging", c("api").put_object(Bucket=staging, Key=probe, Body=b"p"), True), + ("api: delete staging (janitor)", c("api").delete_object(Bucket=staging, Key=probe), True), + ("api: read staging", c("api").head_object(Bucket=staging, Key=SENTINEL_KEY), False), + ("api: read final", c("api").head_object(Bucket=final, Key=SENTINEL_KEY), True), + ("api: write final", c("api").put_object(Bucket=final, Key=probe, Body=b"p"), False), + ("api: list final", c("api").list_objects_v2(Bucket=final, MaxKeys=1), False), + ("api: list staging", c("api").list_objects_v2(Bucket=staging, MaxKeys=1), False), + ("api: create bucket", c("api").create_bucket(Bucket=final + "-x"), False), + ("api: read another bucket", c("api").head_object(Bucket=canary, Key="canary"), False), + ("worker: read staging", c("worker").head_object(Bucket=staging, Key=SENTINEL_KEY), True), + ("worker: write final", c("worker").put_object(Bucket=final, Key=probe, Body=b"p"), True), + ("worker: delete final", c("worker").delete_object(Bucket=final, Key=probe), True), + ("worker: write staging", c("worker").put_object(Bucket=staging, Key=probe, Body=b"p"), True), + ("worker: delete staging", c("worker").delete_object(Bucket=staging, Key=probe), True), + ("worker: list final", c("worker").list_objects_v2(Bucket=final, MaxKeys=1), False), + ("worker: create bucket", c("worker").create_bucket(Bucket=final + "-x"), False), + ("worker: read another bucket", c("worker").head_object(Bucket=canary, Key="canary"), False), + ("auditor: list final", c("auditor").list_objects_v2(Bucket=final, MaxKeys=1), True), + ("auditor: list staging", c("auditor").list_objects_v2(Bucket=staging, MaxKeys=1), True), + ("auditor: read final", c("auditor").head_object(Bucket=final, Key=SENTINEL_KEY), True), + ("auditor: write final", c("auditor").put_object(Bucket=final, Key=probe, Body=b"p"), False), + ("auditor: delete staging", c("auditor").delete_object(Bucket=staging, Key=SENTINEL_KEY), False), ] - for what, code, allowed in checks: - ok = (code == "") == allowed + for what, coro, allowed in checks: + code = await error_code(coro) + ok = (code == "") == allowed and code in ("", "AccessDenied", "403") say(f"{what}: {'allowed' if code == '' else 'refused (' + code + ')'} -> {'OK' if ok else 'WRONG'}") if not ok: failures.append(what) - visible = [b["Name"] for b in (await api_s3.client.list_buckets()).get("Buckets", [])] + visible = [b["Name"] for b in (await c("api").list_buckets()).get("Buckets", [])] say(f"api: buckets visible in ListBuckets: {visible} -> {'OK' if canary not in visible else 'WRONG'}") if canary in visible: failures.append("api sees another bucket") if failures: say("access model is WRONG: " + ", ".join(failures)) return 1 - say("done: private bucket, least-privilege keys verified") + say("done: private buckets, least-privilege keys verified") return 0 finally: - for s in (admin_s3, api_s3, rec_s3): + for s in (admin_s3, *clients.values()): await s.close() diff --git a/src/messenger/service.py b/src/messenger/service.py new file mode 100644 index 0000000..8554bff --- /dev/null +++ b/src/messenger/service.py @@ -0,0 +1,67 @@ +"""Common runtime of the background processes (relay, worker): signals, heartbeat file for the +Docker healthcheck, failure injection that fires once per job and leaves an audit row.""" +from __future__ import annotations + +import asyncio +import logging +import os +import signal +import time +from pathlib import Path + +from . import jobs +from .config import Settings +from .db import Database + +log = logging.getLogger("messenger.service") + +HEARTBEAT_FILE = Path(os.environ.get("HEARTBEAT_FILE", "/tmp/heartbeat")) +CRASH_EXIT_CODE = 70 # os._exit: no cleanup, no ack, no COMMIT - like SIGKILL in the middle of work + + +def beat() -> None: + """Healthcheck input: `python -m messenger.healthcheck --file /tmp/heartbeat --max-age 30`.""" + try: + HEARTBEAT_FILE.write_text(str(time.time()), encoding="ascii") + except OSError: + pass + + +def stop_event() -> asyncio.Event: + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + try: + loop.add_signal_handler(sig, stop.set) + except (NotImplementedError, RuntimeError): + pass # Windows unit tests: no signal handlers + return stop + + +async def sleep_or_stop(stop: asyncio.Event, seconds: float) -> None: + try: + await asyncio.wait_for(stop.wait(), seconds) + except asyncio.TimeoutError: + pass + + +async def crash_once(settings: Settings, db: Database, point: str, job_id: str, actor: str, **detail) -> None: + """Experiments only (FAULT_POINTS): kill THIS process at `point`, once per job (and per `detail`), + across restarts and replicas. The decision is an audit row committed before the crash, so the + restarted process (or another replica) continues normally and the evidence stays in job_events.""" + if point not in settings.fault_points: + return + + async def record(conn) -> bool: + cur = await conn.execute( + "SELECT 1 FROM job_events WHERE job_id = %s AND event = 'fault_injected' AND detail->>'point' = %s " + "AND detail @> %s::jsonb", (job_id, point, jobs.json.dumps({k: str(v) for k, v in detail.items()}))) + if await cur.fetchone(): + return False + await jobs.event(conn, job_id, "fault_injected", actor, point=point, **{k: str(v) for k, v in detail.items()}) + return True + + if await db.run(record): + log.warning("fault_injected_crash", extra={"fields": {"point": point, "job_id": job_id, **detail}}) + logging.shutdown() + os._exit(CRASH_EXIT_CODE) diff --git a/src/messenger/worker.py b/src/messenger/worker.py new file mode 100644 index 0000000..ddd24ff --- /dev/null +++ b/src/messenger/worker.py @@ -0,0 +1,334 @@ +"""Image worker: `python -m messenger.worker` (compose service `worker`, separate from the API). + +One delivery of a job message: + + decode + validate (schema v1; invalid -> reject -> dead-letter queue) + -> CLAIM in PostgreSQL (jobs.claim): finished/leased/not due/stale -> no-op, ack + -> lease heartbeat every JOB_LEASE_SECONDS/3 while the attempt runs + -> read the source (staging copy or published original), check size + SHA-256 against the DB + -> variants in a child process (imaging.py: pixel, memory and time limits) + -> PUT the original (first publication) and the variants under keys of THIS attempt + -> ONE transaction: variants + attachment ready, fenced by the lease token (jobs.complete) + -> ack -> delete the staging copy (marked in the DB first, then S3). + +The ack comes only after the durable effect (the COMMIT). A crash anywhere before it makes +RabbitMQ deliver the message again; the job row then says what is left to do. Failures: a +permanent one (invalid_image, image_too_large, source_missing/corrupt) fails the job at once, a +transient one (S3 down, timeout, killed decoder) is retried with backoff via the outbox, at most +JOB_MAX_ATTEMPTS claims in total. + +Concurrency: RabbitMQ prefetch = WORKER_CONCURRENCY unacked messages per worker, and at most +WORKER_CONCURRENCY image child processes. Graceful stop (SIGTERM): stop consuming, let running +attempts finish for WORKER_SHUTDOWN_SECONDS, give the rest back (jobs.release). + +The worker also runs the cleanup of the pipeline objects (staging copies that are no longer +needed, objects of attempts that never published) - always decided under a row lock. +""" +from __future__ import annotations + +import asyncio +import hashlib +import logging +import sys +import time + +import aio_pika +from aio_pika.abc import AbstractChannel, AbstractConnection, AbstractIncomingMessage, AbstractQueue + +from . import broker, imaging, jobs, logging_setup, service +from .config import ConfigError, Settings +from .db import Database +from .errors import Unavailable +from .imaging import ProcessingError +from .storage import ObjectMissing, Storage + +log = logging.getLogger("messenger.worker") + + +class Worker: + def __init__(self, settings: Settings): + self.settings = settings + self.name = settings.instance_id + self.actor = "worker:" + self.name + self.db = Database(settings, name="messenger-worker") + self.staging = Storage(settings, bucket=settings.s3_staging_bucket) + self.final = Storage(settings) + self.slots = asyncio.Semaphore(settings.worker_concurrency) + self.tasks: set[asyncio.Task] = set() + self.connection: AbstractConnection | None = None + self.channel: AbstractChannel | None = None + self.queue: AbstractQueue | None = None + self.consumer_tag: str | None = None + + # ---------------------------------------------------------------- broker + async def connect(self) -> None: + self.connection = await aio_pika.connect( + self.settings.broker_url, timeout=5, + client_properties={"connection_name": "worker@" + self.name}) + self.channel = await self.connection.channel() + # Prefetch: the broker keeps at most this many unacked messages in flight to this worker. + await self.channel.set_qos(prefetch_count=self.settings.worker_concurrency) + self.queue = await self.channel.get_queue(broker.QUEUE, ensure=False) + self.consumer_tag = await self.queue.consume(self.on_message, no_ack=False) + log.info("consuming", extra={"fields": {"queue": broker.QUEUE, "prefetch": self.settings.worker_concurrency}}) + + async def disconnect(self) -> None: + if self.connection is not None: + try: + await self.connection.close() + except Exception: # noqa: BLE001 - already broken + pass + self.connection = self.channel = self.queue = None + self.consumer_tag = None + + async def on_message(self, message: AbstractIncomingMessage) -> None: + task = asyncio.create_task(self.deliver(message)) + self.tasks.add(task) + task.add_done_callback(self.tasks.discard) + + @staticmethod + async def settle(action) -> None: + """ack/nack/reject; the channel may be gone (broker restart): then the broker requeues the + message by itself and the redelivery is a no-op.""" + try: + await action + except Exception as exc: # noqa: BLE001 + log.warning("settle_failed", extra={"fields": {"error": type(exc).__name__}}) + + async def deliver(self, message: AbstractIncomingMessage) -> None: + try: + body = broker.decode(message.body) + except broker.BadMessage as exc: + log.error("bad_message", extra={"fields": {"reason": str(exc), "message_id": message.message_id}}) + await self.settle(message.reject(requeue=False)) # -> messenger.attachment-jobs.dlq + return + try: + async with self.slots: + outcome = await self.process(body["job_id"]) + except Unavailable: + # PostgreSQL (or S3 during a DB step) unavailable: nothing decided. Back to the queue + # after a pause; a claim that did happen is protected by its lease. + log.warning("job_deferred", extra={"fields": {"job_id": body["job_id"]}}) + await asyncio.sleep(2) + await self.settle(message.nack(requeue=True)) + return + except asyncio.CancelledError: + # Shutdown: back to the queue. If the job was claimed it has been released (new outbox + # row), and one of the two deliveries becomes a no-op. + await self.settle(asyncio.shield(message.nack(requeue=True))) + raise + log.info("job_done", extra={"fields": {"job_id": body["job_id"], "outcome": outcome, + "message_attempt": body["attempt"], "redelivered": message.redelivered}}) + await self.settle(message.ack()) + + # ---------------------------------------------------------------- one attempt + async def process(self, job_id: str) -> str: + settings = self.settings + claim = await self.db.run(lambda c: jobs.claim(c, job_id, self.name, settings.job_lease_seconds)) + if isinstance(claim, jobs.Noop): + return "noop_" + claim.reason + await service.crash_once(settings, self.db, "crash_after_claim", job_id, self.actor) + lost = asyncio.Event() + work = asyncio.create_task(self.attempt(claim)) + beat = asyncio.create_task(self.heartbeat(claim, work, lost)) + try: + variants = await work + except asyncio.CancelledError: + if lost.is_set() and not asyncio.current_task().cancelling(): + return "fenced" # lease taken over: this attempt may write nothing more + outcome = await asyncio.shield(self.db.run(lambda c: jobs.release(c, claim, self.name))) + log.info("job_released", extra={"fields": {"job_id": job_id, "outcome": outcome}}) + raise + except ProcessingError as exc: + return await self.failed(claim, exc.code, exc.permanent) + except Unavailable: + return await self.failed(claim, "storage_unavailable", False) + finally: + beat.cancel() + await service.crash_once(settings, self.db, "crash_after_s3_write", job_id, self.actor) + outcome = await self.db.run(lambda c: jobs.complete(c, claim, variants, self.name)) + await service.crash_once(settings, self.db, "crash_after_commit", job_id, self.actor) + if outcome == "succeeded": + await self.delete_staging(claim.attachment_id) + return outcome + + async def failed(self, claim: jobs.Claim, code: str, permanent: bool) -> str: + s = self.settings + outcome, delay = await self.db.run( + lambda c: jobs.retry_or_fail(c, claim, code, permanent, self.name, s.job_retry_base_seconds, + s.job_retry_max_seconds)) + log.info("attempt_failed", extra={"fields": {"job_id": claim.job_id, "attempt": claim.attempt, "code": code, + "permanent": permanent, "outcome": outcome, + "delay_s": round(delay, 3)}}) + return outcome + + async def heartbeat(self, claim: jobs.Claim, work: asyncio.Task, lost: asyncio.Event) -> None: + """Renew the lease while the attempt is alive and within JOB_TIMEOUT_SECONDS. A hung attempt + stops being renewed, so the reaper can hand the job to another worker; if the lease turns out + to be lost, the attempt is cancelled (its result would be fenced anyway).""" + s = self.settings + if "hang" in s.fault_points: + return # experiment: a hung worker that no longer heartbeats + started = time.monotonic() + while True: + await asyncio.sleep(s.job_lease_seconds / 3) + if time.monotonic() - started > s.job_timeout_seconds: + return + try: + alive = await self.db.run(lambda c: jobs.heartbeat(c, claim, s.job_lease_seconds)) + except Unavailable: + continue # the lease may expire meanwhile; the fence protects the result + if not alive: + log.warning("lease_lost", extra={"fields": {"job_id": claim.job_id, "attempt": claim.attempt}}) + lost.set() + work.cancel() + return + + async def attempt(self, claim: jobs.Claim) -> list[imaging.Rendered]: + s = self.settings + if "hang" in s.fault_points: + await asyncio.sleep(s.fault_value("hang", 3600)) # stuck outside of the time limit + try: + return await asyncio.wait_for(self.build(claim), s.job_timeout_seconds) + except asyncio.TimeoutError: + raise ProcessingError("processing_timeout", permanent=False) from None + + async def build(self, claim: jobs.Claim) -> list[imaging.Rendered]: + s = self.settings + started = time.monotonic() + if claim.size_bytes > s.max_upload_bytes: + raise ProcessingError("image_too_large", permanent=True) + source = self.staging if claim.source == "staging" else self.final + try: + data = await source.read_all(claim.source_key, limit=s.max_upload_bytes) + except ObjectMissing: + raise ProcessingError("source_missing", permanent=True) from None + if len(data) != claim.size_bytes or hashlib.sha256(data).digest() != claim.sha256: + raise ProcessingError("source_corrupt", permanent=True) + self.injected(claim) + if "slow" in s.fault_points and s.fault_value("slow_pipeline", claim.pipeline_version) == claim.pipeline_version: + await asyncio.sleep(s.fault_value("slow", 10)) + remaining = max(1.0, s.job_timeout_seconds - (time.monotonic() - started)) + variants = await imaging.render(data, claim.content_type, claim.pipeline_version, + max_pixels=s.max_image_pixels, timeout=remaining, + memory_mb=s.image_max_memory_mb) + meta = {"attachment-id": claim.attachment_id, "pipeline-version": str(claim.pipeline_version)} + puts = [self.final.put(claim.key(v.name), v.data, v.content_type, meta) for v in variants] + if claim.write_original: + puts.append(self.final.put(claim.key("original"), data, claim.content_type, + {**meta, "sha256": claim.sha256.hex()})) + await asyncio.gather(*puts) + return variants + + def injected(self, claim: jobs.Claim) -> None: + s = self.settings + if "permanent_error" in s.fault_points: + raise ProcessingError("invalid_image", permanent=True) + if "transient_error" in s.fault_points and claim.attempt <= s.fault_value("transient_error", 1000): + raise ProcessingError("injected_transient", permanent=False) + + # ---------------------------------------------------------------- cleanup + async def delete_staging(self, attachment_id: str) -> None: + """Mark `deleting` under the row lock (re-checked there), then S3 DELETE, then `deleted`.""" + try: + key = await self.db.run( + lambda c: jobs.mark_staging_deleting(c, attachment_id, self.settings.staging_retention_seconds)) + if key is None: + return + await self.staging.delete(key) + await self.db.run(lambda c: jobs.mark_staging_deleted(c, attachment_id)) + log.info("staging_deleted", extra={"fields": {"attachment_id": attachment_id}}) + except Unavailable: + pass # stays present/deleting; the next cleanup pass repeats (DELETE is idempotent) + + async def cleanup(self) -> None: + s = self.settings + ids = await self.db.run(lambda c: jobs.staging_to_delete(c, s.staging_retention_seconds)) + tokens = await self.db.run(lambda c: jobs.attempts_to_clean(c, s.orphan_grace_seconds)) + if "pause_cleanup" in s.fault_points and (ids or tokens): + # Experiment: the race "cleanup read its candidates -> replay commits -> cleanup goes on". + log.warning("cleanup_paused", extra={"fields": {"attachments": ids, "attempts": tokens}}) + await asyncio.sleep(s.fault_value("pause_cleanup", 10)) + for attachment_id in ids: + await self.delete_staging(attachment_id) + for token in tokens: + keys = await self.db.run(lambda c: jobs.lock_attempt_for_cleanup(c, token)) + if keys is None: + continue + await asyncio.gather(*(self.final.delete(k) for k in keys)) + await self.db.run(lambda c: jobs.mark_attempt_cleaned(c, token)) + log.info("attempt_objects_deleted", extra={"fields": {"token": token, "objects": len(keys)}}) + + async def cleanup_loop(self, stop: asyncio.Event) -> None: + while not stop.is_set() and self.settings.cleanup_interval_seconds > 0: + await service.sleep_or_stop(stop, self.settings.cleanup_interval_seconds) + try: + await self.cleanup() + except Unavailable: + pass + + # ---------------------------------------------------------------- lifecycle + async def run(self) -> int: + stop = service.stop_event() + await self.db.open() + await self.staging.open() + await self.final.open() + cleaner = asyncio.create_task(self.cleanup_loop(stop)) + try: + while not stop.is_set(): + if self.connection is None or self.connection.is_closed: + await self.disconnect() + try: + await self.connect() + except (aio_pika.exceptions.AMQPError, ConnectionError, OSError, asyncio.TimeoutError) as exc: + log.warning("broker_unavailable", extra={"fields": {"error": type(exc).__name__}}) + await self.disconnect() + if self.connection is not None and not self.connection.is_closed: + service.beat() + await service.sleep_or_stop(stop, 2.0) + await self.shutdown() + finally: + cleaner.cancel() + await self.disconnect() + await self.final.close() + await self.staging.close() + await self.db.close() + log.info("worker_stopped") + return 0 + + async def shutdown(self) -> None: + """Stop consuming; running attempts get WORKER_SHUTDOWN_SECONDS, the rest is released.""" + if self.queue is not None and self.consumer_tag is not None: + try: + await self.queue.cancel(self.consumer_tag) + except Exception: # noqa: BLE001 + pass + pending = set(self.tasks) + log.info("worker_draining", extra={"fields": {"in_flight": len(pending)}}) + if pending: + _, pending = await asyncio.wait(pending, timeout=self.settings.worker_shutdown_seconds) + for task in pending: + task.cancel() + if pending: + await asyncio.wait(pending, timeout=10) + + +def main() -> int: + try: + settings = Settings.from_env() + if not settings.broker_url: + raise ConfigError("BROKER_URL_FILE (or BROKER_URL) must be set") + except ConfigError as exc: + print(f"configuration error: {exc}", file=sys.stderr) + return 2 + logging_setup.configure(settings.log_level) + log.info("worker_starting", extra={"fields": { + "instance_id": settings.instance_id, "concurrency": settings.worker_concurrency, + "max_attempts": settings.job_max_attempts, "timeout_s": settings.job_timeout_seconds, + "lease_s": settings.job_lease_seconds, "memory_mb": settings.image_max_memory_mb, + "fault_points": sorted(settings.fault_points)}}) + return asyncio.run(Worker(settings).run()) + + +if __name__ == "__main__": + sys.exit(main())