forked from mtusi2026/mtusi2026lab5
lab5: RabbitMQ, relay и worker в стенде; reconciler lab4 убран
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> broker: RabbitMQ 4.3.7 (тег+digest), quorum queue + DLX/DLQ из definitions, пользователи relay (только publish в exchange) и worker (только consume), без management UI и портов. relay: outbox -> publish с publisher confirm, reaper истёкших lease, страховка от потерянных сообщений. worker: claim + heartbeat lease, варианты, публикация с fencing, ack после COMMIT, graceful stop, уборка staging и мусора попыток. S3: два bucket и четыре identity (admin, api, worker, auditor) с проверкой модели доступа в s3-init. Отдельная сеть queue. Reconciler lab4 (трек прошлой лабы) удалён: его роль выполняет уборка по журналу попыток.
This commit is contained in:
+34
-2
@@ -24,6 +24,8 @@ S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
# Lab 5: private staging bucket for originals accepted before 202 (API: Write; worker: Read+Write).
|
||||
S3_STAGING_BUCKET=messenger-staging
|
||||
# Contract limits (may be lowered, never raised above 10485760 / 40000000).
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
@@ -38,8 +40,33 @@ S3_TIMEOUT_SECONDS=3
|
||||
# Janitor: 'pending' intents older than this are failed uploads (object + row removed).
|
||||
PENDING_GRACE_SECONDS=900
|
||||
JANITOR_INTERVAL_SECONDS=60
|
||||
# Level 4 failure experiments only (after_s3_put, after_ready_commit); keep empty.
|
||||
FAULT_POINTS=
|
||||
# Lab 5: broker credentials are files too (relay and worker get their OWN RabbitMQ user/URL;
|
||||
# secrets/broker_url_relay|worker are mounted as /run/secrets/broker_url). The API has none.
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
# Jobs processed at once by one worker = RabbitMQ prefetch; image child processes at once.
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_REPLICAS=1
|
||||
# Managed attempts (claims) per job, then failed/retry_exhausted; backoff = full jitter.
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_RETRY_BASE_SECONDS=2
|
||||
JOB_RETRY_MAX_SECONDS=60
|
||||
# One attempt: hard time limit (child process killed) and the lease renewed by its heartbeat.
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_LEASE_SECONDS=30
|
||||
# Address-space limit of the image child process (RLIMIT_AS).
|
||||
IMAGE_MAX_MEMORY_MB=768
|
||||
# Staging copy of a FAILED attachment is kept this long for a replay (ready: deleted at once).
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
# Objects of attempts that never published are deleted this long after the attempt ended.
|
||||
ORPHAN_GRACE_SECONDS=600
|
||||
# Queued job without a claim for this long gets a new outbox row (lost-message guard).
|
||||
QUEUED_REPUBLISH_SECONDS=300
|
||||
# Pipeline version of new uploads (track 2: versions 1, 2, 3 in src/messenger/pipeline.py).
|
||||
PIPELINE_VERSION=1
|
||||
# Failure experiments only, per service (see src/messenger/config.py FAULT_POINTS); keep empty.
|
||||
API_FAULT_POINTS=
|
||||
RELAY_FAULT_POINTS=
|
||||
WORKER_FAULT_POINTS=
|
||||
# Additional settings of this implementation (defaults shown; all optional):
|
||||
LOG_LEVEL=info
|
||||
DB_POOL_MIN=1
|
||||
@@ -60,6 +87,11 @@ PG_CPUS=2.0
|
||||
PG_MEMORY=1g
|
||||
VALKEY_CPUS=0.5
|
||||
LB_CPUS=1.0
|
||||
BROKER_CPUS=1.0
|
||||
BROKER_MEMORY=512m
|
||||
WORKER_CPUS=2.0
|
||||
WORKER_MEMORY=1536m
|
||||
RELAY_CPUS=0.5
|
||||
PG_MAX_CONNECTIONS=100
|
||||
# Host ports of the load balancer (published on 127.0.0.1 only; API replicas publish nothing)
|
||||
APP_PUBLISH_PORT=8080
|
||||
|
||||
+3
-2
@@ -1,6 +1,7 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# Messenger API, lab 4 (replicas behind HAProxy, PostgreSQL + Valkey + S3). Built by compose.yaml
|
||||
# (services api, migrate, s3-init, reconciler: one image, different commands).
|
||||
# Messenger, lab 5 (API replicas behind HAProxy, PostgreSQL + Valkey + S3 + RabbitMQ). Built by
|
||||
# compose.yaml (services api, migrate, s3-init, relay, worker, auditor, jobctl: one image, different
|
||||
# commands; only the worker imports Pillow).
|
||||
#
|
||||
# Python is interpreted, so there is no separate compile stage; the image
|
||||
# contains only the interpreter, locked dependencies and the source code.
|
||||
|
||||
@@ -29,23 +29,23 @@ test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Student solution (lab 4, Python). On Windows: PowerShell/Git Bash with GNU make, public
|
||||
# Student solution (lab 5, Python). On Windows: PowerShell/Git Bash with GNU make, public
|
||||
# tests need PYTHONUTF8=1, and the hooks are started through scripts/*.cmd (Git Bash).
|
||||
.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series reconcile reconcile-apply bench-memory bench-reconcile
|
||||
.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series bench-memory worker-stop worker-start jobs failed audit track2
|
||||
|
||||
secrets:
|
||||
$(PYTHON) scripts/init_secrets.py
|
||||
|
||||
# Build, apply migrations ONCE (service migrate), start API replicas and the LB, wait for healthy.
|
||||
# Build, apply migrations ONCE (service migrate), start broker, API replicas, LB, relay and worker.
|
||||
up: secrets
|
||||
docker compose up -d --build --wait
|
||||
|
||||
# Safe stop: containers stop, named volumes (data) stay.
|
||||
# Safe stop: containers stop, named volumes (data, queue) stay.
|
||||
down:
|
||||
docker compose down
|
||||
|
||||
logs:
|
||||
docker compose logs --tail 50 api lb
|
||||
docker compose logs --tail 50 api lb relay worker
|
||||
|
||||
ps:
|
||||
docker compose ps -a
|
||||
@@ -67,10 +67,32 @@ restore:
|
||||
rolling-update:
|
||||
sh scripts/rolling-update.sh
|
||||
|
||||
# Level 4: stop each replica in turn under load, measure RPS/errors/recovery (evidence/failover/).
|
||||
# Lab 3: stop each replica in turn under load, measure RPS/errors/recovery (evidence/failover/).
|
||||
failover-series:
|
||||
$(PYTHON) bench/failover_series.py --output evidence/failover
|
||||
|
||||
# Lab 5 demo: stop the worker (uploads still get 202, content answers 409), start it again.
|
||||
worker-stop:
|
||||
docker compose stop worker
|
||||
|
||||
worker-start:
|
||||
docker compose start worker
|
||||
|
||||
# Lab 5 operator view: jobs by state + outbox backlog; failed registry (replay: see REPORT.md).
|
||||
jobs:
|
||||
docker compose run --rm --no-deps jobctl stats
|
||||
|
||||
failed:
|
||||
docker compose run --rm --no-deps jobctl list-failed
|
||||
|
||||
# Track 2: read-only check of the pipeline invariants over PostgreSQL and both buckets.
|
||||
audit:
|
||||
docker compose run --rm --no-deps auditor
|
||||
|
||||
# Track 2: automatic fault-injection matrix (duplicate, reorder, crashes, stale versions).
|
||||
track2:
|
||||
$(PYTHON) bench/pipeline_faults.py --output evidence/track2
|
||||
|
||||
# Own tests: unit + black-box against the running stand (needs requirements*.txt in a venv).
|
||||
student-test:
|
||||
$(PYTHON) -m pytest student-tests -v
|
||||
@@ -81,17 +103,6 @@ unit-test:
|
||||
explain:
|
||||
$(PYTHON) scripts/explain_history.py --output evidence/explain-history.txt
|
||||
|
||||
# Track 2: reconciler of S3 objects vs attachment rows (dry-run by default, read-only).
|
||||
reconcile:
|
||||
docker compose run --rm reconciler
|
||||
|
||||
reconcile-apply:
|
||||
docker compose run --rm reconciler --apply
|
||||
|
||||
# Level 4: peak RAM of the replicas under 16 simultaneous uploads (evidence/memory/).
|
||||
# Lab 4: peak RAM of the replicas under 16 simultaneous uploads (evidence/memory/).
|
||||
bench-memory:
|
||||
$(PYTHON) bench/upload_memory.py --output evidence/memory
|
||||
|
||||
# Track 2: cost of one reconciler pass for 1k/10k/50k attachments (evidence/reconcile/).
|
||||
bench-reconcile:
|
||||
$(PYTHON) bench/reconcile_cost.py --output evidence/reconcile
|
||||
|
||||
+225
-42
@@ -1,15 +1,20 @@
|
||||
# Lab 4 stand: client -> HAProxy (lb) -> API x N (one scalable service `api`)
|
||||
# -> PostgreSQL (metadata, source of truth) + Valkey (sessions) + SeaweedFS S3 (bytes).
|
||||
# Lab 5 stand: client -> HAProxy (lb) -> API x N (scalable service `api`)
|
||||
# -> PostgreSQL (metadata, jobs, outbox: source of truth) + Valkey (sessions)
|
||||
# + SeaweedFS S3 (staging bucket and final bucket, both private)
|
||||
# relay : outbox in PostgreSQL -> RabbitMQ (publisher confirms) [service `relay`]
|
||||
# broker : RabbitMQ 4.3, durable quorum queue + dead-letter queue [service `broker`]
|
||||
# worker : RabbitMQ -> variants -> final bucket + PostgreSQL, then ack [service `worker`]
|
||||
# From a clean clone (details and Windows notes: REPORT.md, docs/WINDOWS.md):
|
||||
# python scripts/init_secrets.py # once: random local credentials into ./secrets (git-ignored)
|
||||
# cp .env.example .env # optional overrides (API_REPLICAS, limits, ...)
|
||||
# docker compose --env-file .env up -d --build --wait
|
||||
# Order: postgres/valkey/s3 healthy -> migrate + s3-init (ONE run each, exit 0) -> api healthy -> lb.
|
||||
# Order: postgres/valkey/s3/broker healthy -> migrate + s3-init (ONE run each, exit 0)
|
||||
# -> api healthy -> lb; relay and worker start after migrate/s3-init.
|
||||
# Images are pinned by version AND digest. Passwords/keys are passed only as secret files, and
|
||||
# every service gets only the secrets it needs. Only the LB publishes ports (loopback only).
|
||||
# Three networks: `edge` (lb <-> api), `data` (api/migrate/reconciler <-> postgres/valkey) and
|
||||
# `storage` (api/s3-init/reconciler <-> s3); data and storage are internal (no route outside).
|
||||
name: mtusi-lab4
|
||||
# Networks: `edge` (lb <-> api); internal (no route outside): `data` (postgres/valkey),
|
||||
# `storage` (s3), `queue` (broker). The API is NOT on `queue`: it never talks to the broker.
|
||||
name: mtusi-lab5
|
||||
|
||||
x-api-env: &api-env
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
@@ -29,10 +34,11 @@ x-api-env: &api-env
|
||||
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-8}
|
||||
DOCS_ENABLED: ${DOCS_ENABLED:-true}
|
||||
MESSAGE_WRITE_PATH: ${MESSAGE_WRITE_PATH:-function}
|
||||
# Lab 4: S3 (the API's own Read+Write key pair, see infra/seaweedfs/start.sh) and upload limits.
|
||||
# S3: the API's own key pair (Write staging + Read final, see infra/seaweedfs/start.sh).
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3}
|
||||
@@ -46,8 +52,33 @@ x-api-env: &api-env
|
||||
JPEG_MAX_MEMORY_MB: ${JPEG_MAX_MEMORY_MB:-256}
|
||||
PENDING_GRACE_SECONDS: ${PENDING_GRACE_SECONDS:-900}
|
||||
JANITOR_INTERVAL_SECONDS: ${JANITOR_INTERVAL_SECONDS:-60}
|
||||
# Level 4 failure experiments only (compose.faults.yaml); empty = no injected failures.
|
||||
FAULT_POINTS: ${FAULT_POINTS:-}
|
||||
# Lab 5: pipeline version and attempt budget of the jobs created by uploads.
|
||||
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
# Failure experiments only (student-tests, bench/); empty = no injected failures.
|
||||
FAULT_POINTS: ${API_FAULT_POINTS:-}
|
||||
|
||||
# Settings of the pipeline shared by relay and worker (docs/PIPELINE.md).
|
||||
x-pipeline-env: &pipeline-env
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-info}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
DB_POOL_MAX: ${PIPELINE_DB_POOL_MAX:-6}
|
||||
DB_TIMEOUT_SECONDS: ${DB_TIMEOUT_SECONDS:-3}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging}
|
||||
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-30}
|
||||
JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-2}
|
||||
JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:-60}
|
||||
QUEUED_REPUBLISH_SECONDS: ${QUEUED_REPUBLISH_SECONDS:-300}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
@@ -102,15 +133,15 @@ services:
|
||||
resources:
|
||||
limits: {cpus: "${VALKEY_CPUS:-0.5}", memory: 256m}
|
||||
|
||||
# Lab 4: S3-compatible object storage (SeaweedFS: master + volume + filer + S3 gateway in one
|
||||
# process). Bytes of the attachments live ONLY here, on the named volume s3data. Identities
|
||||
# and exposure: infra/seaweedfs/start.sh. Not published on the host; its web UIs listen on
|
||||
# 127.0.0.1 inside the container only.
|
||||
# S3-compatible object storage (SeaweedFS: master + volume + filer + S3 gateway in one process).
|
||||
# Bytes live ONLY here, on the named volume s3data: staging bucket (originals accepted before
|
||||
# 202) and final bucket (published originals and variants). Identities: infra/seaweedfs/start.sh.
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:4.46@sha256:08d516132314207d10c8e37cbffc1f32b147d870169688734cc61c6231625b62
|
||||
entrypoint: ["/bin/sh", "/etc/seaweedfs/start.sh"]
|
||||
environment:
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging}
|
||||
# Go heap target below the container limit: the GC returns memory before the cgroup OOM.
|
||||
GOMEMLIMIT: ${S3_GOMEMLIMIT:-600MiB}
|
||||
secrets:
|
||||
@@ -118,8 +149,10 @@ services:
|
||||
- s3_admin_secret_access_key
|
||||
- s3_api_access_key_id
|
||||
- s3_api_secret_access_key
|
||||
- s3_reconciler_access_key_id
|
||||
- s3_reconciler_secret_access_key
|
||||
- s3_worker_access_key_id
|
||||
- s3_worker_secret_access_key
|
||||
- s3_auditor_access_key_id
|
||||
- s3_auditor_secret_access_key
|
||||
volumes:
|
||||
- ./infra/seaweedfs/start.sh:/etc/seaweedfs/start.sh:ro
|
||||
- s3data:/data
|
||||
@@ -137,6 +170,34 @@ services:
|
||||
resources:
|
||||
limits: {cpus: "${S3_CPUS:-1.0}", memory: "${S3_MEMORY:-1g}"}
|
||||
|
||||
# Lab 5: RabbitMQ. Topology (exchange, quorum queue, DLX/DLQ) and users come from definition
|
||||
# files imported at boot (infra/rabbitmq, secrets/rabbitmq_users.json). Messages of the quorum
|
||||
# queue live on the named volume rabbitmqdata. No management UI, no host port.
|
||||
broker:
|
||||
image: rabbitmq:4.3.7-alpine@sha256:b1f997f74103d08fc1f20a13890dc44fe6cfa51d528cf521b480ada988d3ffa5
|
||||
hostname: broker # node name rabbit@broker: stable across container re-creation
|
||||
secrets:
|
||||
- source: rabbitmq_users
|
||||
target: /etc/rabbitmq/definitions.d/users.json
|
||||
volumes:
|
||||
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/conf.d/20-lab5.conf:ro
|
||||
- ./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro
|
||||
- ./infra/rabbitmq/topology.json:/etc/rabbitmq/definitions.d/topology.json:ro
|
||||
- rabbitmqdata:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
# The node runs and the AMQP listener accepts connections.
|
||||
test: ["CMD-SHELL", "rabbitmq-diagnostics -q ping && rabbitmq-diagnostics -q check_port_connectivity"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 30
|
||||
start_period: 20s
|
||||
stop_grace_period: 30s
|
||||
restart: unless-stopped
|
||||
networks: [queue]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${BROKER_CPUS:-1.0}", memory: "${BROKER_MEMORY:-512m}"}
|
||||
|
||||
# One-shot schema migration: runs ONCE per `up`, before any API replica starts
|
||||
# (api depends_on service_completed_successfully). Replicas never migrate themselves.
|
||||
migrate:
|
||||
@@ -144,7 +205,7 @@ services:
|
||||
context: .
|
||||
args:
|
||||
BUILD_REVISION: ${BUILD_REVISION:-dev}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab4}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ["python", "-m", "messenger.migrate"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
@@ -160,28 +221,33 @@ services:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 256m}
|
||||
|
||||
# One-shot, idempotent S3 bootstrap: bucket, sentinel object, verification of the access model
|
||||
# (private bucket, least-privilege keys). The ONLY holder of the admin key pair besides s3.
|
||||
# One-shot, idempotent S3 bootstrap: both buckets, sentinel object, verification of the access
|
||||
# model (private buckets, least-privilege keys). The ONLY holder of the admin key pair besides s3.
|
||||
s3-init:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab4}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ["python", "-m", "messenger.s3_bootstrap"]
|
||||
environment:
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging}
|
||||
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
|
||||
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_access_key
|
||||
S3_RECONCILER_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id
|
||||
S3_RECONCILER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key
|
||||
S3_WORKER_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key_id
|
||||
S3_WORKER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_access_key
|
||||
S3_AUDITOR_ACCESS_KEY_ID_FILE: /run/secrets/s3_auditor_access_key_id
|
||||
S3_AUDITOR_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_auditor_secret_access_key
|
||||
secrets:
|
||||
- s3_admin_access_key_id
|
||||
- s3_admin_secret_access_key
|
||||
- s3_api_access_key_id
|
||||
- s3_api_secret_access_key
|
||||
- s3_reconciler_access_key_id
|
||||
- s3_reconciler_secret_access_key
|
||||
- s3_worker_access_key_id
|
||||
- s3_worker_secret_access_key
|
||||
- s3_auditor_access_key_id
|
||||
- s3_auditor_secret_access_key
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
@@ -200,7 +266,7 @@ services:
|
||||
context: .
|
||||
args:
|
||||
BUILD_REVISION: ${BUILD_REVISION:-dev}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab4}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
environment: *api-env
|
||||
secrets:
|
||||
- database_url
|
||||
@@ -236,7 +302,7 @@ services:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
# 512m: UPLOAD_CONCURRENCY (4) x 10 MiB bodies + one djpeg (<= JPEG_MAX_MEMORY_MB) + runtime;
|
||||
# measured in evidence/memory/ (bench/upload_memory.py).
|
||||
# measured in lab 4 (bench/upload_memory.py). No resizing happens in the API.
|
||||
limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-512m}"}
|
||||
# No swap on top of the limit (Docker's default would allow as much swap again): the RAM bound
|
||||
# of the experiments is a real bound, and running out of it is visible, not hidden in swap.
|
||||
@@ -268,29 +334,112 @@ services:
|
||||
resources:
|
||||
limits: {cpus: "${LB_CPUS:-1.0}", memory: 128m}
|
||||
|
||||
# Track 2 (lab 4): reconciler of S3 objects vs PostgreSQL metadata. Not a daemon: started on
|
||||
# demand, dry-run by default (read-only on both stores):
|
||||
# docker compose run --rm reconciler # report only
|
||||
# docker compose run --rm reconciler --apply # fix: orphans, stale intents, quarantine
|
||||
reconciler:
|
||||
# Lab 5: outbox relay + lease reaper (src/messenger/relay.py). One process; a second one would
|
||||
# be safe (SKIP LOCKED) but is not needed. Database + broker only, no S3 keys.
|
||||
relay:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ["python", "-m", "messenger.relay"]
|
||||
environment:
|
||||
<<: *pipeline-env
|
||||
RELAY_POLL_SECONDS: ${RELAY_POLL_SECONDS:-0.5}
|
||||
FAULT_POINTS: ${RELAY_FAULT_POINTS:-}
|
||||
secrets:
|
||||
- database_url
|
||||
- source: broker_url_relay
|
||||
target: broker_url
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-m", "messenger.healthcheck", "--file", "/tmp/heartbeat", "--max-age", "30"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
stop_grace_period: 10s
|
||||
restart: unless-stopped
|
||||
networks: [data, queue]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${RELAY_CPUS:-0.5}", memory: 256m}
|
||||
|
||||
# Lab 5: image worker (src/messenger/worker.py). Scalable (`--scale worker=2`): competing
|
||||
# consumers of one queue; the database lease decides who works on a job.
|
||||
worker:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ["python", "-m", "messenger.worker"]
|
||||
environment:
|
||||
<<: *pipeline-env
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_SHUTDOWN_SECONDS: ${WORKER_SHUTDOWN_SECONDS:-20}
|
||||
IMAGE_MAX_MEMORY_MB: ${IMAGE_MAX_MEMORY_MB:-768}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-600}
|
||||
CLEANUP_INTERVAL_SECONDS: ${CLEANUP_INTERVAL_SECONDS:-30}
|
||||
FAULT_POINTS: ${WORKER_FAULT_POINTS:-}
|
||||
secrets:
|
||||
- database_url
|
||||
- source: broker_url_worker
|
||||
target: broker_url
|
||||
- source: s3_worker_access_key_id
|
||||
target: s3_access_key_id
|
||||
- source: s3_worker_secret_access_key
|
||||
target: s3_secret_access_key
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-m", "messenger.healthcheck", "--file", "/tmp/heartbeat", "--max-age", "30"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
stop_grace_period: 30s # > WORKER_SHUTDOWN_SECONDS (20 s) + release
|
||||
restart: unless-stopped
|
||||
networks: [data, storage, queue]
|
||||
deploy:
|
||||
replicas: ${WORKER_REPLICAS:-1}
|
||||
resources:
|
||||
# WORKER_CONCURRENCY (2) child processes x IMAGE_MAX_MEMORY_MB (768) address space + worker.
|
||||
limits: {cpus: "${WORKER_CPUS:-2.0}", memory: "${WORKER_MEMORY:-1536m}"}
|
||||
memswap_limit: ${WORKER_MEMORY:-1536m}
|
||||
|
||||
# Track 2: read-only checker of the pipeline invariants (database in read-only transactions,
|
||||
# S3 auditor key: Read + List, no Write). On demand: docker compose run --rm auditor
|
||||
auditor:
|
||||
profiles: ["tools"]
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab4}
|
||||
entrypoint: ["python", "-m", "messenger.reconcile"]
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
entrypoint: ["python", "-m", "messenger.audit"]
|
||||
command: []
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
DB_TIMEOUT_SECONDS: ${RECONCILE_DB_TIMEOUT_SECONDS:-10}
|
||||
DB_TIMEOUT_SECONDS: ${AUDIT_DB_TIMEOUT_SECONDS:-30}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3}
|
||||
secrets:
|
||||
- database_url
|
||||
- source: s3_reconciler_access_key_id
|
||||
- source: s3_auditor_access_key_id
|
||||
target: s3_access_key_id
|
||||
- source: s3_reconciler_secret_access_key
|
||||
- source: s3_auditor_secret_access_key
|
||||
target: s3_secret_access_key
|
||||
depends_on:
|
||||
postgres:
|
||||
@@ -301,19 +450,43 @@ services:
|
||||
networks: [data, storage]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${RECONCILER_CPUS:-1.0}", memory: 256m}
|
||||
limits: {cpus: "1.0", memory: 256m}
|
||||
|
||||
# Operator CLI (replay of failed jobs, reprocess with a newer pipeline, failed registry):
|
||||
# docker compose run --rm jobctl list-failed
|
||||
# docker compose run --rm jobctl replay <attachment_id> --operator alice --reason "..."
|
||||
jobctl:
|
||||
profiles: ["tools"]
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
entrypoint: ["python", "-m", "messenger.jobctl"]
|
||||
command: []
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
secrets: [database_url]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
networks: [data]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128m}
|
||||
|
||||
networks:
|
||||
edge:
|
||||
data:
|
||||
internal: true # no route outside: stores talk only to api/migrate
|
||||
internal: true # no route outside: stores talk only to the services above
|
||||
storage:
|
||||
internal: true # S3 API reachable only by api, s3-init, reconciler
|
||||
internal: true # S3 API reachable only by api, s3-init, worker, auditor
|
||||
queue:
|
||||
internal: true # broker reachable only by relay and worker
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
valkeydata:
|
||||
s3data:
|
||||
rabbitmqdata:
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
@@ -332,7 +505,17 @@ secrets:
|
||||
file: ./secrets/s3_api_access_key_id
|
||||
s3_api_secret_access_key:
|
||||
file: ./secrets/s3_api_secret_access_key
|
||||
s3_reconciler_access_key_id:
|
||||
file: ./secrets/s3_reconciler_access_key_id
|
||||
s3_reconciler_secret_access_key:
|
||||
file: ./secrets/s3_reconciler_secret_access_key
|
||||
s3_worker_access_key_id:
|
||||
file: ./secrets/s3_worker_access_key_id
|
||||
s3_worker_secret_access_key:
|
||||
file: ./secrets/s3_worker_secret_access_key
|
||||
s3_auditor_access_key_id:
|
||||
file: ./secrets/s3_auditor_access_key_id
|
||||
s3_auditor_secret_access_key:
|
||||
file: ./secrets/s3_auditor_secret_access_key
|
||||
rabbitmq_users:
|
||||
file: ./secrets/rabbitmq_users.json
|
||||
broker_url_relay:
|
||||
file: ./secrets/broker_url_relay
|
||||
broker_url_worker:
|
||||
file: ./secrets/broker_url_worker
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
[].
|
||||
@@ -0,0 +1,24 @@
|
||||
# RabbitMQ 4.3 of the lab 5 stand (compose service `broker`), mounted as conf.d/20-lab5.conf.
|
||||
# Only AMQP 0-9-1 on 5672 inside the internal network `queue`: no management UI, no Prometheus
|
||||
# listener (enabled_plugins is empty), nothing published on the host.
|
||||
listeners.tcp.default = 5672
|
||||
|
||||
# Users, permissions and topology are imported from files at every boot (idempotent):
|
||||
# /etc/rabbitmq/definitions.d/topology.json exchanges, quorum queues, bindings (this repo)
|
||||
# /etc/rabbitmq/definitions.d/users.json users relay/worker + permissions (Docker secret,
|
||||
# generated by scripts/init_secrets.py)
|
||||
# With imported definitions no default `guest` user is created.
|
||||
definitions.import_backend = local_filesystem
|
||||
definitions.local.path = /etc/rabbitmq/definitions.d
|
||||
definitions.skip_if_unchanged = false
|
||||
|
||||
# A delivery that stays unacked longer than this closes the consumer's channel and the message
|
||||
# is returned to the queue (a worker that hangs while connected). Far above JOB_TIMEOUT_SECONDS.
|
||||
consumer_timeout = 300000
|
||||
|
||||
# The container has a 512 MiB limit; flow control starts well before the cgroup OOM.
|
||||
vm_memory_high_watermark.relative = 0.5
|
||||
disk_free_limit.absolute = 200MB
|
||||
|
||||
log.console = true
|
||||
log.console.level = info
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"vhosts": [{"name": "/"}],
|
||||
"exchanges": [
|
||||
{"name": "messenger.jobs", "vhost": "/", "type": "direct", "durable": true, "auto_delete": false,
|
||||
"internal": false, "arguments": {}},
|
||||
{"name": "messenger.jobs.dlx", "vhost": "/", "type": "fanout", "durable": true, "auto_delete": false,
|
||||
"internal": false, "arguments": {}}
|
||||
],
|
||||
"queues": [
|
||||
{"name": "messenger.attachment-jobs", "vhost": "/", "durable": true, "auto_delete": false,
|
||||
"arguments": {"x-queue-type": "quorum", "x-delivery-limit": 20, "x-dead-letter-exchange": "messenger.jobs.dlx"}},
|
||||
{"name": "messenger.attachment-jobs.dlq", "vhost": "/", "durable": true, "auto_delete": false,
|
||||
"arguments": {"x-queue-type": "quorum"}}
|
||||
],
|
||||
"bindings": [
|
||||
{"source": "messenger.jobs", "vhost": "/", "destination": "messenger.attachment-jobs", "destination_type": "queue",
|
||||
"routing_key": "attachment.process.v1", "arguments": {}},
|
||||
{"source": "messenger.jobs.dlx", "vhost": "/", "destination": "messenger.attachment-jobs.dlq",
|
||||
"destination_type": "queue", "routing_key": "", "arguments": {}}
|
||||
]
|
||||
}
|
||||
+19
-12
@@ -2,11 +2,14 @@
|
||||
# Entry point of the compose service `s3` (SeaweedFS 4.46, one container: master + volume +
|
||||
# filer + S3 gateway). Renders the S3 identities from Docker secrets, then execs `weed`.
|
||||
#
|
||||
# Identities (SeaweedFS s3.json, actions scoped to the bucket):
|
||||
# bootstrap-admin Admin (create bucket) -> only s3-init has these keys
|
||||
# messenger-api Read + Write on S3_BUCKET -> API replicas (no List, no Admin)
|
||||
# messenger-reconciler Read + Write + List on S3_BUCKET -> reconciler CLI (needs the listing)
|
||||
# There is NO anonymous identity, so every unsigned request is refused: the bucket is private.
|
||||
# Two private buckets (lab 5): S3_STAGING_BUCKET holds originals accepted by the API before 202,
|
||||
# S3_BUCKET holds the published originals and variants written by the worker.
|
||||
# Identities (SeaweedFS s3.json, actions scoped to a bucket):
|
||||
# bootstrap-admin Admin (create buckets) -> only s3-init has these keys
|
||||
# messenger-api Write staging, Read final -> API replicas (no List, no Admin)
|
||||
# messenger-worker Read + Write staging, Read + Write final -> worker (no List, no Admin)
|
||||
# messenger-auditor Read + List both buckets (no Write) -> invariant checker (tools profile)
|
||||
# There is NO anonymous identity, so every unsigned request is refused: the buckets are private.
|
||||
#
|
||||
# Exposure: master (9333), volume (8080), filer (8888) with their web UIs and gRPC listen on
|
||||
# 127.0.0.1 INSIDE the container only (-ip.bind); only the S3 API port 8333 listens on the
|
||||
@@ -15,6 +18,7 @@ set -eu
|
||||
umask 077
|
||||
secret() { tr -d '\r\n' < "/run/secrets/$1"; }
|
||||
bucket="${S3_BUCKET:?S3_BUCKET is required}"
|
||||
staging="${S3_STAGING_BUCKET:?S3_STAGING_BUCKET is required}"
|
||||
cat > /tmp/s3.json <<EOF
|
||||
{
|
||||
"identities": [
|
||||
@@ -23,17 +27,20 @@ cat > /tmp/s3.json <<EOF
|
||||
"actions": ["Admin", "Read", "Write", "List", "Tagging"]},
|
||||
{"name": "messenger-api",
|
||||
"credentials": [{"accessKey": "$(secret s3_api_access_key_id)", "secretKey": "$(secret s3_api_secret_access_key)"}],
|
||||
"actions": ["Read:${bucket}", "Write:${bucket}"]},
|
||||
{"name": "messenger-reconciler",
|
||||
"credentials": [{"accessKey": "$(secret s3_reconciler_access_key_id)", "secretKey": "$(secret s3_reconciler_secret_access_key)"}],
|
||||
"actions": ["Read:${bucket}", "Write:${bucket}", "List:${bucket}"]}
|
||||
"actions": ["Write:${staging}", "Read:${bucket}"]},
|
||||
{"name": "messenger-worker",
|
||||
"credentials": [{"accessKey": "$(secret s3_worker_access_key_id)", "secretKey": "$(secret s3_worker_secret_access_key)"}],
|
||||
"actions": ["Read:${staging}", "Write:${staging}", "Read:${bucket}", "Write:${bucket}"]},
|
||||
{"name": "messenger-auditor",
|
||||
"credentials": [{"accessKey": "$(secret s3_auditor_access_key_id)", "secretKey": "$(secret s3_auditor_secret_access_key)"}],
|
||||
"actions": ["Read:${staging}", "List:${staging}", "Read:${bucket}", "List:${bucket}"]}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
# Memory: by default SeaweedFS buffers every in-flight upload without a limit and the Go runtime
|
||||
# keeps freed heap; measured ~1 GB under 16 parallel 9.4 MiB PUTs (evidence/memory). The upload
|
||||
# limits make excess PUTs wait inside SeaweedFS instead of growing the heap, and GOMEMLIMIT (set
|
||||
# in compose.yaml below the container limit) makes the Go GC return memory before the cgroup OOM.
|
||||
# keeps freed heap; measured ~1 GB under 16 parallel 9.4 MiB PUTs (lab 4). The upload limits make
|
||||
# excess PUTs wait inside SeaweedFS instead of growing the heap, and GOMEMLIMIT (set in
|
||||
# compose.yaml below the container limit) makes the Go GC return memory before the cgroup OOM.
|
||||
exec weed server \
|
||||
-dir=/data \
|
||||
-ip=127.0.0.1 -ip.bind=127.0.0.1 \
|
||||
|
||||
+32
-5
@@ -9,6 +9,9 @@ Delete ./secrets together with the volumes (`docker compose down -v`) to start o
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import sys
|
||||
@@ -24,8 +27,8 @@ def write(name: str, value: str) -> None:
|
||||
print(f"keep secrets/{name}")
|
||||
return
|
||||
path.write_text(value + "\n", encoding="utf-8", newline="\n")
|
||||
# Readable by the container users (postgres, valkey, seaweedfs, app uid 10001) through the
|
||||
# compose bind mount. Local synthetic lab credentials only.
|
||||
# Readable by the container users (postgres, valkey, seaweedfs, rabbitmq, app uid 10001)
|
||||
# through the compose bind mount. Local synthetic lab credentials only.
|
||||
os.chmod(path, 0o644)
|
||||
print(f"created secrets/{name}")
|
||||
|
||||
@@ -34,17 +37,41 @@ def read(name: str) -> str:
|
||||
return (SECRETS / name).read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def rabbit_hash(password: str) -> str:
|
||||
"""RabbitMQ `rabbit_password_hashing_sha256`: base64(salt + sha256(salt + password)), 4-byte salt.
|
||||
The broker stores and compares only this hash; the plain password is in the client's URL."""
|
||||
salt = secrets.token_bytes(4)
|
||||
return base64.b64encode(salt + hashlib.sha256(salt + password.encode("utf-8")).digest()).decode("ascii")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
SECRETS.mkdir(exist_ok=True)
|
||||
write("postgres_password", secrets.token_urlsafe(24)) # [A-Za-z0-9_-]: safe inside URLs
|
||||
write("valkey_password", secrets.token_urlsafe(24))
|
||||
write("database_url", f"postgresql://messenger:{read('postgres_password')}@postgres:5432/messenger")
|
||||
write("cache_url", f"redis://:{read('valkey_password')}@valkey:6379/0")
|
||||
# Lab 4: three separate S3 key pairs (SeaweedFS identities, infra/seaweedfs/start.sh):
|
||||
# admin = bucket bootstrap only; api = Read+Write on the bucket; reconciler = + List.
|
||||
for who in ("admin", "api", "reconciler"):
|
||||
# S3 key pairs (SeaweedFS identities, infra/seaweedfs/start.sh): admin = bucket bootstrap only;
|
||||
# api = Write staging + Read final; worker = Read/Write both; auditor = Read + List (no Write).
|
||||
for who in ("admin", "api", "worker", "auditor"):
|
||||
write(f"s3_{who}_access_key_id", "AK" + secrets.token_hex(9).upper())
|
||||
write(f"s3_{who}_secret_access_key", secrets.token_hex(20))
|
||||
# Lab 5: RabbitMQ users. relay may only publish to the jobs exchange, worker may only consume
|
||||
# the jobs queue; neither may declare/delete anything (configure "^$").
|
||||
write("rabbitmq_relay_password", secrets.token_urlsafe(24))
|
||||
write("rabbitmq_worker_password", secrets.token_urlsafe(24))
|
||||
users = {
|
||||
"users": [{"name": who, "password_hash": rabbit_hash(read(f"rabbitmq_{who}_password")),
|
||||
"hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []} for who in ("relay", "worker")],
|
||||
"permissions": [
|
||||
{"user": "relay", "vhost": "/", "configure": "^$", "write": "^messenger\\.jobs$", "read": "^$"},
|
||||
{"user": "worker", "vhost": "/", "configure": "^$", "write": "^$", "read": "^messenger\\.attachment-jobs$"},
|
||||
],
|
||||
}
|
||||
write("rabbitmq_users.json", json.dumps(users, indent=1))
|
||||
for who in ("relay", "worker"):
|
||||
# AMQP heartbeat 10 s: a frozen client loses its connection after ~2 missed heartbeats, and
|
||||
# the broker returns its unacked messages to the queue for the other consumers.
|
||||
write(f"broker_url_{who}", f"amqp://{who}:{read(f'rabbitmq_{who}_password')}@broker:5672/?heartbeat=10")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
+12
-9
@@ -1,18 +1,21 @@
|
||||
#!/bin/sh
|
||||
# Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`.
|
||||
# Restarts EVERY process of the stand: LB, all API replicas, PostgreSQL, Valkey and S3 (SeaweedFS).
|
||||
# Volumes, schema and data are preserved: no `down -v`, no FLUSH, no re-seed.
|
||||
# Restarts EVERY process of the stand: LB, all API replicas, relay, worker(s), PostgreSQL, Valkey,
|
||||
# S3 (SeaweedFS) and the broker (RabbitMQ). Volumes, schema, data and queued messages are
|
||||
# preserved: no `down -v`, no FLUSH, no purge, no re-seed.
|
||||
# Output is suppressed by the test; run it by hand to see the steps.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# 1. Entry point and API first: no request is served while the stores restart underneath.
|
||||
docker compose stop lb api
|
||||
# 1. Entry point, API and the pipeline processes first: nothing is served or processed while the
|
||||
# stores restart underneath.
|
||||
docker compose stop lb api relay worker
|
||||
# 2. Stores: clean shutdown (PostgreSQL checkpoint, Valkey AOF fsync, SeaweedFS volume/filer
|
||||
# flush), then start again. The S3 bytes live on the named volume s3data.
|
||||
docker compose restart postgres valkey s3
|
||||
# 3. API replicas (wait for healthy stores via depends_on), then the LB. HAProxy resolves
|
||||
# the new replica addresses through Docker DNS by itself - no stale backend list.
|
||||
# flush, RabbitMQ quorum log), then start again. Bytes live on s3data, messages on rabbitmqdata.
|
||||
docker compose restart postgres valkey s3 broker
|
||||
# 3. API replicas (wait for healthy stores via depends_on), the LB, then relay and worker. HAProxy
|
||||
# resolves the new replica addresses through Docker DNS by itself - no stale backend list.
|
||||
docker compose start api
|
||||
docker compose start lb
|
||||
echo "restart.sh: lb, api (all replicas), postgres, valkey and s3 restarted with volumes preserved"
|
||||
docker compose start relay worker
|
||||
echo "restart.sh: lb, api (all replicas), relay, worker, postgres, valkey, s3 and broker restarted with volumes preserved"
|
||||
|
||||
@@ -1,12 +1,24 @@
|
||||
"""Docker HEALTHCHECK probe without curl/wget in the image: `python -m messenger.healthcheck`."""
|
||||
"""Docker HEALTHCHECK probe without curl/wget in the image: `python -m messenger.healthcheck`.
|
||||
|
||||
(no arguments) API: GET /health/ready on the local port
|
||||
--file /tmp/heartbeat --max-age 30 relay/worker: the main loop wrote its heartbeat recently
|
||||
(it writes it only while its database/broker work)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
|
||||
def main() -> int:
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) == 4 and argv[0] == "--file" and argv[2] == "--max-age":
|
||||
try:
|
||||
with open(argv[1], encoding="ascii") as f:
|
||||
return 0 if time.time() - float(f.read().strip()) <= float(argv[3]) else 1
|
||||
except (OSError, ValueError):
|
||||
return 1
|
||||
port = os.environ.get("APP_PORT", "8080").strip() or "8080"
|
||||
try:
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/ready", timeout=2) as response:
|
||||
@@ -16,4 +28,4 @@ def main() -> int:
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
"""Outbox relay: `python -m messenger.relay` (compose service `relay`, one process).
|
||||
|
||||
loop: SELECT due outbox rows FOR UPDATE SKIP LOCKED
|
||||
-> publish each job message to RabbitMQ, wait for the publisher CONFIRM
|
||||
-> UPDATE outbox SET published_at -> COMMIT
|
||||
every few seconds: reaper (expired leases -> queued/failed), lost-message guard.
|
||||
|
||||
Why an outbox: the API commits the job and the outbox row in ONE PostgreSQL transaction and never
|
||||
talks to the broker. A crash between COMMIT and publish leaves the row unpublished; the relay
|
||||
publishes it when it (or the broker) is back - nobody re-creates a job by hand. While the broker
|
||||
is down, uploads still get 202 and the rows wait in the outbox.
|
||||
|
||||
A crash after the confirm but before the COMMIT of published_at publishes the row once more after
|
||||
the restart: the queue then holds a duplicate. That is at-least-once delivery; the worker turns
|
||||
the duplicate into a no-op (jobs.claim). The relay never deletes or rewrites jobs.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
|
||||
import aio_pika
|
||||
from aio_pika.abc import AbstractChannel, AbstractConnection, AbstractExchange
|
||||
|
||||
from . import broker, jobs, logging_setup, service
|
||||
from .config import ConfigError, Settings
|
||||
from .db import Database
|
||||
from .errors import Unavailable
|
||||
|
||||
log = logging.getLogger("messenger.relay")
|
||||
ACTOR = "relay"
|
||||
MAINTENANCE_SECONDS = 5.0
|
||||
PURGE_SECONDS = 600.0
|
||||
|
||||
|
||||
class Relay:
|
||||
def __init__(self, settings: Settings):
|
||||
self.settings = settings
|
||||
self.db = Database(settings, name="messenger-relay")
|
||||
self.connection: AbstractConnection | None = None
|
||||
self.channel: AbstractChannel | None = None
|
||||
self.exchange: AbstractExchange | None = None
|
||||
|
||||
async def connect(self) -> None:
|
||||
self.connection = await aio_pika.connect(
|
||||
self.settings.broker_url, timeout=5,
|
||||
client_properties={"connection_name": "relay@" + self.settings.instance_id})
|
||||
# Confirm mode: every publish() waits for basic.ack of the broker (message in the quorum log).
|
||||
self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True)
|
||||
# No declare: the topology belongs to the broker definitions; the relay may only write.
|
||||
self.exchange = await self.channel.get_exchange(broker.EXCHANGE, ensure=False)
|
||||
log.info("broker_connected", extra={"fields": {"exchange": broker.EXCHANGE}})
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
if self.connection is not None:
|
||||
try:
|
||||
await self.connection.close()
|
||||
except Exception: # noqa: BLE001 - already broken
|
||||
pass
|
||||
self.connection = self.channel = self.exchange = None
|
||||
|
||||
async def publish_due(self) -> int:
|
||||
settings = self.settings
|
||||
exchange = self.exchange
|
||||
|
||||
async def work(conn) -> list[jobs.Due]:
|
||||
rows = await jobs.due_outbox(conn, settings.relay_batch)
|
||||
if "reverse_order" in settings.fault_points:
|
||||
rows.reverse()
|
||||
# Rows of jobs that are no longer queued (finished meanwhile) need no message.
|
||||
sendable = [r for r in rows if r.job_state == "queued"]
|
||||
copies = 2 if "duplicate_publish" in settings.fault_points else 1
|
||||
for row in sendable:
|
||||
message = aio_pika.Message(
|
||||
broker.encode(row.job_id, row.attachment_id, row.pipeline_version, row.attempt, row.created_at),
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT, content_type="application/json",
|
||||
message_id=f"{row.job_id}:{row.outbox_id}", type=broker.JOB_TYPE)
|
||||
for _ in range(copies):
|
||||
await exchange.publish(message, routing_key=broker.ROUTING_KEY, mandatory=True, timeout=5)
|
||||
for row in sendable:
|
||||
# Confirmed by the broker, outbox not marked yet: the window "after publish, before
|
||||
# the confirm is recorded". The crash rolls this transaction back.
|
||||
await service.crash_once(settings, self.db, "crash_after_publish", row.job_id, ACTOR,
|
||||
outbox_id=row.outbox_id)
|
||||
await jobs.mark_published(conn, [r.outbox_id for r in rows])
|
||||
return sendable
|
||||
|
||||
sent = await self.db.run(work, deadline=10 + 5 * settings.relay_batch)
|
||||
for row in sent:
|
||||
log.info("job_published", extra={"fields": {"job_id": row.job_id, "attempt": row.attempt,
|
||||
"outbox_id": row.outbox_id}})
|
||||
return len(sent)
|
||||
|
||||
async def maintenance(self) -> None:
|
||||
expired = await self.db.run(lambda c: jobs.expire_leases(c, ACTOR))
|
||||
republished = await self.db.run(lambda c: jobs.republish_stale(c, self.settings.queued_republish_seconds, ACTOR))
|
||||
if expired or republished:
|
||||
log.info("maintenance", extra={"fields": {"leases_expired": expired, "republished": republished}})
|
||||
|
||||
async def run(self) -> int:
|
||||
stop = service.stop_event()
|
||||
await self.db.open()
|
||||
next_maintenance = next_purge = 0.0
|
||||
failures = 0
|
||||
try:
|
||||
while not stop.is_set():
|
||||
healthy = True
|
||||
now = time.monotonic()
|
||||
try:
|
||||
if now >= next_maintenance:
|
||||
await self.maintenance()
|
||||
next_maintenance = now + MAINTENANCE_SECONDS
|
||||
if now >= next_purge:
|
||||
purged = await self.db.run(jobs.purge_outbox)
|
||||
next_purge = now + PURGE_SECONDS
|
||||
if purged:
|
||||
log.info("outbox_purged", extra={"fields": {"rows": purged}})
|
||||
except Unavailable:
|
||||
healthy = False
|
||||
try:
|
||||
if self.exchange is None or self.connection is None or self.connection.is_closed:
|
||||
await self.disconnect()
|
||||
await self.connect()
|
||||
sent = await self.publish_due()
|
||||
failures = 0
|
||||
except Unavailable:
|
||||
healthy, sent = False, 0
|
||||
except (aio_pika.exceptions.AMQPError, ConnectionError, OSError, asyncio.TimeoutError) as exc:
|
||||
# Broker down/unreachable or a message returned/nacked: nothing marked, the rows
|
||||
# stay in the outbox (transaction rolled back) and are published later.
|
||||
failures += 1
|
||||
healthy, sent = False, 0
|
||||
log.warning("publish_failed", extra={"fields": {"error": type(exc).__name__, "failures": failures}})
|
||||
await self.disconnect()
|
||||
if healthy:
|
||||
service.beat()
|
||||
if not sent:
|
||||
await service.sleep_or_stop(stop, min(5.0, self.settings.relay_poll_seconds * (2 ** min(failures, 3))))
|
||||
finally:
|
||||
await self.disconnect()
|
||||
await self.db.close()
|
||||
log.info("relay_stopped")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
settings = Settings.from_env(s3_credentials=False)
|
||||
if not settings.broker_url:
|
||||
raise ConfigError("BROKER_URL_FILE (or BROKER_URL) must be set")
|
||||
except ConfigError as exc:
|
||||
print(f"configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
logging_setup.configure(settings.log_level)
|
||||
log.info("relay_starting", extra={"fields": {"instance_id": settings.instance_id,
|
||||
"fault_points": sorted(settings.fault_points)}})
|
||||
return asyncio.run(Relay(settings).run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -3,14 +3,15 @@
|
||||
Idempotent, safe to run on every `docker compose up`:
|
||||
|
||||
1. waits until the S3 endpoint answers;
|
||||
2. creates the bucket with the ADMIN key pair if it does not exist yet (an existing bucket and
|
||||
its objects are left untouched);
|
||||
3. writes the sentinel object `health/sentinel` that the API's readiness probe HEADs;
|
||||
2. creates the final bucket (S3_BUCKET) and the staging bucket (S3_STAGING_BUCKET) with the ADMIN
|
||||
key pair if they do not exist yet (existing buckets and their objects are left untouched);
|
||||
3. writes the sentinel object `health/sentinel` into the final bucket (the API readiness HEADs it);
|
||||
4. proves the access model and FAILS the stand (exit 1) if it is wrong:
|
||||
* anonymous (unsigned) GET of an object and anonymous LIST of the bucket are refused;
|
||||
* the API key pair can read/write objects of ITS bucket, but can neither list it, nor create
|
||||
a bucket, nor see or read another bucket (a canary bucket created here for this check);
|
||||
* the reconciler key pair can list the bucket but cannot create one.
|
||||
* anonymous (unsigned) GET of an object and anonymous LIST are refused on both buckets;
|
||||
* api: writes staging, reads final; cannot read staging, write final, list or create buckets,
|
||||
nor read another bucket (a canary bucket created here for this check);
|
||||
* worker: reads and writes both buckets; cannot list or create buckets;
|
||||
* auditor: reads and lists both buckets; cannot write.
|
||||
Only file names and verdicts are printed, never key values.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
@@ -57,14 +58,33 @@ def settings_with(keys: tuple[str, str]) -> Settings:
|
||||
return Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024, **base)
|
||||
|
||||
|
||||
def pair(prefix: str) -> tuple[str, str]:
|
||||
return _secret(f"{prefix}ACCESS_KEY_ID", True), _secret(f"{prefix}SECRET_ACCESS_KEY", True)
|
||||
|
||||
|
||||
async def ensure_bucket(admin_s3: Storage, bucket: str) -> bool:
|
||||
code = await error_code(admin_s3.client.head_bucket(Bucket=bucket))
|
||||
if code == "":
|
||||
say(f"bucket {bucket}: exists, kept as is")
|
||||
return True
|
||||
if code in ("404", "NoSuchBucket", "NotFound"):
|
||||
code = await error_code(admin_s3.client.create_bucket(Bucket=bucket))
|
||||
if code not in ("", "BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
|
||||
say(f"bucket {bucket}: create failed ({code})")
|
||||
return False
|
||||
say(f"bucket {bucket}: created")
|
||||
return True
|
||||
say(f"bucket {bucket}: head failed ({code})")
|
||||
return False
|
||||
|
||||
|
||||
async def run() -> int:
|
||||
admin = (_secret("S3_ADMIN_ACCESS_KEY_ID", True), _secret("S3_ADMIN_SECRET_ACCESS_KEY", True))
|
||||
api = (_secret("S3_ACCESS_KEY_ID", True), _secret("S3_SECRET_ACCESS_KEY", True))
|
||||
reconciler = (_secret("S3_RECONCILER_ACCESS_KEY_ID", True), _secret("S3_RECONCILER_SECRET_ACCESS_KEY", True))
|
||||
settings = settings_with(admin)
|
||||
bucket = settings.s3_bucket
|
||||
admin_s3, api_s3, rec_s3 = Storage(settings), Storage(settings_with(api)), Storage(settings_with(reconciler))
|
||||
for s in (admin_s3, api_s3, rec_s3):
|
||||
settings = settings_with(pair("S3_ADMIN_"))
|
||||
final, staging = settings.s3_bucket, settings.s3_staging_bucket
|
||||
admin_s3 = Storage(settings)
|
||||
clients = {"api": Storage(settings_with(pair("S3_"))), "worker": Storage(settings_with(pair("S3_WORKER_"))),
|
||||
"auditor": Storage(settings_with(pair("S3_AUDITOR_")))}
|
||||
for s in (admin_s3, *clients.values()):
|
||||
await s.open()
|
||||
try:
|
||||
deadline = time.monotonic() + 60
|
||||
@@ -77,64 +97,75 @@ async def run() -> int:
|
||||
say("S3 endpoint does not answer within 60 s")
|
||||
return 1
|
||||
await asyncio.sleep(1)
|
||||
|
||||
code = await error_code(admin_s3.client.head_bucket(Bucket=bucket))
|
||||
if code == "":
|
||||
say(f"bucket {bucket}: exists, kept as is")
|
||||
elif code in ("404", "NoSuchBucket", "NotFound"):
|
||||
code = await error_code(admin_s3.client.create_bucket(Bucket=bucket))
|
||||
if code not in ("", "BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
|
||||
say(f"bucket {bucket}: create failed ({code})")
|
||||
for bucket in (final, staging):
|
||||
if not await ensure_bucket(admin_s3, bucket):
|
||||
return 1
|
||||
say(f"bucket {bucket}: created")
|
||||
else:
|
||||
say(f"bucket {bucket}: head failed ({code})")
|
||||
return 1
|
||||
await admin_s3.put(SENTINEL_KEY, b"ok\n", "text/plain", {})
|
||||
await admin_s3.client.put_object(Bucket=staging, Key=SENTINEL_KEY, Body=b"ok\n")
|
||||
say(f"sentinel {SENTINEL_KEY}: written")
|
||||
canary = bucket + "-canary"
|
||||
canary = final + "-canary"
|
||||
if await error_code(admin_s3.client.head_bucket(Bucket=canary)):
|
||||
await admin_s3.client.create_bucket(Bucket=canary)
|
||||
await admin_s3.client.put_object(Bucket=canary, Key="canary", Body=b"secret of another bucket")
|
||||
|
||||
failures = []
|
||||
base = settings.s3_endpoint.rstrip("/")
|
||||
for what, url in (("anonymous GET object", f"{base}/{bucket}/{SENTINEL_KEY}"),
|
||||
("anonymous LIST bucket", f"{base}/{bucket}?list-type=2")):
|
||||
status = anonymous_status(url)
|
||||
verdict = "refused" if status in (401, 403) else "ALLOWED"
|
||||
say(f"{what}: HTTP {status} -> {verdict}")
|
||||
if verdict != "refused":
|
||||
failures.append(what)
|
||||
for bucket in (final, staging):
|
||||
for what, url in ((f"anonymous GET object of {bucket}", f"{base}/{bucket}/{SENTINEL_KEY}"),
|
||||
(f"anonymous LIST of {bucket}", f"{base}/{bucket}?list-type=2")):
|
||||
status = anonymous_status(url)
|
||||
verdict = "refused" if status in (401, 403) else "ALLOWED"
|
||||
say(f"{what}: HTTP {status} -> {verdict}")
|
||||
if verdict != "refused":
|
||||
failures.append(what)
|
||||
|
||||
probe = "health/bootstrap-probe"
|
||||
|
||||
def c(who: str):
|
||||
return clients[who].client
|
||||
|
||||
# (what, coroutine, allowed?) - every refusal must be a real AccessDenied, not an outage.
|
||||
checks = [
|
||||
("api: read object", await error_code(api_s3.client.head_object(Bucket=bucket, Key=SENTINEL_KEY)), True),
|
||||
("api: write object", await error_code(api_s3.client.put_object(Bucket=bucket, Key=probe, Body=b"p")), True),
|
||||
("api: delete object", await error_code(api_s3.client.delete_object(Bucket=bucket, Key=probe)), True),
|
||||
("api: list bucket", await error_code(api_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), False),
|
||||
("api: create bucket", await error_code(api_s3.client.create_bucket(Bucket=bucket + "-x")), False),
|
||||
("api: read object of another bucket",
|
||||
await error_code(api_s3.client.head_object(Bucket=canary, Key="canary")), False),
|
||||
("reconciler: list bucket", await error_code(rec_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), True),
|
||||
("reconciler: create bucket", await error_code(rec_s3.client.create_bucket(Bucket=bucket + "-x")), False),
|
||||
("api: write staging", c("api").put_object(Bucket=staging, Key=probe, Body=b"p"), True),
|
||||
("api: delete staging (janitor)", c("api").delete_object(Bucket=staging, Key=probe), True),
|
||||
("api: read staging", c("api").head_object(Bucket=staging, Key=SENTINEL_KEY), False),
|
||||
("api: read final", c("api").head_object(Bucket=final, Key=SENTINEL_KEY), True),
|
||||
("api: write final", c("api").put_object(Bucket=final, Key=probe, Body=b"p"), False),
|
||||
("api: list final", c("api").list_objects_v2(Bucket=final, MaxKeys=1), False),
|
||||
("api: list staging", c("api").list_objects_v2(Bucket=staging, MaxKeys=1), False),
|
||||
("api: create bucket", c("api").create_bucket(Bucket=final + "-x"), False),
|
||||
("api: read another bucket", c("api").head_object(Bucket=canary, Key="canary"), False),
|
||||
("worker: read staging", c("worker").head_object(Bucket=staging, Key=SENTINEL_KEY), True),
|
||||
("worker: write final", c("worker").put_object(Bucket=final, Key=probe, Body=b"p"), True),
|
||||
("worker: delete final", c("worker").delete_object(Bucket=final, Key=probe), True),
|
||||
("worker: write staging", c("worker").put_object(Bucket=staging, Key=probe, Body=b"p"), True),
|
||||
("worker: delete staging", c("worker").delete_object(Bucket=staging, Key=probe), True),
|
||||
("worker: list final", c("worker").list_objects_v2(Bucket=final, MaxKeys=1), False),
|
||||
("worker: create bucket", c("worker").create_bucket(Bucket=final + "-x"), False),
|
||||
("worker: read another bucket", c("worker").head_object(Bucket=canary, Key="canary"), False),
|
||||
("auditor: list final", c("auditor").list_objects_v2(Bucket=final, MaxKeys=1), True),
|
||||
("auditor: list staging", c("auditor").list_objects_v2(Bucket=staging, MaxKeys=1), True),
|
||||
("auditor: read final", c("auditor").head_object(Bucket=final, Key=SENTINEL_KEY), True),
|
||||
("auditor: write final", c("auditor").put_object(Bucket=final, Key=probe, Body=b"p"), False),
|
||||
("auditor: delete staging", c("auditor").delete_object(Bucket=staging, Key=SENTINEL_KEY), False),
|
||||
]
|
||||
for what, code, allowed in checks:
|
||||
ok = (code == "") == allowed
|
||||
for what, coro, allowed in checks:
|
||||
code = await error_code(coro)
|
||||
ok = (code == "") == allowed and code in ("", "AccessDenied", "403")
|
||||
say(f"{what}: {'allowed' if code == '' else 'refused (' + code + ')'} -> {'OK' if ok else 'WRONG'}")
|
||||
if not ok:
|
||||
failures.append(what)
|
||||
visible = [b["Name"] for b in (await api_s3.client.list_buckets()).get("Buckets", [])]
|
||||
visible = [b["Name"] for b in (await c("api").list_buckets()).get("Buckets", [])]
|
||||
say(f"api: buckets visible in ListBuckets: {visible} -> {'OK' if canary not in visible else 'WRONG'}")
|
||||
if canary in visible:
|
||||
failures.append("api sees another bucket")
|
||||
if failures:
|
||||
say("access model is WRONG: " + ", ".join(failures))
|
||||
return 1
|
||||
say("done: private bucket, least-privilege keys verified")
|
||||
say("done: private buckets, least-privilege keys verified")
|
||||
return 0
|
||||
finally:
|
||||
for s in (admin_s3, api_s3, rec_s3):
|
||||
for s in (admin_s3, *clients.values()):
|
||||
await s.close()
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Common runtime of the background processes (relay, worker): signals, heartbeat file for the
|
||||
Docker healthcheck, failure injection that fires once per job and leaves an audit row."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from . import jobs
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
|
||||
log = logging.getLogger("messenger.service")
|
||||
|
||||
HEARTBEAT_FILE = Path(os.environ.get("HEARTBEAT_FILE", "/tmp/heartbeat"))
|
||||
CRASH_EXIT_CODE = 70 # os._exit: no cleanup, no ack, no COMMIT - like SIGKILL in the middle of work
|
||||
|
||||
|
||||
def beat() -> None:
|
||||
"""Healthcheck input: `python -m messenger.healthcheck --file /tmp/heartbeat --max-age 30`."""
|
||||
try:
|
||||
HEARTBEAT_FILE.write_text(str(time.time()), encoding="ascii")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def stop_event() -> asyncio.Event:
|
||||
stop = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
try:
|
||||
loop.add_signal_handler(sig, stop.set)
|
||||
except (NotImplementedError, RuntimeError):
|
||||
pass # Windows unit tests: no signal handlers
|
||||
return stop
|
||||
|
||||
|
||||
async def sleep_or_stop(stop: asyncio.Event, seconds: float) -> None:
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), seconds)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
|
||||
async def crash_once(settings: Settings, db: Database, point: str, job_id: str, actor: str, **detail) -> None:
|
||||
"""Experiments only (FAULT_POINTS): kill THIS process at `point`, once per job (and per `detail`),
|
||||
across restarts and replicas. The decision is an audit row committed before the crash, so the
|
||||
restarted process (or another replica) continues normally and the evidence stays in job_events."""
|
||||
if point not in settings.fault_points:
|
||||
return
|
||||
|
||||
async def record(conn) -> bool:
|
||||
cur = await conn.execute(
|
||||
"SELECT 1 FROM job_events WHERE job_id = %s AND event = 'fault_injected' AND detail->>'point' = %s "
|
||||
"AND detail @> %s::jsonb", (job_id, point, jobs.json.dumps({k: str(v) for k, v in detail.items()})))
|
||||
if await cur.fetchone():
|
||||
return False
|
||||
await jobs.event(conn, job_id, "fault_injected", actor, point=point, **{k: str(v) for k, v in detail.items()})
|
||||
return True
|
||||
|
||||
if await db.run(record):
|
||||
log.warning("fault_injected_crash", extra={"fields": {"point": point, "job_id": job_id, **detail}})
|
||||
logging.shutdown()
|
||||
os._exit(CRASH_EXIT_CODE)
|
||||
@@ -0,0 +1,334 @@
|
||||
"""Image worker: `python -m messenger.worker` (compose service `worker`, separate from the API).
|
||||
|
||||
One delivery of a job message:
|
||||
|
||||
decode + validate (schema v1; invalid -> reject -> dead-letter queue)
|
||||
-> CLAIM in PostgreSQL (jobs.claim): finished/leased/not due/stale -> no-op, ack
|
||||
-> lease heartbeat every JOB_LEASE_SECONDS/3 while the attempt runs
|
||||
-> read the source (staging copy or published original), check size + SHA-256 against the DB
|
||||
-> variants in a child process (imaging.py: pixel, memory and time limits)
|
||||
-> PUT the original (first publication) and the variants under keys of THIS attempt
|
||||
-> ONE transaction: variants + attachment ready, fenced by the lease token (jobs.complete)
|
||||
-> ack -> delete the staging copy (marked in the DB first, then S3).
|
||||
|
||||
The ack comes only after the durable effect (the COMMIT). A crash anywhere before it makes
|
||||
RabbitMQ deliver the message again; the job row then says what is left to do. Failures: a
|
||||
permanent one (invalid_image, image_too_large, source_missing/corrupt) fails the job at once, a
|
||||
transient one (S3 down, timeout, killed decoder) is retried with backoff via the outbox, at most
|
||||
JOB_MAX_ATTEMPTS claims in total.
|
||||
|
||||
Concurrency: RabbitMQ prefetch = WORKER_CONCURRENCY unacked messages per worker, and at most
|
||||
WORKER_CONCURRENCY image child processes. Graceful stop (SIGTERM): stop consuming, let running
|
||||
attempts finish for WORKER_SHUTDOWN_SECONDS, give the rest back (jobs.release).
|
||||
|
||||
The worker also runs the cleanup of the pipeline objects (staging copies that are no longer
|
||||
needed, objects of attempts that never published) - always decided under a row lock.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
|
||||
import aio_pika
|
||||
from aio_pika.abc import AbstractChannel, AbstractConnection, AbstractIncomingMessage, AbstractQueue
|
||||
|
||||
from . import broker, imaging, jobs, logging_setup, service
|
||||
from .config import ConfigError, Settings
|
||||
from .db import Database
|
||||
from .errors import Unavailable
|
||||
from .imaging import ProcessingError
|
||||
from .storage import ObjectMissing, Storage
|
||||
|
||||
log = logging.getLogger("messenger.worker")
|
||||
|
||||
|
||||
class Worker:
|
||||
def __init__(self, settings: Settings):
|
||||
self.settings = settings
|
||||
self.name = settings.instance_id
|
||||
self.actor = "worker:" + self.name
|
||||
self.db = Database(settings, name="messenger-worker")
|
||||
self.staging = Storage(settings, bucket=settings.s3_staging_bucket)
|
||||
self.final = Storage(settings)
|
||||
self.slots = asyncio.Semaphore(settings.worker_concurrency)
|
||||
self.tasks: set[asyncio.Task] = set()
|
||||
self.connection: AbstractConnection | None = None
|
||||
self.channel: AbstractChannel | None = None
|
||||
self.queue: AbstractQueue | None = None
|
||||
self.consumer_tag: str | None = None
|
||||
|
||||
# ---------------------------------------------------------------- broker
|
||||
async def connect(self) -> None:
|
||||
self.connection = await aio_pika.connect(
|
||||
self.settings.broker_url, timeout=5,
|
||||
client_properties={"connection_name": "worker@" + self.name})
|
||||
self.channel = await self.connection.channel()
|
||||
# Prefetch: the broker keeps at most this many unacked messages in flight to this worker.
|
||||
await self.channel.set_qos(prefetch_count=self.settings.worker_concurrency)
|
||||
self.queue = await self.channel.get_queue(broker.QUEUE, ensure=False)
|
||||
self.consumer_tag = await self.queue.consume(self.on_message, no_ack=False)
|
||||
log.info("consuming", extra={"fields": {"queue": broker.QUEUE, "prefetch": self.settings.worker_concurrency}})
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
if self.connection is not None:
|
||||
try:
|
||||
await self.connection.close()
|
||||
except Exception: # noqa: BLE001 - already broken
|
||||
pass
|
||||
self.connection = self.channel = self.queue = None
|
||||
self.consumer_tag = None
|
||||
|
||||
async def on_message(self, message: AbstractIncomingMessage) -> None:
|
||||
task = asyncio.create_task(self.deliver(message))
|
||||
self.tasks.add(task)
|
||||
task.add_done_callback(self.tasks.discard)
|
||||
|
||||
@staticmethod
|
||||
async def settle(action) -> None:
|
||||
"""ack/nack/reject; the channel may be gone (broker restart): then the broker requeues the
|
||||
message by itself and the redelivery is a no-op."""
|
||||
try:
|
||||
await action
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning("settle_failed", extra={"fields": {"error": type(exc).__name__}})
|
||||
|
||||
async def deliver(self, message: AbstractIncomingMessage) -> None:
|
||||
try:
|
||||
body = broker.decode(message.body)
|
||||
except broker.BadMessage as exc:
|
||||
log.error("bad_message", extra={"fields": {"reason": str(exc), "message_id": message.message_id}})
|
||||
await self.settle(message.reject(requeue=False)) # -> messenger.attachment-jobs.dlq
|
||||
return
|
||||
try:
|
||||
async with self.slots:
|
||||
outcome = await self.process(body["job_id"])
|
||||
except Unavailable:
|
||||
# PostgreSQL (or S3 during a DB step) unavailable: nothing decided. Back to the queue
|
||||
# after a pause; a claim that did happen is protected by its lease.
|
||||
log.warning("job_deferred", extra={"fields": {"job_id": body["job_id"]}})
|
||||
await asyncio.sleep(2)
|
||||
await self.settle(message.nack(requeue=True))
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
# Shutdown: back to the queue. If the job was claimed it has been released (new outbox
|
||||
# row), and one of the two deliveries becomes a no-op.
|
||||
await self.settle(asyncio.shield(message.nack(requeue=True)))
|
||||
raise
|
||||
log.info("job_done", extra={"fields": {"job_id": body["job_id"], "outcome": outcome,
|
||||
"message_attempt": body["attempt"], "redelivered": message.redelivered}})
|
||||
await self.settle(message.ack())
|
||||
|
||||
# ---------------------------------------------------------------- one attempt
|
||||
async def process(self, job_id: str) -> str:
|
||||
settings = self.settings
|
||||
claim = await self.db.run(lambda c: jobs.claim(c, job_id, self.name, settings.job_lease_seconds))
|
||||
if isinstance(claim, jobs.Noop):
|
||||
return "noop_" + claim.reason
|
||||
await service.crash_once(settings, self.db, "crash_after_claim", job_id, self.actor)
|
||||
lost = asyncio.Event()
|
||||
work = asyncio.create_task(self.attempt(claim))
|
||||
beat = asyncio.create_task(self.heartbeat(claim, work, lost))
|
||||
try:
|
||||
variants = await work
|
||||
except asyncio.CancelledError:
|
||||
if lost.is_set() and not asyncio.current_task().cancelling():
|
||||
return "fenced" # lease taken over: this attempt may write nothing more
|
||||
outcome = await asyncio.shield(self.db.run(lambda c: jobs.release(c, claim, self.name)))
|
||||
log.info("job_released", extra={"fields": {"job_id": job_id, "outcome": outcome}})
|
||||
raise
|
||||
except ProcessingError as exc:
|
||||
return await self.failed(claim, exc.code, exc.permanent)
|
||||
except Unavailable:
|
||||
return await self.failed(claim, "storage_unavailable", False)
|
||||
finally:
|
||||
beat.cancel()
|
||||
await service.crash_once(settings, self.db, "crash_after_s3_write", job_id, self.actor)
|
||||
outcome = await self.db.run(lambda c: jobs.complete(c, claim, variants, self.name))
|
||||
await service.crash_once(settings, self.db, "crash_after_commit", job_id, self.actor)
|
||||
if outcome == "succeeded":
|
||||
await self.delete_staging(claim.attachment_id)
|
||||
return outcome
|
||||
|
||||
async def failed(self, claim: jobs.Claim, code: str, permanent: bool) -> str:
|
||||
s = self.settings
|
||||
outcome, delay = await self.db.run(
|
||||
lambda c: jobs.retry_or_fail(c, claim, code, permanent, self.name, s.job_retry_base_seconds,
|
||||
s.job_retry_max_seconds))
|
||||
log.info("attempt_failed", extra={"fields": {"job_id": claim.job_id, "attempt": claim.attempt, "code": code,
|
||||
"permanent": permanent, "outcome": outcome,
|
||||
"delay_s": round(delay, 3)}})
|
||||
return outcome
|
||||
|
||||
async def heartbeat(self, claim: jobs.Claim, work: asyncio.Task, lost: asyncio.Event) -> None:
|
||||
"""Renew the lease while the attempt is alive and within JOB_TIMEOUT_SECONDS. A hung attempt
|
||||
stops being renewed, so the reaper can hand the job to another worker; if the lease turns out
|
||||
to be lost, the attempt is cancelled (its result would be fenced anyway)."""
|
||||
s = self.settings
|
||||
if "hang" in s.fault_points:
|
||||
return # experiment: a hung worker that no longer heartbeats
|
||||
started = time.monotonic()
|
||||
while True:
|
||||
await asyncio.sleep(s.job_lease_seconds / 3)
|
||||
if time.monotonic() - started > s.job_timeout_seconds:
|
||||
return
|
||||
try:
|
||||
alive = await self.db.run(lambda c: jobs.heartbeat(c, claim, s.job_lease_seconds))
|
||||
except Unavailable:
|
||||
continue # the lease may expire meanwhile; the fence protects the result
|
||||
if not alive:
|
||||
log.warning("lease_lost", extra={"fields": {"job_id": claim.job_id, "attempt": claim.attempt}})
|
||||
lost.set()
|
||||
work.cancel()
|
||||
return
|
||||
|
||||
async def attempt(self, claim: jobs.Claim) -> list[imaging.Rendered]:
|
||||
s = self.settings
|
||||
if "hang" in s.fault_points:
|
||||
await asyncio.sleep(s.fault_value("hang", 3600)) # stuck outside of the time limit
|
||||
try:
|
||||
return await asyncio.wait_for(self.build(claim), s.job_timeout_seconds)
|
||||
except asyncio.TimeoutError:
|
||||
raise ProcessingError("processing_timeout", permanent=False) from None
|
||||
|
||||
async def build(self, claim: jobs.Claim) -> list[imaging.Rendered]:
|
||||
s = self.settings
|
||||
started = time.monotonic()
|
||||
if claim.size_bytes > s.max_upload_bytes:
|
||||
raise ProcessingError("image_too_large", permanent=True)
|
||||
source = self.staging if claim.source == "staging" else self.final
|
||||
try:
|
||||
data = await source.read_all(claim.source_key, limit=s.max_upload_bytes)
|
||||
except ObjectMissing:
|
||||
raise ProcessingError("source_missing", permanent=True) from None
|
||||
if len(data) != claim.size_bytes or hashlib.sha256(data).digest() != claim.sha256:
|
||||
raise ProcessingError("source_corrupt", permanent=True)
|
||||
self.injected(claim)
|
||||
if "slow" in s.fault_points and s.fault_value("slow_pipeline", claim.pipeline_version) == claim.pipeline_version:
|
||||
await asyncio.sleep(s.fault_value("slow", 10))
|
||||
remaining = max(1.0, s.job_timeout_seconds - (time.monotonic() - started))
|
||||
variants = await imaging.render(data, claim.content_type, claim.pipeline_version,
|
||||
max_pixels=s.max_image_pixels, timeout=remaining,
|
||||
memory_mb=s.image_max_memory_mb)
|
||||
meta = {"attachment-id": claim.attachment_id, "pipeline-version": str(claim.pipeline_version)}
|
||||
puts = [self.final.put(claim.key(v.name), v.data, v.content_type, meta) for v in variants]
|
||||
if claim.write_original:
|
||||
puts.append(self.final.put(claim.key("original"), data, claim.content_type,
|
||||
{**meta, "sha256": claim.sha256.hex()}))
|
||||
await asyncio.gather(*puts)
|
||||
return variants
|
||||
|
||||
def injected(self, claim: jobs.Claim) -> None:
|
||||
s = self.settings
|
||||
if "permanent_error" in s.fault_points:
|
||||
raise ProcessingError("invalid_image", permanent=True)
|
||||
if "transient_error" in s.fault_points and claim.attempt <= s.fault_value("transient_error", 1000):
|
||||
raise ProcessingError("injected_transient", permanent=False)
|
||||
|
||||
# ---------------------------------------------------------------- cleanup
|
||||
async def delete_staging(self, attachment_id: str) -> None:
|
||||
"""Mark `deleting` under the row lock (re-checked there), then S3 DELETE, then `deleted`."""
|
||||
try:
|
||||
key = await self.db.run(
|
||||
lambda c: jobs.mark_staging_deleting(c, attachment_id, self.settings.staging_retention_seconds))
|
||||
if key is None:
|
||||
return
|
||||
await self.staging.delete(key)
|
||||
await self.db.run(lambda c: jobs.mark_staging_deleted(c, attachment_id))
|
||||
log.info("staging_deleted", extra={"fields": {"attachment_id": attachment_id}})
|
||||
except Unavailable:
|
||||
pass # stays present/deleting; the next cleanup pass repeats (DELETE is idempotent)
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
s = self.settings
|
||||
ids = await self.db.run(lambda c: jobs.staging_to_delete(c, s.staging_retention_seconds))
|
||||
tokens = await self.db.run(lambda c: jobs.attempts_to_clean(c, s.orphan_grace_seconds))
|
||||
if "pause_cleanup" in s.fault_points and (ids or tokens):
|
||||
# Experiment: the race "cleanup read its candidates -> replay commits -> cleanup goes on".
|
||||
log.warning("cleanup_paused", extra={"fields": {"attachments": ids, "attempts": tokens}})
|
||||
await asyncio.sleep(s.fault_value("pause_cleanup", 10))
|
||||
for attachment_id in ids:
|
||||
await self.delete_staging(attachment_id)
|
||||
for token in tokens:
|
||||
keys = await self.db.run(lambda c: jobs.lock_attempt_for_cleanup(c, token))
|
||||
if keys is None:
|
||||
continue
|
||||
await asyncio.gather(*(self.final.delete(k) for k in keys))
|
||||
await self.db.run(lambda c: jobs.mark_attempt_cleaned(c, token))
|
||||
log.info("attempt_objects_deleted", extra={"fields": {"token": token, "objects": len(keys)}})
|
||||
|
||||
async def cleanup_loop(self, stop: asyncio.Event) -> None:
|
||||
while not stop.is_set() and self.settings.cleanup_interval_seconds > 0:
|
||||
await service.sleep_or_stop(stop, self.settings.cleanup_interval_seconds)
|
||||
try:
|
||||
await self.cleanup()
|
||||
except Unavailable:
|
||||
pass
|
||||
|
||||
# ---------------------------------------------------------------- lifecycle
|
||||
async def run(self) -> int:
|
||||
stop = service.stop_event()
|
||||
await self.db.open()
|
||||
await self.staging.open()
|
||||
await self.final.open()
|
||||
cleaner = asyncio.create_task(self.cleanup_loop(stop))
|
||||
try:
|
||||
while not stop.is_set():
|
||||
if self.connection is None or self.connection.is_closed:
|
||||
await self.disconnect()
|
||||
try:
|
||||
await self.connect()
|
||||
except (aio_pika.exceptions.AMQPError, ConnectionError, OSError, asyncio.TimeoutError) as exc:
|
||||
log.warning("broker_unavailable", extra={"fields": {"error": type(exc).__name__}})
|
||||
await self.disconnect()
|
||||
if self.connection is not None and not self.connection.is_closed:
|
||||
service.beat()
|
||||
await service.sleep_or_stop(stop, 2.0)
|
||||
await self.shutdown()
|
||||
finally:
|
||||
cleaner.cancel()
|
||||
await self.disconnect()
|
||||
await self.final.close()
|
||||
await self.staging.close()
|
||||
await self.db.close()
|
||||
log.info("worker_stopped")
|
||||
return 0
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
"""Stop consuming; running attempts get WORKER_SHUTDOWN_SECONDS, the rest is released."""
|
||||
if self.queue is not None and self.consumer_tag is not None:
|
||||
try:
|
||||
await self.queue.cancel(self.consumer_tag)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
pending = set(self.tasks)
|
||||
log.info("worker_draining", extra={"fields": {"in_flight": len(pending)}})
|
||||
if pending:
|
||||
_, pending = await asyncio.wait(pending, timeout=self.settings.worker_shutdown_seconds)
|
||||
for task in pending:
|
||||
task.cancel()
|
||||
if pending:
|
||||
await asyncio.wait(pending, timeout=10)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
settings = Settings.from_env()
|
||||
if not settings.broker_url:
|
||||
raise ConfigError("BROKER_URL_FILE (or BROKER_URL) must be set")
|
||||
except ConfigError as exc:
|
||||
print(f"configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
logging_setup.configure(settings.log_level)
|
||||
log.info("worker_starting", extra={"fields": {
|
||||
"instance_id": settings.instance_id, "concurrency": settings.worker_concurrency,
|
||||
"max_attempts": settings.job_max_attempts, "timeout_s": settings.job_timeout_seconds,
|
||||
"lease_s": settings.job_lease_seconds, "memory_mb": settings.image_max_memory_mb,
|
||||
"fault_points": sorted(settings.fault_points)}})
|
||||
return asyncio.run(Worker(settings).run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user