feat(lab5): durable async attachment pipeline through grade 4

- RabbitMQ quorum queue, persistent messages and publisher confirms
- S3 staging plus PostgreSQL attachments/jobs outbox transaction
- worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper
- jittered retry, permanent failures, explicit replay and admission cap
- deterministic crash hooks, resilience script and evidence for grade 4
- async-aware custom tests, oversized pixel case, docs and changelog
This commit is contained in:
Mikhail Verkovykh
2026-11-15 00:30:36 +03:00
parent 28d68dc30b
commit 9e43faa536
48 changed files with 4077 additions and 333 deletions
+78
View File
@@ -24,6 +24,11 @@ x-api-environment: &api-environment
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
IMAGE_DECODE_MEMORY_MB: ${IMAGE_DECODE_MEMORY_MB:-256}
UPLOAD_TEMP_DIR: /tmp/messenger-uploads
# Lab5 async pipeline. The API publishes outbox rows; the worker consumes.
BROKER_URL: "amqp://${RABBITMQ_USER:-messenger}:${RABBITMQ_PASSWORD:?set RABBITMQ_PASSWORD in .env}@rabbitmq:5672/"
JOB_QUEUE: ${JOB_QUEUE:-attachment.process.v1}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-1000}
services:
postgres:
@@ -74,6 +79,29 @@ services:
cpus: "${REDIS_CPU_LIMIT:-0.50}"
memory: "${REDIS_MEM_LIMIT:-256M}"
rabbitmq:
image: rabbitmq:4.2-management-alpine
restart: unless-stopped
environment:
RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-messenger}
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?set RABBITMQ_PASSWORD in .env}
volumes:
- rabbitmqdata:/var/lib/rabbitmq
ports:
# Management UI is only for the local learning stand.
- "127.0.0.1:${RABBITMQ_MGMT_PORT:-15672}:15672"
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 3s
timeout: 5s
retries: 20
start_period: 10s
deploy:
resources:
limits:
cpus: "${RABBITMQ_CPU_LIMIT:-1.0}"
memory: "${RABBITMQ_MEM_LIMIT:-512M}"
rustfs:
image: rustfs/rustfs:1.0.4
restart: unless-stopped
@@ -155,6 +183,8 @@ services:
condition: service_healthy
rustfs:
condition: service_healthy
rabbitmq:
condition: service_healthy
s3-init:
condition: service_completed_successfully
migrate:
@@ -179,6 +209,53 @@ services:
cpus: "${API_CPU_LIMIT:-0.50}"
memory: "${API_MEM_LIMIT:-512M}"
worker:
<<: *api-build
restart: unless-stopped
depends_on:
postgres:
condition: service_healthy
rustfs:
condition: service_healthy
rabbitmq:
condition: service_healthy
s3-init:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
environment:
<<: *api-environment
RUN_MIGRATIONS: "false"
WORKER_ONLY: "true"
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-2}
JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:-30}
LEASE_SWEEP_SECONDS: ${LEASE_SWEEP_SECONDS:-15}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
WORK_TEMP_DIR: /tmp/messenger-work
# Test hooks, off by default. They make the grade-4 failure windows
# deterministic; see evidence/ and REPORT.md.
WORKER_FAULT_MODE: ${WORKER_FAULT_MODE:-none}
WORKER_CRASH_AFTER_EFFECT: ${WORKER_CRASH_AFTER_EFFECT:-false}
WORKER_CRASH_AFTER_CLAIM: ${WORKER_CRASH_AFTER_CLAIM:-false}
entrypoint: ["/usr/local/bin/worker"]
healthcheck:
test: ["CMD-SHELL", "pgrep -f /usr/local/bin/worker >/dev/null"]
interval: 3s
timeout: 2s
retries: 10
start_period: 5s
stop_grace_period: 20s
deploy:
replicas: ${WORKER_REPLICAS:-1}
resources:
limits:
cpus: "${WORKER_CPU_LIMIT:-1.0}"
memory: "${WORKER_MEM_LIMIT:-1G}"
# AGENT: the reconciler uses the same scoped credential as the API. It never
# touches ready attachments; it only removes stale pending rows/objects and
# objects with no metadata row at all.
@@ -256,3 +333,4 @@ volumes:
pgdata:
redisdata:
rustfsdata:
rabbitmqdata: