forked from mtusi2026/mtusi2026lab5
feat(lab5): durable async attachment pipeline through grade 4
- RabbitMQ quorum queue, persistent messages and publisher confirms - S3 staging plus PostgreSQL attachments/jobs outbox transaction - worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper - jittered retry, permanent failures, explicit replay and admission cap - deterministic crash hooks, resilience script and evidence for grade 4 - async-aware custom tests, oversized pixel case, docs and changelog
This commit is contained in:
@@ -24,6 +24,11 @@ x-api-environment: &api-environment
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
IMAGE_DECODE_MEMORY_MB: ${IMAGE_DECODE_MEMORY_MB:-256}
|
||||
UPLOAD_TEMP_DIR: /tmp/messenger-uploads
|
||||
# Lab5 async pipeline. The API publishes outbox rows; the worker consumes.
|
||||
BROKER_URL: "amqp://${RABBITMQ_USER:-messenger}:${RABBITMQ_PASSWORD:?set RABBITMQ_PASSWORD in .env}@rabbitmq:5672/"
|
||||
JOB_QUEUE: ${JOB_QUEUE:-attachment.process.v1}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-1000}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
@@ -74,6 +79,29 @@ services:
|
||||
cpus: "${REDIS_CPU_LIMIT:-0.50}"
|
||||
memory: "${REDIS_MEM_LIMIT:-256M}"
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:4.2-management-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-messenger}
|
||||
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?set RABBITMQ_PASSWORD in .env}
|
||||
volumes:
|
||||
- rabbitmqdata:/var/lib/rabbitmq
|
||||
ports:
|
||||
# Management UI is only for the local learning stand.
|
||||
- "127.0.0.1:${RABBITMQ_MGMT_PORT:-15672}:15672"
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 3s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${RABBITMQ_CPU_LIMIT:-1.0}"
|
||||
memory: "${RABBITMQ_MEM_LIMIT:-512M}"
|
||||
|
||||
rustfs:
|
||||
image: rustfs/rustfs:1.0.4
|
||||
restart: unless-stopped
|
||||
@@ -155,6 +183,8 @@ services:
|
||||
condition: service_healthy
|
||||
rustfs:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
@@ -179,6 +209,53 @@ services:
|
||||
cpus: "${API_CPU_LIMIT:-0.50}"
|
||||
memory: "${API_MEM_LIMIT:-512M}"
|
||||
|
||||
worker:
|
||||
<<: *api-build
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
rustfs:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
<<: *api-environment
|
||||
RUN_MIGRATIONS: "false"
|
||||
WORKER_ONLY: "true"
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
|
||||
JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-2}
|
||||
JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:-30}
|
||||
LEASE_SWEEP_SECONDS: ${LEASE_SWEEP_SECONDS:-15}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
WORK_TEMP_DIR: /tmp/messenger-work
|
||||
# Test hooks, off by default. They make the grade-4 failure windows
|
||||
# deterministic; see evidence/ and REPORT.md.
|
||||
WORKER_FAULT_MODE: ${WORKER_FAULT_MODE:-none}
|
||||
WORKER_CRASH_AFTER_EFFECT: ${WORKER_CRASH_AFTER_EFFECT:-false}
|
||||
WORKER_CRASH_AFTER_CLAIM: ${WORKER_CRASH_AFTER_CLAIM:-false}
|
||||
entrypoint: ["/usr/local/bin/worker"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pgrep -f /usr/local/bin/worker >/dev/null"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 10
|
||||
start_period: 5s
|
||||
stop_grace_period: 20s
|
||||
deploy:
|
||||
replicas: ${WORKER_REPLICAS:-1}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${WORKER_CPU_LIMIT:-1.0}"
|
||||
memory: "${WORKER_MEM_LIMIT:-1G}"
|
||||
|
||||
# AGENT: the reconciler uses the same scoped credential as the API. It never
|
||||
# touches ready attachments; it only removes stale pending rows/objects and
|
||||
# objects with no metadata row at all.
|
||||
@@ -256,3 +333,4 @@ volumes:
|
||||
pgdata:
|
||||
redisdata:
|
||||
rustfsdata:
|
||||
rabbitmqdata:
|
||||
|
||||
Reference in New Issue
Block a user