Run the broker, the relay and the worker in the stand

Embed the checked lab 5 preparation: RabbitMQ 4.1.8 pinned by digest
with a fixed node hostname, a volume, a read-only root and 512 MiB,
whose topology (vhost messenger, exchange attachments, quorum queue
attachments.process with a dead letter exchange, delivery limit 10 and
reject-publish overflow) and users come from definitions imported on
every boot. The relay may only write to the exchange, the worker may
only read the queue, configure belongs to the operator, there is no
guest, and the management UI is published on 127.0.0.1 only. The relay
and the worker run from the application image with a heartbeat health
check, init, a read-only root and their own limits; the worker gets its
own S3 user that reads staging and writes originals and variants, and
s3-init creates it idempotently. The API has no broker credentials.

make secrets also creates the broker passwords, URLs, hashed users and
the worker S3 keys; the restart hook now stops the balancer, the API,
the relay and the worker and restarts PostgreSQL, Valkey, S3 and the
broker with their volumes; worker.sh and broker.sh stop, start and
inspect the processes for the failure demonstrations; the limits
policy covers the new services and the loopback UI port.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Artyom
2026-10-29 01:19:04 +03:00
co-authored by Claude Opus 5.5
parent 34741dc2d2
commit c55fb1b998
15 changed files with 374 additions and 13 deletions
+10
View File
@@ -55,3 +55,13 @@ UPLOAD_SWEEP_INTERVAL_SECONDS=30
UPLOAD_SWEEP_BATCH=100
# Fixed glibc mmap threshold for the API: buffers above 128 KiB are returned to the system on free.
MALLOC_MMAP_THRESHOLD_=131072
# Solution-specific for lab5: RabbitMQ management UI on 127.0.0.1, backoff base of a transient
# retry (doubles per attempt up to 60 s), relay poll and sweep intervals, worker limits and the
# PostgreSQL pool of relay and worker. FAULT_POINT is set only by infra/compose.faults.yaml.
BROKER_MANAGEMENT_PORT=15672
JOB_BACKOFF_BASE_SECONDS=2
OUTBOX_POLL_SECONDS=1
SWEEP_INTERVAL_SECONDS=5
WORKER_CPUS=2
WORKER_MEMORY=768m
PIPELINE_DB_POOL_MAX_SIZE=4
+1
View File
@@ -72,6 +72,7 @@ student-test:
secrets:
sh scripts/secrets.sh
sh scripts/broker-secrets.sh
up: secrets
$(COMPOSE) up -d --build --wait
+122 -1
View File
@@ -18,6 +18,44 @@ x-s3-env: &s3-env
UPLOAD_SWEEP_INTERVAL_SECONDS: ${UPLOAD_SWEEP_INTERVAL_SECONDS:-30}
UPLOAD_SWEEP_BATCH: ${UPLOAD_SWEEP_BATCH:-100}
x-pipeline: &pipeline
image: ${API_IMAGE:-mtusi-messenger:lab5}
pull_policy: never
restart: unless-stopped
init: true
read_only: true
tmpfs: [/tmp]
security_opt: ["no-new-privileges:true"]
cap_drop: [ALL]
stop_grace_period: 10s
networks: [backend]
healthcheck:
test: ["CMD", "python", "-c", "import os,sys,time;sys.exit(time.time()-os.stat('/tmp/heartbeat').st_mtime>30)"]
interval: 10s
timeout: 5s
retries: 3
start_period: 30s
start_interval: 2s
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
broker:
condition: service_healthy
x-pipeline-env: &pipeline-env
<<: *s3-env
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
DB_POOL_MAX_SIZE: ${PIPELINE_DB_POOL_MAX_SIZE:-4}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
HEARTBEAT_FILE: /tmp/heartbeat
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
x-s3-client: &s3-client
image: rustfs/rc:v0.1.36@sha256:ab024bfebee49a750ce886b4c70963ccd9ddaa03f491704a90710641d7a26699
read_only: true
@@ -27,7 +65,9 @@ x-s3-client: &s3-client
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
RC_CONFIG_DIR: /tmp/rc
volumes: ["./infra/s3:/s3:ro"]
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
secrets:
[s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key,
s3_worker_access_key_id, s3_worker_secret_access_key]
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
restart: "no"
@@ -234,6 +274,74 @@ services:
profiles: [tools]
entrypoint: ["/bin/sh", "/s3/cli.sh"]
broker:
image: rabbitmq:4.1.8-management-alpine@sha256:54ef5ff505352c1beef05a97ee9bd6ab95b3559a76629ba0119b1d63d8d6e4b2
hostname: broker
user: "100:101"
restart: unless-stopped
read_only: true
tmpfs: [/tmp]
security_opt: ["no-new-privileges:true"]
cap_drop: [ALL]
ports:
- "127.0.0.1:${BROKER_MANAGEMENT_PORT:-15672}:15672"
networks: [backend, edge]
volumes:
- broker-data:/var/lib/rabbitmq
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/conf.d/20-messenger.conf:ro
- ./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro
- ./infra/rabbitmq/definitions.json:/etc/rabbitmq/definitions.d/10-topology.json:ro
secrets:
- source: rabbitmq_users
target: /etc/rabbitmq/definitions.d/20-users.json
- rabbitmq_ops_password
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
interval: 10s
timeout: 10s
retries: 5
start_period: 60s
start_interval: 2s
stop_grace_period: 10s
deploy:
resources:
limits: {cpus: "1", memory: 512m}
relay:
<<: *pipeline
command: ["python", "-m", "messenger.relay"]
environment:
<<: *pipeline-env
OUTBOX_POLL_SECONDS: ${OUTBOX_POLL_SECONDS:-1}
SWEEP_INTERVAL_SECONDS: ${SWEEP_INTERVAL_SECONDS:-5}
secrets:
- database_url
- source: broker_url_api
target: broker_url
- s3_access_key_id
- s3_secret_access_key
deploy:
resources:
limits: {cpus: "0.5", memory: 192m}
worker:
<<: *pipeline
command: ["python", "-m", "messenger.worker"]
environment:
<<: *pipeline-env
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
secrets:
- database_url
- source: broker_url_worker
target: broker_url
- source: s3_worker_access_key_id
target: s3_access_key_id
- source: s3_worker_secret_access_key
target: s3_secret_access_key
deploy:
resources:
limits: {cpus: "${WORKER_CPUS:-2}", memory: "${WORKER_MEMORY:-768m}"}
k6:
image: grafana/k6:2.3.0@sha256:2dd4756ec6e14e62778916e1eb850108e4b499304c02df56760fa0bdb0fba0b8
profiles: [bench]
@@ -264,11 +372,24 @@ secrets:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
s3_worker_access_key_id:
file: ./secrets/s3_worker_access_key_id
s3_worker_secret_access_key:
file: ./secrets/s3_worker_secret_access_key
rabbitmq_users:
file: ./secrets/rabbitmq_users.json
rabbitmq_ops_password:
file: ./secrets/rabbitmq_ops_password
broker_url_api:
file: ./secrets/broker_url_api
broker_url_worker:
file: ./secrets/broker_url_worker
volumes:
pgdata:
valkeydata:
s3data:
broker-data:
networks:
edge:
+15
View File
@@ -0,0 +1,15 @@
services:
relay:
restart: "no"
environment:
FAULT_POINT: ${RELAY_FAULT_POINT:-}
OUTBOX_POLL_SECONDS: "0.5"
SWEEP_INTERVAL_SECONDS: "1"
worker:
restart: "no"
environment:
FAULT_POINT: ${WORKER_FAULT_POINT:-}
FAULT_STALL_SECONDS: ${FAULT_STALL_SECONDS:-15}
JOB_TIMEOUT_SECONDS: ${FAULT_JOB_TIMEOUT_SECONDS:-5}
JOB_BACKOFF_BASE_SECONDS: ${FAULT_BACKOFF_BASE_SECONDS:-1}
+44
View File
@@ -0,0 +1,44 @@
{
"rabbit_version": "4.1.8",
"vhosts": [
{"name": "messenger", "description": "lab5 attachment pipeline", "default_queue_type": "quorum"}
],
"exchanges": [
{"name": "attachments", "vhost": "messenger", "type": "direct", "durable": true, "auto_delete": false, "internal": false, "arguments": {}},
{"name": "attachments.dlx", "vhost": "messenger", "type": "fanout", "durable": true, "auto_delete": false, "internal": false, "arguments": {}}
],
"queues": [
{"name": "attachments.process", "vhost": "messenger", "durable": true, "auto_delete": false, "arguments": {"x-queue-type": "quorum"}},
{"name": "attachments.process.dlq", "vhost": "messenger", "durable": true, "auto_delete": false, "arguments": {"x-queue-type": "quorum"}}
],
"bindings": [
{"source": "attachments", "vhost": "messenger", "destination": "attachments.process", "destination_type": "queue", "routing_key": "attachment.process.v1", "arguments": {}},
{"source": "attachments.dlx", "vhost": "messenger", "destination": "attachments.process.dlq", "destination_type": "queue", "routing_key": "", "arguments": {}}
],
"policies": [
{
"vhost": "messenger",
"name": "attachments-process",
"pattern": "^attachments\\.process$",
"apply-to": "queues",
"priority": 10,
"definition": {
"dead-letter-exchange": "attachments.dlx",
"dead-letter-strategy": "at-least-once",
"overflow": "reject-publish",
"delivery-limit": 10
}
},
{
"vhost": "messenger",
"name": "attachments-dlq",
"pattern": "^attachments\\.process\\.dlq$",
"apply-to": "queues",
"priority": 10,
"definition": {
"max-length": 100000,
"overflow": "reject-publish"
}
}
]
}
+1
View File
@@ -0,0 +1 @@
[rabbitmq_management,rabbitmq_prometheus].
+21
View File
@@ -0,0 +1,21 @@
listeners.tcp.default = 5672
management.tcp.port = 15672
prometheus.tcp.port = 15692
loopback_users.guest = true
log.console = true
log.console.level = info
log.file = false
definitions.import_backend = local_filesystem
definitions.local.path = /etc/rabbitmq/definitions.d
vm_memory_high_watermark.absolute = 256MiB
disk_free_limit.absolute = 256MB
raft.wal_max_size_bytes = 64000000
consumer_timeout = 300000
heartbeat = 30
deprecated_features.permit.transient_nonexcl_queues = false
deprecated_features.permit.global_qos = false
+7 -1
View File
@@ -14,7 +14,13 @@ sed "s/messenger-attachments/$bucket/g" /s3/api-policy.json > /tmp/api-policy.js
rc admin policy create root "$policy" /tmp/api-policy.json >/dev/null
rc admin user add root "$api_key" "$(secret s3_secret_access_key)" >/dev/null
rc admin policy attach root "$policy" --user "$api_key" >/dev/null
worker_key=$(secret s3_worker_access_key_id)
sed "s/messenger-attachments/$bucket/g" /s3/worker-policy.json > /tmp/worker-policy.json
rc admin policy create root messenger-worker /tmp/worker-policy.json >/dev/null
rc admin user add root "$worker_key" "$(secret s3_worker_secret_access_key)" >/dev/null
rc admin policy attach root messenger-worker --user "$worker_key" >/dev/null
rc bucket lifecycle rule import "root/$bucket" /s3/lifecycle.json >/dev/null
rc bucket anonymous get "root/$bucket" --json | grep -q '"permission": *"private"'
rc admin user info root "$api_key" --json | grep -q "\"$policy\""
echo "s3-init: bucket=$bucket anonymous=private policy=$policy lifecycle=abort-incomplete-uploads"
rc admin user info root "$worker_key" --json | grep -q '"messenger-worker"'
echo "s3-init: bucket=$bucket anonymous=private policy=$policy worker=messenger-worker lifecycle=abort-incomplete-uploads"
+24
View File
@@ -0,0 +1,24 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "MessengerWorkerRead",
"Effect": "Allow",
"Action": ["s3:GetObject"],
"Resource": [
"arn:aws:s3:::messenger-attachments/staging/*",
"arn:aws:s3:::messenger-attachments/originals/*",
"arn:aws:s3:::messenger-attachments/variants/*"
]
},
{
"Sid": "MessengerWorkerWrite",
"Effect": "Allow",
"Action": ["s3:PutObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts"],
"Resource": [
"arn:aws:s3:::messenger-attachments/originals/*",
"arn:aws:s3:::messenger-attachments/variants/*"
]
}
]
}
@@ -33,10 +33,10 @@
## 6. Стенд
- [ ] 6.1 `infra/rabbitmq/`, `infra/s3/worker-policy.json`, `infra/compose.faults.yaml` из `.prep/lab5`; пользователь worker в `infra/s3/init.sh`
- [ ] 6.2 `compose.yaml`: `broker`, `relay`, `worker`, volume `broker-data`, секреты брокера и S3-ключа worker; `scripts/broker-secrets.sh` в `make secrets`; `.env.example` (`BROKER_MANAGEMENT_PORT`, `JOB_BACKOFF_BASE_SECONDS`, `OUTBOX_POLL_SECONDS`, `SWEEP_INTERVAL_SECONDS`)
- [ ] 6.3 Хуки: `restart.sh` с relay, worker и broker; `worker.sh`, `broker.sh`
- [ ] 6.4 Проверка группы: `docker compose config --quiet`, `make up` из чистого состояния, все сервисы `healthy`, топология и consumer с `prefetch_count`, опубликованы только `lb` и UI брокера на `127.0.0.1`
- [x] 6.1 `infra/rabbitmq/`, `infra/s3/worker-policy.json`, `infra/compose.faults.yaml` из `.prep/lab5`; пользователь worker в `infra/s3/init.sh`
- [x] 6.2 `compose.yaml`: `broker`, `relay`, `worker`, volume `broker-data`, секреты брокера и S3-ключа worker; `scripts/broker-secrets.sh` в `make secrets`; `.env.example` (`BROKER_MANAGEMENT_PORT`, `JOB_BACKOFF_BASE_SECONDS`, `OUTBOX_POLL_SECONDS`, `SWEEP_INTERVAL_SECONDS`)
- [x] 6.3 Хуки: `restart.sh` с relay, worker и broker; `worker.sh`, `broker.sh`
- [x] 6.4 Проверка группы: `docker compose config --quiet`, `make up` из чистого состояния, все сервисы `healthy`, топология и consumer с `prefetch_count`, опубликованы только `lb` и UI брокера на `127.0.0.1`
## 7. Сценарии и обвязка
+60
View File
@@ -0,0 +1,60 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
umask 077
mkdir -p secrets
for role in api worker ops; do
file="secrets/rabbitmq_${role}_password"
[ -s "$file" ] || od -An -tx1 -N24 /dev/urandom | tr -d ' \n' >"$file"
done
[ -s secrets/s3_worker_access_key_id ] || printf 'worker%s' "$(od -An -tx1 -N8 /dev/urandom | tr -d ' \n')" >secrets/s3_worker_access_key_id
[ -s secrets/s3_worker_secret_access_key ] || od -An -tx1 -N32 /dev/urandom | tr -d ' \n' >secrets/s3_worker_secret_access_key
for role in api worker; do
file="secrets/broker_url_${role}"
[ -s "$file" ] || printf 'amqp://messenger-%s:%s@%s:5672/messenger' \
"$role" "$(cat "secrets/rabbitmq_${role}_password")" "${BROKER_HOST:-broker}" >"$file"
done
python3 - <<'PY'
import base64
import hashlib
import json
import os
def user(role, tags):
salt = os.urandom(4)
with open("secrets/rabbitmq_%s_password" % role, "rb") as source:
secret = source.read().strip()
digest = hashlib.sha256(salt + secret).digest()
return {
"name": "messenger-" + role,
"password_hash": base64.b64encode(salt + digest).decode(),
"hashing_algorithm": "rabbit_password_hashing_sha256",
"tags": tags,
}
def grant(role, configure, write, read):
return {"user": "messenger-" + role, "vhost": "messenger", "configure": configure, "write": write, "read": read}
definitions = {
"users": [user("api", []), user("worker", []), user("ops", ["administrator"])],
"permissions": [
grant("api", "^$", "^attachments$", "^$"),
grant("worker", "^$", "^$", "^attachments\\.process$"),
grant("ops", ".*", ".*", ".*"),
],
}
with open("secrets/rabbitmq_users.json", "w") as target:
json.dump(definitions, target, indent=2)
target.write("\n")
PY
chmod 700 secrets
chmod 644 secrets/rabbitmq_users.json secrets/rabbitmq_ops_password secrets/broker_url_api secrets/broker_url_worker \
secrets/s3_worker_access_key_id secrets/s3_worker_secret_access_key
echo "broker and S3 worker secrets ready in secrets/ (values not printed)"
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
COMPOSE=${COMPOSE:-docker compose}
service=${BROKER_SERVICE:-broker}
queues() {
$COMPOSE exec -T "$service" rabbitmqctl -q list_queues -p messenger \
name type messages_ready messages_unacknowledged consumers
}
case "${1:-}" in
restart) $COMPOSE restart -t 20 "$service" && $COMPOSE up -d --no-deps --no-recreate --wait --wait-timeout 90 "$service" ;;
queues) queues ;;
*) echo "usage: $0 restart|queues" >&2; exit 2 ;;
esac
+31 -2
View File
@@ -7,7 +7,14 @@ import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
S3_CLIENT_SECRETS = ["s3_access_key_id", "s3_root_password", "s3_root_user", "s3_secret_access_key"]
S3_CLIENT_SECRETS = [
"s3_access_key_id",
"s3_root_password",
"s3_root_user",
"s3_secret_access_key",
"s3_worker_access_key_id",
"s3_worker_secret_access_key",
]
POLICY = {
"lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True},
"api": {
@@ -32,7 +39,29 @@ POLICY = {
},
"s3-init": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]},
"s3-cli": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]},
"broker": {
"secrets": ["rabbitmq_ops_password", "rabbitmq_users"],
"volumes": ["broker-data"],
"networks": ["backend", "edge"],
"ports": True,
},
"relay": {
"secrets": ["broker_url_api", "database_url", "s3_access_key_id", "s3_secret_access_key"],
"volumes": [],
"networks": ["backend"],
},
"worker": {
"secrets": [
"broker_url_worker",
"database_url",
"s3_worker_access_key_id",
"s3_worker_secret_access_key",
],
"volumes": [],
"networks": ["backend"],
},
}
PUBLISHING = frozenset(name for name, rule in POLICY.items() if rule.get("ports"))
def run(*command):
@@ -94,7 +123,7 @@ def running(problems):
]
if int(cpus) <= 0 or int(memory) <= 0:
problems.append(f"container {service}/{host}: no CPU or memory limit applied")
if published and service != "lb":
if published and service not in PUBLISHING:
problems.append(f"container {service}/{host}: publishes {published}")
rows.append(
[service, host, f"{int(cpus) / 1e9:g}", f"{int(memory) >> 20}MiB", ",".join(published)]
+8 -5
View File
@@ -2,11 +2,14 @@
set -eu
cd "$(dirname "$0")/.."
defined=$(docker compose config --services)
stores=""
for service in ${STATE_SERVICES:-postgres valkey s3}; do
if printf '%s\n' "$defined" | grep -qx "$service"; then stores="$stores $service"; fi
done
pick() {
for service in $1; do
if printf '%s\n' "$defined" | grep -qx "$service"; then printf ' %s' "$service"; fi
done
}
apps=$(pick "${APP_SERVICES:-lb api relay worker}")
stores=$(pick "${STATE_SERVICES:-postgres valkey s3 broker}")
[ -n "$stores" ] || { echo "no state services found" >&2; exit 3; }
docker compose stop lb "${API_SERVICE:-api}"
docker compose stop $apps
docker compose restart $stores
docker compose up -d --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-75}"
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
COMPOSE=${COMPOSE:-docker compose}
service=${WORKER_SERVICE:-worker}
case "${1:-}" in
stop) $COMPOSE stop -t 10 "$service" ;;
kill) $COMPOSE kill -s KILL "$service" ;;
start) $COMPOSE up -d --no-deps --wait --wait-timeout 60 "$service" ;;
status) $COMPOSE ps -a --format '{{.Service}} {{.State}} {{.Status}}' "$service" ;;
*) echo "usage: $0 stop|kill|start|status" >&2; exit 2 ;;
esac