diff --git a/.env.example b/.env.example index 8d8f7f77..2d08f1e9 100644 --- a/.env.example +++ b/.env.example @@ -55,3 +55,13 @@ UPLOAD_SWEEP_INTERVAL_SECONDS=30 UPLOAD_SWEEP_BATCH=100 # Fixed glibc mmap threshold for the API: buffers above 128 KiB are returned to the system on free. MALLOC_MMAP_THRESHOLD_=131072 +# Solution-specific for lab5: RabbitMQ management UI on 127.0.0.1, backoff base of a transient +# retry (doubles per attempt up to 60 s), relay poll and sweep intervals, worker limits and the +# PostgreSQL pool of relay and worker. FAULT_POINT is set only by infra/compose.faults.yaml. +BROKER_MANAGEMENT_PORT=15672 +JOB_BACKOFF_BASE_SECONDS=2 +OUTBOX_POLL_SECONDS=1 +SWEEP_INTERVAL_SECONDS=5 +WORKER_CPUS=2 +WORKER_MEMORY=768m +PIPELINE_DB_POOL_MAX_SIZE=4 diff --git a/Makefile b/Makefile index ff1f2e64..12c86f06 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,7 @@ student-test: secrets: sh scripts/secrets.sh + sh scripts/broker-secrets.sh up: secrets $(COMPOSE) up -d --build --wait diff --git a/compose.yaml b/compose.yaml index 8efa4a7a..2372d91f 100644 --- a/compose.yaml +++ b/compose.yaml @@ -18,6 +18,44 @@ x-s3-env: &s3-env UPLOAD_SWEEP_INTERVAL_SECONDS: ${UPLOAD_SWEEP_INTERVAL_SECONDS:-30} UPLOAD_SWEEP_BATCH: ${UPLOAD_SWEEP_BATCH:-100} +x-pipeline: &pipeline + image: ${API_IMAGE:-mtusi-messenger:lab5} + pull_policy: never + restart: unless-stopped + init: true + read_only: true + tmpfs: [/tmp] + security_opt: ["no-new-privileges:true"] + cap_drop: [ALL] + stop_grace_period: 10s + networks: [backend] + healthcheck: + test: ["CMD", "python", "-c", "import os,sys,time;sys.exit(time.time()-os.stat('/tmp/heartbeat').st_mtime>30)"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + start_interval: 2s + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + broker: + condition: service_healthy + +x-pipeline-env: &pipeline-env + <<: *s3-env + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + DB_POOL_MAX_SIZE: ${PIPELINE_DB_POOL_MAX_SIZE:-4} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + HEARTBEAT_FILE: /tmp/heartbeat + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + x-s3-client: &s3-client image: rustfs/rc:v0.1.36@sha256:ab024bfebee49a750ce886b4c70963ccd9ddaa03f491704a90710641d7a26699 read_only: true @@ -27,7 +65,9 @@ x-s3-client: &s3-client S3_BUCKET: ${S3_BUCKET:-messenger-attachments} RC_CONFIG_DIR: /tmp/rc volumes: ["./infra/s3:/s3:ro"] - secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key] + secrets: + [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key, + s3_worker_access_key_id, s3_worker_secret_access_key] cap_drop: [ALL] security_opt: ["no-new-privileges:true"] restart: "no" @@ -234,6 +274,74 @@ services: profiles: [tools] entrypoint: ["/bin/sh", "/s3/cli.sh"] + broker: + image: rabbitmq:4.1.8-management-alpine@sha256:54ef5ff505352c1beef05a97ee9bd6ab95b3559a76629ba0119b1d63d8d6e4b2 + hostname: broker + user: "100:101" + restart: unless-stopped + read_only: true + tmpfs: [/tmp] + security_opt: ["no-new-privileges:true"] + cap_drop: [ALL] + ports: + - "127.0.0.1:${BROKER_MANAGEMENT_PORT:-15672}:15672" + networks: [backend, edge] + volumes: + - broker-data:/var/lib/rabbitmq + - ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/conf.d/20-messenger.conf:ro + - ./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro + - ./infra/rabbitmq/definitions.json:/etc/rabbitmq/definitions.d/10-topology.json:ro + secrets: + - source: rabbitmq_users + target: /etc/rabbitmq/definitions.d/20-users.json + - rabbitmq_ops_password + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"] + interval: 10s + timeout: 10s + retries: 5 + start_period: 60s + start_interval: 2s + stop_grace_period: 10s + deploy: + resources: + limits: {cpus: "1", memory: 512m} + + relay: + <<: *pipeline + command: ["python", "-m", "messenger.relay"] + environment: + <<: *pipeline-env + OUTBOX_POLL_SECONDS: ${OUTBOX_POLL_SECONDS:-1} + SWEEP_INTERVAL_SECONDS: ${SWEEP_INTERVAL_SECONDS:-5} + secrets: + - database_url + - source: broker_url_api + target: broker_url + - s3_access_key_id + - s3_secret_access_key + deploy: + resources: + limits: {cpus: "0.5", memory: 192m} + + worker: + <<: *pipeline + command: ["python", "-m", "messenger.worker"] + environment: + <<: *pipeline-env + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + secrets: + - database_url + - source: broker_url_worker + target: broker_url + - source: s3_worker_access_key_id + target: s3_access_key_id + - source: s3_worker_secret_access_key + target: s3_secret_access_key + deploy: + resources: + limits: {cpus: "${WORKER_CPUS:-2}", memory: "${WORKER_MEMORY:-768m}"} + k6: image: grafana/k6:2.3.0@sha256:2dd4756ec6e14e62778916e1eb850108e4b499304c02df56760fa0bdb0fba0b8 profiles: [bench] @@ -264,11 +372,24 @@ secrets: file: ./secrets/s3_access_key_id s3_secret_access_key: file: ./secrets/s3_secret_access_key + s3_worker_access_key_id: + file: ./secrets/s3_worker_access_key_id + s3_worker_secret_access_key: + file: ./secrets/s3_worker_secret_access_key + rabbitmq_users: + file: ./secrets/rabbitmq_users.json + rabbitmq_ops_password: + file: ./secrets/rabbitmq_ops_password + broker_url_api: + file: ./secrets/broker_url_api + broker_url_worker: + file: ./secrets/broker_url_worker volumes: pgdata: valkeydata: s3data: + broker-data: networks: edge: diff --git a/infra/compose.faults.yaml b/infra/compose.faults.yaml new file mode 100644 index 00000000..2b9a521e --- /dev/null +++ b/infra/compose.faults.yaml @@ -0,0 +1,15 @@ +services: + relay: + restart: "no" + environment: + FAULT_POINT: ${RELAY_FAULT_POINT:-} + OUTBOX_POLL_SECONDS: "0.5" + SWEEP_INTERVAL_SECONDS: "1" + + worker: + restart: "no" + environment: + FAULT_POINT: ${WORKER_FAULT_POINT:-} + FAULT_STALL_SECONDS: ${FAULT_STALL_SECONDS:-15} + JOB_TIMEOUT_SECONDS: ${FAULT_JOB_TIMEOUT_SECONDS:-5} + JOB_BACKOFF_BASE_SECONDS: ${FAULT_BACKOFF_BASE_SECONDS:-1} diff --git a/infra/rabbitmq/definitions.json b/infra/rabbitmq/definitions.json new file mode 100644 index 00000000..c4f9ec1b --- /dev/null +++ b/infra/rabbitmq/definitions.json @@ -0,0 +1,44 @@ +{ + "rabbit_version": "4.1.8", + "vhosts": [ + {"name": "messenger", "description": "lab5 attachment pipeline", "default_queue_type": "quorum"} + ], + "exchanges": [ + {"name": "attachments", "vhost": "messenger", "type": "direct", "durable": true, "auto_delete": false, "internal": false, "arguments": {}}, + {"name": "attachments.dlx", "vhost": "messenger", "type": "fanout", "durable": true, "auto_delete": false, "internal": false, "arguments": {}} + ], + "queues": [ + {"name": "attachments.process", "vhost": "messenger", "durable": true, "auto_delete": false, "arguments": {"x-queue-type": "quorum"}}, + {"name": "attachments.process.dlq", "vhost": "messenger", "durable": true, "auto_delete": false, "arguments": {"x-queue-type": "quorum"}} + ], + "bindings": [ + {"source": "attachments", "vhost": "messenger", "destination": "attachments.process", "destination_type": "queue", "routing_key": "attachment.process.v1", "arguments": {}}, + {"source": "attachments.dlx", "vhost": "messenger", "destination": "attachments.process.dlq", "destination_type": "queue", "routing_key": "", "arguments": {}} + ], + "policies": [ + { + "vhost": "messenger", + "name": "attachments-process", + "pattern": "^attachments\\.process$", + "apply-to": "queues", + "priority": 10, + "definition": { + "dead-letter-exchange": "attachments.dlx", + "dead-letter-strategy": "at-least-once", + "overflow": "reject-publish", + "delivery-limit": 10 + } + }, + { + "vhost": "messenger", + "name": "attachments-dlq", + "pattern": "^attachments\\.process\\.dlq$", + "apply-to": "queues", + "priority": 10, + "definition": { + "max-length": 100000, + "overflow": "reject-publish" + } + } + ] +} diff --git a/infra/rabbitmq/enabled_plugins b/infra/rabbitmq/enabled_plugins new file mode 100644 index 00000000..cc972a47 --- /dev/null +++ b/infra/rabbitmq/enabled_plugins @@ -0,0 +1 @@ +[rabbitmq_management,rabbitmq_prometheus]. diff --git a/infra/rabbitmq/rabbitmq.conf b/infra/rabbitmq/rabbitmq.conf new file mode 100644 index 00000000..06b35dcf --- /dev/null +++ b/infra/rabbitmq/rabbitmq.conf @@ -0,0 +1,21 @@ +listeners.tcp.default = 5672 +management.tcp.port = 15672 +prometheus.tcp.port = 15692 + +loopback_users.guest = true +log.console = true +log.console.level = info +log.file = false + +definitions.import_backend = local_filesystem +definitions.local.path = /etc/rabbitmq/definitions.d + +vm_memory_high_watermark.absolute = 256MiB +disk_free_limit.absolute = 256MB +raft.wal_max_size_bytes = 64000000 + +consumer_timeout = 300000 +heartbeat = 30 + +deprecated_features.permit.transient_nonexcl_queues = false +deprecated_features.permit.global_qos = false diff --git a/infra/s3/init.sh b/infra/s3/init.sh index 3686b1c6..c9bd299f 100755 --- a/infra/s3/init.sh +++ b/infra/s3/init.sh @@ -14,7 +14,13 @@ sed "s/messenger-attachments/$bucket/g" /s3/api-policy.json > /tmp/api-policy.js rc admin policy create root "$policy" /tmp/api-policy.json >/dev/null rc admin user add root "$api_key" "$(secret s3_secret_access_key)" >/dev/null rc admin policy attach root "$policy" --user "$api_key" >/dev/null +worker_key=$(secret s3_worker_access_key_id) +sed "s/messenger-attachments/$bucket/g" /s3/worker-policy.json > /tmp/worker-policy.json +rc admin policy create root messenger-worker /tmp/worker-policy.json >/dev/null +rc admin user add root "$worker_key" "$(secret s3_worker_secret_access_key)" >/dev/null +rc admin policy attach root messenger-worker --user "$worker_key" >/dev/null rc bucket lifecycle rule import "root/$bucket" /s3/lifecycle.json >/dev/null rc bucket anonymous get "root/$bucket" --json | grep -q '"permission": *"private"' rc admin user info root "$api_key" --json | grep -q "\"$policy\"" -echo "s3-init: bucket=$bucket anonymous=private policy=$policy lifecycle=abort-incomplete-uploads" +rc admin user info root "$worker_key" --json | grep -q '"messenger-worker"' +echo "s3-init: bucket=$bucket anonymous=private policy=$policy worker=messenger-worker lifecycle=abort-incomplete-uploads" diff --git a/infra/s3/worker-policy.json b/infra/s3/worker-policy.json new file mode 100644 index 00000000..b7d4f805 --- /dev/null +++ b/infra/s3/worker-policy.json @@ -0,0 +1,24 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "MessengerWorkerRead", + "Effect": "Allow", + "Action": ["s3:GetObject"], + "Resource": [ + "arn:aws:s3:::messenger-attachments/staging/*", + "arn:aws:s3:::messenger-attachments/originals/*", + "arn:aws:s3:::messenger-attachments/variants/*" + ] + }, + { + "Sid": "MessengerWorkerWrite", + "Effect": "Allow", + "Action": ["s3:PutObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts"], + "Resource": [ + "arn:aws:s3:::messenger-attachments/originals/*", + "arn:aws:s3:::messenger-attachments/variants/*" + ] + } + ] +} diff --git a/openspec/changes/add-lab5-async-attachments/tasks.md b/openspec/changes/add-lab5-async-attachments/tasks.md index 575ed156..1c7e967d 100644 --- a/openspec/changes/add-lab5-async-attachments/tasks.md +++ b/openspec/changes/add-lab5-async-attachments/tasks.md @@ -33,10 +33,10 @@ ## 6. Стенд -- [ ] 6.1 `infra/rabbitmq/`, `infra/s3/worker-policy.json`, `infra/compose.faults.yaml` из `.prep/lab5`; пользователь worker в `infra/s3/init.sh` -- [ ] 6.2 `compose.yaml`: `broker`, `relay`, `worker`, volume `broker-data`, секреты брокера и S3-ключа worker; `scripts/broker-secrets.sh` в `make secrets`; `.env.example` (`BROKER_MANAGEMENT_PORT`, `JOB_BACKOFF_BASE_SECONDS`, `OUTBOX_POLL_SECONDS`, `SWEEP_INTERVAL_SECONDS`) -- [ ] 6.3 Хуки: `restart.sh` с relay, worker и broker; `worker.sh`, `broker.sh` -- [ ] 6.4 Проверка группы: `docker compose config --quiet`, `make up` из чистого состояния, все сервисы `healthy`, топология и consumer с `prefetch_count`, опубликованы только `lb` и UI брокера на `127.0.0.1` +- [x] 6.1 `infra/rabbitmq/`, `infra/s3/worker-policy.json`, `infra/compose.faults.yaml` из `.prep/lab5`; пользователь worker в `infra/s3/init.sh` +- [x] 6.2 `compose.yaml`: `broker`, `relay`, `worker`, volume `broker-data`, секреты брокера и S3-ключа worker; `scripts/broker-secrets.sh` в `make secrets`; `.env.example` (`BROKER_MANAGEMENT_PORT`, `JOB_BACKOFF_BASE_SECONDS`, `OUTBOX_POLL_SECONDS`, `SWEEP_INTERVAL_SECONDS`) +- [x] 6.3 Хуки: `restart.sh` с relay, worker и broker; `worker.sh`, `broker.sh` +- [x] 6.4 Проверка группы: `docker compose config --quiet`, `make up` из чистого состояния, все сервисы `healthy`, топология и consumer с `prefetch_count`, опубликованы только `lb` и UI брокера на `127.0.0.1` ## 7. Сценарии и обвязка diff --git a/scripts/broker-secrets.sh b/scripts/broker-secrets.sh new file mode 100755 index 00000000..4d0ee49c --- /dev/null +++ b/scripts/broker-secrets.sh @@ -0,0 +1,60 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +umask 077 +mkdir -p secrets + +for role in api worker ops; do + file="secrets/rabbitmq_${role}_password" + [ -s "$file" ] || od -An -tx1 -N24 /dev/urandom | tr -d ' \n' >"$file" +done + +[ -s secrets/s3_worker_access_key_id ] || printf 'worker%s' "$(od -An -tx1 -N8 /dev/urandom | tr -d ' \n')" >secrets/s3_worker_access_key_id +[ -s secrets/s3_worker_secret_access_key ] || od -An -tx1 -N32 /dev/urandom | tr -d ' \n' >secrets/s3_worker_secret_access_key + +for role in api worker; do + file="secrets/broker_url_${role}" + [ -s "$file" ] || printf 'amqp://messenger-%s:%s@%s:5672/messenger' \ + "$role" "$(cat "secrets/rabbitmq_${role}_password")" "${BROKER_HOST:-broker}" >"$file" +done + +python3 - <<'PY' +import base64 +import hashlib +import json +import os + + +def user(role, tags): + salt = os.urandom(4) + with open("secrets/rabbitmq_%s_password" % role, "rb") as source: + secret = source.read().strip() + digest = hashlib.sha256(salt + secret).digest() + return { + "name": "messenger-" + role, + "password_hash": base64.b64encode(salt + digest).decode(), + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": tags, + } + + +def grant(role, configure, write, read): + return {"user": "messenger-" + role, "vhost": "messenger", "configure": configure, "write": write, "read": read} + + +definitions = { + "users": [user("api", []), user("worker", []), user("ops", ["administrator"])], + "permissions": [ + grant("api", "^$", "^attachments$", "^$"), + grant("worker", "^$", "^$", "^attachments\\.process$"), + grant("ops", ".*", ".*", ".*"), + ], +} +with open("secrets/rabbitmq_users.json", "w") as target: + json.dump(definitions, target, indent=2) + target.write("\n") +PY +chmod 700 secrets +chmod 644 secrets/rabbitmq_users.json secrets/rabbitmq_ops_password secrets/broker_url_api secrets/broker_url_worker \ + secrets/s3_worker_access_key_id secrets/s3_worker_secret_access_key +echo "broker and S3 worker secrets ready in secrets/ (values not printed)" diff --git a/scripts/broker.sh b/scripts/broker.sh new file mode 100755 index 00000000..ec6ac021 --- /dev/null +++ b/scripts/broker.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +COMPOSE=${COMPOSE:-docker compose} +service=${BROKER_SERVICE:-broker} +queues() { + $COMPOSE exec -T "$service" rabbitmqctl -q list_queues -p messenger \ + name type messages_ready messages_unacknowledged consumers +} +case "${1:-}" in + restart) $COMPOSE restart -t 20 "$service" && $COMPOSE up -d --no-deps --no-recreate --wait --wait-timeout 90 "$service" ;; + queues) queues ;; + *) echo "usage: $0 restart|queues" >&2; exit 2 ;; +esac diff --git a/scripts/limits.py b/scripts/limits.py index c487ef6f..c1da642a 100644 --- a/scripts/limits.py +++ b/scripts/limits.py @@ -7,7 +7,14 @@ import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -S3_CLIENT_SECRETS = ["s3_access_key_id", "s3_root_password", "s3_root_user", "s3_secret_access_key"] +S3_CLIENT_SECRETS = [ + "s3_access_key_id", + "s3_root_password", + "s3_root_user", + "s3_secret_access_key", + "s3_worker_access_key_id", + "s3_worker_secret_access_key", +] POLICY = { "lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True}, "api": { @@ -32,7 +39,29 @@ POLICY = { }, "s3-init": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]}, "s3-cli": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]}, + "broker": { + "secrets": ["rabbitmq_ops_password", "rabbitmq_users"], + "volumes": ["broker-data"], + "networks": ["backend", "edge"], + "ports": True, + }, + "relay": { + "secrets": ["broker_url_api", "database_url", "s3_access_key_id", "s3_secret_access_key"], + "volumes": [], + "networks": ["backend"], + }, + "worker": { + "secrets": [ + "broker_url_worker", + "database_url", + "s3_worker_access_key_id", + "s3_worker_secret_access_key", + ], + "volumes": [], + "networks": ["backend"], + }, } +PUBLISHING = frozenset(name for name, rule in POLICY.items() if rule.get("ports")) def run(*command): @@ -94,7 +123,7 @@ def running(problems): ] if int(cpus) <= 0 or int(memory) <= 0: problems.append(f"container {service}/{host}: no CPU or memory limit applied") - if published and service != "lb": + if published and service not in PUBLISHING: problems.append(f"container {service}/{host}: publishes {published}") rows.append( [service, host, f"{int(cpus) / 1e9:g}", f"{int(memory) >> 20}MiB", ",".join(published)] diff --git a/scripts/restart.sh b/scripts/restart.sh index c78416db..c5b30a67 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -2,11 +2,14 @@ set -eu cd "$(dirname "$0")/.." defined=$(docker compose config --services) -stores="" -for service in ${STATE_SERVICES:-postgres valkey s3}; do - if printf '%s\n' "$defined" | grep -qx "$service"; then stores="$stores $service"; fi -done +pick() { + for service in $1; do + if printf '%s\n' "$defined" | grep -qx "$service"; then printf ' %s' "$service"; fi + done +} +apps=$(pick "${APP_SERVICES:-lb api relay worker}") +stores=$(pick "${STATE_SERVICES:-postgres valkey s3 broker}") [ -n "$stores" ] || { echo "no state services found" >&2; exit 3; } -docker compose stop lb "${API_SERVICE:-api}" +docker compose stop $apps docker compose restart $stores docker compose up -d --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-75}" diff --git a/scripts/worker.sh b/scripts/worker.sh new file mode 100755 index 00000000..e8f0a8b5 --- /dev/null +++ b/scripts/worker.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +COMPOSE=${COMPOSE:-docker compose} +service=${WORKER_SERVICE:-worker} +case "${1:-}" in + stop) $COMPOSE stop -t 10 "$service" ;; + kill) $COMPOSE kill -s KILL "$service" ;; + start) $COMPOSE up -d --no-deps --wait --wait-timeout 60 "$service" ;; + status) $COMPOSE ps -a --format '{{.Service}} {{.State}} {{.Status}}' "$service" ;; + *) echo "usage: $0 stop|kill|start|status" >&2; exit 2 ;; +esac