feat: Bootstrap durable RabbitMQ queue (U1)

This commit is contained in:
Самохвалов Андрей Ремович
2026-10-02 13:28:26 +03:00
parent a96711a51a
commit af7213900b
6 changed files with 153 additions and 0 deletions
+1
View File
@@ -19,6 +19,7 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
BROKER_QUEUE=attachments.process
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
+36
View File
@@ -0,0 +1,36 @@
# Independent broker bootstrap fragment. Merge these roles into compose.yaml
# when the lab4 application base is ready.
services:
broker:
image: rabbitmq:4.1.8-management
hostname: broker
restart: unless-stopped
volumes:
- broker_data:/var/lib/rabbitmq
- ./secrets/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 24
networks: [backend]
broker-init:
image: python:3.12.14-alpine3.24
restart: "no"
depends_on:
broker:
condition: service_healthy
environment:
BROKER_URL_FILE: /run/secrets/broker_url
volumes:
- ./scripts/bootstrap_broker.py:/app/bootstrap_broker.py:ro
- ./secrets/broker_url:/run/secrets/broker_url:ro
command: ["python", "/app/bootstrap_broker.py"]
networks: [backend]
networks:
backend:
volumes:
broker_data:
+22
View File
@@ -0,0 +1,22 @@
Lab5 broker-only bootstrap check, 2026-10-02, local Colima Docker.
Commands (from repository root; local ignored secrets generated first):
python3 scripts/init_local_broker.py
docker compose -f compose.broker.yaml config --quiet
docker compose -f compose.broker.yaml up -d
docker compose -f compose.broker.yaml ps -a
docker compose -f compose.broker.yaml restart broker
docker compose -f compose.broker.yaml exec -T broker rabbitmqctl list_queues name durable type
Observed after first bootstrap:
broker: Up (healthy)
broker-init: Exited (0)
Durable quorum queue attachments.process is ready
Observed after broker restart, before another bootstrap:
name durable type
attachments.process true quorum
No broker ports were published. This verifies queue metadata survives a
container restart with broker_data; it does not verify message persistence,
publisher confirms, worker acknowledgements, or full lab5 readiness.
+12
View File
@@ -5,3 +5,15 @@
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
## Локальный RabbitMQ
```sh
python3 scripts/init_local_broker.py
docker compose -f compose.broker.yaml up -d
docker compose -f compose.broker.yaml ps -a
```
Скрипт создаёт игнорируемые Git файлы `secrets/rabbitmq.conf` и `secrets/broker_url` с локальным паролем, не печатая его. `broker-init` повторяемо объявляет `attachments.process` как durable quorum queue; API publisher и worker должны использовать тот же `BROKER_QUEUE` и файл `BROKER_URL_FILE`. Порт RabbitMQ наружу не публикуется. Volume `broker_data` переживает `docker compose restart broker`; `down -v` намеренно удаляет учебные данные.
`compose.broker.yaml` — самостоятельный фрагмент для подготовки брокера. При переносе реализации lab4 включите его сервисы в общий `compose.yaml` и добавьте отдельные publisher/worker; до этого полный стенд лабораторной 5 не запустится. Один узел RabbitMQ с quorum queue обеспечивает сохранность при рестарте с volume, но не отказоустойчивость при потере узла или диска.
+44
View File
@@ -0,0 +1,44 @@
#!/usr/bin/env python3
"""Idempotently declare the durable processing queue via RabbitMQ management API."""
import base64
import json
import os
import time
from urllib.error import HTTPError, URLError
from urllib.parse import quote, unquote, urlsplit
from urllib.request import Request, urlopen
broker_url = open(os.environ.get("BROKER_URL_FILE", "/run/secrets/broker_url"), encoding="utf-8").read().strip()
parsed = urlsplit(broker_url)
if parsed.scheme != "amqp" or not parsed.username or not parsed.password:
raise SystemExit("Invalid broker URL file")
credentials = f"{unquote(parsed.username)}:{unquote(parsed.password)}".encode()
headers = {
"Authorization": "Basic " + base64.b64encode(credentials).decode("ascii"),
"Content-Type": "application/json",
}
url = "http://broker:15672/api/queues/%2F/" + quote("attachments.process", safe="")
body = json.dumps(
{"durable": True, "auto_delete": False, "arguments": {"x-queue-type": "quorum"}}
).encode()
for attempt in range(30):
try:
with urlopen(Request(url, data=body, headers=headers, method="PUT"), timeout=3):
pass
with urlopen(Request(url, headers=headers), timeout=3) as response:
queue = json.load(response)
if queue.get("type") != "quorum" or not queue.get("durable"):
raise SystemExit("Processing queue is not durable quorum")
print("Durable quorum queue attachments.process is ready")
break
except HTTPError as exc:
if exc.code in (400, 401, 403, 406):
raise SystemExit(f"RabbitMQ queue declaration failed (HTTP {exc.code})") from None
except URLError:
pass
if attempt == 29:
raise SystemExit("RabbitMQ management API did not become ready")
time.sleep(1)
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env python3
"""Create local RabbitMQ credentials without writing them to tracked files."""
from pathlib import Path
from secrets import token_hex
from urllib.parse import quote
root = Path(__file__).resolve().parents[1]
secret_dir = root / "secrets"
secret_dir.mkdir(mode=0o700, exist_ok=True)
secret_dir.chmod(0o700)
config = secret_dir / "rabbitmq.conf"
url_file = secret_dir / "broker_url"
username = "lab5_app"
if config.exists():
values = dict(
line.split(" = ", 1)
for line in config.read_text(encoding="utf-8").splitlines()
if " = " in line
)
if values.get("default_user") != username or not values.get("default_pass"):
raise SystemExit("Existing RabbitMQ config cannot be reused")
password = values["default_pass"]
else:
password = token_hex(24)
config.write_text(
f"default_user = {username}\ndefault_pass = {password}\n",
encoding="utf-8",
)
config.chmod(0o600)
url_file.write_text(
f"amqp://{username}:{quote(password, safe='')}@broker:5672/%2F\n",
encoding="utf-8",
)
url_file.chmod(0o600)
print("Local broker credentials are ready in ignored secrets/")