feat: Bootstrap durable RabbitMQ queue (U1)
This commit is contained in:
@@ -19,6 +19,7 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
BROKER_QUEUE=attachments.process
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# Independent broker bootstrap fragment. Merge these roles into compose.yaml
|
||||
# when the lab4 application base is ready.
|
||||
services:
|
||||
broker:
|
||||
image: rabbitmq:4.1.8-management
|
||||
hostname: broker
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- broker_data:/var/lib/rabbitmq
|
||||
- ./secrets/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 24
|
||||
networks: [backend]
|
||||
|
||||
broker-init:
|
||||
image: python:3.12.14-alpine3.24
|
||||
restart: "no"
|
||||
depends_on:
|
||||
broker:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
volumes:
|
||||
- ./scripts/bootstrap_broker.py:/app/bootstrap_broker.py:ro
|
||||
- ./secrets/broker_url:/run/secrets/broker_url:ro
|
||||
command: ["python", "/app/bootstrap_broker.py"]
|
||||
networks: [backend]
|
||||
|
||||
networks:
|
||||
backend:
|
||||
|
||||
volumes:
|
||||
broker_data:
|
||||
@@ -0,0 +1,22 @@
|
||||
Lab5 broker-only bootstrap check, 2026-10-02, local Colima Docker.
|
||||
|
||||
Commands (from repository root; local ignored secrets generated first):
|
||||
python3 scripts/init_local_broker.py
|
||||
docker compose -f compose.broker.yaml config --quiet
|
||||
docker compose -f compose.broker.yaml up -d
|
||||
docker compose -f compose.broker.yaml ps -a
|
||||
docker compose -f compose.broker.yaml restart broker
|
||||
docker compose -f compose.broker.yaml exec -T broker rabbitmqctl list_queues name durable type
|
||||
|
||||
Observed after first bootstrap:
|
||||
broker: Up (healthy)
|
||||
broker-init: Exited (0)
|
||||
Durable quorum queue attachments.process is ready
|
||||
|
||||
Observed after broker restart, before another bootstrap:
|
||||
name durable type
|
||||
attachments.process true quorum
|
||||
|
||||
No broker ports were published. This verifies queue metadata survives a
|
||||
container restart with broker_data; it does not verify message persistence,
|
||||
publisher confirms, worker acknowledgements, or full lab5 readiness.
|
||||
@@ -5,3 +5,15 @@
|
||||
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
|
||||
|
||||
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
|
||||
|
||||
## Локальный RabbitMQ
|
||||
|
||||
```sh
|
||||
python3 scripts/init_local_broker.py
|
||||
docker compose -f compose.broker.yaml up -d
|
||||
docker compose -f compose.broker.yaml ps -a
|
||||
```
|
||||
|
||||
Скрипт создаёт игнорируемые Git файлы `secrets/rabbitmq.conf` и `secrets/broker_url` с локальным паролем, не печатая его. `broker-init` повторяемо объявляет `attachments.process` как durable quorum queue; API publisher и worker должны использовать тот же `BROKER_QUEUE` и файл `BROKER_URL_FILE`. Порт RabbitMQ наружу не публикуется. Volume `broker_data` переживает `docker compose restart broker`; `down -v` намеренно удаляет учебные данные.
|
||||
|
||||
`compose.broker.yaml` — самостоятельный фрагмент для подготовки брокера. При переносе реализации lab4 включите его сервисы в общий `compose.yaml` и добавьте отдельные publisher/worker; до этого полный стенд лабораторной 5 не запустится. Один узел RabbitMQ с quorum queue обеспечивает сохранность при рестарте с volume, но не отказоустойчивость при потере узла или диска.
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Idempotently declare the durable processing queue via RabbitMQ management API."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import quote, unquote, urlsplit
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
broker_url = open(os.environ.get("BROKER_URL_FILE", "/run/secrets/broker_url"), encoding="utf-8").read().strip()
|
||||
parsed = urlsplit(broker_url)
|
||||
if parsed.scheme != "amqp" or not parsed.username or not parsed.password:
|
||||
raise SystemExit("Invalid broker URL file")
|
||||
credentials = f"{unquote(parsed.username)}:{unquote(parsed.password)}".encode()
|
||||
headers = {
|
||||
"Authorization": "Basic " + base64.b64encode(credentials).decode("ascii"),
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
url = "http://broker:15672/api/queues/%2F/" + quote("attachments.process", safe="")
|
||||
body = json.dumps(
|
||||
{"durable": True, "auto_delete": False, "arguments": {"x-queue-type": "quorum"}}
|
||||
).encode()
|
||||
|
||||
for attempt in range(30):
|
||||
try:
|
||||
with urlopen(Request(url, data=body, headers=headers, method="PUT"), timeout=3):
|
||||
pass
|
||||
with urlopen(Request(url, headers=headers), timeout=3) as response:
|
||||
queue = json.load(response)
|
||||
if queue.get("type") != "quorum" or not queue.get("durable"):
|
||||
raise SystemExit("Processing queue is not durable quorum")
|
||||
print("Durable quorum queue attachments.process is ready")
|
||||
break
|
||||
except HTTPError as exc:
|
||||
if exc.code in (400, 401, 403, 406):
|
||||
raise SystemExit(f"RabbitMQ queue declaration failed (HTTP {exc.code})") from None
|
||||
except URLError:
|
||||
pass
|
||||
if attempt == 29:
|
||||
raise SystemExit("RabbitMQ management API did not become ready")
|
||||
time.sleep(1)
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local RabbitMQ credentials without writing them to tracked files."""
|
||||
|
||||
from pathlib import Path
|
||||
from secrets import token_hex
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
secret_dir = root / "secrets"
|
||||
secret_dir.mkdir(mode=0o700, exist_ok=True)
|
||||
secret_dir.chmod(0o700)
|
||||
config = secret_dir / "rabbitmq.conf"
|
||||
url_file = secret_dir / "broker_url"
|
||||
username = "lab5_app"
|
||||
|
||||
if config.exists():
|
||||
values = dict(
|
||||
line.split(" = ", 1)
|
||||
for line in config.read_text(encoding="utf-8").splitlines()
|
||||
if " = " in line
|
||||
)
|
||||
if values.get("default_user") != username or not values.get("default_pass"):
|
||||
raise SystemExit("Existing RabbitMQ config cannot be reused")
|
||||
password = values["default_pass"]
|
||||
else:
|
||||
password = token_hex(24)
|
||||
config.write_text(
|
||||
f"default_user = {username}\ndefault_pass = {password}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
config.chmod(0o600)
|
||||
url_file.write_text(
|
||||
f"amqp://{username}:{quote(password, safe='')}@broker:5672/%2F\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
url_file.chmod(0o600)
|
||||
print("Local broker credentials are ready in ignored secrets/")
|
||||
Reference in New Issue
Block a user