diff --git a/.env.example b/.env.example index 119384b5..ca76d513 100644 --- a/.env.example +++ b/.env.example @@ -19,6 +19,7 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 BROKER_URL_FILE=/run/secrets/broker_url +BROKER_QUEUE=attachments.process WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 diff --git a/compose.broker.yaml b/compose.broker.yaml new file mode 100644 index 00000000..8fcedf52 --- /dev/null +++ b/compose.broker.yaml @@ -0,0 +1,36 @@ +# Independent broker bootstrap fragment. Merge these roles into compose.yaml +# when the lab4 application base is ready. +services: + broker: + image: rabbitmq:4.1.8-management + hostname: broker + restart: unless-stopped + volumes: + - broker_data:/var/lib/rabbitmq + - ./secrets/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 24 + networks: [backend] + + broker-init: + image: python:3.12.14-alpine3.24 + restart: "no" + depends_on: + broker: + condition: service_healthy + environment: + BROKER_URL_FILE: /run/secrets/broker_url + volumes: + - ./scripts/bootstrap_broker.py:/app/bootstrap_broker.py:ro + - ./secrets/broker_url:/run/secrets/broker_url:ro + command: ["python", "/app/bootstrap_broker.py"] + networks: [backend] + +networks: + backend: + +volumes: + broker_data: diff --git a/evidence/broker-restart.txt b/evidence/broker-restart.txt new file mode 100644 index 00000000..d4178ea5 --- /dev/null +++ b/evidence/broker-restart.txt @@ -0,0 +1,22 @@ +Lab5 broker-only bootstrap check, 2026-10-02, local Colima Docker. + +Commands (from repository root; local ignored secrets generated first): + python3 scripts/init_local_broker.py + docker compose -f compose.broker.yaml config --quiet + docker compose -f compose.broker.yaml up -d + docker compose -f compose.broker.yaml ps -a + docker compose -f compose.broker.yaml restart broker + docker compose -f compose.broker.yaml exec -T broker rabbitmqctl list_queues name durable type + +Observed after first bootstrap: + broker: Up (healthy) + broker-init: Exited (0) + Durable quorum queue attachments.process is ready + +Observed after broker restart, before another bootstrap: + name durable type + attachments.process true quorum + +No broker ports were published. This verifies queue metadata survives a +container restart with broker_data; it does not verify message persistence, +publisher confirms, worker acknowledgements, or full lab5 readiness. diff --git a/infra/README.md b/infra/README.md index 5a39e862..4d32be2f 100644 --- a/infra/README.md +++ b/infra/README.md @@ -5,3 +5,15 @@ В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. + +## Локальный RabbitMQ + +```sh +python3 scripts/init_local_broker.py +docker compose -f compose.broker.yaml up -d +docker compose -f compose.broker.yaml ps -a +``` + +Скрипт создаёт игнорируемые Git файлы `secrets/rabbitmq.conf` и `secrets/broker_url` с локальным паролем, не печатая его. `broker-init` повторяемо объявляет `attachments.process` как durable quorum queue; API publisher и worker должны использовать тот же `BROKER_QUEUE` и файл `BROKER_URL_FILE`. Порт RabbitMQ наружу не публикуется. Volume `broker_data` переживает `docker compose restart broker`; `down -v` намеренно удаляет учебные данные. + +`compose.broker.yaml` — самостоятельный фрагмент для подготовки брокера. При переносе реализации lab4 включите его сервисы в общий `compose.yaml` и добавьте отдельные publisher/worker; до этого полный стенд лабораторной 5 не запустится. Один узел RabbitMQ с quorum queue обеспечивает сохранность при рестарте с volume, но не отказоустойчивость при потере узла или диска. diff --git a/scripts/bootstrap_broker.py b/scripts/bootstrap_broker.py new file mode 100644 index 00000000..25d27806 --- /dev/null +++ b/scripts/bootstrap_broker.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Idempotently declare the durable processing queue via RabbitMQ management API.""" + +import base64 +import json +import os +import time +from urllib.error import HTTPError, URLError +from urllib.parse import quote, unquote, urlsplit +from urllib.request import Request, urlopen + + +broker_url = open(os.environ.get("BROKER_URL_FILE", "/run/secrets/broker_url"), encoding="utf-8").read().strip() +parsed = urlsplit(broker_url) +if parsed.scheme != "amqp" or not parsed.username or not parsed.password: + raise SystemExit("Invalid broker URL file") +credentials = f"{unquote(parsed.username)}:{unquote(parsed.password)}".encode() +headers = { + "Authorization": "Basic " + base64.b64encode(credentials).decode("ascii"), + "Content-Type": "application/json", +} +url = "http://broker:15672/api/queues/%2F/" + quote("attachments.process", safe="") +body = json.dumps( + {"durable": True, "auto_delete": False, "arguments": {"x-queue-type": "quorum"}} +).encode() + +for attempt in range(30): + try: + with urlopen(Request(url, data=body, headers=headers, method="PUT"), timeout=3): + pass + with urlopen(Request(url, headers=headers), timeout=3) as response: + queue = json.load(response) + if queue.get("type") != "quorum" or not queue.get("durable"): + raise SystemExit("Processing queue is not durable quorum") + print("Durable quorum queue attachments.process is ready") + break + except HTTPError as exc: + if exc.code in (400, 401, 403, 406): + raise SystemExit(f"RabbitMQ queue declaration failed (HTTP {exc.code})") from None + except URLError: + pass + if attempt == 29: + raise SystemExit("RabbitMQ management API did not become ready") + time.sleep(1) diff --git a/scripts/init_local_broker.py b/scripts/init_local_broker.py new file mode 100644 index 00000000..e24bf49f --- /dev/null +++ b/scripts/init_local_broker.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Create local RabbitMQ credentials without writing them to tracked files.""" + +from pathlib import Path +from secrets import token_hex +from urllib.parse import quote + + +root = Path(__file__).resolve().parents[1] +secret_dir = root / "secrets" +secret_dir.mkdir(mode=0o700, exist_ok=True) +secret_dir.chmod(0o700) +config = secret_dir / "rabbitmq.conf" +url_file = secret_dir / "broker_url" +username = "lab5_app" + +if config.exists(): + values = dict( + line.split(" = ", 1) + for line in config.read_text(encoding="utf-8").splitlines() + if " = " in line + ) + if values.get("default_user") != username or not values.get("default_pass"): + raise SystemExit("Existing RabbitMQ config cannot be reused") + password = values["default_pass"] +else: + password = token_hex(24) + config.write_text( + f"default_user = {username}\ndefault_pass = {password}\n", + encoding="utf-8", + ) +config.chmod(0o600) +url_file.write_text( + f"amqp://{username}:{quote(password, safe='')}@broker:5672/%2F\n", + encoding="utf-8", +) +url_file.chmod(0o600) +print("Local broker credentials are ready in ignored secrets/")