forked from mtusi2026/mtusi2026lab5
абораторная 5: брокер, worker, outbox, варианты изображений, тесты отказов (оценка 4)
This commit is contained in:
@@ -0,0 +1,153 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = ROOT / "secrets"
|
||||
|
||||
|
||||
def write(name: str, value: str) -> None:
|
||||
with open(SECRETS / name, "w", encoding="utf-8", newline="") as file:
|
||||
file.write(value)
|
||||
(SECRETS / name).chmod(0o644) # контейнеры читают секрет не от root; папка secrets/ в .gitignore
|
||||
|
||||
|
||||
def bucket_name() -> str:
|
||||
"""S3_BUCKET из .env (если задан), иначе значение по умолчанию."""
|
||||
env = ROOT / ".env"
|
||||
if env.exists():
|
||||
for line in env.read_text(encoding="utf-8").splitlines():
|
||||
if line.strip().startswith("S3_BUCKET="):
|
||||
return line.split("=", 1)[1].strip()
|
||||
return "messenger-attachments"
|
||||
|
||||
|
||||
def s3_config(bucket: str) -> str:
|
||||
"""Конфиг ключей SeaweedFS: admin — только для s3-init, messenger-api — один bucket.
|
||||
|
||||
Права SeaweedFS: Read (GetObject/HeadObject), Write (PutObject/DeleteObject), List (ListObjects)
|
||||
с суффиксом :<bucket> действуют только в этом bucket. Admin (создание/удаление bucket и т.п.) у API нет.
|
||||
Identity "anonymous" не объявлена, поэтому запросы без подписи отклоняются.
|
||||
"""
|
||||
def read(name):
|
||||
return (SECRETS / name).read_text(encoding="utf-8").strip()
|
||||
config = {"identities": [
|
||||
{"name": "admin",
|
||||
"credentials": [{"accessKey": read("s3_root_user"), "secretKey": read("s3_root_password")}],
|
||||
"actions": ["Admin", "Read", "Write", "List", "Tagging"]},
|
||||
{"name": "messenger-api",
|
||||
"credentials": [{"accessKey": read("s3_access_key_id"), "secretKey": read("s3_secret_access_key")}],
|
||||
"actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]},
|
||||
]}
|
||||
return json.dumps(config, indent=2)
|
||||
|
||||
|
||||
BROKER_VHOST = "messenger"
|
||||
# Имена из pipeline.py (скрипт не импортирует код приложения, чтобы работать без зависимостей).
|
||||
EXCHANGE, QUEUE, ROUTING_KEY = "attachments", "attachments.process.v1", "attachment.process.v1"
|
||||
DLX, DLQ, DLQ_ROUTING_KEY = "attachments.dlx", "attachments.process.v1.dlq", "attachment.process.v1.dlq"
|
||||
|
||||
|
||||
def rabbit_hash(password: str) -> str:
|
||||
"""Формат rabbit_password_hashing_sha256: base64(salt(4 байта) + SHA-256(salt + пароль))."""
|
||||
salt = secrets.token_bytes(4)
|
||||
return base64.b64encode(salt + hashlib.sha256(salt + password.encode("utf-8")).digest()).decode("ascii")
|
||||
|
||||
|
||||
def rabbit_definitions(passwords: dict) -> str:
|
||||
"""Топология и пользователи RabbitMQ; брокер загружает файл при каждом старте (infra/rabbitmq.conf).
|
||||
|
||||
Права (configure / write / read, регулярные выражения по именам ресурсов):
|
||||
dispatcher — публикует только в exchange attachments; читает DLQ (ops.py dlq-replay); configure нет;
|
||||
worker — читает только основную очередь; не публикует (повторы идут через outbox); configure нет.
|
||||
Ни у кого из сервисов нет тега administrator и права создать/удалить очередь.
|
||||
"""
|
||||
queue_args = {
|
||||
"x-queue-type": "quorum",
|
||||
"x-delivery-limit": 20, # ядовитое сообщение не крутится бесконечно -> DLQ
|
||||
"x-dead-letter-exchange": DLX,
|
||||
"x-dead-letter-routing-key": DLQ_ROUTING_KEY,
|
||||
"x-dead-letter-strategy": "at-least-once", # перенос в DLQ без потери (требует reject-publish)
|
||||
"x-overflow": "reject-publish", # переполнение -> nack издателю, outbox повторит
|
||||
"x-max-length": 100000,
|
||||
}
|
||||
definitions = {
|
||||
"users": [{"name": name, "password_hash": rabbit_hash(password),
|
||||
"hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []}
|
||||
for name, password in passwords.items()],
|
||||
"vhosts": [{"name": BROKER_VHOST}],
|
||||
"permissions": [
|
||||
{"user": "dispatcher", "vhost": BROKER_VHOST, "configure": "^$",
|
||||
"write": f"^{EXCHANGE}$", "read": "^" + DLQ.replace(".", r"\.") + "$"},
|
||||
{"user": "worker", "vhost": BROKER_VHOST, "configure": "^$",
|
||||
"write": "^$", "read": "^" + QUEUE.replace(".", r"\.") + "$"},
|
||||
],
|
||||
"exchanges": [
|
||||
{"name": EXCHANGE, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False,
|
||||
"internal": False, "arguments": {}},
|
||||
{"name": DLX, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False,
|
||||
"internal": False, "arguments": {}},
|
||||
],
|
||||
"queues": [
|
||||
{"name": QUEUE, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False, "arguments": queue_args},
|
||||
# DLQ — classic durable: у quorum в 4.x delivery-limit по умолчанию 20, и многократный
|
||||
# просмотр (basic_get + nack) в ops.py мог бы выбросить сообщение. Здесь потерь от просмотра нет.
|
||||
{"name": DLQ, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False,
|
||||
"arguments": {"x-queue-type": "classic"}},
|
||||
],
|
||||
"bindings": [
|
||||
{"source": EXCHANGE, "vhost": BROKER_VHOST, "destination": QUEUE, "destination_type": "queue",
|
||||
"routing_key": ROUTING_KEY, "arguments": {}},
|
||||
{"source": DLX, "vhost": BROKER_VHOST, "destination": DLQ, "destination_type": "queue",
|
||||
"routing_key": DLQ_ROUTING_KEY, "arguments": {}},
|
||||
],
|
||||
}
|
||||
return json.dumps(definitions, indent=2)
|
||||
|
||||
|
||||
def broker_secrets(created: list, host: str = "rabbitmq") -> None:
|
||||
if (SECRETS / "rabbitmq_definitions.json").exists():
|
||||
return
|
||||
passwords = {"dispatcher": secrets.token_hex(24), "worker": secrets.token_hex(24)}
|
||||
for name, password in passwords.items():
|
||||
write(f"broker_url_{name}", f"amqp://{name}:{password}@{host}:5672/{BROKER_VHOST}")
|
||||
write("rabbitmq_definitions.json", rabbit_definitions(passwords))
|
||||
created += ["broker_url_dispatcher", "broker_url_worker", "rabbitmq_definitions.json"]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Создаёт недостающие секреты. Уже существующие не меняет (иначе разошлись бы с данными в volumes)."""
|
||||
SECRETS.mkdir(exist_ok=True)
|
||||
created = []
|
||||
if not (SECRETS / "postgres_password").exists():
|
||||
db_password = secrets.token_hex(24)
|
||||
redis_password = secrets.token_hex(24)
|
||||
write("postgres_password", db_password)
|
||||
write("database_url", f"postgresql://messenger:{db_password}@postgres:5432/messenger")
|
||||
write("cache_url", f"redis://:{redis_password}@redis:6379/0")
|
||||
write("redis_auth", f"requirepass {redis_password}\n")
|
||||
created += ["postgres_password", "database_url", "cache_url", "redis_auth"]
|
||||
if not (SECRETS / "s3_root_password").exists():
|
||||
# admin-ключ S3: только для сервисов s3 (в конфиге) и s3-init
|
||||
write("s3_root_user", "root-" + secrets.token_hex(6))
|
||||
write("s3_root_password", secrets.token_hex(24))
|
||||
created += ["s3_root_user", "s3_root_password"]
|
||||
if not (SECRETS / "s3_secret_access_key").exists():
|
||||
# отдельный ключ API с политикой только на свой bucket (создаёт s3-init)
|
||||
write("s3_access_key_id", "api-" + secrets.token_hex(8))
|
||||
write("s3_secret_access_key", secrets.token_hex(24))
|
||||
created += ["s3_access_key_id", "s3_secret_access_key"]
|
||||
if not (SECRETS / "s3_config.json").exists():
|
||||
write("s3_config.json", s3_config(bucket_name()))
|
||||
created.append("s3_config.json")
|
||||
broker_secrets(created)
|
||||
if created:
|
||||
print("Created in secrets/: " + ", ".join(created) + " (values are not printed).")
|
||||
else:
|
||||
print("secrets/ is complete, nothing changed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user