Асинхронная обработка вложений: RabbitMQ 4.1 (quorum-очереди, publisher confirms, manual ack, DLQ), transactional outbox, отдельный worker. - upload: оригинал в staging S3, затем одной транзакцией queued + jobs + outbox -> 202; API к брокеру не подключается - relay: публикация outbox с confirm, reaper истёкших аренд, redrive - worker: claim по строке jobs, аренда с heartbeat и fencing по lease_token, transform в подпроцессе с RLIMIT_AS/RLIMIT_CPU/таймаутом, thumbnail 256, crop 128, WebP 1280 по детерминированным ключам - retry с backoff и jitter, постоянные/временные ошибки, реестр failed, ops replay (ready не повторяется), DLQ для невалидных сообщений - janitor: очистка staging и сирот; миграция 004 с backfill вложений lab4 - тесты отказов: окна COMMIT/publish, крахи worker до/после S3 и ack, заморозка worker, всплеск загрузок, права брокера; evidence/ и REPORT.md
113 lines
6.1 KiB
Bash
Executable File
113 lines
6.1 KiB
Bash
Executable File
#!/bin/sh
|
|
# Создаёт локальные секреты и .env для учебного стенда. Повторный запуск ничего не перезаписывает.
|
|
# Значения секретов не печатаются.
|
|
#
|
|
# S3 (SeaweedFS) — РАЗНЫЕ ключи с разными правами (secrets/s3_identities.json):
|
|
# admin — Admin/Read/Write/List: только job s3-init (создание bucket);
|
|
# api — Read+Write на bucket: staging PUT и выдача байтов, без List и без управления bucket;
|
|
# worker — Read+Write на bucket: чтение staging, запись результатов, удаление staging;
|
|
# janitor — List+Write на bucket: обход и удаление, без чтения содержимого.
|
|
# RabbitMQ — пользователи с минимальными правами в vhost messenger (secrets/rabbitmq_definitions.json):
|
|
# relay — только publish в exchange attachments;
|
|
# worker — только consume из attachments.process.v1;
|
|
# ops — consume из DLQ и publish в attachments (replay/учебные проверки).
|
|
# Ни у кого из них нет права configure: топология создаётся из definitions при старте брокера.
|
|
set -eu
|
|
cd "$(dirname "$0")/.."
|
|
umask 077
|
|
mkdir -p secrets
|
|
chmod 700 secrets
|
|
gen() { # gen <file> <bytes>
|
|
if [ ! -s "secrets/$1" ]; then
|
|
python3 -c "import secrets,sys; print(secrets.token_urlsafe(int(sys.argv[1])))" "$2" > "secrets/$1"
|
|
echo "created secrets/$1"
|
|
fi
|
|
}
|
|
gen postgres_password 24
|
|
for who in admin api worker janitor; do
|
|
if [ ! -s "secrets/s3_${who}_access_key" ]; then
|
|
python3 -c "import secrets; print('MTUSI' + secrets.token_hex(8).upper())" > "secrets/s3_${who}_access_key"
|
|
echo "created secrets/s3_${who}_access_key"
|
|
fi
|
|
gen "s3_${who}_secret_key" 30
|
|
done
|
|
for who in relay worker ops; do
|
|
gen "broker_${who}_password" 24
|
|
done
|
|
BUCKET=$(grep -E '^S3_BUCKET=' .env 2>/dev/null | cut -d= -f2 || true)
|
|
BUCKET=${BUCKET:-messenger-attachments}
|
|
# Производные файлы пересобираются из базовых секретов (детерминированно, кроме соли хеша).
|
|
python3 - "$BUCKET" <<'PY'
|
|
import base64, hashlib, json, os, sys
|
|
b = sys.argv[1]
|
|
def read(name):
|
|
return open("secrets/" + name).read().strip()
|
|
def ident(name, actions):
|
|
return {"name": name, "credentials": [{"accessKey": read("s3_%s_access_key" % name),
|
|
"secretKey": read("s3_%s_secret_key" % name)}], "actions": actions}
|
|
cfg = {"identities": [
|
|
ident("admin", ["Admin", "Read", "List", "Tagging", "Write"]),
|
|
ident("api", ["Read:" + b, "Write:" + b]),
|
|
ident("worker", ["Read:" + b, "Write:" + b]),
|
|
ident("janitor", ["List:" + b, "Write:" + b]),
|
|
]}
|
|
with open("secrets/s3_identities.json", "w") as f:
|
|
json.dump(cfg, f, indent=1)
|
|
|
|
# RabbitMQ: rabbit_password_hashing_sha256 = base64(salt4 + sha256(salt4 + password)).
|
|
def rmq_hash(password):
|
|
salt = os.urandom(4)
|
|
return base64.b64encode(salt + hashlib.sha256(salt + password.encode()).digest()).decode()
|
|
vhost = "messenger"
|
|
users, perms = [], []
|
|
rights = {
|
|
"relay": {"configure": "^$", "write": "^attachments$", "read": "^$"},
|
|
"worker": {"configure": "^$", "write": "^$", "read": r"^attachments\.process\.v1$"},
|
|
"ops": {"configure": "^$", "write": "^attachments$", "read": r"^attachments\.dlq$"},
|
|
}
|
|
for user, r in rights.items():
|
|
password = read("broker_%s_password" % user)
|
|
users.append({"name": user, "password_hash": rmq_hash(password),
|
|
"hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []})
|
|
perms.append(dict(user=user, vhost=vhost, **r))
|
|
with open("secrets/broker_%s_url" % user, "w") as f:
|
|
f.write("amqp://%s:%s@broker:5672/%s\n" % (user, password, vhost))
|
|
x = lambda name, kind: {"name": name, "vhost": vhost, "type": kind, "durable": True, "auto_delete": False,
|
|
"internal": False, "arguments": {}}
|
|
definitions = {
|
|
"users": users,
|
|
"vhosts": [{"name": vhost}],
|
|
"permissions": perms,
|
|
"exchanges": [x("attachments", "direct"), x("attachments.dlx", "fanout")],
|
|
"queues": [
|
|
# Quorum-очередь: durable, сообщения пишутся на диск (Raft-журнал) до publisher confirm.
|
|
# delivery-limit: сообщение, которое 10 раз вернулось без ack (crash-loop worker),
|
|
# уходит в DLQ. at-least-once dead-lettering требует overflow=reject-publish.
|
|
{"name": "attachments.process.v1", "vhost": vhost, "durable": True, "auto_delete": False,
|
|
"arguments": {"x-queue-type": "quorum", "x-delivery-limit": 10,
|
|
"x-dead-letter-exchange": "attachments.dlx",
|
|
"x-dead-letter-strategy": "at-least-once", "x-overflow": "reject-publish",
|
|
"x-max-length": 100000}},
|
|
{"name": "attachments.dlq", "vhost": vhost, "durable": True, "auto_delete": False,
|
|
"arguments": {"x-queue-type": "quorum"}},
|
|
],
|
|
"bindings": [
|
|
{"source": "attachments", "vhost": vhost, "destination": "attachments.process.v1",
|
|
"destination_type": "queue", "routing_key": "attachment.process.v1", "arguments": {}},
|
|
{"source": "attachments.dlx", "vhost": vhost, "destination": "attachments.dlq",
|
|
"destination_type": "queue", "routing_key": "", "arguments": {}},
|
|
],
|
|
}
|
|
if not os.path.exists("secrets/rabbitmq_definitions.json"):
|
|
with open("secrets/rabbitmq_definitions.json", "w") as f:
|
|
json.dump(definitions, f, indent=1)
|
|
print("created secrets/rabbitmq_definitions.json")
|
|
PY
|
|
# Файлы монтируются в контейнеры (uid 70 postgres, uid 10001 app, uid 100 rabbitmq, root в seaweedfs) —
|
|
# нужны на чтение; от других пользователей хоста их закрывает каталог secrets/ с правами 700.
|
|
chmod 644 secrets/*
|
|
if [ ! -f .env ]; then
|
|
cp .env.example .env
|
|
echo "created .env from .env.example"
|
|
fi
|