Собрать стенд лабораторной 5: очередь, worker и идемпотентность pipeline.

This commit is contained in:
Wladislava1
2026-11-12 13:05:02 +03:00
parent 276dbdad8a
commit a1b972f7d9
39 changed files with 5492 additions and 34 deletions
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env python3
"""Создать локальные .env и secrets для стенда. Пароль в консоль не печатается."""
import secrets
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRET_DIR = ROOT / "secrets"
PASSWORD_FILE = SECRET_DIR / "postgres_password"
DATABASE_FILE = SECRET_DIR / "database_url"
CACHE_FILE = SECRET_DIR / "cache_url"
MINIO_USER_FILE = SECRET_DIR / "minio_root_user"
MINIO_PASSWORD_FILE = SECRET_DIR / "minio_root_password"
S3_ACCESS_FILE = SECRET_DIR / "s3_access_key"
S3_SECRET_FILE = SECRET_DIR / "s3_secret_key"
RABBIT_USER_FILE = SECRET_DIR / "rabbit_user"
RABBIT_PASSWORD_FILE = SECRET_DIR / "rabbit_password"
BROKER_FILE = SECRET_DIR / "broker_url"
ENV_FILE = ROOT / ".env"
ENV_EXAMPLE = ROOT / ".env.example"
def main():
files = (PASSWORD_FILE, DATABASE_FILE, CACHE_FILE)
if any(path.exists() for path in files) and not all(path.exists() for path in files):
print("secrets/ заполнены не полностью, ничего не перезаписываю", file=sys.stderr)
return 1
SECRET_DIR.mkdir(exist_ok=True)
if all(path.exists() for path in files):
print("secrets already exist")
else:
password = secrets.token_hex(24)
PASSWORD_FILE.write_text(password + "\n", encoding="utf-8")
DATABASE_FILE.write_text(
"postgresql://messenger:{}@postgres:5432/messenger\n".format(password),
encoding="utf-8",
)
CACHE_FILE.write_text("redis://redis:6379/0\n", encoding="utf-8")
print("secrets created")
s3_files = (MINIO_USER_FILE, MINIO_PASSWORD_FILE, S3_ACCESS_FILE, S3_SECRET_FILE)
if any(path.exists() for path in s3_files) and not all(path.exists() for path in s3_files):
print("secrets/ для MinIO заполнены не полностью, ничего не перезаписываю", file=sys.stderr)
return 1
if all(path.exists() for path in s3_files):
print("s3 secrets already exist")
else:
MINIO_USER_FILE.write_text("labroot", encoding="utf-8")
MINIO_PASSWORD_FILE.write_text(secrets.token_hex(24), encoding="utf-8")
S3_ACCESS_FILE.write_text("api" + secrets.token_hex(8), encoding="utf-8")
S3_SECRET_FILE.write_text(secrets.token_hex(24), encoding="utf-8")
print("s3 secrets created")
broker_files = (RABBIT_USER_FILE, RABBIT_PASSWORD_FILE, BROKER_FILE)
if any(path.exists() for path in broker_files) and not all(path.exists() for path in broker_files):
print("secrets/ для брокера заполнены не полностью, ничего не перезаписываю", file=sys.stderr)
return 1
if all(path.exists() for path in broker_files):
print("broker secrets already exist")
else:
user = "lab"
password = secrets.token_hex(24)
RABBIT_USER_FILE.write_text(user, encoding="utf-8")
RABBIT_PASSWORD_FILE.write_text(password, encoding="utf-8")
BROKER_FILE.write_text(
"amqp://{}:{}@rabbitmq:5672/%2F\n".format(user, password),
encoding="utf-8",
)
print("broker secrets created")
if ENV_FILE.exists():
print(".env already exists")
else:
ENV_FILE.write_text(ENV_EXAMPLE.read_text(encoding="utf-8"), encoding="utf-8")
print(".env created")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
# Идемпотентно создаёт приватный bucket и отдельного пользователя API.
# Корневой пароль остаётся только у этого контейнера и у MinIO.
set -eu
ROOT_USER=$(tr -d '\r\n' < /run/secrets/minio_root_user)
ROOT_PW=$(tr -d '\r\n' < /run/secrets/minio_root_password)
ACCESS=$(tr -d '\r\n' < /run/secrets/s3_access_key)
SECRET=$(tr -d '\r\n' < /run/secrets/s3_secret_key)
attempt=0
until mc alias set local http://minio:9000 "$ROOT_USER" "$ROOT_PW" >/dev/null; do
attempt=$((attempt + 1))
if [ "$attempt" -gt 30 ]; then
echo "minio alias failed" >&2
exit 1
fi
sleep 1
done
mc mb --ignore-existing local/messenger-attachments >/dev/null
mc anonymous set none local/messenger-attachments >/dev/null
if ! mc admin user info local "$ACCESS" >/dev/null 2>&1; then
mc admin user add local "$ACCESS" "$SECRET" >/dev/null
fi
mc admin policy detach local messenger-api --user "$ACCESS" >/dev/null 2>&1 || true
mc admin policy rm local messenger-api >/dev/null 2>&1 || true
mc admin policy create local messenger-api /init/policy.json >/dev/null
mc admin policy attach local messenger-api --user "$ACCESS" >/dev/null
echo "minio bucket ready"
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
# Пароль читается из secret и попадает только в файл конфигурации внутри контейнера.
set -eu
user=$(tr -d '\r\n' < /run/secrets/rabbit_user)
pass=$(tr -d '\r\n' < /run/secrets/rabbit_password)
mkdir -p /tmp/rabbit-conf
printf 'default_user = %s\ndefault_pass = %s\nvm_memory_high_watermark.absolute = 768MB\n' "$user" "$pass" > /tmp/rabbit-conf/rabbitmq.conf
chmod 644 /tmp/rabbit-conf/rabbitmq.conf
export RABBITMQ_CONFIG_FILE=/tmp/rabbit-conf/rabbitmq
exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# Рестарт API и всех хранилищ со состоянием. Volume не удаляем, миграции заново не заливаем.
# Балансировщик без своего состояния не трогаем: он продолжает слушать тот же порт.
docker compose --env-file .env restart postgres redis minio rabbitmq
docker compose --env-file .env up -d --wait --no-deps postgres redis minio rabbitmq
docker compose --env-file .env restart api
+23
View File
@@ -0,0 +1,23 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?}"
found=""
for id in $(docker ps -q \
--filter label=com.docker.compose.project=mtusi-lab5 \
--filter label=com.docker.compose.service=api); do
host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
short=$(printf %s "$id" | cut -c1-12)
if [ "$host" = "$TARGET_INSTANCE_ID" ] || [ "$short" = "$TARGET_INSTANCE_ID" ]; then
found=$id
break
fi
done
if [ -z "$found" ]; then
echo "api replica not found" >&2
exit 1
fi
# SIGTERM, затем процесс остаётся остановленным (restart: unless-stopped).
docker stop --time 5 "$found" >/dev/null
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env python3
"""Ждать GET /health/ready через балансировщик. Без токенов и тел ошибок в выводе."""
import os
import sys
import time
import urllib.error
import urllib.request
URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") + "/health/ready"
def main():
deadline = time.monotonic() + 90
last = "not-started"
while time.monotonic() < deadline:
try:
response = urllib.request.urlopen(URL, timeout=2)
if response.status == 200 and response.headers.get("X-Instance-Id"):
print("ready")
return 0
last = str(response.status)
except (urllib.error.URLError, TimeoutError, OSError) as exc:
last = type(exc).__name__
time.sleep(1)
print("not ready: " + last, file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())