53 changed files with 3711 additions and 29 deletions
+2
View File
@@ -30,3 +30,5 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
venv/
+21
View File
@@ -0,0 +1,21 @@
FROM python:3.11.9-slim
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1
WORKDIR /app
COPY requirements.txt .
RUN pip install -r requirements.txt
COPY main.py .
COPY app ./app
COPY scripts ./scripts
RUN useradd --system --uid 10001 --no-create-home --shell /usr/sbin/nologin appuser
USER 10001
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=10 \
CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2)"]
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "10"]
+29 -29
View File
@@ -1,37 +1,37 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Никифорова Екатерина, БВТ2301
- Целевая оценка: **4** (уровни 3 и 4, трек 5 не заявлен)
- Commit/PR: https://gitea.privcloud.tech/eqktrin/mtusi2026lab6 (PR после публикации)
- Источники, библиотеки, помощь ИИ: FastAPI 0.115, Uvicorn 0.30, PostgreSQL 17.6,
Valkey 8.1.3, Silo (`pgsty/silo`), RabbitMQ 3.13-management-alpine, Nginx 1.27,
psycopg 3.2, boto3 1.43.107, Pillow 12.3.0, pika 1.3.2, Argon2id;
OpenTelemetry SDK 1.27.0, `opentelemetry-exporter-otlp-proto-http` 1.27.0,
`opentelemetry-instrumentation-{fastapi,psycopg,botocore,pika,logging}` 0.48b0;
OTel Collector Contrib 0.105.0, Prometheus 2.54.1, Loki 3.1.1, Tempo 2.6.0,
Grafana 11.2.0. Помощь ИИ при написании кода и отчёта.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
**Требования:** Docker Engine ≥ 24 с Compose v2, Python ≥ 3.9,
PowerShell (Windows) или sh (Linux), `PYTHONUTF8=1` для публичных тестов
на Windows.
## Архитектура и решения
```powershell
git clone https://gitea.privcloud.tech/eqktrin/mtusi2026lab6.git
cd mtusi2026lab6
# секреты: см. README, генерируются локально через init-secrets (PowerShell/sh).
# Проверить наличие secrets/grafana_admin_password, secrets/broker_url,
# secrets/rabbitmq_password и т.д.
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
# дождаться /health/ready у API (и healthy у rabbitmq, которая при первом
# старте на пустом volume инициализирует mnesia до 60 сек)
Invoke-RestMethod http://localhost:8080/health/ready
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
# Публичный smoke (lab5-функциональность, кумулятивно)
$env:PYTHONUTF8 = "1"
python tests/smoke.py --base-url http://localhost:8080
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
# Свои тесты lab6 (нагрузка + 2 инцидента + telemetry-down)
python student-tests/test_lab6.py --base-url http://localhost:8080
View File
+139
View File
@@ -0,0 +1,139 @@
"""lab5: API только принимает staging + пишет job. Ресайз ушёл в worker."""
import hashlib
import io
import logging
import uuid
from PIL import Image, UnidentifiedImageError
from . import config, db, jobs, repo, s3
from .errors import ApiError, bad_request, not_found
log = logging.getLogger("attachments")
ALLOWED_FORMATS = {"PNG": "image/png", "JPEG": "image/jpeg"}
Image.MAX_IMAGE_PIXELS = config.MAX_IMAGE_PIXELS
def validate_filename(filename):
if not isinstance(filename, str) or not (1 <= len(filename) <= 255):
raise bad_request("Некорректное имя файла")
for c in filename:
if c in ("/", "\\", "\x00") or ord(c) < 0x20 or ord(c) == 0x7F:
raise bad_request("Недопустимое имя файла")
return filename
def inspect_image_header(fileobj, total_bytes):
"""Только заголовок: формат и размеры. Полное декодирование — в worker.
load() в API не вызываем — дорого и не нужно для 202.
"""
if total_bytes == 0:
raise bad_request("Пустой файл")
if total_bytes > config.MAX_UPLOAD_BYTES:
raise ApiError(413, "payload_too_large", "Файл больше 10 MiB")
fileobj.seek(0)
try:
with Image.open(fileobj) as img:
fmt = img.format
width, height = img.size
except UnidentifiedImageError:
raise ApiError(415, "unsupported_media_type", "Неизвестный формат изображения")
except Image.DecompressionBombError:
raise ApiError(413, "payload_too_large", "Слишком много пикселей")
finally:
fileobj.seek(0)
if fmt not in ALLOWED_FORMATS:
raise ApiError(415, "unsupported_media_type", "Ожидается PNG или JPEG")
if width <= 0 or height <= 0:
raise ApiError(415, "unsupported_media_type", "Повреждённое изображение")
if width * height > config.MAX_IMAGE_PIXELS:
raise ApiError(413, "payload_too_large", "Слишком много пикселей")
return ALLOWED_FORMATS[fmt]
def upload_to_staging(chat_id, owner_id, filename, fileobj, total_bytes, digest, content_type):
"""POST /attachments: staging в S3 → INSERT attachments(queued) + job + outbox → 202."""
validate_filename(filename)
staging_key = "staging/%s/%s" % (chat_id, uuid.uuid4().hex)
with db.connection() as conn:
if not repo.is_member(conn, chat_id, owner_id):
raise not_found()
# 1. staging в S3 — до INSERT, чтобы не было queued без байтов.
try:
fileobj.seek(0)
s3.put_object(staging_key, fileobj, content_type, content_length=total_bytes)
except Exception as exc:
log.warning("staging PUT failed: %s", type(exc).__name__)
raise ApiError(503, "unavailable", "Хранилище временно недоступно")
# 2. Одна транзакция: attachments + job + outbox.
try:
with db.connection() as conn:
row = repo.insert_attachment_queued(
conn, chat_id, owner_id, filename, content_type,
total_bytes, digest, staging_key)
jobs.create_job_for_attachment(conn, row["id"], config.PIPELINE_VERSION)
except Exception:
# Компенсация: удалить staging, чтобы не оставить orphan.
s3.delete_object(staging_key)
raise
# 3. Публикуем outbox сразу — если брокер жив, latency короче.
# Если упадёт — worker или API-поток подхватит на следующей итерации.
try:
with db.connection() as conn:
jobs.publish_pending_outbox(conn, _broker_publish)
except Exception:
log.warning("inline outbox publish failed, будет повторено")
return row
def _broker_publish(payload):
from . import broker
broker.publish(payload)
def download_original(attachment_id, user_id):
with db.connection() as conn:
row = repo.get_attachment_for_member(conn, attachment_id, user_id)
if row is None:
raise not_found()
if row["status"] in ("queued", "processing"):
raise ApiError(409, "conflict", "Вложение ещё не готово")
if row["status"] == "failed":
raise ApiError(409, "conflict", "Вложение недоступно")
if not row["object_key"]:
raise ApiError(409, "conflict", "Вложение ещё не готово")
try:
body = s3.get_object(row["object_key"])
except Exception as exc:
log.warning("S3 GET original failed: %s", type(exc).__name__)
raise ApiError(503, "unavailable", "Хранилище временно недоступно")
return row, body
def download_variant(attachment_id, user_id, variant_name):
with db.connection() as conn:
row = repo.get_attachment_for_member(conn, attachment_id, user_id)
if row is None:
raise not_found()
if row["status"] in ("queued", "processing"):
raise ApiError(409, "conflict", "Вложение ещё не готово")
if row["status"] == "failed":
raise ApiError(409, "conflict", "Вложение недоступно")
with db.connection() as conn:
v = repo.get_variant(conn, attachment_id, variant_name)
if v is None:
raise not_found()
try:
body = s3.get_object(v["object_key"])
except Exception as exc:
log.warning("S3 GET variant failed: %s", type(exc).__name__)
raise ApiError(503, "unavailable", "Хранилище временно недоступно")
return v, body
+104
View File
@@ -0,0 +1,104 @@
"""RabbitMQ: publish с publisher confirm, consume с ручным ack.
Topology:
exchange: lab5 (direct, durable)
routing_key: attachment.process.v1 → queue attachment.process.v1 (durable)
DLX: lab5.dlx (direct)
routing_key: attachment.failed → queue attachment.failed (durable)
"""
import json
import logging
import threading
import time
import pika
from . import config
log = logging.getLogger("broker")
EXCHANGE = "lab5"
DLX = "lab5.dlx"
QUEUE = "attachment.process.v1"
FAILED_QUEUE = "attachment.failed"
_lock = threading.Lock()
_channel = None
def _connect():
params = pika.URLParameters(config.BROKER_URL)
params.heartbeat = 30
params.blocked_connection_timeout = 30
return pika.BlockingConnection(params)
def get_channel():
"""Один канал на процесс; worker также использует для consume."""
global _channel
if _channel is None or _channel.is_closed:
with _lock:
if _channel is None or _channel.is_closed:
conn = _connect()
ch = conn.channel()
ch.exchange_declare(EXCHANGE, exchange_type="direct", durable=True)
ch.exchange_declare(DLX, exchange_type="direct", durable=True)
ch.queue_declare(
QUEUE, durable=True,
arguments={"x-dead-letter-exchange": DLX,
"x-dead-letter-routing-key": "attachment.failed"})
ch.queue_declare(FAILED_QUEUE, durable=True)
ch.queue_bind(QUEUE, EXCHANGE, routing_key=QUEUE)
ch.queue_bind(FAILED_QUEUE, DLX, routing_key="attachment.failed")
_channel = ch
return _channel
def publish(job_payload: dict) -> None:
"""Publish с publisher confirm + W3C trace context в headers.
Заголовки `traceparent`/`tracestate` создаёт OTel propagator из текущего span.
Worker их извлечёт и создаст child span — сквозная трасса API → broker → worker.
"""
from . import telemetry
ch = get_channel()
ch.confirm_delivery()
body = json.dumps(job_payload, separators=(",", ":")).encode("utf-8")
headers = {}
telemetry.inject_headers(headers)
ch.basic_publish(
exchange=EXCHANGE,
routing_key=QUEUE,
body=body,
properties=pika.BasicProperties(
content_type="application/json",
delivery_mode=2,
message_id=job_payload["job_id"],
headers=headers or None,
),
mandatory=True,
)
def declare_only() -> None:
"""Идемпотентное создание topology (при старте API и worker)."""
get_channel()
def wait_ready(timeout=30) -> bool:
"""Проверка брокера для /health/ready."""
deadline = time.monotonic() + timeout
last = None
while time.monotonic() < deadline:
try:
conn = _connect()
try:
conn.channel().queue_declare(QUEUE, passive=True)
return True
finally:
conn.close()
except Exception as exc:
last = exc
time.sleep(0.5)
log.warning("broker not ready: %s", type(last).__name__ if last else "timeout")
return False
+52
View File
@@ -0,0 +1,52 @@
"""Тонкая обёртка над Redis/Valkey."""
import logging
import threading
import time
import redis
from . import config
log = logging.getLogger("cache")
_client = None
_down_until = 0.0
_lock = threading.Lock()
def _get_client():
global _client
if not config.CACHE_URL:
return None
if _client is None:
with _lock:
if _client is None:
_client = redis.Redis.from_url(
config.CACHE_URL, socket_timeout=config.CACHE_TIMEOUT,
socket_connect_timeout=config.CACHE_TIMEOUT, max_connections=20,
decode_responses=True, retry_on_timeout=False)
return _client
def _call(fn):
global _down_until
client = _get_client()
if client is None or time.monotonic() < _down_until:
return None
try:
return fn(client)
except redis.RedisError as exc:
_down_until = time.monotonic() + config.CACHE_BACKOFF
log.warning("cache unavailable (%s); bypassing for %.0fs", type(exc).__name__, config.CACHE_BACKOFF)
return None
def get(key):
return _call(lambda c: c.get(key))
def set_if_absent(key, value, ttl_ms):
return _call(lambda c: c.set(key, value, nx=True, px=ttl_ms))
def set_always(key, value, ttl_ms):
return _call(lambda c: c.set(key, value, px=ttl_ms))
+72
View File
@@ -0,0 +1,72 @@
"""Конфигурация из переменных окружения. Секреты читаются из файлов (*_FILE)."""
import os
import socket
from pathlib import Path
def _read(name):
path = os.environ.get(name + "_FILE")
if path:
return Path(path).read_text(encoding="utf-8").strip()
return os.environ.get(name) or None
def _int(name, default, low, high):
value = int(os.environ.get(name, default))
if not low <= value <= high:
raise ValueError("%s must be in %d..%d" % (name, low, high))
return value
APP_HOST = os.environ.get("APP_HOST", "0.0.0.0")
APP_PORT = _int("APP_PORT", 8080, 1, 65535)
APP_ENV = os.environ.get("APP_ENV", "lab")
SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600, 1, 30 * 24 * 3600)
SESSION_STORE = os.environ.get("SESSION_STORE", "postgres+cache")
SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 60, 1, 3600)
DATABASE_URL = _read("DATABASE_URL")
CACHE_URL = _read("CACHE_URL") if SESSION_STORE == "postgres+cache" else None
DB_POOL_MIN = _int("DB_POOL_MIN", 1, 0, 50)
DB_POOL_MAX = _int("DB_POOL_MAX", 16, 1, 50)
DB_POOL_TIMEOUT = float(os.environ.get("DB_POOL_TIMEOUT", "3"))
DB_STATEMENT_TIMEOUT_MS = _int("DB_STATEMENT_TIMEOUT_MS", 5000, 100, 60000)
CACHE_TIMEOUT = float(os.environ.get("CACHE_TIMEOUT", "0.25"))
CACHE_BACKOFF = float(os.environ.get("CACHE_BACKOFF", "2"))
ARGON2_MEMORY_KIB = _int("ARGON2_MEMORY_KIB", 19456, 8, 1048576)
ARGON2_TIME_COST = _int("ARGON2_TIME_COST", 2, 1, 10)
MAX_BODY_BYTES = 64 * 1024
THREAD_LIMIT = _int("THREAD_LIMIT", 24, 1, 200)
HASH_CONCURRENCY = _int("HASH_CONCURRENCY", 4, 1, 32)
# lab3
INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname()
IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400, 60, 86400 * 7)
# lab4: S3
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://minio:9000")
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
S3_ACCESS_KEY_ID = _read("S3_ACCESS_KEY_ID")
S3_SECRET_ACCESS_KEY = _read("S3_SECRET_ACCESS_KEY")
S3_CONNECT_TIMEOUT = float(os.environ.get("S3_CONNECT_TIMEOUT", "3"))
S3_READ_TIMEOUT = float(os.environ.get("S3_READ_TIMEOUT", "30"))
MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10485760, 1, 10485760)
MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40000000, 1, 40000000)
S3_ORPHAN_GRACE_SECONDS = _int("S3_ORPHAN_GRACE_SECONDS", 3600, 60, 7 * 24 * 3600)
UPLOAD_CHUNK_BYTES = _int("UPLOAD_CHUNK_BYTES", 256 * 1024, 4096, 4 * 1024 * 1024)
UPLOAD_READ_TIMEOUT_SECONDS = float(os.environ.get("UPLOAD_READ_TIMEOUT", "30"))
# lab5: broker / worker / jobs
BROKER_URL = _read("BROKER_URL")
WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2, 1, 32)
JOB_MAX_ATTEMPTS = _int("JOB_MAX_ATTEMPTS", 5, 1, 100)
JOB_TIMEOUT_SECONDS = _int("JOB_TIMEOUT_SECONDS", 60, 10, 3600)
STAGING_RETENTION_SECONDS = _int("STAGING_RETENTION_SECONDS", 86400, 60, 30 * 24 * 3600)
PIPELINE_VERSION = _int("PIPELINE_VERSION", 1, 1, 100)
OTEL_ENABLED = os.environ.get("OTEL_SDK_DISABLED", "false").lower() != "true"
OTEL_SERVICE_VERSION = os.environ.get("OTEL_SERVICE_VERSION", "1.6.0")
+27
View File
@@ -0,0 +1,27 @@
"""Непрозрачный keyset-курсор (created_at, id)."""
import base64
import json
import uuid
from datetime import datetime, timezone
from .errors import bad_request
def encode(kind, scope, created_at, row_id):
payload = {"k": kind, "s": str(scope), "t": created_at.astimezone(timezone.utc).isoformat(), "i": str(row_id)}
return base64.urlsafe_b64encode(json.dumps(payload, separators=(",", ":")).encode()).rstrip(b"=").decode()
def decode(value, kind, scope):
if not value:
raise bad_request("Некорректный cursor")
try:
raw = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
payload = json.loads(raw.decode("utf-8"))
created_at = datetime.fromisoformat(payload["t"])
row_id = uuid.UUID(payload["i"])
if payload["k"] != kind or payload["s"] != str(scope) or created_at.tzinfo is None:
raise ValueError("scope mismatch")
return created_at, row_id
except (ValueError, KeyError, TypeError, UnicodeDecodeError):
raise bad_request("Некорректный cursor")
+53
View File
@@ -0,0 +1,53 @@
"""Пул соединений PostgreSQL."""
import threading
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolClosed, PoolTimeout
from . import config
TRANSIENT = (psycopg.OperationalError, PoolTimeout, PoolClosed, psycopg.errors.QueryCanceled,
psycopg.errors.LockNotAvailable)
_pool = None
_lock = threading.Lock()
def get_pool():
global _pool
if _pool is None:
with _lock:
if _pool is None:
if not config.DATABASE_URL:
raise RuntimeError("DATABASE_URL(_FILE) is not configured")
options = ("-c statement_timeout=%d -c lock_timeout=3000 "
"-c idle_in_transaction_session_timeout=10000" % config.DB_STATEMENT_TIMEOUT_MS)
pool = ConnectionPool(
config.DATABASE_URL,
min_size=config.DB_POOL_MIN, max_size=config.DB_POOL_MAX,
timeout=config.DB_POOL_TIMEOUT, max_lifetime=1800, max_idle=300,
reconnect_timeout=5,
check=ConnectionPool.check_connection,
kwargs={"row_factory": dict_row, "connect_timeout": 3, "options": options,
"application_name": "messenger-api"},
open=False)
pool.open(wait=False)
_pool = pool
return _pool
def connection(timeout=None):
return get_pool().connection(timeout=timeout)
def ping():
with connection(timeout=1.5) as conn:
conn.execute("SELECT 1")
def close():
global _pool
if _pool is not None:
_pool.close(timeout=5)
_pool = None
+20
View File
@@ -0,0 +1,20 @@
class ApiError(Exception):
def __init__(self, status, code, message, headers=None):
super().__init__(message)
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
BASIC_CHALLENGE = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
BEARER_CHALLENGE = {"WWW-Authenticate": 'Bearer realm="messenger"'}
def bad_request(message="Некорректный запрос"):
return ApiError(400, "invalid_request", message)
def unauthorized(challenge):
return ApiError(401, "unauthorized", "Требуется аутентификация", challenge)
def not_found():
return ApiError(404, "not_found", "Объект не найден")
+66
View File
@@ -0,0 +1,66 @@
"""Логика идемпотентности для POST запросов."""
import hashlib
import json
import re
from .errors import ApiError, bad_request
# Контракт: ASCII [A-Za-z0-9._:-]{1,128}
_IDEM_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
def get_request_hash(data: dict) -> str:
"""Хеш тела, независимый от порядка ключей.
Отсутствующий attachment_ids семантически равен [] — нормализуем,
иначе {"text":"x"} и {"text":"x","attachment_ids":[]} дают разные хеши.
"""
normalized = dict(data)
normalized.setdefault("attachment_ids", [])
return hashlib.sha256(json.dumps(normalized, sort_keys=True).encode("utf-8")).hexdigest()
def check_and_lock(conn, user_id: str, method: str, path: str, key: str, req_hash: str):
if not key:
raise bad_request("Missing Idempotency-Key")
if not _IDEM_KEY_RE.fullmatch(key):
raise bad_request("Invalid Idempotency-Key format")
lock_id = f"{user_id}:{method}:{path}:{key}"
conn.execute("SELECT pg_advisory_xact_lock(hashtext(%s))", (lock_id,))
row = conn.execute("""
SELECT request_hash, status_code, response_body
FROM idempotency_keys
WHERE user_id = %s AND method = %s AND path = %s AND key = %s
""", (user_id, method, path, key)).fetchone()
if row:
if row["request_hash"] != req_hash:
raise ApiError(409, "conflict", "Idempotency-Key conflict: body differs")
if row["status_code"] is not None and row["response_body"] is not None:
return False, json.loads(row["response_body"])
return True, None
return True, None
def save_result(conn, user_id, method, path, key, req_hash, status_code, response_body):
conn.execute("""
INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response_body)
VALUES (%s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (user_id, method, path, key) DO UPDATE
SET status_code = EXCLUDED.status_code, response_body = EXCLUDED.response_body
""", (user_id, method, path, key, req_hash, status_code, json.dumps(response_body)))
def cleanup(conn, ttl_seconds: int):
conn.execute("""
DELETE FROM idempotency_keys
WHERE (user_id, method, path, key) IN (
SELECT user_id, method, path, key
FROM idempotency_keys
WHERE created_at < now() - make_interval(secs => %s)
LIMIT 100
)
""", (ttl_seconds,))
+170
View File
@@ -0,0 +1,170 @@
"""Репозиторий jobs и outbox + реестр failed + replay."""
import json
import logging
import uuid
from datetime import datetime, timezone
from . import config, repo
log = logging.getLogger("jobs")
# --- создание задания (API) ----------------------------------------------
def create_job_for_attachment(conn, attachment_id: uuid.UUID, pipeline_version: int = 1) -> dict:
"""INSERT job + INSERT outbox в одной транзакции.
Вызывается API после того, как staging-объект сохранён и строка attachments
вставлена со status='queued' и staging_key.
"""
job_id = uuid.uuid4()
job = conn.execute(
"""INSERT INTO jobs (job_id, attachment_id, pipeline_version, status, max_attempts)
VALUES (%s, %s, %s, 'queued', %s)
ON CONFLICT (attachment_id, pipeline_version) DO NOTHING
RETURNING job_id, attachment_id, pipeline_version, attempt, type""",
(job_id, attachment_id, pipeline_version, config.JOB_MAX_ATTEMPTS)).fetchone()
if job is None:
# Уже есть — идемпотентно, но outbox тоже не создаём.
return None
payload = {
"schema_version": 1,
"job_id": str(job["job_id"]),
"type": job["type"],
"attachment_id": str(job["attachment_id"]),
"pipeline_version": job["pipeline_version"],
"attempt": 1,
"created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
}
# INSERT без RETURNING — НЕ вызываем .fetchone().
conn.execute(
"""INSERT INTO outbox (topic, payload) VALUES (%s, %s::jsonb)""",
(job["type"], json.dumps(payload)))
return payload
# --- outbox publisher -----------------------------------------------------
def publish_pending_outbox(conn, publish_fn) -> int:
"""Публикует неопубликованные outbox-записи. Возвращает число опубликованных.
Ошибки не роняют цикл — запись помечается attempts+1 и last_error.
"""
rows = conn.execute(
"""SELECT id, topic, payload FROM outbox
WHERE published_at IS NULL AND attempts < 20
ORDER BY id LIMIT 50 FOR UPDATE SKIP LOCKED""").fetchall()
published = 0
for row in rows:
try:
publish_fn(row["payload"] if isinstance(row["payload"], dict) else json.loads(row["payload"]))
except Exception as exc:
conn.execute(
"""UPDATE outbox SET attempts = attempts + 1, last_error = %s WHERE id = %s""",
(type(exc).__name__, row["id"]))
log.warning("outbox %s publish failed: %s", row["id"], type(exc).__name__)
break # стоп — брокер, возможно, недоступен
conn.execute(
"""UPDATE outbox SET published_at = clock_timestamp() WHERE id = %s""",
(row["id"],))
published += 1
return published
# --- worker: claim, processing, retry, failed ----------------------------
def claim_job(conn, job_id: uuid.UUID) -> dict:
"""Атомарно переводит queued → processing, выдаёт lease.
Возвращает строку job, если claim успешен (attempt++). Иначе None —
значит, задачу уже взял другой worker или она в другом статусе.
"""
return conn.execute(
"""UPDATE jobs
SET status = 'processing',
attempt = attempt + 1,
lease_until = clock_timestamp() + make_interval(secs => %s),
updated_at = clock_timestamp()
WHERE job_id = %s
AND status = 'queued'
AND (next_attempt_at IS NULL OR next_attempt_at <= clock_timestamp())
RETURNING job_id, attachment_id, pipeline_version, attempt, max_attempts, type""",
(config.JOB_TIMEOUT_SECONDS, job_id)).fetchone()
def mark_ready(conn, job_id: uuid.UUID) -> None:
conn.execute(
"""UPDATE jobs SET status = 'ready', error_code = NULL, lease_until = NULL,
updated_at = clock_timestamp()
WHERE job_id = %s""", (job_id,))
def mark_retry(conn, job_id: uuid.UUID, error_code: str, backoff_seconds: float) -> None:
conn.execute(
"""UPDATE jobs
SET status = 'queued',
lease_until = NULL,
next_attempt_at = clock_timestamp() + make_interval(secs => %s),
error_code = %s,
updated_at = clock_timestamp()
WHERE job_id = %s""",
(backoff_seconds, error_code, job_id))
def mark_failed(conn, job_id: uuid.UUID, error_code: str) -> None:
conn.execute(
"""UPDATE jobs SET status = 'failed', error_code = %s, lease_until = NULL,
updated_at = clock_timestamp()
WHERE job_id = %s""", (error_code, job_id))
def find_expired_leases(conn, limit: int = 50):
"""Возвращает job, у которых истёк lease — зависший processing."""
return conn.execute(
"""SELECT job_id, attachment_id, attempt, max_attempts
FROM jobs
WHERE status = 'processing' AND lease_until < clock_timestamp()
ORDER BY lease_until LIMIT %s
FOR UPDATE SKIP LOCKED""", (limit,)).fetchall()
def requeue_expired(conn, row) -> str:
"""Expired lease: attempt уже инкрементирован при claim. Если попытки исчерпаны — failed,
иначе — queued с backoff."""
if row["attempt"] >= row["max_attempts"]:
mark_failed(conn, row["job_id"], "worker_timeout")
return "failed"
backoff = min(60.0, 2.0 ** row["attempt"]) * (0.5 + 0.5 * (uuid.uuid4().int & 0xFFFF) / 0xFFFF)
mark_retry(conn, row["job_id"], "worker_timeout", backoff)
return "queued"
# --- operator replay ------------------------------------------------------
def replay_failed(conn, job_id: uuid.UUID) -> dict:
"""Операторский replay: failed → queued, attempt=0, INSERT outbox.
Не перезапускает успешные задачи. Ошибки не глушит.
"""
row = conn.execute(
"""UPDATE jobs
SET status = 'queued', attempt = 0, error_code = NULL,
lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp()
WHERE job_id = %s AND status = 'failed'
RETURNING job_id, attachment_id, pipeline_version, type""",
(job_id,)).fetchone()
if row is None:
return None
payload = {
"schema_version": 1,
"job_id": str(row["job_id"]),
"type": row["type"],
"attachment_id": str(row["attachment_id"]),
"pipeline_version": row["pipeline_version"],
"attempt": 1,
"created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
}
conn.execute("INSERT INTO outbox (topic, payload) VALUES (%s, %s::jsonb)",
(row["type"], json.dumps(payload)))
return payload
+61
View File
@@ -0,0 +1,61 @@
"""Версионируемые миграции."""
import hashlib
import logging
import sys
import time
from pathlib import Path
import psycopg
from . import config
LOCK_ID = 727_002_001
log = logging.getLogger("migrate")
def migration_files():
return sorted((Path(__file__).parent / "migrations").glob("[0-9][0-9][0-9][0-9]_*.sql"))
def apply_all(conninfo):
files = migration_files()
with psycopg.connect(conninfo, connect_timeout=3) as conn:
conn.execute("SELECT pg_advisory_xact_lock(%s)", (LOCK_ID,))
conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations (
version text PRIMARY KEY,
checksum text NOT NULL,
applied_at timestamptz NOT NULL DEFAULT now())""")
applied = dict(conn.execute("SELECT version, checksum FROM schema_migrations").fetchall())
done = []
for path in files:
sql = path.read_text(encoding="utf-8")
checksum = hashlib.sha256(sql.encode()).hexdigest()
if path.stem in applied:
if applied[path.stem] != checksum:
raise RuntimeError("migration %s was modified after being applied" % path.stem)
continue
conn.execute(sql)
conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)", (path.stem, checksum))
done.append(path.stem)
return done
def main():
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")
if not config.DATABASE_URL:
sys.exit("DATABASE_URL(_FILE) is not configured")
deadline = time.monotonic() + 60
while True:
try:
done = apply_all(config.DATABASE_URL)
break
except psycopg.OperationalError as exc:
if time.monotonic() > deadline:
raise
log.info("database not ready (%s), retrying", type(exc).__name__)
time.sleep(1)
log.info("applied: %s", ", ".join(done) if done else "nothing to do (schema is up to date)")
if __name__ == "__main__":
main()
+39
View File
@@ -0,0 +1,39 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL,
display_name text NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT users_username_key UNIQUE (username),
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_display_name_length CHECK (char_length(display_name) BETWEEN 1 AND 100)
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_by uuid NOT NULL REFERENCES users (id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT chats_title_length CHECK (char_length(title) BETWEEN 1 AND 100)
);
CREATE INDEX chats_created_at_id_idx ON chats (created_at, id);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users (id),
added_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL,
sender_id uuid NOT NULL,
text text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 1 AND 4000),
CONSTRAINT messages_sender_is_member FOREIGN KEY (chat_id, sender_id)
REFERENCES chat_members (chat_id, user_id)
);
CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id);
+10
View File
@@ -0,0 +1,10 @@
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users (id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
CONSTRAINT sessions_hash_length CHECK (octet_length(token_hash) = 32),
CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at)
);
CREATE INDEX sessions_expires_at_idx ON sessions (expires_at);
CREATE INDEX sessions_user_idx ON sessions (user_id);
+13
View File
@@ -0,0 +1,13 @@
CREATE TABLE idempotency_keys (
user_id uuid NOT NULL REFERENCES users(id),
method text NOT NULL,
path text NOT NULL,
key text NOT NULL,
request_hash text NOT NULL,
status_code int,
response_body text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (user_id, method, path, key)
);
CREATE INDEX idempotency_keys_created_at_idx ON idempotency_keys (created_at);
+36
View File
@@ -0,0 +1,36 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users (id),
filename text NOT NULL,
content_type text NOT NULL,
size_bytes bigint NOT NULL,
sha256 char(64) NOT NULL,
object_key text NOT NULL,
status text NOT NULL,
error_code text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT attachments_filename_length CHECK (char_length(filename) BETWEEN 1 AND 255),
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size_positive CHECK (size_bytes > 0 AND size_bytes <= 10485760),
CONSTRAINT attachments_sha256_format CHECK (sha256 ~ '^[a-f0-9]{64}$'),
CONSTRAINT attachments_object_key_unique UNIQUE (object_key),
CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready', 'failed')),
CONSTRAINT attachments_error_code CHECK (
(status = 'ready' AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL) OR
(status = 'pending')
)
);
CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at);
CREATE INDEX attachments_pending_idx ON attachments (created_at) WHERE status = 'pending';
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE RESTRICT,
position int NOT NULL,
PRIMARY KEY (message_id, attachment_id),
CONSTRAINT message_attachments_position CHECK (position >= 0)
);
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
@@ -0,0 +1,5 @@
-- lab4: разрешаем пустой text, если есть вложения. Проверка «text или вложения»
-- делается в приложении, а БД гарантирует только верхнюю границу.
ALTER TABLE messages DROP CONSTRAINT messages_text_length;
ALTER TABLE messages ADD CONSTRAINT messages_text_length
CHECK (char_length(text) BETWEEN 0 AND 4000);
@@ -0,0 +1,75 @@
-- lab5: статусы queued/processing, variants, jobs, outbox, staging_key.
-- 1. Расширяем статусы attachment (было pending/ready/failed).
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
ALTER TABLE attachments ADD CONSTRAINT attachments_status
CHECK (status IN ('queued', 'processing', 'ready', 'failed'));
-- 2. Существующие pending → queued (совместимость с lab4).
UPDATE attachments SET status = 'queued' WHERE status = 'pending';
-- 3. Разделяем staging и финальный объект.
-- object_key = финальный путь (ставит worker после копирования).
-- staging_key = куда API положил байты при upload.
ALTER TABLE attachments ADD COLUMN staging_key text;
ALTER TABLE attachments ALTER COLUMN object_key DROP NOT NULL;
ALTER TABLE attachments ADD CONSTRAINT attachments_ready_has_object
CHECK (status <> 'ready' OR object_key IS NOT NULL);
-- 4. Варианты (thumbnail, crop, optimized).
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
name text NOT NULL,
object_key text NOT NULL,
content_type text NOT NULL,
size_bytes bigint NOT NULL,
width int NOT NULL,
height int NOT NULL,
sha256 char(64) NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (attachment_id, name),
CONSTRAINT attachment_variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')),
CONSTRAINT attachment_variants_type CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')),
CONSTRAINT attachment_variants_size CHECK (size_bytes > 0),
CONSTRAINT attachment_variants_dims CHECK (width > 0 AND height > 0),
CONSTRAINT attachment_variants_sha CHECK (sha256 ~ '^[a-f0-9]{64}$')
);
-- 5. Jobs: одна логическая работа на (attachment_id, pipeline_version).
-- job_id — стабильный идентификатор; attempt растёт при retry.
-- lease_until защищает от двух воркеров на одной задаче.
CREATE TABLE jobs (
job_id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
pipeline_version int NOT NULL DEFAULT 1,
type text NOT NULL DEFAULT 'attachment.process.v1',
status text NOT NULL DEFAULT 'queued',
attempt int NOT NULL DEFAULT 0,
max_attempts int NOT NULL DEFAULT 5,
lease_until timestamptz,
next_attempt_at timestamptz,
error_code text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT jobs_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
CONSTRAINT jobs_type CHECK (type = 'attachment.process.v1'),
CONSTRAINT jobs_attempt CHECK (attempt >= 0),
CONSTRAINT jobs_max_att CHECK (max_attempts BETWEEN 1 AND 100),
CONSTRAINT jobs_pipeline CHECK (pipeline_version >= 1),
UNIQUE (attachment_id, pipeline_version)
);
CREATE INDEX jobs_claim_idx ON jobs (status, next_attempt_at) WHERE status = 'queued';
CREATE INDEX jobs_lease_idx ON jobs (lease_until) WHERE status = 'processing';
-- 6. Transactional outbox: INSERT в той же транзакции, что и jobs.
-- Отдельный publisher читает непубликованные и подтверждает.
CREATE TABLE outbox (
id bigserial PRIMARY KEY,
topic text NOT NULL,
payload jsonb NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
published_at timestamptz,
attempts int NOT NULL DEFAULT 0,
last_error text
);
CREATE INDEX outbox_pending_idx ON outbox (id) WHERE published_at IS NULL;
@@ -0,0 +1,6 @@
-- lab5: constraint attachments_error_code должен знать про queued/processing.
-- Идемпотентно: DROP IF EXISTS — можно применять повторно.
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_error_code;
ALTER TABLE attachments ADD CONSTRAINT attachments_error_code CHECK (
(status = 'failed') = (error_code IS NOT NULL)
);
+99
View File
@@ -0,0 +1,99 @@
"""Обработка изображений: thumbnail 256×256, crop 128×128, optimized WebP."""
import hashlib
import io
import logging
from PIL import Image, ImageOps
from . import config
log = logging.getLogger("pipeline")
PIPELINE_VERSION = 1
Image.MAX_IMAGE_PIXELS = config.MAX_IMAGE_PIXELS
THUMB_MAX = (256, 256)
CROP_SIDE = 128
OPT_MAX = (1280, 1280)
OPT_QUALITY = 82
JPEG_QUALITY = 85
def _open_normalized(data: bytes) -> Image.Image:
"""Открывает, применяет EXIF orientation, приводит к RGB с белым фоном для alpha."""
img = Image.open(io.BytesIO(data))
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
background = Image.new("RGB", img.size, (255, 255, 255))
img_rgba = img.convert("RGBA")
background.paste(img_rgba, mask=img_rgba.split()[-1])
img = background
elif img.mode != "RGB":
img = img.convert("RGB")
img.load()
return img
def _encode_jpeg(img: Image.Image) -> bytes:
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=JPEG_QUALITY, optimize=True, progressive=True)
return buf.getvalue()
def _encode_webp(img: Image.Image) -> bytes:
buf = io.BytesIO()
img.save(buf, format="WEBP", quality=OPT_QUALITY, method=4)
return buf.getvalue()
def _make_thumbnail(img: Image.Image):
thumb = img.copy()
thumb.thumbnail(THUMB_MAX, Image.Resampling.LANCZOS)
if thumb.width < 1 or thumb.height < 1:
raise ValueError("thumbnail degenerate")
return _encode_jpeg(thumb), thumb.width, thumb.height
def _make_crop(img: Image.Image):
side = CROP_SIDE
# ImageOps.fit центрирует и ресайзит; для маленьких исходников увеличивает — разрешено.
crop = ImageOps.fit(img, (side, side), Image.Resampling.LANCZOS)
return _encode_jpeg(crop), side, side
def _make_optimized(img: Image.Image):
opt = img.copy()
opt.thumbnail(OPT_MAX, Image.Resampling.LANCZOS)
return _encode_webp(opt), opt.width, opt.height
def process(data: bytes):
"""Возвращает список вариантов: [{name, content_type, bytes, width, height, sha256, size_bytes}]."""
img = _open_normalized(data)
if img.width * img.height > config.MAX_IMAGE_PIXELS:
raise ValueError("too many pixels")
results = []
thumb_bytes, tw, th = _make_thumbnail(img)
results.append({
"name": "thumbnail", "content_type": "image/jpeg",
"bytes": thumb_bytes, "width": tw, "height": th,
"sha256": hashlib.sha256(thumb_bytes).hexdigest(), "size_bytes": len(thumb_bytes),
})
crop_bytes, cw, ch = _make_crop(img)
results.append({
"name": "crop", "content_type": "image/jpeg",
"bytes": crop_bytes, "width": cw, "height": ch,
"sha256": hashlib.sha256(crop_bytes).hexdigest(), "size_bytes": len(crop_bytes),
})
opt_bytes, ow, oh = _make_optimized(img)
results.append({
"name": "optimized", "content_type": "image/webp",
"bytes": opt_bytes, "width": ow, "height": oh,
"sha256": hashlib.sha256(opt_bytes).hexdigest(), "size_bytes": len(opt_bytes),
})
return results
+205
View File
@@ -0,0 +1,205 @@
"""Доступ к данным. Добавлены queued-статусы, variants, json с variants."""
import uuid
from . import cursor
from .errors import bad_request, not_found
from .sessions import fmt_ts
# ---------- пользователи / чаты (без изменений) ----------
def username_exists(conn, username):
return conn.execute("SELECT 1 FROM users WHERE username = %s", (username,)).fetchone() is not None
def insert_user(conn, username, display_name, password_hash):
return conn.execute(
"""INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s)
RETURNING id, username, display_name, created_at""",
(uuid.uuid4(), username, display_name, password_hash)).fetchone()
def user_for_login(conn, username):
return conn.execute(
"SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = %s",
(username,)).fetchone()
def update_password_hash(conn, user_id, new_hash):
conn.execute("UPDATE users SET password_hash = %s WHERE id = %s", (new_hash, user_id))
def chat_json(row):
return {"id": str(row["id"]), "title": row["title"],
"member_ids": [str(x) for x in row["member_ids"]],
"created_at": fmt_ts(row["created_at"])}
def create_chat(conn, creator_id, title, member_ids, check_members=True):
if check_members:
found = conn.execute("SELECT count(*) AS n FROM users WHERE id = ANY(%s)",
(member_ids,)).fetchone()["n"]
if found != len(member_ids):
raise bad_request("Неизвестный участник чата")
chat = conn.execute(
"INSERT INTO chats (id, title, created_by) VALUES (%s, %s, %s) RETURNING id, title, created_at",
(uuid.uuid4(), title, creator_id)).fetchone()
conn.execute("INSERT INTO chat_members (chat_id, user_id) SELECT %s, unnest(%s::uuid[])",
(chat["id"], member_ids))
chat["member_ids"] = sorted(member_ids)
return chat
def get_chat(conn, chat_id, user_id):
return conn.execute(
"""SELECT c.id, c.title, c.created_at,
ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids
FROM chats c
WHERE c.id = %s AND EXISTS (SELECT 1 FROM chat_members me WHERE me.chat_id = c.id AND me.user_id = %s)""",
(chat_id, user_id)).fetchone()
def is_member(conn, chat_id, user_id):
return conn.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, user_id)).fetchone() is not None
def list_chats(conn, user_id, after, limit):
base = """SELECT c.id, c.title, c.created_at,
ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids
FROM chat_members me JOIN chats c ON c.id = me.chat_id
WHERE me.user_id = %s"""
params = [user_id]
if after:
base += " AND (c.created_at, c.id) > (%s, %s)"
params += list(after)
rows = conn.execute(base + " ORDER BY c.created_at, c.id LIMIT %s", params + [limit + 1]).fetchall()
page, more = rows[:limit], len(rows) > limit
nxt = cursor.encode("chats", user_id, page[-1]["created_at"], page[-1]["id"]) if more else None
return [chat_json(r) for r in page], nxt
# ---------- сообщения (без изменений логики) ----------
def _message_json(row):
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]),
"text": row["text"], "created_at": fmt_ts(row["created_at"]),
"attachment_ids": [str(x) for x in (row.get("attachment_ids") or [])]}
def message_json(row):
return _message_json(row)
def insert_message(conn, chat_id, sender_id, text, attachment_ids=None):
attachment_ids = attachment_ids or []
if attachment_ids:
rows = conn.execute(
"""SELECT id, status FROM attachments WHERE id = ANY(%s) AND chat_id = %s""",
(attachment_ids, chat_id)).fetchall()
by_id = {r["id"]: r for r in rows}
if len(by_id) != len(attachment_ids):
raise not_found()
for att_id in attachment_ids:
if by_id[att_id]["status"] == "failed":
raise bad_request("Нельзя прикрепить failed-вложение")
# queued/processing разрешены с lab5 — можно ссылаться до готовности.
row = conn.execute(
"""INSERT INTO messages (id, chat_id, sender_id, text) VALUES (%s, %s, %s, %s)
RETURNING id, chat_id, sender_id, text, created_at""",
(uuid.uuid4(), chat_id, sender_id, text)).fetchone()
if attachment_ids:
conn.execute(
"""INSERT INTO message_attachments (message_id, attachment_id, position)
SELECT %s, unnest(%s::uuid[]), generate_subscripts(%s::uuid[], 1) - 1""",
(row["id"], attachment_ids, attachment_ids))
row["attachment_ids"] = attachment_ids
return row
def list_messages(conn, chat_id, after, limit):
sql = """SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(SELECT ma.attachment_id FROM message_attachments ma
WHERE ma.message_id = m.id ORDER BY ma.position) AS attachment_ids
FROM messages m WHERE m.chat_id = %s"""
params = [chat_id]
if after:
sql += " AND (m.created_at, m.id) > (%s, %s)"
params += list(after)
rows = conn.execute(sql + " ORDER BY m.created_at, m.id LIMIT %s", params + [limit + 1]).fetchall()
page, more = rows[:limit], len(rows) > limit
nxt = cursor.encode("messages", chat_id, page[-1]["created_at"], page[-1]["id"]) if more else None
return [_message_json(r) for r in page], nxt
# ---------- вложения lab5 ----------
def insert_attachment_queued(conn, chat_id, owner_id, filename, content_type,
size_bytes, sha256, staging_key):
return conn.execute(
"""INSERT INTO attachments
(id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
staging_key, status)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'queued')
RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
status, error_code, created_at, updated_at""",
(uuid.uuid4(), chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key)
).fetchone()
def mark_attachment_failed(conn, attachment_id, error_code):
conn.execute(
"""UPDATE attachments
SET status='failed', error_code=%s, updated_at=clock_timestamp()
WHERE id=%s AND status IN ('queued','processing')""",
(error_code, attachment_id))
def get_attachment_for_member(conn, attachment_id, user_id):
return conn.execute(
"""SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256,
a.object_key, a.staging_key, a.status, a.error_code, a.created_at, a.updated_at
FROM attachments a
WHERE a.id = %s
AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = a.chat_id AND m.user_id = %s)""",
(attachment_id, user_id)).fetchone()
def get_variant(conn, attachment_id, variant_name):
return conn.execute(
"""SELECT attachment_id, name, object_key, content_type, size_bytes, width, height, sha256
FROM attachment_variants WHERE attachment_id = %s AND name = %s""",
(attachment_id, variant_name)).fetchone()
def list_variants(conn, attachment_id):
return conn.execute(
"""SELECT name, content_type, size_bytes, width, height, sha256
FROM attachment_variants WHERE attachment_id = %s ORDER BY name""",
(attachment_id,)).fetchall()
def attachment_json(row, variants=None):
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"],
"status": row["status"],
"variants": [
{
"name": v["name"],
"content_type": v["content_type"],
"size_bytes": v["size_bytes"],
"width": v["width"],
"height": v["height"],
"sha256": v["sha256"],
} for v in (variants or [])
],
"error_code": row["error_code"],
"created_at": fmt_ts(row["created_at"]),
"updated_at": fmt_ts(row["updated_at"]),
}
+74
View File
@@ -0,0 +1,74 @@
"""S3-совместимое хранилище (Silo/MinIO): потоковые put/get, копирование, удаление."""
import logging
import threading
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from . import config
log = logging.getLogger("s3")
_client = None
_lock = threading.Lock()
def _get_client():
global _client
if _client is None:
with _lock:
if _client is None:
_client = boto3.client(
"s3",
endpoint_url=config.S3_ENDPOINT,
aws_access_key_id=config.S3_ACCESS_KEY_ID,
aws_secret_access_key=config.S3_SECRET_ACCESS_KEY,
region_name=config.S3_REGION,
config=Config(
signature_version="s3v4",
s3={"addressing_style": "path"},
retries={"max_attempts": 3, "mode": "standard"},
connect_timeout=config.S3_CONNECT_TIMEOUT,
read_timeout=config.S3_READ_TIMEOUT,
),
)
return _client
def ensure_bucket():
client = _get_client()
try:
client.head_bucket(Bucket=config.S3_BUCKET)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in ("404", "NoSuchBucket"):
client.create_bucket(Bucket=config.S3_BUCKET)
else:
raise
def put_object(key: str, body, content_type: str, content_length: int):
client = _get_client()
client.put_object(
Bucket=config.S3_BUCKET, Key=key, Body=body,
ContentType=content_type, ContentLength=content_length,
)
def get_object(key: str):
"""Возвращает StreamingBody."""
client = _get_client()
resp = client.get_object(Bucket=config.S3_BUCKET, Key=key)
return resp["Body"]
def delete_object(key: str):
client = _get_client()
try:
client.delete_object(Bucket=config.S3_BUCKET, Key=key)
except (ClientError, BotoCoreError):
log.warning("failed to delete object %s", key)
def ping():
client = _get_client()
client.head_bucket(Bucket=config.S3_BUCKET)
+72
View File
@@ -0,0 +1,72 @@
"""Пароли (Argon2id), токены, разбор заголовка Authorization."""
import base64
import binascii
import hashlib
import re
import secrets
import threading
from argon2 import PasswordHasher, Type
from argon2.exceptions import InvalidHashError, VerificationError
from . import config
_hasher = PasswordHasher(time_cost=config.ARGON2_TIME_COST, memory_cost=config.ARGON2_MEMORY_KIB,
parallelism=1, hash_len=32, salt_len=16, type=Type.ID)
_DUMMY_HASH = _hasher.hash("dummy-password-for-constant-time")
_hash_slots = threading.BoundedSemaphore(config.HASH_CONCURRENCY)
_TOKEN_RE = re.compile(r"[A-Za-z0-9._~+/=-]{16,512}")
def hash_password(password):
with _hash_slots:
return _hasher.hash(password)
def verify_password(stored_hash, password):
with _hash_slots:
try:
_hasher.verify(stored_hash, password)
except (VerificationError, InvalidHashError):
return False, None
return True, (_hasher.hash(password) if _hasher.check_needs_rehash(stored_hash) else None)
def burn_verify(password):
with _hash_slots:
try:
_hasher.verify(_DUMMY_HASH, password)
except VerificationError:
pass
def new_token():
return secrets.token_urlsafe(32)
def token_hash(token):
return hashlib.sha256(token.encode("ascii")).digest()
def parse_basic(header):
if not header:
return None
scheme, _, rest = header.partition(" ")
if scheme.lower() != "basic":
return None
try:
text = base64.b64decode(rest.strip(), validate=True).decode("utf-8")
except (binascii.Error, UnicodeDecodeError, ValueError):
return None
username, sep, password = text.partition(":")
return (username, password) if sep else None
def parse_bearer(header):
if not header:
return None
scheme, _, rest = header.partition(" ")
token = rest.strip()
if scheme.lower() != "bearer" or not _TOKEN_RE.fullmatch(token):
return None
return token
+81
View File
@@ -0,0 +1,81 @@
"""Сессии: PostgreSQL — источник истины, Redis/Valkey — read-through кэш."""
import json
import time
from datetime import timezone
from . import cache, config, db, security
from .errors import ApiError
TOMBSTONE = "revoked"
_KEY = "sess:v1:"
def fmt_ts(value):
return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
def user_json(row):
return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"],
"created_at": fmt_ts(row["created_at"])}
def _ttl_ms(expires_epoch):
remaining = expires_epoch - time.time()
return int(min(config.SESSION_CACHE_TTL_SECONDS, remaining) * 1000) if remaining > 0 else 0
def create(conn, user_row):
token = security.new_token()
row = conn.execute(
"""INSERT INTO sessions (token_hash, user_id, expires_at)
VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s))
RETURNING expires_at""",
(security.token_hash(token), user_row["id"], config.SESSION_TTL_SECONDS)).fetchone()
conn.execute("""DELETE FROM sessions WHERE token_hash IN (
SELECT token_hash FROM sessions WHERE expires_at < now() - interval '1 hour' LIMIT 100)""")
return token, row["expires_at"]
def warm(token, user, expires_at):
entry = {"user": user, "exp": expires_at.timestamp()}
ttl = _ttl_ms(entry["exp"])
if ttl > 0:
cache.set_if_absent(_KEY + security.token_hash(token).hex(), json.dumps(entry), ttl)
def lookup(token):
digest = security.token_hash(token)
key = _KEY + digest.hex()
cached = cache.get(key)
if cached == TOMBSTONE:
return None
if cached:
try:
entry = json.loads(cached)
if entry["exp"] > time.time():
return {"user": entry["user"], "digest": digest}
return None
except (ValueError, KeyError, TypeError):
pass
with db.connection() as conn:
row = conn.execute(
"""SELECT u.id, u.username, u.display_name, u.created_at,
extract(epoch FROM s.expires_at) AS exp
FROM sessions s JOIN users u ON u.id = s.user_id
WHERE s.token_hash = %s AND s.expires_at > clock_timestamp()""", (digest,)).fetchone()
if row is None:
return None
user = user_json(row)
ttl = _ttl_ms(float(row["exp"]))
if ttl > 0:
cache.set_if_absent(key, json.dumps({"user": user, "exp": float(row["exp"])}), ttl)
return {"user": user, "digest": digest}
def revoke(digest):
if config.CACHE_URL:
wrote = cache.set_always(_KEY + digest.hex(), TOMBSTONE, config.SESSION_CACHE_TTL_SECONDS * 1000)
if not wrote:
raise ApiError(503, "unavailable", "Сервис временно недоступен")
with db.connection() as conn:
conn.execute("DELETE FROM sessions WHERE token_hash = %s", (digest,))
+98
View File
@@ -0,0 +1,98 @@
"""lab6: общая инициализация OTel для API и worker.
Каждый процесс вызывает init(service_name=...) один раз. После этого:
- get_tracer() / get_meter() — SDK-хендлы;
- inject_headers(dict) / extract_context(dict) — W3C trace context через RabbitMQ.
"""
import logging
import os
from opentelemetry import metrics, trace
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.propagate import inject as _inject, extract as _extract
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource, SERVICE_NAME, SERVICE_VERSION
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry._logs import set_logger_provider
log = logging.getLogger("telemetry")
_initialized = False
def init(service_name: str) -> None:
"""Идемпотентная инициализация. Если OTEL выключен — no-op.
Ресурс: service.name, service.version, deployment.environment.name, service.instance.id.
"""
global _initialized
if _initialized:
return
# OTEL_SDK_DISABLED=true отключает SDK полностью (для теста «telemetry off»).
if os.environ.get("OTEL_SDK_DISABLED", "false").lower() == "true":
log.info("OTel disabled by OTEL_SDK_DISABLED, using no-op providers")
_initialized = True
return
endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318")
version = os.environ.get("OTEL_SERVICE_VERSION", "1.6.0")
instance_id = os.environ.get("HOSTNAME") or os.environ.get("INSTANCE_ID") or "unknown"
resource = Resource.create({
SERVICE_NAME: service_name,
SERVICE_VERSION: version,
"deployment.environment.name": "lab",
"service.instance.id": instance_id,
})
# Traces
tracer_provider = TracerProvider(resource=resource)
tracer_provider.add_span_processor(
BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint + "/v1/traces")))
trace.set_tracer_provider(tracer_provider)
# Metrics — периодический экспорт каждые 10 секунд.
reader = PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=endpoint + "/v1/metrics"),
export_interval_millis=10_000)
meter_provider = MeterProvider(resource=resource, metric_readers=[reader])
metrics.set_meter_provider(meter_provider)
# Logs — OTLP HTTP в Collector → Loki. Формат структурный, JSON.
logger_provider = LoggerProvider(resource=resource)
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint + "/v1/logs")))
set_logger_provider(logger_provider)
# Прикрепляем OTel handler к root logger, чтобы все наши log.info(...)
# уходили в Collector. Существующий StreamHandler остаётся — логи видны и в docker logs.
root = logging.getLogger()
root.addHandler(LoggingHandler(level=logging.INFO, logger_provider=logger_provider))
_initialized = True
log.info("OTel initialized: service=%s endpoint=%s", service_name, endpoint)
def get_tracer(name: str = None):
return trace.get_tracer(name or "messenger")
def get_meter(name: str = None):
return metrics.get_meter(name or "messenger")
def inject_headers(carrier: dict) -> dict:
"""W3C traceparent/tracestate → headers брокера. Level 4: сквозная связь."""
_inject(carrier)
return carrier
def extract_context(carrier: dict):
"""Из headers брокера → OTel Context. Используется в worker."""
return _extract(carrier or {})
+83
View File
@@ -0,0 +1,83 @@
"""Строгая валидация входа."""
import json
import re
import uuid
from .errors import ApiError, bad_request
_UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
_USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
_INT_RE = re.compile(r"[0-9]{1,6}")
def parse_uuid(value):
if isinstance(value, str) and _UUID_RE.fullmatch(value):
return uuid.UUID(value)
return None
def require_uuid(value, name="id"):
parsed = parse_uuid(value)
if parsed is None:
raise bad_request("Некорректный %s" % name)
return parsed
def _reject_constant(name):
raise ValueError("non-finite number")
def _no_duplicates(pairs):
result = {}
for key, value in pairs:
if key in result:
raise ValueError("duplicate key")
result[key] = value
return result
def json_body(content_type, raw, allowed, required):
if content_type.split(";")[0].strip().lower() != "application/json":
raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: application/json")
try:
data = json.loads(raw.decode("utf-8"), parse_constant=_reject_constant, object_pairs_hook=_no_duplicates)
except (ValueError, UnicodeDecodeError, RecursionError):
raise bad_request("Некорректный JSON")
if not isinstance(data, dict) or set(data) - set(allowed) or not set(required) <= set(data):
raise bad_request("Некорректный набор полей")
return data
def text_field(value, name, low, high, allow_blank=False):
if not isinstance(value, str):
raise bad_request("Поле %s должно быть строкой" % name)
if "\x00" in value or not _encodable(value):
raise bad_request("Поле %s содержит недопустимые символы" % name)
if not low <= len(value) <= high:
raise bad_request("Поле %s: длина должна быть %d–%d" % (name, low, high))
if not allow_blank and not value.strip():
raise bad_request("Поле %s не должно состоять только из пробелов" % name)
return value
def _encodable(value):
try:
value.encode("utf-8")
return True
except UnicodeEncodeError:
return False
def username_field(value):
if not isinstance(value, str) or not _USERNAME_RE.fullmatch(value):
raise bad_request("username должен соответствовать [a-z0-9_]{3,32}")
return value
def limit_param(params):
raw = params.get("limit")
if raw is None:
return 50
if not _INT_RE.fullmatch(raw) or not 1 <= int(raw) <= 100:
raise bad_request("limit должен быть целым числом 1–100")
return int(raw)
+373
View File
@@ -0,0 +1,373 @@
"""Worker lab6: consume из RabbitMQ, обработка изображения, OTel-инструментация."""
import hashlib
import json
import logging
import signal
import sys
import threading
import time
import uuid
import pika
from opentelemetry import trace
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor
from opentelemetry.trace import SpanKind
from . import broker, config, db, jobs, pipeline, repo, s3, telemetry
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
logging.getLogger("pika").setLevel(logging.WARNING)
logging.getLogger("opentelemetry").setLevel(logging.WARNING)
log = logging.getLogger("worker")
_stop = threading.Event()
# OTel-хендлы и метрики инициализируются один раз в main().
_tracer = None
_m_jobs = None
_m_duration = None
_m_time_to_ready = None
_m_busy = None
def _init_metrics():
global _tracer, _m_jobs, _m_duration, _m_time_to_ready, _m_busy
if _tracer is not None:
return
_tracer = telemetry.get_tracer("messenger-worker")
meter = telemetry.get_meter("messenger-worker")
_m_jobs = meter.create_counter(
"messenger_jobs_processed_total",
description="Jobs processed, by outcome")
_m_duration = meter.create_histogram(
"messenger_job_processing_duration_seconds",
description="Job processing duration",
unit="s")
_m_time_to_ready = meter.create_histogram(
"messenger_time_to_ready_seconds",
description="Upload accepted -> status ready",
unit="s")
_m_busy = meter.create_up_down_counter(
"messenger_worker_busy",
description="Worker slots in use")
def _on_sigterm(signum, frame):
log.info("SIGTERM received, stopping after current message")
_stop.set()
def _backoff(attempt: int) -> float:
"""Экспоненциальный backoff с jitter: 2^attempt секунд ± 50%."""
base = min(60.0, 2.0 ** min(attempt, 6))
jitter = 0.5 + 0.5 * (uuid.uuid4().int & 0xFFFF) / 0xFFFF
return base * jitter
def _is_permanent(exc: Exception) -> bool:
if isinstance(exc, ValueError):
return True
return False
def _load_attachment(conn, attachment_id: uuid.UUID):
return conn.execute(
"""SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, staging_key, status
FROM attachments WHERE id = %s""", (attachment_id,)).fetchone()
def _already_ready(conn, attachment_id: uuid.UUID) -> bool:
"""Идемпотентность: если ready и есть все три варианта — no-op."""
row = conn.execute(
"""SELECT a.status,
(SELECT count(*) FROM attachment_variants v WHERE v.attachment_id = a.id) AS n
FROM attachments a WHERE a.id = %s""", (attachment_id,)).fetchone()
if row is None:
return False
return row["status"] == "ready" and row["n"] >= 3
def _process_one(conn, job_row) -> None:
"""Один job: скачать staging, обработать, залить финал+варианты, транзакционно записать."""
attachment_id = job_row["attachment_id"]
job_id = job_row["job_id"]
if _already_ready(conn, attachment_id):
jobs.mark_ready(conn, job_id)
log.info("job %s: already ready, ack", job_id)
return
att = _load_attachment(conn, attachment_id)
if att is None:
jobs.mark_failed(conn, job_id, "attachment_missing")
return
if not att["staging_key"]:
jobs.mark_failed(conn, job_id, "staging_missing")
return
# 1. Прочитать staging из S3. BotocoreInstrumentor создаст client span.
body = s3.get_object(att["staging_key"])
try:
data = body.read()
finally:
body.close()
if hashlib.sha256(data).hexdigest() != att["sha256"]:
raise RuntimeError("staging checksum mismatch")
# 2. Обработать (thumbnail, crop, optimized). Не инструментируется отдельно —
# CPU-bound, метрика duration уже покрывает всё.
variants = pipeline.process(data)
# 3. Финализировать: original + варианты в S3.
final_key = "originals/%s/%s" % (att["chat_id"], uuid.uuid4().hex)
s3.put_object(final_key, data, att["content_type"], content_length=len(data))
for v in variants:
v_key = "variants/%s/%s" % (attachment_id, v["name"])
s3.put_object(v_key, v["bytes"], v["content_type"], content_length=len(v["bytes"]))
v["object_key"] = v_key
# 4. Зафиксировать в БД атомарно.
with db.connection() as txn:
check = txn.execute(
"""SELECT status FROM attachments WHERE id = %s FOR UPDATE""",
(attachment_id,)).fetchone()
if check["status"] == "ready":
jobs.mark_ready(txn, job_id)
return
for v in variants:
txn.execute(
"""INSERT INTO attachment_variants
(attachment_id, name, object_key, content_type, size_bytes, width, height, sha256)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (attachment_id, name) DO UPDATE
SET object_key = EXCLUDED.object_key,
content_type = EXCLUDED.content_type,
size_bytes = EXCLUDED.size_bytes,
width = EXCLUDED.width,
height = EXCLUDED.height,
sha256 = EXCLUDED.sha256""",
(attachment_id, v["name"], v["object_key"], v["content_type"],
v["size_bytes"], v["width"], v["height"], v["sha256"]))
txn.execute(
"""UPDATE attachments
SET status = 'ready', object_key = %s, error_code = NULL,
staging_key = NULL, updated_at = clock_timestamp()
WHERE id = %s""",
(final_key, attachment_id))
jobs.mark_ready(txn, job_id)
# 5. После COMMIT — удалить staging (best-effort).
try:
s3.delete_object(att["staging_key"])
except Exception:
log.warning("staging delete failed for %s", att["staging_key"])
def _record_time_to_ready(attachment_id: uuid.UUID) -> None:
"""histogram: from attachments.created_at (upload accepted) to now (ready)."""
try:
with db.connection() as conn:
row = conn.execute(
"""SELECT EXTRACT(EPOCH FROM now() - created_at)::float AS delta
FROM attachments WHERE id = %s""",
(attachment_id,)).fetchone()
if row and row["delta"] is not None:
_m_time_to_ready.record(float(row["delta"]),
{"pipeline.version": "1"})
except Exception:
pass
def _handle_message(ch, method, properties, body):
"""Consumer: span + W3C trace context + ack только после COMMIT."""
_init_metrics()
# W3C trace context из headers. Pika хранит значения как bytes — декодируем.
raw_headers = (properties.headers or {}) if properties else {}
headers = {}
for k, v in raw_headers.items():
if isinstance(v, bytes):
headers[k] = v.decode("utf-8", errors="replace")
else:
headers[k] = v
ctx = telemetry.extract_context(headers)
try:
payload = json.loads(body)
job_id = uuid.UUID(payload["job_id"])
except Exception:
log.error("malformed job payload, drop")
ch.basic_nack(method.delivery_tag, requeue=False)
return
with _tracer.start_as_current_span(
"attachment.process",
context=ctx,
kind=SpanKind.CONSUMER,
attributes={
"messaging.system": "rabbitmq",
"messaging.destination.name": broker.QUEUE,
"messaging.message.id": str(job_id),
"messaging.operation": "process",
"job.id": str(job_id),
"attachment.id": payload.get("attachment_id", ""),
"service.instance.id": config.INSTANCE_ID,
}) as span:
t0 = time.monotonic()
_m_busy.add(1, {"worker": config.INSTANCE_ID})
try:
with db.connection() as conn:
job_row = jobs.claim_job(conn, job_id)
if job_row is None:
span.set_attribute("job.skip", "not_claimable")
ch.basic_ack(method.delivery_tag)
return
if job_row["attempt"] > job_row["max_attempts"]:
with db.connection() as conn:
jobs.mark_failed(conn, job_id, "retry_exhausted")
_m_jobs.add(1, {"outcome": "failed", "job.type": "attachment.process.v1"})
span.set_attribute("job.outcome", "failed")
span.set_attribute("job.error_code", "retry_exhausted")
ch.basic_ack(method.delivery_tag)
return
with db.connection() as conn:
_process_one(conn, job_row)
duration = time.monotonic() - t0
_m_duration.record(duration,
{"outcome": "ready", "job.type": "attachment.process.v1"})
_m_jobs.add(1, {"outcome": "ready", "job.type": "attachment.process.v1"})
_record_time_to_ready(job_row["attachment_id"])
span.set_attribute("job.outcome", "ready")
span.set_attribute("job.attempt", job_row["attempt"])
span.set_attribute("job.duration_seconds", round(duration, 3))
ch.basic_ack(method.delivery_tag)
log.info("job %s: done (attempt %d, %.2fs)", job_id, job_row["attempt"], duration)
except Exception as exc:
span.set_attribute("job.outcome", "error")
span.record_exception(exc)
log.warning("job %s: error %s", job_id, type(exc).__name__)
permanent = _is_permanent(exc)
attempt = 1
max_attempts = config.JOB_MAX_ATTEMPTS
try:
with db.connection() as conn:
row = conn.execute(
"SELECT attempt, max_attempts FROM jobs WHERE job_id = %s",
(job_id,)).fetchone()
if row:
attempt = row["attempt"]
max_attempts = row["max_attempts"]
if permanent or attempt >= max_attempts:
jobs.mark_failed(conn, job_id,
"invalid_image" if permanent else "retry_exhausted")
_m_jobs.add(1, {"outcome": "failed", "job.type": "attachment.process.v1"})
span.set_attribute("job.error_code",
"invalid_image" if permanent else "retry_exhausted")
else:
jobs.mark_retry(conn, job_id, "processing_failed", _backoff(attempt))
_m_jobs.add(1, {"outcome": "retry", "job.type": "attachment.process.v1"})
span.set_attribute("job.error_code", "processing_failed")
span.set_attribute("job.retry_after", round(_backoff(attempt), 1))
finally:
ch.basic_ack(method.delivery_tag)
finally:
_m_busy.add(-1, {"worker": config.INSTANCE_ID})
def _reaper_loop():
"""Возвращает в queued задачи с истёкшим lease."""
while not _stop.wait(30):
try:
with db.connection() as conn:
rows = jobs.find_expired_leases(conn)
for row in rows:
outcome = jobs.requeue_expired(conn, row)
log.info("lease reaper: job %s -> %s", row["job_id"], outcome)
except Exception as exc:
log.warning("reaper error: %s", type(exc).__name__)
def _outbox_loop():
"""Публикует outbox — на случай, если API упал между COMMIT и publish."""
while not _stop.wait(3):
try:
with db.connection() as conn:
jobs.publish_pending_outbox(conn, broker.publish)
except Exception as exc:
log.warning("outbox error: %s", type(exc).__name__)
def main():
signal.signal(signal.SIGTERM, _on_sigterm)
signal.signal(signal.SIGINT, _on_sigterm)
# lab6: OTel. service.name=messenger-worker (отличается от API).
telemetry.init("messenger-worker")
PsycopgInstrumentor().instrument()
BotocoreInstrumentor().instrument()
_init_metrics()
# Ждём broker до 120 секунд — на случай рестарта RabbitMQ.
ready = False
for attempt in range(60):
if _stop.is_set():
return
try:
broker.declare_only()
ready = True
break
except Exception as exc:
log.warning("broker not ready (attempt %d): %s",
attempt + 1, type(exc).__name__)
if _stop.wait(2):
return
if not ready:
log.error("broker unavailable after 120s, exiting")
sys.exit(1)
log.info("worker started, queue=%s concurrency=%d",
broker.QUEUE, config.WORKER_CONCURRENCY)
threading.Thread(target=_reaper_loop, daemon=True).start()
threading.Thread(target=_outbox_loop, daemon=True).start()
# Consume с ручным ack и ограниченным prefetch.
conn = pika.BlockingConnection(pika.URLParameters(config.BROKER_URL))
ch = conn.channel()
ch.basic_qos(prefetch_count=config.WORKER_CONCURRENCY)
ch.basic_consume(queue=broker.QUEUE, on_message_callback=_handle_message, auto_ack=False)
while not _stop.is_set():
try:
conn.process_data_events(time_limit=1)
except pika.exceptions.AMQPConnectionError:
log.warning("broker connection lost, reconnecting in 2s")
time.sleep(2)
try:
conn = pika.BlockingConnection(pika.URLParameters(config.BROKER_URL))
ch = conn.channel()
ch.basic_qos(prefetch_count=config.WORKER_CONCURRENCY)
ch.basic_consume(queue=broker.QUEUE, on_message_callback=_handle_message,
auto_ack=False)
except Exception as exc:
log.warning("reconnect failed: %s", type(exc).__name__)
try:
ch.stop_consuming()
conn.close()
except Exception:
pass
db.close()
log.info("worker stopped")
if __name__ == "__main__":
main()
+334
View File
@@ -0,0 +1,334 @@
name: messenger-lab6
x-app-env: &app-env
APP_HOST: "0.0.0.0"
APP_PORT: "8080"
APP_ENV: ${APP_ENV:-lab}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: ${SESSION_STORE:-postgres+cache}
SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
S3_ENDPOINT: http://minio:9000
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
S3_ORPHAN_GRACE_SECONDS: ${S3_ORPHAN_GRACE_SECONDS:-3600}
BROKER_URL_FILE: /run/secrets/broker_url
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
# lab6: OTEL — общие значения, каждый сервис переопределяет OTEL_SERVICE_NAME
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab,service.version=1.6.0
OTEL_TRACES_SAMPLER: parentbased_traceidratio
OTEL_TRACES_SAMPLER_ARG: "1.0"
OTEL_METRICS_EXPORTER: otlp
OTEL_LOGS_EXPORTER: otlp
OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED: "true"
services:
postgres:
image: postgres:17.6-alpine
restart: unless-stopped
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on",
"-c", "full_page_writes=on", "-c", "max_connections=100"]
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 3s
timeout: 3s
retries: 20
stop_grace_period: 30s
networks: [backend]
mem_limit: 1g
cpus: 1.0
valkey:
image: valkey/valkey:8.1.3-alpine
restart: unless-stopped
secrets: [valkey_password]
command:
- sh
- -c
- >-
exec valkey-server --requirepass "$$(cat /run/secrets/valkey_password)"
--appendonly yes --appendfsync everysec --save ""
--maxmemory 64mb --maxmemory-policy volatile-ttl
volumes:
- valkeydata:/data
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/valkey_password)\" valkey-cli ping | grep -q PONG"]
interval: 3s
timeout: 3s
retries: 20
networks: [backend]
mem_limit: 256m
cpus: 0.5
rabbitmq:
image: rabbitmq:3.13-management-alpine
restart: unless-stopped
secrets: [rabbitmq_password]
entrypoint: ["/bin/sh", "-c"]
command:
- |
export RABBITMQ_DEFAULT_USER=messenger
export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/rabbitmq_password)"
exec docker-entrypoint.sh rabbitmq-server
volumes:
- rabbitmqdata:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 20
start_period: 60s
stop_grace_period: 30s
networks: [backend]
mem_limit: 512m
cpus: 0.5
minio:
image: pgsty/silo:latest
restart: unless-stopped
command: ["server", "/data", "--console-address", ":9001"]
environment:
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
secrets: [minio_root_user, minio_root_password]
volumes:
- miniodata:/data
healthcheck:
test: ["CMD", "silo", "healthcheck", "ready"]
interval: 5s
timeout: 3s
retries: 20
start_period: 10s
networks: [backend]
mem_limit: 512m
cpus: 0.5
minio-init:
image: pgsty/mc:latest
depends_on:
minio: {condition: service_healthy}
secrets: [minio_root_user, minio_root_password, s3_access_key_id, s3_secret_access_key]
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
ROOT_USER=$$(cat /run/secrets/minio_root_user)
ROOT_PASS=$$(cat /run/secrets/minio_root_password)
APP_KEY=$$(cat /run/secrets/s3_access_key_id)
APP_SECRET=$$(cat /run/secrets/s3_secret_access_key)
mc alias set local http://minio:9000 "$$ROOT_USER" "$$ROOT_PASS"
mc mb --ignore-existing --with-lock=false local/messenger-attachments
mc anonymous set none local/messenger-attachments
mc admin user add local "$$APP_KEY" "$$APP_SECRET" || true
cat >/tmp/app-policy.json <<EOF
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListBucket"],
"Resource": [
"arn:aws:s3:::messenger-attachments",
"arn:aws:s3:::messenger-attachments/*"
]
}]
}
EOF
mc admin policy create local app-attachment-only /tmp/app-policy.json || true
mc admin policy attach local app-attachment-only --user "$$APP_KEY" || true
restart: "no"
networks: [backend]
migrate:
build: .
command: ["python", "-m", "app.migrate"]
environment: *app-env
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
depends_on:
postgres: {condition: service_healthy}
restart: "no"
networks: [backend]
api:
build: .
restart: unless-stopped
environment:
<<: *app-env
OTEL_SERVICE_NAME: messenger-api
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
depends_on:
migrate: {condition: service_completed_successfully}
postgres: {condition: service_healthy}
valkey: {condition: service_started}
minio-init: {condition: service_completed_successfully}
rabbitmq: {condition: service_healthy}
otel-collector: {condition: service_started}
read_only: true
tmpfs: ["/tmp", "/dev/shm"]
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request as u; u.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
interval: 5s
timeout: 3s
retries: 20
start_period: 30s
stop_grace_period: 25s
networks: [backend]
deploy:
replicas: 2
mem_limit: 512m
cpus: 1.0
worker:
build: .
command: ["python", "-m", "app.worker"]
healthcheck:
disable: true
restart: unless-stopped
environment:
<<: *app-env
OTEL_SERVICE_NAME: messenger-worker
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
depends_on:
migrate: {condition: service_completed_successfully}
postgres: {condition: service_healthy}
minio-init: {condition: service_completed_successfully}
rabbitmq: {condition: service_healthy}
otel-collector: {condition: service_started}
read_only: true
tmpfs: ["/tmp", "/dev/shm"]
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
stop_grace_period: 60s
networks: [backend]
mem_limit: 768m
cpus: 1.0
nginx:
image: nginx:1.27-alpine
ports: ["127.0.0.1:8080:80"]
volumes:
- ./nginx.conf:/etc/nginx/nginx.conf:ro
depends_on:
api: {condition: service_healthy}
networks: [backend]
mem_limit: 128m
cpus: 0.25
# ---------------- telemetry stack (lab6) ----------------
otel-collector:
image: otel/opentelemetry-collector-contrib:0.105.0
restart: unless-stopped
command: ["--config=/etc/otelcol/config.yaml"]
volumes:
- ./infra/otel-collector.yaml:/etc/otelcol/config.yaml:ro
networks: [backend]
mem_limit: 256m
cpus: 0.5
# Порты НЕ публикуем: 4317/4318/8889 доступны только внутри backend.
prometheus:
image: prom/prometheus:v2.54.1
restart: unless-stopped
command:
- "--config.file=/etc/prometheus/prometheus.yaml"
- "--storage.tsdb.path=/prometheus"
- "--storage.tsdb.retention.time=24h"
- "--web.enable-lifecycle"
volumes:
- ./infra/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
- ./infra/alert-rules.yml:/etc/prometheus/alert-rules.yml:ro
- prometheusdata:/prometheus
networks: [backend]
mem_limit: 512m
cpus: 0.5
loki:
image: grafana/loki:3.1.1
restart: unless-stopped
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki.yaml:/etc/loki/loki.yaml:ro
- lokidata:/loki
networks: [backend]
mem_limit: 512m
cpus: 0.5
tempo:
image: grafana/tempo:2.6.0
restart: unless-stopped
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempodata:/var/tempo
networks: [backend]
mem_limit: 512m
cpus: 0.5
grafana:
image: grafana/grafana:11.2.0
restart: unless-stopped
ports: ["127.0.0.1:3000:3000"]
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD_FILE: /run/secrets/grafana_admin_password
GF_USERS_ALLOW_SIGN_UP: "false"
GF_AUTH_ANONYMOUS_ENABLED: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
secrets: [grafana_admin_password]
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafanadata:/var/lib/grafana
depends_on:
prometheus: {condition: service_started}
loki: {condition: service_started}
tempo: {condition: service_started}
networks: [backend]
mem_limit: 256m
cpus: 0.5
networks:
backend: {}
volumes:
pgdata: {}
valkeydata: {}
miniodata: {}
rabbitmqdata: {}
prometheusdata: {}
lokidata: {}
tempodata: {}
grafanadata: {}
secrets:
postgres_password: {file: ./secrets/postgres_password}
valkey_password: {file: ./secrets/valkey_password}
database_url: {file: ./secrets/database_url}
cache_url: {file: ./secrets/cache_url}
minio_root_user: {file: ./secrets/minio_root_user}
minio_root_password: {file: ./secrets/minio_root_password}
s3_access_key_id: {file: ./secrets/s3_access_key_id}
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
rabbitmq_password: {file: ./secrets/rabbitmq_password}
broker_url: {file: ./secrets/broker_url}
grafana_admin_password: {file: ./secrets/grafana_admin_password}
+44
View File
@@ -0,0 +1,44 @@
groups:
- name: messenger-api
interval: 15s
rules:
# Доля 5xx среди бизнес-запросов (health исключён) > 5% за 2 минуты.
- alert: ApiHighErrorRate
expr: |
(
sum(rate(http_server_duration_milliseconds_count{
http_status_code=~"5..",
http_target!~"/health/.*"
}[2m]))
/
sum(rate(http_server_duration_milliseconds_count{
http_target!~"/health/.*"
}[2m]))
) > 0.05
for: 2m
labels:
severity: warning
annotations:
summary: "API error rate > 5% за 2 минуты"
description: "5xx доля бизнес-запросов превысила 5% в течение 2 минут."
runbook: "Проверить /api/v1 logs в Loki по trace_id из ошибки; проверить health БД/S3/broker."
- alert: QueueLagHigh
expr: messenger_queue_oldest_job_age_seconds > 60
for: 2m
labels:
severity: warning
annotations:
summary: "Очередь обработки отстаёт > 60s"
description: "Самая старая задача в queued старше 60 секунд."
runbook: "Проверить worker (docker compose ps worker, logs worker); проверить RabbitMQ; проверить S3/DB latency."
- alert: QueueLagCritical
expr: messenger_queue_oldest_job_age_seconds > 300
for: 1m
labels:
severity: critical
annotations:
summary: "Очередь обработки отстаёт > 5 минут"
description: "Обработка не продвигается; возможен отказ worker."
runbook: "Остановить worker → проверить логи; проверить RabbitMQ; replay failed."
+64
View File
@@ -0,0 +1,64 @@
{
"annotations": {"list": []},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"panels": [
{"type":"row","title":"API HTTP","gridPos":{"h":1,"w":24,"x":0,"y":0},"collapsed":false,"panels":[]},
{"type":"timeseries","title":"RPS по route template","gridPos":{"h":8,"w":12,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},
"fieldConfig":{"defaults":{"unit":"reqps","custom":{"drawStyle":"line","fillOpacity":10}},"overrides":[]},
"targets":[{"expr":"sum by (http_target) (rate(http_server_duration_milliseconds_count{http_target!~\"/health/.*\"}[1m]))","legendFormat":"{{http_target}}"}]},
{"type":"timeseries","title":"Error ratio (5xx / всё, health исключён)","gridPos":{"h":8,"w":12,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},
"fieldConfig":{"defaults":{"unit":"percentunit","custom":{"drawStyle":"line","fillOpacity":10}},"overrides":[]},
"targets":[{"expr":"(sum(rate(http_server_duration_milliseconds_count{http_status_code=~\"5..\",http_target!~\"/health/.*\"}[2m])) or vector(0)) / sum(rate(http_server_duration_milliseconds_count{http_target!~\"/health/.*\"}[2m]))","legendFormat":"5xx ratio"}]},
{"type":"timeseries","title":"Latency p50/p95/p99 (API, ms)","gridPos":{"h":8,"w":24,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},
"fieldConfig":{"defaults":{"unit":"ms","custom":{"drawStyle":"line","fillOpacity":0}},"overrides":[]},
"targets":[
{"expr":"histogram_quantile(0.50, sum by (le) (rate(http_server_duration_milliseconds_bucket{http_target!~\"/health/.*\"}[2m])))","legendFormat":"p50"},
{"expr":"histogram_quantile(0.95, sum by (le) (rate(http_server_duration_milliseconds_bucket{http_target!~\"/health/.*\"}[2m])))","legendFormat":"p95"},
{"expr":"histogram_quantile(0.99, sum by (le) (rate(http_server_duration_milliseconds_bucket{http_target!~\"/health/.*\"}[2m])))","legendFormat":"p99"}
]},
{"type":"row","title":"Worker / Queue","gridPos":{"h":1,"w":24,"x":0,"y":17},"collapsed":false,"panels":[]},
{"type":"timeseries","title":"Jobs по outcome (rate, 1m)","gridPos":{"h":8,"w":12,"x":0,"y":18},"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},
"fieldConfig":{"defaults":{"unit":"ops","custom":{"drawStyle":"line","fillOpacity":10}},"overrides":[]},
"targets":[{"expr":"sum by (outcome) (rate(messenger_jobs_processed_total[1m]))","legendFormat":"{{outcome}}"}]},
{"type":"timeseries","title":"Job processing duration p50/p95/p99 (s)","gridPos":{"h":8,"w":12,"x":12,"y":18},"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},
"fieldConfig":{"defaults":{"unit":"s","custom":{"drawStyle":"line","fillOpacity":0}},"overrides":[]},
"targets":[
{"expr":"histogram_quantile(0.50, sum by (le) (rate(messenger_job_processing_duration_seconds_bucket[5m])))","legendFormat":"p50"},
{"expr":"histogram_quantile(0.95, sum by (le) (rate(messenger_job_processing_duration_seconds_bucket[5m])))","legendFormat":"p95"},
{"expr":"histogram_quantile(0.99, sum by (le) (rate(messenger_job_processing_duration_seconds_bucket[5m])))","legendFormat":"p99"}
]},
{"type":"timeseries","title":"Queue depth (queued)","gridPos":{"h":8,"w":8,"x":0,"y":26},"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},
"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"line","fillOpacity":10}},"overrides":[]},
"targets":[{"expr":"sum(messenger_queue_depth)","legendFormat":"depth"}]},
{"type":"timeseries","title":"Oldest job age (s)","gridPos":{"h":8,"w":8,"x":8,"y":26},"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},
"fieldConfig":{"defaults":{"unit":"s","custom":{"drawStyle":"line","fillOpacity":10},"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":60},{"color":"red","value":300}]}},"overrides":[]},
"targets":[{"expr":"max(messenger_queue_oldest_job_age_seconds)","legendFormat":"oldest age"}]},
{"type":"timeseries","title":"Time to ready p50/p95 (s)","gridPos":{"h":8,"w":8,"x":16,"y":26},"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},
"fieldConfig":{"defaults":{"unit":"s","custom":{"drawStyle":"line","fillOpacity":0}},"overrides":[]},
"targets":[
{"expr":"histogram_quantile(0.50, sum by (le) (rate(messenger_time_to_ready_seconds_bucket[5m])))","legendFormat":"p50"},
{"expr":"histogram_quantile(0.95, sum by (le) (rate(messenger_time_to_ready_seconds_bucket[5m])))","legendFormat":"p95"}
]},
{"type":"row","title":"Saturation","gridPos":{"h":1,"w":24,"x":0,"y":34},"collapsed":false,"panels":[]},
{"type":"timeseries","title":"Worker busy slots (capacity 2)","gridPos":{"h":8,"w":8,"x":0,"y":35},"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},
"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"line","fillOpacity":10},"max":2},"overrides":[]},
"targets":[{"expr":"sum(messenger_worker_busy)","legendFormat":"busy"}]},
{"type":"timeseries","title":"DB pool (available / waiting)","gridPos":{"h":8,"w":8,"x":8,"y":35},"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},
"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"line","fillOpacity":10}},"overrides":[]},
"targets":[{"expr":"sum by (state) (messenger_db_pool)","legendFormat":"{{state}}"}]},
{"type":"timeseries","title":"Active HTTP requests","gridPos":{"h":8,"w":8,"x":16,"y":35},"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},
"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"line","fillOpacity":10}},"overrides":[]},
"targets":[{"expr":"sum(http_server_active_requests)","legendFormat":"active"}]}
],
"refresh": "10s",
"schemaVersion": 39,
"tags": ["messenger","lab6"],
"templating": {"list": []},
"time": {"from": "now-15m", "to": "now"},
"timezone": "browser",
"title": "Messenger — lab6",
"uid": "messenger-lab6"
}
@@ -0,0 +1,12 @@
apiVersion: 1
providers:
- name: messenger
orgId: 1
folder: Messenger
type: file
disableDeletion: false
updateIntervalSeconds: 30
options:
path: /var/lib/grafana/dashboards
foldersFromFilesStructure: false
@@ -0,0 +1,31 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
jsonData:
timeInterval: 10s
- name: Loki
type: loki
access: proxy
url: http://loki:3100
jsonData:
derivedFields:
- datasourceUid: tempo
matcherRegex: '"trace_id":"([a-f0-9]+)"'
name: TraceID
url: "$${__value.raw}"
- name: Tempo
type: tempo
access: proxy
url: http://tempo:3200
uid: tempo
jsonData:
tracesToLogsV2:
datasourceUid: loki
filterByTraceID: true
serviceMap:
datasourceUid: prometheus
+38
View File
@@ -0,0 +1,38 @@
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
log_level: warn
common:
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
instance_addr: 127.0.0.1
kvstore:
store: inmemory
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 24h
reject_old_samples: false
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
+61
View File
@@ -0,0 +1,61 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
grpc:
endpoint: 0.0.0.0:4317
processors:
# Ограничивает RAM коллектора: если превысили — начинает отбрасывать данные,
# а не падает по OOM. Это требование lab6 уровня 4.
memory_limiter:
check_interval: 1s
limit_mib: 200
spike_limit_mib: 50
# Батчинг: уменьшает число запросов к backend.
batch:
timeout: 5s
send_batch_size: 1024
# Ограничивает кардинальность метрик: удаляет потенциально опасные атрибуты.
attributes/scrub:
actions:
- key: http.request.header.authorization
action: delete
- key: http.request.header.cookie
action: delete
exporters:
# Метрики — в формате Prometheus. Prometheus сам scrape'ит :8889.
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion:
enabled: true
# Логи — в Loki через OTLP endpoint (Loki 3.x умеет /otlp).
otlphttp/loki:
endpoint: http://loki:3100/otlp
tls:
insecure: true
# Трассы — в Tempo через OTLP gRPC.
otlp/tempo:
endpoint: tempo:4317
tls:
insecure: true
service:
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch, attributes/scrub]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch, attributes/scrub]
exporters: [otlphttp/loki]
telemetry:
logs:
level: info
+16
View File
@@ -0,0 +1,16 @@
global:
scrape_interval: 10s
evaluation_interval: 10s
rule_files:
- /etc/prometheus/alert-rules.yml
scrape_configs:
# Метрики от OTel Collector (он в формате Prometheus на :8889).
- job_name: otel-collector
static_configs:
- targets: ['otel-collector:8889']
# Сам Prometheus — для наблюдения за собой.
- job_name: prometheus
static_configs:
- targets: ['localhost:9090']
+35
View File
@@ -0,0 +1,35 @@
server:
http_listen_port: 3200
grpc_listen_port: 9095
log_level: warn
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
ingester:
max_block_duration: 5m
compactor:
compaction:
block_retention: 24h
storage:
trace:
backend: local
local:
path: /var/tempo/traces
wal:
path: /var/tempo/wal
metrics_generator:
registry:
external_labels:
source: tempo
storage:
path: /var/tempo/generator/wal
+482
View File
@@ -0,0 +1,482 @@
"""Мессенджер, лабораторная 6: lab5 + OpenTelemetry-инструментация."""
import hashlib
import logging
import re
import tempfile
import threading
import uuid
from contextlib import asynccontextmanager
from typing import NamedTuple
from urllib.parse import quote
import anyio.to_thread
import psycopg
from fastapi import Depends, FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, Response
from opentelemetry import metrics as metrics_module
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor
from starlette.exceptions import HTTPException as StarletteHTTPException
from app import (attachments, broker, config, cursor, db, idempotency, jobs,
repo, security, sessions, s3, telemetry, validation)
from app.errors import (BASIC_CHALLENGE, BEARER_CHALLENGE, ApiError,
bad_request, not_found, unauthorized)
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
for _noisy in ("pika", "aiohttp", "urllib3", "botocore", "opentelemetry"):
_l = logging.getLogger(_noisy)
_l.setLevel(logging.WARNING)
_l.propagate = False
log = logging.getLogger("api")
_REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._-]{1,64}")
INSTANCE_ID = config.INSTANCE_ID
_stop = threading.Event()
def _outbox_loop():
"""API тоже публикует outbox — снижает latency до broker."""
while not _stop.wait(2):
try:
with db.connection() as conn:
jobs.publish_pending_outbox(conn, broker.publish)
except Exception:
pass
def _register_metrics():
"""Observable gauges: SDK сам вызывает callbacks при каждом экспорте.
Никакого отдельного потока не нужно — MeterProvider с PeriodicExportingMetricReader
сам опрашивает при экспорте.
"""
meter = telemetry.get_meter("messenger-api")
def _queue_depth(_options):
try:
with db.connection() as conn:
row = conn.execute(
"SELECT count(*) AS c FROM jobs WHERE status='queued'").fetchone()
yield metrics_module.Observation(
int(row["c"] or 0), {"queue.name": broker.QUEUE})
except Exception:
return
def _queue_age(_options):
try:
with db.connection() as conn:
row = conn.execute(
"""SELECT EXTRACT(EPOCH FROM
COALESCE(now() - min(created_at), '0'::interval))::float AS age
FROM jobs WHERE status='queued'""").fetchone()
yield metrics_module.Observation(
float(row["age"] or 0.0), {"queue.name": broker.QUEUE})
except Exception:
return
def _db_pool(_options):
try:
stats = db.get_pool().get_stats()
yield metrics_module.Observation(
int(stats.get("pool_available", 0)), {"state": "available"})
yield metrics_module.Observation(
int(stats.get("requests_waiting", 0)), {"state": "waiting"})
except Exception:
return
meter.create_observable_gauge(
"messenger_queue_depth", callbacks=[_queue_depth],
description="Jobs in queued status")
meter.create_observable_gauge(
"messenger_queue_oldest_job_age_seconds", callbacks=[_queue_age],
description="Age of oldest queued job (seconds)")
meter.create_observable_gauge(
"messenger_db_pool", callbacks=[_db_pool],
description="DB pool state (available/waiting)")
@asynccontextmanager
async def lifespan(_app):
anyio.to_thread.current_default_thread_limiter().total_tokens = config.THREAD_LIMIT
# Psycopg/Botocore можно инструментировать и здесь — они не привязаны к ASGI-приложению.
PsycopgInstrumentor().instrument()
BotocoreInstrumentor().instrument()
_register_metrics()
threading.Thread(target=_outbox_loop, daemon=True).start()
try:
broker.declare_only()
except Exception as exc:
log.warning("broker topology init failed: %s", type(exc).__name__)
yield
_stop.set()
db.close()
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
# lab6: OTel инициализируется до старта приложения.
# FastAPIInstrumentor.instrument_app() обязан вызываться ДО первого запроса,
# иначе Starlette не даёт добавить middleware.
telemetry.init("messenger-api")
FastAPIInstrumentor.instrument_app(app, excluded_urls="health")
def make_error(code, message, request_id):
return {"error": {"code": code, "message": message, "request_id": request_id}}
def _rid(request):
return getattr(request.state, "request_id", None) or uuid.uuid4().hex
def error_response(request, status, code, message, headers=None):
rid = _rid(request)
out = {"X-Request-Id": rid, "X-Instance-Id": INSTANCE_ID, "Cache-Control": "no-store"}
out.update(headers or {})
return JSONResponse(status_code=status, content=make_error(code, message, rid), headers=out)
@app.middleware("http")
async def add_request_id(request: Request, call_next):
incoming = request.headers.get("X-Request-Id", "")
rid = incoming if _REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex
request.state.request_id = rid
response = await call_next(request)
response.headers["X-Request-Id"] = rid
response.headers["X-Instance-Id"] = INSTANCE_ID
response.headers["Cache-Control"] = "no-store"
return response
@app.exception_handler(ApiError)
async def api_error_handler(request: Request, exc: ApiError):
return error_response(request, exc.status, exc.code, exc.message, exc.headers)
@app.exception_handler(RequestValidationError)
async def validation_handler(request: Request, exc):
return error_response(request, 400, "invalid_request", "Некорректный запрос")
@app.exception_handler(StarletteHTTPException)
async def http_exc_handler(request: Request, exc: StarletteHTTPException):
mapping = {404: ("not_found", "Объект не найден"), 405: ("invalid_request", "Метод не поддерживается")}
code, message = mapping.get(exc.status_code, ("invalid_request", "Ошибка запроса"))
return error_response(request, exc.status_code, code, message)
@app.exception_handler(Exception)
async def generic_handler(request: Request, exc: Exception):
if isinstance(exc, db.TRANSIENT):
log.warning("dependency unavailable [%s]: %s", _rid(request), type(exc).__name__)
return error_response(request, 503, "unavailable", "Сервис временно недоступен")
log.error("unhandled error [%s]", _rid(request), exc_info=exc)
return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера")
class Auth(NamedTuple):
user: dict
user_id: uuid.UUID
digest: bytes
def authenticated(request: Request) -> Auth:
token = security.parse_bearer(request.headers.get("Authorization"))
if token is None:
raise unauthorized(BEARER_CHALLENGE)
found = sessions.lookup(token)
if found is None:
raise unauthorized(BEARER_CHALLENGE)
return Auth(found["user"], uuid.UUID(found["user"]["id"]), found["digest"])
class RawBody(NamedTuple):
content_type: str
data: bytes
async def raw_body(request: Request) -> RawBody:
declared = request.headers.get("content-length")
if declared is not None and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES:
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое")
chunks, size = [], 0
async for chunk in request.stream():
size += len(chunk)
if size > config.MAX_BODY_BYTES:
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое")
chunks.append(chunk)
return RawBody(request.headers.get("content-type", ""), b"".join(chunks))
@app.get("/health/live")
def live():
return {"status": "ok"}
@app.get("/health/ready")
def ready():
try:
db.ping()
except Exception:
raise ApiError(503, "unavailable", "PostgreSQL недоступен")
try:
s3.ping()
except Exception:
raise ApiError(503, "unavailable", "S3 недоступен")
try:
if not broker.wait_ready(timeout=2):
raise ApiError(503, "unavailable", "Broker недоступен")
except ApiError:
raise
except Exception:
raise ApiError(503, "unavailable", "Broker недоступен")
return {"status": "ok"}
# ---------- пользователи / сессии / чаты ----------
@app.post("/api/v1/users", status_code=201)
def create_user(body: RawBody = Depends(raw_body)):
data = validation.json_body(body.content_type, body.data,
{"username", "display_name", "password"},
{"username", "display_name", "password"})
username = validation.username_field(data["username"])
display_name = validation.text_field(data["display_name"], "display_name", 1, 100)
password = validation.text_field(data["password"], "password", 12, 128)
with db.connection() as conn:
if repo.username_exists(conn, username):
raise ApiError(409, "conflict", "Имя пользователя занято")
password_hash = security.hash_password(password)
try:
with db.connection() as conn:
row = repo.insert_user(conn, username, display_name, password_hash)
except psycopg.errors.UniqueViolation:
raise ApiError(409, "conflict", "Имя пользователя занято")
return sessions.user_json(row)
@app.get("/api/v1/users/me")
def get_me(auth: Auth = Depends(authenticated)):
return auth.user
@app.post("/api/v1/auth/sessions", status_code=201)
def login(request: Request):
creds = security.parse_basic(request.headers.get("Authorization"))
if creds is None:
raise unauthorized(BASIC_CHALLENGE)
username, password = creds
with db.connection() as conn:
user = repo.user_for_login(conn, username) if validation._USERNAME_RE.fullmatch(username) else None
if user is None:
security.burn_verify(password)
raise unauthorized(BASIC_CHALLENGE)
ok, new_hash = security.verify_password(user["password_hash"], password)
if not ok:
raise unauthorized(BASIC_CHALLENGE)
with db.connection() as conn:
if new_hash:
repo.update_password_hash(conn, user["id"], new_hash)
token, expires_at = sessions.create(conn, user)
user_json = sessions.user_json(user)
sessions.warm(token, user_json, expires_at)
return {"token": token, "token_type": "Bearer",
"expires_at": sessions.fmt_ts(expires_at), "user": user_json}
@app.delete("/api/v1/auth/sessions/current")
def logout(auth: Auth = Depends(authenticated)):
sessions.revoke(auth.digest)
return Response(status_code=204)
@app.post("/api/v1/chats", status_code=201)
def create_chat(auth: Auth = Depends(authenticated), body: RawBody = Depends(raw_body)):
data = validation.json_body(body.content_type, body.data,
{"title", "member_ids"}, {"title", "member_ids"})
title = validation.text_field(data["title"], "title", 1, 100)
raw = data["member_ids"]
if not isinstance(raw, list) or not 1 <= len(raw) <= 100:
raise bad_request("member_ids: от 1 до 100 элементов")
members = [validation.require_uuid(x, "member_id") for x in raw]
if len(set(members)) != len(members) or auth.user_id not in members:
raise bad_request("member_ids должны быть уникальны и включать вызывающего")
with db.connection() as conn:
chat = repo.create_chat(conn, auth.user_id, title, members)
return repo.chat_json(chat)
@app.get("/api/v1/chats")
def list_chats(request: Request, auth: Auth = Depends(authenticated)):
limit = validation.limit_param(request.query_params)
raw = request.query_params.get("cursor")
after = cursor.decode(raw, "chats", auth.user_id) if raw is not None else None
with db.connection() as conn:
items, nxt = repo.list_chats(conn, auth.user_id, after, limit)
return {"items": items, "next_cursor": nxt}
@app.get("/api/v1/chats/{chat_id}")
def get_chat(chat_id: str, auth: Auth = Depends(authenticated)):
cid = validation.require_uuid(chat_id, "chat_id")
with db.connection() as conn:
chat = repo.get_chat(conn, cid, auth.user_id)
if chat is None:
raise not_found()
return repo.chat_json(chat)
# ---------- сообщения ----------
def _parse_attachment_ids(data):
raw = data.get("attachment_ids", [])
if not isinstance(raw, list) or len(raw) > 10:
raise bad_request("attachment_ids: до 10 элементов")
ids = [validation.require_uuid(x, "attachment_id") for x in raw]
if len(set(ids)) != len(ids):
raise bad_request("attachment_ids должны быть уникальны")
return ids
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
def create_message(chat_id: str, request: Request, auth: Auth = Depends(authenticated),
body: RawBody = Depends(raw_body)):
cid = validation.require_uuid(chat_id, "chat_id")
with db.connection() as conn:
if not repo.is_member(conn, cid, auth.user_id):
raise not_found()
data = validation.json_body(body.content_type, body.data,
{"text", "attachment_ids"}, {"text"})
text = validation.text_field(data["text"], "text", 0, 4000, allow_blank=True)
attachment_ids = _parse_attachment_ids(data)
if not text.strip() and not attachment_ids:
raise bad_request("Пустое сообщение без вложений")
idem_key = request.headers.get("Idempotency-Key")
path = "/api/v1/chats/%s/messages" % chat_id
req_hash = idempotency.get_request_hash(data)
with db.connection() as conn:
idempotency.cleanup(conn, config.IDEMPOTENCY_TTL_SECONDS)
need_exec, cached = idempotency.check_and_lock(
conn, str(auth.user_id), "POST", path, idem_key, req_hash)
if not need_exec:
return JSONResponse(status_code=201, content=cached)
msg = repo.insert_message(conn, cid, auth.user_id, text, attachment_ids)
msg_json = repo.message_json(msg)
idempotency.save_result(conn, str(auth.user_id), "POST", path, idem_key,
req_hash, 201, msg_json)
return msg_json
@app.get("/api/v1/chats/{chat_id}/messages")
def list_messages(chat_id: str, request: Request, auth: Auth = Depends(authenticated)):
cid = validation.require_uuid(chat_id, "chat_id")
limit = validation.limit_param(request.query_params)
raw = request.query_params.get("cursor")
after = cursor.decode(raw, "messages", cid) if raw is not None else None
with db.connection() as conn:
if not repo.is_member(conn, cid, auth.user_id):
raise not_found()
items, nxt = repo.list_messages(conn, cid, after, limit)
return {"items": items, "next_cursor": nxt}
# ---------- вложения (lab5: 202 queued) ----------
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def upload_attachment(chat_id: str, request: Request, auth: Auth = Depends(authenticated)):
cid = validation.require_uuid(chat_id, "chat_id")
filename = request.query_params.get("filename")
if filename is None:
raise bad_request("filename обязателен")
declared = request.headers.get("content-type", "").split(";")[0].strip().lower()
if declared not in ("image/png", "image/jpeg"):
raise ApiError(415, "unsupported_media_type", "Ожидается image/png или image/jpeg")
def _check_member():
with db.connection() as conn:
return repo.is_member(conn, cid, auth.user_id)
if not await anyio.to_thread.run_sync(_check_member):
raise not_found()
declared_len = request.headers.get("content-length")
if declared_len is not None and declared_len.isdigit() and int(declared_len) > config.MAX_UPLOAD_BYTES:
raise ApiError(413, "payload_too_large", "Файл больше 10 MiB")
spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024)
hasher = hashlib.sha256()
total = 0
try:
async for chunk in request.stream():
total += len(chunk)
if total > config.MAX_UPLOAD_BYTES:
raise ApiError(413, "payload_too_large", "Файл больше 10 MiB")
hasher.update(chunk)
spool.write(chunk)
if total == 0:
raise bad_request("Пустой файл")
spool.seek(0)
digest = hasher.hexdigest()
def _process():
actual = attachments.inspect_image_header(spool, total)
if actual != declared:
raise ApiError(415, "unsupported_media_type", "MIME не соответствует содержимому")
return attachments.upload_to_staging(
cid, auth.user_id, filename, spool, total, digest, actual)
row = await anyio.to_thread.run_sync(_process)
return repo.attachment_json(row)
finally:
spool.close()
@app.get("/api/v1/attachments/{attachment_id}")
def get_attachment(attachment_id: str, auth: Auth = Depends(authenticated)):
aid = validation.require_uuid(attachment_id, "attachment_id")
with db.connection() as conn:
row = repo.get_attachment_for_member(conn, aid, auth.user_id)
if row is None:
raise not_found()
variants = repo.list_variants(conn, aid)
return repo.attachment_json(row, variants)
@app.get("/api/v1/attachments/{attachment_id}/content")
async def download_attachment(attachment_id: str, request: Request,
auth: Auth = Depends(authenticated)):
aid = validation.require_uuid(attachment_id, "attachment_id")
variant_name = request.query_params.get("variant", "original")
if variant_name not in ("original", "thumbnail", "crop", "optimized"):
raise bad_request("variant: original|thumbnail|crop|optimized")
def _read_all():
if variant_name == "original":
row, body = attachments.download_original(aid, auth.user_id)
try:
data = body.read()
finally:
body.close()
return row, data
v, body = attachments.download_variant(aid, auth.user_id, variant_name)
try:
data = body.read()
finally:
body.close()
return v, data
row, data = await anyio.to_thread.run_sync(_read_all)
if variant_name == "original":
headers = {"Content-Disposition": "attachment; filename*=UTF-8''%s" % quote(row["filename"], safe="")}
return Response(content=data, media_type=row["content_type"], headers=headers)
return Response(content=data, media_type=row["content_type"])
+44
View File
@@ -0,0 +1,44 @@
worker_processes auto;
events { worker_connections 1024; }
http {
# Nginx OSS резолвит DNS в upstream один раз при старте.
# Поэтому перечисляем обе реплики явно по DNS-именам контейнеров Compose.
upstream api_pool {
server messenger-lab6-api-1:8080 max_fails=1 fail_timeout=5s;
server messenger-lab6-api-2:8080 max_fails=1 fail_timeout=5s;
keepalive 16;
}
server {
listen 80;
location / {
proxy_pass http://api_pool;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Connection "";
# Retry только на connection errors/timeouts (backend не ответил).
proxy_next_upstream error timeout;
proxy_next_upstream_tries 2;
proxy_connect_timeout 2s;
# 15s вместо 60s: мёртвый backend помечается быстрее,
# failover укладывается в 10-секундный бюджет.
proxy_send_timeout 15s;
proxy_read_timeout 15s;
# 11m, а не 10m: пропускаем до backend, чтобы он сам вернул 413
# с X-Request-Id и телом ошибки. Верхний лимит 10 MiB — в приложении.
client_max_body_size 11m;
# ВАЖНО: on, а не off. С off nginx стримит тело в backend
# и не может повторить POST на живую реплику, если backend умер
# посреди запроса.
proxy_request_buffering on;
}
}
}
+37
View File
@@ -0,0 +1,37 @@
boto3==1.43.107
Pillow==12.3.0
annotated-types==0.8.0
anyio==4.15.1
argon2-cffi==25.1.0
argon2-cffi-bindings==26.1.0
cffi==2.1.1
click==8.5.0
fastapi==0.115.0
h11==0.16.0
httptools==0.9.0
idna==3.20
psycopg==3.2.9
psycopg-binary==3.2.9
psycopg-pool==3.2.6
pycparser==3.11
pydantic==2.14.1
pydantic_core==2.50.1
python-dotenv==1.2.4
PyYAML==6.0.3
redis==6.4.0
starlette==0.38.6
typing-inspection==0.4.4
typing_extensions==4.16.0
uvicorn==0.30.6
uvloop==0.23.0
watchfiles==1.3.0
websockets==17.2
pika==1.3.2
opentelemetry-api==1.27.0
opentelemetry-sdk==1.27.0
opentelemetry-exporter-otlp-proto-http==1.27.0
opentelemetry-instrumentation-fastapi==0.48b0
opentelemetry-instrumentation-psycopg==0.48b0
opentelemetry-instrumentation-botocore==0.48b0
opentelemetry-instrumentation-pika==0.48b0
opentelemetry-instrumentation-logging==0.48b0
+56
View File
@@ -0,0 +1,56 @@
#!/bin/sh
set -eu
umask 077
cd "$(dirname "$0")/.."
mkdir -p secrets
gen() { head -c 32 /dev/urandom | base64 | tr -d '/+=\n' | cut -c1-32; }
ensure() {
f="secrets/$1"
if [ ! -s "$f" ]; then
printf '%s' "$2" > "$f"
chmod 600 "$f"
echo "created $f"
fi
}
PG_PASS=$(cat secrets/postgres_password 2>/dev/null || gen)
VALKEY_PASS=$(cat secrets/valkey_password 2>/dev/null || gen)
MINIO_ROOT_PASS=$(cat secrets/minio_root_password 2>/dev/null || gen)
S3_KEY=$(cat secrets/s3_access_key_id 2>/dev/null || "app$(gen | cut -c1-20)")
S3_SECRET=$(cat secrets/s3_secret_access_key 2>/dev/null || gen)
RMQ_PASS=$(cat secrets/rabbitmq_password 2>/dev/null || gen)
ensure postgres_password "$PG_PASS"
ensure valkey_password "$VALKEY_PASS"
ensure minio_root_user "minioadmin"
ensure minio_root_password "$MINIO_ROOT_PASS"
ensure s3_access_key_id "$S3_KEY"
ensure s3_secret_access_key "$S3_SECRET"
ensure rabbitmq_password "$RMQ_PASS"
ensure database_url "postgresql://messenger:${PG_PASS}@postgres:5432/messenger"
ensure cache_url "redis://:${VALKEY_PASS}@valkey:6379/0"
ensure broker_url "amqp://messenger:${RMQ_PASS}@rabbitmq:5672/%2F"
if [ ! -f .env ]; then
cat > .env <<'EOF'
APP_ENV=lab
SESSION_TTL_SECONDS=3600
SESSION_STORE=postgres+cache
SESSION_CACHE_TTL_SECONDS=60
IDEMPOTENCY_TTL_SECONDS=86400
API_REPLICAS=2
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
S3_ORPHAN_GRACE_SECONDS=3600
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
EOF
echo "created .env"
fi
+115
View File
@@ -0,0 +1,115 @@
#!/usr/bin/env python3
"""Сверка S3 и метаданных PostgreSQL.
Классы проблем:
orphan_object — объект в S3 без строки attachments (или с pending > grace)
missing_object — attachments.status=ready, но объекта в S3 нет
failed_referenced — attachments.status=failed, объект всё ещё лежит в S3
Действия:
orphan_object → DELETE объекта (только старше grace)
missing_object → UPDATE status='failed', error_code='object_missing'
failed_referenced → DELETE объекта
Использование:
python scripts/reconcile_s3.py --dry-run
python scripts/reconcile_s3.py --apply
"""
import argparse
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from app import config, db, repo, s3 # noqa: E402
def _paginate(client, prefix):
"""Возвращает {key: last_modified}."""
token = None
out = {}
while True:
kwargs = {"Bucket": config.S3_BUCKET, "Prefix": prefix}
if token:
kwargs["ContinuationToken"] = token
resp = client.list_objects_v2(**kwargs)
for obj in resp.get("Contents", []):
out[obj["Key"]] = obj["LastModified"]
if not resp.get("IsTruncated"):
return out
token = resp["NextContinuationToken"]
def reconcile(dry_run: bool, grace_seconds: int):
client = s3._get_client()
now = time.time()
report = {"orphans_deleted": [], "orphans_skipped_fresh": [],
"missing_marked_failed": [], "failed_objects_deleted": []}
with db.connection() as conn:
rows = conn.execute(
"""SELECT id, object_key, status, created_at,
extract(epoch FROM created_at) AS created_epoch
FROM attachments""").fetchall()
by_key = {r["object_key"]: r for r in rows}
referenced = set(by_key)
for key, last_modified in _paginate(client, "originals/").items():
if key in referenced:
continue
age = now - last_modified.timestamp()
if age < grace_seconds:
report["orphans_skipped_fresh"].append(key)
continue
report["orphans_deleted"].append(key)
if not dry_run:
s3.delete_object(key)
for row in rows:
key = row["object_key"]
if row["status"] == "ready" and key not in _paginate(client, key):
report["missing_marked_failed"].append(key)
if not dry_run:
with db.connection() as conn:
conn.execute(
"""UPDATE attachments
SET status='failed', error_code='object_missing',
updated_at=clock_timestamp()
WHERE id=%s AND status='ready'""", (row["id"],))
elif row["status"] == "failed":
if key in _paginate(client, key):
report["failed_objects_deleted"].append(key)
if not dry_run:
s3.delete_object(key)
elif row["status"] == "pending":
age = now - float(row["created_epoch"])
if age >= grace_seconds and key in _paginate(client, key):
# pending дольше grace, а объект есть → API упал между S3 PUT и COMMIT
report["orphans_deleted"].append(key)
if not dry_run:
with db.connection() as conn:
repo.mark_attachment_failed(conn, row["id"], "orphaned_pending")
s3.delete_object(key)
import json
print(json.dumps(report, ensure_ascii=False, indent=2, default=str))
print("dry-run" if dry_run else "applied")
def main():
p = argparse.ArgumentParser()
p.add_argument("--apply", action="store_true")
p.add_argument("--dry-run", action="store_true")
p.add_argument("--grace-seconds", type=int, default=config.S3_ORPHAN_GRACE_SECONDS)
args = p.parse_args()
if args.apply and args.dry_run:
sys.exit("use --apply or --dry-run, not both")
if not (args.apply or args.dry_run):
sys.exit("choose --dry-run or --apply")
reconcile(dry_run=not args.apply, grace_seconds=args.grace_seconds)
if __name__ == "__main__":
main()
+28
View File
@@ -0,0 +1,28 @@
#!/usr/bin/env python3
"""Операторский replay failed-job. Запуск: python scripts/replay-failed.py <job_id>."""
import sys
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from app import db, jobs # noqa: E402
def main():
if len(sys.argv) != 2:
sys.exit("usage: replay-failed.py <job_id>")
try:
job_id = uuid.UUID(sys.argv[1])
except ValueError:
sys.exit("invalid job_id")
with db.connection() as conn:
payload = jobs.replay_failed(conn, job_id)
if payload is None:
sys.exit("job not found or not in failed state")
print("requeued:", payload["job_id"])
if __name__ == "__main__":
main()
+6
View File
@@ -0,0 +1,6 @@
@echo off
cd /d "%~dp0\.."
docker compose --env-file .env stop
timeout /t 2 /nobreak >nul
docker compose --env-file .env up -d
exit /b 0
Executable → Regular
View File
+17
View File
@@ -0,0 +1,17 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose --env-file .env stop
sleep 2
docker compose --env-file .env up -d
for i in $(seq 1 120); do
if curl -fs -o /dev/null http://127.0.0.1:8080/health/ready 2>/dev/null; then
echo "stack ready after ${i}s"
exit 0
fi
sleep 1
done
echo "stack not ready in 120s" >&2
exit 1
+19
View File
@@ -0,0 +1,19 @@
@echo off
REM Остановить одну API-реплику по её X-Instance-Id (== Hostname контейнера).
setlocal
if "%TARGET_INSTANCE_ID%"=="" (
echo TARGET_INSTANCE_ID is not set
exit /b 1
)
for /f "tokens=*" %%c in ('docker ps -q --filter "name=messenger-lab4-api"') do (
for /f "tokens=*" %%h in ('docker inspect -f "{{.Config.Hostname}}" %%c') do (
if "%%h"=="%TARGET_INSTANCE_ID%" (
echo Stopping container %%c
docker stop %%c
exit /b 0
)
)
)
echo Container with instance ID %TARGET_INSTANCE_ID% not found
exit /b 1
Executable → Regular
View File
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
# Останавливает конкретную реплику API по её X-Instance-Id.
set -eu
if [ -z "${TARGET_INSTANCE_ID:-}" ]; then
echo "TARGET_INSTANCE_ID is not set" >&2
exit 1
fi
# INSTANCE_ID = socket.gethostname(), в Docker это короткий ID контейнера.
cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" || true)
if [ -z "$cid" ]; then
# Fallback: искать по имени проекта и сверять hostname
for c in $(docker ps -q --filter "name=messenger-lab4"); do
h=$(docker inspect -f '{{.Config.Hostname}}' "$c")
if [ "$h" = "$TARGET_INSTANCE_ID" ]; then
cid="$c"
break
fi
done
fi
if [ -n "$cid" ]; then
echo "Stopping container $cid (Instance ID: $TARGET_INSTANCE_ID)"
docker stop "$cid"
else
echo "Container with instance ID $TARGET_INSTANCE_ID not found" >&2
exit 1
fi
+4
View File
@@ -0,0 +1,4 @@
@echo off
cd /d "%~dp0\.."
docker compose --env-file .env stop worker
echo worker stopped
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose --env-file .env stop worker
echo "worker stopped"
+148
View File
@@ -0,0 +1,148 @@
#!/usr/bin/env python3
"""Lab6: controlled incidents + observability correlation."""
import base64, hashlib, json, os, subprocess, sys, time, urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
def prom(expr):
"""Запрос к Prometheus через docker exec."""
url = f"http://localhost:9090/api/v1/query?query={urllib.parse.quote(expr)}"
r = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "prometheus",
"wget", "-qO-", url],
capture_output=True, text=True, cwd=ROOT, timeout=15)
if r.returncode != 0:
return None
try:
return json.loads(r.stdout)["data"]["result"]
except Exception:
return None
def compose(*args, timeout=120):
r = subprocess.run(["docker", "compose", "--env-file", ".env", *args],
capture_output=True, text=True, cwd=ROOT, timeout=timeout)
return r.returncode, r.stdout + r.stderr
def loki(service_name):
url = ("http://localhost:3100/loki/api/v1/query_range?"
"query=%7Bservice_name%3D%22" + service_name + "%22%7D&limit=5")
r = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "loki",
"wget", "-qO-", url], capture_output=True, text=True, cwd=ROOT, timeout=15)
return r.stdout
def tempo():
r = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "tempo",
"wget", "-qO-", "http://localhost:3200/api/search?limit=10"],
capture_output=True, text=True, cwd=ROOT, timeout=15)
return r.stdout
def main():
args = arguments("Lab6 controlled incidents")
c = Client(args.base_url, args.ca_file)
# 1. Smoke: данные текут.
alice, bob, eve, chat = c.bootstrap()
print("1. bootstrap OK, chat:", chat["id"])
att, _ = c.attachment(alice, chat)
ready = c.wait_ready(bob, att, wait_seconds=60)
check(ready["status"] == "ready", "worker pipeline not ready")
# 2. Метрики существуют.
jobs = prom('messenger_jobs_processed_total{outcome="ready"}')
check(jobs and len(jobs) > 0, "messenger_jobs_processed_total missing")
print("2. jobs metric OK:", jobs[0]["value"][1])
http_count = prom('sum(http_server_duration_milliseconds_count{http_target!~"/health/.*"})')
check(http_count and len(http_count) > 0, "http_server_duration_milliseconds_count missing")
print(" http metric OK:", http_count[0]["value"][1])
# 3. Корреляция trace → log.
l = loki("messenger-api")
check("trace_id" in l, "no trace_id in Loki logs")
print("3. Loki correlation OK (trace_id present in logs)")
t = tempo()
check("traces" in t and "messenger-api" in t, "no traces in Tempo")
print(" Tempo traces OK")
# 4. Инцидент 1: остановить worker → очередь растёт.
print("4. stopping worker...")
compose("stop", "worker", timeout=60)
time.sleep(2)
for _ in range(5):
c.attachment(alice, chat)
print(" 5 uploads queued")
# Ждём, пока alert станет firing (for: 2m).
firing = False
deadline = time.monotonic() + 200
while time.monotonic() < deadline:
time.sleep(15)
rules = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "prometheus",
"wget", "-qO-", "http://localhost:9090/api/v1/rules"],
capture_output=True, text=True, cwd=ROOT, timeout=15).stdout
if '"name":"QueueLagHigh"' in rules and '"state":"firing"' in rules:
firing = True
break
check(firing, "QueueLagHigh did not reach firing")
print(" QueueLagHigh firing ✓")
# 5. Инцидент 2: восстановить worker → alert resolved.
print("5. starting worker...")
compose("start", "worker", timeout=60)
resolved = False
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
time.sleep(15)
rules = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "prometheus",
"wget", "-qO-", "http://localhost:9090/api/v1/rules"],
capture_output=True, text=True, cwd=ROOT, timeout=15).stdout
if ('"name":"QueueLagHigh"' in rules
and '"state":"inactive"' in rules.split('"name":"QueueLagHigh"')[1][:2000]):
resolved = True
break
check(resolved, "QueueLagHigh did not resolve after worker restart")
print(" QueueLagHigh resolved ✓")
# 6. Backlog рассосался.
time.sleep(5)
depth = prom("sum(messenger_queue_depth)")
if depth and depth[0]["value"][1]:
check(float(depth[0]["value"][1]) == 0.0, "queue not drained: " + depth[0]["value"][1])
print("6. queue drained")
# 7. Телеметрия-бэкенд недоступен → API продолжает работать.
print("7. stopping telemetry backends (loki, tempo)...")
compose("stop", "loki", "tempo", timeout=60)
time.sleep(3)
# smoke через API — должен пройти, несмотря на потерю traces/logs.
# В этом состоянии в api пойдёт WARN от exporter — но оно не падает.
a2, b2, e2, chat2 = c.bootstrap()
att2, _ = c.attachment(a2, chat2)
ready2 = c.wait_ready(b2, att2, wait_seconds=60)
check(ready2["status"] == "ready", "API broken when telemetry down")
print(" API still works with telemetry down ✓")
compose("start", "loki", "tempo", timeout=60)
time.sleep(10)
print("\nPASS: lab6 observability (metrics, traces, logs, alerts firing→resolved, telemetry-down resilience)")
if __name__ == "__main__":
run(main)