ЛР6: OpenTelemetry, Collector, Prometheus/Loki/Tempo, Grafana, алерты, исследование цены наблюдаемости

This commit is contained in:
kolya5544
2026-10-03 05:10:45 +03:00
parent 470d9d6f1e
commit 1d908578e3
79 changed files with 10501 additions and 38 deletions
+208
View File
@@ -0,0 +1,208 @@
"""Вложения (lab4): потоковая загрузка PNG/JPEG в приватный S3, метаданные в PostgreSQL, авторизованное скачивание."""
import asyncio
import hashlib
import tempfile
import unicodedata
import uuid
from . import config, images, telemetry
from .web import ApiError, bad, format_dt, log, parse_uuid, unavailable
ALLOWED_TYPES = ("image/png", "image/jpeg")
VARIANT_NAMES = ("original", "thumbnail", "crop", "optimized")
ATT_COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.status, a.error_code, "
"a.s3_key, a.created_at, a.updated_at, a.pipeline_version")
VARIANT_ORDER = {"thumbnail": 0, "crop": 1, "optimized": 2}
def attachment_json(row, variants=()):
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), "filename": row["filename"],
"content_type": row["content_type"], "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"],
"variants": list(variants), "error_code": row["error_code"], "created_at": format_dt(row["created_at"]),
"updated_at": format_dt(row["updated_at"])}
def valid_filename(query):
values = query.get("filename")
if not values or len(values) != 1:
raise bad("Требуется параметр filename")
name = values[0]
if not 1 <= len(name) <= 255 or not name.strip():
raise bad("filename: длина 1–255 символов")
try:
name.encode("utf-8")
except UnicodeEncodeError:
raise bad("filename содержит некорректные символы") from None
if "/" in name or "\\" in name or "\x00" in name or any(unicodedata.category(ch) == "Cc" for ch in name):
raise bad("filename не должен содержать / \\ NUL и управляющие символы")
return name
class StreamResponse:
"""Потоковый ответ: тело читается из S3 порциями и сразу отдаётся клиенту (без загрузки файла в память)."""
def __init__(self, content_type, length, stream):
self.content_type, self.length, self.stream = content_type, length, stream
async def send(self, send, request_id, instance_id):
headers = [(b"content-type", self.content_type.encode()), (b"content-length", str(self.length).encode()),
(b"x-request-id", request_id.encode()), (b"x-content-type-options", b"nosniff"),
(b"cache-control", b"private, no-store")]
if instance_id:
headers.append((b"x-instance-id", instance_id.encode()))
await send({"type": "http.response.start", "status": 200, "headers": headers})
try:
while True:
chunk = await self.stream.read(config.DOWNLOAD_CHUNK_BYTES)
if not chunk:
break
await send({"type": "http.response.body", "body": chunk, "more_body": True})
finally:
self.stream.close()
await send({"type": "http.response.body", "body": b""})
class AttachmentMixin:
"""Подмешивается в Messenger: требует self.db, self.s3, self.authenticate()."""
def _init_attachments(self):
self._upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY)
self._upload_waiting = 0
async def _receive_upload(self, req):
"""Принимает тело в SpooledTemporaryFile (до 1 MiB в памяти, далее диск), считает SHA-256, ограничивает размер."""
declared = req.headers.get("content-length")
too_large = ApiError(413, "payload_too_large", "Файл больше 10 MiB", {"connection": "close"})
if declared is not None:
if not declared.isdigit():
raise bad("Некорректный Content-Length")
if int(declared) > config.MAX_UPLOAD_BYTES:
raise too_large
spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024)
digest, size = hashlib.sha256(), 0
try:
while True:
event = await req.receive()
if event["type"] == "http.disconnect":
raise bad("Соединение закрыто клиентом")
chunk = event.get("body", b"")
size += len(chunk)
if size > config.MAX_UPLOAD_BYTES:
raise too_large
digest.update(chunk)
spool.write(chunk)
if not event.get("more_body"):
break
except BaseException:
spool.close()
raise
if size == 0:
spool.close()
raise bad("Пустой файл")
return spool, digest.hexdigest(), size
async def upload_attachment(self, req, params):
user_id = await self.authenticate(req)
chat_id = await self._require_member(user_id, params)
filename = valid_filename(req.query)
media = req.headers.get("content-type", "").split(";")[0].strip().lower()
if media not in ALLOWED_TYPES:
raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: image/png или image/jpeg")
backlog = await self.db.fetchval("SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')")
if backlog >= config.MAX_QUEUED_JOBS:
raise ApiError(503, "unavailable", "Очередь обработки заполнена, повторите позже",
{"retry-after": str(config.BACKLOG_RETRY_AFTER_SECONDS)})
if self._upload_waiting >= config.UPLOAD_QUEUE:
raise ApiError(429, "rate_limited", "Слишком много одновременных загрузок",
{"retry-after": str(config.RETRY_AFTER_SECONDS)})
self._upload_waiting += 1
try:
await self._upload_slots.acquire() # ограничение одновременных загрузок на реплику (RAM/диск/CPU)
finally:
self._upload_waiting -= 1
try:
spool, sha256, size = await self._receive_upload(req)
try:
try:
loop = asyncio.get_running_loop()
content_type, width, height = await loop.run_in_executor(self.image_pool, images.validate, spool, media)
except images.ImageRejected as exc:
raise ApiError(exc.status, exc.code, exc.message) from None
att_id = uuid.uuid4()
staging_key = "staging/%s" % att_id
final_key = "att/%s/original" % att_id
spool.seek(0)
await self.s3.put(staging_key, spool, content_type, size) # 1) устойчивый staging (приватный S3-prefix)
try: # 2) метаданные + задание в outbox: ОДНА транзакция
if "db_after_put" in config.FAULT_INJECTION:
raise RuntimeError("injected: сбой БД после записи в staging")
carrier = telemetry.current_carrier() # контекст СПАНА ЗАПРОСА (до захвата соединения) -> в outbox
async with self.db.conn() as c:
async with c.transaction():
row = await c.fetchrow(
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, "
"height, status, s3_key, staging_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'queued', $10, $11) "
"RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, error_code, "
"s3_key, created_at, updated_at",
att_id, uuid.UUID(chat_id), uuid.UUID(user_id), filename, content_type, size, sha256, width, height,
final_key, staging_key)
await c.execute(
"INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) "
"VALUES ($1, $2, 1, 1, $3, $4)", uuid.uuid4(), att_id, carrier.get("traceparent"),
carrier.get("tracestate"))
except BaseException:
try:
if "s3_delete_fails" in config.FAULT_INJECTION:
raise OSError("injected: компенсирующее удаление не удалось")
await self.s3.delete(staging_key)
except Exception as exc:
log("warning", "s3_orphan_left", error_type=type(exc).__name__)
raise
finally:
spool.close()
finally:
self._upload_slots.release()
self.relay.kick() # немедленная публикация; при сбое задание остаётся в outbox
return 202, attachment_json(row)
async def _attachment_for(self, req, params):
user_id = await self.authenticate(req)
att_id = parse_uuid(params["attachment_id"], "attachment_id")
row = await self.db.fetchrow(
"SELECT " + ATT_COLUMNS + " FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 "
"WHERE a.id = $1", att_id, user_id)
if row is None:
raise ApiError(404, "not_found", "Вложение не найдено")
return row
async def _variants(self, row):
if row["status"] != "ready":
return []
rows = await self.db.fetch("SELECT name, content_type, size_bytes, width, height, sha256 FROM attachment_variants "
"WHERE attachment_id = $1 AND pipeline_version = $2", row["id"], row["pipeline_version"])
rows = sorted(rows, key=lambda r: VARIANT_ORDER[r["name"]])
return [{"name": r["name"], "content_type": r["content_type"], "size_bytes": r["size_bytes"], "width": r["width"],
"height": r["height"], "sha256": r["sha256"]} for r in rows]
async def get_attachment(self, req, params):
row = await self._attachment_for(req, params)
return 200, attachment_json(row, await self._variants(row))
async def download_attachment(self, req, params):
row = await self._attachment_for(req, params)
variant = "original"
if "variant" in req.query:
if len(req.query["variant"]) != 1 or req.query["variant"][0] not in VARIANT_NAMES:
raise bad("variant: original | thumbnail | crop | optimized")
variant = req.query["variant"][0]
if row["status"] != "ready":
raise ApiError(409, "conflict", "Вложение ещё обрабатывается или обработка не удалась")
key, content_type = row["s3_key"], row["content_type"]
if variant != "original":
v = await self.db.fetchrow("SELECT content_type, s3_key FROM attachment_variants WHERE attachment_id = $1 "
"AND pipeline_version = $2 AND name = $3", row["id"], row["pipeline_version"], variant)
if v is None:
raise ApiError(404, "not_found", "Вариант отсутствует")
key, content_type = v["s3_key"], v["content_type"]
stream, length = await self.s3.open_stream(key)
return 200, StreamResponse(content_type, length, stream)
+67
View File
@@ -0,0 +1,67 @@
"""RabbitMQ: durable quorum-очередь, persistent-сообщения, publisher confirms, DLX для нечитаемых сообщений.
В брокер попадают только служебные поля задания (jobs.schema.json): ID, версия пайплайна, номер попытки, время.
Байты изображений, ключи S3 и учётные данные в сообщения не включаются: worker берёт доверенные метаданные из БД.
"""
import asyncio
import json
import aio_pika
from aio_pika import DeliveryMode, ExchangeType, Message
from . import config
from .web import log
QUEUE = "attachment.process"
DEAD_QUEUE = "attachment.process.dead"
DEAD_EXCHANGE = "attachment.dlx"
class Broker:
def __init__(self):
self.connection = None
self.channel = None
self.queue = None
self._lock = asyncio.Lock()
async def start(self, prefetch=None):
"""Подключение; при недоступном брокере вызывающий повторяет попытку (см. ensure)."""
self.connection = await aio_pika.connect_robust(config.BROKER_URL, timeout=config.BROKER_TIMEOUT,
client_properties={"connection_name": config.SERVICE_NAME})
self.channel = await self.connection.channel(publisher_confirms=True)
if prefetch:
await self.channel.set_qos(prefetch_count=prefetch)
dlx = await self.channel.declare_exchange(DEAD_EXCHANGE, ExchangeType.DIRECT, durable=True)
dead = await self.channel.declare_queue(DEAD_QUEUE, durable=True, arguments={"x-queue-type": "quorum"})
await dead.bind(dlx, routing_key="dead")
self.queue = await self.channel.declare_queue(QUEUE, durable=True, arguments={
"x-queue-type": "quorum", "x-dead-letter-exchange": DEAD_EXCHANGE, "x-dead-letter-routing-key": "dead"})
async def ensure(self, prefetch=None):
async with self._lock:
if self.channel is None or self.channel.is_closed:
await self.start(prefetch)
async def stop(self):
if self.connection:
try:
await asyncio.wait_for(self.connection.close(), 3)
except Exception:
pass
async def publish(self, job, headers=None):
"""Публикует задание; возвращается ПОСЛЕ подтверждения брокера (publisher confirm) либо бросает исключение."""
await self.ensure()
body = json.dumps(job, separators=(",", ":")).encode("utf-8")
message = Message(body, delivery_mode=DeliveryMode.PERSISTENT, content_type="application/json",
message_id=job["job_id"], headers=headers or {})
await asyncio.wait_for(self.channel.default_exchange.publish(message, routing_key=QUEUE, mandatory=True),
config.BROKER_TIMEOUT)
async def queue_depth(self):
await self.ensure()
q = await self.channel.declare_queue(QUEUE, durable=True, passive=True)
return q.declaration_result.message_count
async def ping(self):
return self.channel is not None and not self.channel.is_closed
+112
View File
@@ -0,0 +1,112 @@
"""Конфигурация из переменных окружения (см. .env.example)."""
import os
def _int(name, default):
value = os.environ.get(name)
return int(value) if value not in (None, "") else default
def _float(name, default):
value = os.environ.get(name)
return float(value) if value not in (None, "") else default
def _secret(name, default=""):
"""Значение из переменной NAME либо из файла NAME_FILE (Docker secrets); файл читается один раз."""
if os.environ.get(name):
return os.environ[name]
path = os.environ.get(name + "_FILE")
if path and os.path.exists(path):
with open(path, encoding="utf-8") as fh:
return fh.read().strip()
return default
APP_HOST = os.environ.get("APP_HOST", "0.0.0.0")
APP_PORT = _int("APP_PORT", 8080)
APP_ENV = os.environ.get("APP_ENV", "lab")
DATABASE_URL = _secret("DATABASE_URL")
DB_POOL_MIN = _int("DB_POOL_MIN", 2)
DB_POOL_MAX = _int("DB_POOL_MAX", 10)
DB_ACQUIRE_TIMEOUT = _float("DB_ACQUIRE_TIMEOUT", 2.0) # ожидание свободного соединения пула
DB_COMMAND_TIMEOUT = _float("DB_COMMAND_TIMEOUT", 5.0) # таймаут одного запроса
DB_CONNECT_TIMEOUT = _float("DB_CONNECT_TIMEOUT", 2.0)
VALKEY_URL = os.environ.get("VALKEY_URL", "") # пусто = кэш сессий выключен
VALKEY_TIMEOUT = _float("VALKEY_TIMEOUT", 0.2)
VALKEY_BREAKER_SECONDS = _float("VALKEY_BREAKER_SECONDS", 5.0)
SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600)
SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 60)
ARGON2_TIME_COST = _int("ARGON2_TIME_COST", 2)
ARGON2_MEMORY_KIB = _int("ARGON2_MEMORY_KIB", 19456) # 19 MiB, рекомендация OWASP
ARGON2_PARALLELISM = _int("ARGON2_PARALLELISM", 1)
HASH_CONCURRENCY = _int("HASH_CONCURRENCY", 4)
MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536)
MAX_INFLIGHT = _int("MAX_INFLIGHT", 64)
LIMIT_CONCURRENCY = _int("LIMIT_CONCURRENCY", 1000)
RETRY_AFTER_SECONDS = _int("RETRY_AFTER_SECONDS", 1)
MIGRATIONS_DIR = os.environ.get("MIGRATIONS_DIR", os.path.join(os.path.dirname(__file__), "..", "..", "migrations"))
DRAIN_SECONDS = _float("DRAIN_SECONDS", 4.0) # после SIGTERM: readiness=503, сервер ещё принимает запросы
GRACEFUL_SHUTDOWN_SECONDS = _int("GRACEFUL_SHUTDOWN_SECONDS", 8)
KEEPALIVE_SECONDS = _int("KEEPALIVE_SECONDS", 5)
IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 24 * 3600)
# --- S3 / вложения (lab4) ---
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "")
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
S3_ACCESS_KEY = _secret("S3_ACCESS_KEY")
S3_SECRET_KEY = _secret("S3_SECRET_KEY")
S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 2.0)
S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 15.0)
S3_POOL = _int("S3_POOL", 20)
MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024) # 10 MiB = 10 485 760
MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40_000_000)
UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 4) # одновременных загрузок на реплику
UPLOAD_QUEUE = _int("UPLOAD_QUEUE", 16) # ожидающих загрузок; сверх — 429
IMAGE_THREADS = _int("IMAGE_THREADS", 2)
DOWNLOAD_CHUNK_BYTES = _int("DOWNLOAD_CHUNK_BYTES", 65536)
ORPHAN_GRACE_SECONDS = _int("ORPHAN_GRACE_SECONDS", 900) # объект без метаданных моложе этого возраста не трогаем
FAULT_INJECTION = {x for x in os.environ.get("FAULT_INJECTION", "").split(",") if x} # db_after_put, s3_delete_fails (эксперименты)
# --- lab5: брокер, outbox, worker ---
SERVICE_NAME = os.environ.get("SERVICE_NAME", "messenger-api")
BROKER_URL = _secret("BROKER_URL")
BROKER_TIMEOUT = _float("BROKER_TIMEOUT", 5.0)
RELAY_BATCH = _int("RELAY_BATCH", 50)
RELAY_INTERVAL = _float("RELAY_INTERVAL", 1.0)
WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2) # prefetch == число одновременных заданий == потоков обработки
LEASE_SECONDS = _int("LEASE_SECONDS", 60) # аренда задания; истекла -> sweeper возвращает в очередь
JOB_TIMEOUT = _float("JOB_TIMEOUT", 45.0) # лимит времени одной обработки (меньше lease)
MAX_ATTEMPTS = _int("MAX_ATTEMPTS", 5)
RETRY_BASE_DELAY = _float("RETRY_BASE_DELAY", 2.0)
RETRY_MAX_DELAY = _float("RETRY_MAX_DELAY", 60.0)
SWEEP_INTERVAL = _float("SWEEP_INTERVAL", 5.0)
STAGING_FAILED_RETENTION_SECONDS = _int("STAGING_FAILED_RETENTION_SECONDS", 7 * 24 * 3600)
MAX_QUEUED_JOBS = _int("MAX_QUEUED_JOBS", 200) # backpressure: при превышении upload -> 503 + Retry-After
BACKLOG_RETRY_AFTER_SECONDS = _int("BACKLOG_RETRY_AFTER_SECONDS", 5)
# --- lab6: наблюдаемость ---
OTEL_ENABLED = os.environ.get("OTEL_ENABLED", "1") not in ("0", "false", "")
OTEL_ENDPOINT = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318")
OTEL_TRACE_RATIO = _float("OTEL_TRACES_SAMPLER_ARG", 1.0) # head-sampling: доля трасс (ParentBased)
OTEL_METRIC_INTERVAL_MS = _int("OTEL_METRIC_EXPORT_INTERVAL", 5000)
OTEL_QUEUE_SIZE = _int("OTEL_QUEUE_SIZE", 2048)
OTEL_BATCH_SIZE = _int("OTEL_BATCH_SIZE", 512) # спанов в одном экспорте (max_export_batch_size) # ограниченная очередь SDK; при переполнении данные отбрасываются
SERVICE_VERSION = os.environ.get("APP_REVISION", "dev")
INSTANCE_ID = (os.environ.get("HOSTNAME") or "local")[:12]
# Внедрение отказов в HTTP-путь для экспериментов наблюдаемости (по умолчанию пусто): "http500:0.02" — 2 % запросов дают 500;
# "slow:0.01:0.5" — 1 % запросов задерживаются на 0.5 с. Health-маршруты не затрагиваются.
def _fault_rate(prefix):
for item in FAULT_INJECTION:
if item.startswith(prefix + ":"):
return item.split(":")[1:]
return None
HTTP_FAULT_500 = _fault_rate("http500")
HTTP_FAULT_SLOW = _fault_rate("slow")
+88
View File
@@ -0,0 +1,88 @@
"""PostgreSQL: ограниченный пул asyncpg, таймауты, перевод сбоев зависимости в 503."""
import asyncio
from contextlib import asynccontextmanager
import asyncpg
from . import config, telemetry
from .web import log, unavailable
DEPENDENCY_ERRORS = (asyncpg.PostgresConnectionError, asyncpg.InterfaceError, asyncpg.exceptions.OperatorInterventionError,
asyncpg.exceptions.InsufficientResourcesError, asyncio.TimeoutError, OSError)
_RETRYABLE = (asyncpg.ConnectionDoesNotExistError, asyncpg.InterfaceError, ConnectionResetError)
class Database:
def __init__(self):
self.pool = None
self.waiting = 0
telemetry.gauge("db.client.connections.usage", self._usage, "{connection}", "Соединения пула PostgreSQL")
telemetry.gauge("db.client.connections.waiting", lambda: [(self.waiting, {})], "{request}", "Запросы, ожидающие соединение")
def _usage(self):
if self.pool is None:
return []
size, idle = self.pool.get_size(), self.pool.get_idle_size()
return [(size - idle, {"state": "used"}), (idle, {"state": "idle"})]
async def start(self):
if not config.DATABASE_URL:
log("error", "database_url_missing")
return
# min_size=0: приложение стартует и без БД (readiness покажет 503), пул подключается лениво.
self.pool = await asyncpg.create_pool(
dsn=config.DATABASE_URL, min_size=0, max_size=config.DB_POOL_MAX,
command_timeout=config.DB_COMMAND_TIMEOUT, timeout=config.DB_CONNECT_TIMEOUT,
max_inactive_connection_lifetime=60,
server_settings={"application_name": "messenger-api", "timezone": "UTC"})
async def stop(self):
if self.pool:
try:
await asyncio.wait_for(self.pool.close(), 3)
except Exception:
self.pool.terminate()
@asynccontextmanager
async def conn(self):
if self.pool is None:
raise unavailable("База данных не настроена")
try:
self.waiting += 1
try:
acquired = await self.pool.acquire(timeout=config.DB_ACQUIRE_TIMEOUT)
finally:
self.waiting -= 1
try:
with telemetry.child_span("db.connection", "client", {"db.system": "postgresql"}):
yield acquired
finally:
await self.pool.release(acquired)
except DEPENDENCY_ERRORS as exc:
log("warning", "db_unavailable", error_type=type(exc).__name__)
raise unavailable() from exc
async def _run(self, method, sql, args, retry):
for attempt in (0, 1):
try:
async with self.conn() as c:
return await getattr(c, method)(sql, *args)
except Exception as exc:
if attempt == 0 and retry and isinstance(exc.__cause__, _RETRYABLE):
continue # разорванное соединение пула (рестарт БД): один повтор только для чтения
raise
async def fetch(self, sql, *args):
return await self._run("fetch", sql, args, True)
async def fetchrow(self, sql, *args, retry=True):
return await self._run("fetchrow", sql, args, retry)
async def fetchval(self, sql, *args):
return await self._run("fetchval", sql, args, True)
async def execute(self, sql, *args):
return await self._run("execute", sql, args, False)
async def ping(self):
return await asyncio.wait_for(self.fetchval("SELECT 1"), 1.0)
+102
View File
@@ -0,0 +1,102 @@
"""Проверка загружаемых изображений ДО полного декодирования: сигнатура, формат Pillow, размеры, целостность.
PNG/JPEG определяются по содержимому (а не по Content-Type). Лимит пикселей проверяется по заголовку,
поэтому «decompression bomb» отклоняется без декодирования. Целостность: PNG — verify() (CRC всех чанков, без
распаковки в память), JPEG — декодирование в режиме draft (1/8 размера => около 1/64 памяти).
"""
import struct
import zlib
from PIL import Image, UnidentifiedImageError
from . import config
Image.MAX_IMAGE_PIXELS = None # лимит применяем сами по заголовку (ниже), без предупреждений Pillow
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
class ImageRejected(Exception):
def __init__(self, status, code, message):
super().__init__(message)
self.status, self.code, self.message = status, code, message
def sniff(head):
if head.startswith(PNG_SIGNATURE):
return "image/png"
if head[:3] == b"\xff\xd8\xff":
return "image/jpeg"
return None
def _check_png_stream(fileobj, width, height):
"""Целостность потока IDAT без аллокации изображения: распаковка порциями, проверка конца потока и размера данных."""
fileobj.seek(8)
inflater = zlib.decompressobj()
total, channels, bit_depth, interlaced = 0, 0, 0, 0
while True:
head = fileobj.read(8)
if len(head) < 8:
raise ValueError("truncated")
length, kind = struct.unpack(">I4s", head)
if length > 0x7FFFFFFF:
raise ValueError("chunk")
if kind == b"IHDR":
data = fileobj.read(length)
bit_depth, color_type, _, _, interlaced = struct.unpack(">BBBBB", data[8:13])
channels = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}[color_type]
fileobj.read(4)
elif kind == b"IDAT":
remaining = length
while remaining:
piece = fileobj.read(min(remaining, 65536))
if not piece:
raise ValueError("truncated")
remaining -= len(piece)
pending = piece
while pending:
total += len(inflater.decompress(pending, 1 << 20))
pending = inflater.unconsumed_tail
fileobj.read(4)
elif kind == b"IEND":
break
else:
fileobj.seek(length + 4, 1)
if not inflater.eof:
raise ValueError("zlib stream incomplete")
if not interlaced:
row = (width * channels * bit_depth + 7) // 8
if total != height * (row + 1):
raise ValueError("unexpected image data size")
def validate(fileobj, declared_type):
"""Возвращает (content_type, width, height) либо бросает ImageRejected (415 — не то/повреждено, 413 — много пикселей)."""
fileobj.seek(0)
detected = sniff(fileobj.read(16))
if detected is None or detected != declared_type:
raise ImageRejected(415, "unsupported_media_type", "Содержимое не соответствует PNG/JPEG или заявленному Content-Type")
fileobj.seek(0)
try:
with Image.open(fileobj) as img:
width, height = img.size
fmt = img.format
except (UnidentifiedImageError, OSError, ValueError, SyntaxError, Image.DecompressionBombError):
raise ImageRejected(415, "unsupported_media_type", "Изображение повреждено или не поддерживается") from None
if fmt not in ("PNG", "JPEG") or width < 1 or height < 1:
raise ImageRejected(415, "unsupported_media_type", "Неподдерживаемый формат изображения")
if width * height > config.MAX_IMAGE_PIXELS:
raise ImageRejected(413, "payload_too_large", "Слишком много пикселей в изображении")
fileobj.seek(0)
try:
img = Image.open(fileobj)
if fmt == "PNG":
img.verify()
_check_png_stream(fileobj, width, height)
else:
img.draft("RGB", (max(1, width // 8), max(1, height // 8)))
img.load()
img.close()
except Exception: # любая ошибка декодера = повреждённый файл
raise ImageRejected(415, "unsupported_media_type", "Изображение повреждено") from None
return detected, width, height
+464
View File
@@ -0,0 +1,464 @@
"""ASGI-приложение мессенджера (lab2): REST API, PostgreSQL, сессии Bearer, кэш Valkey."""
import asyncio
import contextlib
from concurrent.futures import ThreadPoolExecutor
import base64
import binascii
import hashlib
import json
import os
import re
import time
import uuid
import asyncpg
from . import config, passwords, telemetry
from .attachments import AttachmentMixin, StreamResponse
from .db import DEPENDENCY_ERRORS, Database
from .broker import Broker
from .outbox import Relay
from .s3 import S3
from .sessions import SessionCache, Sessions
from .web import (ApiError, Request, bad, dt_to_us, encode_cursor, format_dt, log, new_request_id, page_params,
parse_json_object, parse_uuid, respond, unavailable, us_to_dt, valid_text, USERNAME_RE)
ROUTES = [
("health_live", re.compile(r"/health/live"), {"GET"}),
("health_ready", re.compile(r"/health/ready"), {"GET"}),
("users", re.compile(r"/api/v1/users"), {"POST"}),
("users_me", re.compile(r"/api/v1/users/me"), {"GET"}),
("create_session", re.compile(r"/api/v1/auth/sessions"), {"POST"}),
("session_current", re.compile(r"/api/v1/auth/sessions/current"), {"DELETE"}),
("chats", re.compile(r"/api/v1/chats"), {"GET", "POST"}),
("chat", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)"), {"GET"}),
("messages", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)/messages"), {"GET", "POST"}),
("upload_attachment", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)/attachments"), {"POST"}),
("get_attachment", re.compile(r"/api/v1/attachments/(?P<attachment_id>[^/]+)"), {"GET"}),
("download_attachment", re.compile(r"/api/v1/attachments/(?P<attachment_id>[^/]+)/content"), {"GET"}),
]
ROUTE_LABELS = {
"health_live": "/health/live", "health_ready": "/health/ready", "users": "/api/v1/users",
"users_me": "/api/v1/users/me", "create_session": "/api/v1/auth/sessions",
"session_current": "/api/v1/auth/sessions/current", "chats": "/api/v1/chats",
"chat": "/api/v1/chats/{chat_id}", "messages": "/api/v1/chats/{chat_id}/messages",
"upload_attachment": "/api/v1/chats/{chat_id}/attachments", "get_attachment": "/api/v1/attachments/{attachment_id}",
"download_attachment": "/api/v1/attachments/{attachment_id}/content",
}
TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{32,128}")
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
BEARER = 'Bearer realm="messenger"'
BASIC = 'Basic realm="messenger", charset="UTF-8"'
# Идентификатор реплики: имя хоста контейнера (12 hex, == короткий ID контейнера) -> легко сопоставить с docker ps.
INSTANCE_ID = os.environ.get("INSTANCE_ID") or (os.environ.get("HOSTNAME") or uuid.uuid4().hex)[:12]
MSG_ATTS = ("COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) FROM message_attachments ma "
"WHERE ma.message_id = messages.id), '{}'::uuid[]) AS attachment_ids")
CHAT_SELECT = ("SELECT c.id, c.title, c.created_at, "
"(SELECT array_agg(m2.user_id ORDER BY m2.ord) FROM chat_members m2 WHERE m2.chat_id = c.id) AS member_ids ")
def user_json(row):
return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"],
"created_at": format_dt(row["created_at"])}
def chat_json(row):
return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(x) for x in row["member_ids"]],
"created_at": format_dt(row["created_at"])}
def message_json(row):
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]),
"text": row["text"], "created_at": format_dt(row["created_at"]),
"attachment_ids": [str(x) for x in (row["attachment_ids"] or [])]}
def unauthorized(challenge=BEARER, message="Требуется действительная сессия"):
return ApiError(401, "unauthorized", message, {"www-authenticate": challenge})
class Messenger(AttachmentMixin):
def __init__(self):
self.db = Database()
self.s3 = S3()
self.broker = Broker()
self.relay = Relay(self.db, self.broker)
self.image_pool = ThreadPoolExecutor(max_workers=config.IMAGE_THREADS, thread_name_prefix="image")
self._init_attachments()
self.cache = SessionCache()
self.sessions = Sessions(self.db, self.cache)
self.inflight = 0
self.draining = False
# Метрики очереди считает API (любая реплика), а не worker: при остановке worker алерт по возрасту задач продолжает работать.
self.queue_stats = (0, 0.0)
telemetry.gauge("messenger.queue.depth", lambda: [(self.queue_stats[0], {"queue.name": "attachment.process"})], "{job}", "queued + processing")
telemetry.gauge("messenger.queue.oldest_job_age", lambda: [(self.queue_stats[1], {"queue.name": "attachment.process"})], "s", "Возраст старейшего задания")
# --- аутентификация ------------------------------------------------------
@staticmethod
def bearer_token(req):
header = req.headers.get("authorization", "")
scheme, _, token = header.partition(" ")
if scheme.lower() != "bearer" or not TOKEN_RE.fullmatch(token.strip()):
raise unauthorized()
return token.strip()
async def authenticate(self, req):
token = self.bearer_token(req)
user_id = await self.sessions.resolve(token)
if user_id is None:
raise unauthorized()
req.user_id, req.token = user_id, token
return user_id
# --- обработчики -----------------------------------------------------------
async def health_live(self, req, params):
return 200, {"status": "ok"}
async def health_ready(self, req, params):
# Критичная зависимость — PostgreSQL. Valkey — необязательный кэш: его отказ не останавливает API.
if self.draining:
raise unavailable("Сервис останавливается")
try:
await self.db.ping()
except ApiError:
raise
except Exception:
raise unavailable() from None
return 200, {"status": "ok"}
async def users(self, req, params):
req.require_json_media_type()
data = parse_json_object(await req.body(), {"username", "display_name", "password"},
{"username", "display_name", "password"})
username = data["username"]
if not isinstance(username, str) or not USERNAME_RE.fullmatch(username):
raise bad("username должен соответствовать [a-z0-9_]{3,32}")
display_name = valid_text(data["display_name"], "display_name", 100)
password = valid_text(data["password"], "password", 128, allow_empty=True)
if len(password) < 12:
raise bad("password должен содержать от 12 до 128 символов")
pw_hash = await passwords.hash_password(password)
row = await self.db.fetchrow(
"INSERT INTO users (id, username, display_name, password_hash) VALUES ($1, $2, $3, $4) "
"ON CONFLICT (username) DO NOTHING RETURNING id, username, display_name, created_at",
uuid.uuid4(), username, display_name, pw_hash)
if row is None:
raise ApiError(409, "conflict", "username уже занят")
return 201, user_json(row)
async def users_me(self, req, params):
user_id = await self.authenticate(req)
row = await self.db.fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", user_id)
if row is None:
raise unauthorized()
return 200, user_json(row)
async def create_session(self, req, params):
header = req.headers.get("authorization", "")
scheme, _, value = header.partition(" ")
if scheme.lower() != "basic" or not value.strip():
raise unauthorized(BASIC, "Неверные учётные данные")
try:
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
username, sep, password = decoded.partition(":")
if not sep:
raise ValueError
except (binascii.Error, ValueError, UnicodeDecodeError):
raise unauthorized(BASIC, "Неверные учётные данные") from None
row = None
if USERNAME_RE.fullmatch(username):
row = await self.db.fetchrow("SELECT id, username, display_name, created_at, password_hash FROM users "
"WHERE username = $1", username)
ok = await passwords.verify_password(row["password_hash"] if row else None, password)
if not ok:
raise unauthorized(BASIC, "Неверные учётные данные")
token, expires_at = await self.sessions.create(row["id"])
return 201, {"token": token, "token_type": "Bearer", "expires_at": format_dt(expires_at), "user": user_json(row)}
async def session_current(self, req, params):
await self.authenticate(req)
await self.sessions.revoke(req.token)
return 204, None
async def chats(self, req, params):
user_id = await self.authenticate(req)
if req.method == "GET":
limit, after = page_params(req.query, "c", user_id)
if after is None:
rows = await self.db.fetch(
CHAT_SELECT + "FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = $1 "
"ORDER BY m.chat_created_at, m.chat_id LIMIT $2", user_id, limit + 1)
else:
rows = await self.db.fetch(
CHAT_SELECT + "FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = $1 "
"AND (m.chat_created_at, m.chat_id) > ($2, $3) ORDER BY m.chat_created_at, m.chat_id LIMIT $4",
user_id, us_to_dt(after[0]), uuid.UUID(after[1]), limit + 1)
more = len(rows) > limit
rows = rows[:limit]
cursor = encode_cursor("c", user_id, (dt_to_us(rows[-1]["created_at"]), str(rows[-1]["id"]))) if more else None
return 200, {"items": [chat_json(r) for r in rows], "next_cursor": cursor}
req.require_json_media_type()
data = parse_json_object(await req.body(), {"title", "member_ids"}, {"title", "member_ids"})
title = valid_text(data["title"], "title", 100)
members = data["member_ids"]
if not isinstance(members, list) or not 1 <= len(members) <= 100:
raise bad("member_ids должен содержать от 1 до 100 элементов")
ids = [parse_uuid(m, "member_id") for m in members]
if len(set(ids)) != len(ids):
raise bad("member_ids содержит дубликаты")
if user_id not in ids:
raise bad("Создатель должен быть в member_ids")
chat_id = uuid.uuid4()
uuids = [uuid.UUID(i) for i in ids]
try:
async with self.db.conn() as c:
async with c.transaction(): # чат и участники создаются атомарно
# Существование участников проверяет внешний ключ chat_members.user_id: при неизвестном
# участнике вставка падает и откатывается вся транзакция, включая уже созданный чат.
created = await c.fetchval("INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING created_at",
chat_id, title)
await c.execute(
"INSERT INTO chat_members (chat_id, user_id, chat_created_at, ord) "
"SELECT $1, u, $2, o FROM unnest($3::uuid[]) WITH ORDINALITY AS t(u, o)",
chat_id, created, uuids)
except asyncpg.ForeignKeyViolationError:
raise bad("Неизвестный участник") from None
return 201, {"id": str(chat_id), "title": title, "member_ids": ids, "created_at": format_dt(created)}
async def _require_member(self, user_id, params):
chat_id = parse_uuid(params["chat_id"], "chat_id")
ok = await self.db.fetchval("SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, user_id)
if not ok:
raise ApiError(404, "not_found", "Чат не найден")
return chat_id
async def chat(self, req, params):
user_id = await self.authenticate(req)
chat_id = parse_uuid(params["chat_id"], "chat_id")
row = await self.db.fetchrow(
CHAT_SELECT + "FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2 WHERE c.id = $1",
chat_id, user_id)
if row is None:
raise ApiError(404, "not_found", "Чат не найден")
return 200, chat_json(row)
async def messages(self, req, params):
user_id = await self.authenticate(req)
if req.method == "POST":
row = await self._post_message(req, parse_uuid(params["chat_id"], "chat_id"), user_id)
return 201, message_json(row)
chat_id = await self._require_member(user_id, params)
if req.method == "GET":
limit, after = page_params(req.query, "m", chat_id)
if after is None:
rows = await self.db.fetch(
"SELECT id, chat_id, sender_id, text, created_at, " + MSG_ATTS + " FROM messages WHERE chat_id = $1 "
"ORDER BY created_at, id LIMIT $2", chat_id, limit + 1)
else:
rows = await self.db.fetch(
"SELECT id, chat_id, sender_id, text, created_at, " + MSG_ATTS + " FROM messages WHERE chat_id = $1 "
"AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT $4",
chat_id, us_to_dt(after[0]), uuid.UUID(after[1]), limit + 1)
more = len(rows) > limit
rows = rows[:limit]
cursor = encode_cursor("m", chat_id, (dt_to_us(rows[-1]["created_at"]), str(rows[-1]["id"]))) if more else None
return 200, {"items": [message_json(r) for r in rows], "next_cursor": cursor}
async def _post_message(self, req, chat_id, user_id):
# Сначала разбор запроса; ошибка валидации (400/415) возвращается только участнику чата — постороннему всегда 404.
try:
key = req.headers.get("idempotency-key", "")
if not IDEMPOTENCY_KEY_RE.fullmatch(key):
raise bad("Требуется заголовок Idempotency-Key ([A-Za-z0-9._:-]{1,128})")
req.require_json_media_type()
data = parse_json_object(await req.body(), {"text"}, {"text", "attachment_ids"})
raw_atts = data.get("attachment_ids", [])
if not isinstance(raw_atts, list) or len(raw_atts) > 10:
raise bad("attachment_ids: массив до 10 элементов")
atts = [parse_uuid(a, "attachment_id") for a in raw_atts]
if len(set(atts)) != len(atts):
raise bad("attachment_ids содержит дубликаты")
text = valid_text(data["text"], "text", 4000, allow_empty=bool(atts))
except ApiError:
await self._require_member(user_id, {"chat_id": chat_id})
raise
# Нормализованное представление запроса: порядок ключей JSON не важен, отсутствие attachment_ids == [].
request_hash = hashlib.sha256(json.dumps({"text": text, "attachment_ids": atts}, sort_keys=True,
ensure_ascii=False).encode("utf-8")).digest()
row = await self.db.fetchrow("SELECT * FROM send_message($1, $2, $3, $4, $5, $6, $7)", uuid.UUID(chat_id),
uuid.UUID(user_id), key, request_hash, text, float(config.IDEMPOTENCY_TTL_SECONDS),
[uuid.UUID(a) for a in atts], retry=False)
if row["outcome"] == "not_member":
raise ApiError(404, "not_found", "Чат не найден")
if row["outcome"] == "attachment_not_found":
raise ApiError(404, "not_found", "Вложение не найдено")
if row["outcome"] == "attachment_failed":
raise ApiError(409, "conflict", "Вложение не удалось обработать")
if bytes(row["request_hash"]) != request_hash:
raise ApiError(409, "conflict", "Idempotency-Key уже использован с другим телом запроса")
return {"id": row["msg_id"], "chat_id": row["msg_chat"], "sender_id": row["msg_sender"], "text": row["msg_text"],
"created_at": row["msg_created"], "attachment_ids": row["msg_atts"]}
# --- ASGI ----------------------------------------------------------------------
async def __call__(self, scope, receive, send):
if scope["type"] == "lifespan":
return await self.lifespan(receive, send)
if scope["type"] != "http":
return
started = time.perf_counter()
request_id = new_request_id()
route_name, status, headers = "unknown", 500, {}
raw_headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope["headers"]}
carrier = {k: raw_headers[k] for k in ("traceparent", "tracestate") if k in raw_headers}
is_health = scope.get("path", "").startswith("/health") # проверки LB раз в секунду: без спанов и access-log (метрики остаются)
span_cm = (contextlib.nullcontext() if is_health else
telemetry.span("HTTP %s" % scope.get("method"), "server", {"http.request.method": scope.get("method")}, carrier))
sp = span_cm.__enter__()
try:
req = Request(scope, receive, request_id)
handler, params, route_name = self.route(req.method, req.path)
telemetry.set_attr(sp, "http.route", ROUTE_LABELS.get(route_name, "unmatched"))
req.span = sp
if not route_name.startswith("health") and self.inflight >= config.MAX_INFLIGHT:
raise ApiError(429, "rate_limited", "Слишком много одновременных запросов",
{"retry-after": str(config.RETRY_AFTER_SECONDS)})
self.inflight += 1
try:
await asyncio.sleep(0)
if not route_name.startswith("health") and (config.HTTP_FAULT_500 or config.HTTP_FAULT_SLOW):
await self.inject_http_faults()
status, body = await handler(req, params)
finally:
self.inflight -= 1
except ApiError as exc:
status, headers = exc.status, exc.headers
body = {"error": {"code": exc.code, "message": exc.message, "request_id": request_id}}
except DEPENDENCY_ERRORS as exc:
log("warning", "dependency_error", request_id=request_id, error_type=type(exc).__name__)
status, headers = 503, {"retry-after": str(config.RETRY_AFTER_SECONDS)}
body = {"error": {"code": "unavailable", "message": "Зависимость временно недоступна", "request_id": request_id}}
except Exception as exc:
log("error", "unhandled_exception", request_id=request_id, error_type=type(exc).__name__)
status = 500
body = {"error": {"code": "internal_error", "message": "Внутренняя ошибка", "request_id": request_id}}
route_label = ROUTE_LABELS.get(route_name, "unmatched")
duration = time.perf_counter() - started
telemetry.set_attr(sp, "http.response.status_code", status)
telemetry.set_attr(sp, "http.request_id", request_id)
if sp is not None:
sp.update_name("HTTP %s %s" % (scope.get("method"), route_label))
if status >= 500:
telemetry.mark_error(sp, "HTTP %d" % status)
telemetry.record_http(scope.get("method"), route_label, status, duration)
if isinstance(body, StreamResponse):
try:
await body.send(send, request_id, INSTANCE_ID)
except Exception as exc: # клиент оборвал загрузку/ошибка S3 посреди потока
log("warning", "stream_aborted", request_id=request_id, error_type=type(exc).__name__)
else:
await respond(send, status, body, request_id, headers, INSTANCE_ID)
if os.environ.get("ACCESS_LOG", "1") != "0" and (not is_health or status >= 500):
log("info", "request", request_id=request_id, method=scope.get("method"),
route=route_label, status=status, duration_ms=round(duration * 1000, 3))
span_cm.__exit__(None, None, None)
async def janitor(self):
"""Удаляет давно просроченные ключи идемпотентности и сессии (безопасно запускать на каждой реплике)."""
import random
await asyncio.sleep(random.uniform(5, 30))
while True:
try:
await self.db.execute("DELETE FROM idempotency_keys WHERE ctid IN (SELECT ctid FROM idempotency_keys "
"WHERE created_at < now() - make_interval(secs => $1) LIMIT 5000)",
float(config.IDEMPOTENCY_TTL_SECONDS) + 3600)
await self.db.execute("DELETE FROM sessions WHERE ctid IN (SELECT ctid FROM sessions "
"WHERE expires_at < now() - interval '1 day' LIMIT 5000)")
except Exception as exc:
log("warning", "janitor_failed", error_type=type(exc).__name__)
await asyncio.sleep(3600)
async def queue_stats_loop(self):
while True:
try:
row = await self.db.fetchrow("SELECT count(*) AS n, coalesce(extract(epoch FROM now() - min(accepted_at)), 0) AS age "
"FROM attachments WHERE status IN ('queued', 'processing')")
self.queue_stats = (row["n"], float(row["age"]))
except Exception:
pass
await asyncio.sleep(2)
@staticmethod
async def inject_http_faults():
import random
if config.HTTP_FAULT_SLOW and random.random() < float(config.HTTP_FAULT_SLOW[0]):
await asyncio.sleep(float(config.HTTP_FAULT_SLOW[1]))
if config.HTTP_FAULT_500 and random.random() < float(config.HTTP_FAULT_500[0]):
raise RuntimeError("injected fault")
def route(self, method, path):
for name, pattern, methods in ROUTES:
m = pattern.fullmatch(path)
if m:
if method not in methods:
raise ApiError(405, "invalid_request", "Метод не поддерживается", {"allow": ", ".join(sorted(methods))})
return getattr(self, name), m.groupdict(), name
raise ApiError(404, "not_found", "Ресурс не найден")
async def lifespan(self, receive, send):
while True:
event = await receive()
if event["type"] == "lifespan.startup":
telemetry.setup(config.SERVICE_NAME)
telemetry.process_metrics()
await self.db.start()
await self.cache.start()
await self.s3.start()
self._relay_task = asyncio.create_task(self.relay.run())
self._queue_task = asyncio.create_task(self.queue_stats_loop())
self._janitor = asyncio.create_task(self.janitor())
log("info", "startup", env=config.APP_ENV, instance=INSTANCE_ID, revision=os.environ.get("APP_REVISION", "dev"), cache=bool(config.VALKEY_URL),
session_ttl=config.SESSION_TTL_SECONDS, pool_max=config.DB_POOL_MAX)
await send({"type": "lifespan.startup.complete"})
elif event["type"] == "lifespan.shutdown":
self.draining = True
self._janitor.cancel()
await self.cache.stop()
telemetry.shutdown()
self.relay.stop()
self._relay_task.cancel()
self._queue_task.cancel()
await self.broker.stop()
await self.s3.stop()
await self.db.stop()
log("info", "shutdown")
await send({"type": "lifespan.shutdown.complete"})
return
async def overloaded_app(scope, receive, send):
"""Ответ uvicorn при превышении --limit-concurrency в контрактном формате."""
if scope["type"] != "http":
return
request_id = new_request_id()
body = {"error": {"code": "unavailable", "message": "Сервер перегружен: слишком много соединений",
"request_id": request_id}}
await respond(send, 503, body, request_id, {"retry-after": str(config.RETRY_AFTER_SECONDS), "connection": "close"}, INSTANCE_ID)
def _install_overload_response():
for name in ("uvicorn.protocols.http.httptools_impl", "uvicorn.protocols.http.h11_impl"):
try:
module = __import__(name, fromlist=["service_unavailable"])
except ImportError:
continue
module.service_unavailable = overloaded_app
def create_app():
_install_overload_response()
return Messenger()
app = create_app()
+64
View File
@@ -0,0 +1,64 @@
"""Применение версионируемых SQL-миграций: python -m app.migrate
Свойства:
* повторный запуск ничего не делает (таблица schema_migrations);
* одновременный запуск нескольких экземпляров безопасен (advisory lock);
* каждая миграция — одна транзакция вместе с записью о применении;
* ждёт готовности PostgreSQL до MIGRATE_WAIT_SECONDS.
"""
import asyncio
import hashlib
import os
import sys
import time
import asyncpg
from . import config
from .web import log
LOCK_ID = 7352001
async def connect(deadline):
while True:
try:
return await asyncpg.connect(config.DATABASE_URL, timeout=3)
except (OSError, asyncpg.PostgresError, asyncio.TimeoutError) as exc:
if time.monotonic() > deadline:
raise
log("info", "waiting_for_database", error_type=type(exc).__name__)
await asyncio.sleep(1)
async def main():
wait = float(os.environ.get("MIGRATE_WAIT_SECONDS", "60"))
conn = await connect(time.monotonic() + wait)
try:
await conn.execute("SELECT pg_advisory_lock($1)", LOCK_ID)
await conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations ("
"version text PRIMARY KEY, checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())")
applied = {r["version"]: r["checksum"] for r in await conn.fetch("SELECT version, checksum FROM schema_migrations")}
directory = os.path.abspath(config.MIGRATIONS_DIR)
files = sorted(f for f in os.listdir(directory) if f.endswith(".sql"))
count = 0
for name in files:
sql = open(os.path.join(directory, name), encoding="utf-8").read()
checksum = hashlib.sha256(sql.encode("utf-8")).hexdigest()
if name in applied:
if applied[name] != checksum:
log("error", "migration_modified", version=name)
sys.exit(2)
continue
async with conn.transaction():
await conn.execute(sql)
await conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES ($1, $2)", name, checksum)
count += 1
log("info", "migration_applied", version=name)
log("info", "migrations_done", applied=count, total=len(files))
finally:
await conn.close()
if __name__ == "__main__":
asyncio.run(main())
+82
View File
@@ -0,0 +1,82 @@
"""Transactional outbox: relay публикует задания из PostgreSQL в брокер.
Гарантия: задание, зафиксированное в БД (COMMIT), будет опубликовано даже если процесс упал между COMMIT и publish.
Relay запускается в каждой реплике API и в worker; строки берутся `FOR UPDATE SKIP LOCKED`, поэтому одну строку
публикует один relay. Если процесс упадёт после подтверждения брокера, но до отметки published_at, задание будет
опубликовано повторно — доставка at-least-once, дубликаты безвредны (обработка идемпотентна).
"""
import asyncio
from datetime import timezone
from . import config, telemetry
from .web import format_dt, log
JOB_TYPE = "attachment.process.v1"
def job_message(row, carrier=None):
msg = {"schema_version": 1, "job_id": str(row["job_id"]), "type": JOB_TYPE, "attachment_id": str(row["attachment_id"]),
"pipeline_version": row["pipeline_version"], "attempt": row["attempt"],
"created_at": format_dt(row["created_at"].astimezone(timezone.utc))}
msg.update(carrier or {}) # traceparent / tracestate (W3C Trace Context) — поля схемы jobs.schema.json
return msg
class Relay:
def __init__(self, db, broker):
self.db, self.broker = db, broker
self._kick = asyncio.Event()
self._stop = False
self.last_error = None
def kick(self):
self._kick.set()
def stop(self):
self._stop = True
self._kick.set()
async def relay_once(self):
published = 0
async with self.db.conn() as c:
async with c.transaction():
rows = await c.fetch(
"SELECT id, job_id, attachment_id, pipeline_version, attempt, created_at, traceparent, tracestate FROM outbox "
"WHERE published_at IS NULL AND available_at <= now() ORDER BY id LIMIT $1 FOR UPDATE SKIP LOCKED",
config.RELAY_BATCH)
for row in rows:
parent = {k: row[k] for k in ("traceparent", "tracestate") if row[k]}
# PRODUCER-спан — дочерний к спану запроса upload (контекст восстановлен из outbox)
with telemetry.span("attachment.process publish", "producer",
{"messaging.system": "rabbitmq", "messaging.destination.name": "attachment.process",
"messaging.operation": "publish", "outbox.attempt": row["attempt"]}, parent):
carrier = telemetry.current_carrier() or parent
await self.broker.publish(job_message(row, carrier), headers=carrier)
await c.execute("UPDATE outbox SET published_at = now() WHERE id = $1", row["id"])
published += 1
return published
def headers_for(self, row):
return {}
async def run(self):
delay = config.RELAY_INTERVAL
while not self._stop:
try:
n = await self.relay_once()
self.last_error = None
delay = config.RELAY_INTERVAL
if n >= config.RELAY_BATCH:
continue # возможно, есть ещё
except asyncio.CancelledError:
raise
except Exception as exc:
if self.last_error != type(exc).__name__:
log("warning", "outbox_relay_failed", error_type=type(exc).__name__)
self.last_error = type(exc).__name__
delay = min(delay * 2, 10.0) # брокер/БД недоступны: пауза с ростом до 10 с, задания остаются в outbox
try:
await asyncio.wait_for(self._kick.wait(), delay)
except asyncio.TimeoutError:
pass
self._kick.clear()
+31
View File
@@ -0,0 +1,31 @@
"""Хеширование паролей: Argon2id (библиотека argon2-cffi), вычисления вне event loop."""
import asyncio
from concurrent.futures import ThreadPoolExecutor
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
from . import config
_hasher = PasswordHasher(time_cost=config.ARGON2_TIME_COST, memory_cost=config.ARGON2_MEMORY_KIB,
parallelism=config.ARGON2_PARALLELISM)
_pool = ThreadPoolExecutor(max_workers=config.HASH_CONCURRENCY, thread_name_prefix="argon2")
# Для неизвестного username проверяем фиктивный хеш: время ответа не выдаёт существование пользователя.
_DUMMY = _hasher.hash("dummy-password-for-timing-equalisation")
async def hash_password(password):
return await asyncio.get_running_loop().run_in_executor(_pool, _hasher.hash, password)
def _verify(stored, password):
try:
return _hasher.verify(stored, password)
except (VerifyMismatchError, VerificationError, InvalidHashError):
return False
async def verify_password(stored, password):
"""stored=None -> проверка фиктивного хеша, результат всегда False."""
ok = await asyncio.get_running_loop().run_in_executor(_pool, _verify, stored or _DUMMY, password)
return ok and stored is not None
+97
View File
@@ -0,0 +1,97 @@
"""Обработка изображений (pipeline_version=1): thumbnail, crop, optimized.
Правила (contracts/README.md):
* thumbnail: JPEG, вписан в 256x256 с сохранением пропорций, БЕЗ увеличения, EXIF orientation применяется, alpha — на белый
фон; размеры округляются вниз, минимум 1 px (320x200 -> 256x160); качество 85, метаданные не сохраняются;
* crop: JPEG, центрированный квадрат 128x128 (маленькие исходники допускается увеличивать), качество 85;
* optimized: WebP, вписан в 1280x1280 без увеличения, quality=80 method=4, EXIF/ICC не сохраняются (ориентация применена).
Память: JPEG декодируется через draft (масштабирование при декодировании), число пикселей ограничено MAX_IMAGE_PIXELS.
"""
import hashlib
import io
import math
from PIL import Image, ImageOps
from . import config
Image.MAX_IMAGE_PIXELS = None
PIPELINE_VERSION = 1
JPEG_QUALITY = 85
WEBP_QUALITY = 80
class PermanentError(Exception):
"""Постоянная ошибка входных данных: повтор не поможет (-> failed, error_code)."""
def __init__(self, code):
super().__init__(code)
self.code = code
def _flatten(img):
"""RGBA/LA/P с прозрачностью -> RGB на белом фоне."""
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
rgba = img.convert("RGBA")
background = Image.new("RGB", rgba.size, (255, 255, 255))
background.paste(rgba, mask=rgba.getchannel("A"))
return background
return img.convert("RGB")
def _fit_size(width, height, box, upscale=False):
scale = min(box / width, box / height)
if not upscale:
scale = min(scale, 1.0)
return max(1, math.floor(width * scale)), max(1, math.floor(height * scale))
def _encode(img, fmt, **params):
buf = io.BytesIO()
img.save(buf, fmt, **params)
data = buf.getvalue()
return data, hashlib.sha256(data).hexdigest()
def process(fileobj):
"""Возвращает список вариантов: dict(name, content_type, data, sha256, width, height). Бросает PermanentError."""
try:
fileobj.seek(0)
src = Image.open(fileobj)
fmt = src.format
width, height = src.size
if fmt not in ("PNG", "JPEG"):
raise PermanentError("invalid_image")
if width * height > config.MAX_IMAGE_PIXELS:
raise PermanentError("image_too_large")
if fmt == "JPEG" and width > 2560 and height > 2560:
# экономия памяти: JPEG декодируется сразу в масштабе 1/2 (результат всё равно вписывается в 1280x1280)
src.draft("RGB", (width // 2, height // 2))
src.load()
img = ImageOps.exif_transpose(src)
except PermanentError:
raise
except MemoryError:
raise
except Exception:
raise PermanentError("invalid_image") from None
base = _flatten(img)
out = []
# thumbnail
tw, th = _fit_size(base.width, base.height, 256)
thumb = base.resize((tw, th), Image.LANCZOS) if (tw, th) != base.size else base
data, sha = _encode(thumb, "JPEG", quality=JPEG_QUALITY, optimize=True)
out.append({"name": "thumbnail", "content_type": "image/jpeg", "data": data, "sha256": sha, "width": tw, "height": th})
# crop 128x128 (центрированный квадрат)
side = min(base.width, base.height)
left, top = (base.width - side) // 2, (base.height - side) // 2
crop = base.crop((left, top, left + side, top + side)).resize((128, 128), Image.LANCZOS)
data, sha = _encode(crop, "JPEG", quality=JPEG_QUALITY, optimize=True)
out.append({"name": "crop", "content_type": "image/jpeg", "data": data, "sha256": sha, "width": 128, "height": 128})
# optimized WebP (прозрачность сохраняется)
opt_src = img.convert("RGBA") if (img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info)) else base
ow, oh = _fit_size(opt_src.width, opt_src.height, 1280)
opt = opt_src.resize((ow, oh), Image.LANCZOS) if (ow, oh) != opt_src.size else opt_src
data, sha = _encode(opt, "WEBP", quality=WEBP_QUALITY, method=4)
out.append({"name": "optimized", "content_type": "image/webp", "data": data, "sha256": sha, "width": ow, "height": oh})
return out
+91
View File
@@ -0,0 +1,91 @@
"""Сверка S3 и PostgreSQL (reconcile): python -m app.reconcile [--apply] [--grace СЕКУНДЫ] [--prefix att/]
Находит и (с --apply) безопасно исправляет расхождения двух хранилищ:
* orphan_object — объект в S3 без строки в attachments (сбой между записью в S3 и COMMIT метаданных, прерванная загрузка);
удаляется ТОЛЬКО если старше grace (по умолчанию ORPHAN_GRACE_SECONDS=900 с): свежий объект может быть
загрузкой, метаданные которой ещё не зафиксированы;
* missing_object — строка attachments (ready) без объекта в S3 (потеря/удаление объекта): строка НЕ удаляется,
статус -> failed, error_code=object_missing (сообщения со ссылкой остаются, скачивание даёт 409);
* size_mismatch — размер объекта не совпадает с size_bytes: статус -> failed, error_code=size_mismatch.
Режим по умолчанию — dry-run (ничего не меняет). Повторный запуск идемпотентен. Используемые (привязанные к сообщению)
вложения и свежие объекты не удаляются никогда: удаляются только объекты, у которых нет строки в БД.
Выход: JSON-отчёт со счётчиками и временем обхода.
"""
import argparse
import asyncio
import json
import time
from datetime import datetime, timedelta, timezone
import asyncpg
from . import config
from .s3 import S3
async def reconcile(apply, grace, prefix):
started = time.perf_counter()
s3 = S3()
await s3.start()
conn = await asyncpg.connect(config.DATABASE_URL, timeout=5)
try:
t0 = time.perf_counter()
rows = await conn.fetch("SELECT id, s3_key, size_bytes, status, created_at FROM attachments")
db_ms = (time.perf_counter() - t0) * 1000
by_key = {r["s3_key"]: r for r in rows}
t0 = time.perf_counter()
objects = {}
async for key, size, modified in s3.list_keys(prefix):
objects[key] = (size, modified)
list_ms = (time.perf_counter() - t0) * 1000
now = datetime.now(timezone.utc)
cutoff = now - timedelta(seconds=grace)
report = {"mode": "apply" if apply else "dry-run", "grace_seconds": grace, "db_rows": len(rows), "s3_objects": len(objects),
"orphan_objects": [], "orphans_skipped_fresh": 0, "missing_objects": [], "size_mismatch": [], "fixed": {"deleted_orphans": 0, "marked_failed": 0}}
for key, (size, modified) in objects.items():
row = by_key.get(key)
if row is None:
if modified < cutoff:
report["orphan_objects"].append(key)
else:
report["orphans_skipped_fresh"] += 1
elif row["status"] == "ready" and row["size_bytes"] != size:
report["size_mismatch"].append(str(row["id"]))
for key, row in by_key.items():
if key.startswith(prefix) and key not in objects and row["status"] == "ready" and row["created_at"] < cutoff:
report["missing_objects"].append(str(row["id"]))
if apply:
for key in report["orphan_objects"]:
# повторная проверка перед удалением: за время обхода строка могла появиться
if await conn.fetchval("SELECT 1 FROM attachments WHERE s3_key = $1", key) is None:
await s3.delete(key)
report["fixed"]["deleted_orphans"] += 1
for ids, code in ((report["missing_objects"], "object_missing"), (report["size_mismatch"], "size_mismatch")):
for att_id in ids:
n = await conn.execute("UPDATE attachments SET status = 'failed', error_code = $2, updated_at = now() "
"WHERE id = $1 AND status = 'ready'", att_id, code)
report["fixed"]["marked_failed"] += int(n.split()[-1])
report["timing_ms"] = {"db_query": round(db_ms, 1), "s3_listing": round(list_ms, 1),
"total": round((time.perf_counter() - started) * 1000, 1)}
return report
finally:
await conn.close()
await s3.stop()
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--apply", action="store_true", help="исправлять (по умолчанию dry-run)")
ap.add_argument("--grace", type=int, default=config.ORPHAN_GRACE_SECONDS)
ap.add_argument("--prefix", default="att/")
ap.add_argument("--max-listed", type=int, default=20, help="сколько идентификаторов печатать в отчёте")
args = ap.parse_args()
report = asyncio.run(reconcile(args.apply, args.grace, args.prefix))
for k in ("orphan_objects", "missing_objects", "size_mismatch"):
report[k + "_count"] = len(report[k])
report[k] = report[k][:args.max_listed]
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+70
View File
@@ -0,0 +1,70 @@
"""Операторский replay и реестр неудавшихся заданий: python -m app.replay [list | replay <attachment_id> | replay-all]
list — реестр failed-вложений: id, error_code, число попыток, возраст, наличие staging;
replay <attachment_id> — возвращает ТОЛЬКО failed-вложение в queued: новая попытка (attempt сбрасывается на 1), новый job_id,
запись в job_events (аудит), задание в outbox. Если у вложения уже есть варианты (ready/успешный
эффект), повторного исполнения не будет: вложение не в статусе failed -> replay отклоняется.
Эффекты защищены ключом (attachment_id, pipeline_version, name): даже повторное выполнение не создаёт второй набор вариантов.
"""
import argparse
import asyncio
import json
import sys
import uuid
import asyncpg
from . import config
async def list_failed(conn):
rows = await conn.fetch(
"SELECT id, error_code, attempt, staging_key IS NOT NULL AS has_staging, updated_at FROM attachments "
"WHERE status = 'failed' ORDER BY updated_at DESC LIMIT 200")
return [{"attachment_id": str(r["id"]), "error_code": r["error_code"], "attempt": r["attempt"],
"staging_present": r["has_staging"], "failed_at": r["updated_at"].isoformat()} for r in rows]
async def replay(conn, att_id):
async with conn.transaction():
row = await conn.fetchrow("SELECT id, status, pipeline_version, staging_key FROM attachments WHERE id = $1 FOR UPDATE", att_id)
if row is None:
return {"attachment_id": str(att_id), "result": "not_found"}
if row["status"] != "failed":
return {"attachment_id": str(att_id), "result": "skipped", "reason": "status=%s (replay только для failed)" % row["status"]}
if not row["staging_key"]:
return {"attachment_id": str(att_id), "result": "skipped", "reason": "staging удалён по retention, исходник недоступен"}
job_id = uuid.uuid4()
await conn.execute("UPDATE attachments SET status = 'queued', error_code = NULL, attempt = 0, job_id = $2, lease_owner = NULL, "
"lease_expires_at = NULL, updated_at = now() WHERE id = $1", att_id, job_id)
await conn.execute("INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt) VALUES ($1, $2, $3, 1)",
job_id, att_id, row["pipeline_version"])
await conn.execute("INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) VALUES ($1, $2, 'operator_replay', 1, 'failed -> queued')",
att_id, job_id)
return {"attachment_id": str(att_id), "result": "requeued", "job_id": str(job_id)}
async def main_async(args):
conn = await asyncpg.connect(config.DATABASE_URL, timeout=5)
try:
if args.command == "list":
return await list_failed(conn)
if args.command == "replay":
return await replay(conn, uuid.UUID(args.attachment_id))
return [await replay(conn, uuid.UUID(f["attachment_id"])) for f in await list_failed(conn)]
finally:
await conn.close()
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = ap.add_subparsers(dest="command", required=True)
sub.add_parser("list")
r = sub.add_parser("replay")
r.add_argument("attachment_id")
sub.add_parser("replay-all")
print(json.dumps(asyncio.run(main_async(ap.parse_args())), ensure_ascii=False, indent=2))
if __name__ == "__main__":
sys.exit(main())
+92
View File
@@ -0,0 +1,92 @@
"""Приватное S3-хранилище (MinIO) через aiobotocore: потоковая загрузка/скачивание, таймауты, перевод сбоев в 503."""
import asyncio
from aiobotocore.config import AioConfig
from aiobotocore.session import get_session
from botocore.exceptions import BotoCoreError, ClientError
from . import config, telemetry
from .web import log, unavailable
S3_ERRORS = (BotoCoreError, ClientError, asyncio.TimeoutError, OSError)
class S3:
def __init__(self):
self._ctx = None
self.client = None
async def start(self):
if not config.S3_ENDPOINT:
log("error", "s3_endpoint_missing")
return
session = get_session()
self._ctx = session.create_client(
"s3", endpoint_url=config.S3_ENDPOINT, region_name=config.S3_REGION,
aws_access_key_id=config.S3_ACCESS_KEY, aws_secret_access_key=config.S3_SECRET_KEY,
config=AioConfig(connect_timeout=config.S3_CONNECT_TIMEOUT, read_timeout=config.S3_READ_TIMEOUT,
retries={"max_attempts": 2, "mode": "standard"}, max_pool_connections=config.S3_POOL,
s3={"addressing_style": "path"}))
self.client = await self._ctx.__aenter__()
async def stop(self):
if self._ctx:
await self._ctx.__aexit__(None, None, None)
def _check(self):
if self.client is None:
raise unavailable("Хранилище файлов не настроено")
async def put(self, key, fileobj, content_type, size):
self._check()
try:
with telemetry.child_span("s3.put_object", "client", {"rpc.system": "aws-api", "rpc.method": "PutObject"}):
await self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentType=content_type,
ContentLength=size)
except S3_ERRORS as exc:
log("warning", "s3_put_failed", error_type=type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from exc
async def put_bytes(self, key, data, content_type):
self._check()
with telemetry.child_span("s3.put_object", "client", {"rpc.system": "aws-api", "rpc.method": "PutObject"}):
await self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=data, ContentType=content_type)
async def open_stream(self, key):
"""Возвращает (stream, content_length). Вызывающий обязан закрыть stream."""
self._check()
try:
with telemetry.child_span("s3.get_object", "client", {"rpc.system": "aws-api", "rpc.method": "GetObject"}):
resp = await self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
except S3_ERRORS as exc:
log("warning", "s3_get_failed", error_type=type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from exc
return resp["Body"], resp["ContentLength"]
async def delete(self, key):
self._check()
await self.client.delete_object(Bucket=config.S3_BUCKET, Key=key)
async def head(self, key):
"""Метаданные объекта или None, если его нет."""
self._check()
try:
return await self.client.head_object(Bucket=config.S3_BUCKET, Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in ("404", "NoSuchKey", "NotFound"):
return None
raise unavailable("Хранилище файлов недоступно") from exc
except S3_ERRORS as exc:
raise unavailable("Хранилище файлов недоступно") from exc
async def ping(self):
self._check()
await asyncio.wait_for(self.client.head_bucket(Bucket=config.S3_BUCKET), 2.0)
async def list_keys(self, prefix):
"""Асинхронный итератор (key, size, last_modified) по префиксу — для сверки (reconcile)."""
self._check()
paginator = self.client.get_paginator("list_objects_v2")
async for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix):
for obj in page.get("Contents", []):
yield obj["Key"], obj["Size"], obj["LastModified"]
+85
View File
@@ -0,0 +1,85 @@
"""Идемпотентный bootstrap S3: python -m app.s3_bootstrap (one-shot сервис s3-init при каждом `docker compose up`).
1) bucket создаётся администратором, если его ещё нет (повтор ничего не меняет);
2) проверяется, что анонимный запрос к bucket отклоняется (private);
3) проверяется, что отдельный ключ API (права только на этот bucket) может писать/читать/удалять и НЕ может создавать bucket.
Ключи читаются из файлов секретов; в вывод не попадают. Код выхода != 0, если любая проверка не пройдена.
"""
import asyncio
import os
import sys
import time
import urllib.error
import urllib.request
from aiobotocore.config import AioConfig
from aiobotocore.session import get_session
from botocore.exceptions import ClientError
from . import config
def secret(name):
path = os.environ.get(name + "_FILE")
with open(path, encoding="utf-8") as fh:
return fh.read().strip()
def client(session, access, secret_key):
return session.create_client("s3", endpoint_url=config.S3_ENDPOINT, region_name=config.S3_REGION, aws_access_key_id=access,
aws_secret_access_key=secret_key,
config=AioConfig(connect_timeout=3, read_timeout=10, retries={"max_attempts": 1},
s3={"addressing_style": "path"}))
async def main():
bucket = config.S3_BUCKET
session = get_session()
deadline = time.time() + 60
async with client(session, secret("S3_ADMIN_ACCESS_KEY"), secret("S3_ADMIN_SECRET_KEY")) as admin:
while True:
try:
await admin.list_buckets()
break
except Exception as exc:
if time.time() > deadline:
print("S3 недоступен:", type(exc).__name__)
return 1
await asyncio.sleep(1)
try:
await admin.head_bucket(Bucket=bucket)
print("bucket %s уже существует" % bucket)
except ClientError:
await admin.create_bucket(Bucket=bucket)
print("bucket %s создан" % bucket)
# приватность: запрос без подписи отклоняется
try:
urllib.request.urlopen("%s/%s/" % (config.S3_ENDPOINT, bucket), timeout=5)
print("ОШИБКА: bucket доступен анонимно")
return 2
except urllib.error.HTTPError as exc:
if exc.code not in (401, 403):
print("ОШИБКА: анонимный доступ вернул %d, ожидался 401/403" % exc.code)
return 2
print("анонимный доступ запрещён (HTTP %d)" % exc.code)
# ключ API: рабочие права на bucket, нет прав администратора
async with client(session, config.S3_ACCESS_KEY, config.S3_SECRET_KEY) as api:
key = "bootstrap-check/%d" % int(time.time())
await api.put_object(Bucket=bucket, Key=key, Body=b"ok")
got = await (await api.get_object(Bucket=bucket, Key=key))["Body"].read()
await api.delete_object(Bucket=bucket, Key=key)
if got != b"ok":
print("ОШИБКА: чтение/запись ключом API")
return 3
try:
await api.create_bucket(Bucket="not-allowed-" + str(int(time.time())))
print("ОШИБКА: ключ API смог создать bucket (избыточные права)")
return 4
except ClientError as exc:
print("ключ API ограничен одним bucket (create_bucket отклонён: %s)" % exc.response["Error"]["Code"])
print("bootstrap завершён")
return 0
if __name__ == "__main__":
sys.exit(asyncio.run(main()))
+40
View File
@@ -0,0 +1,40 @@
"""Точка входа: uvicorn с управляемым завершением (drain).
SIGTERM -> readiness сразу 503 -> DRAIN_SECONDS продолжаем обслуживать запросы, пока балансировщик
исключает реплику -> uvicorn прекращает приём и дожидается текущих запросов (timeout_graceful_shutdown).
Так плановая остановка/rolling update не даёт ошибок клиентам.
"""
import signal
import threading
import uvicorn
from . import config
from .main import app
from .web import log
class DrainingServer(uvicorn.Server):
_draining = False
def handle_exit(self, sig, frame):
if sig == signal.SIGTERM and not self._draining and config.DRAIN_SECONDS > 0:
self._draining = True
app.draining = True
log("info", "draining", seconds=config.DRAIN_SECONDS)
timer = threading.Timer(config.DRAIN_SECONDS, super().handle_exit, args=(sig, frame))
timer.daemon = True
timer.start()
return
super().handle_exit(sig, frame)
def main():
cfg = uvicorn.Config(app, host=config.APP_HOST, port=config.APP_PORT, workers=1, access_log=False,
log_level="warning", timeout_graceful_shutdown=config.GRACEFUL_SHUTDOWN_SECONDS,
timeout_keep_alive=config.KEEPALIVE_SECONDS, limit_concurrency=config.LIMIT_CONCURRENCY)
DrainingServer(cfg).run()
if __name__ == "__main__":
main()
+131
View File
@@ -0,0 +1,131 @@
"""Сессии: PostgreSQL — источник истины; Valkey — кэш с TTL (read-through).
Политики:
* токен — 256 бит из secrets; в БД и Valkey хранится только SHA-256 токена;
* expiry: expires_at в БД (по часам БД). В кэше TTL = min(остаток жизни, SESSION_CACHE_TTL_SECONDS);
* инвалидация: logout помечает revoked_at в БД и удаляет ключ в Valkey;
* Valkey недоступен -> кэш обходится (чтение из БД), анонимный запрос никогда не считается доверенным;
circuit breaker не даёт каждому запросу ждать таймаут;
* окно устаревания: если Valkey был недоступен во время logout, ключ может прожить до SESSION_CACHE_TTL_SECONDS.
"""
import hashlib
import secrets
import time
from . import config
from .web import log
try:
import redis.asyncio as aioredis
except ImportError: # кэш опционален
aioredis = None
def token_hash(token):
return hashlib.sha256(token.encode("ascii")).digest()
class SessionCache:
def __init__(self):
self.client = None
self.down_until = 0.0
self.stats = {"hit": 0, "miss": 0, "error": 0}
async def start(self):
if config.VALKEY_URL and aioredis:
self.client = aioredis.Redis.from_url(config.VALKEY_URL, socket_timeout=config.VALKEY_TIMEOUT,
socket_connect_timeout=config.VALKEY_TIMEOUT)
async def stop(self):
if self.client:
await self.client.aclose()
def _usable(self):
return self.client is not None and time.monotonic() >= self.down_until
def _fail(self, exc):
self.stats["error"] += 1
if time.monotonic() >= self.down_until:
log("warning", "session_cache_unavailable", error_type=type(exc).__name__)
self.down_until = time.monotonic() + config.VALKEY_BREAKER_SECONDS
async def get(self, key):
if not self._usable():
return None
try:
value = await self.client.get(b"s:" + key)
except Exception as exc:
self._fail(exc)
return None
self.stats["hit" if value is not None else "miss"] += 1
return value
async def put(self, key, value, ttl):
if not self._usable() or ttl <= 0:
return
try:
await self.client.set(b"s:" + key, value, ex=max(1, int(ttl)))
except Exception as exc:
self._fail(exc)
async def delete(self, key):
if not self._usable():
return
try:
await self.client.delete(b"s:" + key)
except Exception as exc:
self._fail(exc)
async def ping(self):
if self.client is None:
return None
try:
return bool(await self.client.ping())
except Exception:
return False
class Sessions:
def __init__(self, db, cache):
self.db, self.cache = db, cache
@staticmethod
def _cache_value(user_id, expires_at):
return ("%s|%d" % (user_id, int(expires_at.timestamp() * 1000))).encode()
async def create(self, user_id):
token = secrets.token_urlsafe(32) # 256 бит энтропии
digest = token_hash(token)
row = await self.db.fetchrow(
"INSERT INTO sessions (token_hash, user_id, expires_at) "
"VALUES ($1, $2, now() + make_interval(secs => $3)) RETURNING expires_at",
digest, user_id, float(config.SESSION_TTL_SECONDS))
remaining = row["expires_at"].timestamp() - time.time()
await self.cache.put(digest, self._cache_value(user_id, row["expires_at"]),
min(remaining, config.SESSION_CACHE_TTL_SECONDS))
return token, row["expires_at"]
async def resolve(self, token):
"""token -> user_id (str) либо None, если сессии нет/истекла/отозвана."""
digest = token_hash(token)
cached = await self.cache.get(digest)
if cached is not None:
user_id, _, expires_ms = cached.decode().partition("|")
if int(expires_ms) > time.time() * 1000:
return user_id
await self.cache.delete(digest)
row = await self.db.fetchrow(
"SELECT user_id, expires_at FROM sessions "
"WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > now()", digest)
if row is None:
return None
user_id = str(row["user_id"])
remaining = row["expires_at"].timestamp() - time.time()
await self.cache.put(digest, self._cache_value(user_id, row["expires_at"]),
min(remaining, config.SESSION_CACHE_TTL_SECONDS))
return user_id
async def revoke(self, token):
digest = token_hash(token)
await self.db.execute("UPDATE sessions SET revoked_at = now() WHERE token_hash = $1 AND revoked_at IS NULL", digest)
await self.cache.delete(digest)
+229
View File
@@ -0,0 +1,229 @@
"""Наблюдаемость (lab6): OpenTelemetry SDK — трассы, метрики, логи по OTLP/HTTP в Collector.
Выключается переменной OTEL_ENABLED=0 (нужно для эксперимента «цена наблюдаемости»): тогда все функции — no-op и
накладных расходов на экспорт нет. Секреты, тела сообщений, токены и изображения в телеметрию не попадают:
в атрибутах только шаблон маршрута, метод, статус, тип задания и исход; ID объектов — только в логах/спанах
(не в labels метрик).
Соответствие сигналов contracts/observability.md и имён метрик (после Prometheus-экспортёра Collector):
http.server.request.duration (s) -> http_server_request_duration_seconds_{bucket,sum,count} (HTTP requests = _count)
messenger.jobs.processed -> messenger_jobs_processed_total{job_type,outcome}
messenger.job.duration (s) -> messenger_job_duration_seconds_*
messenger.job.time_to_ready (s) -> messenger_job_time_to_ready_seconds_* (от приёма до ready)
messenger.queue.depth / .oldest_age -> messenger_queue_depth{queue_name}, messenger_queue_oldest_job_age_seconds
db.client.connections.usage/waiting -> db_client_connections_usage{state}, db_client_connections_waiting
messenger.worker.busy / .capacity -> messenger_worker_busy, messenger_worker_capacity
process.memory.rss / process.cpu.time -> process_memory_rss_bytes, process_cpu_time_seconds_total
"""
import contextlib
import json
import logging
import os
import time
from . import config
ENABLED = config.OTEL_ENABLED
_state = {"tracer": None, "meter": None, "logger": None, "propagator": None, "inst": {}}
HTTP_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]
JOB_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]
TTR_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300]
def setup(service_name):
"""Инициализирует провайдеры. Вызывается один раз при старте процесса (API или worker)."""
if not ENABLED:
return
from opentelemetry import metrics, trace
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.propagate import get_global_textmap
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
endpoint = config.OTEL_ENDPOINT.rstrip("/")
resource = Resource.create({"service.name": service_name, "service.version": config.SERVICE_VERSION,
"service.instance.id": config.INSTANCE_ID, "deployment.environment.name": "lab"})
# --- трассы: head-sampling по доле (ParentBased: решение родителя, в т.ч. из traceparent брокера, сохраняется)
tp = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(config.OTEL_TRACE_RATIO)))
tp.add_span_processor(BatchSpanProcessor(
OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=3),
max_queue_size=config.OTEL_QUEUE_SIZE, max_export_batch_size=config.OTEL_BATCH_SIZE, schedule_delay_millis=2000, export_timeout_millis=3000))
trace.set_tracer_provider(tp)
# --- метрики (histogram с явными границами, секунды)
views = [View(instrument_name="http.server.request.duration", aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)),
View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
View(instrument_name="messenger.job.time_to_ready", aggregation=ExplicitBucketHistogramAggregation(TTR_BUCKETS))]
reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=3),
export_interval_millis=config.OTEL_METRIC_INTERVAL_MS, export_timeout_millis=3000)
mp = MeterProvider(resource=resource, metric_readers=[reader], views=views)
metrics.set_meter_provider(mp)
# --- логи: стандартный logging -> OTLP (trace_id/span_id добавляются автоматически из активного спана)
lp = LoggerProvider(resource=resource)
lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=3),
max_queue_size=config.OTEL_QUEUE_SIZE, schedule_delay_millis=2000))
logger = logging.getLogger("messenger")
logger.setLevel(logging.INFO)
logger.propagate = False
logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=lp))
_state.update(tracer=trace.get_tracer("messenger"), meter=metrics.get_meter("messenger"), logger=logger,
propagator=get_global_textmap(), providers=(tp, mp, lp))
m = _state["meter"]
_state["inst"] = {
"http": m.create_histogram("http.server.request.duration", unit="s", description="Длительность HTTP-запроса"),
"jobs": m.create_counter("messenger.jobs.processed", description="Обработанные задания по исходам"),
"job_dur": m.create_histogram("messenger.job.duration", unit="s", description="Длительность обработки задания"),
"ttr": m.create_histogram("messenger.job.time_to_ready", unit="s", description="От приёма загрузки до ready"),
}
for args in list(_pending_gauges):
(observable_counter if len(args) == 5 else gauge)(*args[:4])
_pending_gauges.clear()
def shutdown():
for p in _state.get("providers", ()):
try:
p.shutdown()
except Exception:
pass
# --- трассы ----------------------------------------------------------------------------------
@contextlib.contextmanager
def span(name, kind="internal", attributes=None, carrier=None):
"""Контекстный менеджер спана. carrier — dict с traceparent/tracestate для продолжения чужого trace."""
if not ENABLED or _state["tracer"] is None:
yield None
return
from opentelemetry import trace
from opentelemetry.trace import SpanKind, Status, StatusCode
ctx = _state["propagator"].extract(carrier) if carrier else None
kinds = {"server": SpanKind.SERVER, "client": SpanKind.CLIENT, "producer": SpanKind.PRODUCER,
"consumer": SpanKind.CONSUMER, "internal": SpanKind.INTERNAL}
with _state["tracer"].start_as_current_span(name, context=ctx, kind=kinds[kind], attributes=attributes or {},
record_exception=False, set_status_on_exception=False) as sp:
try:
yield sp
except Exception as exc:
sp.set_status(Status(StatusCode.ERROR, type(exc).__name__))
sp.set_attribute("error.type", type(exc).__name__)
raise
def child_span(name, kind="internal", attributes=None):
"""Спан создаётся только внутри активного спана (запрос/задание); фоновые опросы БД не порождают корневых трасс."""
if not ENABLED or _state["tracer"] is None:
return contextlib.nullcontext()
from opentelemetry import trace
if not trace.get_current_span().get_span_context().is_valid:
return contextlib.nullcontext()
return span(name, kind, attributes)
def set_attr(sp, key, value):
if sp is not None:
sp.set_attribute(key, value)
def mark_error(sp, description):
if sp is not None:
from opentelemetry.trace import Status, StatusCode
sp.set_status(Status(StatusCode.ERROR, description))
def current_carrier():
"""traceparent/tracestate текущего спана (для передачи через outbox/брокер)."""
if not ENABLED or _state["propagator"] is None:
return {}
carrier = {}
_state["propagator"].inject(carrier)
return carrier
def trace_ids():
if not ENABLED:
return None, None
from opentelemetry import trace
ctx = trace.get_current_span().get_span_context()
if not ctx.is_valid:
return None, None
return "%032x" % ctx.trace_id, "%016x" % ctx.span_id
# --- метрики ----------------------------------------------------------------------------------
def record_http(method, route, status, seconds):
if ENABLED and _state["inst"]:
_state["inst"]["http"].record(seconds, {"http.request.method": method, "http.route": route,
"http.response.status_code": status})
def record_job(job_type, outcome, seconds, time_to_ready=None):
if ENABLED and _state["inst"]:
attrs = {"job.type": job_type, "outcome": outcome}
_state["inst"]["jobs"].add(1, attrs)
_state["inst"]["job_dur"].record(seconds, attrs)
if time_to_ready is not None:
_state["inst"]["ttr"].record(time_to_ready, {"job.type": job_type})
_pending_gauges = []
def gauge(name, callback, unit="1", description=""):
"""Observable gauge: callback() -> [(value, {attrs})]. Регистрация откладывается до setup(), если провайдер ещё не создан."""
if not ENABLED:
return
if _state["meter"] is None:
_pending_gauges.append((name, callback, unit, description))
return
from opentelemetry.metrics import Observation
def cb(_options):
try:
return [Observation(v, a) for v, a in callback()]
except Exception:
return []
_state["meter"].create_observable_gauge(name, callbacks=[cb], unit=unit, description=description)
def observable_counter(name, callback, unit="1", description=""):
"""Монотонный счётчик, значение которого читается при экспорте (например, накопленное CPU-время)."""
if not ENABLED or _state["meter"] is None:
_pending_gauges.append((name, callback, unit, description, "counter"))
return
from opentelemetry.metrics import Observation
def cb(_options):
try:
return [Observation(v, a) for v, a in callback()]
except Exception:
return []
_state["meter"].create_observable_counter(name, callbacks=[cb], unit=unit, description=description)
def process_metrics():
def rss():
try:
with open("/proc/self/statm") as fh:
return [(int(fh.read().split()[1]) * os.sysconf("SC_PAGE_SIZE"), {})]
except (OSError, ValueError):
return []
gauge("process.memory.rss", rss, "By", "Резидентная память процесса")
observable_counter("process.cpu.time", lambda: [(time.process_time(), {})], "s", "Накопленное CPU-время процесса")
# --- логи ---------------------------------------------------------------------------------------
def emit_log(level, record):
"""Отправляет запись в OTLP (если включено). record — dict без секретов."""
if not ENABLED or _state["logger"] is None:
return
lvl = {"info": logging.INFO, "warning": logging.WARNING, "error": logging.ERROR}.get(level, logging.INFO)
_state["logger"].log(lvl, record.get("msg", ""), extra={k: v for k, v in record.items() if k not in ("msg", "level", "ts")})
+192
View File
@@ -0,0 +1,192 @@
"""Общий HTTP-слой: ошибки, запрос, валидация, курсоры, ответы, журнал."""
import base64
import json
import re
import sys
import time
import uuid
from datetime import datetime, timedelta, timezone
from urllib.parse import parse_qsl
from . import config, telemetry
UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
LIMIT_RE = re.compile(r"[0-9]{1,9}")
EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
class ApiError(Exception):
def __init__(self, status, code, message, headers=None):
super().__init__(message)
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
def bad(message):
return ApiError(400, "invalid_request", message)
def unavailable(message="Зависимость временно недоступна"):
return ApiError(503, "unavailable", message, {"retry-after": str(config.RETRY_AFTER_SECONDS)})
def log(level, msg, **fields):
"""Структурный лог: JSON в stdout + OTLP (если включено). trace_id/span_id — из активного спана; тела и токены не логируются."""
record = {"ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "level": level, "service": config.SERVICE_NAME, "msg": msg}
trace_id, span_id = telemetry.trace_ids()
if trace_id:
record["trace_id"], record["span_id"] = trace_id, span_id
record.update(fields)
sys.stdout.write(json.dumps(record, ensure_ascii=False) + "\n")
sys.stdout.flush()
telemetry.emit_log(level, record)
# --- время -------------------------------------------------------------------
def format_dt(value):
value = value.astimezone(timezone.utc)
return value.strftime("%Y-%m-%dT%H:%M:%S.") + "%06dZ" % value.microsecond
def dt_to_us(value):
return (value - EPOCH) // timedelta(microseconds=1)
def us_to_dt(us):
return EPOCH + timedelta(microseconds=us)
# --- валидация ---------------------------------------------------------------
def parse_uuid(value, what="id"):
if not isinstance(value, str) or not UUID_RE.fullmatch(value):
raise bad("Некорректный %s" % what)
return value.lower()
def valid_text(value, field, maximum, allow_empty=False):
if not isinstance(value, str):
raise bad("Поле %s должно быть строкой" % field)
try:
value.encode("utf-8")
except UnicodeEncodeError:
raise bad("Поле %s содержит некорректные символы" % field) from None
if "\x00" in value:
raise bad("Поле %s содержит NUL" % field)
if len(value) > maximum:
raise bad("Поле %s слишком длинное" % field)
if not allow_empty and not value.strip():
raise bad("Поле %s не может быть пустым" % field)
return value
def _reject_constant(name):
raise ValueError(name)
def parse_json_object(raw, required, allowed):
try:
data = json.loads(raw.decode("utf-8"), parse_constant=_reject_constant)
except (UnicodeDecodeError, ValueError, RecursionError):
raise bad("Тело запроса — некорректный JSON") from None
if not isinstance(data, dict):
raise bad("Ожидается JSON-объект")
if set(data) - allowed:
raise bad("Лишние поля в запросе")
if any(key not in data for key in required):
raise bad("Не хватает обязательных полей")
return data
def encode_cursor(kind, resource, key):
payload = json.dumps({"k": kind, "r": resource, "t": key[0], "i": key[1]}, separators=(",", ":"))
return base64.urlsafe_b64encode(payload.encode()).decode().rstrip("=")
def decode_cursor(text, kind, resource):
try:
data = json.loads(base64.urlsafe_b64decode(text + "=" * (-len(text) % 4)).decode())
if (not isinstance(data, dict) or data.get("k") != kind or data.get("r") != resource
or not isinstance(data.get("t"), int) or isinstance(data.get("t"), bool)
or not isinstance(data.get("i"), str) or not UUID_RE.fullmatch(data["i"])
or not -62135596800000000 < data["t"] < 253402300799000000):
raise ValueError
return data["t"], data["i"]
except (ValueError, UnicodeError):
raise bad("Некорректный cursor") from None
def page_params(query, kind, resource):
limit, cursor = 50, None
if "limit" in query:
if len(query["limit"]) != 1 or not LIMIT_RE.fullmatch(query["limit"][0]):
raise bad("limit должен быть целым числом 1–100")
limit = int(query["limit"][0])
if not 1 <= limit <= 100:
raise bad("limit должен быть целым числом 1–100")
if "cursor" in query:
if len(query["cursor"]) != 1 or not query["cursor"][0]:
raise bad("Некорректный cursor")
cursor = decode_cursor(query["cursor"][0], kind, resource)
return limit, cursor
def parse_query(raw):
result = {}
if raw:
for key, value in parse_qsl(raw.decode("latin-1"), keep_blank_values=True):
result.setdefault(key, []).append(value)
return result
class Request:
def __init__(self, scope, receive, request_id):
self.method = scope["method"]
self.path = scope["path"]
self.query = parse_query(scope.get("query_string", b""))
self.headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope["headers"]}
self.receive = receive
self.request_id = request_id
self.user_id = None
async def body(self):
declared = self.headers.get("content-length")
if declared is not None:
if not declared.isdigit():
raise bad("Некорректный Content-Length")
if int(declared) > config.MAX_BODY_BYTES:
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"connection": "close"})
chunks, size = [], 0
while True:
event = await self.receive()
if event["type"] == "http.disconnect":
raise bad("Соединение закрыто клиентом")
chunk = event.get("body", b"")
size += len(chunk)
if size > config.MAX_BODY_BYTES:
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"connection": "close"})
chunks.append(chunk)
if not event.get("more_body"):
return b"".join(chunks)
def require_json_media_type(self):
media = self.headers.get("content-type", "").split(";")[0].strip().lower()
if media != "application/json":
raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: application/json")
async def respond(send, status, body, request_id, extra=None, instance_id=None):
payload = b"" if status == 204 else json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
headers = [(b"x-request-id", request_id.encode())]
if status != 204:
headers += [(b"content-type", b"application/json"), (b"content-length", str(len(payload)).encode())]
else:
headers.append((b"content-length", b"0"))
if instance_id:
headers.append((b"x-instance-id", instance_id.encode()))
headers += [(k.encode(), v.encode()) for k, v in (extra or {}).items()]
await send({"type": "http.response.start", "status": status, "headers": headers})
await send({"type": "http.response.body", "body": payload})
def new_request_id():
return uuid.uuid4().hex
+345
View File
@@ -0,0 +1,345 @@
"""Worker вложений: python -m app.worker
Конвейер задания (jobs.schema.json, состояния — contracts/attachment-states.md):
claim (queued -> processing, lease) -> чтение staging -> варианты (Pillow, в потоках) -> запись в конечный S3-prefix
-> ОДНА транзакция: варианты + ready -> ack -> очистка staging.
Ack отправляется только после устойчивого эффекта (COMMIT результата). Повторная доставка уже готовой задачи — no-op с ack.
Временные ошибки: возврат в queued с backoff (экспоненциальный + jitter) через outbox; после MAX_ATTEMPTS — failed/retry_exhausted.
Постоянные ошибки (невалидное изображение): failed с error_code сразу. Зависшие processing возвращает sweeper по истечении lease.
"""
import asyncio
import json
import os
import random
import signal
import tempfile
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
import aio_pika
from . import config, pipeline, telemetry
from .broker import QUEUE, Broker
from .db import Database
from .outbox import Relay
from .s3 import S3
from .web import log
WORKER_ID = "%s-%s" % ((os.environ.get("HOSTNAME") or "worker")[:12], uuid.uuid4().hex[:6])
def backoff_seconds(attempt):
"""Экспоненциальная задержка с полным jitter: случайное значение в [0, min(cap, base * 2^(attempt-1))]."""
ceiling = min(config.RETRY_MAX_DELAY, config.RETRY_BASE_DELAY * (2 ** max(0, attempt - 1)))
return random.uniform(ceiling * 0.5, ceiling)
class Worker:
def __init__(self):
self.db, self.s3, self.broker = Database(), S3(), Broker()
self.relay = Relay(self.db, self.broker)
self.pool = ThreadPoolExecutor(max_workers=config.WORKER_CONCURRENCY, thread_name_prefix="img")
self.busy = 0
self.stopping = False
self.tasks = []
telemetry.gauge("messenger.worker.busy", lambda: [(self.busy, {})], "{job}", "Задания в обработке")
telemetry.gauge("messenger.worker.capacity", lambda: [(config.WORKER_CONCURRENCY, {})], "{job}", "Ёмкость worker")
# --- claim / finalize ---------------------------------------------------------
async def claim(self, job):
return await self.db.fetchrow(
"UPDATE attachments SET status = 'processing', lease_owner = $2, processing_started_at = now(), "
"lease_expires_at = now() + make_interval(secs => $3), attempt = $4, job_id = $5, updated_at = now() "
"WHERE id = $1 AND status = 'queued' AND pipeline_version <= $6 "
"RETURNING id, chat_id, staging_key, s3_key, content_type, size_bytes, attempt",
uuid.UUID(job["attachment_id"]), WORKER_ID, float(config.LEASE_SECONDS), job["attempt"],
uuid.UUID(job["job_id"]), job["pipeline_version"], retry=False)
async def fail(self, att_id, job, code, event="failed"):
await self.db.execute(
"WITH u AS (UPDATE attachments SET status = 'failed', error_code = $2, lease_owner = NULL, lease_expires_at = NULL, "
"updated_at = now() WHERE id = $1 AND status = 'processing' AND lease_owner = $3 RETURNING id) "
"INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $4, $5, $6, $2 FROM u",
att_id, code, WORKER_ID, uuid.UUID(job["job_id"]), event, job["attempt"])
async def reschedule(self, att_id, job, reason):
"""Временная ошибка: processing -> queued, новая запись outbox с отложенной публикацией (backoff + jitter)."""
nxt = job["attempt"] + 1
if nxt > config.MAX_ATTEMPTS:
log("warning", "job_retry_exhausted", attachment_id=str(att_id), attempts=job["attempt"])
await self.fail(att_id, job, "retry_exhausted", "retry_exhausted")
return
delay = backoff_seconds(job["attempt"])
await self.db.execute(
"WITH u AS (UPDATE attachments SET status = 'queued', lease_owner = NULL, lease_expires_at = NULL, updated_at = now() "
"WHERE id = $1 AND status = 'processing' AND lease_owner = $2 RETURNING id, pipeline_version), "
"o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, available_at, traceparent, tracestate) "
"SELECT $3, id, pipeline_version, $4, now() + make_interval(secs => $5), $7, $8 FROM u RETURNING 1) "
"INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $3, 'retry_scheduled', $4, $6 FROM u",
att_id, WORKER_ID, uuid.UUID(job["job_id"]), nxt, delay, reason, job.get("traceparent"), job.get("tracestate"))
log("info", "job_rescheduled", attachment_id=str(att_id), next_attempt=nxt, delay_seconds=round(delay, 2), reason=reason)
async def publish_result(self, row, job, variants):
"""Одна транзакция: варианты (идемпотентно) + ready. Только владелец lease может опубликовать результат."""
async with self.db.conn() as c:
async with c.transaction():
owner = await c.fetchval("SELECT 1 FROM attachments WHERE id = $1 AND status = 'processing' AND lease_owner = $2 "
"FOR UPDATE", row["id"], WORKER_ID)
if owner is None:
return False # lease потерян (sweeper вернул задачу): результат не публикуем, чужой повтор доделает
for v in variants:
await c.execute(
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, size_bytes, width, "
"height, sha256, s3_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) "
"ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING",
row["id"], job["pipeline_version"], v["name"], v["content_type"], len(v["data"]), v["width"], v["height"],
v["sha256"], "att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"]))
await c.execute("UPDATE attachments SET status = 'ready', error_code = NULL, ready_at = now(), updated_at = now(), "
"lease_owner = NULL, lease_expires_at = NULL WHERE id = $1", row["id"])
await c.execute("INSERT INTO job_events (attachment_id, job_id, event, attempt) VALUES ($1, $2, 'ready', $3)",
row["id"], uuid.UUID(job["job_id"]), job["attempt"])
return True
# --- обработка ----------------------------------------------------------------
async def process(self, job):
att_id = uuid.UUID(job["attachment_id"])
row = await self.claim(job)
if row is None:
state = await self.db.fetchval("SELECT status FROM attachments WHERE id = $1", att_id)
log("info", "job_noop", attachment_id=job["attachment_id"], status=state or "unknown")
telemetry.record_job("attachment.process.v1", "noop", 0.0)
return # ready/failed/processing у другого владельца/неизвестное вложение: идемпотентный no-op, ack
started = time.perf_counter()
outcome = "ready"
try:
await self.fault("after_claim")
stream, _ = await self.s3.open_stream(row["staging_key"])
spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024)
try:
try:
while True:
chunk = await stream.read(256 * 1024)
if not chunk:
break
spool.write(chunk)
finally:
stream.close()
loop = asyncio.get_running_loop()
variants = await asyncio.wait_for(loop.run_in_executor(self.pool, pipeline.process, spool), config.JOB_TIMEOUT)
finally:
spool.close()
# оригинал и варианты в конечный prefix: ключи детерминированы, повторная запись перезаписывает те же байты
await self.copy_original(row)
for v in variants:
await self.s3.put_bytes("att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"]), v["data"], v["content_type"])
await self.fault("after_s3_before_commit")
if not await self.publish_result(row, job, variants):
log("warning", "lease_lost", attachment_id=job["attachment_id"])
return
await self.fault("after_commit_before_ack")
log("info", "job_ready", attachment_id=job["attachment_id"], attempt=job["attempt"],
duration_ms=round((time.perf_counter() - started) * 1000, 1))
try:
await self.s3.delete(row["staging_key"]) # staging безопасно удалять после COMMIT результата
except Exception as exc:
log("warning", "staging_cleanup_failed", error_type=type(exc).__name__)
except pipeline.PermanentError as exc:
outcome = "failed"
await self.fail(att_id, job, exc.code)
log("warning", "job_failed", attachment_id=job["attachment_id"], error_code=exc.code)
except MemoryError:
outcome = "failed"
await self.fail(att_id, job, "processing_failed")
log("error", "job_failed_oom", attachment_id=job["attachment_id"])
except asyncio.CancelledError:
raise
except InjectedCrash:
raise
except Exception as exc: # S3/БД/таймаут/неожиданное: временная ошибка
outcome = "retried"
await self.reschedule(att_id, job, type(exc).__name__)
finally:
ttr = None
if outcome == "ready":
ttr = await self.db.fetchval("SELECT extract(epoch FROM ready_at - accepted_at) FROM attachments WHERE id = $1 "
"AND status = 'ready'", att_id)
telemetry.record_job("attachment.process.v1", outcome, time.perf_counter() - started, float(ttr) if ttr is not None else None)
async def copy_original(self, row):
stream, size = await self.s3.open_stream(row["staging_key"])
spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024)
try:
while True:
chunk = await stream.read(256 * 1024)
if not chunk:
break
spool.write(chunk)
stream.close()
spool.seek(0)
await self.s3.put(row["s3_key"], spool, row["content_type"], size)
finally:
spool.close()
async def fault(self, point):
"""Точки внедрения сбоев для экспериментов (FAULT_INJECTION=crash:<точка> => процесс умирает мгновенно)."""
if "crash_once:" + point in config.FAULT_INJECTION:
marker = "/tmp/fault-%s.done" % point # сохраняется при рестарте того же контейнера -> сбой ровно один раз
if not os.path.exists(marker):
open(marker, "w").close()
log("error", "fault_crash_once", point=point)
os._exit(137)
if "crash:" + point in config.FAULT_INJECTION:
log("error", "fault_crash", point=point)
os._exit(137)
if "fail:" + point in config.FAULT_INJECTION:
raise OSError("injected failure at " + point)
async def on_message(self, message):
try:
job = json.loads(message.body)
if (job.get("schema_version") != 1 or job.get("type") != "attachment.process.v1"
or not isinstance(job.get("attempt"), int) or not isinstance(job.get("pipeline_version"), int)):
raise ValueError("bad job")
uuid.UUID(job["attachment_id"]), uuid.UUID(job["job_id"])
except Exception:
log("error", "job_unreadable")
await message.reject(requeue=False) # -> DLQ
return
self.busy += 1
carrier = {k: job[k] for k in ("traceparent", "tracestate") if k in job}
try:
with telemetry.span("attachment.process process", "consumer",
{"messaging.system": "rabbitmq", "messaging.destination.name": "attachment.process",
"messaging.operation": "process", "job.type": job["type"], "job.attempt": job["attempt"]}, carrier):
await self.process(job)
await message.ack()
except InjectedCrash:
raise
except Exception as exc:
log("error", "job_unhandled", error_type=type(exc).__name__)
await message.nack(requeue=True)
finally:
self.busy -= 1
# --- sweeper -------------------------------------------------------------------
async def sweep_once(self):
"""Истёкший lease -> queued (новая попытка) или failed/retry_exhausted; чистка старого staging."""
rows = await self.db.fetch(
"WITH exp AS (SELECT id FROM attachments WHERE status = 'processing' AND lease_expires_at < now() FOR UPDATE SKIP LOCKED), "
"u AS (UPDATE attachments a SET status = CASE WHEN a.attempt >= $1 THEN 'failed' ELSE 'queued' END, "
"error_code = CASE WHEN a.attempt >= $1 THEN 'retry_exhausted' ELSE NULL END, lease_owner = NULL, lease_expires_at = NULL, "
"updated_at = now() FROM exp WHERE a.id = exp.id RETURNING a.id, a.status, a.pipeline_version, a.attempt, a.job_id), "
"o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) "
"SELECT COALESCE(job_id, gen_random_uuid()), id, pipeline_version, attempt + 1, "
"(SELECT traceparent FROM outbox WHERE attachment_id = u.id ORDER BY id DESC LIMIT 1), "
"(SELECT tracestate FROM outbox WHERE attachment_id = u.id ORDER BY id DESC LIMIT 1) FROM u WHERE status = 'queued' RETURNING 1), "
"e AS (INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, job_id, 'lease_expired', attempt, status FROM u RETURNING 1) "
"SELECT id, status FROM u", config.MAX_ATTEMPTS)
for r in rows:
log("warning", "lease_expired", attachment_id=str(r["id"]), new_status=r["status"])
if rows:
self.relay.kick()
async def sweeper(self):
while not self.stopping:
try:
await self.sweep_once()
except Exception as exc:
log("warning", "sweeper_failed", error_type=type(exc).__name__)
await asyncio.sleep(config.SWEEP_INTERVAL)
async def staging_janitor(self):
"""Удаляет staging для ready (если не удалён после COMMIT) и для failed старше retention; текущую работу не трогает."""
while not self.stopping:
await asyncio.sleep(max(30, config.SWEEP_INTERVAL * 6))
try:
rows = await self.db.fetch(
"SELECT id, staging_key FROM attachments WHERE staging_key IS NOT NULL AND (status = 'ready' OR "
"(status = 'failed' AND updated_at < now() - make_interval(secs => $1))) LIMIT 200",
float(config.STAGING_FAILED_RETENTION_SECONDS))
for r in rows:
await self.s3.delete(r["staging_key"])
await self.db.execute("UPDATE attachments SET staging_key = NULL WHERE id = $1", r["id"])
except Exception as exc:
log("warning", "staging_janitor_failed", error_type=type(exc).__name__)
# --- жизненный цикл ------------------------------------------------------------
async def connect_broker(self):
while not self.stopping:
try:
await self.broker.ensure(prefetch=config.WORKER_CONCURRENCY)
return
except Exception as exc:
log("warning", "broker_unavailable", error_type=type(exc).__name__)
await asyncio.sleep(2)
async def consume(self):
while not self.stopping:
await self.connect_broker()
try:
queue = await self.broker.channel.declare_queue(QUEUE, durable=True, passive=True)
tag = await queue.consume(self.on_message)
log("info", "consuming", worker=WORKER_ID, prefetch=config.WORKER_CONCURRENCY)
while not self.stopping and not self.broker.channel.is_closed:
await asyncio.sleep(1)
if self.stopping:
await queue.cancel(tag)
return
except Exception as exc:
log("warning", "consume_failed", error_type=type(exc).__name__)
await asyncio.sleep(2)
async def run(self):
telemetry.setup(config.SERVICE_NAME)
telemetry.process_metrics()
await self.db.start()
await self.s3.start()
log("info", "worker_started", worker=WORKER_ID, concurrency=config.WORKER_CONCURRENCY, lease=config.LEASE_SECONDS,
max_attempts=config.MAX_ATTEMPTS)
loop = asyncio.get_running_loop()
for sig in (signal.SIGTERM, signal.SIGINT):
try:
loop.add_signal_handler(sig, self.request_stop)
except NotImplementedError:
signal.signal(sig, lambda *_: self.request_stop())
self.tasks = [asyncio.create_task(self.consume()), asyncio.create_task(self.relay.run()),
asyncio.create_task(self.sweeper()), asyncio.create_task(self.staging_janitor())]
beat = 0.0
while not self.stopping:
if time.monotonic() - beat > 5: # heartbeat для HEALTHCHECK контейнера
beat = time.monotonic()
try:
with open("/tmp/worker.alive", "w") as fh:
fh.write(str(time.time()))
except OSError:
pass
await asyncio.sleep(0.5)
# graceful: перестаём брать новые задания, ждём текущие до GRACEFUL_SHUTDOWN_SECONDS
deadline = time.monotonic() + config.GRACEFUL_SHUTDOWN_SECONDS
while self.busy and time.monotonic() < deadline:
await asyncio.sleep(0.2)
self.relay.stop()
for t in self.tasks:
t.cancel()
await self.broker.stop()
await self.s3.stop()
await self.db.stop()
log("info", "worker_stopped", unfinished=self.busy)
telemetry.shutdown()
def request_stop(self):
log("info", "worker_stopping")
self.stopping = True
class InjectedCrash(Exception):
pass
def main():
asyncio.run(Worker().run())
if __name__ == "__main__":
main()