forked from mtusi2026/mtusi2026lab6
ЛР6: OpenTelemetry, Collector, Prometheus/Loki/Tempo, Grafana, алерты, исследование цены наблюдаемости
This commit is contained in:
@@ -0,0 +1,208 @@
|
||||
"""Вложения (lab4): потоковая загрузка PNG/JPEG в приватный S3, метаданные в PostgreSQL, авторизованное скачивание."""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import uuid
|
||||
|
||||
from . import config, images, telemetry
|
||||
from .web import ApiError, bad, format_dt, log, parse_uuid, unavailable
|
||||
|
||||
ALLOWED_TYPES = ("image/png", "image/jpeg")
|
||||
VARIANT_NAMES = ("original", "thumbnail", "crop", "optimized")
|
||||
ATT_COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.status, a.error_code, "
|
||||
"a.s3_key, a.created_at, a.updated_at, a.pipeline_version")
|
||||
VARIANT_ORDER = {"thumbnail": 0, "crop": 1, "optimized": 2}
|
||||
|
||||
|
||||
def attachment_json(row, variants=()):
|
||||
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), "filename": row["filename"],
|
||||
"content_type": row["content_type"], "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"],
|
||||
"variants": list(variants), "error_code": row["error_code"], "created_at": format_dt(row["created_at"]),
|
||||
"updated_at": format_dt(row["updated_at"])}
|
||||
|
||||
|
||||
def valid_filename(query):
|
||||
values = query.get("filename")
|
||||
if not values or len(values) != 1:
|
||||
raise bad("Требуется параметр filename")
|
||||
name = values[0]
|
||||
if not 1 <= len(name) <= 255 or not name.strip():
|
||||
raise bad("filename: длина 1–255 символов")
|
||||
try:
|
||||
name.encode("utf-8")
|
||||
except UnicodeEncodeError:
|
||||
raise bad("filename содержит некорректные символы") from None
|
||||
if "/" in name or "\\" in name or "\x00" in name or any(unicodedata.category(ch) == "Cc" for ch in name):
|
||||
raise bad("filename не должен содержать / \\ NUL и управляющие символы")
|
||||
return name
|
||||
|
||||
|
||||
class StreamResponse:
|
||||
"""Потоковый ответ: тело читается из S3 порциями и сразу отдаётся клиенту (без загрузки файла в память)."""
|
||||
|
||||
def __init__(self, content_type, length, stream):
|
||||
self.content_type, self.length, self.stream = content_type, length, stream
|
||||
|
||||
async def send(self, send, request_id, instance_id):
|
||||
headers = [(b"content-type", self.content_type.encode()), (b"content-length", str(self.length).encode()),
|
||||
(b"x-request-id", request_id.encode()), (b"x-content-type-options", b"nosniff"),
|
||||
(b"cache-control", b"private, no-store")]
|
||||
if instance_id:
|
||||
headers.append((b"x-instance-id", instance_id.encode()))
|
||||
await send({"type": "http.response.start", "status": 200, "headers": headers})
|
||||
try:
|
||||
while True:
|
||||
chunk = await self.stream.read(config.DOWNLOAD_CHUNK_BYTES)
|
||||
if not chunk:
|
||||
break
|
||||
await send({"type": "http.response.body", "body": chunk, "more_body": True})
|
||||
finally:
|
||||
self.stream.close()
|
||||
await send({"type": "http.response.body", "body": b""})
|
||||
|
||||
|
||||
class AttachmentMixin:
|
||||
"""Подмешивается в Messenger: требует self.db, self.s3, self.authenticate()."""
|
||||
|
||||
def _init_attachments(self):
|
||||
self._upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY)
|
||||
self._upload_waiting = 0
|
||||
|
||||
async def _receive_upload(self, req):
|
||||
"""Принимает тело в SpooledTemporaryFile (до 1 MiB в памяти, далее диск), считает SHA-256, ограничивает размер."""
|
||||
declared = req.headers.get("content-length")
|
||||
too_large = ApiError(413, "payload_too_large", "Файл больше 10 MiB", {"connection": "close"})
|
||||
if declared is not None:
|
||||
if not declared.isdigit():
|
||||
raise bad("Некорректный Content-Length")
|
||||
if int(declared) > config.MAX_UPLOAD_BYTES:
|
||||
raise too_large
|
||||
spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024)
|
||||
digest, size = hashlib.sha256(), 0
|
||||
try:
|
||||
while True:
|
||||
event = await req.receive()
|
||||
if event["type"] == "http.disconnect":
|
||||
raise bad("Соединение закрыто клиентом")
|
||||
chunk = event.get("body", b"")
|
||||
size += len(chunk)
|
||||
if size > config.MAX_UPLOAD_BYTES:
|
||||
raise too_large
|
||||
digest.update(chunk)
|
||||
spool.write(chunk)
|
||||
if not event.get("more_body"):
|
||||
break
|
||||
except BaseException:
|
||||
spool.close()
|
||||
raise
|
||||
if size == 0:
|
||||
spool.close()
|
||||
raise bad("Пустой файл")
|
||||
return spool, digest.hexdigest(), size
|
||||
|
||||
async def upload_attachment(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
chat_id = await self._require_member(user_id, params)
|
||||
filename = valid_filename(req.query)
|
||||
media = req.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if media not in ALLOWED_TYPES:
|
||||
raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: image/png или image/jpeg")
|
||||
backlog = await self.db.fetchval("SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')")
|
||||
if backlog >= config.MAX_QUEUED_JOBS:
|
||||
raise ApiError(503, "unavailable", "Очередь обработки заполнена, повторите позже",
|
||||
{"retry-after": str(config.BACKLOG_RETRY_AFTER_SECONDS)})
|
||||
if self._upload_waiting >= config.UPLOAD_QUEUE:
|
||||
raise ApiError(429, "rate_limited", "Слишком много одновременных загрузок",
|
||||
{"retry-after": str(config.RETRY_AFTER_SECONDS)})
|
||||
self._upload_waiting += 1
|
||||
try:
|
||||
await self._upload_slots.acquire() # ограничение одновременных загрузок на реплику (RAM/диск/CPU)
|
||||
finally:
|
||||
self._upload_waiting -= 1
|
||||
try:
|
||||
spool, sha256, size = await self._receive_upload(req)
|
||||
try:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
content_type, width, height = await loop.run_in_executor(self.image_pool, images.validate, spool, media)
|
||||
except images.ImageRejected as exc:
|
||||
raise ApiError(exc.status, exc.code, exc.message) from None
|
||||
att_id = uuid.uuid4()
|
||||
staging_key = "staging/%s" % att_id
|
||||
final_key = "att/%s/original" % att_id
|
||||
spool.seek(0)
|
||||
await self.s3.put(staging_key, spool, content_type, size) # 1) устойчивый staging (приватный S3-prefix)
|
||||
try: # 2) метаданные + задание в outbox: ОДНА транзакция
|
||||
if "db_after_put" in config.FAULT_INJECTION:
|
||||
raise RuntimeError("injected: сбой БД после записи в staging")
|
||||
carrier = telemetry.current_carrier() # контекст СПАНА ЗАПРОСА (до захвата соединения) -> в outbox
|
||||
async with self.db.conn() as c:
|
||||
async with c.transaction():
|
||||
row = await c.fetchrow(
|
||||
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, "
|
||||
"height, status, s3_key, staging_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'queued', $10, $11) "
|
||||
"RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, error_code, "
|
||||
"s3_key, created_at, updated_at",
|
||||
att_id, uuid.UUID(chat_id), uuid.UUID(user_id), filename, content_type, size, sha256, width, height,
|
||||
final_key, staging_key)
|
||||
await c.execute(
|
||||
"INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) "
|
||||
"VALUES ($1, $2, 1, 1, $3, $4)", uuid.uuid4(), att_id, carrier.get("traceparent"),
|
||||
carrier.get("tracestate"))
|
||||
except BaseException:
|
||||
try:
|
||||
if "s3_delete_fails" in config.FAULT_INJECTION:
|
||||
raise OSError("injected: компенсирующее удаление не удалось")
|
||||
await self.s3.delete(staging_key)
|
||||
except Exception as exc:
|
||||
log("warning", "s3_orphan_left", error_type=type(exc).__name__)
|
||||
raise
|
||||
finally:
|
||||
spool.close()
|
||||
finally:
|
||||
self._upload_slots.release()
|
||||
self.relay.kick() # немедленная публикация; при сбое задание остаётся в outbox
|
||||
return 202, attachment_json(row)
|
||||
|
||||
async def _attachment_for(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
att_id = parse_uuid(params["attachment_id"], "attachment_id")
|
||||
row = await self.db.fetchrow(
|
||||
"SELECT " + ATT_COLUMNS + " FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 "
|
||||
"WHERE a.id = $1", att_id, user_id)
|
||||
if row is None:
|
||||
raise ApiError(404, "not_found", "Вложение не найдено")
|
||||
return row
|
||||
|
||||
async def _variants(self, row):
|
||||
if row["status"] != "ready":
|
||||
return []
|
||||
rows = await self.db.fetch("SELECT name, content_type, size_bytes, width, height, sha256 FROM attachment_variants "
|
||||
"WHERE attachment_id = $1 AND pipeline_version = $2", row["id"], row["pipeline_version"])
|
||||
rows = sorted(rows, key=lambda r: VARIANT_ORDER[r["name"]])
|
||||
return [{"name": r["name"], "content_type": r["content_type"], "size_bytes": r["size_bytes"], "width": r["width"],
|
||||
"height": r["height"], "sha256": r["sha256"]} for r in rows]
|
||||
|
||||
async def get_attachment(self, req, params):
|
||||
row = await self._attachment_for(req, params)
|
||||
return 200, attachment_json(row, await self._variants(row))
|
||||
|
||||
async def download_attachment(self, req, params):
|
||||
row = await self._attachment_for(req, params)
|
||||
variant = "original"
|
||||
if "variant" in req.query:
|
||||
if len(req.query["variant"]) != 1 or req.query["variant"][0] not in VARIANT_NAMES:
|
||||
raise bad("variant: original | thumbnail | crop | optimized")
|
||||
variant = req.query["variant"][0]
|
||||
if row["status"] != "ready":
|
||||
raise ApiError(409, "conflict", "Вложение ещё обрабатывается или обработка не удалась")
|
||||
key, content_type = row["s3_key"], row["content_type"]
|
||||
if variant != "original":
|
||||
v = await self.db.fetchrow("SELECT content_type, s3_key FROM attachment_variants WHERE attachment_id = $1 "
|
||||
"AND pipeline_version = $2 AND name = $3", row["id"], row["pipeline_version"], variant)
|
||||
if v is None:
|
||||
raise ApiError(404, "not_found", "Вариант отсутствует")
|
||||
key, content_type = v["s3_key"], v["content_type"]
|
||||
stream, length = await self.s3.open_stream(key)
|
||||
return 200, StreamResponse(content_type, length, stream)
|
||||
@@ -0,0 +1,67 @@
|
||||
"""RabbitMQ: durable quorum-очередь, persistent-сообщения, publisher confirms, DLX для нечитаемых сообщений.
|
||||
|
||||
В брокер попадают только служебные поля задания (jobs.schema.json): ID, версия пайплайна, номер попытки, время.
|
||||
Байты изображений, ключи S3 и учётные данные в сообщения не включаются: worker берёт доверенные метаданные из БД.
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import aio_pika
|
||||
from aio_pika import DeliveryMode, ExchangeType, Message
|
||||
|
||||
from . import config
|
||||
from .web import log
|
||||
|
||||
QUEUE = "attachment.process"
|
||||
DEAD_QUEUE = "attachment.process.dead"
|
||||
DEAD_EXCHANGE = "attachment.dlx"
|
||||
|
||||
|
||||
class Broker:
|
||||
def __init__(self):
|
||||
self.connection = None
|
||||
self.channel = None
|
||||
self.queue = None
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def start(self, prefetch=None):
|
||||
"""Подключение; при недоступном брокере вызывающий повторяет попытку (см. ensure)."""
|
||||
self.connection = await aio_pika.connect_robust(config.BROKER_URL, timeout=config.BROKER_TIMEOUT,
|
||||
client_properties={"connection_name": config.SERVICE_NAME})
|
||||
self.channel = await self.connection.channel(publisher_confirms=True)
|
||||
if prefetch:
|
||||
await self.channel.set_qos(prefetch_count=prefetch)
|
||||
dlx = await self.channel.declare_exchange(DEAD_EXCHANGE, ExchangeType.DIRECT, durable=True)
|
||||
dead = await self.channel.declare_queue(DEAD_QUEUE, durable=True, arguments={"x-queue-type": "quorum"})
|
||||
await dead.bind(dlx, routing_key="dead")
|
||||
self.queue = await self.channel.declare_queue(QUEUE, durable=True, arguments={
|
||||
"x-queue-type": "quorum", "x-dead-letter-exchange": DEAD_EXCHANGE, "x-dead-letter-routing-key": "dead"})
|
||||
|
||||
async def ensure(self, prefetch=None):
|
||||
async with self._lock:
|
||||
if self.channel is None or self.channel.is_closed:
|
||||
await self.start(prefetch)
|
||||
|
||||
async def stop(self):
|
||||
if self.connection:
|
||||
try:
|
||||
await asyncio.wait_for(self.connection.close(), 3)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def publish(self, job, headers=None):
|
||||
"""Публикует задание; возвращается ПОСЛЕ подтверждения брокера (publisher confirm) либо бросает исключение."""
|
||||
await self.ensure()
|
||||
body = json.dumps(job, separators=(",", ":")).encode("utf-8")
|
||||
message = Message(body, delivery_mode=DeliveryMode.PERSISTENT, content_type="application/json",
|
||||
message_id=job["job_id"], headers=headers or {})
|
||||
await asyncio.wait_for(self.channel.default_exchange.publish(message, routing_key=QUEUE, mandatory=True),
|
||||
config.BROKER_TIMEOUT)
|
||||
|
||||
async def queue_depth(self):
|
||||
await self.ensure()
|
||||
q = await self.channel.declare_queue(QUEUE, durable=True, passive=True)
|
||||
return q.declaration_result.message_count
|
||||
|
||||
async def ping(self):
|
||||
return self.channel is not None and not self.channel.is_closed
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Конфигурация из переменных окружения (см. .env.example)."""
|
||||
import os
|
||||
|
||||
|
||||
def _int(name, default):
|
||||
value = os.environ.get(name)
|
||||
return int(value) if value not in (None, "") else default
|
||||
|
||||
|
||||
def _float(name, default):
|
||||
value = os.environ.get(name)
|
||||
return float(value) if value not in (None, "") else default
|
||||
|
||||
|
||||
def _secret(name, default=""):
|
||||
"""Значение из переменной NAME либо из файла NAME_FILE (Docker secrets); файл читается один раз."""
|
||||
if os.environ.get(name):
|
||||
return os.environ[name]
|
||||
path = os.environ.get(name + "_FILE")
|
||||
if path and os.path.exists(path):
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
return fh.read().strip()
|
||||
return default
|
||||
|
||||
|
||||
APP_HOST = os.environ.get("APP_HOST", "0.0.0.0")
|
||||
APP_PORT = _int("APP_PORT", 8080)
|
||||
APP_ENV = os.environ.get("APP_ENV", "lab")
|
||||
|
||||
DATABASE_URL = _secret("DATABASE_URL")
|
||||
DB_POOL_MIN = _int("DB_POOL_MIN", 2)
|
||||
DB_POOL_MAX = _int("DB_POOL_MAX", 10)
|
||||
DB_ACQUIRE_TIMEOUT = _float("DB_ACQUIRE_TIMEOUT", 2.0) # ожидание свободного соединения пула
|
||||
DB_COMMAND_TIMEOUT = _float("DB_COMMAND_TIMEOUT", 5.0) # таймаут одного запроса
|
||||
DB_CONNECT_TIMEOUT = _float("DB_CONNECT_TIMEOUT", 2.0)
|
||||
|
||||
VALKEY_URL = os.environ.get("VALKEY_URL", "") # пусто = кэш сессий выключен
|
||||
VALKEY_TIMEOUT = _float("VALKEY_TIMEOUT", 0.2)
|
||||
VALKEY_BREAKER_SECONDS = _float("VALKEY_BREAKER_SECONDS", 5.0)
|
||||
|
||||
SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600)
|
||||
SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 60)
|
||||
|
||||
ARGON2_TIME_COST = _int("ARGON2_TIME_COST", 2)
|
||||
ARGON2_MEMORY_KIB = _int("ARGON2_MEMORY_KIB", 19456) # 19 MiB, рекомендация OWASP
|
||||
ARGON2_PARALLELISM = _int("ARGON2_PARALLELISM", 1)
|
||||
HASH_CONCURRENCY = _int("HASH_CONCURRENCY", 4)
|
||||
|
||||
MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536)
|
||||
MAX_INFLIGHT = _int("MAX_INFLIGHT", 64)
|
||||
LIMIT_CONCURRENCY = _int("LIMIT_CONCURRENCY", 1000)
|
||||
RETRY_AFTER_SECONDS = _int("RETRY_AFTER_SECONDS", 1)
|
||||
MIGRATIONS_DIR = os.environ.get("MIGRATIONS_DIR", os.path.join(os.path.dirname(__file__), "..", "..", "migrations"))
|
||||
DRAIN_SECONDS = _float("DRAIN_SECONDS", 4.0) # после SIGTERM: readiness=503, сервер ещё принимает запросы
|
||||
GRACEFUL_SHUTDOWN_SECONDS = _int("GRACEFUL_SHUTDOWN_SECONDS", 8)
|
||||
KEEPALIVE_SECONDS = _int("KEEPALIVE_SECONDS", 5)
|
||||
IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 24 * 3600)
|
||||
|
||||
# --- S3 / вложения (lab4) ---
|
||||
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "")
|
||||
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
|
||||
S3_ACCESS_KEY = _secret("S3_ACCESS_KEY")
|
||||
S3_SECRET_KEY = _secret("S3_SECRET_KEY")
|
||||
S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 2.0)
|
||||
S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 15.0)
|
||||
S3_POOL = _int("S3_POOL", 20)
|
||||
MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024) # 10 MiB = 10 485 760
|
||||
MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40_000_000)
|
||||
UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 4) # одновременных загрузок на реплику
|
||||
UPLOAD_QUEUE = _int("UPLOAD_QUEUE", 16) # ожидающих загрузок; сверх — 429
|
||||
IMAGE_THREADS = _int("IMAGE_THREADS", 2)
|
||||
DOWNLOAD_CHUNK_BYTES = _int("DOWNLOAD_CHUNK_BYTES", 65536)
|
||||
ORPHAN_GRACE_SECONDS = _int("ORPHAN_GRACE_SECONDS", 900) # объект без метаданных моложе этого возраста не трогаем
|
||||
FAULT_INJECTION = {x for x in os.environ.get("FAULT_INJECTION", "").split(",") if x} # db_after_put, s3_delete_fails (эксперименты)
|
||||
|
||||
# --- lab5: брокер, outbox, worker ---
|
||||
SERVICE_NAME = os.environ.get("SERVICE_NAME", "messenger-api")
|
||||
BROKER_URL = _secret("BROKER_URL")
|
||||
BROKER_TIMEOUT = _float("BROKER_TIMEOUT", 5.0)
|
||||
RELAY_BATCH = _int("RELAY_BATCH", 50)
|
||||
RELAY_INTERVAL = _float("RELAY_INTERVAL", 1.0)
|
||||
WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2) # prefetch == число одновременных заданий == потоков обработки
|
||||
LEASE_SECONDS = _int("LEASE_SECONDS", 60) # аренда задания; истекла -> sweeper возвращает в очередь
|
||||
JOB_TIMEOUT = _float("JOB_TIMEOUT", 45.0) # лимит времени одной обработки (меньше lease)
|
||||
MAX_ATTEMPTS = _int("MAX_ATTEMPTS", 5)
|
||||
RETRY_BASE_DELAY = _float("RETRY_BASE_DELAY", 2.0)
|
||||
RETRY_MAX_DELAY = _float("RETRY_MAX_DELAY", 60.0)
|
||||
SWEEP_INTERVAL = _float("SWEEP_INTERVAL", 5.0)
|
||||
STAGING_FAILED_RETENTION_SECONDS = _int("STAGING_FAILED_RETENTION_SECONDS", 7 * 24 * 3600)
|
||||
MAX_QUEUED_JOBS = _int("MAX_QUEUED_JOBS", 200) # backpressure: при превышении upload -> 503 + Retry-After
|
||||
BACKLOG_RETRY_AFTER_SECONDS = _int("BACKLOG_RETRY_AFTER_SECONDS", 5)
|
||||
|
||||
# --- lab6: наблюдаемость ---
|
||||
OTEL_ENABLED = os.environ.get("OTEL_ENABLED", "1") not in ("0", "false", "")
|
||||
OTEL_ENDPOINT = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318")
|
||||
OTEL_TRACE_RATIO = _float("OTEL_TRACES_SAMPLER_ARG", 1.0) # head-sampling: доля трасс (ParentBased)
|
||||
OTEL_METRIC_INTERVAL_MS = _int("OTEL_METRIC_EXPORT_INTERVAL", 5000)
|
||||
OTEL_QUEUE_SIZE = _int("OTEL_QUEUE_SIZE", 2048)
|
||||
OTEL_BATCH_SIZE = _int("OTEL_BATCH_SIZE", 512) # спанов в одном экспорте (max_export_batch_size) # ограниченная очередь SDK; при переполнении данные отбрасываются
|
||||
SERVICE_VERSION = os.environ.get("APP_REVISION", "dev")
|
||||
INSTANCE_ID = (os.environ.get("HOSTNAME") or "local")[:12]
|
||||
|
||||
# Внедрение отказов в HTTP-путь для экспериментов наблюдаемости (по умолчанию пусто): "http500:0.02" — 2 % запросов дают 500;
|
||||
# "slow:0.01:0.5" — 1 % запросов задерживаются на 0.5 с. Health-маршруты не затрагиваются.
|
||||
def _fault_rate(prefix):
|
||||
for item in FAULT_INJECTION:
|
||||
if item.startswith(prefix + ":"):
|
||||
return item.split(":")[1:]
|
||||
return None
|
||||
HTTP_FAULT_500 = _fault_rate("http500")
|
||||
HTTP_FAULT_SLOW = _fault_rate("slow")
|
||||
@@ -0,0 +1,88 @@
|
||||
"""PostgreSQL: ограниченный пул asyncpg, таймауты, перевод сбоев зависимости в 503."""
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import config, telemetry
|
||||
from .web import log, unavailable
|
||||
|
||||
DEPENDENCY_ERRORS = (asyncpg.PostgresConnectionError, asyncpg.InterfaceError, asyncpg.exceptions.OperatorInterventionError,
|
||||
asyncpg.exceptions.InsufficientResourcesError, asyncio.TimeoutError, OSError)
|
||||
_RETRYABLE = (asyncpg.ConnectionDoesNotExistError, asyncpg.InterfaceError, ConnectionResetError)
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self):
|
||||
self.pool = None
|
||||
self.waiting = 0
|
||||
telemetry.gauge("db.client.connections.usage", self._usage, "{connection}", "Соединения пула PostgreSQL")
|
||||
telemetry.gauge("db.client.connections.waiting", lambda: [(self.waiting, {})], "{request}", "Запросы, ожидающие соединение")
|
||||
|
||||
def _usage(self):
|
||||
if self.pool is None:
|
||||
return []
|
||||
size, idle = self.pool.get_size(), self.pool.get_idle_size()
|
||||
return [(size - idle, {"state": "used"}), (idle, {"state": "idle"})]
|
||||
|
||||
async def start(self):
|
||||
if not config.DATABASE_URL:
|
||||
log("error", "database_url_missing")
|
||||
return
|
||||
# min_size=0: приложение стартует и без БД (readiness покажет 503), пул подключается лениво.
|
||||
self.pool = await asyncpg.create_pool(
|
||||
dsn=config.DATABASE_URL, min_size=0, max_size=config.DB_POOL_MAX,
|
||||
command_timeout=config.DB_COMMAND_TIMEOUT, timeout=config.DB_CONNECT_TIMEOUT,
|
||||
max_inactive_connection_lifetime=60,
|
||||
server_settings={"application_name": "messenger-api", "timezone": "UTC"})
|
||||
|
||||
async def stop(self):
|
||||
if self.pool:
|
||||
try:
|
||||
await asyncio.wait_for(self.pool.close(), 3)
|
||||
except Exception:
|
||||
self.pool.terminate()
|
||||
|
||||
@asynccontextmanager
|
||||
async def conn(self):
|
||||
if self.pool is None:
|
||||
raise unavailable("База данных не настроена")
|
||||
try:
|
||||
self.waiting += 1
|
||||
try:
|
||||
acquired = await self.pool.acquire(timeout=config.DB_ACQUIRE_TIMEOUT)
|
||||
finally:
|
||||
self.waiting -= 1
|
||||
try:
|
||||
with telemetry.child_span("db.connection", "client", {"db.system": "postgresql"}):
|
||||
yield acquired
|
||||
finally:
|
||||
await self.pool.release(acquired)
|
||||
except DEPENDENCY_ERRORS as exc:
|
||||
log("warning", "db_unavailable", error_type=type(exc).__name__)
|
||||
raise unavailable() from exc
|
||||
|
||||
async def _run(self, method, sql, args, retry):
|
||||
for attempt in (0, 1):
|
||||
try:
|
||||
async with self.conn() as c:
|
||||
return await getattr(c, method)(sql, *args)
|
||||
except Exception as exc:
|
||||
if attempt == 0 and retry and isinstance(exc.__cause__, _RETRYABLE):
|
||||
continue # разорванное соединение пула (рестарт БД): один повтор только для чтения
|
||||
raise
|
||||
|
||||
async def fetch(self, sql, *args):
|
||||
return await self._run("fetch", sql, args, True)
|
||||
|
||||
async def fetchrow(self, sql, *args, retry=True):
|
||||
return await self._run("fetchrow", sql, args, retry)
|
||||
|
||||
async def fetchval(self, sql, *args):
|
||||
return await self._run("fetchval", sql, args, True)
|
||||
|
||||
async def execute(self, sql, *args):
|
||||
return await self._run("execute", sql, args, False)
|
||||
|
||||
async def ping(self):
|
||||
return await asyncio.wait_for(self.fetchval("SELECT 1"), 1.0)
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Проверка загружаемых изображений ДО полного декодирования: сигнатура, формат Pillow, размеры, целостность.
|
||||
|
||||
PNG/JPEG определяются по содержимому (а не по Content-Type). Лимит пикселей проверяется по заголовку,
|
||||
поэтому «decompression bomb» отклоняется без декодирования. Целостность: PNG — verify() (CRC всех чанков, без
|
||||
распаковки в память), JPEG — декодирование в режиме draft (1/8 размера => около 1/64 памяти).
|
||||
"""
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from . import config
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = None # лимит применяем сами по заголовку (ниже), без предупреждений Pillow
|
||||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
|
||||
class ImageRejected(Exception):
|
||||
def __init__(self, status, code, message):
|
||||
super().__init__(message)
|
||||
self.status, self.code, self.message = status, code, message
|
||||
|
||||
|
||||
def sniff(head):
|
||||
if head.startswith(PNG_SIGNATURE):
|
||||
return "image/png"
|
||||
if head[:3] == b"\xff\xd8\xff":
|
||||
return "image/jpeg"
|
||||
return None
|
||||
|
||||
|
||||
def _check_png_stream(fileobj, width, height):
|
||||
"""Целостность потока IDAT без аллокации изображения: распаковка порциями, проверка конца потока и размера данных."""
|
||||
fileobj.seek(8)
|
||||
inflater = zlib.decompressobj()
|
||||
total, channels, bit_depth, interlaced = 0, 0, 0, 0
|
||||
while True:
|
||||
head = fileobj.read(8)
|
||||
if len(head) < 8:
|
||||
raise ValueError("truncated")
|
||||
length, kind = struct.unpack(">I4s", head)
|
||||
if length > 0x7FFFFFFF:
|
||||
raise ValueError("chunk")
|
||||
if kind == b"IHDR":
|
||||
data = fileobj.read(length)
|
||||
bit_depth, color_type, _, _, interlaced = struct.unpack(">BBBBB", data[8:13])
|
||||
channels = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}[color_type]
|
||||
fileobj.read(4)
|
||||
elif kind == b"IDAT":
|
||||
remaining = length
|
||||
while remaining:
|
||||
piece = fileobj.read(min(remaining, 65536))
|
||||
if not piece:
|
||||
raise ValueError("truncated")
|
||||
remaining -= len(piece)
|
||||
pending = piece
|
||||
while pending:
|
||||
total += len(inflater.decompress(pending, 1 << 20))
|
||||
pending = inflater.unconsumed_tail
|
||||
fileobj.read(4)
|
||||
elif kind == b"IEND":
|
||||
break
|
||||
else:
|
||||
fileobj.seek(length + 4, 1)
|
||||
if not inflater.eof:
|
||||
raise ValueError("zlib stream incomplete")
|
||||
if not interlaced:
|
||||
row = (width * channels * bit_depth + 7) // 8
|
||||
if total != height * (row + 1):
|
||||
raise ValueError("unexpected image data size")
|
||||
|
||||
|
||||
def validate(fileobj, declared_type):
|
||||
"""Возвращает (content_type, width, height) либо бросает ImageRejected (415 — не то/повреждено, 413 — много пикселей)."""
|
||||
fileobj.seek(0)
|
||||
detected = sniff(fileobj.read(16))
|
||||
if detected is None or detected != declared_type:
|
||||
raise ImageRejected(415, "unsupported_media_type", "Содержимое не соответствует PNG/JPEG или заявленному Content-Type")
|
||||
fileobj.seek(0)
|
||||
try:
|
||||
with Image.open(fileobj) as img:
|
||||
width, height = img.size
|
||||
fmt = img.format
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError, Image.DecompressionBombError):
|
||||
raise ImageRejected(415, "unsupported_media_type", "Изображение повреждено или не поддерживается") from None
|
||||
if fmt not in ("PNG", "JPEG") or width < 1 or height < 1:
|
||||
raise ImageRejected(415, "unsupported_media_type", "Неподдерживаемый формат изображения")
|
||||
if width * height > config.MAX_IMAGE_PIXELS:
|
||||
raise ImageRejected(413, "payload_too_large", "Слишком много пикселей в изображении")
|
||||
fileobj.seek(0)
|
||||
try:
|
||||
img = Image.open(fileobj)
|
||||
if fmt == "PNG":
|
||||
img.verify()
|
||||
_check_png_stream(fileobj, width, height)
|
||||
else:
|
||||
img.draft("RGB", (max(1, width // 8), max(1, height // 8)))
|
||||
img.load()
|
||||
img.close()
|
||||
except Exception: # любая ошибка декодера = повреждённый файл
|
||||
raise ImageRejected(415, "unsupported_media_type", "Изображение повреждено") from None
|
||||
return detected, width, height
|
||||
+464
@@ -0,0 +1,464 @@
|
||||
"""ASGI-приложение мессенджера (lab2): REST API, PostgreSQL, сессии Bearer, кэш Valkey."""
|
||||
import asyncio
|
||||
import contextlib
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import config, passwords, telemetry
|
||||
from .attachments import AttachmentMixin, StreamResponse
|
||||
from .db import DEPENDENCY_ERRORS, Database
|
||||
from .broker import Broker
|
||||
from .outbox import Relay
|
||||
from .s3 import S3
|
||||
from .sessions import SessionCache, Sessions
|
||||
from .web import (ApiError, Request, bad, dt_to_us, encode_cursor, format_dt, log, new_request_id, page_params,
|
||||
parse_json_object, parse_uuid, respond, unavailable, us_to_dt, valid_text, USERNAME_RE)
|
||||
|
||||
ROUTES = [
|
||||
("health_live", re.compile(r"/health/live"), {"GET"}),
|
||||
("health_ready", re.compile(r"/health/ready"), {"GET"}),
|
||||
("users", re.compile(r"/api/v1/users"), {"POST"}),
|
||||
("users_me", re.compile(r"/api/v1/users/me"), {"GET"}),
|
||||
("create_session", re.compile(r"/api/v1/auth/sessions"), {"POST"}),
|
||||
("session_current", re.compile(r"/api/v1/auth/sessions/current"), {"DELETE"}),
|
||||
("chats", re.compile(r"/api/v1/chats"), {"GET", "POST"}),
|
||||
("chat", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)"), {"GET"}),
|
||||
("messages", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)/messages"), {"GET", "POST"}),
|
||||
("upload_attachment", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)/attachments"), {"POST"}),
|
||||
("get_attachment", re.compile(r"/api/v1/attachments/(?P<attachment_id>[^/]+)"), {"GET"}),
|
||||
("download_attachment", re.compile(r"/api/v1/attachments/(?P<attachment_id>[^/]+)/content"), {"GET"}),
|
||||
]
|
||||
ROUTE_LABELS = {
|
||||
"health_live": "/health/live", "health_ready": "/health/ready", "users": "/api/v1/users",
|
||||
"users_me": "/api/v1/users/me", "create_session": "/api/v1/auth/sessions",
|
||||
"session_current": "/api/v1/auth/sessions/current", "chats": "/api/v1/chats",
|
||||
"chat": "/api/v1/chats/{chat_id}", "messages": "/api/v1/chats/{chat_id}/messages",
|
||||
"upload_attachment": "/api/v1/chats/{chat_id}/attachments", "get_attachment": "/api/v1/attachments/{attachment_id}",
|
||||
"download_attachment": "/api/v1/attachments/{attachment_id}/content",
|
||||
}
|
||||
TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{32,128}")
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
BEARER = 'Bearer realm="messenger"'
|
||||
BASIC = 'Basic realm="messenger", charset="UTF-8"'
|
||||
# Идентификатор реплики: имя хоста контейнера (12 hex, == короткий ID контейнера) -> легко сопоставить с docker ps.
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID") or (os.environ.get("HOSTNAME") or uuid.uuid4().hex)[:12]
|
||||
|
||||
MSG_ATTS = ("COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) FROM message_attachments ma "
|
||||
"WHERE ma.message_id = messages.id), '{}'::uuid[]) AS attachment_ids")
|
||||
CHAT_SELECT = ("SELECT c.id, c.title, c.created_at, "
|
||||
"(SELECT array_agg(m2.user_id ORDER BY m2.ord) FROM chat_members m2 WHERE m2.chat_id = c.id) AS member_ids ")
|
||||
|
||||
|
||||
def user_json(row):
|
||||
return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"],
|
||||
"created_at": format_dt(row["created_at"])}
|
||||
|
||||
|
||||
def chat_json(row):
|
||||
return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(x) for x in row["member_ids"]],
|
||||
"created_at": format_dt(row["created_at"])}
|
||||
|
||||
|
||||
def message_json(row):
|
||||
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]),
|
||||
"text": row["text"], "created_at": format_dt(row["created_at"]),
|
||||
"attachment_ids": [str(x) for x in (row["attachment_ids"] or [])]}
|
||||
|
||||
|
||||
def unauthorized(challenge=BEARER, message="Требуется действительная сессия"):
|
||||
return ApiError(401, "unauthorized", message, {"www-authenticate": challenge})
|
||||
|
||||
|
||||
class Messenger(AttachmentMixin):
|
||||
def __init__(self):
|
||||
self.db = Database()
|
||||
self.s3 = S3()
|
||||
self.broker = Broker()
|
||||
self.relay = Relay(self.db, self.broker)
|
||||
self.image_pool = ThreadPoolExecutor(max_workers=config.IMAGE_THREADS, thread_name_prefix="image")
|
||||
self._init_attachments()
|
||||
self.cache = SessionCache()
|
||||
self.sessions = Sessions(self.db, self.cache)
|
||||
self.inflight = 0
|
||||
self.draining = False
|
||||
# Метрики очереди считает API (любая реплика), а не worker: при остановке worker алерт по возрасту задач продолжает работать.
|
||||
self.queue_stats = (0, 0.0)
|
||||
telemetry.gauge("messenger.queue.depth", lambda: [(self.queue_stats[0], {"queue.name": "attachment.process"})], "{job}", "queued + processing")
|
||||
telemetry.gauge("messenger.queue.oldest_job_age", lambda: [(self.queue_stats[1], {"queue.name": "attachment.process"})], "s", "Возраст старейшего задания")
|
||||
|
||||
# --- аутентификация ------------------------------------------------------
|
||||
@staticmethod
|
||||
def bearer_token(req):
|
||||
header = req.headers.get("authorization", "")
|
||||
scheme, _, token = header.partition(" ")
|
||||
if scheme.lower() != "bearer" or not TOKEN_RE.fullmatch(token.strip()):
|
||||
raise unauthorized()
|
||||
return token.strip()
|
||||
|
||||
async def authenticate(self, req):
|
||||
token = self.bearer_token(req)
|
||||
user_id = await self.sessions.resolve(token)
|
||||
if user_id is None:
|
||||
raise unauthorized()
|
||||
req.user_id, req.token = user_id, token
|
||||
return user_id
|
||||
|
||||
# --- обработчики -----------------------------------------------------------
|
||||
async def health_live(self, req, params):
|
||||
return 200, {"status": "ok"}
|
||||
|
||||
async def health_ready(self, req, params):
|
||||
# Критичная зависимость — PostgreSQL. Valkey — необязательный кэш: его отказ не останавливает API.
|
||||
if self.draining:
|
||||
raise unavailable("Сервис останавливается")
|
||||
try:
|
||||
await self.db.ping()
|
||||
except ApiError:
|
||||
raise
|
||||
except Exception:
|
||||
raise unavailable() from None
|
||||
return 200, {"status": "ok"}
|
||||
|
||||
async def users(self, req, params):
|
||||
req.require_json_media_type()
|
||||
data = parse_json_object(await req.body(), {"username", "display_name", "password"},
|
||||
{"username", "display_name", "password"})
|
||||
username = data["username"]
|
||||
if not isinstance(username, str) or not USERNAME_RE.fullmatch(username):
|
||||
raise bad("username должен соответствовать [a-z0-9_]{3,32}")
|
||||
display_name = valid_text(data["display_name"], "display_name", 100)
|
||||
password = valid_text(data["password"], "password", 128, allow_empty=True)
|
||||
if len(password) < 12:
|
||||
raise bad("password должен содержать от 12 до 128 символов")
|
||||
pw_hash = await passwords.hash_password(password)
|
||||
row = await self.db.fetchrow(
|
||||
"INSERT INTO users (id, username, display_name, password_hash) VALUES ($1, $2, $3, $4) "
|
||||
"ON CONFLICT (username) DO NOTHING RETURNING id, username, display_name, created_at",
|
||||
uuid.uuid4(), username, display_name, pw_hash)
|
||||
if row is None:
|
||||
raise ApiError(409, "conflict", "username уже занят")
|
||||
return 201, user_json(row)
|
||||
|
||||
async def users_me(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
row = await self.db.fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", user_id)
|
||||
if row is None:
|
||||
raise unauthorized()
|
||||
return 200, user_json(row)
|
||||
|
||||
async def create_session(self, req, params):
|
||||
header = req.headers.get("authorization", "")
|
||||
scheme, _, value = header.partition(" ")
|
||||
if scheme.lower() != "basic" or not value.strip():
|
||||
raise unauthorized(BASIC, "Неверные учётные данные")
|
||||
try:
|
||||
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
|
||||
username, sep, password = decoded.partition(":")
|
||||
if not sep:
|
||||
raise ValueError
|
||||
except (binascii.Error, ValueError, UnicodeDecodeError):
|
||||
raise unauthorized(BASIC, "Неверные учётные данные") from None
|
||||
row = None
|
||||
if USERNAME_RE.fullmatch(username):
|
||||
row = await self.db.fetchrow("SELECT id, username, display_name, created_at, password_hash FROM users "
|
||||
"WHERE username = $1", username)
|
||||
ok = await passwords.verify_password(row["password_hash"] if row else None, password)
|
||||
if not ok:
|
||||
raise unauthorized(BASIC, "Неверные учётные данные")
|
||||
token, expires_at = await self.sessions.create(row["id"])
|
||||
return 201, {"token": token, "token_type": "Bearer", "expires_at": format_dt(expires_at), "user": user_json(row)}
|
||||
|
||||
async def session_current(self, req, params):
|
||||
await self.authenticate(req)
|
||||
await self.sessions.revoke(req.token)
|
||||
return 204, None
|
||||
|
||||
async def chats(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
if req.method == "GET":
|
||||
limit, after = page_params(req.query, "c", user_id)
|
||||
if after is None:
|
||||
rows = await self.db.fetch(
|
||||
CHAT_SELECT + "FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = $1 "
|
||||
"ORDER BY m.chat_created_at, m.chat_id LIMIT $2", user_id, limit + 1)
|
||||
else:
|
||||
rows = await self.db.fetch(
|
||||
CHAT_SELECT + "FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = $1 "
|
||||
"AND (m.chat_created_at, m.chat_id) > ($2, $3) ORDER BY m.chat_created_at, m.chat_id LIMIT $4",
|
||||
user_id, us_to_dt(after[0]), uuid.UUID(after[1]), limit + 1)
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
cursor = encode_cursor("c", user_id, (dt_to_us(rows[-1]["created_at"]), str(rows[-1]["id"]))) if more else None
|
||||
return 200, {"items": [chat_json(r) for r in rows], "next_cursor": cursor}
|
||||
req.require_json_media_type()
|
||||
data = parse_json_object(await req.body(), {"title", "member_ids"}, {"title", "member_ids"})
|
||||
title = valid_text(data["title"], "title", 100)
|
||||
members = data["member_ids"]
|
||||
if not isinstance(members, list) or not 1 <= len(members) <= 100:
|
||||
raise bad("member_ids должен содержать от 1 до 100 элементов")
|
||||
ids = [parse_uuid(m, "member_id") for m in members]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise bad("member_ids содержит дубликаты")
|
||||
if user_id not in ids:
|
||||
raise bad("Создатель должен быть в member_ids")
|
||||
chat_id = uuid.uuid4()
|
||||
uuids = [uuid.UUID(i) for i in ids]
|
||||
try:
|
||||
async with self.db.conn() as c:
|
||||
async with c.transaction(): # чат и участники создаются атомарно
|
||||
# Существование участников проверяет внешний ключ chat_members.user_id: при неизвестном
|
||||
# участнике вставка падает и откатывается вся транзакция, включая уже созданный чат.
|
||||
created = await c.fetchval("INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING created_at",
|
||||
chat_id, title)
|
||||
await c.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id, chat_created_at, ord) "
|
||||
"SELECT $1, u, $2, o FROM unnest($3::uuid[]) WITH ORDINALITY AS t(u, o)",
|
||||
chat_id, created, uuids)
|
||||
except asyncpg.ForeignKeyViolationError:
|
||||
raise bad("Неизвестный участник") from None
|
||||
return 201, {"id": str(chat_id), "title": title, "member_ids": ids, "created_at": format_dt(created)}
|
||||
|
||||
async def _require_member(self, user_id, params):
|
||||
chat_id = parse_uuid(params["chat_id"], "chat_id")
|
||||
ok = await self.db.fetchval("SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, user_id)
|
||||
if not ok:
|
||||
raise ApiError(404, "not_found", "Чат не найден")
|
||||
return chat_id
|
||||
|
||||
async def chat(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
chat_id = parse_uuid(params["chat_id"], "chat_id")
|
||||
row = await self.db.fetchrow(
|
||||
CHAT_SELECT + "FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2 WHERE c.id = $1",
|
||||
chat_id, user_id)
|
||||
if row is None:
|
||||
raise ApiError(404, "not_found", "Чат не найден")
|
||||
return 200, chat_json(row)
|
||||
|
||||
async def messages(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
if req.method == "POST":
|
||||
row = await self._post_message(req, parse_uuid(params["chat_id"], "chat_id"), user_id)
|
||||
return 201, message_json(row)
|
||||
chat_id = await self._require_member(user_id, params)
|
||||
if req.method == "GET":
|
||||
limit, after = page_params(req.query, "m", chat_id)
|
||||
if after is None:
|
||||
rows = await self.db.fetch(
|
||||
"SELECT id, chat_id, sender_id, text, created_at, " + MSG_ATTS + " FROM messages WHERE chat_id = $1 "
|
||||
"ORDER BY created_at, id LIMIT $2", chat_id, limit + 1)
|
||||
else:
|
||||
rows = await self.db.fetch(
|
||||
"SELECT id, chat_id, sender_id, text, created_at, " + MSG_ATTS + " FROM messages WHERE chat_id = $1 "
|
||||
"AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT $4",
|
||||
chat_id, us_to_dt(after[0]), uuid.UUID(after[1]), limit + 1)
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
cursor = encode_cursor("m", chat_id, (dt_to_us(rows[-1]["created_at"]), str(rows[-1]["id"]))) if more else None
|
||||
return 200, {"items": [message_json(r) for r in rows], "next_cursor": cursor}
|
||||
|
||||
async def _post_message(self, req, chat_id, user_id):
|
||||
# Сначала разбор запроса; ошибка валидации (400/415) возвращается только участнику чата — постороннему всегда 404.
|
||||
try:
|
||||
key = req.headers.get("idempotency-key", "")
|
||||
if not IDEMPOTENCY_KEY_RE.fullmatch(key):
|
||||
raise bad("Требуется заголовок Idempotency-Key ([A-Za-z0-9._:-]{1,128})")
|
||||
req.require_json_media_type()
|
||||
data = parse_json_object(await req.body(), {"text"}, {"text", "attachment_ids"})
|
||||
raw_atts = data.get("attachment_ids", [])
|
||||
if not isinstance(raw_atts, list) or len(raw_atts) > 10:
|
||||
raise bad("attachment_ids: массив до 10 элементов")
|
||||
atts = [parse_uuid(a, "attachment_id") for a in raw_atts]
|
||||
if len(set(atts)) != len(atts):
|
||||
raise bad("attachment_ids содержит дубликаты")
|
||||
text = valid_text(data["text"], "text", 4000, allow_empty=bool(atts))
|
||||
except ApiError:
|
||||
await self._require_member(user_id, {"chat_id": chat_id})
|
||||
raise
|
||||
# Нормализованное представление запроса: порядок ключей JSON не важен, отсутствие attachment_ids == [].
|
||||
request_hash = hashlib.sha256(json.dumps({"text": text, "attachment_ids": atts}, sort_keys=True,
|
||||
ensure_ascii=False).encode("utf-8")).digest()
|
||||
row = await self.db.fetchrow("SELECT * FROM send_message($1, $2, $3, $4, $5, $6, $7)", uuid.UUID(chat_id),
|
||||
uuid.UUID(user_id), key, request_hash, text, float(config.IDEMPOTENCY_TTL_SECONDS),
|
||||
[uuid.UUID(a) for a in atts], retry=False)
|
||||
if row["outcome"] == "not_member":
|
||||
raise ApiError(404, "not_found", "Чат не найден")
|
||||
if row["outcome"] == "attachment_not_found":
|
||||
raise ApiError(404, "not_found", "Вложение не найдено")
|
||||
if row["outcome"] == "attachment_failed":
|
||||
raise ApiError(409, "conflict", "Вложение не удалось обработать")
|
||||
if bytes(row["request_hash"]) != request_hash:
|
||||
raise ApiError(409, "conflict", "Idempotency-Key уже использован с другим телом запроса")
|
||||
return {"id": row["msg_id"], "chat_id": row["msg_chat"], "sender_id": row["msg_sender"], "text": row["msg_text"],
|
||||
"created_at": row["msg_created"], "attachment_ids": row["msg_atts"]}
|
||||
|
||||
# --- ASGI ----------------------------------------------------------------------
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] == "lifespan":
|
||||
return await self.lifespan(receive, send)
|
||||
if scope["type"] != "http":
|
||||
return
|
||||
started = time.perf_counter()
|
||||
request_id = new_request_id()
|
||||
route_name, status, headers = "unknown", 500, {}
|
||||
raw_headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope["headers"]}
|
||||
carrier = {k: raw_headers[k] for k in ("traceparent", "tracestate") if k in raw_headers}
|
||||
is_health = scope.get("path", "").startswith("/health") # проверки LB раз в секунду: без спанов и access-log (метрики остаются)
|
||||
span_cm = (contextlib.nullcontext() if is_health else
|
||||
telemetry.span("HTTP %s" % scope.get("method"), "server", {"http.request.method": scope.get("method")}, carrier))
|
||||
sp = span_cm.__enter__()
|
||||
try:
|
||||
req = Request(scope, receive, request_id)
|
||||
handler, params, route_name = self.route(req.method, req.path)
|
||||
telemetry.set_attr(sp, "http.route", ROUTE_LABELS.get(route_name, "unmatched"))
|
||||
req.span = sp
|
||||
if not route_name.startswith("health") and self.inflight >= config.MAX_INFLIGHT:
|
||||
raise ApiError(429, "rate_limited", "Слишком много одновременных запросов",
|
||||
{"retry-after": str(config.RETRY_AFTER_SECONDS)})
|
||||
self.inflight += 1
|
||||
try:
|
||||
await asyncio.sleep(0)
|
||||
if not route_name.startswith("health") and (config.HTTP_FAULT_500 or config.HTTP_FAULT_SLOW):
|
||||
await self.inject_http_faults()
|
||||
status, body = await handler(req, params)
|
||||
finally:
|
||||
self.inflight -= 1
|
||||
except ApiError as exc:
|
||||
status, headers = exc.status, exc.headers
|
||||
body = {"error": {"code": exc.code, "message": exc.message, "request_id": request_id}}
|
||||
except DEPENDENCY_ERRORS as exc:
|
||||
log("warning", "dependency_error", request_id=request_id, error_type=type(exc).__name__)
|
||||
status, headers = 503, {"retry-after": str(config.RETRY_AFTER_SECONDS)}
|
||||
body = {"error": {"code": "unavailable", "message": "Зависимость временно недоступна", "request_id": request_id}}
|
||||
except Exception as exc:
|
||||
log("error", "unhandled_exception", request_id=request_id, error_type=type(exc).__name__)
|
||||
status = 500
|
||||
body = {"error": {"code": "internal_error", "message": "Внутренняя ошибка", "request_id": request_id}}
|
||||
route_label = ROUTE_LABELS.get(route_name, "unmatched")
|
||||
duration = time.perf_counter() - started
|
||||
telemetry.set_attr(sp, "http.response.status_code", status)
|
||||
telemetry.set_attr(sp, "http.request_id", request_id)
|
||||
if sp is not None:
|
||||
sp.update_name("HTTP %s %s" % (scope.get("method"), route_label))
|
||||
if status >= 500:
|
||||
telemetry.mark_error(sp, "HTTP %d" % status)
|
||||
telemetry.record_http(scope.get("method"), route_label, status, duration)
|
||||
if isinstance(body, StreamResponse):
|
||||
try:
|
||||
await body.send(send, request_id, INSTANCE_ID)
|
||||
except Exception as exc: # клиент оборвал загрузку/ошибка S3 посреди потока
|
||||
log("warning", "stream_aborted", request_id=request_id, error_type=type(exc).__name__)
|
||||
else:
|
||||
await respond(send, status, body, request_id, headers, INSTANCE_ID)
|
||||
if os.environ.get("ACCESS_LOG", "1") != "0" and (not is_health or status >= 500):
|
||||
log("info", "request", request_id=request_id, method=scope.get("method"),
|
||||
route=route_label, status=status, duration_ms=round(duration * 1000, 3))
|
||||
span_cm.__exit__(None, None, None)
|
||||
|
||||
async def janitor(self):
|
||||
"""Удаляет давно просроченные ключи идемпотентности и сессии (безопасно запускать на каждой реплике)."""
|
||||
import random
|
||||
await asyncio.sleep(random.uniform(5, 30))
|
||||
while True:
|
||||
try:
|
||||
await self.db.execute("DELETE FROM idempotency_keys WHERE ctid IN (SELECT ctid FROM idempotency_keys "
|
||||
"WHERE created_at < now() - make_interval(secs => $1) LIMIT 5000)",
|
||||
float(config.IDEMPOTENCY_TTL_SECONDS) + 3600)
|
||||
await self.db.execute("DELETE FROM sessions WHERE ctid IN (SELECT ctid FROM sessions "
|
||||
"WHERE expires_at < now() - interval '1 day' LIMIT 5000)")
|
||||
except Exception as exc:
|
||||
log("warning", "janitor_failed", error_type=type(exc).__name__)
|
||||
await asyncio.sleep(3600)
|
||||
|
||||
async def queue_stats_loop(self):
|
||||
while True:
|
||||
try:
|
||||
row = await self.db.fetchrow("SELECT count(*) AS n, coalesce(extract(epoch FROM now() - min(accepted_at)), 0) AS age "
|
||||
"FROM attachments WHERE status IN ('queued', 'processing')")
|
||||
self.queue_stats = (row["n"], float(row["age"]))
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(2)
|
||||
|
||||
@staticmethod
|
||||
async def inject_http_faults():
|
||||
import random
|
||||
if config.HTTP_FAULT_SLOW and random.random() < float(config.HTTP_FAULT_SLOW[0]):
|
||||
await asyncio.sleep(float(config.HTTP_FAULT_SLOW[1]))
|
||||
if config.HTTP_FAULT_500 and random.random() < float(config.HTTP_FAULT_500[0]):
|
||||
raise RuntimeError("injected fault")
|
||||
|
||||
def route(self, method, path):
|
||||
for name, pattern, methods in ROUTES:
|
||||
m = pattern.fullmatch(path)
|
||||
if m:
|
||||
if method not in methods:
|
||||
raise ApiError(405, "invalid_request", "Метод не поддерживается", {"allow": ", ".join(sorted(methods))})
|
||||
return getattr(self, name), m.groupdict(), name
|
||||
raise ApiError(404, "not_found", "Ресурс не найден")
|
||||
|
||||
async def lifespan(self, receive, send):
|
||||
while True:
|
||||
event = await receive()
|
||||
if event["type"] == "lifespan.startup":
|
||||
telemetry.setup(config.SERVICE_NAME)
|
||||
telemetry.process_metrics()
|
||||
await self.db.start()
|
||||
await self.cache.start()
|
||||
await self.s3.start()
|
||||
self._relay_task = asyncio.create_task(self.relay.run())
|
||||
self._queue_task = asyncio.create_task(self.queue_stats_loop())
|
||||
self._janitor = asyncio.create_task(self.janitor())
|
||||
log("info", "startup", env=config.APP_ENV, instance=INSTANCE_ID, revision=os.environ.get("APP_REVISION", "dev"), cache=bool(config.VALKEY_URL),
|
||||
session_ttl=config.SESSION_TTL_SECONDS, pool_max=config.DB_POOL_MAX)
|
||||
await send({"type": "lifespan.startup.complete"})
|
||||
elif event["type"] == "lifespan.shutdown":
|
||||
self.draining = True
|
||||
self._janitor.cancel()
|
||||
await self.cache.stop()
|
||||
telemetry.shutdown()
|
||||
self.relay.stop()
|
||||
self._relay_task.cancel()
|
||||
self._queue_task.cancel()
|
||||
await self.broker.stop()
|
||||
await self.s3.stop()
|
||||
await self.db.stop()
|
||||
log("info", "shutdown")
|
||||
await send({"type": "lifespan.shutdown.complete"})
|
||||
return
|
||||
|
||||
|
||||
async def overloaded_app(scope, receive, send):
|
||||
"""Ответ uvicorn при превышении --limit-concurrency в контрактном формате."""
|
||||
if scope["type"] != "http":
|
||||
return
|
||||
request_id = new_request_id()
|
||||
body = {"error": {"code": "unavailable", "message": "Сервер перегружен: слишком много соединений",
|
||||
"request_id": request_id}}
|
||||
await respond(send, 503, body, request_id, {"retry-after": str(config.RETRY_AFTER_SECONDS), "connection": "close"}, INSTANCE_ID)
|
||||
|
||||
|
||||
def _install_overload_response():
|
||||
for name in ("uvicorn.protocols.http.httptools_impl", "uvicorn.protocols.http.h11_impl"):
|
||||
try:
|
||||
module = __import__(name, fromlist=["service_unavailable"])
|
||||
except ImportError:
|
||||
continue
|
||||
module.service_unavailable = overloaded_app
|
||||
|
||||
|
||||
def create_app():
|
||||
_install_overload_response()
|
||||
return Messenger()
|
||||
|
||||
|
||||
app = create_app()
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Применение версионируемых SQL-миграций: python -m app.migrate
|
||||
|
||||
Свойства:
|
||||
* повторный запуск ничего не делает (таблица schema_migrations);
|
||||
* одновременный запуск нескольких экземпляров безопасен (advisory lock);
|
||||
* каждая миграция — одна транзакция вместе с записью о применении;
|
||||
* ждёт готовности PostgreSQL до MIGRATE_WAIT_SECONDS.
|
||||
"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import config
|
||||
from .web import log
|
||||
|
||||
LOCK_ID = 7352001
|
||||
|
||||
|
||||
async def connect(deadline):
|
||||
while True:
|
||||
try:
|
||||
return await asyncpg.connect(config.DATABASE_URL, timeout=3)
|
||||
except (OSError, asyncpg.PostgresError, asyncio.TimeoutError) as exc:
|
||||
if time.monotonic() > deadline:
|
||||
raise
|
||||
log("info", "waiting_for_database", error_type=type(exc).__name__)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
async def main():
|
||||
wait = float(os.environ.get("MIGRATE_WAIT_SECONDS", "60"))
|
||||
conn = await connect(time.monotonic() + wait)
|
||||
try:
|
||||
await conn.execute("SELECT pg_advisory_lock($1)", LOCK_ID)
|
||||
await conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
"version text PRIMARY KEY, checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())")
|
||||
applied = {r["version"]: r["checksum"] for r in await conn.fetch("SELECT version, checksum FROM schema_migrations")}
|
||||
directory = os.path.abspath(config.MIGRATIONS_DIR)
|
||||
files = sorted(f for f in os.listdir(directory) if f.endswith(".sql"))
|
||||
count = 0
|
||||
for name in files:
|
||||
sql = open(os.path.join(directory, name), encoding="utf-8").read()
|
||||
checksum = hashlib.sha256(sql.encode("utf-8")).hexdigest()
|
||||
if name in applied:
|
||||
if applied[name] != checksum:
|
||||
log("error", "migration_modified", version=name)
|
||||
sys.exit(2)
|
||||
continue
|
||||
async with conn.transaction():
|
||||
await conn.execute(sql)
|
||||
await conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES ($1, $2)", name, checksum)
|
||||
count += 1
|
||||
log("info", "migration_applied", version=name)
|
||||
log("info", "migrations_done", applied=count, total=len(files))
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Transactional outbox: relay публикует задания из PostgreSQL в брокер.
|
||||
|
||||
Гарантия: задание, зафиксированное в БД (COMMIT), будет опубликовано даже если процесс упал между COMMIT и publish.
|
||||
Relay запускается в каждой реплике API и в worker; строки берутся `FOR UPDATE SKIP LOCKED`, поэтому одну строку
|
||||
публикует один relay. Если процесс упадёт после подтверждения брокера, но до отметки published_at, задание будет
|
||||
опубликовано повторно — доставка at-least-once, дубликаты безвредны (обработка идемпотентна).
|
||||
"""
|
||||
import asyncio
|
||||
from datetime import timezone
|
||||
|
||||
from . import config, telemetry
|
||||
from .web import format_dt, log
|
||||
|
||||
JOB_TYPE = "attachment.process.v1"
|
||||
|
||||
|
||||
def job_message(row, carrier=None):
|
||||
msg = {"schema_version": 1, "job_id": str(row["job_id"]), "type": JOB_TYPE, "attachment_id": str(row["attachment_id"]),
|
||||
"pipeline_version": row["pipeline_version"], "attempt": row["attempt"],
|
||||
"created_at": format_dt(row["created_at"].astimezone(timezone.utc))}
|
||||
msg.update(carrier or {}) # traceparent / tracestate (W3C Trace Context) — поля схемы jobs.schema.json
|
||||
return msg
|
||||
|
||||
|
||||
class Relay:
|
||||
def __init__(self, db, broker):
|
||||
self.db, self.broker = db, broker
|
||||
self._kick = asyncio.Event()
|
||||
self._stop = False
|
||||
self.last_error = None
|
||||
|
||||
def kick(self):
|
||||
self._kick.set()
|
||||
|
||||
def stop(self):
|
||||
self._stop = True
|
||||
self._kick.set()
|
||||
|
||||
async def relay_once(self):
|
||||
published = 0
|
||||
async with self.db.conn() as c:
|
||||
async with c.transaction():
|
||||
rows = await c.fetch(
|
||||
"SELECT id, job_id, attachment_id, pipeline_version, attempt, created_at, traceparent, tracestate FROM outbox "
|
||||
"WHERE published_at IS NULL AND available_at <= now() ORDER BY id LIMIT $1 FOR UPDATE SKIP LOCKED",
|
||||
config.RELAY_BATCH)
|
||||
for row in rows:
|
||||
parent = {k: row[k] for k in ("traceparent", "tracestate") if row[k]}
|
||||
# PRODUCER-спан — дочерний к спану запроса upload (контекст восстановлен из outbox)
|
||||
with telemetry.span("attachment.process publish", "producer",
|
||||
{"messaging.system": "rabbitmq", "messaging.destination.name": "attachment.process",
|
||||
"messaging.operation": "publish", "outbox.attempt": row["attempt"]}, parent):
|
||||
carrier = telemetry.current_carrier() or parent
|
||||
await self.broker.publish(job_message(row, carrier), headers=carrier)
|
||||
await c.execute("UPDATE outbox SET published_at = now() WHERE id = $1", row["id"])
|
||||
published += 1
|
||||
return published
|
||||
|
||||
def headers_for(self, row):
|
||||
return {}
|
||||
|
||||
async def run(self):
|
||||
delay = config.RELAY_INTERVAL
|
||||
while not self._stop:
|
||||
try:
|
||||
n = await self.relay_once()
|
||||
self.last_error = None
|
||||
delay = config.RELAY_INTERVAL
|
||||
if n >= config.RELAY_BATCH:
|
||||
continue # возможно, есть ещё
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
if self.last_error != type(exc).__name__:
|
||||
log("warning", "outbox_relay_failed", error_type=type(exc).__name__)
|
||||
self.last_error = type(exc).__name__
|
||||
delay = min(delay * 2, 10.0) # брокер/БД недоступны: пауза с ростом до 10 с, задания остаются в outbox
|
||||
try:
|
||||
await asyncio.wait_for(self._kick.wait(), delay)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
self._kick.clear()
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Хеширование паролей: Argon2id (библиотека argon2-cffi), вычисления вне event loop."""
|
||||
import asyncio
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
|
||||
|
||||
from . import config
|
||||
|
||||
_hasher = PasswordHasher(time_cost=config.ARGON2_TIME_COST, memory_cost=config.ARGON2_MEMORY_KIB,
|
||||
parallelism=config.ARGON2_PARALLELISM)
|
||||
_pool = ThreadPoolExecutor(max_workers=config.HASH_CONCURRENCY, thread_name_prefix="argon2")
|
||||
# Для неизвестного username проверяем фиктивный хеш: время ответа не выдаёт существование пользователя.
|
||||
_DUMMY = _hasher.hash("dummy-password-for-timing-equalisation")
|
||||
|
||||
|
||||
async def hash_password(password):
|
||||
return await asyncio.get_running_loop().run_in_executor(_pool, _hasher.hash, password)
|
||||
|
||||
|
||||
def _verify(stored, password):
|
||||
try:
|
||||
return _hasher.verify(stored, password)
|
||||
except (VerifyMismatchError, VerificationError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
async def verify_password(stored, password):
|
||||
"""stored=None -> проверка фиктивного хеша, результат всегда False."""
|
||||
ok = await asyncio.get_running_loop().run_in_executor(_pool, _verify, stored or _DUMMY, password)
|
||||
return ok and stored is not None
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Обработка изображений (pipeline_version=1): thumbnail, crop, optimized.
|
||||
|
||||
Правила (contracts/README.md):
|
||||
* thumbnail: JPEG, вписан в 256x256 с сохранением пропорций, БЕЗ увеличения, EXIF orientation применяется, alpha — на белый
|
||||
фон; размеры округляются вниз, минимум 1 px (320x200 -> 256x160); качество 85, метаданные не сохраняются;
|
||||
* crop: JPEG, центрированный квадрат 128x128 (маленькие исходники допускается увеличивать), качество 85;
|
||||
* optimized: WebP, вписан в 1280x1280 без увеличения, quality=80 method=4, EXIF/ICC не сохраняются (ориентация применена).
|
||||
Память: JPEG декодируется через draft (масштабирование при декодировании), число пикселей ограничено MAX_IMAGE_PIXELS.
|
||||
"""
|
||||
import hashlib
|
||||
import io
|
||||
import math
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
from . import config
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = None
|
||||
PIPELINE_VERSION = 1
|
||||
JPEG_QUALITY = 85
|
||||
WEBP_QUALITY = 80
|
||||
|
||||
|
||||
class PermanentError(Exception):
|
||||
"""Постоянная ошибка входных данных: повтор не поможет (-> failed, error_code)."""
|
||||
|
||||
def __init__(self, code):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
def _flatten(img):
|
||||
"""RGBA/LA/P с прозрачностью -> RGB на белом фоне."""
|
||||
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
|
||||
rgba = img.convert("RGBA")
|
||||
background = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||
background.paste(rgba, mask=rgba.getchannel("A"))
|
||||
return background
|
||||
return img.convert("RGB")
|
||||
|
||||
|
||||
def _fit_size(width, height, box, upscale=False):
|
||||
scale = min(box / width, box / height)
|
||||
if not upscale:
|
||||
scale = min(scale, 1.0)
|
||||
return max(1, math.floor(width * scale)), max(1, math.floor(height * scale))
|
||||
|
||||
|
||||
def _encode(img, fmt, **params):
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, fmt, **params)
|
||||
data = buf.getvalue()
|
||||
return data, hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def process(fileobj):
|
||||
"""Возвращает список вариантов: dict(name, content_type, data, sha256, width, height). Бросает PermanentError."""
|
||||
try:
|
||||
fileobj.seek(0)
|
||||
src = Image.open(fileobj)
|
||||
fmt = src.format
|
||||
width, height = src.size
|
||||
if fmt not in ("PNG", "JPEG"):
|
||||
raise PermanentError("invalid_image")
|
||||
if width * height > config.MAX_IMAGE_PIXELS:
|
||||
raise PermanentError("image_too_large")
|
||||
if fmt == "JPEG" and width > 2560 and height > 2560:
|
||||
# экономия памяти: JPEG декодируется сразу в масштабе 1/2 (результат всё равно вписывается в 1280x1280)
|
||||
src.draft("RGB", (width // 2, height // 2))
|
||||
src.load()
|
||||
img = ImageOps.exif_transpose(src)
|
||||
except PermanentError:
|
||||
raise
|
||||
except MemoryError:
|
||||
raise
|
||||
except Exception:
|
||||
raise PermanentError("invalid_image") from None
|
||||
base = _flatten(img)
|
||||
out = []
|
||||
# thumbnail
|
||||
tw, th = _fit_size(base.width, base.height, 256)
|
||||
thumb = base.resize((tw, th), Image.LANCZOS) if (tw, th) != base.size else base
|
||||
data, sha = _encode(thumb, "JPEG", quality=JPEG_QUALITY, optimize=True)
|
||||
out.append({"name": "thumbnail", "content_type": "image/jpeg", "data": data, "sha256": sha, "width": tw, "height": th})
|
||||
# crop 128x128 (центрированный квадрат)
|
||||
side = min(base.width, base.height)
|
||||
left, top = (base.width - side) // 2, (base.height - side) // 2
|
||||
crop = base.crop((left, top, left + side, top + side)).resize((128, 128), Image.LANCZOS)
|
||||
data, sha = _encode(crop, "JPEG", quality=JPEG_QUALITY, optimize=True)
|
||||
out.append({"name": "crop", "content_type": "image/jpeg", "data": data, "sha256": sha, "width": 128, "height": 128})
|
||||
# optimized WebP (прозрачность сохраняется)
|
||||
opt_src = img.convert("RGBA") if (img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info)) else base
|
||||
ow, oh = _fit_size(opt_src.width, opt_src.height, 1280)
|
||||
opt = opt_src.resize((ow, oh), Image.LANCZOS) if (ow, oh) != opt_src.size else opt_src
|
||||
data, sha = _encode(opt, "WEBP", quality=WEBP_QUALITY, method=4)
|
||||
out.append({"name": "optimized", "content_type": "image/webp", "data": data, "sha256": sha, "width": ow, "height": oh})
|
||||
return out
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Сверка S3 и PostgreSQL (reconcile): python -m app.reconcile [--apply] [--grace СЕКУНДЫ] [--prefix att/]
|
||||
|
||||
Находит и (с --apply) безопасно исправляет расхождения двух хранилищ:
|
||||
* orphan_object — объект в S3 без строки в attachments (сбой между записью в S3 и COMMIT метаданных, прерванная загрузка);
|
||||
удаляется ТОЛЬКО если старше grace (по умолчанию ORPHAN_GRACE_SECONDS=900 с): свежий объект может быть
|
||||
загрузкой, метаданные которой ещё не зафиксированы;
|
||||
* missing_object — строка attachments (ready) без объекта в S3 (потеря/удаление объекта): строка НЕ удаляется,
|
||||
статус -> failed, error_code=object_missing (сообщения со ссылкой остаются, скачивание даёт 409);
|
||||
* size_mismatch — размер объекта не совпадает с size_bytes: статус -> failed, error_code=size_mismatch.
|
||||
Режим по умолчанию — dry-run (ничего не меняет). Повторный запуск идемпотентен. Используемые (привязанные к сообщению)
|
||||
вложения и свежие объекты не удаляются никогда: удаляются только объекты, у которых нет строки в БД.
|
||||
Выход: JSON-отчёт со счётчиками и временем обхода.
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import config
|
||||
from .s3 import S3
|
||||
|
||||
|
||||
async def reconcile(apply, grace, prefix):
|
||||
started = time.perf_counter()
|
||||
s3 = S3()
|
||||
await s3.start()
|
||||
conn = await asyncpg.connect(config.DATABASE_URL, timeout=5)
|
||||
try:
|
||||
t0 = time.perf_counter()
|
||||
rows = await conn.fetch("SELECT id, s3_key, size_bytes, status, created_at FROM attachments")
|
||||
db_ms = (time.perf_counter() - t0) * 1000
|
||||
by_key = {r["s3_key"]: r for r in rows}
|
||||
t0 = time.perf_counter()
|
||||
objects = {}
|
||||
async for key, size, modified in s3.list_keys(prefix):
|
||||
objects[key] = (size, modified)
|
||||
list_ms = (time.perf_counter() - t0) * 1000
|
||||
now = datetime.now(timezone.utc)
|
||||
cutoff = now - timedelta(seconds=grace)
|
||||
report = {"mode": "apply" if apply else "dry-run", "grace_seconds": grace, "db_rows": len(rows), "s3_objects": len(objects),
|
||||
"orphan_objects": [], "orphans_skipped_fresh": 0, "missing_objects": [], "size_mismatch": [], "fixed": {"deleted_orphans": 0, "marked_failed": 0}}
|
||||
for key, (size, modified) in objects.items():
|
||||
row = by_key.get(key)
|
||||
if row is None:
|
||||
if modified < cutoff:
|
||||
report["orphan_objects"].append(key)
|
||||
else:
|
||||
report["orphans_skipped_fresh"] += 1
|
||||
elif row["status"] == "ready" and row["size_bytes"] != size:
|
||||
report["size_mismatch"].append(str(row["id"]))
|
||||
for key, row in by_key.items():
|
||||
if key.startswith(prefix) and key not in objects and row["status"] == "ready" and row["created_at"] < cutoff:
|
||||
report["missing_objects"].append(str(row["id"]))
|
||||
if apply:
|
||||
for key in report["orphan_objects"]:
|
||||
# повторная проверка перед удалением: за время обхода строка могла появиться
|
||||
if await conn.fetchval("SELECT 1 FROM attachments WHERE s3_key = $1", key) is None:
|
||||
await s3.delete(key)
|
||||
report["fixed"]["deleted_orphans"] += 1
|
||||
for ids, code in ((report["missing_objects"], "object_missing"), (report["size_mismatch"], "size_mismatch")):
|
||||
for att_id in ids:
|
||||
n = await conn.execute("UPDATE attachments SET status = 'failed', error_code = $2, updated_at = now() "
|
||||
"WHERE id = $1 AND status = 'ready'", att_id, code)
|
||||
report["fixed"]["marked_failed"] += int(n.split()[-1])
|
||||
report["timing_ms"] = {"db_query": round(db_ms, 1), "s3_listing": round(list_ms, 1),
|
||||
"total": round((time.perf_counter() - started) * 1000, 1)}
|
||||
return report
|
||||
finally:
|
||||
await conn.close()
|
||||
await s3.stop()
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--apply", action="store_true", help="исправлять (по умолчанию dry-run)")
|
||||
ap.add_argument("--grace", type=int, default=config.ORPHAN_GRACE_SECONDS)
|
||||
ap.add_argument("--prefix", default="att/")
|
||||
ap.add_argument("--max-listed", type=int, default=20, help="сколько идентификаторов печатать в отчёте")
|
||||
args = ap.parse_args()
|
||||
report = asyncio.run(reconcile(args.apply, args.grace, args.prefix))
|
||||
for k in ("orphan_objects", "missing_objects", "size_mismatch"):
|
||||
report[k + "_count"] = len(report[k])
|
||||
report[k] = report[k][:args.max_listed]
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Операторский replay и реестр неудавшихся заданий: python -m app.replay [list | replay <attachment_id> | replay-all]
|
||||
|
||||
list — реестр failed-вложений: id, error_code, число попыток, возраст, наличие staging;
|
||||
replay <attachment_id> — возвращает ТОЛЬКО failed-вложение в queued: новая попытка (attempt сбрасывается на 1), новый job_id,
|
||||
запись в job_events (аудит), задание в outbox. Если у вложения уже есть варианты (ready/успешный
|
||||
эффект), повторного исполнения не будет: вложение не в статусе failed -> replay отклоняется.
|
||||
Эффекты защищены ключом (attachment_id, pipeline_version, name): даже повторное выполнение не создаёт второй набор вариантов.
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import config
|
||||
|
||||
|
||||
async def list_failed(conn):
|
||||
rows = await conn.fetch(
|
||||
"SELECT id, error_code, attempt, staging_key IS NOT NULL AS has_staging, updated_at FROM attachments "
|
||||
"WHERE status = 'failed' ORDER BY updated_at DESC LIMIT 200")
|
||||
return [{"attachment_id": str(r["id"]), "error_code": r["error_code"], "attempt": r["attempt"],
|
||||
"staging_present": r["has_staging"], "failed_at": r["updated_at"].isoformat()} for r in rows]
|
||||
|
||||
|
||||
async def replay(conn, att_id):
|
||||
async with conn.transaction():
|
||||
row = await conn.fetchrow("SELECT id, status, pipeline_version, staging_key FROM attachments WHERE id = $1 FOR UPDATE", att_id)
|
||||
if row is None:
|
||||
return {"attachment_id": str(att_id), "result": "not_found"}
|
||||
if row["status"] != "failed":
|
||||
return {"attachment_id": str(att_id), "result": "skipped", "reason": "status=%s (replay только для failed)" % row["status"]}
|
||||
if not row["staging_key"]:
|
||||
return {"attachment_id": str(att_id), "result": "skipped", "reason": "staging удалён по retention, исходник недоступен"}
|
||||
job_id = uuid.uuid4()
|
||||
await conn.execute("UPDATE attachments SET status = 'queued', error_code = NULL, attempt = 0, job_id = $2, lease_owner = NULL, "
|
||||
"lease_expires_at = NULL, updated_at = now() WHERE id = $1", att_id, job_id)
|
||||
await conn.execute("INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt) VALUES ($1, $2, $3, 1)",
|
||||
job_id, att_id, row["pipeline_version"])
|
||||
await conn.execute("INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) VALUES ($1, $2, 'operator_replay', 1, 'failed -> queued')",
|
||||
att_id, job_id)
|
||||
return {"attachment_id": str(att_id), "result": "requeued", "job_id": str(job_id)}
|
||||
|
||||
|
||||
async def main_async(args):
|
||||
conn = await asyncpg.connect(config.DATABASE_URL, timeout=5)
|
||||
try:
|
||||
if args.command == "list":
|
||||
return await list_failed(conn)
|
||||
if args.command == "replay":
|
||||
return await replay(conn, uuid.UUID(args.attachment_id))
|
||||
return [await replay(conn, uuid.UUID(f["attachment_id"])) for f in await list_failed(conn)]
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = ap.add_subparsers(dest="command", required=True)
|
||||
sub.add_parser("list")
|
||||
r = sub.add_parser("replay")
|
||||
r.add_argument("attachment_id")
|
||||
sub.add_parser("replay-all")
|
||||
print(json.dumps(asyncio.run(main_async(ap.parse_args())), ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Приватное S3-хранилище (MinIO) через aiobotocore: потоковая загрузка/скачивание, таймауты, перевод сбоев в 503."""
|
||||
import asyncio
|
||||
|
||||
from aiobotocore.config import AioConfig
|
||||
from aiobotocore.session import get_session
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from . import config, telemetry
|
||||
from .web import log, unavailable
|
||||
|
||||
S3_ERRORS = (BotoCoreError, ClientError, asyncio.TimeoutError, OSError)
|
||||
|
||||
|
||||
class S3:
|
||||
def __init__(self):
|
||||
self._ctx = None
|
||||
self.client = None
|
||||
|
||||
async def start(self):
|
||||
if not config.S3_ENDPOINT:
|
||||
log("error", "s3_endpoint_missing")
|
||||
return
|
||||
session = get_session()
|
||||
self._ctx = session.create_client(
|
||||
"s3", endpoint_url=config.S3_ENDPOINT, region_name=config.S3_REGION,
|
||||
aws_access_key_id=config.S3_ACCESS_KEY, aws_secret_access_key=config.S3_SECRET_KEY,
|
||||
config=AioConfig(connect_timeout=config.S3_CONNECT_TIMEOUT, read_timeout=config.S3_READ_TIMEOUT,
|
||||
retries={"max_attempts": 2, "mode": "standard"}, max_pool_connections=config.S3_POOL,
|
||||
s3={"addressing_style": "path"}))
|
||||
self.client = await self._ctx.__aenter__()
|
||||
|
||||
async def stop(self):
|
||||
if self._ctx:
|
||||
await self._ctx.__aexit__(None, None, None)
|
||||
|
||||
def _check(self):
|
||||
if self.client is None:
|
||||
raise unavailable("Хранилище файлов не настроено")
|
||||
|
||||
async def put(self, key, fileobj, content_type, size):
|
||||
self._check()
|
||||
try:
|
||||
with telemetry.child_span("s3.put_object", "client", {"rpc.system": "aws-api", "rpc.method": "PutObject"}):
|
||||
await self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentType=content_type,
|
||||
ContentLength=size)
|
||||
except S3_ERRORS as exc:
|
||||
log("warning", "s3_put_failed", error_type=type(exc).__name__)
|
||||
raise unavailable("Хранилище файлов недоступно") from exc
|
||||
|
||||
async def put_bytes(self, key, data, content_type):
|
||||
self._check()
|
||||
with telemetry.child_span("s3.put_object", "client", {"rpc.system": "aws-api", "rpc.method": "PutObject"}):
|
||||
await self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=data, ContentType=content_type)
|
||||
|
||||
async def open_stream(self, key):
|
||||
"""Возвращает (stream, content_length). Вызывающий обязан закрыть stream."""
|
||||
self._check()
|
||||
try:
|
||||
with telemetry.child_span("s3.get_object", "client", {"rpc.system": "aws-api", "rpc.method": "GetObject"}):
|
||||
resp = await self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
except S3_ERRORS as exc:
|
||||
log("warning", "s3_get_failed", error_type=type(exc).__name__)
|
||||
raise unavailable("Хранилище файлов недоступно") from exc
|
||||
return resp["Body"], resp["ContentLength"]
|
||||
|
||||
async def delete(self, key):
|
||||
self._check()
|
||||
await self.client.delete_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
|
||||
async def head(self, key):
|
||||
"""Метаданные объекта или None, если его нет."""
|
||||
self._check()
|
||||
try:
|
||||
return await self.client.head_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in ("404", "NoSuchKey", "NotFound"):
|
||||
return None
|
||||
raise unavailable("Хранилище файлов недоступно") from exc
|
||||
except S3_ERRORS as exc:
|
||||
raise unavailable("Хранилище файлов недоступно") from exc
|
||||
|
||||
async def ping(self):
|
||||
self._check()
|
||||
await asyncio.wait_for(self.client.head_bucket(Bucket=config.S3_BUCKET), 2.0)
|
||||
|
||||
async def list_keys(self, prefix):
|
||||
"""Асинхронный итератор (key, size, last_modified) по префиксу — для сверки (reconcile)."""
|
||||
self._check()
|
||||
paginator = self.client.get_paginator("list_objects_v2")
|
||||
async for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix):
|
||||
for obj in page.get("Contents", []):
|
||||
yield obj["Key"], obj["Size"], obj["LastModified"]
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Идемпотентный bootstrap S3: python -m app.s3_bootstrap (one-shot сервис s3-init при каждом `docker compose up`).
|
||||
|
||||
1) bucket создаётся администратором, если его ещё нет (повтор ничего не меняет);
|
||||
2) проверяется, что анонимный запрос к bucket отклоняется (private);
|
||||
3) проверяется, что отдельный ключ API (права только на этот bucket) может писать/читать/удалять и НЕ может создавать bucket.
|
||||
Ключи читаются из файлов секретов; в вывод не попадают. Код выхода != 0, если любая проверка не пройдена.
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from aiobotocore.config import AioConfig
|
||||
from aiobotocore.session import get_session
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from . import config
|
||||
|
||||
|
||||
def secret(name):
|
||||
path = os.environ.get(name + "_FILE")
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
return fh.read().strip()
|
||||
|
||||
|
||||
def client(session, access, secret_key):
|
||||
return session.create_client("s3", endpoint_url=config.S3_ENDPOINT, region_name=config.S3_REGION, aws_access_key_id=access,
|
||||
aws_secret_access_key=secret_key,
|
||||
config=AioConfig(connect_timeout=3, read_timeout=10, retries={"max_attempts": 1},
|
||||
s3={"addressing_style": "path"}))
|
||||
|
||||
|
||||
async def main():
|
||||
bucket = config.S3_BUCKET
|
||||
session = get_session()
|
||||
deadline = time.time() + 60
|
||||
async with client(session, secret("S3_ADMIN_ACCESS_KEY"), secret("S3_ADMIN_SECRET_KEY")) as admin:
|
||||
while True:
|
||||
try:
|
||||
await admin.list_buckets()
|
||||
break
|
||||
except Exception as exc:
|
||||
if time.time() > deadline:
|
||||
print("S3 недоступен:", type(exc).__name__)
|
||||
return 1
|
||||
await asyncio.sleep(1)
|
||||
try:
|
||||
await admin.head_bucket(Bucket=bucket)
|
||||
print("bucket %s уже существует" % bucket)
|
||||
except ClientError:
|
||||
await admin.create_bucket(Bucket=bucket)
|
||||
print("bucket %s создан" % bucket)
|
||||
# приватность: запрос без подписи отклоняется
|
||||
try:
|
||||
urllib.request.urlopen("%s/%s/" % (config.S3_ENDPOINT, bucket), timeout=5)
|
||||
print("ОШИБКА: bucket доступен анонимно")
|
||||
return 2
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code not in (401, 403):
|
||||
print("ОШИБКА: анонимный доступ вернул %d, ожидался 401/403" % exc.code)
|
||||
return 2
|
||||
print("анонимный доступ запрещён (HTTP %d)" % exc.code)
|
||||
# ключ API: рабочие права на bucket, нет прав администратора
|
||||
async with client(session, config.S3_ACCESS_KEY, config.S3_SECRET_KEY) as api:
|
||||
key = "bootstrap-check/%d" % int(time.time())
|
||||
await api.put_object(Bucket=bucket, Key=key, Body=b"ok")
|
||||
got = await (await api.get_object(Bucket=bucket, Key=key))["Body"].read()
|
||||
await api.delete_object(Bucket=bucket, Key=key)
|
||||
if got != b"ok":
|
||||
print("ОШИБКА: чтение/запись ключом API")
|
||||
return 3
|
||||
try:
|
||||
await api.create_bucket(Bucket="not-allowed-" + str(int(time.time())))
|
||||
print("ОШИБКА: ключ API смог создать bucket (избыточные права)")
|
||||
return 4
|
||||
except ClientError as exc:
|
||||
print("ключ API ограничен одним bucket (create_bucket отклонён: %s)" % exc.response["Error"]["Code"])
|
||||
print("bootstrap завершён")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Точка входа: uvicorn с управляемым завершением (drain).
|
||||
|
||||
SIGTERM -> readiness сразу 503 -> DRAIN_SECONDS продолжаем обслуживать запросы, пока балансировщик
|
||||
исключает реплику -> uvicorn прекращает приём и дожидается текущих запросов (timeout_graceful_shutdown).
|
||||
Так плановая остановка/rolling update не даёт ошибок клиентам.
|
||||
"""
|
||||
import signal
|
||||
import threading
|
||||
|
||||
import uvicorn
|
||||
|
||||
from . import config
|
||||
from .main import app
|
||||
from .web import log
|
||||
|
||||
|
||||
class DrainingServer(uvicorn.Server):
|
||||
_draining = False
|
||||
|
||||
def handle_exit(self, sig, frame):
|
||||
if sig == signal.SIGTERM and not self._draining and config.DRAIN_SECONDS > 0:
|
||||
self._draining = True
|
||||
app.draining = True
|
||||
log("info", "draining", seconds=config.DRAIN_SECONDS)
|
||||
timer = threading.Timer(config.DRAIN_SECONDS, super().handle_exit, args=(sig, frame))
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
return
|
||||
super().handle_exit(sig, frame)
|
||||
|
||||
|
||||
def main():
|
||||
cfg = uvicorn.Config(app, host=config.APP_HOST, port=config.APP_PORT, workers=1, access_log=False,
|
||||
log_level="warning", timeout_graceful_shutdown=config.GRACEFUL_SHUTDOWN_SECONDS,
|
||||
timeout_keep_alive=config.KEEPALIVE_SECONDS, limit_concurrency=config.LIMIT_CONCURRENCY)
|
||||
DrainingServer(cfg).run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Сессии: PostgreSQL — источник истины; Valkey — кэш с TTL (read-through).
|
||||
|
||||
Политики:
|
||||
* токен — 256 бит из secrets; в БД и Valkey хранится только SHA-256 токена;
|
||||
* expiry: expires_at в БД (по часам БД). В кэше TTL = min(остаток жизни, SESSION_CACHE_TTL_SECONDS);
|
||||
* инвалидация: logout помечает revoked_at в БД и удаляет ключ в Valkey;
|
||||
* Valkey недоступен -> кэш обходится (чтение из БД), анонимный запрос никогда не считается доверенным;
|
||||
circuit breaker не даёт каждому запросу ждать таймаут;
|
||||
* окно устаревания: если Valkey был недоступен во время logout, ключ может прожить до SESSION_CACHE_TTL_SECONDS.
|
||||
"""
|
||||
import hashlib
|
||||
import secrets
|
||||
import time
|
||||
|
||||
from . import config
|
||||
from .web import log
|
||||
|
||||
try:
|
||||
import redis.asyncio as aioredis
|
||||
except ImportError: # кэш опционален
|
||||
aioredis = None
|
||||
|
||||
|
||||
def token_hash(token):
|
||||
return hashlib.sha256(token.encode("ascii")).digest()
|
||||
|
||||
|
||||
class SessionCache:
|
||||
def __init__(self):
|
||||
self.client = None
|
||||
self.down_until = 0.0
|
||||
self.stats = {"hit": 0, "miss": 0, "error": 0}
|
||||
|
||||
async def start(self):
|
||||
if config.VALKEY_URL and aioredis:
|
||||
self.client = aioredis.Redis.from_url(config.VALKEY_URL, socket_timeout=config.VALKEY_TIMEOUT,
|
||||
socket_connect_timeout=config.VALKEY_TIMEOUT)
|
||||
|
||||
async def stop(self):
|
||||
if self.client:
|
||||
await self.client.aclose()
|
||||
|
||||
def _usable(self):
|
||||
return self.client is not None and time.monotonic() >= self.down_until
|
||||
|
||||
def _fail(self, exc):
|
||||
self.stats["error"] += 1
|
||||
if time.monotonic() >= self.down_until:
|
||||
log("warning", "session_cache_unavailable", error_type=type(exc).__name__)
|
||||
self.down_until = time.monotonic() + config.VALKEY_BREAKER_SECONDS
|
||||
|
||||
async def get(self, key):
|
||||
if not self._usable():
|
||||
return None
|
||||
try:
|
||||
value = await self.client.get(b"s:" + key)
|
||||
except Exception as exc:
|
||||
self._fail(exc)
|
||||
return None
|
||||
self.stats["hit" if value is not None else "miss"] += 1
|
||||
return value
|
||||
|
||||
async def put(self, key, value, ttl):
|
||||
if not self._usable() or ttl <= 0:
|
||||
return
|
||||
try:
|
||||
await self.client.set(b"s:" + key, value, ex=max(1, int(ttl)))
|
||||
except Exception as exc:
|
||||
self._fail(exc)
|
||||
|
||||
async def delete(self, key):
|
||||
if not self._usable():
|
||||
return
|
||||
try:
|
||||
await self.client.delete(b"s:" + key)
|
||||
except Exception as exc:
|
||||
self._fail(exc)
|
||||
|
||||
async def ping(self):
|
||||
if self.client is None:
|
||||
return None
|
||||
try:
|
||||
return bool(await self.client.ping())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class Sessions:
|
||||
def __init__(self, db, cache):
|
||||
self.db, self.cache = db, cache
|
||||
|
||||
@staticmethod
|
||||
def _cache_value(user_id, expires_at):
|
||||
return ("%s|%d" % (user_id, int(expires_at.timestamp() * 1000))).encode()
|
||||
|
||||
async def create(self, user_id):
|
||||
token = secrets.token_urlsafe(32) # 256 бит энтропии
|
||||
digest = token_hash(token)
|
||||
row = await self.db.fetchrow(
|
||||
"INSERT INTO sessions (token_hash, user_id, expires_at) "
|
||||
"VALUES ($1, $2, now() + make_interval(secs => $3)) RETURNING expires_at",
|
||||
digest, user_id, float(config.SESSION_TTL_SECONDS))
|
||||
remaining = row["expires_at"].timestamp() - time.time()
|
||||
await self.cache.put(digest, self._cache_value(user_id, row["expires_at"]),
|
||||
min(remaining, config.SESSION_CACHE_TTL_SECONDS))
|
||||
return token, row["expires_at"]
|
||||
|
||||
async def resolve(self, token):
|
||||
"""token -> user_id (str) либо None, если сессии нет/истекла/отозвана."""
|
||||
digest = token_hash(token)
|
||||
cached = await self.cache.get(digest)
|
||||
if cached is not None:
|
||||
user_id, _, expires_ms = cached.decode().partition("|")
|
||||
if int(expires_ms) > time.time() * 1000:
|
||||
return user_id
|
||||
await self.cache.delete(digest)
|
||||
row = await self.db.fetchrow(
|
||||
"SELECT user_id, expires_at FROM sessions "
|
||||
"WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > now()", digest)
|
||||
if row is None:
|
||||
return None
|
||||
user_id = str(row["user_id"])
|
||||
remaining = row["expires_at"].timestamp() - time.time()
|
||||
await self.cache.put(digest, self._cache_value(user_id, row["expires_at"]),
|
||||
min(remaining, config.SESSION_CACHE_TTL_SECONDS))
|
||||
return user_id
|
||||
|
||||
async def revoke(self, token):
|
||||
digest = token_hash(token)
|
||||
await self.db.execute("UPDATE sessions SET revoked_at = now() WHERE token_hash = $1 AND revoked_at IS NULL", digest)
|
||||
await self.cache.delete(digest)
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Наблюдаемость (lab6): OpenTelemetry SDK — трассы, метрики, логи по OTLP/HTTP в Collector.
|
||||
|
||||
Выключается переменной OTEL_ENABLED=0 (нужно для эксперимента «цена наблюдаемости»): тогда все функции — no-op и
|
||||
накладных расходов на экспорт нет. Секреты, тела сообщений, токены и изображения в телеметрию не попадают:
|
||||
в атрибутах только шаблон маршрута, метод, статус, тип задания и исход; ID объектов — только в логах/спанах
|
||||
(не в labels метрик).
|
||||
|
||||
Соответствие сигналов contracts/observability.md и имён метрик (после Prometheus-экспортёра Collector):
|
||||
http.server.request.duration (s) -> http_server_request_duration_seconds_{bucket,sum,count} (HTTP requests = _count)
|
||||
messenger.jobs.processed -> messenger_jobs_processed_total{job_type,outcome}
|
||||
messenger.job.duration (s) -> messenger_job_duration_seconds_*
|
||||
messenger.job.time_to_ready (s) -> messenger_job_time_to_ready_seconds_* (от приёма до ready)
|
||||
messenger.queue.depth / .oldest_age -> messenger_queue_depth{queue_name}, messenger_queue_oldest_job_age_seconds
|
||||
db.client.connections.usage/waiting -> db_client_connections_usage{state}, db_client_connections_waiting
|
||||
messenger.worker.busy / .capacity -> messenger_worker_busy, messenger_worker_capacity
|
||||
process.memory.rss / process.cpu.time -> process_memory_rss_bytes, process_cpu_time_seconds_total
|
||||
"""
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
from . import config
|
||||
|
||||
ENABLED = config.OTEL_ENABLED
|
||||
_state = {"tracer": None, "meter": None, "logger": None, "propagator": None, "inst": {}}
|
||||
HTTP_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]
|
||||
JOB_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]
|
||||
TTR_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300]
|
||||
|
||||
|
||||
def setup(service_name):
|
||||
"""Инициализирует провайдеры. Вызывается один раз при старте процесса (API или worker)."""
|
||||
if not ENABLED:
|
||||
return
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.propagate import get_global_textmap
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
|
||||
|
||||
endpoint = config.OTEL_ENDPOINT.rstrip("/")
|
||||
resource = Resource.create({"service.name": service_name, "service.version": config.SERVICE_VERSION,
|
||||
"service.instance.id": config.INSTANCE_ID, "deployment.environment.name": "lab"})
|
||||
# --- трассы: head-sampling по доле (ParentBased: решение родителя, в т.ч. из traceparent брокера, сохраняется)
|
||||
tp = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(config.OTEL_TRACE_RATIO)))
|
||||
tp.add_span_processor(BatchSpanProcessor(
|
||||
OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=3),
|
||||
max_queue_size=config.OTEL_QUEUE_SIZE, max_export_batch_size=config.OTEL_BATCH_SIZE, schedule_delay_millis=2000, export_timeout_millis=3000))
|
||||
trace.set_tracer_provider(tp)
|
||||
# --- метрики (histogram с явными границами, секунды)
|
||||
views = [View(instrument_name="http.server.request.duration", aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)),
|
||||
View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
|
||||
View(instrument_name="messenger.job.time_to_ready", aggregation=ExplicitBucketHistogramAggregation(TTR_BUCKETS))]
|
||||
reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=3),
|
||||
export_interval_millis=config.OTEL_METRIC_INTERVAL_MS, export_timeout_millis=3000)
|
||||
mp = MeterProvider(resource=resource, metric_readers=[reader], views=views)
|
||||
metrics.set_meter_provider(mp)
|
||||
# --- логи: стандартный logging -> OTLP (trace_id/span_id добавляются автоматически из активного спана)
|
||||
lp = LoggerProvider(resource=resource)
|
||||
lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=3),
|
||||
max_queue_size=config.OTEL_QUEUE_SIZE, schedule_delay_millis=2000))
|
||||
logger = logging.getLogger("messenger")
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.propagate = False
|
||||
logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=lp))
|
||||
_state.update(tracer=trace.get_tracer("messenger"), meter=metrics.get_meter("messenger"), logger=logger,
|
||||
propagator=get_global_textmap(), providers=(tp, mp, lp))
|
||||
m = _state["meter"]
|
||||
_state["inst"] = {
|
||||
"http": m.create_histogram("http.server.request.duration", unit="s", description="Длительность HTTP-запроса"),
|
||||
"jobs": m.create_counter("messenger.jobs.processed", description="Обработанные задания по исходам"),
|
||||
"job_dur": m.create_histogram("messenger.job.duration", unit="s", description="Длительность обработки задания"),
|
||||
"ttr": m.create_histogram("messenger.job.time_to_ready", unit="s", description="От приёма загрузки до ready"),
|
||||
}
|
||||
for args in list(_pending_gauges):
|
||||
(observable_counter if len(args) == 5 else gauge)(*args[:4])
|
||||
_pending_gauges.clear()
|
||||
|
||||
|
||||
def shutdown():
|
||||
for p in _state.get("providers", ()):
|
||||
try:
|
||||
p.shutdown()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# --- трассы ----------------------------------------------------------------------------------
|
||||
@contextlib.contextmanager
|
||||
def span(name, kind="internal", attributes=None, carrier=None):
|
||||
"""Контекстный менеджер спана. carrier — dict с traceparent/tracestate для продолжения чужого trace."""
|
||||
if not ENABLED or _state["tracer"] is None:
|
||||
yield None
|
||||
return
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
ctx = _state["propagator"].extract(carrier) if carrier else None
|
||||
kinds = {"server": SpanKind.SERVER, "client": SpanKind.CLIENT, "producer": SpanKind.PRODUCER,
|
||||
"consumer": SpanKind.CONSUMER, "internal": SpanKind.INTERNAL}
|
||||
with _state["tracer"].start_as_current_span(name, context=ctx, kind=kinds[kind], attributes=attributes or {},
|
||||
record_exception=False, set_status_on_exception=False) as sp:
|
||||
try:
|
||||
yield sp
|
||||
except Exception as exc:
|
||||
sp.set_status(Status(StatusCode.ERROR, type(exc).__name__))
|
||||
sp.set_attribute("error.type", type(exc).__name__)
|
||||
raise
|
||||
|
||||
|
||||
def child_span(name, kind="internal", attributes=None):
|
||||
"""Спан создаётся только внутри активного спана (запрос/задание); фоновые опросы БД не порождают корневых трасс."""
|
||||
if not ENABLED or _state["tracer"] is None:
|
||||
return contextlib.nullcontext()
|
||||
from opentelemetry import trace
|
||||
if not trace.get_current_span().get_span_context().is_valid:
|
||||
return contextlib.nullcontext()
|
||||
return span(name, kind, attributes)
|
||||
|
||||
|
||||
def set_attr(sp, key, value):
|
||||
if sp is not None:
|
||||
sp.set_attribute(key, value)
|
||||
|
||||
|
||||
def mark_error(sp, description):
|
||||
if sp is not None:
|
||||
from opentelemetry.trace import Status, StatusCode
|
||||
sp.set_status(Status(StatusCode.ERROR, description))
|
||||
|
||||
|
||||
def current_carrier():
|
||||
"""traceparent/tracestate текущего спана (для передачи через outbox/брокер)."""
|
||||
if not ENABLED or _state["propagator"] is None:
|
||||
return {}
|
||||
carrier = {}
|
||||
_state["propagator"].inject(carrier)
|
||||
return carrier
|
||||
|
||||
|
||||
def trace_ids():
|
||||
if not ENABLED:
|
||||
return None, None
|
||||
from opentelemetry import trace
|
||||
ctx = trace.get_current_span().get_span_context()
|
||||
if not ctx.is_valid:
|
||||
return None, None
|
||||
return "%032x" % ctx.trace_id, "%016x" % ctx.span_id
|
||||
|
||||
|
||||
# --- метрики ----------------------------------------------------------------------------------
|
||||
def record_http(method, route, status, seconds):
|
||||
if ENABLED and _state["inst"]:
|
||||
_state["inst"]["http"].record(seconds, {"http.request.method": method, "http.route": route,
|
||||
"http.response.status_code": status})
|
||||
|
||||
|
||||
def record_job(job_type, outcome, seconds, time_to_ready=None):
|
||||
if ENABLED and _state["inst"]:
|
||||
attrs = {"job.type": job_type, "outcome": outcome}
|
||||
_state["inst"]["jobs"].add(1, attrs)
|
||||
_state["inst"]["job_dur"].record(seconds, attrs)
|
||||
if time_to_ready is not None:
|
||||
_state["inst"]["ttr"].record(time_to_ready, {"job.type": job_type})
|
||||
|
||||
|
||||
_pending_gauges = []
|
||||
|
||||
|
||||
def gauge(name, callback, unit="1", description=""):
|
||||
"""Observable gauge: callback() -> [(value, {attrs})]. Регистрация откладывается до setup(), если провайдер ещё не создан."""
|
||||
if not ENABLED:
|
||||
return
|
||||
if _state["meter"] is None:
|
||||
_pending_gauges.append((name, callback, unit, description))
|
||||
return
|
||||
from opentelemetry.metrics import Observation
|
||||
|
||||
def cb(_options):
|
||||
try:
|
||||
return [Observation(v, a) for v, a in callback()]
|
||||
except Exception:
|
||||
return []
|
||||
_state["meter"].create_observable_gauge(name, callbacks=[cb], unit=unit, description=description)
|
||||
|
||||
|
||||
def observable_counter(name, callback, unit="1", description=""):
|
||||
"""Монотонный счётчик, значение которого читается при экспорте (например, накопленное CPU-время)."""
|
||||
if not ENABLED or _state["meter"] is None:
|
||||
_pending_gauges.append((name, callback, unit, description, "counter"))
|
||||
return
|
||||
from opentelemetry.metrics import Observation
|
||||
|
||||
def cb(_options):
|
||||
try:
|
||||
return [Observation(v, a) for v, a in callback()]
|
||||
except Exception:
|
||||
return []
|
||||
_state["meter"].create_observable_counter(name, callbacks=[cb], unit=unit, description=description)
|
||||
|
||||
|
||||
def process_metrics():
|
||||
def rss():
|
||||
try:
|
||||
with open("/proc/self/statm") as fh:
|
||||
return [(int(fh.read().split()[1]) * os.sysconf("SC_PAGE_SIZE"), {})]
|
||||
except (OSError, ValueError):
|
||||
return []
|
||||
gauge("process.memory.rss", rss, "By", "Резидентная память процесса")
|
||||
observable_counter("process.cpu.time", lambda: [(time.process_time(), {})], "s", "Накопленное CPU-время процесса")
|
||||
|
||||
|
||||
# --- логи ---------------------------------------------------------------------------------------
|
||||
def emit_log(level, record):
|
||||
"""Отправляет запись в OTLP (если включено). record — dict без секретов."""
|
||||
if not ENABLED or _state["logger"] is None:
|
||||
return
|
||||
lvl = {"info": logging.INFO, "warning": logging.WARNING, "error": logging.ERROR}.get(level, logging.INFO)
|
||||
_state["logger"].log(lvl, record.get("msg", ""), extra={k: v for k, v in record.items() if k not in ("msg", "level", "ts")})
|
||||
+192
@@ -0,0 +1,192 @@
|
||||
"""Общий HTTP-слой: ошибки, запрос, валидация, курсоры, ответы, журнал."""
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
from . import config, telemetry
|
||||
|
||||
UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
|
||||
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
|
||||
LIMIT_RE = re.compile(r"[0-9]{1,9}")
|
||||
EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status, code, message, headers=None):
|
||||
super().__init__(message)
|
||||
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
|
||||
|
||||
|
||||
def bad(message):
|
||||
return ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def unavailable(message="Зависимость временно недоступна"):
|
||||
return ApiError(503, "unavailable", message, {"retry-after": str(config.RETRY_AFTER_SECONDS)})
|
||||
|
||||
|
||||
def log(level, msg, **fields):
|
||||
"""Структурный лог: JSON в stdout + OTLP (если включено). trace_id/span_id — из активного спана; тела и токены не логируются."""
|
||||
record = {"ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "level": level, "service": config.SERVICE_NAME, "msg": msg}
|
||||
trace_id, span_id = telemetry.trace_ids()
|
||||
if trace_id:
|
||||
record["trace_id"], record["span_id"] = trace_id, span_id
|
||||
record.update(fields)
|
||||
sys.stdout.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
sys.stdout.flush()
|
||||
telemetry.emit_log(level, record)
|
||||
|
||||
|
||||
# --- время -------------------------------------------------------------------
|
||||
def format_dt(value):
|
||||
value = value.astimezone(timezone.utc)
|
||||
return value.strftime("%Y-%m-%dT%H:%M:%S.") + "%06dZ" % value.microsecond
|
||||
|
||||
|
||||
def dt_to_us(value):
|
||||
return (value - EPOCH) // timedelta(microseconds=1)
|
||||
|
||||
|
||||
def us_to_dt(us):
|
||||
return EPOCH + timedelta(microseconds=us)
|
||||
|
||||
|
||||
# --- валидация ---------------------------------------------------------------
|
||||
def parse_uuid(value, what="id"):
|
||||
if not isinstance(value, str) or not UUID_RE.fullmatch(value):
|
||||
raise bad("Некорректный %s" % what)
|
||||
return value.lower()
|
||||
|
||||
|
||||
def valid_text(value, field, maximum, allow_empty=False):
|
||||
if not isinstance(value, str):
|
||||
raise bad("Поле %s должно быть строкой" % field)
|
||||
try:
|
||||
value.encode("utf-8")
|
||||
except UnicodeEncodeError:
|
||||
raise bad("Поле %s содержит некорректные символы" % field) from None
|
||||
if "\x00" in value:
|
||||
raise bad("Поле %s содержит NUL" % field)
|
||||
if len(value) > maximum:
|
||||
raise bad("Поле %s слишком длинное" % field)
|
||||
if not allow_empty and not value.strip():
|
||||
raise bad("Поле %s не может быть пустым" % field)
|
||||
return value
|
||||
|
||||
|
||||
def _reject_constant(name):
|
||||
raise ValueError(name)
|
||||
|
||||
|
||||
def parse_json_object(raw, required, allowed):
|
||||
try:
|
||||
data = json.loads(raw.decode("utf-8"), parse_constant=_reject_constant)
|
||||
except (UnicodeDecodeError, ValueError, RecursionError):
|
||||
raise bad("Тело запроса — некорректный JSON") from None
|
||||
if not isinstance(data, dict):
|
||||
raise bad("Ожидается JSON-объект")
|
||||
if set(data) - allowed:
|
||||
raise bad("Лишние поля в запросе")
|
||||
if any(key not in data for key in required):
|
||||
raise bad("Не хватает обязательных полей")
|
||||
return data
|
||||
|
||||
|
||||
def encode_cursor(kind, resource, key):
|
||||
payload = json.dumps({"k": kind, "r": resource, "t": key[0], "i": key[1]}, separators=(",", ":"))
|
||||
return base64.urlsafe_b64encode(payload.encode()).decode().rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(text, kind, resource):
|
||||
try:
|
||||
data = json.loads(base64.urlsafe_b64decode(text + "=" * (-len(text) % 4)).decode())
|
||||
if (not isinstance(data, dict) or data.get("k") != kind or data.get("r") != resource
|
||||
or not isinstance(data.get("t"), int) or isinstance(data.get("t"), bool)
|
||||
or not isinstance(data.get("i"), str) or not UUID_RE.fullmatch(data["i"])
|
||||
or not -62135596800000000 < data["t"] < 253402300799000000):
|
||||
raise ValueError
|
||||
return data["t"], data["i"]
|
||||
except (ValueError, UnicodeError):
|
||||
raise bad("Некорректный cursor") from None
|
||||
|
||||
|
||||
def page_params(query, kind, resource):
|
||||
limit, cursor = 50, None
|
||||
if "limit" in query:
|
||||
if len(query["limit"]) != 1 or not LIMIT_RE.fullmatch(query["limit"][0]):
|
||||
raise bad("limit должен быть целым числом 1–100")
|
||||
limit = int(query["limit"][0])
|
||||
if not 1 <= limit <= 100:
|
||||
raise bad("limit должен быть целым числом 1–100")
|
||||
if "cursor" in query:
|
||||
if len(query["cursor"]) != 1 or not query["cursor"][0]:
|
||||
raise bad("Некорректный cursor")
|
||||
cursor = decode_cursor(query["cursor"][0], kind, resource)
|
||||
return limit, cursor
|
||||
|
||||
|
||||
def parse_query(raw):
|
||||
result = {}
|
||||
if raw:
|
||||
for key, value in parse_qsl(raw.decode("latin-1"), keep_blank_values=True):
|
||||
result.setdefault(key, []).append(value)
|
||||
return result
|
||||
|
||||
|
||||
class Request:
|
||||
def __init__(self, scope, receive, request_id):
|
||||
self.method = scope["method"]
|
||||
self.path = scope["path"]
|
||||
self.query = parse_query(scope.get("query_string", b""))
|
||||
self.headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope["headers"]}
|
||||
self.receive = receive
|
||||
self.request_id = request_id
|
||||
self.user_id = None
|
||||
|
||||
async def body(self):
|
||||
declared = self.headers.get("content-length")
|
||||
if declared is not None:
|
||||
if not declared.isdigit():
|
||||
raise bad("Некорректный Content-Length")
|
||||
if int(declared) > config.MAX_BODY_BYTES:
|
||||
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"connection": "close"})
|
||||
chunks, size = [], 0
|
||||
while True:
|
||||
event = await self.receive()
|
||||
if event["type"] == "http.disconnect":
|
||||
raise bad("Соединение закрыто клиентом")
|
||||
chunk = event.get("body", b"")
|
||||
size += len(chunk)
|
||||
if size > config.MAX_BODY_BYTES:
|
||||
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"connection": "close"})
|
||||
chunks.append(chunk)
|
||||
if not event.get("more_body"):
|
||||
return b"".join(chunks)
|
||||
|
||||
def require_json_media_type(self):
|
||||
media = self.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if media != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: application/json")
|
||||
|
||||
|
||||
async def respond(send, status, body, request_id, extra=None, instance_id=None):
|
||||
payload = b"" if status == 204 else json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
headers = [(b"x-request-id", request_id.encode())]
|
||||
if status != 204:
|
||||
headers += [(b"content-type", b"application/json"), (b"content-length", str(len(payload)).encode())]
|
||||
else:
|
||||
headers.append((b"content-length", b"0"))
|
||||
if instance_id:
|
||||
headers.append((b"x-instance-id", instance_id.encode()))
|
||||
headers += [(k.encode(), v.encode()) for k, v in (extra or {}).items()]
|
||||
await send({"type": "http.response.start", "status": status, "headers": headers})
|
||||
await send({"type": "http.response.body", "body": payload})
|
||||
|
||||
|
||||
def new_request_id():
|
||||
return uuid.uuid4().hex
|
||||
@@ -0,0 +1,345 @@
|
||||
"""Worker вложений: python -m app.worker
|
||||
|
||||
Конвейер задания (jobs.schema.json, состояния — contracts/attachment-states.md):
|
||||
claim (queued -> processing, lease) -> чтение staging -> варианты (Pillow, в потоках) -> запись в конечный S3-prefix
|
||||
-> ОДНА транзакция: варианты + ready -> ack -> очистка staging.
|
||||
Ack отправляется только после устойчивого эффекта (COMMIT результата). Повторная доставка уже готовой задачи — no-op с ack.
|
||||
Временные ошибки: возврат в queued с backoff (экспоненциальный + jitter) через outbox; после MAX_ATTEMPTS — failed/retry_exhausted.
|
||||
Постоянные ошибки (невалидное изображение): failed с error_code сразу. Зависшие processing возвращает sweeper по истечении lease.
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import signal
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import aio_pika
|
||||
|
||||
from . import config, pipeline, telemetry
|
||||
from .broker import QUEUE, Broker
|
||||
from .db import Database
|
||||
from .outbox import Relay
|
||||
from .s3 import S3
|
||||
from .web import log
|
||||
|
||||
WORKER_ID = "%s-%s" % ((os.environ.get("HOSTNAME") or "worker")[:12], uuid.uuid4().hex[:6])
|
||||
|
||||
|
||||
def backoff_seconds(attempt):
|
||||
"""Экспоненциальная задержка с полным jitter: случайное значение в [0, min(cap, base * 2^(attempt-1))]."""
|
||||
ceiling = min(config.RETRY_MAX_DELAY, config.RETRY_BASE_DELAY * (2 ** max(0, attempt - 1)))
|
||||
return random.uniform(ceiling * 0.5, ceiling)
|
||||
|
||||
|
||||
class Worker:
|
||||
def __init__(self):
|
||||
self.db, self.s3, self.broker = Database(), S3(), Broker()
|
||||
self.relay = Relay(self.db, self.broker)
|
||||
self.pool = ThreadPoolExecutor(max_workers=config.WORKER_CONCURRENCY, thread_name_prefix="img")
|
||||
self.busy = 0
|
||||
self.stopping = False
|
||||
self.tasks = []
|
||||
telemetry.gauge("messenger.worker.busy", lambda: [(self.busy, {})], "{job}", "Задания в обработке")
|
||||
telemetry.gauge("messenger.worker.capacity", lambda: [(config.WORKER_CONCURRENCY, {})], "{job}", "Ёмкость worker")
|
||||
|
||||
# --- claim / finalize ---------------------------------------------------------
|
||||
async def claim(self, job):
|
||||
return await self.db.fetchrow(
|
||||
"UPDATE attachments SET status = 'processing', lease_owner = $2, processing_started_at = now(), "
|
||||
"lease_expires_at = now() + make_interval(secs => $3), attempt = $4, job_id = $5, updated_at = now() "
|
||||
"WHERE id = $1 AND status = 'queued' AND pipeline_version <= $6 "
|
||||
"RETURNING id, chat_id, staging_key, s3_key, content_type, size_bytes, attempt",
|
||||
uuid.UUID(job["attachment_id"]), WORKER_ID, float(config.LEASE_SECONDS), job["attempt"],
|
||||
uuid.UUID(job["job_id"]), job["pipeline_version"], retry=False)
|
||||
|
||||
async def fail(self, att_id, job, code, event="failed"):
|
||||
await self.db.execute(
|
||||
"WITH u AS (UPDATE attachments SET status = 'failed', error_code = $2, lease_owner = NULL, lease_expires_at = NULL, "
|
||||
"updated_at = now() WHERE id = $1 AND status = 'processing' AND lease_owner = $3 RETURNING id) "
|
||||
"INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $4, $5, $6, $2 FROM u",
|
||||
att_id, code, WORKER_ID, uuid.UUID(job["job_id"]), event, job["attempt"])
|
||||
|
||||
async def reschedule(self, att_id, job, reason):
|
||||
"""Временная ошибка: processing -> queued, новая запись outbox с отложенной публикацией (backoff + jitter)."""
|
||||
nxt = job["attempt"] + 1
|
||||
if nxt > config.MAX_ATTEMPTS:
|
||||
log("warning", "job_retry_exhausted", attachment_id=str(att_id), attempts=job["attempt"])
|
||||
await self.fail(att_id, job, "retry_exhausted", "retry_exhausted")
|
||||
return
|
||||
delay = backoff_seconds(job["attempt"])
|
||||
await self.db.execute(
|
||||
"WITH u AS (UPDATE attachments SET status = 'queued', lease_owner = NULL, lease_expires_at = NULL, updated_at = now() "
|
||||
"WHERE id = $1 AND status = 'processing' AND lease_owner = $2 RETURNING id, pipeline_version), "
|
||||
"o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, available_at, traceparent, tracestate) "
|
||||
"SELECT $3, id, pipeline_version, $4, now() + make_interval(secs => $5), $7, $8 FROM u RETURNING 1) "
|
||||
"INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $3, 'retry_scheduled', $4, $6 FROM u",
|
||||
att_id, WORKER_ID, uuid.UUID(job["job_id"]), nxt, delay, reason, job.get("traceparent"), job.get("tracestate"))
|
||||
log("info", "job_rescheduled", attachment_id=str(att_id), next_attempt=nxt, delay_seconds=round(delay, 2), reason=reason)
|
||||
|
||||
async def publish_result(self, row, job, variants):
|
||||
"""Одна транзакция: варианты (идемпотентно) + ready. Только владелец lease может опубликовать результат."""
|
||||
async with self.db.conn() as c:
|
||||
async with c.transaction():
|
||||
owner = await c.fetchval("SELECT 1 FROM attachments WHERE id = $1 AND status = 'processing' AND lease_owner = $2 "
|
||||
"FOR UPDATE", row["id"], WORKER_ID)
|
||||
if owner is None:
|
||||
return False # lease потерян (sweeper вернул задачу): результат не публикуем, чужой повтор доделает
|
||||
for v in variants:
|
||||
await c.execute(
|
||||
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, size_bytes, width, "
|
||||
"height, sha256, s3_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) "
|
||||
"ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING",
|
||||
row["id"], job["pipeline_version"], v["name"], v["content_type"], len(v["data"]), v["width"], v["height"],
|
||||
v["sha256"], "att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"]))
|
||||
await c.execute("UPDATE attachments SET status = 'ready', error_code = NULL, ready_at = now(), updated_at = now(), "
|
||||
"lease_owner = NULL, lease_expires_at = NULL WHERE id = $1", row["id"])
|
||||
await c.execute("INSERT INTO job_events (attachment_id, job_id, event, attempt) VALUES ($1, $2, 'ready', $3)",
|
||||
row["id"], uuid.UUID(job["job_id"]), job["attempt"])
|
||||
return True
|
||||
|
||||
# --- обработка ----------------------------------------------------------------
|
||||
async def process(self, job):
|
||||
att_id = uuid.UUID(job["attachment_id"])
|
||||
row = await self.claim(job)
|
||||
if row is None:
|
||||
state = await self.db.fetchval("SELECT status FROM attachments WHERE id = $1", att_id)
|
||||
log("info", "job_noop", attachment_id=job["attachment_id"], status=state or "unknown")
|
||||
telemetry.record_job("attachment.process.v1", "noop", 0.0)
|
||||
return # ready/failed/processing у другого владельца/неизвестное вложение: идемпотентный no-op, ack
|
||||
started = time.perf_counter()
|
||||
outcome = "ready"
|
||||
try:
|
||||
await self.fault("after_claim")
|
||||
stream, _ = await self.s3.open_stream(row["staging_key"])
|
||||
spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024)
|
||||
try:
|
||||
try:
|
||||
while True:
|
||||
chunk = await stream.read(256 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
spool.write(chunk)
|
||||
finally:
|
||||
stream.close()
|
||||
loop = asyncio.get_running_loop()
|
||||
variants = await asyncio.wait_for(loop.run_in_executor(self.pool, pipeline.process, spool), config.JOB_TIMEOUT)
|
||||
finally:
|
||||
spool.close()
|
||||
# оригинал и варианты в конечный prefix: ключи детерминированы, повторная запись перезаписывает те же байты
|
||||
await self.copy_original(row)
|
||||
for v in variants:
|
||||
await self.s3.put_bytes("att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"]), v["data"], v["content_type"])
|
||||
await self.fault("after_s3_before_commit")
|
||||
if not await self.publish_result(row, job, variants):
|
||||
log("warning", "lease_lost", attachment_id=job["attachment_id"])
|
||||
return
|
||||
await self.fault("after_commit_before_ack")
|
||||
log("info", "job_ready", attachment_id=job["attachment_id"], attempt=job["attempt"],
|
||||
duration_ms=round((time.perf_counter() - started) * 1000, 1))
|
||||
try:
|
||||
await self.s3.delete(row["staging_key"]) # staging безопасно удалять после COMMIT результата
|
||||
except Exception as exc:
|
||||
log("warning", "staging_cleanup_failed", error_type=type(exc).__name__)
|
||||
except pipeline.PermanentError as exc:
|
||||
outcome = "failed"
|
||||
await self.fail(att_id, job, exc.code)
|
||||
log("warning", "job_failed", attachment_id=job["attachment_id"], error_code=exc.code)
|
||||
except MemoryError:
|
||||
outcome = "failed"
|
||||
await self.fail(att_id, job, "processing_failed")
|
||||
log("error", "job_failed_oom", attachment_id=job["attachment_id"])
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except InjectedCrash:
|
||||
raise
|
||||
except Exception as exc: # S3/БД/таймаут/неожиданное: временная ошибка
|
||||
outcome = "retried"
|
||||
await self.reschedule(att_id, job, type(exc).__name__)
|
||||
finally:
|
||||
ttr = None
|
||||
if outcome == "ready":
|
||||
ttr = await self.db.fetchval("SELECT extract(epoch FROM ready_at - accepted_at) FROM attachments WHERE id = $1 "
|
||||
"AND status = 'ready'", att_id)
|
||||
telemetry.record_job("attachment.process.v1", outcome, time.perf_counter() - started, float(ttr) if ttr is not None else None)
|
||||
|
||||
async def copy_original(self, row):
|
||||
stream, size = await self.s3.open_stream(row["staging_key"])
|
||||
spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024)
|
||||
try:
|
||||
while True:
|
||||
chunk = await stream.read(256 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
spool.write(chunk)
|
||||
stream.close()
|
||||
spool.seek(0)
|
||||
await self.s3.put(row["s3_key"], spool, row["content_type"], size)
|
||||
finally:
|
||||
spool.close()
|
||||
|
||||
async def fault(self, point):
|
||||
"""Точки внедрения сбоев для экспериментов (FAULT_INJECTION=crash:<точка> => процесс умирает мгновенно)."""
|
||||
if "crash_once:" + point in config.FAULT_INJECTION:
|
||||
marker = "/tmp/fault-%s.done" % point # сохраняется при рестарте того же контейнера -> сбой ровно один раз
|
||||
if not os.path.exists(marker):
|
||||
open(marker, "w").close()
|
||||
log("error", "fault_crash_once", point=point)
|
||||
os._exit(137)
|
||||
if "crash:" + point in config.FAULT_INJECTION:
|
||||
log("error", "fault_crash", point=point)
|
||||
os._exit(137)
|
||||
if "fail:" + point in config.FAULT_INJECTION:
|
||||
raise OSError("injected failure at " + point)
|
||||
|
||||
async def on_message(self, message):
|
||||
try:
|
||||
job = json.loads(message.body)
|
||||
if (job.get("schema_version") != 1 or job.get("type") != "attachment.process.v1"
|
||||
or not isinstance(job.get("attempt"), int) or not isinstance(job.get("pipeline_version"), int)):
|
||||
raise ValueError("bad job")
|
||||
uuid.UUID(job["attachment_id"]), uuid.UUID(job["job_id"])
|
||||
except Exception:
|
||||
log("error", "job_unreadable")
|
||||
await message.reject(requeue=False) # -> DLQ
|
||||
return
|
||||
self.busy += 1
|
||||
carrier = {k: job[k] for k in ("traceparent", "tracestate") if k in job}
|
||||
try:
|
||||
with telemetry.span("attachment.process process", "consumer",
|
||||
{"messaging.system": "rabbitmq", "messaging.destination.name": "attachment.process",
|
||||
"messaging.operation": "process", "job.type": job["type"], "job.attempt": job["attempt"]}, carrier):
|
||||
await self.process(job)
|
||||
await message.ack()
|
||||
except InjectedCrash:
|
||||
raise
|
||||
except Exception as exc:
|
||||
log("error", "job_unhandled", error_type=type(exc).__name__)
|
||||
await message.nack(requeue=True)
|
||||
finally:
|
||||
self.busy -= 1
|
||||
|
||||
# --- sweeper -------------------------------------------------------------------
|
||||
async def sweep_once(self):
|
||||
"""Истёкший lease -> queued (новая попытка) или failed/retry_exhausted; чистка старого staging."""
|
||||
rows = await self.db.fetch(
|
||||
"WITH exp AS (SELECT id FROM attachments WHERE status = 'processing' AND lease_expires_at < now() FOR UPDATE SKIP LOCKED), "
|
||||
"u AS (UPDATE attachments a SET status = CASE WHEN a.attempt >= $1 THEN 'failed' ELSE 'queued' END, "
|
||||
"error_code = CASE WHEN a.attempt >= $1 THEN 'retry_exhausted' ELSE NULL END, lease_owner = NULL, lease_expires_at = NULL, "
|
||||
"updated_at = now() FROM exp WHERE a.id = exp.id RETURNING a.id, a.status, a.pipeline_version, a.attempt, a.job_id), "
|
||||
"o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) "
|
||||
"SELECT COALESCE(job_id, gen_random_uuid()), id, pipeline_version, attempt + 1, "
|
||||
"(SELECT traceparent FROM outbox WHERE attachment_id = u.id ORDER BY id DESC LIMIT 1), "
|
||||
"(SELECT tracestate FROM outbox WHERE attachment_id = u.id ORDER BY id DESC LIMIT 1) FROM u WHERE status = 'queued' RETURNING 1), "
|
||||
"e AS (INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, job_id, 'lease_expired', attempt, status FROM u RETURNING 1) "
|
||||
"SELECT id, status FROM u", config.MAX_ATTEMPTS)
|
||||
for r in rows:
|
||||
log("warning", "lease_expired", attachment_id=str(r["id"]), new_status=r["status"])
|
||||
if rows:
|
||||
self.relay.kick()
|
||||
|
||||
async def sweeper(self):
|
||||
while not self.stopping:
|
||||
try:
|
||||
await self.sweep_once()
|
||||
except Exception as exc:
|
||||
log("warning", "sweeper_failed", error_type=type(exc).__name__)
|
||||
await asyncio.sleep(config.SWEEP_INTERVAL)
|
||||
|
||||
async def staging_janitor(self):
|
||||
"""Удаляет staging для ready (если не удалён после COMMIT) и для failed старше retention; текущую работу не трогает."""
|
||||
while not self.stopping:
|
||||
await asyncio.sleep(max(30, config.SWEEP_INTERVAL * 6))
|
||||
try:
|
||||
rows = await self.db.fetch(
|
||||
"SELECT id, staging_key FROM attachments WHERE staging_key IS NOT NULL AND (status = 'ready' OR "
|
||||
"(status = 'failed' AND updated_at < now() - make_interval(secs => $1))) LIMIT 200",
|
||||
float(config.STAGING_FAILED_RETENTION_SECONDS))
|
||||
for r in rows:
|
||||
await self.s3.delete(r["staging_key"])
|
||||
await self.db.execute("UPDATE attachments SET staging_key = NULL WHERE id = $1", r["id"])
|
||||
except Exception as exc:
|
||||
log("warning", "staging_janitor_failed", error_type=type(exc).__name__)
|
||||
|
||||
# --- жизненный цикл ------------------------------------------------------------
|
||||
async def connect_broker(self):
|
||||
while not self.stopping:
|
||||
try:
|
||||
await self.broker.ensure(prefetch=config.WORKER_CONCURRENCY)
|
||||
return
|
||||
except Exception as exc:
|
||||
log("warning", "broker_unavailable", error_type=type(exc).__name__)
|
||||
await asyncio.sleep(2)
|
||||
|
||||
async def consume(self):
|
||||
while not self.stopping:
|
||||
await self.connect_broker()
|
||||
try:
|
||||
queue = await self.broker.channel.declare_queue(QUEUE, durable=True, passive=True)
|
||||
tag = await queue.consume(self.on_message)
|
||||
log("info", "consuming", worker=WORKER_ID, prefetch=config.WORKER_CONCURRENCY)
|
||||
while not self.stopping and not self.broker.channel.is_closed:
|
||||
await asyncio.sleep(1)
|
||||
if self.stopping:
|
||||
await queue.cancel(tag)
|
||||
return
|
||||
except Exception as exc:
|
||||
log("warning", "consume_failed", error_type=type(exc).__name__)
|
||||
await asyncio.sleep(2)
|
||||
|
||||
async def run(self):
|
||||
telemetry.setup(config.SERVICE_NAME)
|
||||
telemetry.process_metrics()
|
||||
await self.db.start()
|
||||
await self.s3.start()
|
||||
log("info", "worker_started", worker=WORKER_ID, concurrency=config.WORKER_CONCURRENCY, lease=config.LEASE_SECONDS,
|
||||
max_attempts=config.MAX_ATTEMPTS)
|
||||
loop = asyncio.get_running_loop()
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
try:
|
||||
loop.add_signal_handler(sig, self.request_stop)
|
||||
except NotImplementedError:
|
||||
signal.signal(sig, lambda *_: self.request_stop())
|
||||
self.tasks = [asyncio.create_task(self.consume()), asyncio.create_task(self.relay.run()),
|
||||
asyncio.create_task(self.sweeper()), asyncio.create_task(self.staging_janitor())]
|
||||
beat = 0.0
|
||||
while not self.stopping:
|
||||
if time.monotonic() - beat > 5: # heartbeat для HEALTHCHECK контейнера
|
||||
beat = time.monotonic()
|
||||
try:
|
||||
with open("/tmp/worker.alive", "w") as fh:
|
||||
fh.write(str(time.time()))
|
||||
except OSError:
|
||||
pass
|
||||
await asyncio.sleep(0.5)
|
||||
# graceful: перестаём брать новые задания, ждём текущие до GRACEFUL_SHUTDOWN_SECONDS
|
||||
deadline = time.monotonic() + config.GRACEFUL_SHUTDOWN_SECONDS
|
||||
while self.busy and time.monotonic() < deadline:
|
||||
await asyncio.sleep(0.2)
|
||||
self.relay.stop()
|
||||
for t in self.tasks:
|
||||
t.cancel()
|
||||
await self.broker.stop()
|
||||
await self.s3.stop()
|
||||
await self.db.stop()
|
||||
log("info", "worker_stopped", unfinished=self.busy)
|
||||
telemetry.shutdown()
|
||||
|
||||
def request_stop(self):
|
||||
log("info", "worker_stopping")
|
||||
self.stopping = True
|
||||
|
||||
|
||||
class InjectedCrash(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
asyncio.run(Worker().run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user