forked from mtusi2026/mtusi2026lab6
Лабораторная работа 6
This commit is contained in:
+221
@@ -0,0 +1,221 @@
|
||||
"""Брокер заданий: Redis Streams с consumer group (lab5).
|
||||
|
||||
Почему это durable at-least-once очередь (подробно — REPORT.md):
|
||||
|
||||
* сервис `broker` — отдельный Redis с AOF и `appendfsync always`: ответ на XADD
|
||||
приходит только после fsync записи в журнал. Успешный XADD — наш аналог
|
||||
publisher confirm: сообщение переживёт рестарт/падение брокера с volume;
|
||||
* `maxmemory-policy noeviction`: поток никогда не вытесняется (при нехватке
|
||||
памяти XADD получит ошибку, строка outbox останется и будет опубликована позже);
|
||||
* consumer group: XREADGROUP выдаёт сообщение одному consumer и кладёт его в
|
||||
PEL (pending entries list) до XACK. Consumer ack — отдельная граница: worker
|
||||
делает XACK только ПОСЛЕ устойчивого эффекта (COMMIT результата в PostgreSQL);
|
||||
* упавший consumer: его сообщения остаются в PEL. Тот же consumer после
|
||||
рестарта контейнера (то же имя) читает свой PEL с id 0; иначе любой живой
|
||||
consumer забирает их XAUTOCLAIM после JOB_CLAIM_IDLE_SECONDS простоя.
|
||||
|
||||
В сообщении — только JSON по contracts/jobs.schema.json (ID и служебные поля):
|
||||
ни байтов файла, ни credentials, ни URL, ни filename.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import redis
|
||||
|
||||
log = logging.getLogger("messenger.broker")
|
||||
|
||||
FIELD = "job"
|
||||
|
||||
|
||||
class BrokerUnavailable(Exception):
|
||||
"""Брокер недоступен/таймаут/ошибка записи."""
|
||||
|
||||
|
||||
class Delivery:
|
||||
__slots__ = ("message_id", "raw", "deliveries")
|
||||
|
||||
def __init__(self, message_id: str, raw: Optional[str], deliveries: int = 1) -> None:
|
||||
self.message_id = message_id
|
||||
self.raw = raw
|
||||
self.deliveries = deliveries
|
||||
|
||||
def payload(self) -> Optional[dict]:
|
||||
if self.raw is None:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(self.raw)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
def _s(value) -> str:
|
||||
return value.decode("utf-8", "replace") if isinstance(value, (bytes, bytearray)) else str(value)
|
||||
|
||||
|
||||
class Broker:
|
||||
def __init__(self, url: str, stream: str, group: str, timeout: float = 2.0,
|
||||
block_ms: int = 2000) -> None:
|
||||
self.stream = stream
|
||||
self.group = group
|
||||
self.timeout = timeout
|
||||
self.block_ms = block_ms
|
||||
common = dict(socket_connect_timeout=timeout, health_check_interval=15,
|
||||
retry_on_timeout=False, decode_responses=False)
|
||||
# Обычные команды: короткий таймаут сокета -> быстрый BrokerUnavailable.
|
||||
self.client = redis.Redis.from_url(url, socket_timeout=timeout, **common)
|
||||
# Блокирующий XREADGROUP: таймаут сокета больше времени блокировки.
|
||||
self.blocking = redis.Redis.from_url(url, socket_timeout=timeout + block_ms / 1000.0, **common)
|
||||
self._group_ready = False
|
||||
|
||||
def close(self) -> None:
|
||||
for c in (self.client, self.blocking):
|
||||
try:
|
||||
c.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# -------------------------------------------------------------- producer
|
||||
|
||||
def ensure_group(self) -> None:
|
||||
"""Идемпотентно: поток + группа с позиции 0 (сообщения, опубликованные до
|
||||
создания группы, тоже будут доставлены)."""
|
||||
if self._group_ready:
|
||||
return
|
||||
try:
|
||||
self.client.xgroup_create(self.stream, self.group, id="0", mkstream=True)
|
||||
log.info("broker: created stream %s group %s", self.stream, self.group)
|
||||
except redis.ResponseError as exc:
|
||||
if "BUSYGROUP" not in str(exc):
|
||||
raise BrokerUnavailable() from exc
|
||||
except redis.RedisError as exc:
|
||||
raise BrokerUnavailable() from exc
|
||||
self._group_ready = True
|
||||
|
||||
def publish(self, payload: Dict) -> str:
|
||||
"""XADD; возврат = подтверждение приёма брокером (AOF fsync always)."""
|
||||
body = json.dumps(payload, separators=(",", ":"), sort_keys=True)
|
||||
try:
|
||||
self.ensure_group()
|
||||
return _s(self.client.xadd(self.stream, {FIELD: body}))
|
||||
except redis.RedisError as exc:
|
||||
raise BrokerUnavailable() from exc
|
||||
|
||||
# -------------------------------------------------------------- consumer
|
||||
|
||||
@staticmethod
|
||||
def _parse(entries) -> List[Delivery]:
|
||||
out = []
|
||||
for message_id, fields in entries or []:
|
||||
if fields is None: # запись удалена из потока, но осталась в PEL
|
||||
out.append(Delivery(_s(message_id), None))
|
||||
continue
|
||||
raw = fields.get(FIELD.encode()) if isinstance(fields, dict) else None
|
||||
out.append(Delivery(_s(message_id), _s(raw) if raw is not None else None))
|
||||
return out
|
||||
|
||||
def read(self, consumer: str, own_pending: bool = False) -> List[Delivery]:
|
||||
"""Одно сообщение (prefetch = 1 на consumer). own_pending: свой PEL с id 0 —
|
||||
сообщения, полученные прошлой инкарнацией этого consumer и не подтверждённые."""
|
||||
try:
|
||||
self.ensure_group()
|
||||
if own_pending:
|
||||
resp = self.client.xreadgroup(self.group, consumer, {self.stream: "0"}, count=1)
|
||||
else:
|
||||
resp = self.blocking.xreadgroup(self.group, consumer, {self.stream: ">"},
|
||||
count=1, block=self.block_ms)
|
||||
except redis.RedisError as exc:
|
||||
raise BrokerUnavailable() from exc
|
||||
result = []
|
||||
for _stream, entries in resp or []:
|
||||
result.extend(self._parse(entries))
|
||||
return result
|
||||
|
||||
def autoclaim(self, consumer: str, min_idle_ms: int) -> List[Delivery]:
|
||||
"""Забрать одно сообщение, которое другой consumer держит дольше min_idle_ms."""
|
||||
try:
|
||||
self.ensure_group()
|
||||
resp = self.client.xautoclaim(self.stream, self.group, consumer, min_idle_ms,
|
||||
start_id="0-0", count=1)
|
||||
except redis.RedisError as exc:
|
||||
raise BrokerUnavailable() from exc
|
||||
entries = resp[1] if resp and len(resp) > 1 else []
|
||||
deliveries = self._parse(entries)
|
||||
for d in deliveries:
|
||||
d.deliveries = self.delivery_count(d.message_id)
|
||||
return deliveries
|
||||
|
||||
def delivery_count(self, message_id: str) -> int:
|
||||
try:
|
||||
info = self.client.xpending_range(self.stream, self.group, message_id, message_id, 1)
|
||||
except redis.RedisError as exc:
|
||||
raise BrokerUnavailable() from exc
|
||||
return int(info[0]["times_delivered"]) if info else 1
|
||||
|
||||
def ack(self, message_id: str) -> None:
|
||||
try:
|
||||
self.client.xack(self.stream, self.group, message_id)
|
||||
except redis.RedisError as exc:
|
||||
raise BrokerUnavailable() from exc
|
||||
|
||||
# -------------------------------------------------------- housekeeping
|
||||
|
||||
def trim_acknowledged(self) -> int:
|
||||
"""XTRIM MINID: удалить только записи, которые уже доставлены И подтверждены
|
||||
(id меньше самого старого pending и не больше last-delivered-id группы).
|
||||
Недоставленные и неподтверждённые записи никогда не удаляются."""
|
||||
try:
|
||||
groups = self.client.xinfo_groups(self.stream)
|
||||
if not groups:
|
||||
return 0
|
||||
bounds = []
|
||||
for g in groups:
|
||||
last = _s(g.get("last-delivered-id") or g.get(b"last-delivered-id") or "0-0")
|
||||
bounds.append(_increment(last))
|
||||
pending = self.client.xpending(self.stream, _s(g["name"]))
|
||||
if pending and pending.get("pending"):
|
||||
bounds.append(_s(pending["min"]))
|
||||
minid = min(bounds, key=_id_key)
|
||||
if minid == "0-0" or minid == "0-1":
|
||||
return 0
|
||||
return int(self.client.xtrim(self.stream, minid=minid, approximate=False) or 0)
|
||||
except redis.ResponseError:
|
||||
return 0
|
||||
except redis.RedisError as exc:
|
||||
raise BrokerUnavailable() from exc
|
||||
|
||||
def stats(self) -> Dict:
|
||||
"""Длина потока, pending (в обработке/не подтверждено), lag (ещё не выдано)."""
|
||||
try:
|
||||
length = int(self.client.xlen(self.stream))
|
||||
out = {"stream": self.stream, "length": length, "groups": []}
|
||||
for g in self.client.xinfo_groups(self.stream):
|
||||
out["groups"].append({
|
||||
"name": _s(g["name"]),
|
||||
"consumers": int(g["consumers"]),
|
||||
"pending": int(g["pending"]),
|
||||
"lag": g.get("lag"),
|
||||
"last_delivered_id": _s(g["last-delivered-id"]),
|
||||
})
|
||||
return out
|
||||
except redis.ResponseError:
|
||||
return {"stream": self.stream, "length": 0, "groups": []}
|
||||
except redis.RedisError as exc:
|
||||
raise BrokerUnavailable() from exc
|
||||
|
||||
def ping(self) -> bool:
|
||||
try:
|
||||
return bool(self.client.ping())
|
||||
except redis.RedisError:
|
||||
return False
|
||||
|
||||
|
||||
def _id_key(message_id: str) -> Tuple[int, int]:
|
||||
ms, _, seq = message_id.partition("-")
|
||||
return int(ms), int(seq or 0)
|
||||
|
||||
|
||||
def _increment(message_id: str) -> str:
|
||||
ms, seq = _id_key(message_id)
|
||||
return "%d-%d" % (ms, seq + 1)
|
||||
+244
@@ -0,0 +1,244 @@
|
||||
"""Конфигурация из переменных окружения.
|
||||
|
||||
Секреты (строки подключения с паролями) читаются из файлов *_FILE,
|
||||
которые Compose монтирует как secrets в /run/secrets/. Для локального
|
||||
запуска без Docker допускаются и прямые переменные DATABASE_URL / CACHE_URL.
|
||||
"""
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def _read_secret(name: str) -> Optional[str]:
|
||||
path = os.environ.get(name + "_FILE")
|
||||
if path:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
value = handle.read().strip()
|
||||
return value or None
|
||||
value = os.environ.get(name, "").strip()
|
||||
return value or None
|
||||
|
||||
|
||||
def _int(name: str, default: int) -> int:
|
||||
raw = os.environ.get(name)
|
||||
if raw is None or raw.strip() == "":
|
||||
return default
|
||||
return int(raw)
|
||||
|
||||
|
||||
def _float(name: str, default: float) -> float:
|
||||
raw = os.environ.get(name)
|
||||
if raw is None or raw.strip() == "":
|
||||
return default
|
||||
return float(raw)
|
||||
|
||||
|
||||
def _bool(name: str, default: bool) -> bool:
|
||||
raw = os.environ.get(name)
|
||||
if raw is None or raw.strip() == "":
|
||||
return default
|
||||
return raw.strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
app_env: str
|
||||
app_version: str
|
||||
instance_id: str
|
||||
database_url: str
|
||||
cache_url: Optional[str]
|
||||
session_store: str
|
||||
session_ttl_seconds: int
|
||||
session_cache_ttl_seconds: int
|
||||
db_pool_min: int
|
||||
db_pool_max: int
|
||||
db_pool_timeout: float
|
||||
db_connect_timeout: int
|
||||
db_statement_timeout_ms: int
|
||||
cache_timeout_seconds: float
|
||||
cache_retry_after_seconds: float
|
||||
migrate_wait_seconds: int
|
||||
run_migrations_on_start: bool
|
||||
idempotency_ttl_seconds: int
|
||||
idempotency_cleanup_interval_seconds: int
|
||||
ready_db_timeout_seconds: float
|
||||
drain_seconds: float
|
||||
graceful_timeout_seconds: int
|
||||
keep_alive_seconds: int
|
||||
password_hash_concurrency: int
|
||||
# lab4: S3 и вложения
|
||||
s3_endpoint: Optional[str]
|
||||
s3_region: str
|
||||
s3_bucket: str
|
||||
s3_access_key_id: Optional[str]
|
||||
s3_secret_access_key: Optional[str]
|
||||
s3_connect_timeout_seconds: float
|
||||
s3_read_timeout_seconds: float
|
||||
s3_max_attempts: int
|
||||
s3_download_deadline_seconds: float
|
||||
s3_upload_deadline_seconds: float
|
||||
s3_circuit_open_seconds: float
|
||||
max_upload_bytes: int
|
||||
max_image_pixels: int
|
||||
max_concurrent_uploads: int
|
||||
upload_queue_timeout_seconds: float
|
||||
upload_read_timeout_seconds: float
|
||||
image_decode_concurrency: int
|
||||
attachment_gc_interval_seconds: int
|
||||
attachment_gc_grace_seconds: int
|
||||
orphan_sweep_interval_seconds: int
|
||||
fault_injection: str
|
||||
# lab5: брокер, outbox и worker
|
||||
broker_url: Optional[str]
|
||||
broker_stream: str
|
||||
broker_group: str
|
||||
broker_timeout_seconds: float
|
||||
pipeline_version: int
|
||||
worker_concurrency: int
|
||||
job_max_attempts: int
|
||||
job_timeout_seconds: int
|
||||
job_lease_grace_seconds: int
|
||||
job_claim_idle_seconds: int
|
||||
job_max_deliveries: int
|
||||
job_retry_base_seconds: float
|
||||
job_retry_max_seconds: float
|
||||
worker_image_timeout_seconds: float
|
||||
worker_image_memory_mb: int
|
||||
worker_faults: bool
|
||||
staging_retention_seconds: int
|
||||
relay_enabled: bool
|
||||
relay_poll_interval_seconds: float
|
||||
relay_fast_path_grace_seconds: float
|
||||
relay_batch: int
|
||||
|
||||
|
||||
# Жёсткие пределы контракта lab4/lab5; конфигурация может их только ужесточить.
|
||||
CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024
|
||||
CONTRACT_MAX_IMAGE_PIXELS = 40_000_000
|
||||
# Grace period сборщика должен быть заметно больше самой долгой загрузки
|
||||
# (чтение тела + S3 PUT с повтором), см. REPORT.md.
|
||||
MIN_GC_GRACE_SECONDS = 300
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
database_url = _read_secret("DATABASE_URL")
|
||||
if not database_url:
|
||||
raise RuntimeError("DATABASE_URL or DATABASE_URL_FILE must be set")
|
||||
|
||||
ttl = _int("SESSION_TTL_SECONDS", 3600)
|
||||
if ttl < 1:
|
||||
raise RuntimeError("SESSION_TTL_SECONDS must be positive")
|
||||
|
||||
session_store = os.environ.get("SESSION_STORE", "postgres").strip().lower()
|
||||
# Источник истины для сессий в этой реализации всегда PostgreSQL;
|
||||
# Redis (CACHE_URL) — только кэш проверок токена.
|
||||
if session_store != "postgres":
|
||||
raise RuntimeError("only SESSION_STORE=postgres is supported")
|
||||
|
||||
idem_ttl = _int("IDEMPOTENCY_TTL_SECONDS", 86400)
|
||||
if idem_ttl < 86400:
|
||||
# Контракт: повтор в течение 24 часов возвращает тот же результат.
|
||||
raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be >= 86400")
|
||||
|
||||
# X-Instance-Id: по умолчанию hostname контейнера (= короткий ID контейнера
|
||||
# Docker), стабилен в пределах жизни процесса. INSTANCE_ID — для запуска без Docker.
|
||||
from .runtime import INSTANCE_ID as instance_id
|
||||
|
||||
max_upload = _int("MAX_UPLOAD_BYTES", CONTRACT_MAX_UPLOAD_BYTES)
|
||||
max_pixels = _int("MAX_IMAGE_PIXELS", CONTRACT_MAX_IMAGE_PIXELS)
|
||||
if not 1 <= max_upload <= CONTRACT_MAX_UPLOAD_BYTES:
|
||||
raise RuntimeError("MAX_UPLOAD_BYTES must be within 1..10485760")
|
||||
if not 1 <= max_pixels <= CONTRACT_MAX_IMAGE_PIXELS:
|
||||
raise RuntimeError("MAX_IMAGE_PIXELS must be within 1..40000000")
|
||||
gc_grace = _int("ATTACHMENT_GC_GRACE_SECONDS", 900)
|
||||
# Меньше минимума — только для явных тестов сверки (ALLOW_SHORT_GC_GRACE=true).
|
||||
if gc_grace < MIN_GC_GRACE_SECONDS and not _bool("ALLOW_SHORT_GC_GRACE", False):
|
||||
raise RuntimeError("ATTACHMENT_GC_GRACE_SECONDS must be >= %d" % MIN_GC_GRACE_SECONDS)
|
||||
|
||||
job_timeout = _int("JOB_TIMEOUT_SECONDS", 60)
|
||||
image_timeout = _float("WORKER_IMAGE_TIMEOUT_SECONDS", 30.0)
|
||||
# Аренда обязана покрывать самую долгую попытку: обработка изображения +
|
||||
# чтение staging и запись 4 объектов (дедлайны S3) + COMMIT.
|
||||
if job_timeout < image_timeout + 20:
|
||||
raise RuntimeError("JOB_TIMEOUT_SECONDS must be >= WORKER_IMAGE_TIMEOUT_SECONDS + 20")
|
||||
max_attempts = _int("JOB_MAX_ATTEMPTS", 5)
|
||||
if not 1 <= max_attempts <= 20:
|
||||
raise RuntimeError("JOB_MAX_ATTEMPTS must be within 1..20")
|
||||
claim_idle = _int("JOB_CLAIM_IDLE_SECONDS", job_timeout + 15)
|
||||
if claim_idle <= job_timeout:
|
||||
raise RuntimeError("JOB_CLAIM_IDLE_SECONDS must be > JOB_TIMEOUT_SECONDS")
|
||||
|
||||
return Settings(
|
||||
app_env=os.environ.get("APP_ENV", "lab"),
|
||||
app_version=os.environ.get("APP_VERSION", "lab5"),
|
||||
instance_id=instance_id,
|
||||
database_url=database_url,
|
||||
cache_url=_read_secret("CACHE_URL"),
|
||||
session_store=session_store,
|
||||
session_ttl_seconds=ttl,
|
||||
session_cache_ttl_seconds=_int("SESSION_CACHE_TTL_SECONDS", 60),
|
||||
db_pool_min=_int("DB_POOL_MIN", 1),
|
||||
db_pool_max=_int("DB_POOL_MAX", 10),
|
||||
db_pool_timeout=_float("DB_POOL_TIMEOUT_SECONDS", 3.0),
|
||||
db_connect_timeout=_int("DB_CONNECT_TIMEOUT_SECONDS", 3),
|
||||
db_statement_timeout_ms=_int("DB_STATEMENT_TIMEOUT_MS", 5000),
|
||||
cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.25),
|
||||
cache_retry_after_seconds=_float("CACHE_RETRY_AFTER_SECONDS", 5.0),
|
||||
migrate_wait_seconds=_int("MIGRATE_WAIT_SECONDS", 60),
|
||||
# lab3: миграции выполняет отдельный one-shot сервис `migrate`,
|
||||
# реплики API их не запускают (нет гонки N реплик за схему).
|
||||
run_migrations_on_start=_bool("RUN_MIGRATIONS_ON_START", False),
|
||||
idempotency_ttl_seconds=idem_ttl,
|
||||
idempotency_cleanup_interval_seconds=_int("IDEMPOTENCY_CLEANUP_INTERVAL_SECONDS", 300),
|
||||
ready_db_timeout_seconds=_float("READY_DB_TIMEOUT_SECONDS", 1.0),
|
||||
drain_seconds=_float("DRAIN_SECONDS", 3.0),
|
||||
graceful_timeout_seconds=_int("GRACEFUL_TIMEOUT_SECONDS", 10),
|
||||
keep_alive_seconds=_int("KEEP_ALIVE_SECONDS", 30),
|
||||
password_hash_concurrency=_int("PASSWORD_HASH_CONCURRENCY", 4),
|
||||
s3_endpoint=os.environ.get("S3_ENDPOINT", "").strip() or None,
|
||||
s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1",
|
||||
s3_bucket=os.environ.get("S3_BUCKET", "messenger-attachments").strip(),
|
||||
s3_access_key_id=_read_secret("S3_ACCESS_KEY_ID"),
|
||||
s3_secret_access_key=_read_secret("S3_SECRET_ACCESS_KEY"),
|
||||
s3_connect_timeout_seconds=_float("S3_CONNECT_TIMEOUT_SECONDS", 2.0),
|
||||
s3_read_timeout_seconds=_float("S3_READ_TIMEOUT_SECONDS", 10.0),
|
||||
s3_max_attempts=_int("S3_MAX_ATTEMPTS", 2),
|
||||
# Полный бюджет вызова S3 из запроса (DNS + connect + повторы). Скачивание
|
||||
# обязано уложиться в timeout server HAProxy (10 с), загрузка — 10 с (PUT 10 MiB локально ~0.1 с).
|
||||
s3_download_deadline_seconds=_float("S3_DOWNLOAD_DEADLINE_SECONDS", 5.0),
|
||||
s3_upload_deadline_seconds=_float("S3_UPLOAD_DEADLINE_SECONDS", 10.0),
|
||||
s3_circuit_open_seconds=_float("S3_CIRCUIT_OPEN_SECONDS", 5.0),
|
||||
max_upload_bytes=max_upload,
|
||||
max_image_pixels=max_pixels,
|
||||
max_concurrent_uploads=max(1, _int("MAX_CONCURRENT_UPLOADS", 4)),
|
||||
upload_queue_timeout_seconds=_float("UPLOAD_QUEUE_TIMEOUT_SECONDS", 5.0),
|
||||
upload_read_timeout_seconds=_float("UPLOAD_READ_TIMEOUT_SECONDS", 60.0),
|
||||
image_decode_concurrency=max(1, _int("IMAGE_DECODE_CONCURRENCY", 2)),
|
||||
attachment_gc_interval_seconds=_int("ATTACHMENT_GC_INTERVAL_SECONDS", 300),
|
||||
attachment_gc_grace_seconds=gc_grace,
|
||||
orphan_sweep_interval_seconds=_int("ORPHAN_SWEEP_INTERVAL_SECONDS", 3600),
|
||||
# Только для тестового экземпляра api-faults (профиль faults), см. REPORT.md.
|
||||
fault_injection=os.environ.get("FAULT_INJECTION", "").strip().lower(),
|
||||
broker_url=_read_secret("BROKER_URL"),
|
||||
broker_stream=os.environ.get("BROKER_STREAM", "attachments.jobs").strip(),
|
||||
broker_group=os.environ.get("BROKER_GROUP", "attachment-workers").strip(),
|
||||
broker_timeout_seconds=_float("BROKER_TIMEOUT_SECONDS", 2.0),
|
||||
pipeline_version=max(1, _int("PIPELINE_VERSION", 1)),
|
||||
worker_concurrency=max(1, _int("WORKER_CONCURRENCY", 2)),
|
||||
job_max_attempts=max_attempts,
|
||||
job_timeout_seconds=job_timeout,
|
||||
job_lease_grace_seconds=_int("JOB_LEASE_GRACE_SECONDS", 10),
|
||||
job_claim_idle_seconds=claim_idle,
|
||||
job_max_deliveries=max(2, _int("JOB_MAX_DELIVERIES", 10)),
|
||||
job_retry_base_seconds=_float("JOB_RETRY_BASE_SECONDS", 2.0),
|
||||
job_retry_max_seconds=_float("JOB_RETRY_MAX_SECONDS", 60.0),
|
||||
worker_image_timeout_seconds=image_timeout,
|
||||
worker_image_memory_mb=_int("WORKER_IMAGE_MEMORY_MB", 1024),
|
||||
# Только для стенда отказов (compose.faults.yaml): маркеры в filename.
|
||||
worker_faults=_bool("WORKER_FAULTS", False),
|
||||
staging_retention_seconds=_int("STAGING_RETENTION_SECONDS", 86400),
|
||||
relay_enabled=_bool("RELAY_ENABLED", True),
|
||||
relay_poll_interval_seconds=_float("RELAY_POLL_INTERVAL_SECONDS", 1.0),
|
||||
relay_fast_path_grace_seconds=_float("RELAY_FAST_PATH_GRACE_SECONDS", 2.0),
|
||||
relay_batch=max(1, _int("RELAY_BATCH", 100)),
|
||||
)
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Пул соединений PostgreSQL и версионируемые миграции."""
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import psycopg
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
from .config import Settings
|
||||
|
||||
log = logging.getLogger("messenger.db")
|
||||
|
||||
MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations"
|
||||
|
||||
# Произвольная константа: один процесс за раз применяет миграции,
|
||||
# даже если одновременно стартуют несколько реплик API (lab3).
|
||||
MIGRATION_LOCK_ID = 2026_0002
|
||||
|
||||
|
||||
def conninfo_kwargs(settings: Settings) -> dict:
|
||||
return {
|
||||
"connect_timeout": settings.db_connect_timeout,
|
||||
# Все timestamptz приходят в UTC; любой запрос ограничен по времени.
|
||||
"options": "-c TimeZone=UTC -c statement_timeout=%d"
|
||||
% settings.db_statement_timeout_ms,
|
||||
"application_name": "messenger-api",
|
||||
}
|
||||
|
||||
|
||||
def create_pool(settings: Settings) -> ConnectionPool:
|
||||
pool = ConnectionPool(
|
||||
conninfo=settings.database_url,
|
||||
kwargs=conninfo_kwargs(settings),
|
||||
min_size=settings.db_pool_min,
|
||||
max_size=settings.db_pool_max,
|
||||
# Сколько запрос ждёт свободное соединение, прежде чем получить 503.
|
||||
timeout=settings.db_pool_timeout,
|
||||
# Битые после рестарта БД соединения отбрасываются при выдаче из пула.
|
||||
check=ConnectionPool.check_connection,
|
||||
max_idle=300,
|
||||
reconnect_timeout=3600,
|
||||
open=False,
|
||||
name="messenger",
|
||||
)
|
||||
# wait=False: процесс поднимается, даже если БД ещё/уже недоступна;
|
||||
# readiness это покажет, а запросы получат 503.
|
||||
pool.open(wait=False)
|
||||
return pool
|
||||
|
||||
|
||||
def _migration_files():
|
||||
files = sorted(MIGRATIONS_DIR.glob("*.sql"))
|
||||
if not files:
|
||||
raise RuntimeError("no migrations found in %s" % MIGRATIONS_DIR)
|
||||
return files
|
||||
|
||||
|
||||
def required_schema_version() -> str:
|
||||
"""Последняя миграция, которую знает этот образ.
|
||||
|
||||
Реплика готова (readiness) только если она уже применена: миграции
|
||||
запускает отдельный сервис `migrate`, а не сама реплика.
|
||||
"""
|
||||
return _migration_files()[-1].stem
|
||||
|
||||
|
||||
def schema_has_version(pool: ConnectionPool, version: str, timeout: float) -> bool:
|
||||
with pool.connection(timeout=timeout) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT to_regclass('schema_migrations') IS NOT NULL"
|
||||
).fetchone()
|
||||
if not row[0]:
|
||||
return False
|
||||
return conn.execute(
|
||||
"SELECT EXISTS (SELECT 1 FROM schema_migrations WHERE version = %s)",
|
||||
(version,),
|
||||
).fetchone()[0]
|
||||
|
||||
|
||||
def apply_migrations(settings: Settings) -> list:
|
||||
"""Применяет недостающие миграции; возвращает список применённых версий."""
|
||||
applied_now = []
|
||||
with psycopg.connect(
|
||||
settings.database_url, autocommit=True, **conninfo_kwargs(settings)
|
||||
) as conn:
|
||||
conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK_ID,))
|
||||
try:
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version text PRIMARY KEY,
|
||||
applied_at timestamptz NOT NULL DEFAULT now()
|
||||
)
|
||||
"""
|
||||
)
|
||||
done = {
|
||||
row[0]
|
||||
for row in conn.execute("SELECT version FROM schema_migrations")
|
||||
}
|
||||
for path in _migration_files():
|
||||
version = path.stem
|
||||
if version in done:
|
||||
continue
|
||||
sql = path.read_text(encoding="utf-8")
|
||||
# DDL и отметка о версии — одна транзакция: либо всё, либо ничего.
|
||||
with conn.transaction():
|
||||
conn.execute(sql)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations (version) VALUES (%s)",
|
||||
(version,),
|
||||
)
|
||||
applied_now.append(version)
|
||||
log.info("applied migration %s", version)
|
||||
finally:
|
||||
conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATION_LOCK_ID,))
|
||||
return applied_now
|
||||
|
||||
|
||||
def wait_and_migrate(settings: Settings) -> list:
|
||||
deadline = time.monotonic() + settings.migrate_wait_seconds
|
||||
last_error: Optional[Exception] = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
return apply_migrations(settings)
|
||||
except psycopg.OperationalError as exc:
|
||||
last_error = exc
|
||||
log.warning("database not ready for migrations, retrying")
|
||||
time.sleep(1)
|
||||
raise RuntimeError("database unavailable for migrations") from last_error
|
||||
+284
@@ -0,0 +1,284 @@
|
||||
"""Проверка загружаемых изображений по содержимому, с ограниченной памятью.
|
||||
|
||||
Порядок для каждого файла (лимит байт уже применён при чтении тела):
|
||||
1. Сигнатура: PNG (\\x89PNG\\r\\n\\x1a\\n) или JPEG (FF D8 FF). Иное -> 415.
|
||||
2. Сигнатура должна совпасть с заявленным Content-Type (подмена MIME -> 415).
|
||||
3. Размеры читаются из заголовка (PNG IHDR / JPEG SOFn) ДО распаковки пикселей;
|
||||
width*height > MAX_IMAGE_PIXELS -> 413. Распаковка не начинается.
|
||||
4. Целостность без полного декодирования в память:
|
||||
* PNG: обход всех чанков с проверкой CRC, порядка IHDR..IDAT..IEND и
|
||||
потоковая распаковка zlib с выводом кусками по 64 KiB «в никуда»:
|
||||
объём распакованных данных должен в точности равняться ожидаемому
|
||||
(height*(1+rowbytes), с учётом Adam7). Память — O(64 KiB) при любом
|
||||
размере картинки; «zip-бомба» обрывается, как только вывод превысил ожидание.
|
||||
* JPEG: обход маркеров (длины сегментов, SOF до SOS, обязательный EOI).
|
||||
Baseline дополнительно декодируется Pillow в режиме draft (масштаб 1/8,
|
||||
~5 MiB даже для 40 Мп). Progressive Pillow не декодируется: libjpeg держит
|
||||
буфер коэффициентов всего кадра (~120 MiB на 40 Мп, замер в REPORT.md),
|
||||
для него — только структурная проверка.
|
||||
5. Pillow открывает файл независимым парсером: формат и размеры должны совпасть.
|
||||
"""
|
||||
import io
|
||||
import struct
|
||||
import zlib
|
||||
from typing import Tuple
|
||||
|
||||
from PIL import Image, ImageFile
|
||||
|
||||
# Свою проверку пикселей делаем сами (шаг 3) — до вызова Pillow.
|
||||
Image.MAX_IMAGE_PIXELS = None
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = False
|
||||
|
||||
PNG_SIG = b"\x89PNG\r\n\x1a\n"
|
||||
JPEG_SIG = b"\xff\xd8\xff"
|
||||
INFLATE_CHUNK = 64 * 1024
|
||||
|
||||
PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
|
||||
PNG_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)}
|
||||
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
||||
# SOF0..SOF15 без DHT(C4), JPG(C8), DAC(CC)
|
||||
SOF_MARKERS = set(range(0xC0, 0xD0)) - {0xC4, 0xC8, 0xCC}
|
||||
PROGRESSIVE_SOF = {0xC2, 0xC6, 0xCA, 0xCE}
|
||||
|
||||
|
||||
class ImageRejected(Exception):
|
||||
def __init__(self, status: int, code: str, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
|
||||
|
||||
def unsupported(message: str = "Unsupported or corrupted image") -> ImageRejected:
|
||||
return ImageRejected(415, "unsupported_media_type", message)
|
||||
|
||||
|
||||
def too_many_pixels() -> ImageRejected:
|
||||
return ImageRejected(413, "payload_too_large", "Image has too many pixels")
|
||||
|
||||
|
||||
def sniff(data) -> str:
|
||||
head = bytes(data[:8])
|
||||
if head.startswith(PNG_SIG):
|
||||
return "image/png"
|
||||
if head.startswith(JPEG_SIG):
|
||||
return "image/jpeg"
|
||||
return ""
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- PNG
|
||||
|
||||
def _png_expected_raw(width: int, height: int, depth: int, color: int, interlace: int) -> int:
|
||||
bpp = depth * PNG_CHANNELS[color]
|
||||
|
||||
def rows(w: int, h: int) -> int:
|
||||
if w == 0 or h == 0:
|
||||
return 0
|
||||
return h * (1 + (w * bpp + 7) // 8)
|
||||
|
||||
if interlace == 0:
|
||||
return rows(width, height)
|
||||
total = 0
|
||||
for x0, y0, dx, dy in ADAM7:
|
||||
pw = (width - x0 + dx - 1) // dx if width > x0 else 0
|
||||
ph = (height - y0 + dy - 1) // dy if height > y0 else 0
|
||||
total += rows(pw, ph)
|
||||
return total
|
||||
|
||||
|
||||
def inspect_png(data, max_pixels: int) -> Tuple[int, int, bool]:
|
||||
view = memoryview(data)
|
||||
size = len(view)
|
||||
pos = 8
|
||||
width = height = 0
|
||||
expected = 0
|
||||
inflater = None
|
||||
produced = 0
|
||||
seen_ihdr = seen_plte = idat_done = False
|
||||
idat_started = False
|
||||
color = 0
|
||||
|
||||
while True:
|
||||
if pos + 12 > size:
|
||||
raise unsupported("Truncated PNG")
|
||||
length, ctype = struct.unpack(">I4s", view[pos:pos + 8])
|
||||
if length > 0x7FFFFFFF or pos + 12 + length > size:
|
||||
raise unsupported("Truncated PNG")
|
||||
if not all(65 <= c <= 90 or 97 <= c <= 122 for c in ctype):
|
||||
raise unsupported("Invalid PNG chunk")
|
||||
body = view[pos + 8:pos + 8 + length]
|
||||
(crc,) = struct.unpack(">I", view[pos + 8 + length:pos + 12 + length])
|
||||
if zlib.crc32(body, zlib.crc32(ctype)) & 0xFFFFFFFF != crc:
|
||||
raise unsupported("PNG CRC mismatch")
|
||||
pos += 12 + length
|
||||
|
||||
if not seen_ihdr:
|
||||
if ctype != b"IHDR" or length != 13:
|
||||
raise unsupported("PNG must start with IHDR")
|
||||
width, height, depth, color, comp, flt, interlace = struct.unpack(">IIBBBBB", body)
|
||||
if not (0 < width <= 0x7FFFFFFF and 0 < height <= 0x7FFFFFFF):
|
||||
raise unsupported("Invalid PNG dimensions")
|
||||
if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color] or comp or flt or interlace > 1:
|
||||
raise unsupported("Invalid PNG header")
|
||||
# Лимит пикселей — до любой распаковки.
|
||||
if width * height > max_pixels:
|
||||
raise too_many_pixels()
|
||||
expected = _png_expected_raw(width, height, depth, color, interlace)
|
||||
seen_ihdr = True
|
||||
continue
|
||||
|
||||
if ctype == b"IHDR":
|
||||
raise unsupported("Duplicate IHDR")
|
||||
if ctype == b"PLTE":
|
||||
if idat_started:
|
||||
raise unsupported("PLTE after IDAT")
|
||||
seen_plte = True
|
||||
elif ctype == b"IDAT":
|
||||
if idat_done:
|
||||
raise unsupported("Non-contiguous IDAT")
|
||||
if color == 3 and not seen_plte:
|
||||
raise unsupported("Missing PLTE")
|
||||
if not idat_started:
|
||||
idat_started = True
|
||||
inflater = zlib.decompressobj()
|
||||
produced += _inflate(inflater, body, expected - produced)
|
||||
elif ctype == b"IEND":
|
||||
if length != 0 or not idat_started:
|
||||
raise unsupported("Invalid PNG end")
|
||||
produced += _inflate(inflater, b"", expected - produced, final=True)
|
||||
if not inflater.eof or produced != expected:
|
||||
raise unsupported("Corrupted PNG image data")
|
||||
# Данные после IEND не используются декодерами; допускаем.
|
||||
return width, height, bool(interlace)
|
||||
else:
|
||||
if idat_started:
|
||||
idat_done = True
|
||||
if ctype[0] & 0x20 == 0:
|
||||
# Неизвестный КРИТИЧЕСКИЙ чанк (заглавная первая буква): декодер обязан
|
||||
# отказаться от такого файла. Вспомогательные (строчная буква) пропускаем.
|
||||
raise unsupported("Unknown critical PNG chunk")
|
||||
|
||||
|
||||
def _inflate(inflater, data, budget: int, final: bool = False) -> int:
|
||||
"""Распаковать data, не держа в памяти больше ~2 x INFLATE_CHUNK; вернуть объём вывода.
|
||||
|
||||
Вход подаётся срезами memoryview по INFLATE_CHUNK: unconsumed_tail — копия
|
||||
остатка входа, поэтому подача всего IDAT целиком давала бы копию размером с
|
||||
файл на каждом шаге (замер: пик 2x размера файла и квадратичное копирование).
|
||||
"""
|
||||
produced = 0
|
||||
view = memoryview(data)
|
||||
try:
|
||||
for start in range(0, len(view), INFLATE_CHUNK):
|
||||
buf = view[start:start + INFLATE_CHUNK]
|
||||
while True:
|
||||
out = inflater.decompress(buf, INFLATE_CHUNK)
|
||||
produced += len(out)
|
||||
if produced > budget:
|
||||
raise unsupported("Corrupted PNG image data")
|
||||
buf = inflater.unconsumed_tail
|
||||
if not buf:
|
||||
break
|
||||
if final:
|
||||
while True:
|
||||
out = inflater.flush(INFLATE_CHUNK) if not inflater.eof else b""
|
||||
produced += len(out)
|
||||
if produced > budget:
|
||||
raise unsupported("Corrupted PNG image data")
|
||||
if not out:
|
||||
break
|
||||
except zlib.error:
|
||||
raise unsupported("Corrupted PNG image data")
|
||||
return produced
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ JPEG
|
||||
|
||||
def inspect_jpeg(data, max_pixels: int) -> Tuple[int, int, bool]:
|
||||
raw = data if isinstance(data, (bytes, bytearray)) else bytes(data)
|
||||
size = len(raw)
|
||||
pos = 2
|
||||
width = height = 0
|
||||
progressive = False
|
||||
seen_sof = False
|
||||
while True:
|
||||
# Маркер: один или несколько 0xFF, затем код.
|
||||
if pos >= size or raw[pos] != 0xFF:
|
||||
raise unsupported("Corrupted JPEG structure")
|
||||
while pos < size and raw[pos] == 0xFF:
|
||||
pos += 1
|
||||
if pos >= size:
|
||||
raise unsupported("Truncated JPEG")
|
||||
marker = raw[pos]
|
||||
pos += 1
|
||||
if marker == 0xD9: # EOI
|
||||
if not seen_sof:
|
||||
raise unsupported("JPEG without frame")
|
||||
return width, height, progressive
|
||||
if marker == 0x01 or 0xD0 <= marker <= 0xD7:
|
||||
continue
|
||||
if marker == 0xD8 or marker == 0x00:
|
||||
raise unsupported("Corrupted JPEG structure")
|
||||
if pos + 2 > size:
|
||||
raise unsupported("Truncated JPEG")
|
||||
(length,) = struct.unpack(">H", raw[pos:pos + 2])
|
||||
if length < 2 or pos + length > size:
|
||||
raise unsupported("Truncated JPEG")
|
||||
segment = raw[pos + 2:pos + length]
|
||||
pos += length
|
||||
if marker in SOF_MARKERS:
|
||||
if seen_sof or len(segment) < 6:
|
||||
raise unsupported("Invalid JPEG frame")
|
||||
_precision, height, width, ncomp = struct.unpack(">BHHB", segment[:6])
|
||||
if width == 0 or height == 0 or ncomp not in (1, 3, 4) or len(segment) < 6 + 3 * ncomp:
|
||||
raise unsupported("Invalid JPEG frame")
|
||||
if width * height > max_pixels:
|
||||
raise too_many_pixels()
|
||||
progressive = marker in PROGRESSIVE_SOF
|
||||
seen_sof = True
|
||||
elif marker == 0xDA: # SOS: дальше энтропийные данные до следующего маркера
|
||||
if not seen_sof:
|
||||
raise unsupported("JPEG scan before frame")
|
||||
while True:
|
||||
nxt = raw.find(b"\xff", pos)
|
||||
if nxt < 0 or nxt + 1 >= size:
|
||||
raise unsupported("Truncated JPEG")
|
||||
code = raw[nxt + 1]
|
||||
if code == 0x00 or 0xD0 <= code <= 0xD7 or code == 0xFF:
|
||||
pos = nxt + 1 if code == 0xFF else nxt + 2
|
||||
continue
|
||||
pos = nxt
|
||||
break
|
||||
|
||||
|
||||
# -------------------------------------------------------------- common
|
||||
|
||||
def inspect(data, declared_type: str, max_pixels: int) -> Tuple[str, int, int]:
|
||||
"""Вернуть (content_type, width, height) или бросить ImageRejected."""
|
||||
detected = sniff(data)
|
||||
if not detected:
|
||||
raise unsupported("Only PNG and JPEG images are accepted")
|
||||
if detected != declared_type:
|
||||
raise unsupported("Content-Type does not match image content")
|
||||
if detected == "image/png":
|
||||
width, height, _ = inspect_png(data, max_pixels)
|
||||
pil_format = "PNG"
|
||||
decode = False # целостность IDAT уже проверена потоковой распаковкой
|
||||
else:
|
||||
width, height, progressive = inspect_jpeg(data, max_pixels)
|
||||
pil_format = "JPEG"
|
||||
decode = not progressive
|
||||
# Независимый парсер Pillow: только заголовок (open ленивый), затем при
|
||||
# необходимости декодирование уменьшенной копии.
|
||||
try:
|
||||
with Image.open(io.BytesIO(data)) as img:
|
||||
if img.format != pil_format or img.size != (width, height):
|
||||
raise unsupported("Inconsistent image header")
|
||||
if decode:
|
||||
img.draft(None, (max(1, width // 8), max(1, height // 8)))
|
||||
img.load()
|
||||
except ImageRejected:
|
||||
raise
|
||||
except (OSError, SyntaxError, ValueError, struct.error, Image.DecompressionBombError, EOFError):
|
||||
raise unsupported("Corrupted image")
|
||||
return detected, width, height
|
||||
+274
@@ -0,0 +1,274 @@
|
||||
"""Варианты изображения (lab5) и их изолированное вычисление.
|
||||
|
||||
Варианты pipeline_version=1 (параметры фиксированы, вывод детерминирован для
|
||||
одной версии Pillow/libjpeg/libwebp, поэтому повтор той же job пишет те же байты):
|
||||
|
||||
thumbnail JPEG, вписан в 256x256 с сохранением пропорций, без увеличения,
|
||||
размеры округляются вниз (минимум 1 px): 320x200 -> 256x160.
|
||||
quality=85, optimize, baseline, 4:2:0.
|
||||
crop JPEG 128x128: центрированный квадрат min(w,h) оригинала, масштаб
|
||||
до 128 (маленькие исходники увеличиваются). quality=85.
|
||||
optimized WebP, вписан в 1280x1280 без увеличения, quality=80, method=4.
|
||||
Уменьшение размера каждого маленького файла НЕ обещается.
|
||||
|
||||
Для всех вариантов: EXIF orientation применяется до вычисления размеров;
|
||||
прозрачность (RGBA/LA/P с tRNS) компонуется на белый фон; EXIF и XMP НЕ
|
||||
копируются (Pillow сохраняет их только при явной передаче exif=/xmp=), ICC-профиль
|
||||
сохраняется, чтобы не исказить цвет широкого охвата.
|
||||
|
||||
Ограничения ресурсов worker (недоверенный файл, даже прошедший проверки API):
|
||||
* bytes: staging-объект больше MAX_UPLOAD_BYTES не читается (worker.py);
|
||||
* pixels: размеры читаются из заголовка до декодирования; > MAX_IMAGE_PIXELS -> отказ;
|
||||
* память: обработка идёт в отдельном процессе с RLIMIT_AS = WORKER_IMAGE_MEMORY_MB;
|
||||
MemoryError/убийство процесса -> постоянная ошибка resource_limit;
|
||||
* время: процесс убивается через WORKER_IMAGE_TIMEOUT_SECONDS (временная ошибка,
|
||||
повтор ограничен JOB_MAX_ATTEMPTS); RLIMIT_CPU — страховка.
|
||||
Процессы создаёт forkserver: чистый однопоточный родитель, без унаследованных
|
||||
соединений PostgreSQL/Redis и потоков worker.
|
||||
"""
|
||||
import hashlib
|
||||
import io
|
||||
import multiprocessing
|
||||
import os
|
||||
import resource
|
||||
import signal
|
||||
from typing import Dict, Tuple
|
||||
|
||||
from PIL import Image, ImageFile, ImageOps
|
||||
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = False
|
||||
|
||||
THUMB_BOX = 256
|
||||
CROP_SIDE = 128
|
||||
OPTIMIZED_BOX = 1280
|
||||
JPEG_QUALITY = 85
|
||||
WEBP_QUALITY = 80
|
||||
WEBP_METHOD = 4
|
||||
|
||||
VARIANT_TYPES = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"}
|
||||
VARIANT_EXT = {"thumbnail": "jpg", "crop": "jpg", "optimized": "webp"}
|
||||
|
||||
# EXIF orientation 5..8 поворачивает изображение на 90°: ширина и высота меняются.
|
||||
SWAPPING_ORIENTATIONS = {5, 6, 7, 8}
|
||||
|
||||
|
||||
class ImageFailure(Exception):
|
||||
"""Постоянная ошибка: повтор того же файла даст тот же результат."""
|
||||
|
||||
def __init__(self, code: str, message: str = "") -> None:
|
||||
super().__init__(message or code)
|
||||
self.code = code
|
||||
|
||||
|
||||
class ImageTimeout(Exception):
|
||||
"""Обработка не уложилась в WORKER_IMAGE_TIMEOUT_SECONDS (временная ошибка)."""
|
||||
|
||||
|
||||
def fit_within(width: int, height: int, box: int) -> Tuple[int, int]:
|
||||
"""Вписать в box x box без увеличения; целочисленно, округление вниз, минимум 1."""
|
||||
if width <= box and height <= box:
|
||||
return width, height
|
||||
if width >= height:
|
||||
return box, max(1, height * box // width)
|
||||
return max(1, width * box // height), box
|
||||
|
||||
|
||||
def _orientation(img: Image.Image) -> int:
|
||||
try:
|
||||
return int(img.getexif().get(0x0112, 1) or 1)
|
||||
except Exception:
|
||||
return 1
|
||||
|
||||
|
||||
def _has_alpha(img: Image.Image) -> bool:
|
||||
return img.mode in ("RGBA", "LA", "PA") or "transparency" in img.info
|
||||
|
||||
|
||||
def _working_mode(img: Image.Image) -> Image.Image:
|
||||
"""RGB или RGBA без лишних копий: уже подходящий режим не конвертируется
|
||||
(convert в тот же режим всё равно копирует кадр — на 40 Мп это +160 MiB)."""
|
||||
if img.mode in ("RGB", "RGBA") and not (img.mode == "RGB" and "transparency" in img.info):
|
||||
return img
|
||||
return img.convert("RGBA" if _has_alpha(img) else "RGB")
|
||||
|
||||
|
||||
def _flatten(img: Image.Image) -> Image.Image:
|
||||
"""Уже УМЕНЬШЕННЫЙ вариант -> RGB: альфа компонуется на белый фон.
|
||||
Масштабирование RGBA в Pillow идёт с premultiplied alpha, ореолов нет."""
|
||||
if img.mode != "RGBA":
|
||||
return img
|
||||
background = Image.new("RGB", img.size, (255, 255, 255))
|
||||
background.paste(img, mask=img.getchannel("A"))
|
||||
return background
|
||||
|
||||
|
||||
def _encode(img: Image.Image, fmt: str, icc) -> bytes:
|
||||
out = io.BytesIO()
|
||||
extra = {"icc_profile": icc} if icc else {}
|
||||
if fmt == "JPEG":
|
||||
img.save(out, "JPEG", quality=JPEG_QUALITY, optimize=True, progressive=False,
|
||||
subsampling="4:2:0", **extra)
|
||||
else:
|
||||
img.save(out, "WEBP", quality=WEBP_QUALITY, method=WEBP_METHOD, lossless=False, **extra)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
def render(data: bytes, max_pixels: int) -> Dict[str, Tuple[bytes, int, int, str]]:
|
||||
"""{name: (bytes, width, height, content_type)} для thumbnail, crop, optimized."""
|
||||
Image.MAX_IMAGE_PIXELS = None # свой лимит ниже, до декодирования
|
||||
try:
|
||||
img = Image.open(io.BytesIO(data))
|
||||
if img.format not in ("PNG", "JPEG"):
|
||||
raise ImageFailure("invalid_image", "unsupported format")
|
||||
raw_w, raw_h = img.size
|
||||
if raw_w < 1 or raw_h < 1 or raw_w * raw_h > max_pixels:
|
||||
raise ImageFailure("invalid_image", "too many pixels")
|
||||
orientation = _orientation(img)
|
||||
icc = img.info.get("icc_profile")
|
||||
# Целевые размеры считаются по ОРИЕНТИРОВАННОМУ оригиналу.
|
||||
w, h = (raw_h, raw_w) if orientation in SWAPPING_ORIENTATIONS else (raw_w, raw_h)
|
||||
thumb_size = fit_within(w, h, THUMB_BOX)
|
||||
opt_size = fit_within(w, h, OPTIMIZED_BOX)
|
||||
if img.format == "JPEG":
|
||||
# Декодирование сразу в уменьшенном масштабе (1/2..1/8): память и время
|
||||
# пропорциональны нужному размеру. draft выбирает масштаб >= запрошенного.
|
||||
want = (opt_size[1], opt_size[0]) if orientation in SWAPPING_ORIENTATIONS else opt_size
|
||||
img.draft("RGB", want)
|
||||
img.load()
|
||||
img = ImageOps.exif_transpose(img) # копия только если orientation != 1
|
||||
base = _working_mode(img)
|
||||
del img
|
||||
bw, bh = base.size
|
||||
|
||||
result: Dict[str, Tuple[bytes, int, int, str]] = {}
|
||||
# Сначала масштабирование (reducing_gap: быстрый целочисленный reduce, затем
|
||||
# LANCZOS), потом компоновка альфы на белый — уже на маленьком кадре.
|
||||
thumb = _flatten(base.resize(thumb_size, Image.LANCZOS, reducing_gap=3.0)
|
||||
if base.size != thumb_size else base)
|
||||
result["thumbnail"] = (_encode(thumb, "JPEG", icc),) + thumb.size + ("image/jpeg",)
|
||||
del thumb
|
||||
|
||||
side = min(bw, bh)
|
||||
left, top = (bw - side) // 2, (bh - side) // 2
|
||||
crop = _flatten(base.resize((CROP_SIDE, CROP_SIDE), Image.LANCZOS, box=(left, top, left + side, top + side),
|
||||
reducing_gap=3.0 if side > CROP_SIDE else None))
|
||||
result["crop"] = (_encode(crop, "JPEG", icc),) + crop.size + ("image/jpeg",)
|
||||
del crop
|
||||
|
||||
opt = _flatten(base.resize(opt_size, Image.LANCZOS, reducing_gap=3.0) if base.size != opt_size else base)
|
||||
result["optimized"] = (_encode(opt, "WEBP", icc),) + opt.size + ("image/webp",)
|
||||
return result
|
||||
except ImageFailure:
|
||||
raise
|
||||
except MemoryError:
|
||||
raise ImageFailure("resource_limit", "memory limit exceeded")
|
||||
except (OSError, SyntaxError, ValueError, EOFError, Image.DecompressionBombError) as exc:
|
||||
raise ImageFailure("invalid_image", type(exc).__name__)
|
||||
|
||||
|
||||
def sha256_hex(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- sandbox
|
||||
|
||||
def _child(conn, memory_mb: int, cpu_seconds: int, max_pixels: int) -> None:
|
||||
"""Тело дочернего процесса: лимиты -> получить байты -> вычислить -> отправить."""
|
||||
try:
|
||||
limit = memory_mb * 1024 * 1024
|
||||
resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds, cpu_seconds + 1))
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
||||
try:
|
||||
data = conn.recv_bytes()
|
||||
result = render(data, max_pixels)
|
||||
del data
|
||||
peak_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024.0
|
||||
conn.send(("ok", (result, peak_mb)))
|
||||
except ImageFailure as exc:
|
||||
conn.send(("fail", exc.code))
|
||||
except MemoryError:
|
||||
try:
|
||||
conn.send(("fail", "resource_limit"))
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc: # неожиданное — тоже постоянная ошибка обработки
|
||||
try:
|
||||
conn.send(("fail", "processing_failed:" + type(exc).__name__))
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
class Sandbox:
|
||||
def __init__(self, memory_mb: int, timeout: float, max_pixels: int) -> None:
|
||||
self.memory_mb = memory_mb
|
||||
self.timeout = timeout
|
||||
self.max_pixels = max_pixels
|
||||
self.last_peak_mb = 0.0
|
||||
method = "forkserver" if "forkserver" in multiprocessing.get_all_start_methods() else "spawn"
|
||||
self.ctx = multiprocessing.get_context(method)
|
||||
if method == "forkserver":
|
||||
self.ctx.set_forkserver_preload(["src.imaging"])
|
||||
|
||||
def render(self, data: bytes) -> Dict[str, Tuple[bytes, int, int, str]]:
|
||||
parent, child = self.ctx.Pipe(duplex=True)
|
||||
proc = self.ctx.Process(
|
||||
target=_child,
|
||||
args=(child, self.memory_mb, int(self.timeout) + 5, self.max_pixels),
|
||||
daemon=True,
|
||||
)
|
||||
proc.start()
|
||||
child.close()
|
||||
try:
|
||||
parent.send_bytes(data)
|
||||
if not parent.poll(self.timeout):
|
||||
raise ImageTimeout()
|
||||
try:
|
||||
status, value = parent.recv()
|
||||
except (EOFError, OSError):
|
||||
# Процесс умер без ответа: OOM/RLIMIT (SIGKILL/SIGSEGV при нехватке памяти).
|
||||
proc.join(1)
|
||||
raise ImageFailure("resource_limit", "worker subprocess died (exit %s)" % proc.exitcode)
|
||||
if status == "ok":
|
||||
result, self.last_peak_mb = value
|
||||
return result
|
||||
code = str(value)
|
||||
if code.startswith("processing_failed"):
|
||||
raise ImageFailure("processing_failed", code)
|
||||
raise ImageFailure(code)
|
||||
finally:
|
||||
parent.close()
|
||||
if proc.is_alive():
|
||||
proc.kill()
|
||||
proc.join(5)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Самопроверка: python -m src.imaging FILE [--memory-mb N] [--timeout S]."""
|
||||
import argparse
|
||||
import json
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("file")
|
||||
parser.add_argument("--memory-mb", type=int, default=1024)
|
||||
parser.add_argument("--timeout", type=float, default=30)
|
||||
parser.add_argument("--max-pixels", type=int, default=40_000_000)
|
||||
args = parser.parse_args()
|
||||
with open(args.file, "rb") as f:
|
||||
data = f.read()
|
||||
try:
|
||||
box = Sandbox(args.memory_mb, args.timeout, args.max_pixels)
|
||||
out = box.render(data)
|
||||
print(json.dumps({**{k: {"bytes": len(v[0]), "width": v[1], "height": v[2], "type": v[3]}
|
||||
for k, v in out.items()}, "peak_rss_mb": round(box.last_peak_mb, 1)}))
|
||||
except ImageFailure as exc:
|
||||
print(json.dumps({"failed": exc.code}))
|
||||
except ImageTimeout:
|
||||
print(json.dumps({"failed": "timeout"}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+473
@@ -0,0 +1,473 @@
|
||||
"""Переходы состояний задания обработки вложения (lab5). Только PostgreSQL.
|
||||
|
||||
Таблица переходов (contracts/attachment-states.md) и где она реализована:
|
||||
|
||||
нет -> queued Storage.accept_attachment (API, одна транзакция с outbox)
|
||||
queued -> processing claim(): аренда = lease_token + lease_owner + lease_until
|
||||
processing -> ready complete(): fenced по lease_token; варианты + state одной транзакцией
|
||||
processing -> queued retry(): временная ошибка, attempt+1, backoff+jitter через outbox
|
||||
processing -> failed fail(): постоянная ошибка / попытки исчерпаны; запись в dead_letters
|
||||
processing -> queued reap_expired(): аренда истекла (worker умер/завис) — через outbox
|
||||
failed -> queued replay(): явная операторская команда, аудит в job_events
|
||||
|
||||
Fencing: любая запись результата worker'а выполняется условием
|
||||
`lease_token = <мой токен> AND status = 'processing'`. Worker, у которого аренду
|
||||
уже отобрали (истекла и задание перезапущено), не может ни опубликовать результат,
|
||||
ни перевести задание в retry/failed — его транзакция ничего не изменит.
|
||||
|
||||
Ключ идемпотентности эффекта — (attachment_id, pipeline_version, variant):
|
||||
PRIMARY KEY attachment_variants + детерминированные ключи объектов S3.
|
||||
"""
|
||||
import json
|
||||
import random
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from uuid import UUID
|
||||
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
from . import telemetry
|
||||
|
||||
JOB_TYPE = "attachment.process.v1"
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
|
||||
def iso_now() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
|
||||
|
||||
def job_payload(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int) -> Dict:
|
||||
"""Сообщение брокера по contracts/jobs.schema.json: только ID и служебные поля.
|
||||
|
||||
lab6: + W3C traceparent/tracestate ТЕКУЩЕГО span (HTTP upload в API, consumer
|
||||
span при retry в worker). Строка outbox хранит контекст в PostgreSQL, поэтому
|
||||
связь upload -> publish -> process сохраняется, даже если публикует sweeper
|
||||
другого процесса через минуту. Без активного span (reaper) полей нет.
|
||||
"""
|
||||
return telemetry.inject({
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"job_id": str(job_id),
|
||||
"type": JOB_TYPE,
|
||||
"attachment_id": str(attachment_id),
|
||||
"pipeline_version": int(pipeline_version),
|
||||
"attempt": int(attempt),
|
||||
"created_at": iso_now(),
|
||||
})
|
||||
|
||||
|
||||
def validate_payload(data: Optional[Dict]) -> Optional[Dict]:
|
||||
"""Строгая проверка входящего сообщения; None — сообщение отравлено (poison)."""
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
allowed = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version",
|
||||
"attempt", "created_at", "traceparent", "tracestate"}
|
||||
required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"}
|
||||
if not required <= set(data) or not set(data) <= allowed:
|
||||
return None
|
||||
try:
|
||||
if data["schema_version"] != SCHEMA_VERSION or data["type"] != JOB_TYPE:
|
||||
return None
|
||||
if type(data["attempt"]) is not int or data["attempt"] < 1:
|
||||
return None
|
||||
if type(data["pipeline_version"]) is not int or data["pipeline_version"] < 1:
|
||||
return None
|
||||
return {
|
||||
"job_id": UUID(str(data["job_id"])),
|
||||
"attachment_id": UUID(str(data["attachment_id"])),
|
||||
"pipeline_version": data["pipeline_version"],
|
||||
"attempt": data["attempt"],
|
||||
}
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
return None
|
||||
|
||||
|
||||
def backoff_seconds(attempt: int, base: float, cap: float, rng=random) -> float:
|
||||
"""Экспоненциальный backoff с «equal jitter»: d = min(cap, base*2^(n-1)),
|
||||
задержка равномерно в [d/2, d]. Нижняя граница d/2 гарантирует паузу,
|
||||
случайная часть разносит повторы многих заданий во времени."""
|
||||
d = min(cap, base * (2 ** max(0, attempt - 1)))
|
||||
return d / 2 + rng.uniform(0, d / 2)
|
||||
|
||||
|
||||
class Claim:
|
||||
__slots__ = ("job_id", "attachment_id", "attempt", "max_attempts", "replay_count",
|
||||
"pipeline_version", "lease_token", "takeover")
|
||||
|
||||
def __init__(self, row: Dict, takeover: bool) -> None:
|
||||
self.job_id = row["job_id"]
|
||||
self.attachment_id = row["attachment_id"]
|
||||
self.attempt = row["attempt"]
|
||||
self.max_attempts = row["max_attempts"]
|
||||
self.replay_count = row["replay_count"]
|
||||
self.pipeline_version = row["pipeline_version"]
|
||||
self.lease_token = row["lease_token"]
|
||||
self.takeover = takeover
|
||||
|
||||
|
||||
class JobStore:
|
||||
def __init__(self, pool: ConnectionPool, settings) -> None:
|
||||
self.pool = pool
|
||||
self.settings = settings
|
||||
|
||||
# ------------------------------------------------------------ helpers
|
||||
|
||||
@staticmethod
|
||||
def _event(conn, job_id, attachment_id, attempt, event, detail=None, actor=None) -> None:
|
||||
conn.execute(
|
||||
"INSERT INTO job_events (job_id, attachment_id, attempt, event, detail, actor) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s)",
|
||||
(job_id, attachment_id, attempt, event, detail, actor),
|
||||
)
|
||||
|
||||
def event(self, job_id, attachment_id, attempt, event, detail=None, actor=None) -> None:
|
||||
with self.pool.connection() as conn:
|
||||
self._event(conn, job_id, attachment_id, attempt, event, detail, actor)
|
||||
|
||||
def has_event(self, job_id, event: str) -> bool:
|
||||
with self.pool.connection() as conn:
|
||||
return conn.execute(
|
||||
"SELECT EXISTS (SELECT 1 FROM job_events WHERE job_id = %s AND event = %s)",
|
||||
(job_id, event),
|
||||
).fetchone()[0]
|
||||
|
||||
# ------------------------------------------------------------ claim
|
||||
|
||||
def claim(self, msg: Dict, owner: str) -> Tuple[Optional[Claim], str]:
|
||||
"""queued -> processing. Возвращает (Claim, "") или (None, причина no-op).
|
||||
|
||||
Захват разрешён, если сообщение относится к ТЕКУЩЕЙ попытке и задание
|
||||
queued, либо processing с истёкшей арендой, либо processing с арендой
|
||||
этого же consumer (его прошлая инкарнация умерла: consumer обрабатывает
|
||||
не больше одного сообщения за раз, поэтому живой аренды с его именем быть не может).
|
||||
"""
|
||||
token = uuid.uuid4()
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
cur = conn.cursor(row_factory=dict_row).execute(
|
||||
"""
|
||||
SELECT job_id, attachment_id, status, attempt, max_attempts, replay_count,
|
||||
pipeline_version, lease_owner,
|
||||
(lease_until IS NOT NULL AND lease_until < clock_timestamp()) AS lease_expired
|
||||
FROM attachment_jobs WHERE job_id = %s
|
||||
FOR UPDATE
|
||||
""",
|
||||
(msg["job_id"],),
|
||||
).fetchone()
|
||||
if cur is None or cur["attachment_id"] != msg["attachment_id"]:
|
||||
return None, "unknown_job"
|
||||
same_attempt = cur["attempt"] == msg["attempt"] and cur["pipeline_version"] == msg["pipeline_version"]
|
||||
takeover = cur["status"] == "processing" and (cur["lease_expired"] or cur["lease_owner"] == owner)
|
||||
if same_attempt and (cur["status"] == "queued" or takeover):
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'processing', lease_token = %s, lease_owner = %s,
|
||||
lease_until = clock_timestamp() + make_interval(secs => %s),
|
||||
started_at = clock_timestamp(), updated_at = clock_timestamp()
|
||||
WHERE job_id = %s
|
||||
""",
|
||||
(token, owner, self.settings.job_timeout_seconds, cur["job_id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET state = 'processing', updated_at = clock_timestamp() "
|
||||
"WHERE id = %s AND state IN ('queued', 'processing')",
|
||||
(cur["attachment_id"],),
|
||||
)
|
||||
self._event(conn, cur["job_id"], cur["attachment_id"], cur["attempt"],
|
||||
"lease_takeover" if takeover else "claimed",
|
||||
"previous owner %s" % cur["lease_owner"] if takeover else None, owner)
|
||||
cur["lease_token"] = token
|
||||
return Claim(cur, takeover), ""
|
||||
if cur["status"] == "succeeded":
|
||||
reason = "already_succeeded"
|
||||
elif cur["status"] == "failed":
|
||||
reason = "already_failed"
|
||||
elif cur["attempt"] > msg["attempt"]:
|
||||
reason = "stale_attempt"
|
||||
elif cur["status"] == "processing":
|
||||
reason = "leased_by_other"
|
||||
else:
|
||||
reason = "attempt_mismatch"
|
||||
# Повторная доставка — нормальна; фиксируем no-op в аудите.
|
||||
self._event(conn, cur["job_id"], cur["attachment_id"], msg["attempt"],
|
||||
"redelivery_noop", reason, owner)
|
||||
return None, reason
|
||||
|
||||
def attachment_source(self, attachment_id: UUID) -> Optional[Dict]:
|
||||
"""Доверенные метаданные для worker'а: только из БД, не из сообщения."""
|
||||
with self.pool.connection() as conn:
|
||||
return conn.cursor(row_factory=dict_row).execute(
|
||||
"SELECT id, content_type, size_bytes, sha256, width, height, object_key, "
|
||||
"staging_key, filename, state FROM attachments WHERE id = %s",
|
||||
(attachment_id,),
|
||||
).fetchone()
|
||||
|
||||
# ------------------------------------------------------------ outcomes
|
||||
|
||||
def complete(self, claim: Claim, variants: List[Dict], original_key: str, actor: str) -> Optional[float]:
|
||||
"""processing -> ready. None — аренда потеряна (fencing), ничего не изменено.
|
||||
|
||||
lab6: возвращает секунды от создания задания (COMMIT при приёме, ответ 202)
|
||||
до COMMIT ready — end-to-end time to ready."""
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'succeeded', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
finished_at = clock_timestamp(), updated_at = clock_timestamp(), last_error_code = NULL
|
||||
WHERE job_id = %s AND lease_token = %s AND status = 'processing'
|
||||
RETURNING attempt, EXTRACT(EPOCH FROM clock_timestamp() - created_at)
|
||||
""",
|
||||
(claim.job_id, claim.lease_token),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"""
|
||||
INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type,
|
||||
size_bytes, width, height, sha256, object_key)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING
|
||||
""",
|
||||
[(claim.attachment_id, claim.pipeline_version, v["name"], v["content_type"],
|
||||
v["size_bytes"], v["width"], v["height"], v["sha256"], v["object_key"])
|
||||
for v in variants],
|
||||
)
|
||||
# Никогда не откатываем более новую версию: pipeline_version <= текущей задачи.
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET state = 'ready', error_code = NULL, object_key = %s,
|
||||
pipeline_version = %s, updated_at = clock_timestamp()
|
||||
WHERE id = %s AND pipeline_version <= %s AND state <> 'pending'
|
||||
""",
|
||||
(original_key, claim.pipeline_version, claim.attachment_id, claim.pipeline_version),
|
||||
)
|
||||
self._event(conn, claim.job_id, claim.attachment_id, claim.attempt, "succeeded",
|
||||
",".join(v["name"] for v in variants), actor)
|
||||
return float(row[1])
|
||||
|
||||
def retry_or_fail(self, claim: Claim, code: str, actor: str) -> Tuple[str, float]:
|
||||
"""Временная ошибка: следующая попытка через outbox (backoff+jitter) или
|
||||
failed retry_exhausted. Возвращает ("retry", delay) | ("failed", 0) | ("lost", 0)."""
|
||||
if claim.attempt >= claim.max_attempts:
|
||||
return ("failed", 0.0) if self.fail(claim, "retry_exhausted", actor, detail=code) else ("lost", 0.0)
|
||||
delay = backoff_seconds(claim.attempt, self.settings.job_retry_base_seconds,
|
||||
self.settings.job_retry_max_seconds)
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = attempt + 1, lease_token = NULL, lease_owner = NULL,
|
||||
lease_until = NULL, last_error_code = %s, updated_at = clock_timestamp(),
|
||||
next_attempt_at = clock_timestamp() + make_interval(secs => %s)
|
||||
WHERE job_id = %s AND lease_token = %s AND status = 'processing'
|
||||
RETURNING attempt
|
||||
""",
|
||||
(code, delay, claim.job_id, claim.lease_token),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return "lost", 0.0
|
||||
self._requeue_side_effects(conn, claim.job_id, claim.attachment_id, claim.pipeline_version,
|
||||
row[0], delay)
|
||||
self._event(conn, claim.job_id, claim.attachment_id, claim.attempt, "retry_scheduled",
|
||||
"%s; next attempt %d in %.2fs" % (code, row[0], delay), actor)
|
||||
return "retry", delay
|
||||
|
||||
def _requeue_side_effects(self, conn, job_id, attachment_id, pipeline_version, attempt, delay) -> None:
|
||||
conn.execute(
|
||||
"UPDATE attachments SET state = 'queued', updated_at = clock_timestamp() "
|
||||
"WHERE id = %s AND state IN ('queued', 'processing', 'failed')",
|
||||
(attachment_id,),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO job_outbox (job_id, payload, available_at) "
|
||||
"VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s))",
|
||||
(job_id, Jsonb(job_payload(job_id, attachment_id, pipeline_version, attempt)), delay),
|
||||
)
|
||||
|
||||
def fail(self, claim: Claim, code: str, actor: str, detail: Optional[str] = None) -> bool:
|
||||
"""processing -> failed (fenced) + запись в реестр dead_letters."""
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
last_error_code = %s, finished_at = clock_timestamp(), updated_at = clock_timestamp()
|
||||
WHERE job_id = %s AND lease_token = %s AND status = 'processing'
|
||||
RETURNING attempt
|
||||
""",
|
||||
(code, claim.job_id, claim.lease_token),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return False
|
||||
self._mark_failed(conn, claim.job_id, claim.attachment_id, code, row[0])
|
||||
self._event(conn, claim.job_id, claim.attachment_id, claim.attempt, "failed",
|
||||
code + ("; " + detail if detail else ""), actor)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _mark_failed(conn, job_id, attachment_id, code, attempts) -> None:
|
||||
conn.execute(
|
||||
"UPDATE attachments SET state = 'failed', error_code = %s, updated_at = clock_timestamp() "
|
||||
"WHERE id = %s AND state IN ('queued', 'processing')",
|
||||
(code, attachment_id),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO dead_letters (job_id, attachment_id, error_code, attempts)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
ON CONFLICT (job_id) DO UPDATE
|
||||
SET error_code = EXCLUDED.error_code, attempts = EXCLUDED.attempts,
|
||||
failed_at = clock_timestamp()
|
||||
""",
|
||||
(job_id, attachment_id, code, attempts),
|
||||
)
|
||||
|
||||
def fail_poison(self, job_id: UUID, code: str, actor: str) -> bool:
|
||||
"""Сообщение доставлялось слишком много раз (worker падает на нём) —
|
||||
перевести задание в failed без аренды, если оно ещё не завершено."""
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
last_error_code = %s, finished_at = clock_timestamp(), updated_at = clock_timestamp()
|
||||
WHERE job_id = %s AND status IN ('queued', 'processing')
|
||||
RETURNING attachment_id, attempt
|
||||
""",
|
||||
(code, job_id),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return False
|
||||
self._mark_failed(conn, job_id, row[0], code, row[1])
|
||||
self._event(conn, job_id, row[0], row[1], "failed", code, actor)
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------ recovery
|
||||
|
||||
def reap_expired(self, actor: str, limit: int = 100) -> List[Tuple[str, str]]:
|
||||
"""processing с истёкшей арендой (+grace) -> queued attempt+1 через outbox,
|
||||
или failed processing_timeout, если попытки исчерпаны. SKIP LOCKED: несколько
|
||||
relay не мешают друг другу."""
|
||||
done = []
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
rows = conn.cursor(row_factory=dict_row).execute(
|
||||
"""
|
||||
SELECT job_id, attachment_id, attempt, max_attempts, pipeline_version, lease_owner
|
||||
FROM attachment_jobs
|
||||
WHERE status = 'processing'
|
||||
AND lease_until < clock_timestamp() - make_interval(secs => %s)
|
||||
ORDER BY lease_until LIMIT %s
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""",
|
||||
(self.settings.job_lease_grace_seconds, limit),
|
||||
).fetchall()
|
||||
for r in rows:
|
||||
if r["attempt"] >= r["max_attempts"]:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
last_error_code = 'processing_timeout', finished_at = clock_timestamp(),
|
||||
updated_at = clock_timestamp()
|
||||
WHERE job_id = %s
|
||||
""",
|
||||
(r["job_id"],),
|
||||
)
|
||||
self._mark_failed(conn, r["job_id"], r["attachment_id"], "processing_timeout", r["attempt"])
|
||||
self._event(conn, r["job_id"], r["attachment_id"], r["attempt"], "failed",
|
||||
"processing_timeout (lease expired, owner %s)" % r["lease_owner"], actor)
|
||||
done.append((str(r["job_id"]), "failed"))
|
||||
continue
|
||||
delay = backoff_seconds(r["attempt"], self.settings.job_retry_base_seconds,
|
||||
self.settings.job_retry_max_seconds)
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = attempt + 1, lease_token = NULL, lease_owner = NULL,
|
||||
lease_until = NULL, last_error_code = 'lease_expired', updated_at = clock_timestamp(),
|
||||
next_attempt_at = clock_timestamp() + make_interval(secs => %s)
|
||||
WHERE job_id = %s
|
||||
""",
|
||||
(delay, r["job_id"]),
|
||||
)
|
||||
self._requeue_side_effects(conn, r["job_id"], r["attachment_id"], r["pipeline_version"],
|
||||
r["attempt"] + 1, delay)
|
||||
self._event(conn, r["job_id"], r["attachment_id"], r["attempt"], "lease_expired",
|
||||
"owner %s; next attempt %d in %.2fs" % (r["lease_owner"], r["attempt"] + 1, delay),
|
||||
actor)
|
||||
done.append((str(r["job_id"]), "requeued"))
|
||||
return done
|
||||
|
||||
def replay(self, attachment_id: UUID, actor: str, pipeline_version: Optional[int] = None) -> Tuple[str, str]:
|
||||
"""failed -> queued: явный replay. Успешное задание НЕ исполняется повторно.
|
||||
|
||||
Новая попытка получает attempt+1 и свежий бюджет JOB_MAX_ATTEMPTS. Логический
|
||||
эффект защищён тем же ключом (attachment_id, pipeline_version, variant).
|
||||
"""
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
job = conn.cursor(row_factory=dict_row).execute(
|
||||
"""
|
||||
SELECT j.job_id, j.status, j.attempt, j.pipeline_version, a.staging_key, a.state
|
||||
FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.attachment_id = %s AND (%s::int IS NULL OR j.pipeline_version = %s)
|
||||
ORDER BY j.pipeline_version DESC LIMIT 1
|
||||
FOR UPDATE OF j, a
|
||||
""",
|
||||
(attachment_id, pipeline_version, pipeline_version),
|
||||
).fetchone()
|
||||
if job is None:
|
||||
return "refused", "no job for this attachment"
|
||||
if job["status"] == "succeeded" or job["state"] == "ready":
|
||||
self._event(conn, job["job_id"], attachment_id, job["attempt"], "replay_refused",
|
||||
"already succeeded (no duplicate execution)", actor)
|
||||
return "noop", "already succeeded; nothing to replay"
|
||||
if job["status"] != "failed":
|
||||
return "refused", "job is %s, only failed jobs can be replayed" % job["status"]
|
||||
if job["staging_key"] is None:
|
||||
self._event(conn, job["job_id"], attachment_id, job["attempt"], "replay_refused",
|
||||
"staging source already removed (retention)", actor)
|
||||
return "refused", "staging source was removed by retention; re-upload required"
|
||||
attempt = job["attempt"] + 1
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = %s, max_attempts = %s, replay_count = replay_count + 1,
|
||||
last_error_code = NULL, next_attempt_at = NULL, finished_at = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE job_id = %s AND status = 'failed'
|
||||
""",
|
||||
(attempt, attempt - 1 + self.settings.job_max_attempts, job["job_id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = clock_timestamp() "
|
||||
"WHERE id = %s AND state = 'failed'",
|
||||
(attachment_id,),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE dead_letters SET replayed_at = clock_timestamp(), replay_count = replay_count + 1 "
|
||||
"WHERE job_id = %s",
|
||||
(job["job_id"],),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO job_outbox (job_id, payload) VALUES (%s, %s)",
|
||||
(job["job_id"], Jsonb(job_payload(job["job_id"], attachment_id, job["pipeline_version"], attempt))),
|
||||
)
|
||||
self._event(conn, job["job_id"], attachment_id, attempt, "replayed",
|
||||
"operator replay; new attempt %d" % attempt, actor)
|
||||
return "replayed", "attempt %d queued" % attempt
|
||||
|
||||
|
||||
def dumps(obj) -> str:
|
||||
return json.dumps(obj, ensure_ascii=False, default=str)
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Операторские команды очереди (lab5). Секреты и токены не печатаются.
|
||||
|
||||
python -m src.jobs_admin stats # состояния заданий, outbox, брокер (длина/pending/lag)
|
||||
python -m src.jobs_admin failed [--limit 50] # реестр dead_letters (DLQ)
|
||||
python -m src.jobs_admin show ATTACHMENT_ID # задание + журнал событий
|
||||
python -m src.jobs_admin replay ATTACHMENT_ID [...] # явный replay failed-задания
|
||||
python -m src.jobs_admin replay --all-failed [--error-code retry_exhausted] [--dry-run]
|
||||
|
||||
Replay безопасен: успешное задание не исполняется повторно (no-op с записью в
|
||||
аудит), replay без staging-источника отклоняется, новая попытка получает attempt+1.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from uuid import UUID
|
||||
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from .broker import Broker, BrokerUnavailable
|
||||
from .config import load_settings
|
||||
from .db import create_pool
|
||||
from .jobs import JobStore
|
||||
|
||||
|
||||
def _print(obj) -> None:
|
||||
print(json.dumps(obj, ensure_ascii=False, indent=2, default=str))
|
||||
|
||||
|
||||
def cmd_stats(pool, settings, _args) -> int:
|
||||
with pool.connection() as conn:
|
||||
jobs = dict(conn.execute("SELECT status, count(*) FROM attachment_jobs GROUP BY status").fetchall())
|
||||
atts = dict(conn.execute("SELECT state, count(*) FROM attachments GROUP BY state").fetchall())
|
||||
outbox = conn.execute(
|
||||
"SELECT count(*), count(*) FILTER (WHERE available_at <= clock_timestamp()), "
|
||||
"COALESCE(EXTRACT(EPOCH FROM clock_timestamp() - min(created_at)), 0) FROM job_outbox"
|
||||
).fetchone()
|
||||
dlq = conn.execute("SELECT count(*) FROM dead_letters WHERE replayed_at IS NULL").fetchone()[0]
|
||||
oldest_queued = conn.execute(
|
||||
"SELECT COALESCE(EXTRACT(EPOCH FROM clock_timestamp() - min(created_at)), 0) "
|
||||
"FROM attachment_jobs WHERE status IN ('queued', 'processing')"
|
||||
).fetchone()[0]
|
||||
out = {
|
||||
"jobs": jobs, "attachments": atts,
|
||||
"outbox": {"rows": outbox[0], "due": outbox[1], "oldest_age_s": round(float(outbox[2]), 1)},
|
||||
"dead_letters_unreplayed": dlq,
|
||||
"oldest_unfinished_job_age_s": round(float(oldest_queued), 1),
|
||||
}
|
||||
if settings.broker_url:
|
||||
broker = Broker(settings.broker_url, settings.broker_stream, settings.broker_group,
|
||||
settings.broker_timeout_seconds)
|
||||
try:
|
||||
out["broker"] = broker.stats()
|
||||
except BrokerUnavailable:
|
||||
out["broker"] = "unavailable"
|
||||
broker.close()
|
||||
_print(out)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_failed(pool, _settings, args) -> int:
|
||||
with pool.connection() as conn:
|
||||
rows = conn.cursor(row_factory=dict_row).execute(
|
||||
"""
|
||||
SELECT d.attachment_id, d.job_id, d.error_code, d.attempts, d.failed_at, d.replayed_at,
|
||||
d.replay_count, j.status AS job_status, (a.staging_key IS NOT NULL) AS replayable
|
||||
FROM dead_letters d
|
||||
JOIN attachment_jobs j ON j.job_id = d.job_id
|
||||
JOIN attachments a ON a.id = d.attachment_id
|
||||
ORDER BY d.failed_at DESC LIMIT %s
|
||||
""",
|
||||
(args.limit,),
|
||||
).fetchall()
|
||||
_print(rows)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_show(pool, _settings, args) -> int:
|
||||
with pool.connection() as conn:
|
||||
att = conn.cursor(row_factory=dict_row).execute(
|
||||
"SELECT id, state, error_code, pipeline_version, size_bytes, sha256, "
|
||||
"(staging_key IS NOT NULL) AS has_staging, created_at, updated_at FROM attachments WHERE id = %s",
|
||||
(args.attachment_id,),
|
||||
).fetchone()
|
||||
job = conn.cursor(row_factory=dict_row).execute(
|
||||
"SELECT job_id, status, attempt, max_attempts, replay_count, lease_owner, lease_until, "
|
||||
"last_error_code, next_attempt_at, created_at, finished_at FROM attachment_jobs "
|
||||
"WHERE attachment_id = %s ORDER BY pipeline_version",
|
||||
(args.attachment_id,),
|
||||
).fetchall()
|
||||
variants = conn.cursor(row_factory=dict_row).execute(
|
||||
"SELECT pipeline_version, name, content_type, width, height, size_bytes, sha256 "
|
||||
"FROM attachment_variants WHERE attachment_id = %s ORDER BY pipeline_version, name",
|
||||
(args.attachment_id,),
|
||||
).fetchall()
|
||||
events = conn.cursor(row_factory=dict_row).execute(
|
||||
"SELECT at, attempt, event, detail, actor FROM job_events WHERE attachment_id = %s "
|
||||
"OR job_id IN (SELECT job_id FROM attachment_jobs WHERE attachment_id = %s) ORDER BY id",
|
||||
(args.attachment_id, args.attachment_id),
|
||||
).fetchall()
|
||||
_print({"attachment": att, "jobs": job, "variants": variants, "events": events})
|
||||
return 0 if att else 1
|
||||
|
||||
|
||||
def cmd_replay(pool, settings, args) -> int:
|
||||
jobs = JobStore(pool, settings)
|
||||
ids = [UUID(x) for x in args.attachment_ids]
|
||||
if args.all_failed:
|
||||
with pool.connection() as conn:
|
||||
q = "SELECT attachment_id FROM attachment_jobs WHERE status = 'failed'"
|
||||
params = []
|
||||
if args.error_code:
|
||||
q += " AND last_error_code = %s"
|
||||
params.append(args.error_code)
|
||||
ids += [r[0] for r in conn.execute(q + " ORDER BY updated_at", params).fetchall()]
|
||||
if not ids:
|
||||
print("nothing to replay")
|
||||
return 0
|
||||
results = []
|
||||
for att in ids:
|
||||
if args.dry_run:
|
||||
results.append({"attachment_id": att, "result": "dry-run"})
|
||||
continue
|
||||
outcome, detail = jobs.replay(att, actor="operator:jobs_admin")
|
||||
results.append({"attachment_id": att, "result": outcome, "detail": detail})
|
||||
_print(results)
|
||||
return 0 if all(r["result"] in ("replayed", "noop", "dry-run") for r in results) else 1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Attachment job queue operations")
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
sub.add_parser("stats")
|
||||
p = sub.add_parser("failed")
|
||||
p.add_argument("--limit", type=int, default=50)
|
||||
p = sub.add_parser("show")
|
||||
p.add_argument("attachment_id", type=UUID)
|
||||
p = sub.add_parser("replay")
|
||||
p.add_argument("attachment_ids", nargs="*")
|
||||
p.add_argument("--all-failed", action="store_true")
|
||||
p.add_argument("--error-code")
|
||||
p.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if args.cmd == "replay" and not (args.attachment_ids or args.all_failed):
|
||||
parser.error("replay: give ATTACHMENT_ID(s) or --all-failed")
|
||||
settings = load_settings()
|
||||
pool = create_pool(settings)
|
||||
pool.wait(timeout=15)
|
||||
try:
|
||||
return {"stats": cmd_stats, "failed": cmd_failed, "show": cmd_show, "replay": cmd_replay}[args.cmd](
|
||||
pool, settings, args)
|
||||
finally:
|
||||
pool.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+894
@@ -0,0 +1,894 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import unicodedata
|
||||
import uuid as uuid_mod
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from typing import Callable, Dict, Optional, Tuple
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import psycopg
|
||||
from opentelemetry import trace
|
||||
from fastapi import BackgroundTasks, Depends, FastAPI, Query, Request, Response
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from psycopg_pool import PoolTimeout
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
from starlette.requests import ClientDisconnect
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
from . import telemetry
|
||||
from .config import load_settings
|
||||
from .db import create_pool, required_schema_version, schema_has_version, wait_and_migrate
|
||||
from .images import ImageRejected, inspect as inspect_image
|
||||
from .models import (
|
||||
Attachment,
|
||||
Chat,
|
||||
ChatCreate,
|
||||
ChatPage,
|
||||
Message,
|
||||
MessageCreate,
|
||||
MessagePage,
|
||||
Session,
|
||||
User,
|
||||
UserCreate,
|
||||
)
|
||||
from .objectstore import ObjectMissing, ObjectStore, ObjectStoreUnavailable
|
||||
from .broker import Broker
|
||||
from .jobs import job_payload
|
||||
from .outbox import Relay
|
||||
from .reconcile import STAGING_PREFIX, gc_loop
|
||||
from .runtime import DRAIN, INSTANCE_ID
|
||||
from .security import (
|
||||
configure_kdf_concurrency,
|
||||
hash_password,
|
||||
parse_basic,
|
||||
parse_bearer,
|
||||
verify_password,
|
||||
)
|
||||
from .sessions import SessionCache, SessionService
|
||||
from .storage import (
|
||||
AttachmentFailed,
|
||||
AttachmentNotFound,
|
||||
IdempotencyConflict,
|
||||
IdempotencyRace,
|
||||
NotChatMember,
|
||||
Storage,
|
||||
UnknownMember,
|
||||
UsernameTaken,
|
||||
)
|
||||
|
||||
# lab6: OTLP traces/metrics/logs + JSON-логи. До create_pool (инструментация psycopg).
|
||||
telemetry.setup("messenger-api")
|
||||
log = logging.getLogger("messenger.api")
|
||||
access_log = logging.getLogger("messenger.http")
|
||||
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
# Контракт lab3: Idempotency-Key — ASCII [A-Za-z0-9._:-]{1,128}.
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
|
||||
def _idempotency_cleanup_loop(storage: Storage, interval: float, stop: threading.Event) -> None:
|
||||
"""Фоновая очистка истёкших ключей. Advisory lock — одна реплика за раз."""
|
||||
while not stop.wait(interval):
|
||||
try:
|
||||
deleted = storage.delete_expired_idempotency_keys()
|
||||
if deleted:
|
||||
log.info("deleted %d expired idempotency keys", deleted)
|
||||
except Exception as exc: # БД недоступна — попробуем в следующий раз
|
||||
log.warning("idempotency cleanup skipped: %s", type(exc).__name__)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
settings = load_settings()
|
||||
if settings.run_migrations_on_start:
|
||||
# Только для локальной разработки. В compose миграции выполняет
|
||||
# отдельный one-shot сервис `migrate` до старта реплик.
|
||||
applied = wait_and_migrate(settings)
|
||||
log.info("migrations applied at startup: %s", applied or "none")
|
||||
configure_kdf_concurrency(settings.password_hash_concurrency)
|
||||
pool = create_pool(settings)
|
||||
storage = Storage(pool)
|
||||
cache = SessionCache(settings)
|
||||
app.state.settings = settings
|
||||
app.state.storage = storage
|
||||
app.state.cache = cache
|
||||
app.state.sessions = SessionService(storage, cache, settings)
|
||||
app.state.required_schema = required_schema_version()
|
||||
# lab6: DB pool и очередь (из PostgreSQL) — метрики живут, даже когда worker остановлен.
|
||||
telemetry.register_pool_metrics(pool)
|
||||
telemetry.register_queue_metrics(pool, settings.broker_stream)
|
||||
app.state.schema_ok = False
|
||||
stop_cleanup = threading.Event()
|
||||
cleanup = threading.Thread(
|
||||
target=_idempotency_cleanup_loop,
|
||||
args=(storage, settings.idempotency_cleanup_interval_seconds, stop_cleanup),
|
||||
name="idempotency-cleanup",
|
||||
daemon=True,
|
||||
)
|
||||
cleanup.start()
|
||||
# lab4: S3 и ограничители ресурсов загрузок (на реплику).
|
||||
app.state.objects = ObjectStore(settings) if ObjectStore.configured(settings) else None
|
||||
app.state.upload_slots = asyncio.Semaphore(settings.max_concurrent_uploads)
|
||||
app.state.decode_slots = threading.BoundedSemaphore(settings.image_decode_concurrency)
|
||||
if app.state.objects is not None:
|
||||
threading.Thread(
|
||||
target=gc_loop,
|
||||
args=(pool, app.state.objects, settings, stop_cleanup),
|
||||
name="attachment-gc",
|
||||
daemon=True,
|
||||
).start()
|
||||
else:
|
||||
log.warning("S3 is not configured: attachment endpoints return 503")
|
||||
# lab5: outbox relay в каждой реплике (быстрый путь после 202 + sweeper).
|
||||
# Брокер НЕ входит в синхронный путь: при его недоступности upload всё равно
|
||||
# отвечает 202, задание ждёт в job_outbox (PostgreSQL).
|
||||
app.state.relay = None
|
||||
if settings.broker_url and settings.relay_enabled:
|
||||
broker = Broker(settings.broker_url, settings.broker_stream, settings.broker_group,
|
||||
settings.broker_timeout_seconds)
|
||||
app.state.relay = Relay(pool, broker, settings, "api:" + INSTANCE_ID)
|
||||
threading.Thread(target=app.state.relay.loop, args=(stop_cleanup,), name="outbox-relay",
|
||||
daemon=True).start()
|
||||
elif not settings.broker_url:
|
||||
log.warning("BROKER_URL is not configured: jobs stay in job_outbox until a relay publishes them")
|
||||
log.info(
|
||||
"attachments: s3=%s bucket=%s max_bytes=%d max_pixels=%d uploads<=%d decode<=%d gc_grace=%ss fault=%s",
|
||||
"on" if app.state.objects is not None else "off",
|
||||
settings.s3_bucket,
|
||||
settings.max_upload_bytes,
|
||||
settings.max_image_pixels,
|
||||
settings.max_concurrent_uploads,
|
||||
settings.image_decode_concurrency,
|
||||
settings.attachment_gc_grace_seconds,
|
||||
settings.fault_injection or "none",
|
||||
)
|
||||
log.info(
|
||||
"started: instance=%s version=%s session_ttl=%ss cache=%s pool=%s..%s schema>=%s",
|
||||
INSTANCE_ID,
|
||||
settings.app_version,
|
||||
settings.session_ttl_seconds,
|
||||
"redis" if cache.client is not None else "disabled",
|
||||
settings.db_pool_min,
|
||||
settings.db_pool_max,
|
||||
app.state.required_schema,
|
||||
)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
log.info("shutdown: instance=%s closing pool", INSTANCE_ID)
|
||||
stop_cleanup.set()
|
||||
pool.close()
|
||||
if cache.client is not None:
|
||||
cache.client.close()
|
||||
telemetry.shutdown()
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="MTUSI Messenger (lab6: observability)",
|
||||
version="6.0.0",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
|
||||
# Request ID
|
||||
|
||||
class RequestIdMiddleware(BaseHTTPMiddleware):
|
||||
async def dispatch(self, request: Request, call_next: Callable):
|
||||
incoming = request.headers.get("X-Request-Id", "")
|
||||
request_id = incoming if REQUEST_ID_RE.match(incoming) else str(uuid4())
|
||||
request.state.request_id = request_id
|
||||
# lab6: X-Request-Id в server span — связь «ответ с ошибкой -> trace -> лог».
|
||||
span = trace.get_current_span()
|
||||
if span.is_recording():
|
||||
span.set_attribute("messenger.request_id", request_id)
|
||||
started = time.perf_counter()
|
||||
response = await call_next(request)
|
||||
path = request.url.path
|
||||
if not path.startswith("/health/"):
|
||||
route = request.scope.get("route")
|
||||
# Только шаблон маршрута, без реальных UUID и query string.
|
||||
access_log.info(
|
||||
"%s %s -> %d", request.method, getattr(route, "path", "unmatched"), response.status_code,
|
||||
extra={"event": "http.request", "http_method": request.method,
|
||||
"http_route": getattr(route, "path", "unmatched"),
|
||||
"http_status_code": response.status_code, "request_id": request_id,
|
||||
"duration_ms": round((time.perf_counter() - started) * 1000, 1)},
|
||||
)
|
||||
response.headers["X-Request-Id"] = request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
if DRAIN.active:
|
||||
# Во время осушения не держим keep-alive: следующий запрос клиент/LB
|
||||
# откроет новым соединением — уже к другой реплике.
|
||||
response.headers["Connection"] = "close"
|
||||
return response
|
||||
|
||||
|
||||
app.add_middleware(RequestIdMiddleware)
|
||||
# Добавляется последним -> самый внешний middleware: RequestIdMiddleware и
|
||||
# обработчики работают внутри server span (логи получают trace_id/span_id).
|
||||
telemetry.instrument_fastapi(app)
|
||||
|
||||
|
||||
# errors
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status_code: int, code: str, message: str, headers: Optional[Dict[str, str]] = None):
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def unauthorized(challenge: str = BEARER_CHALLENGE) -> ApiError:
|
||||
return ApiError(401, "unauthorized", "Authentication required", {"WWW-Authenticate": challenge})
|
||||
|
||||
|
||||
def bad_request(message: str = "Invalid request") -> ApiError:
|
||||
return ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def not_found() -> ApiError:
|
||||
return ApiError(404, "not_found", "Not found")
|
||||
|
||||
|
||||
def error_response(
|
||||
request: Request,
|
||||
status_code: int,
|
||||
code: str,
|
||||
message: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> JSONResponse:
|
||||
request_id = getattr(request.state, "request_id", None) or str(uuid4())
|
||||
all_headers = {"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID}
|
||||
all_headers.update(headers or {})
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content={"error": {"code": code, "message": message, "request_id": request_id}},
|
||||
headers=all_headers,
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def api_error_handler(request: Request, exc: ApiError) -> JSONResponse:
|
||||
return error_response(request, exc.status_code, exc.code, exc.message, exc.headers)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_exception_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
|
||||
return error_response(request, 400, "invalid_request", "Invalid request")
|
||||
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def http_exception_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse:
|
||||
status_to_code = {
|
||||
400: "invalid_request",
|
||||
401: "unauthorized",
|
||||
404: "not_found",
|
||||
409: "conflict",
|
||||
413: "payload_too_large",
|
||||
415: "unsupported_media_type",
|
||||
429: "rate_limited",
|
||||
503: "unavailable",
|
||||
}
|
||||
status_code = exc.status_code if exc.status_code in status_to_code else 500
|
||||
code = status_to_code.get(status_code, "internal_error")
|
||||
message = str(exc.detail) if status_code != 500 else "Internal server error"
|
||||
return error_response(request, status_code, code, message)
|
||||
|
||||
|
||||
async def dependency_unavailable(request: Request, exc: Exception) -> JSONResponse:
|
||||
# Наружу — только общий 503, без SQL, адресов и текста драйвера.
|
||||
log.warning("database unavailable: %s", type(exc).__name__)
|
||||
return error_response(request, 503, "unavailable", "Service temporarily unavailable")
|
||||
|
||||
|
||||
app.add_exception_handler(psycopg.OperationalError, dependency_unavailable)
|
||||
app.add_exception_handler(psycopg.InterfaceError, dependency_unavailable)
|
||||
app.add_exception_handler(PoolTimeout, dependency_unavailable)
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unexpected_exception_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||
log.exception("unexpected error")
|
||||
return error_response(request, 500, "internal_error", "Internal server error")
|
||||
|
||||
|
||||
# helpers
|
||||
|
||||
def require_json_content_type(request: Request) -> None:
|
||||
content_type = request.headers.get("content-type", "")
|
||||
if not content_type.lower().startswith("application/json"):
|
||||
raise ApiError(415, "unsupported_media_type", "Unsupported media type")
|
||||
|
||||
|
||||
def ensure_not_blank(value: str) -> None:
|
||||
if not value.strip():
|
||||
raise bad_request()
|
||||
|
||||
|
||||
def storage_of(request: Request) -> Storage:
|
||||
return request.app.state.storage
|
||||
|
||||
|
||||
def current_user(request: Request) -> User:
|
||||
"""Только Authorization: Bearer. Basic и X-User-Id здесь не дают доступа."""
|
||||
token = parse_bearer(request.headers.get("Authorization"))
|
||||
if token is None:
|
||||
raise unauthorized()
|
||||
user = request.app.state.sessions.resolve(token)
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
request.state.token = token
|
||||
return user
|
||||
|
||||
|
||||
def encode_cursor(resource_id: UUID, created_at: datetime, item_id: UUID) -> str:
|
||||
payload = {
|
||||
"resource_id": str(resource_id),
|
||||
"created_at": created_at.isoformat(),
|
||||
"item_id": str(item_id),
|
||||
}
|
||||
raw = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(cursor: Optional[str], expected_resource_id: UUID) -> Optional[Tuple[datetime, UUID]]:
|
||||
if cursor is None:
|
||||
return None
|
||||
try:
|
||||
padding = "=" * (-len(cursor) % 4)
|
||||
raw = base64.urlsafe_b64decode(cursor + padding)
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
resource_id = UUID(payload["resource_id"])
|
||||
created_at = datetime.fromisoformat(payload["created_at"])
|
||||
item_id = UUID(payload["item_id"])
|
||||
if created_at.tzinfo is None or created_at.utcoffset() is None:
|
||||
raise ValueError("invalid_cursor")
|
||||
except (ValueError, KeyError, TypeError, AttributeError, json.JSONDecodeError):
|
||||
raise bad_request("Invalid cursor")
|
||||
if resource_id != expected_resource_id:
|
||||
raise bad_request("Invalid cursor")
|
||||
return created_at, item_id
|
||||
|
||||
|
||||
# health
|
||||
|
||||
@app.get("/health/live", response_model=Dict[str, str])
|
||||
def liveness() -> Dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready", response_model=Dict[str, str])
|
||||
def readiness(request: Request) -> Dict[str, str]:
|
||||
"""Готовность к обязательному синхронному пути.
|
||||
|
||||
* SIGTERM получен (фаза осушения) -> 503: LB выводит реплику из ротации;
|
||||
* PostgreSQL недоступна за READY_DB_TIMEOUT_SECONDS -> 503;
|
||||
* схема ещё не мигрирована до версии, которую знает образ -> 503
|
||||
(миграции выполняет отдельный сервис `migrate`).
|
||||
Redis не критичен: это кэш с обходом (см. REPORT.md).
|
||||
"""
|
||||
if DRAIN.active:
|
||||
raise ApiError(503, "unavailable", "Instance is shutting down")
|
||||
state = request.app.state
|
||||
timeout = state.settings.ready_db_timeout_seconds
|
||||
try:
|
||||
if state.schema_ok:
|
||||
with state.storage.pool.connection(timeout=timeout) as conn:
|
||||
conn.execute("SELECT 1")
|
||||
elif schema_has_version(state.storage.pool, state.required_schema, timeout):
|
||||
state.schema_ok = True
|
||||
else:
|
||||
raise ApiError(503, "unavailable", "Database schema is not migrated")
|
||||
except (psycopg.Error, PoolTimeout):
|
||||
raise ApiError(503, "unavailable", "Database is not available")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
# users
|
||||
|
||||
@app.post("/api/v1/users", response_model=User, status_code=201)
|
||||
def register_user(request: Request, data: UserCreate) -> User:
|
||||
require_json_content_type(request)
|
||||
ensure_not_blank(data.display_name)
|
||||
ensure_not_blank(data.password)
|
||||
password_hash = hash_password(data.password)
|
||||
try:
|
||||
return storage_of(request).create_user(data.username, data.display_name, password_hash)
|
||||
except UsernameTaken:
|
||||
raise ApiError(409, "conflict", "Username already exists")
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me", response_model=User)
|
||||
def get_current_user(user: User = Depends(current_user)) -> User:
|
||||
return user
|
||||
|
||||
|
||||
# sessions
|
||||
|
||||
async def require_empty_body(request: Request) -> None:
|
||||
"""Вход — только заголовок Authorization: Basic, без тела (контракт).
|
||||
|
||||
Любые байты в теле (JSON или нет) — 400. Проверка выполняется до
|
||||
дорогой проверки пароля и до создания сессии.
|
||||
"""
|
||||
if await request.body():
|
||||
raise bad_request("Request body is not allowed")
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201,
|
||||
dependencies=[Depends(require_empty_body)])
|
||||
def create_session(request: Request) -> Session:
|
||||
credentials = parse_basic(request.headers.get("Authorization"))
|
||||
if credentials is None:
|
||||
raise unauthorized(BASIC_CHALLENGE)
|
||||
username, password = credentials
|
||||
found = storage_of(request).get_credentials(username)
|
||||
user, password_hash = found if found else (None, None)
|
||||
# verify_password выполняется и для несуществующего username (с хешем-пустышкой).
|
||||
if not verify_password(password_hash, password) or user is None:
|
||||
raise unauthorized(BASIC_CHALLENGE)
|
||||
token, expires_at = request.app.state.sessions.create(user)
|
||||
return Session(token=token, token_type="Bearer", expires_at=expires_at, user=user)
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def delete_current_session(request: Request, user: User = Depends(current_user)) -> Response:
|
||||
if not request.app.state.sessions.revoke(request.state.token):
|
||||
raise unauthorized()
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
# chats
|
||||
|
||||
@app.post("/api/v1/chats", response_model=Chat, status_code=201)
|
||||
def create_chat(request: Request, data: ChatCreate, user: User = Depends(current_user)) -> Chat:
|
||||
require_json_content_type(request)
|
||||
ensure_not_blank(data.title)
|
||||
if len(data.member_ids) != len(set(data.member_ids)):
|
||||
raise bad_request("Duplicate member ids")
|
||||
if user.id not in data.member_ids:
|
||||
raise bad_request("Caller must be a chat member")
|
||||
try:
|
||||
return storage_of(request).create_chat(data.title, data.member_ids)
|
||||
except UnknownMember:
|
||||
raise bad_request("Unknown member")
|
||||
|
||||
|
||||
@app.get("/api/v1/chats", response_model=ChatPage)
|
||||
def list_chats(
|
||||
request: Request,
|
||||
user: User = Depends(current_user),
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
cursor: Optional[str] = Query(default=None, min_length=1),
|
||||
) -> ChatPage:
|
||||
after = decode_cursor(cursor, user.id)
|
||||
items, has_more = storage_of(request).list_user_chats(user.id, limit, after)
|
||||
next_cursor = None
|
||||
if has_more and items:
|
||||
next_cursor = encode_cursor(user.id, items[-1].created_at, items[-1].id)
|
||||
return ChatPage(items=items, next_cursor=next_cursor)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}", response_model=Chat)
|
||||
def get_chat(request: Request, chat_id: UUID, user: User = Depends(current_user)) -> Chat:
|
||||
chat = storage_of(request).get_chat_for_member(chat_id, user.id)
|
||||
if chat is None:
|
||||
raise not_found()
|
||||
return chat
|
||||
|
||||
|
||||
# messages
|
||||
|
||||
def request_fingerprint(data: MessageCreate) -> bytes:
|
||||
"""SHA-256 канонического JSON тела.
|
||||
|
||||
Канонизация: проверенная модель -> JSON с сортировкой ключей и без пробелов.
|
||||
Поэтому порядок ключей и форматирование исходного JSON не влияют на hash.
|
||||
lab4: attachment_ids имеет значение по умолчанию [], поэтому отсутствующее
|
||||
поле и явный [] дают одинаковый hash (контракт). Порядок id значим.
|
||||
"""
|
||||
canonical = json.dumps(
|
||||
data.model_dump(mode="json"), sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).digest()
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", response_model=Message, status_code=201)
|
||||
def create_message(
|
||||
request: Request,
|
||||
response: Response,
|
||||
chat_id: UUID,
|
||||
data: MessageCreate,
|
||||
user: User = Depends(current_user),
|
||||
) -> Message:
|
||||
# Порядок: аутентификация (dependency) -> ключ -> тело -> доступ к чату.
|
||||
key = request.headers.get("Idempotency-Key")
|
||||
if key is None or not IDEMPOTENCY_KEY_RE.fullmatch(key):
|
||||
raise bad_request("Missing or invalid Idempotency-Key")
|
||||
require_json_content_type(request)
|
||||
# lab4: пустой/пробельный текст допустим только вместе с вложениями.
|
||||
if not data.attachment_ids:
|
||||
ensure_not_blank(data.text)
|
||||
if len(set(data.attachment_ids)) != len(data.attachment_ids):
|
||||
raise bad_request("Duplicate attachment ids")
|
||||
settings = request.app.state.settings
|
||||
try:
|
||||
message, replayed = storage_of(request).create_message(
|
||||
chat_id,
|
||||
user.id,
|
||||
data.text,
|
||||
key,
|
||||
request_fingerprint(data),
|
||||
settings.idempotency_ttl_seconds,
|
||||
data.attachment_ids,
|
||||
)
|
||||
except (NotChatMember, AttachmentNotFound):
|
||||
# Чужое, неизвестное или ещё не принятое (pending) вложение — как и чужой чат: 404.
|
||||
raise not_found()
|
||||
except AttachmentFailed:
|
||||
raise ApiError(409, "conflict", "Attachment processing failed; it cannot be attached")
|
||||
except IdempotencyConflict:
|
||||
raise ApiError(409, "conflict", "Idempotency-Key was used with a different request")
|
||||
except IdempotencyRace:
|
||||
raise ApiError(503, "unavailable", "Please retry with the same Idempotency-Key")
|
||||
if replayed:
|
||||
# Информационный заголовок (не часть контракта): ответ — повтор.
|
||||
response.headers["Idempotency-Replayed"] = "true"
|
||||
return message
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage)
|
||||
def list_messages(
|
||||
request: Request,
|
||||
chat_id: UUID,
|
||||
user: User = Depends(current_user),
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
cursor: Optional[str] = Query(default=None, min_length=1),
|
||||
) -> MessagePage:
|
||||
after = decode_cursor(cursor, chat_id)
|
||||
result = storage_of(request).list_messages(chat_id, user.id, limit, after)
|
||||
if result is None:
|
||||
raise not_found()
|
||||
items, has_more = result
|
||||
next_cursor = None
|
||||
if has_more and items:
|
||||
next_cursor = encode_cursor(chat_id, items[-1].created_at, items[-1].id)
|
||||
return MessagePage(items=items, next_cursor=next_cursor)
|
||||
|
||||
|
||||
# attachments (lab4)
|
||||
|
||||
ALLOWED_UPLOAD_TYPES = ("image/png", "image/jpeg")
|
||||
UPLOAD_CHUNK = 64 * 1024
|
||||
KNOWN_VARIANTS = ("thumbnail", "crop", "optimized")
|
||||
|
||||
|
||||
async def s3_call(store: ObjectStore, deadline: float, fn, *args):
|
||||
"""Вызов S3 из запроса с жёстким общим дедлайном.
|
||||
|
||||
connect/read timeout boto3 не покрывают DNS: в Docker остановленный
|
||||
контейнер s3 пропадает из DNS, и getaddrinfo висит секундами (замер на
|
||||
Docker Desktop: 10–13 с, у скачивания это давало 504 от HAProxy). Дедлайн
|
||||
ограничивает всё вместе; по истечении — 503 и открытый circuit breaker.
|
||||
Поток с зависшим вызовом доработает в фоне; для PUT это может дать
|
||||
объект-сироту, его уберёт сборщик (src/reconcile.py, проход B).
|
||||
"""
|
||||
try:
|
||||
return await asyncio.wait_for(run_in_threadpool(fn, *args), timeout=deadline)
|
||||
except asyncio.TimeoutError:
|
||||
store.breaker.failure()
|
||||
log.warning("s3 call exceeded %.1fs deadline", deadline)
|
||||
raise ObjectStoreUnavailable()
|
||||
|
||||
|
||||
def objects_of(request: Request) -> ObjectStore:
|
||||
store = request.app.state.objects
|
||||
if store is None:
|
||||
raise ApiError(503, "unavailable", "Attachment storage is not available")
|
||||
return store
|
||||
|
||||
|
||||
def s3_unavailable() -> ApiError:
|
||||
return ApiError(503, "unavailable", "Attachment storage is temporarily unavailable",
|
||||
{"Retry-After": "2"})
|
||||
|
||||
|
||||
def validate_filename(name: Optional[str]) -> str:
|
||||
"""Отображаемое имя: 1–255 code points, без /, \\, NUL и управляющих символов.
|
||||
|
||||
Имя никогда не становится путём или ключом S3; ограничения защищают
|
||||
клиентов, которые покажут/сохранят файл под этим именем.
|
||||
"""
|
||||
if name is None or not name.strip() or len(name) > 255:
|
||||
raise bad_request("Invalid filename")
|
||||
if name in (".", ".."):
|
||||
raise bad_request("Invalid filename")
|
||||
for ch in name:
|
||||
if ch in "/\\\x00" or unicodedata.category(ch) in ("Cc", "Cs") \
|
||||
or ch in "
":
|
||||
raise bad_request("Invalid filename")
|
||||
return name
|
||||
|
||||
|
||||
def upload_media_type(request: Request) -> str:
|
||||
raw = request.headers.get("content-type", "")
|
||||
media = raw.split(";", 1)[0].strip().lower()
|
||||
if media not in ALLOWED_UPLOAD_TYPES:
|
||||
raise ApiError(415, "unsupported_media_type", "Only image/png and image/jpeg are accepted")
|
||||
return media
|
||||
|
||||
|
||||
def declared_length(request: Request) -> Optional[int]:
|
||||
raw = request.headers.get("content-length")
|
||||
if raw is None:
|
||||
return None
|
||||
if not raw.strip().isdigit():
|
||||
raise bad_request("Invalid Content-Length")
|
||||
return int(raw)
|
||||
|
||||
|
||||
async def drain(request: Request, limit: int) -> None:
|
||||
"""Дочитать и выбросить тело (не больше limit байт), чтобы клиент увидел 413,
|
||||
а не обрыв соединения посреди отправки. Память — O(chunk)."""
|
||||
seen = 0
|
||||
try:
|
||||
async for chunk in request.stream():
|
||||
seen += len(chunk)
|
||||
if seen > limit:
|
||||
break
|
||||
except ClientDisconnect:
|
||||
pass
|
||||
|
||||
|
||||
async def read_limited_body(request: Request, limit: int):
|
||||
"""Потоковое чтение тела кусками; не больше limit байт в памяти.
|
||||
|
||||
Возвращает (bytes, md5, sha256). Превышение лимита -> 413 сразу, как только
|
||||
счётчик перешёл limit (остаток тела отбрасывается, не накапливается).
|
||||
"""
|
||||
chunks = []
|
||||
size = 0
|
||||
md5 = hashlib.md5(usedforsecurity=False)
|
||||
sha = hashlib.sha256()
|
||||
async for chunk in request.stream():
|
||||
if not chunk:
|
||||
continue
|
||||
size += len(chunk)
|
||||
if size > limit:
|
||||
chunks.clear()
|
||||
await drain(request, limit)
|
||||
raise ApiError(413, "payload_too_large", "File exceeds %d bytes" % limit)
|
||||
chunks.append(chunk)
|
||||
md5.update(chunk)
|
||||
sha.update(chunk)
|
||||
# Одна склейка в bytes точного размера; список кусков сразу освобождается.
|
||||
# Дальше bytes не копируются: BytesIO(bytes) и memoryview разделяют буфер.
|
||||
data = b"".join(chunks)
|
||||
chunks.clear()
|
||||
return data, md5.digest(), sha.hexdigest()
|
||||
|
||||
|
||||
def _validate_image(request: Request, data: bytes, declared: str):
|
||||
settings = request.app.state.settings
|
||||
with request.app.state.decode_slots: # потолок одновременных разборов на реплику
|
||||
return inspect_image(data, declared, settings.max_image_pixels)
|
||||
|
||||
|
||||
def _inject(settings, point: str) -> None:
|
||||
"""Внедрение отказов — ТОЛЬКО для экземпляра api-faults (профиль compose faults)."""
|
||||
fault = settings.fault_injection
|
||||
if not fault:
|
||||
return
|
||||
if fault == "crash_after_s3_put" and point == "after_put":
|
||||
log.error("FAULT_INJECTION: process exit between S3 PUT and DB COMMIT")
|
||||
os._exit(70)
|
||||
if fault == "db_error_after_s3_put" and point == "after_put":
|
||||
raise psycopg.OperationalError("injected: database failure after S3 PUT")
|
||||
if fault == "s3_error_on_put" and point == "before_put":
|
||||
raise ObjectStoreUnavailable()
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", response_model=Attachment, status_code=202)
|
||||
async def upload_attachment(
|
||||
request: Request,
|
||||
background: BackgroundTasks,
|
||||
chat_id: UUID,
|
||||
filename: Optional[str] = Query(default=None),
|
||||
user: User = Depends(current_user),
|
||||
) -> Attachment:
|
||||
"""lab5: приём оригинала -> 202 queued; обработку выполняет worker.
|
||||
|
||||
Порядок проверок (как в lab4): аутентификация -> filename (400) ->
|
||||
Content-Type (415) -> Content-Length (413 без чтения тела) -> членство (404) ->
|
||||
слот загрузки (503) -> потоковое чтение с лимитом (413) -> содержимое (415/413).
|
||||
Затем durable handoff:
|
||||
[1] INSERT attachments(state=pending, staging_key) COMMIT
|
||||
[2] S3 PUT staging/<random> (Content-MD5, SHA-256 в метаданных)
|
||||
[3] UPDATE pending->queued + INSERT attachment_jobs + job_outbox COMMIT -> 202
|
||||
В момент 202 устойчиво сохранены: все байты оригинала (S3 staging), метаданные и
|
||||
намерение обработать (задание + outbox в PostgreSQL). Публикация в брокер — после
|
||||
ответа (background task) и/или sweeper'ом; остановка API после 202 работу не теряет.
|
||||
"""
|
||||
settings = request.app.state.settings
|
||||
storage = storage_of(request)
|
||||
name = validate_filename(filename)
|
||||
declared = upload_media_type(request)
|
||||
length = declared_length(request)
|
||||
if length is not None and length > settings.max_upload_bytes:
|
||||
if length <= 2 * settings.max_upload_bytes:
|
||||
await drain(request, length)
|
||||
raise ApiError(413, "payload_too_large", "File exceeds %d bytes" % settings.max_upload_bytes,
|
||||
{"Connection": "close"} if length > 2 * settings.max_upload_bytes else None)
|
||||
if not await run_in_threadpool(storage.is_chat_member, chat_id, user.id):
|
||||
raise not_found()
|
||||
store = objects_of(request)
|
||||
|
||||
slots: asyncio.Semaphore = request.app.state.upload_slots
|
||||
try:
|
||||
await asyncio.wait_for(slots.acquire(), timeout=settings.upload_queue_timeout_seconds)
|
||||
except asyncio.TimeoutError:
|
||||
raise ApiError(503, "unavailable", "Too many concurrent uploads", {"Retry-After": "1"})
|
||||
try:
|
||||
try:
|
||||
data, md5_digest, sha256_hex = await asyncio.wait_for(
|
||||
read_limited_body(request, settings.max_upload_bytes),
|
||||
timeout=settings.upload_read_timeout_seconds,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
raise bad_request("Upload was not completed in time")
|
||||
except ClientDisconnect:
|
||||
# Клиент оборвал загрузку: в БД и S3 ничего не записано.
|
||||
log.info("upload aborted by client before completion")
|
||||
raise bad_request("Upload was interrupted")
|
||||
|
||||
try:
|
||||
content_type, width, height = await run_in_threadpool(_validate_image, request, data, declared)
|
||||
except ImageRejected as exc:
|
||||
raise ApiError(exc.status, exc.code, exc.message)
|
||||
|
||||
attachment_id = uuid_mod.uuid4()
|
||||
job_id = uuid_mod.uuid4()
|
||||
# Ключи назначает сервер: не связаны с filename. staging — случайный;
|
||||
# конечный ключ оригинала детерминирован от id (идемпотентная публикация worker'ом).
|
||||
staging_key = "%s%s" % (STAGING_PREFIX, uuid_mod.uuid4().hex)
|
||||
final_key = "final/%s/original" % attachment_id
|
||||
# [1] намерение
|
||||
created = await run_in_threadpool(
|
||||
storage.create_pending_attachment, attachment_id, chat_id, user.id, name,
|
||||
content_type, len(data), sha256_hex, width, height, final_key, staging_key,
|
||||
settings.pipeline_version,
|
||||
)
|
||||
if not created:
|
||||
raise not_found()
|
||||
# [2] байты в устойчивый staging
|
||||
try:
|
||||
_inject(settings, "before_put")
|
||||
await s3_call(store, settings.s3_upload_deadline_seconds,
|
||||
store.put, staging_key, data, content_type, md5_digest, sha256_hex)
|
||||
except ObjectStoreUnavailable:
|
||||
await _compensate(storage, store, attachment_id, None)
|
||||
raise s3_unavailable()
|
||||
finally:
|
||||
del data
|
||||
# [3] задание + outbox одной транзакцией
|
||||
try:
|
||||
_inject(settings, "after_put")
|
||||
accepted = await run_in_threadpool(
|
||||
storage.accept_attachment, attachment_id, job_id,
|
||||
job_payload(job_id, attachment_id, settings.pipeline_version, 1),
|
||||
settings.job_max_attempts, settings.relay_fast_path_grace_seconds,
|
||||
)
|
||||
except (psycopg.Error, PoolTimeout):
|
||||
await _compensate(storage, store, attachment_id, staging_key)
|
||||
raise
|
||||
if accepted is None:
|
||||
# Строку уже забрал сборщик (загрузка шла дольше grace) — объект лишний.
|
||||
await _compensate(storage, store, attachment_id, staging_key)
|
||||
raise ApiError(503, "unavailable", "Upload expired, please retry")
|
||||
attachment, outbox_id = accepted
|
||||
relay = request.app.state.relay
|
||||
if relay is not None:
|
||||
# Выполняется ПОСЛЕ отправки ответа 202.
|
||||
background.add_task(relay.publish_now, outbox_id)
|
||||
return attachment
|
||||
finally:
|
||||
slots.release()
|
||||
|
||||
|
||||
async def _compensate(storage: Storage, store: ObjectStore, attachment_id: UUID, object_key: Optional[str]) -> None:
|
||||
"""Best-effort откат в рамках запроса. Если не удалось — уберёт сборщик."""
|
||||
if object_key is not None:
|
||||
try:
|
||||
await s3_call(store, 5.0, store.delete, object_key)
|
||||
except Exception as exc:
|
||||
log.warning("compensation: object delete deferred to gc (%s)", type(exc).__name__)
|
||||
try:
|
||||
await run_in_threadpool(storage.delete_pending_attachment, attachment_id)
|
||||
except Exception as exc:
|
||||
log.warning("compensation: pending row left for gc (%s)", type(exc).__name__)
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment)
|
||||
def get_attachment(request: Request, attachment_id: UUID, user: User = Depends(current_user)) -> Attachment:
|
||||
found = storage_of(request).get_attachment_for_member(attachment_id, user.id)
|
||||
if found is None:
|
||||
raise not_found()
|
||||
return found[0]
|
||||
|
||||
|
||||
VARIANT_CONTENT_TYPES = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"}
|
||||
|
||||
|
||||
def content_disposition(filename: str) -> str:
|
||||
from urllib.parse import quote
|
||||
ascii_name = "".join(c if 32 <= ord(c) < 127 and c not in ('"', "\\", ";") else "_" for c in filename)
|
||||
return "inline; filename=\"%s\"; filename*=UTF-8''%s" % (ascii_name, quote(filename, safe=""))
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
async def download_attachment(
|
||||
request: Request,
|
||||
attachment_id: UUID,
|
||||
variant: Optional[str] = Query(default=None),
|
||||
user: User = Depends(current_user),
|
||||
):
|
||||
"""Байты оригинала или варианта через авторизованный API (без redirect), потоком из S3.
|
||||
|
||||
lab5: queued/processing/failed -> 409 conflict; у ready нет такого варианта -> 404.
|
||||
"""
|
||||
if variant is not None and variant != "original" and variant not in KNOWN_VARIANTS:
|
||||
raise bad_request("Invalid variant")
|
||||
storage = storage_of(request)
|
||||
found = await run_in_threadpool(storage.get_attachment_for_member, attachment_id, user.id)
|
||||
if found is None:
|
||||
raise not_found()
|
||||
attachment, object_key, pipeline_version = found
|
||||
if attachment.status != "ready":
|
||||
raise ApiError(409, "conflict", "Attachment is not ready (status: %s)" % attachment.status)
|
||||
if variant in KNOWN_VARIANTS:
|
||||
meta = await run_in_threadpool(storage.variant_object, attachment_id, pipeline_version, variant)
|
||||
if meta is None:
|
||||
raise not_found()
|
||||
object_key, media_type, size_bytes, etag = meta
|
||||
disposition_name = "%s-%s.%s" % (variant, attachment.filename.rsplit(".", 1)[0][:200],
|
||||
"webp" if media_type == "image/webp" else "jpg")
|
||||
else:
|
||||
media_type, size_bytes, etag = attachment.content_type, attachment.size_bytes, attachment.sha256
|
||||
disposition_name = attachment.filename
|
||||
store = objects_of(request)
|
||||
try:
|
||||
chunks, length = await s3_call(store, request.app.state.settings.s3_download_deadline_seconds,
|
||||
store.open, object_key)
|
||||
except ObjectStoreUnavailable:
|
||||
raise s3_unavailable()
|
||||
except ObjectMissing:
|
||||
log.error("ready attachment %s has no object in S3 (see python -m src.reconcile)", attachment.id)
|
||||
raise ApiError(500, "internal_error", "Internal server error")
|
||||
if length != size_bytes:
|
||||
chunks.close() # закрыть поток S3
|
||||
log.error("attachment %s: object size differs from metadata", attachment.id)
|
||||
raise ApiError(500, "internal_error", "Internal server error")
|
||||
return StreamingResponse(
|
||||
chunks,
|
||||
media_type=media_type,
|
||||
headers={
|
||||
"Content-Length": str(size_bytes),
|
||||
"Cache-Control": "private, no-store",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"Content-Disposition": content_disposition(disposition_name),
|
||||
"ETag": '"%s"' % etag,
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Однократный запуск миграций: python -m src.migrate
|
||||
|
||||
lab3: это команда one-shot сервиса `migrate` в compose.yaml. Реплики API
|
||||
миграции не применяют (RUN_MIGRATIONS_ON_START=false) и стартуют только после
|
||||
успешного завершения `migrate` (depends_on: service_completed_successfully).
|
||||
Повторный запуск ничего не меняет; одновременный запуск нескольких копий
|
||||
сериализуется pg_advisory_lock (см. src/db.py).
|
||||
"""
|
||||
import logging
|
||||
|
||||
from .config import load_settings
|
||||
from .db import wait_and_migrate
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s %(message)s")
|
||||
applied = wait_and_migrate(load_settings())
|
||||
if applied:
|
||||
print("applied migrations: " + ", ".join(applied))
|
||||
else:
|
||||
print("schema is up to date; nothing applied")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class UserCreate(StrictModel):
|
||||
username: str = Field(
|
||||
min_length=3,
|
||||
max_length=32,
|
||||
pattern=r"^[a-z0-9_]{3,32}$",
|
||||
)
|
||||
display_name: str = Field(
|
||||
min_length=1,
|
||||
max_length=100,
|
||||
)
|
||||
# lab2: пароль обязателен; длина в Unicode code points (len() в Python).
|
||||
password: str = Field(
|
||||
min_length=12,
|
||||
max_length=128,
|
||||
)
|
||||
|
||||
|
||||
class User(StrictModel):
|
||||
id: UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class Session(StrictModel):
|
||||
token: str
|
||||
token_type: str = "Bearer"
|
||||
expires_at: datetime
|
||||
user: User
|
||||
|
||||
|
||||
class ChatCreate(StrictModel):
|
||||
title: str = Field(
|
||||
min_length=1,
|
||||
max_length=100,
|
||||
)
|
||||
member_ids: List[UUID] = Field(
|
||||
min_length=1,
|
||||
max_length=100,
|
||||
)
|
||||
|
||||
|
||||
class Chat(StrictModel):
|
||||
id: UUID
|
||||
title: str
|
||||
member_ids: List[UUID]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class MessageCreate(StrictModel):
|
||||
# lab4: text="" допустим, если есть хотя бы одно вложение (проверка в main.py).
|
||||
text: str = Field(
|
||||
min_length=0,
|
||||
max_length=4000,
|
||||
)
|
||||
# Отсутствующее поле == [] (контракт); значение по умолчанию попадает и в
|
||||
# канонический JSON отпечатка идемпотентности.
|
||||
attachment_ids: List[UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
|
||||
class Message(StrictModel):
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
sender_id: UUID
|
||||
text: str
|
||||
created_at: datetime
|
||||
attachment_ids: List[UUID] = Field(default_factory=list)
|
||||
|
||||
|
||||
class Variant(StrictModel):
|
||||
name: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
|
||||
|
||||
class Attachment(StrictModel):
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
owner_id: UUID
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
# lab5: queued | processing | ready | failed (внутренний pending в API не виден).
|
||||
status: str = "queued"
|
||||
variants: List[Variant] = Field(default_factory=list)
|
||||
error_code: Optional[str] = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class ChatPage(StrictModel):
|
||||
items: List[Chat]
|
||||
next_cursor: Optional[str]
|
||||
|
||||
|
||||
class MessagePage(StrictModel):
|
||||
items: List[Message]
|
||||
next_cursor: Optional[str]
|
||||
@@ -0,0 +1,226 @@
|
||||
"""Приватное S3-совместимое хранилище оригиналов (lab4: SeaweedFS).
|
||||
|
||||
API работает с отдельными credentials, у которых есть только Read/Write/List
|
||||
на один бакет (см. infra/seaweedfs/README.md): создать/удалить бакет или
|
||||
прочитать другой бакет этими ключами нельзя. Административные ключи есть
|
||||
только у one-shot сервиса s3-init (src/s3_bootstrap.py).
|
||||
|
||||
Все вызовы ограничены по времени (connect/read timeout, не более
|
||||
S3_MAX_ATTEMPTS попыток), поэтому недоступность S3 превращается в быстрый 503
|
||||
для операций с вложениями, а не в зависший запрос.
|
||||
"""
|
||||
import base64
|
||||
import hashlib
|
||||
import io
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Iterator, List, Optional, Tuple
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .config import Settings
|
||||
|
||||
log = logging.getLogger("messenger.s3")
|
||||
|
||||
CHUNK = 64 * 1024
|
||||
|
||||
|
||||
class ObjectStoreUnavailable(Exception):
|
||||
"""S3 недоступен, таймаут или 5xx -> 503 для клиента."""
|
||||
|
||||
|
||||
class ObjectMissing(Exception):
|
||||
"""Объекта нет в бакете (404/NoSuchKey)."""
|
||||
|
||||
|
||||
def _code(exc: ClientError) -> str:
|
||||
return str(exc.response.get("Error", {}).get("Code", ""))
|
||||
|
||||
|
||||
def _status(exc: ClientError) -> int:
|
||||
return int(exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode", 0) or 0)
|
||||
|
||||
|
||||
def make_client(endpoint: str, region: str, access_key: str, secret_key: str,
|
||||
connect_timeout: float = 2.0, read_timeout: float = 10.0, max_attempts: int = 2):
|
||||
config = Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
retries={"max_attempts": max_attempts, "mode": "standard"},
|
||||
# Соединения на реплику: не больше, чем одновременных загрузок + скачиваний.
|
||||
max_pool_connections=32,
|
||||
)
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint,
|
||||
region_name=region,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
class CircuitBreaker:
|
||||
"""После отказа S3 следующие open_seconds операции сразу получают
|
||||
ObjectStoreUnavailable, не тратя время на DNS/connect/повторы. Затем —
|
||||
одна пробная попытка (half-open): успех закрывает, отказ снова открывает."""
|
||||
|
||||
def __init__(self, open_seconds: float) -> None:
|
||||
self.open_seconds = open_seconds
|
||||
self._lock = threading.Lock()
|
||||
self._open_until = 0.0
|
||||
|
||||
def check(self) -> None:
|
||||
with self._lock:
|
||||
now = time.monotonic()
|
||||
if now < self._open_until:
|
||||
raise ObjectStoreUnavailable()
|
||||
if self._open_until:
|
||||
# half-open: пропускаем одну попытку, остальные пока отклоняем
|
||||
self._open_until = now + self.open_seconds
|
||||
|
||||
def failure(self) -> None:
|
||||
with self._lock:
|
||||
if not self._open_until or time.monotonic() >= self._open_until - self.open_seconds:
|
||||
log.warning("s3 circuit opened for %.1fs", self.open_seconds)
|
||||
self._open_until = time.monotonic() + self.open_seconds
|
||||
|
||||
def success(self) -> None:
|
||||
with self._lock:
|
||||
self._open_until = 0.0
|
||||
|
||||
@property
|
||||
def is_open(self) -> bool:
|
||||
return time.monotonic() < self._open_until
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self.bucket = settings.s3_bucket
|
||||
self.breaker = CircuitBreaker(settings.s3_circuit_open_seconds)
|
||||
self.client = make_client(
|
||||
settings.s3_endpoint,
|
||||
settings.s3_region,
|
||||
settings.s3_access_key_id,
|
||||
settings.s3_secret_access_key,
|
||||
settings.s3_connect_timeout_seconds,
|
||||
settings.s3_read_timeout_seconds,
|
||||
settings.s3_max_attempts,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def configured(settings: Settings) -> bool:
|
||||
return bool(settings.s3_endpoint and settings.s3_access_key_id and settings.s3_secret_access_key)
|
||||
|
||||
def _wrap(self, exc: Exception, op: str) -> Exception:
|
||||
if isinstance(exc, ClientError):
|
||||
code, status = _code(exc), _status(exc)
|
||||
if status == 404 or code in ("NoSuchKey", "NotFound", "404"):
|
||||
self.breaker.success() # S3 ответил — он доступен
|
||||
return ObjectMissing()
|
||||
log.warning("s3 %s failed: %s/%s", op, status, code)
|
||||
else:
|
||||
log.warning("s3 %s failed: %s", op, type(exc).__name__)
|
||||
self.breaker.failure()
|
||||
return ObjectStoreUnavailable()
|
||||
|
||||
def put(self, key: str, data, content_type: str, md5_digest: bytes, sha256_hex: str) -> None:
|
||||
"""PUT с Content-MD5: S3 сам отклонит повреждённую по пути передачу (BadDigest).
|
||||
|
||||
Тело передаётся как BytesIO поверх bytes (без копирования): botocore
|
||||
отправляет файловый объект блоками. С bytes он склеивает заголовки и тело
|
||||
в один буфер — лишняя копия размером с файл на каждую загрузку (замер в REPORT.md).
|
||||
"""
|
||||
self.breaker.check()
|
||||
try:
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=io.BytesIO(data),
|
||||
ContentType=content_type,
|
||||
ContentMD5=base64.b64encode(md5_digest).decode("ascii"),
|
||||
Metadata={"sha256": sha256_hex},
|
||||
)
|
||||
except (ClientError, BotoCoreError) as exc:
|
||||
raise self._wrap(exc, "put")
|
||||
self.breaker.success()
|
||||
|
||||
def open(self, key: str) -> Tuple[Iterator[bytes], int]:
|
||||
"""Потоковое чтение: (итератор кусков по 64 KiB, длина). Память — O(chunk)."""
|
||||
self.breaker.check()
|
||||
try:
|
||||
obj = self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
except (ClientError, BotoCoreError) as exc:
|
||||
raise self._wrap(exc, "get")
|
||||
self.breaker.success()
|
||||
body = obj["Body"]
|
||||
|
||||
def chunks() -> Iterator[bytes]:
|
||||
try:
|
||||
for chunk in body.iter_chunks(CHUNK):
|
||||
yield chunk
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
return chunks(), int(obj.get("ContentLength", -1))
|
||||
|
||||
def read_all(self, key: str, limit: int) -> bytes:
|
||||
"""Для сверки/тестов: прочитать объект целиком (не больше limit байт)."""
|
||||
it, _ = self.open(key)
|
||||
out = bytearray()
|
||||
for chunk in it:
|
||||
out += chunk
|
||||
if len(out) > limit:
|
||||
raise ValueError("object exceeds limit")
|
||||
return bytes(out)
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
"""Идемпотентно: удаление отсутствующего объекта — не ошибка."""
|
||||
self.breaker.check()
|
||||
try:
|
||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
||||
except (ClientError, BotoCoreError) as exc:
|
||||
err = self._wrap(exc, "delete")
|
||||
if not isinstance(err, ObjectMissing):
|
||||
raise err
|
||||
|
||||
def head(self, key: str) -> Optional[dict]:
|
||||
self.breaker.check()
|
||||
try:
|
||||
return self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
except (ClientError, BotoCoreError) as exc:
|
||||
err = self._wrap(exc, "head")
|
||||
if isinstance(err, ObjectMissing):
|
||||
return None
|
||||
raise err
|
||||
|
||||
def list_page(self, prefix: str, token: Optional[str] = None,
|
||||
page_size: int = 1000) -> Tuple[List[Tuple[str, datetime, int]], Optional[str]]:
|
||||
kwargs = {"Bucket": self.bucket, "Prefix": prefix, "MaxKeys": page_size}
|
||||
if token:
|
||||
kwargs["ContinuationToken"] = token
|
||||
self.breaker.check()
|
||||
try:
|
||||
resp = self.client.list_objects_v2(**kwargs)
|
||||
except (ClientError, BotoCoreError) as exc:
|
||||
raise self._wrap(exc, "list")
|
||||
items = [(o["Key"], o["LastModified"], int(o.get("Size", 0))) for o in resp.get("Contents", [])]
|
||||
next_token = resp.get("NextContinuationToken") if resp.get("IsTruncated") else None
|
||||
return items, next_token
|
||||
|
||||
def ping(self) -> bool:
|
||||
try:
|
||||
self.client.head_bucket(Bucket=self.bucket)
|
||||
return True
|
||||
except (ClientError, BotoCoreError):
|
||||
return False
|
||||
|
||||
|
||||
def md5_and_sha256(data: bytes) -> Tuple[bytes, str]:
|
||||
return hashlib.md5(data).digest(), hashlib.sha256(data).hexdigest()
|
||||
+187
@@ -0,0 +1,187 @@
|
||||
"""Transactional outbox relay (lab5): PostgreSQL job_outbox -> брокер.
|
||||
|
||||
Почему нет разрыва «COMMIT в БД прошёл, а publish не случился»:
|
||||
API пишет задание и строку outbox в ОДНОЙ транзакции с переводом вложения в
|
||||
queued. Публикация — отдельный шаг, который можно безопасно повторять:
|
||||
|
||||
быстрый путь API сразу после ответа 202 (background task) публикует свою строку:
|
||||
SELECT ... FOR UPDATE SKIP LOCKED -> XADD -> DELETE -> COMMIT.
|
||||
sweeper Каждый процесс API и worker раз в RELAY_POLL_INTERVAL_SECONDS
|
||||
публикует строки с available_at <= now(). У свежей строки
|
||||
available_at = created_at + RELAY_FAST_PATH_GRACE_SECONDS, чтобы
|
||||
не соревноваться с быстрым путём; у retry — время следующей попытки.
|
||||
|
||||
Окна отказа:
|
||||
* процесс умер после COMMIT до XADD -> строка осталась -> sweeper опубликует;
|
||||
* умер после XADD до DELETE/COMMIT -> строка осталась -> повторная публикация,
|
||||
дубль в брокере; worker идемпотентен (redelivery_noop);
|
||||
* брокер недоступен -> ROLLBACK, publish_failures+1, повтор позже.
|
||||
Строка блокируется FOR UPDATE SKIP LOCKED: несколько relay публикуют разные строки.
|
||||
Успешный XADD = подтверждение приёма брокером (AOF appendfsync always).
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
import psycopg
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
|
||||
from . import telemetry
|
||||
from .broker import Broker, BrokerUnavailable
|
||||
from .jobs import JobStore
|
||||
|
||||
log = logging.getLogger("messenger.outbox")
|
||||
|
||||
|
||||
class Relay:
|
||||
def __init__(self, pool: ConnectionPool, broker: Broker, settings, actor: str) -> None:
|
||||
self.pool = pool
|
||||
self.broker = broker
|
||||
self.settings = settings
|
||||
self.actor = actor
|
||||
self.jobs = JobStore(pool, settings)
|
||||
self.fault = settings.fault_injection
|
||||
self.published = 0
|
||||
|
||||
def _publish_rows(self, conn, rows, fast: bool = False) -> int:
|
||||
sent = []
|
||||
for outbox_id, job_id, payload in rows:
|
||||
stream_id = self._publish_traced(payload) # BrokerUnavailable -> ROLLBACK всего пакета
|
||||
sent.append((outbox_id, job_id, payload, stream_id))
|
||||
if fast and self.fault == "crash_after_publish":
|
||||
log.error("FAULT_INJECTION: relay exits after XADD %s before outbox DELETE/COMMIT", stream_id)
|
||||
os._exit(70)
|
||||
conn.execute("DELETE FROM job_outbox WHERE id = ANY(%s)", ([s[0] for s in sent],))
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"INSERT INTO job_events (job_id, attachment_id, attempt, event, detail, actor) "
|
||||
"VALUES (%s, %s, %s, 'published', %s, %s)",
|
||||
[(job_id, payload.get("attachment_id"), payload.get("attempt"), "stream id " + stream_id, self.actor)
|
||||
for _oid, job_id, payload, stream_id in sent],
|
||||
)
|
||||
self.published += len(sent)
|
||||
return len(sent)
|
||||
|
||||
def _publish_traced(self, payload: dict) -> str:
|
||||
"""lab6: PRODUCER span — ребёнок span'а, сохранённого в строке outbox
|
||||
(HTTP upload или consumer при retry). В брокер уходит traceparent уже
|
||||
этого producer span: цепочка upload -> publish -> process."""
|
||||
parent = telemetry.extract(payload)
|
||||
with telemetry.tracer().start_as_current_span(
|
||||
"%s publish" % self.broker.stream, context=parent, kind=SpanKind.PRODUCER,
|
||||
attributes={
|
||||
"messaging.system": "redis",
|
||||
"messaging.operation.type": "publish",
|
||||
"messaging.destination.name": self.broker.stream,
|
||||
"messenger.job.type": payload.get("type", ""),
|
||||
"messenger.job.attempt": int(payload.get("attempt") or 0),
|
||||
"messenger.relay.actor": self.actor.split(":", 1)[0],
|
||||
},
|
||||
) as span:
|
||||
message = {k: v for k, v in payload.items() if k not in ("traceparent", "tracestate")}
|
||||
telemetry.inject(message)
|
||||
try:
|
||||
stream_id = self.broker.publish(message)
|
||||
except BrokerUnavailable as exc:
|
||||
span.set_status(Status(StatusCode.ERROR, "broker unavailable"))
|
||||
span.record_exception(exc)
|
||||
raise
|
||||
span.set_attribute("messaging.message.id", stream_id)
|
||||
telemetry.INSTRUMENTS.published.add(1, {"job.type": payload.get("type", "")})
|
||||
return stream_id
|
||||
|
||||
def publish_now(self, outbox_id: int) -> bool:
|
||||
"""Быстрый путь для одной строки (вызывается после ответа 202). Ошибки не
|
||||
пробрасываются: строку опубликует sweeper."""
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
rows = conn.execute(
|
||||
"SELECT id, job_id, payload FROM job_outbox WHERE id = %s FOR UPDATE SKIP LOCKED",
|
||||
(outbox_id,),
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return False
|
||||
if self.fault == "crash_after_commit":
|
||||
# Внедрённый разрыв: COMMIT задания и outbox уже есть, 202 отправлен,
|
||||
# процесс умирает до XADD. Восстановление — sweeper другого процесса.
|
||||
self.jobs.event(rows[0][1], rows[0][2].get("attachment_id"), 1, "fault_injected",
|
||||
"crash between COMMIT and publish", self.actor)
|
||||
log.error("FAULT_INJECTION: process exit between COMMIT and publish (outbox %s)", outbox_id)
|
||||
os._exit(70)
|
||||
if self.fault == "crash_after_publish":
|
||||
self.jobs.event(rows[0][1], rows[0][2].get("attachment_id"), 1, "fault_injected",
|
||||
"crash after XADD before outbox commit", self.actor)
|
||||
return self._publish_rows(conn, rows, fast=True) > 0
|
||||
except (BrokerUnavailable, psycopg.Error, PoolTimeout) as exc:
|
||||
log.warning("fast publish deferred to sweeper: %s", type(exc).__name__)
|
||||
return False
|
||||
|
||||
def sweep_once(self) -> int:
|
||||
"""Опубликовать все готовые строки пакетами; вернуть число опубликованных."""
|
||||
total = 0
|
||||
while True:
|
||||
with self.pool.connection() as conn:
|
||||
try:
|
||||
with conn.transaction():
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, job_id, payload FROM job_outbox
|
||||
WHERE available_at <= clock_timestamp()
|
||||
ORDER BY available_at, id
|
||||
LIMIT %s
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""",
|
||||
(self.settings.relay_batch,),
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return total
|
||||
total += self._publish_rows(conn, rows)
|
||||
except BrokerUnavailable:
|
||||
# Транзакция откатилась (строки целы). Отметим неудачу отдельно.
|
||||
with conn.transaction():
|
||||
conn.execute(
|
||||
"UPDATE job_outbox SET publish_failures = publish_failures + 1 "
|
||||
"WHERE id = ANY(%s)", ([r[0] for r in rows],))
|
||||
raise
|
||||
if len(rows) < self.settings.relay_batch:
|
||||
return total
|
||||
|
||||
def loop(self, stop: threading.Event, wake: Optional[threading.Event] = None) -> None:
|
||||
"""Фоновый цикл: sweeper + сборщик просроченных аренд."""
|
||||
interval = self.settings.relay_poll_interval_seconds
|
||||
last_reap = 0.0
|
||||
broker_down_logged = False
|
||||
while not stop.is_set():
|
||||
try:
|
||||
n = self.sweep_once()
|
||||
if n:
|
||||
log.info("relay %s: published %d outbox rows", self.actor, n)
|
||||
broker_down_logged = False
|
||||
if time.monotonic() - last_reap >= 5:
|
||||
last_reap = time.monotonic()
|
||||
for job_id, outcome in self.jobs.reap_expired(self.actor):
|
||||
log.warning("lease expired: job %s %s", job_id, outcome)
|
||||
except BrokerUnavailable:
|
||||
if not broker_down_logged:
|
||||
log.warning("relay %s: broker unavailable; outbox rows are kept", self.actor)
|
||||
broker_down_logged = True
|
||||
except (psycopg.Error, PoolTimeout) as exc:
|
||||
log.warning("relay %s: database unavailable (%s)", self.actor, type(exc).__name__)
|
||||
except Exception:
|
||||
log.exception("relay %s: unexpected error", self.actor)
|
||||
if wake is not None:
|
||||
wake.wait(interval)
|
||||
wake.clear()
|
||||
else:
|
||||
stop.wait(interval)
|
||||
|
||||
|
||||
def start_relay_thread(pool, broker, settings, actor: str, stop: threading.Event) -> Relay:
|
||||
relay = Relay(pool, broker, settings, actor)
|
||||
threading.Thread(target=relay.loop, args=(stop,), name="outbox-relay", daemon=True).start()
|
||||
return relay
|
||||
@@ -0,0 +1,295 @@
|
||||
"""Согласованность PostgreSQL (метаданные) и S3 (байты): сборка мусора и сверка.
|
||||
|
||||
lab5: префиксы бакета
|
||||
staging/<hex> — принятый оригинал до обработки (API);
|
||||
final/<id>/original — опубликованный оригинал (worker);
|
||||
final/<id>/p<версия>/<вариант>.<ext> — варианты (worker);
|
||||
originals/<hex> — оригиналы lab4 (до перехода на lab5).
|
||||
Строка pending ссылается на staging_key (объект появляется ПОСЛЕ строки).
|
||||
Добавлен проход D — очистка staging: ready-вложения (staging больше не нужен,
|
||||
результат опубликован) и failed старше STAGING_RETENTION_SECONDS (после этого
|
||||
replay невозможен — нужен повторный upload). Текущую работу (queued/processing)
|
||||
сборщик не трогает никогда.
|
||||
|
||||
Почему неучтённых объектов «навсегда» не бывает (подробно — REPORT.md):
|
||||
|
||||
загрузка: [1] INSERT attachments(state=pending, object_key) COMMIT
|
||||
[2] S3 PUT object_key
|
||||
[3] UPDATE state=ready WHERE state=pending COMMIT -> 201
|
||||
Ключ объекта записан в БД ДО того, как объект появится в бакете.
|
||||
|
||||
Сбой/обрыв между шагами:
|
||||
после [1] -> строка pending без объекта;
|
||||
после [2] -> строка pending + объект (S3 прошёл, COMMIT [3] — нет);
|
||||
ошибка S3 в [2] -> API сам компенсирует: DELETE pending;
|
||||
ошибка БД в [3] -> API сам компенсирует: DELETE object (pending уберёт GC).
|
||||
Процесс может умереть в любой точке, поэтому компенсация в запросе —
|
||||
оптимизация, а гарантию даёт сборщик:
|
||||
|
||||
A. stale pending: строки pending старше GRACE. Сначала строка удаляется
|
||||
(DELETE ... WHERE state='pending' RETURNING object_key — атомарный захват:
|
||||
если запрос всё-таки успел сделать [3], строка уже ready и не удаляется),
|
||||
затем удаляется объект. Если удаление объекта не удалось — объект стал
|
||||
«сиротой» и будет убран проходом B.
|
||||
B. orphan objects: объекты в бакете (префикс originals/), для ключа которых
|
||||
в БД нет строки, и LastModified старше GRACE. Строка всегда создаётся
|
||||
раньше объекта и после удаления не возрождается (ключ одноразовый),
|
||||
поэтому «нет строки» + «старше GRACE» = объект не принадлежит живой загрузке.
|
||||
C. missing objects: строки ready без объекта (или с другим размером) — только
|
||||
отчёт: восстановить байты нельзя, удалять метаданные автоматически опасно.
|
||||
|
||||
GRACE (ATTACHMENT_GC_GRACE_SECONDS, по умолчанию 900 с, минимум 300 с) много
|
||||
больше самой долгой загрузки: чтение тела <= UPLOAD_READ_TIMEOUT_SECONDS (60 с)
|
||||
+ проверка + S3 PUT (connect 2 с + read 10 с) x S3_MAX_ATTEMPTS (2).
|
||||
Строки ready и вложения, на которые ссылаются сообщения, сборщик не трогает
|
||||
никогда (FK message_attachments -> attachments ON DELETE RESTRICT).
|
||||
|
||||
CLI (по умолчанию dry-run, ничего не меняет):
|
||||
python -m src.reconcile # отчёт
|
||||
python -m src.reconcile --apply # исправить A и B
|
||||
python -m src.reconcile --apply --grace-seconds 0 # только на учебном стенде
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
from .objectstore import ObjectStore, ObjectStoreUnavailable
|
||||
|
||||
log = logging.getLogger("messenger.reconcile")
|
||||
|
||||
OBJECT_PREFIX = "originals/" # lab4
|
||||
STAGING_PREFIX = "staging/"
|
||||
FINAL_PREFIX = "final/"
|
||||
SCANNED_PREFIXES = (STAGING_PREFIX, FINAL_PREFIX, OBJECT_PREFIX)
|
||||
GC_LOCK_ID = 2026_0004
|
||||
BATCH = 500
|
||||
|
||||
|
||||
def _try_lock(conn) -> bool:
|
||||
return conn.execute("SELECT pg_try_advisory_lock(%s)", (GC_LOCK_ID,)).fetchone()[0]
|
||||
|
||||
|
||||
def _unlock(conn) -> None:
|
||||
conn.execute("SELECT pg_advisory_unlock(%s)", (GC_LOCK_ID,))
|
||||
|
||||
|
||||
def collect_stale_pending(pool: ConnectionPool, store: ObjectStore, grace_seconds: int,
|
||||
apply: bool, report: Dict) -> None:
|
||||
"""Проход A. dry-run: только подсчёт; apply: строка -> затем объект."""
|
||||
with pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, COALESCE(staging_key, object_key) FROM attachments
|
||||
WHERE state = 'pending' AND created_at < clock_timestamp() - make_interval(secs => %s)
|
||||
ORDER BY created_at LIMIT %s
|
||||
""",
|
||||
(grace_seconds, BATCH),
|
||||
).fetchall()
|
||||
report["stale_pending"] = len(rows)
|
||||
report["stale_pending_ids"] = [str(r[0]) for r in rows[:20]]
|
||||
if not apply:
|
||||
return
|
||||
deleted_rows = deleted_objects = 0
|
||||
for att_id, _key in rows:
|
||||
with pool.connection() as conn:
|
||||
claimed = conn.execute(
|
||||
"""
|
||||
DELETE FROM attachments
|
||||
WHERE id = %s AND state = 'pending'
|
||||
AND created_at < clock_timestamp() - make_interval(secs => %s)
|
||||
RETURNING COALESCE(staging_key, object_key)
|
||||
""",
|
||||
(att_id, grace_seconds),
|
||||
).fetchone()
|
||||
if claimed is None:
|
||||
continue # загрузка успела завершиться (ready) — не трогаем
|
||||
deleted_rows += 1
|
||||
try:
|
||||
store.delete(claimed[0])
|
||||
deleted_objects += 1
|
||||
except ObjectStoreUnavailable:
|
||||
# Объект стал сиротой без строки; его уберёт проход B.
|
||||
log.warning("gc: object delete deferred to orphan sweep")
|
||||
report["stale_pending_rows_deleted"] = deleted_rows
|
||||
report["stale_pending_object_deletes"] = deleted_objects
|
||||
|
||||
|
||||
def known_keys(pool: ConnectionPool, keys: List[str]) -> set:
|
||||
"""Ключи, на которые ссылается БД: оригинал, staging или вариант."""
|
||||
with pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT object_key FROM attachments WHERE object_key = ANY(%s)
|
||||
UNION SELECT staging_key FROM attachments WHERE staging_key = ANY(%s)
|
||||
UNION SELECT object_key FROM attachment_variants WHERE object_key = ANY(%s)
|
||||
""",
|
||||
(keys, keys, keys),
|
||||
).fetchall()
|
||||
return {r[0] for r in rows}
|
||||
|
||||
|
||||
def clean_staging(pool: ConnectionPool, store: ObjectStore, retention_seconds: int,
|
||||
apply: bool, report: Dict) -> None:
|
||||
"""Проход D: staging, который больше не нужен. Строка обновляется ПОСЛЕ удаления
|
||||
объекта; сбой между шагами безопасен (повторное удаление отсутствующего — no-op)."""
|
||||
with pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, staging_key, state FROM attachments
|
||||
WHERE staging_key IS NOT NULL
|
||||
AND (state = 'ready'
|
||||
OR (state = 'failed' AND updated_at < clock_timestamp() - make_interval(secs => %s)))
|
||||
ORDER BY updated_at LIMIT %s
|
||||
""",
|
||||
(retention_seconds, BATCH),
|
||||
).fetchall()
|
||||
report["staging_to_clean"] = len(rows)
|
||||
if not apply:
|
||||
return
|
||||
cleaned = 0
|
||||
for att_id, key, state in rows:
|
||||
try:
|
||||
store.delete(key)
|
||||
except ObjectStoreUnavailable:
|
||||
log.warning("gc: staging delete deferred")
|
||||
break
|
||||
with pool.connection() as conn:
|
||||
conn.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s AND staging_key = %s "
|
||||
"AND state = %s", (att_id, key, state))
|
||||
cleaned += 1
|
||||
report["staging_cleaned"] = cleaned
|
||||
|
||||
|
||||
def sweep_orphans(pool: ConnectionPool, store: ObjectStore, grace_seconds: int,
|
||||
apply: bool, report: Dict, verify_ready: bool = True) -> None:
|
||||
"""Проходы B и C: полный обход префикса бакета страницами по 1000."""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
|
||||
token: Optional[str] = None
|
||||
listed = orphans = orphans_young = deleted = 0
|
||||
seen_keys: List[str] = []
|
||||
orphan_sample: List[str] = []
|
||||
started = time.monotonic()
|
||||
for prefix in SCANNED_PREFIXES:
|
||||
token = None
|
||||
while True:
|
||||
items, token = store.list_page(prefix, token)
|
||||
listed += len(items)
|
||||
if items:
|
||||
keys = [k for k, _, _ in items]
|
||||
known = known_keys(pool, keys)
|
||||
for key, modified, _size in items:
|
||||
if key in known:
|
||||
if verify_ready:
|
||||
seen_keys.append(key)
|
||||
continue
|
||||
orphans += 1
|
||||
if modified > cutoff:
|
||||
orphans_young += 1 # возможно, живая загрузка/обработка или ещё не истёк grace
|
||||
continue
|
||||
if len(orphan_sample) < 20:
|
||||
orphan_sample.append(key)
|
||||
if apply and not known_keys(pool, [key]):
|
||||
# Повторная проверка непосредственно перед удалением.
|
||||
store.delete(key)
|
||||
deleted += 1
|
||||
if not token:
|
||||
break
|
||||
report["objects_listed"] = listed
|
||||
report["orphan_objects"] = orphans
|
||||
report["orphan_objects_within_grace"] = orphans_young
|
||||
report["orphan_objects_sample"] = orphan_sample
|
||||
if apply:
|
||||
report["orphan_objects_deleted"] = deleted
|
||||
report["bucket_scan_seconds"] = round(time.monotonic() - started, 3)
|
||||
|
||||
if verify_ready:
|
||||
# C: ready-строки, для которых объекта нет в листинге (или не тот размер).
|
||||
present = set(seen_keys)
|
||||
with pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id, object_key, size_bytes FROM attachments WHERE state = 'ready' "
|
||||
"UNION ALL SELECT attachment_id, object_key, size_bytes FROM attachment_variants"
|
||||
).fetchall()
|
||||
missing = [str(r[0]) for r in rows if r[1] not in present]
|
||||
report["ready_rows"] = len(rows)
|
||||
report["missing_objects"] = len(missing)
|
||||
report["missing_object_ids"] = missing[:20]
|
||||
|
||||
|
||||
def run_once(pool: ConnectionPool, store: ObjectStore, grace_seconds: int, apply: bool,
|
||||
orphan_sweep: bool = True, staging_retention_seconds: int = 86400) -> Optional[Dict]:
|
||||
"""Один проход под advisory lock. None — сейчас работает другая реплика."""
|
||||
report: Dict = {"mode": "apply" if apply else "dry-run", "grace_seconds": grace_seconds}
|
||||
with pool.connection() as lock_conn:
|
||||
lock_conn.autocommit = True
|
||||
if not _try_lock(lock_conn):
|
||||
return None
|
||||
try:
|
||||
collect_stale_pending(pool, store, grace_seconds, apply, report)
|
||||
clean_staging(pool, store, staging_retention_seconds, apply, report)
|
||||
if orphan_sweep:
|
||||
sweep_orphans(pool, store, grace_seconds, apply, report)
|
||||
finally:
|
||||
_unlock(lock_conn)
|
||||
# Соединение вернётся в общий пул — режим транзакций как у остальных.
|
||||
lock_conn.autocommit = False
|
||||
return report
|
||||
|
||||
|
||||
def gc_loop(pool: ConnectionPool, store: ObjectStore, settings, stop) -> None:
|
||||
"""Фон в каждой реплике: проход A каждые N с, проход B — реже. Lock — одна реплика."""
|
||||
last_sweep = time.monotonic()
|
||||
while not stop.wait(settings.attachment_gc_interval_seconds):
|
||||
try:
|
||||
sweep = time.monotonic() - last_sweep >= settings.orphan_sweep_interval_seconds
|
||||
report = run_once(pool, store, settings.attachment_gc_grace_seconds, apply=True,
|
||||
orphan_sweep=sweep, staging_retention_seconds=settings.staging_retention_seconds)
|
||||
if report is None:
|
||||
continue
|
||||
if sweep:
|
||||
last_sweep = time.monotonic()
|
||||
if report.get("stale_pending") or report.get("orphan_objects_deleted") or report.get("missing_objects") \
|
||||
or report.get("staging_cleaned"):
|
||||
log.info("attachment gc: %s", json.dumps(report, ensure_ascii=False))
|
||||
except Exception as exc: # БД/S3 недоступны — следующий проход повторит
|
||||
log.warning("attachment gc skipped: %s", type(exc).__name__)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Reconcile attachments metadata (PostgreSQL) with S3 objects")
|
||||
parser.add_argument("--apply", action="store_true", help="delete stale pending rows and orphan objects")
|
||||
parser.add_argument("--grace-seconds", type=int, default=None,
|
||||
help="override ATTACHMENT_GC_GRACE_SECONDS (values < 300 only for lab tests)")
|
||||
parser.add_argument("--no-orphan-sweep", action="store_true", help="skip bucket listing")
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s %(message)s")
|
||||
|
||||
from .config import load_settings
|
||||
from .db import create_pool
|
||||
|
||||
settings = load_settings()
|
||||
if not ObjectStore.configured(settings):
|
||||
raise SystemExit("S3 is not configured")
|
||||
grace = settings.attachment_gc_grace_seconds if args.grace_seconds is None else args.grace_seconds
|
||||
pool = create_pool(settings)
|
||||
pool.wait(timeout=15)
|
||||
store = ObjectStore(settings)
|
||||
try:
|
||||
started = time.monotonic()
|
||||
report = run_once(pool, store, grace, args.apply, orphan_sweep=not args.no_orphan_sweep,
|
||||
staging_retention_seconds=settings.staging_retention_seconds)
|
||||
if report is None:
|
||||
raise SystemExit("another reconciler holds the lock; retry later")
|
||||
report["total_seconds"] = round(time.monotonic() - started, 3)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
finally:
|
||||
pool.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Состояние процесса, общее для middleware, readiness и сервера.
|
||||
|
||||
INSTANCE_ID — значение X-Instance-Id. По умолчанию hostname контейнера:
|
||||
Docker задаёт его равным короткому ID контейнера, поэтому scripts/ops.py
|
||||
однозначно сопоставляет заголовок конкретному контейнеру (stop-one).
|
||||
Значение не меняется в течение жизни процесса.
|
||||
|
||||
DRAIN — фаза «осушения» после SIGTERM (см. src/server.py): процесс ещё
|
||||
обслуживает запросы, но /health/ready уже отвечает 503, и балансировщик
|
||||
выводит реплику из ротации до того, как она перестанет принимать соединения.
|
||||
"""
|
||||
import os
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname()
|
||||
|
||||
|
||||
class Drain:
|
||||
def __init__(self) -> None:
|
||||
self._event = threading.Event()
|
||||
self.started_at: Optional[float] = None
|
||||
|
||||
def begin(self) -> bool:
|
||||
"""True, если осушение началось этим вызовом."""
|
||||
if self._event.is_set():
|
||||
return False
|
||||
self.started_at = time.monotonic()
|
||||
self._event.set()
|
||||
return True
|
||||
|
||||
@property
|
||||
def active(self) -> bool:
|
||||
return self._event.is_set()
|
||||
|
||||
|
||||
DRAIN = Drain()
|
||||
@@ -0,0 +1,226 @@
|
||||
"""Bootstrap и проверка приватного бакета: python -m src.s3_bootstrap
|
||||
|
||||
Запускается one-shot сервисом `s3-init` до старта API (depends_on
|
||||
service_completed_successfully). Идемпотентно: повторный запуск не меняет
|
||||
данные — BucketAlreadyOwnedByYou считается успехом.
|
||||
|
||||
Проверки (любое нарушение -> exit 1, API не стартует):
|
||||
* API-ключи могут PUT/GET/DELETE в своём бакете (проба в префиксе _bootstrap/,
|
||||
который сборщик мусора не обходит);
|
||||
* API-ключи НЕ могут создать бакет (нет прав администратора);
|
||||
* анонимный GET объекта и анонимный листинг бакета -> 403;
|
||||
* административные HTTP-порты SeaweedFS (master 9333, filer 8888,
|
||||
volume 8080) не слушают в общей сети — доступен только S3 API.
|
||||
|
||||
python -m src.s3_bootstrap # s3-init: создать бакет + проверки
|
||||
python -m src.s3_bootstrap --check-only # только проверки (без admin-ключей)
|
||||
python -m src.s3_bootstrap --check-only --sample-from-db
|
||||
# + анонимный GET реального объекта последнего ready-вложения -> 403,
|
||||
# GET тем же ключом с API-credentials -> 200 и совпадает SHA-256.
|
||||
python -m src.s3_bootstrap --wait-ready 90
|
||||
# прогрев после рестарта: ждать, пока запись/чтение пробного объекта и
|
||||
# чтение СУЩЕСТВУЮЩЕГО объекта занимают < 2 с. SeaweedFS отвечает /healthz
|
||||
# сразу, но первые чтение/запись после рестарта блокируются на 15–30 с
|
||||
# (выборы Raft-лидера master, heartbeat volume-сервера; замер в REPORT.md).
|
||||
Значения ключей не печатаются.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .config import _read_secret
|
||||
from .objectstore import make_client
|
||||
|
||||
ADMIN_PORTS = (9333, 8888, 8080, 19333, 18888)
|
||||
|
||||
|
||||
def anonymous_status(url: str) -> int:
|
||||
req = urllib.request.Request(url, method="GET")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
||||
return resp.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code
|
||||
|
||||
|
||||
def port_open(host: str, port: int) -> bool:
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=1.5):
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def wait_ready(timeout: float) -> int:
|
||||
"""Ждать, пока S3 быстро (< 2 с на операцию) выполняет запись, чтение и удаление
|
||||
пробного объекта и чтение реального (самого старого ready) объекта, если он есть."""
|
||||
import psycopg
|
||||
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").rstrip("/")
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
client = make_client(endpoint, os.environ.get("S3_REGION", "us-east-1"), _read_secret("S3_ACCESS_KEY_ID"),
|
||||
_read_secret("S3_SECRET_ACCESS_KEY"), 1, 2, 1)
|
||||
started = time.monotonic()
|
||||
key = None
|
||||
looked_up = False
|
||||
while time.monotonic() - started < timeout:
|
||||
try:
|
||||
if not looked_up and _read_secret("DATABASE_URL"):
|
||||
with psycopg.connect(_read_secret("DATABASE_URL"), connect_timeout=3) as conn:
|
||||
row = conn.execute("SELECT object_key FROM attachments WHERE state = 'ready' "
|
||||
"ORDER BY created_at LIMIT 1").fetchone()
|
||||
key = row[0] if row else None
|
||||
looked_up = True
|
||||
t = time.monotonic()
|
||||
probe = "_bootstrap/warmup-%s" % uuid.uuid4().hex
|
||||
client.put_object(Bucket=bucket, Key=probe, Body=b"warmup")
|
||||
client.get_object(Bucket=bucket, Key=probe)["Body"].read()
|
||||
client.delete_object(Bucket=bucket, Key=probe)
|
||||
if key:
|
||||
client.get_object(Bucket=bucket, Key=key)["Body"].read()
|
||||
print(json.dumps({"s3_ready_after_s": round(time.monotonic() - started, 2),
|
||||
"last_cycle_s": round(time.monotonic() - t, 3),
|
||||
"existing_object_read": bool(key)}))
|
||||
return 0
|
||||
except (ClientError, BotoCoreError, OSError, psycopg.Error):
|
||||
time.sleep(0.5)
|
||||
print("S3 did not become ready for reads/writes within %.0fs" % timeout, file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--check-only", action="store_true")
|
||||
parser.add_argument("--sample-from-db", action="store_true")
|
||||
parser.add_argument("--wait-ready", type=float, default=0.0, metavar="SECONDS")
|
||||
args = parser.parse_args()
|
||||
if args.wait_ready:
|
||||
return wait_ready(args.wait_ready)
|
||||
|
||||
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").rstrip("/")
|
||||
region = os.environ.get("S3_REGION", "us-east-1")
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
wait_seconds = int(os.environ.get("S3_BOOTSTRAP_WAIT_SECONDS", "90"))
|
||||
report = {"endpoint_host": urlparse(endpoint).hostname, "bucket": bucket, "checks": []}
|
||||
ok = True
|
||||
|
||||
def check(name, condition, detail=""):
|
||||
nonlocal ok
|
||||
ok = ok and bool(condition)
|
||||
report["checks"].append({"check": name, "ok": bool(condition), "detail": detail})
|
||||
|
||||
api_key = _read_secret("S3_ACCESS_KEY_ID")
|
||||
api_secret = _read_secret("S3_SECRET_ACCESS_KEY")
|
||||
if not (api_key and api_secret):
|
||||
print("S3 API credentials are not configured", file=sys.stderr)
|
||||
return 2
|
||||
api = make_client(endpoint, region, api_key, api_secret, 2, 10, 2)
|
||||
|
||||
if not args.check_only:
|
||||
admin_key = _read_secret("S3_ADMIN_ACCESS_KEY_ID")
|
||||
admin_secret = _read_secret("S3_ADMIN_SECRET_ACCESS_KEY")
|
||||
if not (admin_key and admin_secret):
|
||||
print("S3 admin credentials are not configured", file=sys.stderr)
|
||||
return 2
|
||||
admin = make_client(endpoint, region, admin_key, admin_secret, 2, 10, 1)
|
||||
deadline = time.monotonic() + wait_seconds
|
||||
while True:
|
||||
try:
|
||||
admin.list_buckets()
|
||||
break
|
||||
except (ClientError, BotoCoreError) as exc:
|
||||
if time.monotonic() > deadline:
|
||||
print("S3 is not reachable: %s" % type(exc).__name__, file=sys.stderr)
|
||||
return 1
|
||||
time.sleep(1)
|
||||
try:
|
||||
admin.create_bucket(Bucket=bucket)
|
||||
report["bucket_action"] = "created"
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code")
|
||||
if code in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
|
||||
report["bucket_action"] = "already exists (idempotent no-op)"
|
||||
else:
|
||||
print("create_bucket failed: %s" % code, file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Проба API-ключами.
|
||||
probe_key = "_bootstrap/probe-%s" % uuid.uuid4().hex
|
||||
body = b"probe " + uuid.uuid4().bytes
|
||||
try:
|
||||
api.put_object(Bucket=bucket, Key=probe_key, Body=body)
|
||||
got = api.get_object(Bucket=bucket, Key=probe_key)["Body"].read()
|
||||
check("api credentials: put/get own bucket", got == body)
|
||||
except (ClientError, BotoCoreError) as exc:
|
||||
check("api credentials: put/get own bucket", False, type(exc).__name__)
|
||||
got = None
|
||||
|
||||
status = anonymous_status("%s/%s/%s" % (endpoint, bucket, quote(probe_key)))
|
||||
check("anonymous GET object is denied", status in (401, 403), "HTTP %d" % status)
|
||||
status = anonymous_status("%s/%s?list-type=2" % (endpoint, bucket))
|
||||
check("anonymous list bucket is denied", status in (401, 403), "HTTP %d" % status)
|
||||
status = anonymous_status(endpoint + "/")
|
||||
check("anonymous list buckets is denied", status in (401, 403), "HTTP %d" % status)
|
||||
|
||||
other = "forbidden-%s" % uuid.uuid4().hex[:12]
|
||||
try:
|
||||
api.create_bucket(Bucket=other)
|
||||
check("api credentials cannot create buckets", False, "bucket was created")
|
||||
try:
|
||||
api.delete_bucket(Bucket=other)
|
||||
except (ClientError, BotoCoreError):
|
||||
pass
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code")
|
||||
check("api credentials cannot create buckets", code == "AccessDenied", code)
|
||||
try:
|
||||
api.delete_bucket(Bucket=bucket)
|
||||
check("api credentials cannot delete the bucket", False, "bucket deleted")
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code")
|
||||
check("api credentials cannot delete the bucket", code in ("AccessDenied", "BucketNotEmpty"), code)
|
||||
|
||||
try:
|
||||
api.delete_object(Bucket=bucket, Key=probe_key)
|
||||
except (ClientError, BotoCoreError):
|
||||
pass
|
||||
|
||||
host = urlparse(endpoint).hostname
|
||||
exposed = [p for p in ADMIN_PORTS if port_open(host, p)]
|
||||
check("SeaweedFS admin/UI ports are not reachable on the shared network", not exposed,
|
||||
"open: %s" % exposed if exposed else "checked %s" % list(ADMIN_PORTS))
|
||||
|
||||
if args.sample_from_db:
|
||||
import psycopg
|
||||
dsn = _read_secret("DATABASE_URL")
|
||||
with psycopg.connect(dsn, connect_timeout=5) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT object_key, sha256 FROM attachments WHERE state = 'ready' "
|
||||
"ORDER BY created_at DESC LIMIT 1"
|
||||
).fetchone()
|
||||
if row is None:
|
||||
check("sample ready attachment exists in DB", False, "upload an attachment first (make test)")
|
||||
else:
|
||||
key, sha = row
|
||||
status = anonymous_status("%s/%s/%s" % (endpoint, bucket, quote(key)))
|
||||
check("anonymous GET of a real attachment object is denied", status in (401, 403), "HTTP %d" % status)
|
||||
data = api.get_object(Bucket=bucket, Key=key)["Body"].read()
|
||||
check("same object is readable with API credentials and matches sha256",
|
||||
hashlib.sha256(data).hexdigest() == sha, "%d bytes" % len(data))
|
||||
|
||||
report["ok"] = ok
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Пароли, разбор Basic/Bearer и генерация токенов."""
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
import threading
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
|
||||
|
||||
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
|
||||
# Argon2id (библиотека argon2-cffi). Параметры — рекомендация OWASP для
|
||||
# Argon2id: m=19 MiB, t=2, p=1. Пароль хешируется целиком в UTF-8, без усечения.
|
||||
_hasher = PasswordHasher(time_cost=2, memory_cost=19 * 1024, parallelism=1)
|
||||
|
||||
# Хеш-«пустышка»: проверка для несуществующего пользователя стоит столько же,
|
||||
# сколько для существующего, — время ответа не выдаёт наличие username.
|
||||
_DUMMY_HASH = _hasher.hash("dummy-password-for-timing-equalization")
|
||||
|
||||
# 32 байта случайности = 256 бит энтропии; base64url без padding = 43 символа.
|
||||
TOKEN_BYTES = 32
|
||||
MAX_TOKEN_LENGTH = 256
|
||||
|
||||
|
||||
# lab3: у каждой реплики лимит памяти (deploy.resources.limits). Один хеш Argon2id
|
||||
# занимает 19 MiB, а sync-обработчики FastAPI выполняются в threadpool до 40 потоков.
|
||||
# Без ограничения всплеск входов мог бы занять 40 x 19 MiB и привести к OOM-kill
|
||||
# реплики. Семафор ограничивает число одновременных хешей (PASSWORD_HASH_CONCURRENCY).
|
||||
_kdf_slots = threading.BoundedSemaphore(4)
|
||||
|
||||
|
||||
def configure_kdf_concurrency(limit: int) -> None:
|
||||
global _kdf_slots
|
||||
_kdf_slots = threading.BoundedSemaphore(max(1, limit))
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
with _kdf_slots:
|
||||
return _hasher.hash(password)
|
||||
|
||||
|
||||
def verify_password(password_hash: Optional[str], password: str) -> bool:
|
||||
try:
|
||||
with _kdf_slots:
|
||||
return _hasher.verify(password_hash or _DUMMY_HASH, password) and bool(
|
||||
password_hash
|
||||
)
|
||||
except (VerifyMismatchError, VerificationError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
return secrets.token_urlsafe(TOKEN_BYTES)
|
||||
|
||||
|
||||
def token_digest(token: str) -> bytes:
|
||||
"""Токен высокоэнтропийный, поэтому для поиска достаточно быстрого SHA-256.
|
||||
|
||||
В отличие от пароля, перебор 256-битного токена невозможен, и KDF здесь
|
||||
только замедлил бы каждый запрос.
|
||||
"""
|
||||
return hashlib.sha256(token.encode("ascii")).digest()
|
||||
|
||||
|
||||
def parse_basic(header: Optional[str]) -> Optional[Tuple[str, str]]:
|
||||
"""Authorization: Basic base64(username:password) -> (username, password)."""
|
||||
if not header:
|
||||
return None
|
||||
scheme, _, value = header.strip().partition(" ")
|
||||
if scheme.lower() != "basic" or not value.strip():
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(value.strip(), validate=True)
|
||||
decoded = raw.decode("utf-8")
|
||||
except (binascii.Error, ValueError, UnicodeDecodeError):
|
||||
return None
|
||||
username, sep, password = decoded.partition(":") # двоеточие в пароле допустимо
|
||||
if not sep or not USERNAME_RE.match(username):
|
||||
return None
|
||||
if not 12 <= len(password) <= 128:
|
||||
return None
|
||||
return username, password
|
||||
|
||||
|
||||
def parse_bearer(header: Optional[str]) -> Optional[str]:
|
||||
if not header:
|
||||
return None
|
||||
scheme, _, value = header.strip().partition(" ")
|
||||
token = value.strip()
|
||||
if scheme.lower() != "bearer" or not token or len(token) > MAX_TOKEN_LENGTH:
|
||||
return None
|
||||
if not token.isascii() or any(ch.isspace() for ch in token):
|
||||
return None
|
||||
return token
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Точка входа контейнера: Uvicorn с фазой осушения (drain) перед остановкой.
|
||||
|
||||
Последовательность при SIGTERM (docker stop / rolling update / stop-one):
|
||||
|
||||
t=0 SIGTERM. Реплика ПРОДОЛЖАЕТ принимать и обслуживать запросы,
|
||||
но /health/ready -> 503 и ответы идут с Connection: close.
|
||||
t<=~1.5 с HAProxy (inter 1s, fastinter 500ms, fall 2) помечает реплику DOWN
|
||||
и больше не отправляет ей новые запросы.
|
||||
t=DRAIN_SECONDS (3 с)
|
||||
Uvicorn закрывает listening-сокет, дожидается завершения запросов
|
||||
в обработке (не дольше GRACEFUL_TIMEOUT_SECONDS = 10 с), закрывает
|
||||
пул PostgreSQL и выходит.
|
||||
stop_grace_period 20 с > 3 + 10 — Docker не успевает прислать SIGKILL.
|
||||
|
||||
Повторный SIGTERM/SIGINT во время осушения — немедленный переход к остановке.
|
||||
Запуск: python -m src.server (CMD в Dockerfile, процесс — PID 1).
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
|
||||
import uvicorn
|
||||
|
||||
from .runtime import DRAIN, INSTANCE_ID
|
||||
|
||||
log = logging.getLogger("messenger.server")
|
||||
|
||||
|
||||
class DrainingServer(uvicorn.Server):
|
||||
drain_seconds = 3.0
|
||||
|
||||
def handle_exit(self, sig, frame): # вызывается из обработчика сигнала
|
||||
if self.should_exit or not DRAIN.begin():
|
||||
# Второй сигнал (или уже останавливаемся) — без ожидания.
|
||||
return super().handle_exit(sig, frame)
|
||||
log.info(
|
||||
"signal %s: instance %s draining for %.1fs (readiness=503), then graceful stop",
|
||||
sig, INSTANCE_ID, self.drain_seconds,
|
||||
)
|
||||
timer = threading.Timer(self.drain_seconds, super().handle_exit, args=(sig, frame))
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
config = uvicorn.Config(
|
||||
"src.main:app",
|
||||
host=os.environ.get("APP_HOST", "0.0.0.0"),
|
||||
port=int(os.environ.get("APP_PORT", "8080")),
|
||||
# Дольше, чем HAProxy держит простаивающие соединения к серверу
|
||||
# (pool-purge-delay 5s): соединение закрывает LB, а не сервер под запросом.
|
||||
timeout_keep_alive=int(os.environ.get("KEEP_ALIVE_SECONDS", "30")),
|
||||
timeout_graceful_shutdown=int(os.environ.get("GRACEFUL_TIMEOUT_SECONDS", "10")),
|
||||
access_log=False,
|
||||
server_header=False,
|
||||
log_config=None,
|
||||
)
|
||||
server = DrainingServer(config)
|
||||
server.drain_seconds = float(os.environ.get("DRAIN_SECONDS", "3"))
|
||||
server.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+240
@@ -0,0 +1,240 @@
|
||||
"""Сессии: PostgreSQL — источник истины, Redis — кэш проверок токена.
|
||||
|
||||
Роль Redis
|
||||
----------
|
||||
Каждый защищённый запрос сначала ищет `sess:<sha256(token)>` в Redis.
|
||||
* попадание -> пользователь берётся из кэша, PostgreSQL не трогаем;
|
||||
* промах -> читаем sessions JOIN users в PostgreSQL и кладём результат
|
||||
в Redis через SET NX с TTL = min(SESSION_CACHE_TTL_SECONDS,
|
||||
время до expires_at). Запись кэша никогда не переживает сессию.
|
||||
|
||||
Инвалидация (logout)
|
||||
--------------------
|
||||
1. UPDATE sessions SET revoked_at (COMMIT в PostgreSQL — момент отзыва);
|
||||
2. SET sess:<hash> = "revoked" PX=SESSION_CACHE_TTL (обычный SET, перезаписывает).
|
||||
Заполнение кэша идёт через SET NX, поэтому параллельный запрос, прочитавший
|
||||
сессию до отзыва, не может перезаписать «надгробие» валидной записью.
|
||||
|
||||
Надёжность отзыва при сбое Redis
|
||||
--------------------------------
|
||||
Отзыв в PostgreSQL и строка-«долг» в session_cache_invalidations пишутся
|
||||
ОДНОЙ транзакцией (миграция 002). Строка удаляется только после успешной
|
||||
записи «надгробия» в Redis. Если записать его не удалось (таймаут, обрыв
|
||||
связи, Redis заморожен), на кэш больше нельзя полагаться: в нём могла
|
||||
остаться положительная запись отозванного токена. Поэтому:
|
||||
* пока долг не погашен, положительные попадания кэша НЕ принимаются —
|
||||
каждый токен проверяется в PostgreSQL (отозванный -> 401);
|
||||
* не чаще раза в секунду API пытается погасить долг: пишет надгробия в
|
||||
Redis и удаляет строки; строки старше SESSION_CACHE_TTL + 5 с удаляются
|
||||
без записи (любая положительная запись к этому моменту уже истекла);
|
||||
* наличие долга перечитывается из PostgreSQL не реже раза в секунду, так что
|
||||
и другие реплики (lab3) перестают доверять кэшу не позже чем через 1 с;
|
||||
реплика, выполнившая logout, — сразу, до ответа 204.
|
||||
|
||||
Отказ Redis
|
||||
-----------
|
||||
Любая ошибка Redis (таймаут 250 мс) -> кэш пропускается, проверка идёт в
|
||||
PostgreSQL; на CACHE_RETRY_AFTER_SECONDS Redis выключается (простой
|
||||
circuit breaker), чтобы не платить таймаут на каждом запросе.
|
||||
Redis работает без persistence: после рестарта кэш пуст, и это безопасно —
|
||||
следующие запросы просто заполнят его из PostgreSQL.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import redis
|
||||
|
||||
from .config import Settings
|
||||
from .models import User
|
||||
from .security import new_token, token_digest
|
||||
from .storage import Storage
|
||||
|
||||
log = logging.getLogger("messenger.sessions")
|
||||
|
||||
REVOKED = "revoked"
|
||||
|
||||
|
||||
class SessionCache:
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self.ttl_ms = settings.session_cache_ttl_seconds * 1000
|
||||
self.retry_after = settings.cache_retry_after_seconds
|
||||
self._down_until = 0.0
|
||||
self._lock = threading.Lock()
|
||||
self.client: Optional[redis.Redis] = None
|
||||
if settings.cache_url:
|
||||
self.client = redis.Redis.from_url(
|
||||
settings.cache_url,
|
||||
socket_timeout=settings.cache_timeout_seconds,
|
||||
socket_connect_timeout=settings.cache_timeout_seconds,
|
||||
retry_on_timeout=False,
|
||||
health_check_interval=30,
|
||||
max_connections=20,
|
||||
decode_responses=True,
|
||||
)
|
||||
# Счётчики для наблюдения и тестов (GET /health/cache не нужен контракту,
|
||||
# поэтому они только логируются).
|
||||
self.hits = 0
|
||||
self.misses = 0
|
||||
self.errors = 0
|
||||
|
||||
@staticmethod
|
||||
def key(digest: bytes) -> str:
|
||||
return "sess:" + digest.hex()
|
||||
|
||||
def _available(self) -> bool:
|
||||
return self.client is not None and time.monotonic() >= self._down_until
|
||||
|
||||
def _failed(self, exc: Exception) -> None:
|
||||
with self._lock:
|
||||
self.errors += 1
|
||||
if time.monotonic() >= self._down_until:
|
||||
log.warning("redis unavailable (%s); bypassing cache for %.0fs",
|
||||
type(exc).__name__, self.retry_after)
|
||||
self._down_until = time.monotonic() + self.retry_after
|
||||
|
||||
def get(self, digest: bytes):
|
||||
"""None — промах/кэш недоступен; REVOKED — отозвано; иначе (User, expires_at)."""
|
||||
if not self._available():
|
||||
return None
|
||||
try:
|
||||
raw = self.client.get(self.key(digest))
|
||||
except redis.RedisError as exc:
|
||||
self._failed(exc)
|
||||
return None
|
||||
if raw is None:
|
||||
self.misses += 1
|
||||
return None
|
||||
self.hits += 1
|
||||
if raw == REVOKED:
|
||||
return REVOKED
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
expires_at = datetime.fromisoformat(data["expires_at"])
|
||||
return User(**data["user"]), expires_at
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return None
|
||||
|
||||
def put(self, digest: bytes, user: User, expires_at: datetime) -> None:
|
||||
if not self._available():
|
||||
return
|
||||
remaining_ms = int((expires_at - datetime.now(timezone.utc)).total_seconds() * 1000)
|
||||
ttl = min(self.ttl_ms, remaining_ms)
|
||||
if ttl <= 0:
|
||||
return
|
||||
value = json.dumps({"user": user.model_dump(mode="json"), "expires_at": expires_at.isoformat()})
|
||||
try:
|
||||
self.client.set(self.key(digest), value, px=ttl, nx=True)
|
||||
except redis.RedisError as exc:
|
||||
self._failed(exc)
|
||||
|
||||
def revoke(self, digest: bytes) -> bool:
|
||||
"""True — надгробие записано (или кэша нет вовсе); False — не удалось."""
|
||||
if self.client is None:
|
||||
return True
|
||||
if not self._available():
|
||||
return False
|
||||
try:
|
||||
self.client.set(self.key(digest), REVOKED, px=self.ttl_ms)
|
||||
return True
|
||||
except redis.RedisError as exc:
|
||||
self._failed(exc)
|
||||
return False
|
||||
|
||||
def ping(self) -> bool:
|
||||
if self.client is None:
|
||||
return False
|
||||
try:
|
||||
return bool(self.client.ping())
|
||||
except redis.RedisError:
|
||||
return False
|
||||
|
||||
|
||||
class SessionService:
|
||||
# Как часто перечитывать из PostgreSQL наличие непогашенных инвалидаций.
|
||||
RECHECK_SECONDS = 1.0
|
||||
|
||||
def __init__(self, storage: Storage, cache: SessionCache, settings: Settings) -> None:
|
||||
self.storage = storage
|
||||
self.cache = cache
|
||||
self.ttl = settings.session_ttl_seconds
|
||||
# Строки старше этого возраста можно удалять без записи в Redis.
|
||||
self.stale_after = settings.session_cache_ttl_seconds + 5
|
||||
self._lock = threading.Lock()
|
||||
self._pending = True # до первой сверки с БД кэшу не доверяем
|
||||
self._checked_at = float("-inf")
|
||||
|
||||
def create(self, user: User) -> Tuple[str, datetime]:
|
||||
token = new_token()
|
||||
expires_at = self.storage.create_session(token_digest(token), user.id, self.ttl)
|
||||
return token, expires_at
|
||||
|
||||
def _reconcile(self) -> None:
|
||||
"""Погасить долг инвалидации; вызывается под self._lock."""
|
||||
try:
|
||||
rows = self.storage.pending_invalidations()
|
||||
done = []
|
||||
for digest, age in rows:
|
||||
if age > self.stale_after or self.cache.revoke(digest):
|
||||
done.append(digest)
|
||||
else:
|
||||
break # Redis всё ещё недоступен — попробуем позже
|
||||
self.storage.delete_invalidations(done)
|
||||
self._pending = len(done) < len(rows) or len(rows) >= 500
|
||||
if rows and not self._pending:
|
||||
log.info("cache invalidation backlog cleared (%d)", len(done))
|
||||
except Exception as exc: # PostgreSQL недоступна: доверять кэшу нельзя
|
||||
log.warning("cannot verify cache invalidations: %s", type(exc).__name__)
|
||||
self._pending = True
|
||||
# Пробрасываем сразу (-> 503): иначе запрос ждал бы пул второй раз
|
||||
# в find_session и отвечал за 2 x DB_POOL_TIMEOUT_SECONDS.
|
||||
raise
|
||||
|
||||
def cache_trusted(self) -> bool:
|
||||
now = time.monotonic()
|
||||
if now - self._checked_at < self.RECHECK_SECONDS:
|
||||
return not self._pending
|
||||
with self._lock:
|
||||
if now - self._checked_at >= self.RECHECK_SECONDS:
|
||||
self._reconcile()
|
||||
self._checked_at = time.monotonic()
|
||||
return not self._pending
|
||||
|
||||
def resolve(self, token: str) -> Optional[User]:
|
||||
digest = token_digest(token)
|
||||
# Сначала сверка (она может дописать надгробия в Redis), и только
|
||||
# потом чтение кэша — иначе прочитанная до сверки запись устарела бы.
|
||||
trusted = self.cache_trusted()
|
||||
cached = self.cache.get(digest)
|
||||
if cached == REVOKED:
|
||||
return None
|
||||
if cached is not None and trusted:
|
||||
user, expires_at = cached
|
||||
if expires_at > datetime.now(timezone.utc):
|
||||
return user
|
||||
found = self.storage.find_session(digest) # PostgreSQL недоступна -> исключение -> 503
|
||||
if found is None:
|
||||
return None
|
||||
user, expires_at = found
|
||||
self.cache.put(digest, user, expires_at)
|
||||
return user
|
||||
|
||||
def revoke(self, token: str) -> bool:
|
||||
digest = token_digest(token)
|
||||
if not self.storage.revoke_session(digest): # COMMIT: отзыв + долг инвалидации
|
||||
return False
|
||||
if self.cache.revoke(digest):
|
||||
try:
|
||||
self.storage.delete_invalidations([digest])
|
||||
except Exception:
|
||||
pass # строка останется и будет удалена при следующей сверке
|
||||
else:
|
||||
# Надгробие не записано: до погашения долга кэшу не доверяем.
|
||||
with self._lock:
|
||||
self._pending = True
|
||||
self._checked_at = time.monotonic()
|
||||
log.warning("logout: cache invalidation deferred; positive cache disabled")
|
||||
return True
|
||||
+661
@@ -0,0 +1,661 @@
|
||||
"""Доступ к PostgreSQL. Все запросы параметризованы (%s), SQL не склеивается
|
||||
со значениями пользователя. Каждая публичная операция — одна транзакция
|
||||
(`with pool.connection()` делает COMMIT при успехе и ROLLBACK при исключении).
|
||||
"""
|
||||
from datetime import datetime
|
||||
from typing import List, Optional, Sequence, Tuple
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from psycopg import errors
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
from .models import Attachment, Chat, Message, User, Variant
|
||||
|
||||
|
||||
class UsernameTaken(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UnknownMember(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NotChatMember(Exception):
|
||||
"""Чата нет или пользователь не участник -> 404."""
|
||||
|
||||
|
||||
class IdempotencyConflict(Exception):
|
||||
"""Тот же ключ уже использован с другим телом -> 409."""
|
||||
|
||||
|
||||
class IdempotencyRace(Exception):
|
||||
"""Строка ключа исчезла между шагами -> 503, клиент повторяет с тем же ключом."""
|
||||
|
||||
|
||||
class AttachmentNotFound(Exception):
|
||||
"""Вложение не существует, ещё не принято (pending) или принадлежит другому чату -> 404."""
|
||||
|
||||
|
||||
class AttachmentFailed(Exception):
|
||||
"""lab5: вложение в состоянии failed нельзя прикрепить к сообщению -> 409."""
|
||||
|
||||
|
||||
# Одна реплика за раз выполняет сборку мусора вложений (src/reconcile.py).
|
||||
ATTACHMENT_GC_LOCK_ID = 2026_0004
|
||||
|
||||
ATTACHMENT_COLUMNS = (
|
||||
"a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, "
|
||||
"a.sha256, a.state, a.error_code, a.pipeline_version, a.created_at, a.updated_at"
|
||||
)
|
||||
|
||||
# Внутреннее состояние -> публичный status. pending (приём ещё не завершён) наружу
|
||||
# не попадает: такие строки запросы API не выбирают.
|
||||
PUBLIC_STATES = ("queued", "processing", "ready", "failed")
|
||||
|
||||
# Подзапрос: attachment_ids сообщения в исходном порядке (position).
|
||||
MESSAGE_ATTACHMENT_IDS = (
|
||||
"COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) "
|
||||
"FROM message_attachments ma WHERE ma.message_id = m.id), '{}'::uuid[]) AS attachment_ids"
|
||||
)
|
||||
|
||||
|
||||
def _attachment(row, variants=()) -> Attachment:
|
||||
state = row["state"]
|
||||
return Attachment(
|
||||
id=row["id"],
|
||||
chat_id=row["chat_id"],
|
||||
owner_id=row["owner_id"],
|
||||
filename=row["filename"],
|
||||
content_type=row["content_type"],
|
||||
size_bytes=row["size_bytes"],
|
||||
sha256=row["sha256"],
|
||||
status=state,
|
||||
# Варианты публикуются вместе со статусом ready (одна транзакция worker).
|
||||
variants=list(variants) if state == "ready" else [],
|
||||
error_code=row.get("error_code") if state == "failed" else None,
|
||||
created_at=row["created_at"],
|
||||
updated_at=row["updated_at"],
|
||||
)
|
||||
|
||||
|
||||
# Одна реплика за раз чистит истёкшие ключи идемпотентности.
|
||||
IDEMPOTENCY_CLEANUP_LOCK_ID = 2026_0003
|
||||
|
||||
|
||||
Position = Optional[Tuple[datetime, UUID]]
|
||||
|
||||
|
||||
def _user(row) -> User:
|
||||
return User(
|
||||
id=row["id"],
|
||||
username=row["username"],
|
||||
display_name=row["display_name"],
|
||||
created_at=row["created_at"],
|
||||
)
|
||||
|
||||
|
||||
class Storage:
|
||||
def __init__(self, pool: ConnectionPool) -> None:
|
||||
self.pool = pool
|
||||
|
||||
def ping(self) -> None:
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
|
||||
# users
|
||||
|
||||
def create_user(self, username: str, display_name: str, password_hash: str) -> User:
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.cursor(row_factory=dict_row).execute(
|
||||
"""
|
||||
INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
RETURNING id, username, display_name, created_at
|
||||
""",
|
||||
(uuid4(), username, display_name, password_hash),
|
||||
).fetchone()
|
||||
except errors.UniqueViolation:
|
||||
# Гонку двух одинаковых регистраций решает UNIQUE(username) в БД,
|
||||
# а не проверка «SELECT, потом INSERT» в приложении.
|
||||
raise UsernameTaken()
|
||||
return _user(row)
|
||||
|
||||
def get_credentials(self, username: str) -> Optional[Tuple[User, str]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.cursor(row_factory=dict_row).execute(
|
||||
"""
|
||||
SELECT id, username, display_name, created_at, password_hash
|
||||
FROM users WHERE username = %s
|
||||
""",
|
||||
(username,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return _user(row), row["password_hash"]
|
||||
|
||||
# sessions (источник истины)
|
||||
|
||||
def create_session(self, token_hash: bytes, user_id: UUID, ttl_seconds: int) -> datetime:
|
||||
with self.pool.connection() as conn:
|
||||
# Заодно убираем давно истёкшие сессии этого пользователя.
|
||||
conn.execute(
|
||||
"DELETE FROM sessions WHERE user_id = %s AND expires_at < now() - interval '1 day'",
|
||||
(user_id,),
|
||||
)
|
||||
row = conn.execute(
|
||||
"""
|
||||
INSERT INTO sessions (token_hash, user_id, expires_at)
|
||||
VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s))
|
||||
RETURNING expires_at
|
||||
""",
|
||||
(token_hash, user_id, ttl_seconds),
|
||||
).fetchone()
|
||||
return row[0]
|
||||
|
||||
def find_session(self, token_hash: bytes) -> Optional[Tuple[User, datetime]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.cursor(row_factory=dict_row).execute(
|
||||
"""
|
||||
SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at
|
||||
FROM sessions s JOIN users u ON u.id = s.user_id
|
||||
WHERE s.token_hash = %s
|
||||
AND s.revoked_at IS NULL
|
||||
AND s.expires_at > clock_timestamp()
|
||||
""",
|
||||
(token_hash,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return _user(row), row["expires_at"]
|
||||
|
||||
def revoke_session(self, token_hash: bytes) -> bool:
|
||||
"""Отзыв и долг «инвалидировать кэш» фиксируются одним COMMIT."""
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"""
|
||||
UPDATE sessions SET revoked_at = clock_timestamp()
|
||||
WHERE token_hash = %s
|
||||
AND revoked_at IS NULL
|
||||
AND expires_at > clock_timestamp()
|
||||
RETURNING 1
|
||||
""",
|
||||
(token_hash,),
|
||||
).fetchone()
|
||||
if row is not None:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO session_cache_invalidations (token_hash)
|
||||
VALUES (%s) ON CONFLICT (token_hash) DO NOTHING
|
||||
""",
|
||||
(token_hash,),
|
||||
)
|
||||
return row is not None
|
||||
|
||||
# долг инвалидации кэша
|
||||
|
||||
def has_pending_invalidations(self) -> bool:
|
||||
with self.pool.connection() as conn:
|
||||
return conn.execute(
|
||||
"SELECT EXISTS (SELECT 1 FROM session_cache_invalidations)"
|
||||
).fetchone()[0]
|
||||
|
||||
def pending_invalidations(self, limit: int = 500) -> List[Tuple[bytes, float]]:
|
||||
"""[(token_hash, возраст в секундах)] — самые старые первыми."""
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT token_hash, EXTRACT(EPOCH FROM clock_timestamp() - revoked_at)
|
||||
FROM session_cache_invalidations
|
||||
ORDER BY revoked_at LIMIT %s
|
||||
""",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
return [(bytes(h), float(age)) for h, age in rows]
|
||||
|
||||
def delete_invalidations(self, token_hashes: Sequence[bytes]) -> None:
|
||||
if not token_hashes:
|
||||
return
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM session_cache_invalidations WHERE token_hash = ANY(%s)",
|
||||
(list(token_hashes),),
|
||||
)
|
||||
|
||||
# chats
|
||||
|
||||
def _insert_members(self, conn, chat_id: UUID, member_ids: Sequence[UUID]) -> None:
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)",
|
||||
[(chat_id, member_id, pos) for pos, member_id in enumerate(member_ids)],
|
||||
)
|
||||
|
||||
def create_chat(self, title: str, member_ids: Sequence[UUID]) -> Chat:
|
||||
"""Чат и все участники создаются в одной транзакции.
|
||||
|
||||
Существование участников проверяет внешний ключ chat_members.user_id:
|
||||
если хоть один id неизвестен, INSERT падает, и откатывается вся
|
||||
транзакция, включая уже вставленную строку chats и других участников.
|
||||
"""
|
||||
chat_id = uuid4()
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
created_at = conn.execute(
|
||||
"INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING created_at",
|
||||
(chat_id, title),
|
||||
).fetchone()[0]
|
||||
self._insert_members(conn, chat_id, member_ids)
|
||||
except errors.ForeignKeyViolation:
|
||||
raise UnknownMember()
|
||||
return Chat(id=chat_id, title=title, member_ids=list(member_ids), created_at=created_at)
|
||||
|
||||
def _members_for(self, conn, chat_ids: List[UUID]) -> dict:
|
||||
members = {chat_id: [] for chat_id in chat_ids}
|
||||
if not chat_ids:
|
||||
return members
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT chat_id, user_id FROM chat_members
|
||||
WHERE chat_id = ANY(%s) ORDER BY chat_id, position
|
||||
""",
|
||||
(chat_ids,),
|
||||
).fetchall()
|
||||
for chat_id, user_id in rows:
|
||||
members[chat_id].append(user_id)
|
||||
return members
|
||||
|
||||
def get_chat_for_member(self, chat_id: UUID, user_id: UUID) -> Optional[Chat]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.cursor(row_factory=dict_row).execute(
|
||||
"""
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s
|
||||
WHERE c.id = %s
|
||||
""",
|
||||
(user_id, chat_id),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
members = self._members_for(conn, [row["id"]])
|
||||
return Chat(id=row["id"], title=row["title"], member_ids=members[row["id"]], created_at=row["created_at"])
|
||||
|
||||
def list_user_chats(self, user_id: UUID, limit: int, after: Position) -> Tuple[List[Chat], bool]:
|
||||
params: list = [user_id]
|
||||
keyset = ""
|
||||
if after is not None:
|
||||
keyset = "AND (c.created_at, c.id) > (%s, %s)"
|
||||
params.extend(after)
|
||||
params.append(limit + 1)
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.cursor(row_factory=dict_row).execute(
|
||||
"""
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chat_members m JOIN chats c ON c.id = m.chat_id
|
||||
WHERE m.user_id = %s {keyset}
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT %s
|
||||
""".format(keyset=keyset),
|
||||
params,
|
||||
).fetchall()
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
members = self._members_for(conn, [r["id"] for r in rows])
|
||||
chats = [
|
||||
Chat(id=r["id"], title=r["title"], member_ids=members[r["id"]], created_at=r["created_at"])
|
||||
for r in rows
|
||||
]
|
||||
return chats, has_more
|
||||
|
||||
# messages
|
||||
|
||||
@staticmethod
|
||||
def _insert_message(conn, message_id: UUID, chat_id: UUID, sender_id: UUID, text: str) -> Optional[Message]:
|
||||
"""Проверка членства и вставка — один оператор, без окна гонки."""
|
||||
row = conn.cursor(row_factory=dict_row).execute(
|
||||
"""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text)
|
||||
SELECT %s, %s, %s, %s
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s
|
||||
)
|
||||
RETURNING id, chat_id, sender_id, text, created_at
|
||||
""",
|
||||
(message_id, chat_id, sender_id, text, chat_id, sender_id),
|
||||
).fetchone()
|
||||
return Message(**row) if row is not None else None
|
||||
|
||||
@staticmethod
|
||||
def _link_attachments(conn, message: Message, attachment_ids: Sequence[UUID]) -> None:
|
||||
"""Привязка вложений в той же транзакции, что и сообщение.
|
||||
|
||||
lab5: можно ссылаться на queued/processing/ready вложения ЭТОГО чата.
|
||||
Неизвестное/чужое/ещё не принятое (pending) -> AttachmentNotFound (404);
|
||||
failed -> AttachmentFailed (409). Любое исключение -> ROLLBACK сообщения и
|
||||
ключа идемпотентности. FOR KEY SHARE не даёт удалить строку до COMMIT;
|
||||
составной FK (attachment_id, chat_id) дублирует проверку чата в БД.
|
||||
"""
|
||||
if not attachment_ids:
|
||||
return
|
||||
ids = list(attachment_ids)
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, state FROM attachments
|
||||
WHERE id = ANY(%s) AND chat_id = %s AND state <> 'pending'
|
||||
FOR KEY SHARE
|
||||
""",
|
||||
(ids, message.chat_id),
|
||||
).fetchall()
|
||||
if len(rows) != len(ids):
|
||||
raise AttachmentNotFound()
|
||||
if any(state == "failed" for _id, state in rows):
|
||||
raise AttachmentFailed()
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"""
|
||||
INSERT INTO message_attachments (message_id, attachment_id, chat_id, position)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
""",
|
||||
[(message.id, aid, message.chat_id, pos) for pos, aid in enumerate(ids)],
|
||||
)
|
||||
message.attachment_ids = ids
|
||||
|
||||
def create_message(
|
||||
self,
|
||||
chat_id: UUID,
|
||||
sender_id: UUID,
|
||||
text: str,
|
||||
idem_key: str,
|
||||
request_hash: bytes,
|
||||
idem_ttl_seconds: int,
|
||||
attachment_ids: Sequence[UUID] = (),
|
||||
) -> Tuple[Message, bool]:
|
||||
"""Идемпотентная отправка. Возвращает (сообщение, replayed).
|
||||
|
||||
Всё — одна транзакция READ COMMITTED (см. migrations/003):
|
||||
1. INSERT строки ключа. Если такой ключ уже есть и он не истёк,
|
||||
ON CONFLICT ... WHERE ничего не меняет и RETURNING пуст. Если
|
||||
конкурентная транзакция с тем же ключом ещё не завершилась, INSERT
|
||||
ЖДЁТ её COMMIT/ROLLBACK на уникальном индексе — поэтому одновременные
|
||||
повторы через разные реплики создают ровно одно сообщение.
|
||||
2a. Ключ наш -> INSERT сообщения (с проверкой членства). Не участник ->
|
||||
исключение -> ROLLBACK: ключ не «сгорает» на 404.
|
||||
2b. Ключ чужой (уже подтверждён) -> снова проверяем доступ, сравниваем
|
||||
hash тела: совпал -> то же сообщение; нет -> 409.
|
||||
Ошибка/обрыв на любом шаге откатывает и ключ, и сообщение.
|
||||
"""
|
||||
new_id = uuid4()
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
owned = conn.execute(
|
||||
"""
|
||||
INSERT INTO idempotency_keys
|
||||
(user_id, method, chat_id, idem_key, request_hash, message_id, expires_at)
|
||||
VALUES (%s, 'POST', %s, %s, %s, %s,
|
||||
clock_timestamp() + make_interval(secs => %s))
|
||||
ON CONFLICT (user_id, method, chat_id, idem_key) DO UPDATE
|
||||
SET request_hash = EXCLUDED.request_hash,
|
||||
message_id = EXCLUDED.message_id,
|
||||
created_at = clock_timestamp(),
|
||||
expires_at = EXCLUDED.expires_at
|
||||
-- истёкший (ещё не удалённый очисткой) ключ можно занять заново
|
||||
WHERE idempotency_keys.expires_at <= clock_timestamp()
|
||||
RETURNING message_id
|
||||
""",
|
||||
(sender_id, chat_id, idem_key, request_hash, new_id, idem_ttl_seconds),
|
||||
).fetchone()
|
||||
if owned is not None:
|
||||
message = self._insert_message(conn, new_id, chat_id, sender_id, text)
|
||||
if message is None:
|
||||
raise NotChatMember() # -> ROLLBACK вместе со строкой ключа
|
||||
# lab4: чужое/неготовое вложение -> исключение -> ROLLBACK всего.
|
||||
self._link_attachments(conn, message, attachment_ids)
|
||||
return message, False
|
||||
|
||||
# Ключ уже подтверждён другой (возможно, конкурентной) транзакцией.
|
||||
member = conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, sender_id),
|
||||
).fetchone()
|
||||
if member is None:
|
||||
raise NotChatMember()
|
||||
row = conn.cursor(row_factory=dict_row).execute(
|
||||
"""
|
||||
SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
{attachments}
|
||||
FROM idempotency_keys k
|
||||
JOIN messages m ON m.id = k.message_id
|
||||
WHERE k.user_id = %s AND k.method = 'POST'
|
||||
AND k.chat_id = %s AND k.idem_key = %s
|
||||
""".format(attachments=MESSAGE_ATTACHMENT_IDS),
|
||||
(sender_id, chat_id, idem_key),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
# Теоретически: строка удалена между шагами (очистка/каскад).
|
||||
# Отдаём 503-подобную транзиентную ошибку, клиент повторит с тем же ключом.
|
||||
raise IdempotencyRace()
|
||||
if bytes(row.pop("request_hash")) != request_hash:
|
||||
raise IdempotencyConflict()
|
||||
return Message(**row), True
|
||||
|
||||
def delete_expired_idempotency_keys(self, batch: int = 1000) -> Optional[int]:
|
||||
"""Удаляет истёкшие ключи пачкой. None — очистку сейчас выполняет другая реплика."""
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
got = conn.execute(
|
||||
"SELECT pg_try_advisory_xact_lock(%s)", (IDEMPOTENCY_CLEANUP_LOCK_ID,)
|
||||
).fetchone()[0]
|
||||
if not got:
|
||||
return None
|
||||
cur = conn.execute(
|
||||
"""
|
||||
DELETE FROM idempotency_keys
|
||||
WHERE ctid IN (
|
||||
SELECT ctid FROM idempotency_keys
|
||||
WHERE expires_at <= clock_timestamp()
|
||||
LIMIT %s
|
||||
)
|
||||
""",
|
||||
(batch,),
|
||||
)
|
||||
return cur.rowcount
|
||||
|
||||
def list_messages(
|
||||
self, chat_id: UUID, user_id: UUID, limit: int, after: Position
|
||||
) -> Optional[Tuple[List[Message], bool]]:
|
||||
params: list = [chat_id]
|
||||
keyset = ""
|
||||
if after is not None:
|
||||
keyset = "AND (created_at, id) > (%s, %s)"
|
||||
params.extend(after)
|
||||
params.append(limit + 1)
|
||||
with self.pool.connection() as conn:
|
||||
member = conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, user_id),
|
||||
).fetchone()
|
||||
if member is None:
|
||||
return None
|
||||
# Keyset-пагинация по индексу messages_chat_created_id_idx:
|
||||
# Index Scan сразу с позиции курсора, читается limit+1 строк.
|
||||
rows = conn.cursor(row_factory=dict_row).execute(
|
||||
"""
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
{attachments}
|
||||
FROM messages m
|
||||
WHERE m.chat_id = %s {keyset}
|
||||
ORDER BY m.created_at, m.id
|
||||
LIMIT %s
|
||||
""".format(keyset=keyset.replace("(created_at, id)", "(m.created_at, m.id)"),
|
||||
attachments=MESSAGE_ATTACHMENT_IDS),
|
||||
params,
|
||||
).fetchall()
|
||||
has_more = len(rows) > limit
|
||||
return [Message(**r) for r in rows[:limit]], has_more
|
||||
|
||||
# attachments (lab4)
|
||||
|
||||
def is_chat_member(self, chat_id: UUID, user_id: UUID) -> bool:
|
||||
with self.pool.connection() as conn:
|
||||
return conn.execute(
|
||||
"SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)",
|
||||
(chat_id, user_id),
|
||||
).fetchone()[0]
|
||||
|
||||
def create_pending_attachment(
|
||||
self,
|
||||
attachment_id: UUID,
|
||||
chat_id: UUID,
|
||||
owner_id: UUID,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
size_bytes: int,
|
||||
sha256: str,
|
||||
width: int,
|
||||
height: int,
|
||||
object_key: str,
|
||||
staging_key: Optional[str] = None,
|
||||
pipeline_version: int = 1,
|
||||
) -> bool:
|
||||
"""Шаг 1 из 3: намерение (state=pending) COMMIT до S3 PUT в staging.
|
||||
|
||||
У любого объекта, который API кладёт в бакет, уже есть строка с его ключом
|
||||
(staging_key): объект без строки может появиться только после удаления
|
||||
строки сборщиком и никогда не «потеряется» навсегда (src/reconcile.py).
|
||||
object_key — будущий конечный ключ оригинала, его запишет worker.
|
||||
Проверка членства и вставка — один оператор. False -> не участник.
|
||||
"""
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
INSERT INTO attachments (id, chat_id, owner_id, filename, content_type,
|
||||
size_bytes, sha256, width, height, object_key,
|
||||
staging_key, pipeline_version, state)
|
||||
SELECT %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending'
|
||||
WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)
|
||||
RETURNING 1
|
||||
""",
|
||||
(attachment_id, chat_id, owner_id, filename, content_type, size_bytes,
|
||||
sha256, width, height, object_key, staging_key, pipeline_version, chat_id, owner_id),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
def accept_attachment(
|
||||
self,
|
||||
attachment_id: UUID,
|
||||
job_id: UUID,
|
||||
payload: dict,
|
||||
max_attempts: int,
|
||||
outbox_delay_seconds: float,
|
||||
) -> Optional[Tuple[Attachment, int]]:
|
||||
"""Шаг 3 из 3 (lab5): после успешного PUT в staging — ОДНА транзакция:
|
||||
pending -> queued, запись задания и строки outbox. COMMIT => можно 202.
|
||||
|
||||
Возвращает (вложение, outbox_id) или None, если строку уже забрал сборщик.
|
||||
"""
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.cursor(row_factory=dict_row).execute(
|
||||
"""
|
||||
UPDATE attachments a SET state = 'queued', updated_at = clock_timestamp()
|
||||
WHERE a.id = %s AND a.state = 'pending'
|
||||
RETURNING {cols}
|
||||
""".format(cols=ATTACHMENT_COLUMNS),
|
||||
(attachment_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status,
|
||||
attempt, max_attempts)
|
||||
VALUES (%s, %s, %s, 'queued', 1, %s)
|
||||
""",
|
||||
(job_id, attachment_id, row["pipeline_version"], max_attempts),
|
||||
)
|
||||
outbox_id = conn.execute(
|
||||
"""
|
||||
INSERT INTO job_outbox (job_id, payload, available_at)
|
||||
VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s))
|
||||
RETURNING id
|
||||
""",
|
||||
(job_id, Jsonb(payload), outbox_delay_seconds),
|
||||
).fetchone()[0]
|
||||
conn.execute(
|
||||
"INSERT INTO job_events (job_id, attachment_id, attempt, event, actor) "
|
||||
"VALUES (%s, %s, 1, 'accepted', 'api')",
|
||||
(job_id, attachment_id),
|
||||
)
|
||||
return _attachment(row), outbox_id
|
||||
|
||||
def mark_attachment_ready(self, attachment_id: UUID) -> Optional[Attachment]:
|
||||
"""lab4-совместимость (синхронная загрузка без worker). В lab5 не используется."""
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.cursor(row_factory=dict_row).execute(
|
||||
"""
|
||||
UPDATE attachments a SET state = 'ready', updated_at = clock_timestamp()
|
||||
WHERE a.id = %s AND a.state = 'pending'
|
||||
RETURNING {cols}
|
||||
""".format(cols=ATTACHMENT_COLUMNS),
|
||||
(attachment_id,),
|
||||
).fetchone()
|
||||
return _attachment(row) if row is not None else None
|
||||
|
||||
def delete_pending_attachment(self, attachment_id: UUID) -> bool:
|
||||
"""Компенсация при ошибке S3 PUT: удалить только своё незавершённое намерение."""
|
||||
with self.pool.connection() as conn:
|
||||
cur = conn.execute(
|
||||
"DELETE FROM attachments WHERE id = %s AND state = 'pending'",
|
||||
(attachment_id,),
|
||||
)
|
||||
return cur.rowcount == 1
|
||||
|
||||
@staticmethod
|
||||
def _variants(conn, attachment_id: UUID, pipeline_version: int) -> List[Variant]:
|
||||
rows = conn.cursor(row_factory=dict_row).execute(
|
||||
"""
|
||||
SELECT name, content_type, size_bytes, width, height, sha256
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = %s AND pipeline_version = %s
|
||||
ORDER BY CASE name WHEN 'thumbnail' THEN 1 WHEN 'crop' THEN 2 ELSE 3 END
|
||||
""",
|
||||
(attachment_id, pipeline_version),
|
||||
).fetchall()
|
||||
return [Variant(**r) for r in rows]
|
||||
|
||||
def get_attachment_for_member(self, attachment_id: UUID, user_id: UUID) -> Optional[Tuple[Attachment, str, int]]:
|
||||
"""(метаданные, object_key оригинала, pipeline_version) принятого вложения, если user — участник чата.
|
||||
|
||||
pending (приём не завершён) — как несуществующее. object_key читаем только в ready.
|
||||
"""
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.cursor(row_factory=dict_row).execute(
|
||||
"""
|
||||
SELECT {cols}, a.object_key
|
||||
FROM attachments a
|
||||
JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s
|
||||
WHERE a.id = %s AND a.state <> 'pending'
|
||||
""".format(cols=ATTACHMENT_COLUMNS),
|
||||
(user_id, attachment_id),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
variants = self._variants(conn, attachment_id, row["pipeline_version"]) if row["state"] == "ready" else []
|
||||
return _attachment(row, variants), row["object_key"], row["pipeline_version"]
|
||||
|
||||
def variant_object(self, attachment_id: UUID, pipeline_version: int, name: str) -> Optional[Tuple[str, str, int, str]]:
|
||||
"""(object_key, content_type, size_bytes, sha256) варианта или None."""
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT object_key, content_type, size_bytes, sha256 FROM attachment_variants
|
||||
WHERE attachment_id = %s AND pipeline_version = %s AND name = %s
|
||||
""",
|
||||
(attachment_id, pipeline_version, name),
|
||||
).fetchone()
|
||||
return tuple(row) if row else None
|
||||
@@ -0,0 +1,397 @@
|
||||
"""lab6: OpenTelemetry для API и worker — traces, metrics, logs по OTLP в Collector.
|
||||
|
||||
Что настраивается в setup():
|
||||
* Resource: service.name (messenger-api / messenger-worker), service.version,
|
||||
service.instance.id (= X-Instance-Id), deployment.environment.name.
|
||||
* Traces: BatchSpanProcessor (очередь ограничена OTEL_BSP_MAX_QUEUE_SIZE, при
|
||||
переполнении новые spans отбрасываются, процесс не растёт). Sampler отбрасывает
|
||||
«сиротские» CLIENT-spans без родителя: это SQL/S3 фоновых циклов (outbox
|
||||
sweeper, GC, readiness, callbacks метрик) — иначе в Tempo тысячи пустых трасс.
|
||||
Всё, что происходит внутри HTTP server / producer / consumer span, сохраняется.
|
||||
* Metrics: PeriodicExportingMetricReader (10 с). Корзины гистограмм заданы
|
||||
явно в секундах (View), лишние метрики размера тела отброшены.
|
||||
* Logs: JSON в stdout (для docker logs) + OTLP LoggingHandler. В каждой записи
|
||||
timestamp, severity, service.name, event, trace_id/span_id (если есть span).
|
||||
* Автоинструментация: psycopg (SQL без параметров), botocore (S3).
|
||||
Redis НЕ инструментируется: команды AUTH/кэш сессий могли бы утечь в spans.
|
||||
|
||||
Что никогда не экспортируется: тела сообщений, байты изображений, заголовки
|
||||
Authorization/Cookie (FastAPI-инструментация заголовки не пишет без явной
|
||||
настройки), параметры SQL, URL с подписью. Метрики не имеют меток user/chat/message.
|
||||
Отключение: OTEL_SDK_DISABLED=true (остаются только JSON-логи в stdout).
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import resource
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Callable, Dict, Iterable, Optional
|
||||
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry.context import Context
|
||||
from opentelemetry.metrics import CallbackOptions, Observation
|
||||
from opentelemetry.trace import SpanKind
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
|
||||
from .runtime import INSTANCE_ID
|
||||
|
||||
log = logging.getLogger("messenger.telemetry")
|
||||
|
||||
# Только W3C traceparent/tracestate: baggage в сообщение брокера не попадает
|
||||
# (jobs.schema.json: additionalProperties=false).
|
||||
PROPAGATOR = TraceContextTextMapPropagator()
|
||||
|
||||
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0)
|
||||
JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0)
|
||||
READY_BUCKETS = (0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 900.0)
|
||||
|
||||
_STATE = {"service": "messenger", "enabled": False, "configured": False}
|
||||
|
||||
|
||||
def enabled() -> bool:
|
||||
return _STATE["enabled"]
|
||||
|
||||
|
||||
def tracer():
|
||||
return trace.get_tracer("messenger")
|
||||
|
||||
|
||||
def meter():
|
||||
return metrics.get_meter("messenger")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ logging
|
||||
|
||||
_STD_ATTRS = set(logging.LogRecord("", 0, "", 0, "", (), None).__dict__) | {"message", "asctime", "event"}
|
||||
|
||||
|
||||
class _EventFilter(logging.Filter):
|
||||
"""У каждой записи есть поле event: явное (extra={"event": ...}) или имя логгера."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
if not getattr(record, "event", None):
|
||||
record.event = record.name
|
||||
return True
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def __init__(self, compact: bool = False) -> None:
|
||||
super().__init__()
|
||||
self.compact = compact
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
out = {
|
||||
"timestamp": datetime.fromtimestamp(record.created, timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
|
||||
"severity": record.levelname,
|
||||
"service.name": _STATE["service"],
|
||||
"service.instance.id": INSTANCE_ID,
|
||||
"event": getattr(record, "event", record.name),
|
||||
"logger": record.name,
|
||||
"message": record.getMessage(),
|
||||
}
|
||||
ctx = trace.get_current_span().get_span_context()
|
||||
if ctx.is_valid:
|
||||
out["trace_id"] = format(ctx.trace_id, "032x")
|
||||
out["span_id"] = format(ctx.span_id, "016x")
|
||||
for key, value in record.__dict__.items():
|
||||
if key not in _STD_ATTRS and not key.startswith("_") and key not in out:
|
||||
out[key] = value if isinstance(value, (str, int, float, bool)) or value is None else str(value)
|
||||
if record.exc_info:
|
||||
if self.compact:
|
||||
out["error"] = type(record.exc_info[1]).__name__
|
||||
else:
|
||||
out["exception"] = self.formatException(record.exc_info)
|
||||
return json.dumps(out, ensure_ascii=False)
|
||||
|
||||
|
||||
def _configure_logging(otel_handler: Optional[logging.Handler]) -> None:
|
||||
root = logging.getLogger()
|
||||
for h in list(root.handlers):
|
||||
root.removeHandler(h)
|
||||
stdout = logging.StreamHandler(sys.stdout)
|
||||
stdout.setFormatter(JsonFormatter())
|
||||
stdout.addFilter(_EventFilter())
|
||||
root.addHandler(stdout)
|
||||
if otel_handler is not None:
|
||||
otel_handler.addFilter(_EventFilter())
|
||||
root.addHandler(otel_handler)
|
||||
root.setLevel(logging.INFO)
|
||||
# Шумные библиотеки: их DEBUG/INFO не нужны ни в stdout, ни в Loki.
|
||||
for name in ("botocore", "boto3", "urllib3"):
|
||||
logging.getLogger(name).setLevel(logging.WARNING)
|
||||
# Ошибки самого SDK (Collector недоступен): только в stdout, одной строкой,
|
||||
# без трейсбека и НЕ в OTLP — иначе петля «лог об ошибке экспорта логов».
|
||||
otel = logging.getLogger("opentelemetry")
|
||||
for h in list(otel.handlers):
|
||||
otel.removeHandler(h)
|
||||
sdk_handler = logging.StreamHandler(sys.stdout)
|
||||
sdk_handler.setFormatter(JsonFormatter(compact=True))
|
||||
sdk_handler.addFilter(_EventFilter())
|
||||
otel.addHandler(sdk_handler)
|
||||
otel.propagate = False
|
||||
otel.setLevel(logging.WARNING)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ setup
|
||||
|
||||
def _sampler():
|
||||
from opentelemetry.sdk.trace.sampling import ALWAYS_ON, Decision, ParentBased, Sampler, SamplingResult
|
||||
|
||||
class DropOrphanClientSpans(Sampler):
|
||||
"""CLIENT-span без родителя (фоновый SQL/S3) не сэмплируется; остальное — ParentBased(AlwaysOn)."""
|
||||
|
||||
_base = ParentBased(ALWAYS_ON)
|
||||
|
||||
def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None,
|
||||
trace_state=None):
|
||||
parent = trace.get_current_span(parent_context).get_span_context()
|
||||
if not parent.is_valid and kind == SpanKind.CLIENT:
|
||||
return SamplingResult(Decision.DROP)
|
||||
return self._base.should_sample(parent_context, trace_id, name, kind, attributes, links, trace_state)
|
||||
|
||||
def get_description(self) -> str:
|
||||
return "DropOrphanClientSpans{ParentBased{AlwaysOn}}"
|
||||
|
||||
return DropOrphanClientSpans()
|
||||
|
||||
|
||||
def setup(service_name: str) -> None:
|
||||
"""Вызывать один раз в процессе ДО создания пула БД и клиента S3."""
|
||||
if _STATE["configured"]:
|
||||
return
|
||||
_STATE["configured"] = True
|
||||
_STATE["service"] = service_name
|
||||
if os.environ.get("OTEL_SDK_DISABLED", "false").strip().lower() == "true":
|
||||
_configure_logging(None)
|
||||
log.info("OpenTelemetry disabled (OTEL_SDK_DISABLED=true)", extra={"event": "telemetry.disabled"})
|
||||
return
|
||||
# Стабильные HTTP semconv: http.server.request.duration в секундах,
|
||||
# атрибуты http.request.method / http.route / http.response.status_code.
|
||||
os.environ.setdefault("OTEL_SEMCONV_STABILITY_OPT_IN", "http")
|
||||
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import DropAggregation, ExplicitBucketHistogramAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
|
||||
# Resource.create добавляет OTEL_RESOURCE_ATTRIBUTES из окружения.
|
||||
res = Resource.create({
|
||||
"service.name": service_name,
|
||||
"service.version": os.environ.get("APP_VERSION", "lab6"),
|
||||
"service.instance.id": INSTANCE_ID,
|
||||
"deployment.environment.name": os.environ.get("APP_ENV", "lab"),
|
||||
})
|
||||
|
||||
# Размеры очередей/таймауты — OTEL_BSP_* / OTEL_BLRP_* / OTEL_EXPORTER_OTLP_TIMEOUT из окружения.
|
||||
tp = TracerProvider(resource=res, sampler=_sampler())
|
||||
tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
|
||||
trace.set_tracer_provider(tp)
|
||||
|
||||
views = [
|
||||
View(instrument_name="http.server.request.duration",
|
||||
aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)),
|
||||
View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
|
||||
View(instrument_name="messenger.attachment.time_to_ready",
|
||||
aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)),
|
||||
View(instrument_name="http.server.request.body.size", aggregation=DropAggregation()),
|
||||
View(instrument_name="http.server.response.body.size", aggregation=DropAggregation()),
|
||||
]
|
||||
interval = int(os.environ.get("OTEL_METRIC_EXPORT_INTERVAL", "10000"))
|
||||
reader = PeriodicExportingMetricReader(OTLPMetricExporter(), export_interval_millis=interval)
|
||||
metrics.set_meter_provider(MeterProvider(resource=res, metric_readers=[reader], views=views))
|
||||
|
||||
lp = LoggerProvider(resource=res)
|
||||
lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
|
||||
set_logger_provider(lp)
|
||||
_configure_logging(LoggingHandler(level=logging.INFO, logger_provider=lp))
|
||||
|
||||
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
|
||||
from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor
|
||||
|
||||
PsycopgInstrumentor().instrument(skip_dep_check=True)
|
||||
BotocoreInstrumentor().instrument(skip_dep_check=True)
|
||||
_register_process_metrics()
|
||||
_STATE["enabled"] = True
|
||||
log.info("OpenTelemetry enabled: endpoint=%s", os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "default"),
|
||||
extra={"event": "telemetry.enabled"})
|
||||
|
||||
|
||||
def shutdown() -> None:
|
||||
"""Сбросить буферы при штатной остановке (ограничено таймаутами экспортёров)."""
|
||||
if not _STATE["enabled"]:
|
||||
return
|
||||
from opentelemetry._logs import get_logger_provider
|
||||
|
||||
for provider in (trace.get_tracer_provider(), metrics.get_meter_provider(), get_logger_provider()):
|
||||
try:
|
||||
provider.shutdown()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def instrument_fastapi(app) -> None:
|
||||
if not _STATE["enabled"]:
|
||||
return
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||
|
||||
# Health-пробы (HAProxy каждую секунду) не входят ни в трассы, ни в метрики
|
||||
# HTTP: доля ошибок считается по бизнес-запросам.
|
||||
FastAPIInstrumentor.instrument_app(app, excluded_urls="/health/live,/health/ready",
|
||||
exclude_spans=["receive", "send"])
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ messaging context
|
||||
|
||||
def inject(carrier: Dict) -> Dict:
|
||||
"""Записать traceparent/tracestate текущего span в словарь (сообщение задания)."""
|
||||
PROPAGATOR.inject(carrier)
|
||||
return carrier
|
||||
|
||||
|
||||
def extract(carrier: Optional[Dict]) -> Context:
|
||||
if not isinstance(carrier, dict):
|
||||
return Context()
|
||||
return PROPAGATOR.extract({k: v for k, v in carrier.items() if k in ("traceparent", "tracestate")
|
||||
and isinstance(v, str)})
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ app metrics
|
||||
|
||||
class _Lazy:
|
||||
"""Инструменты создаются при первом обращении (после setup())."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._made = False
|
||||
|
||||
def _make(self) -> None:
|
||||
if self._made:
|
||||
return
|
||||
m = meter()
|
||||
self.jobs_processed = m.create_counter(
|
||||
"messenger.jobs.processed", unit="{job}",
|
||||
description="Обработанные сообщения заданий по исходу (succeeded/retry/failed/noop/...)")
|
||||
self.job_duration = m.create_histogram(
|
||||
"messenger.job.duration", unit="s", description="Длительность обработки задания worker'ом")
|
||||
self.time_to_ready = m.create_histogram(
|
||||
"messenger.attachment.time_to_ready", unit="s",
|
||||
description="От durable acceptance (COMMIT задания, ответ 202) до публикации ready")
|
||||
self.published = m.create_counter(
|
||||
"messenger.jobs.published", unit="{job}", description="Сообщения, опубликованные outbox relay в брокер")
|
||||
self._made = True
|
||||
|
||||
def __getattr__(self, item):
|
||||
if item.startswith("_"):
|
||||
raise AttributeError(item)
|
||||
self._make()
|
||||
return self.__dict__[item]
|
||||
|
||||
|
||||
INSTRUMENTS = _Lazy()
|
||||
|
||||
|
||||
def _safe(callback: Callable[[], Iterable[Observation]]) -> Callable[[CallbackOptions], Iterable[Observation]]:
|
||||
last = {"t": 0.0}
|
||||
|
||||
def wrapped(_options: CallbackOptions) -> Iterable[Observation]:
|
||||
try:
|
||||
return list(callback())
|
||||
except Exception as exc: # БД недоступна: точку пропускаем, а не роняем экспорт
|
||||
if time.monotonic() - last["t"] > 60: # не чаще раза в минуту
|
||||
last["t"] = time.monotonic()
|
||||
log.warning("metric callback skipped: %s", type(exc).__name__,
|
||||
extra={"event": "telemetry.callback_failed"})
|
||||
return []
|
||||
return wrapped
|
||||
|
||||
|
||||
def _register_process_metrics() -> None:
|
||||
m = meter()
|
||||
page = os.sysconf("SC_PAGE_SIZE")
|
||||
|
||||
def rss():
|
||||
with open("/proc/self/statm") as fh:
|
||||
yield Observation(int(fh.read().split()[1]) * page)
|
||||
|
||||
def cpu():
|
||||
r = resource.getrusage(resource.RUSAGE_SELF)
|
||||
c = resource.getrusage(resource.RUSAGE_CHILDREN) # подпроцессы обработки изображений
|
||||
yield Observation(r.ru_utime + r.ru_stime + c.ru_utime + c.ru_stime)
|
||||
|
||||
m.create_observable_gauge("process.memory.rss", [_safe(rss)], unit="By", description="RSS процесса")
|
||||
m.create_observable_counter("process.cpu.time", [_safe(cpu)], unit="s",
|
||||
description="CPU-время процесса и его подпроцессов")
|
||||
|
||||
|
||||
def register_pool_metrics(pool) -> None:
|
||||
"""DB pool: занято/свободно/ожидающих/максимум (psycopg_pool.get_stats)."""
|
||||
m = meter()
|
||||
|
||||
def conns():
|
||||
st = pool.get_stats()
|
||||
size, idle = st.get("pool_size", 0), st.get("pool_available", 0)
|
||||
yield Observation(size - idle, {"state": "used"})
|
||||
yield Observation(idle, {"state": "idle"})
|
||||
|
||||
m.create_observable_gauge("messenger.db.pool.connections", [_safe(conns)], unit="{connection}")
|
||||
m.create_observable_gauge("messenger.db.pool.waiting",
|
||||
[_safe(lambda: [Observation(pool.get_stats().get("requests_waiting", 0))])],
|
||||
unit="{request}", description="Запросы, ждущие свободное соединение")
|
||||
m.create_observable_gauge("messenger.db.pool.max", [_safe(lambda: [Observation(pool.max_size)])],
|
||||
unit="{connection}")
|
||||
|
||||
|
||||
def register_queue_metrics(pool, queue: str) -> None:
|
||||
"""Глубина очереди и возраст старейшего задания — из PostgreSQL (источник истины).
|
||||
|
||||
Публикуется API-репликами: метрика должна жить, когда worker остановлен
|
||||
(именно в этот момент она важнее всего). Несколько реплик дают одинаковые
|
||||
значения — в PromQL берётся max().
|
||||
"""
|
||||
m = meter()
|
||||
cache = {"t": 0.0, "row": None}
|
||||
|
||||
def row():
|
||||
if time.monotonic() - cache["t"] > 5 or cache["row"] is None:
|
||||
with pool.connection(timeout=1) as conn:
|
||||
cache["row"] = conn.execute(
|
||||
"""
|
||||
SELECT count(*) FILTER (WHERE status = 'queued'),
|
||||
count(*) FILTER (WHERE status = 'processing'),
|
||||
COALESCE(EXTRACT(EPOCH FROM clock_timestamp() - min(created_at)), 0),
|
||||
(SELECT count(*) FROM job_outbox)
|
||||
FROM attachment_jobs WHERE status IN ('queued', 'processing')
|
||||
"""
|
||||
).fetchone()
|
||||
cache["t"] = time.monotonic()
|
||||
return cache["row"]
|
||||
|
||||
attrs = {"queue": queue}
|
||||
m.create_observable_gauge(
|
||||
"messenger.queue.depth",
|
||||
[_safe(lambda: [Observation(row()[0], dict(attrs, state="queued")),
|
||||
Observation(row()[1], dict(attrs, state="processing"))])],
|
||||
unit="{job}", description="Незавершённые задания")
|
||||
m.create_observable_gauge(
|
||||
"messenger.queue.oldest_age", [_safe(lambda: [Observation(float(row()[2]), attrs)])],
|
||||
unit="s", description="Возраст старейшего незавершённого задания")
|
||||
m.create_observable_gauge(
|
||||
"messenger.outbox.pending", [_safe(lambda: [Observation(row()[3], attrs)])],
|
||||
unit="{job}", description="Строки outbox, ещё не опубликованные в брокер")
|
||||
|
||||
|
||||
def register_worker_metrics(busy: Callable[[], int], capacity: int) -> None:
|
||||
m = meter()
|
||||
m.create_observable_gauge("messenger.worker.busy", [_safe(lambda: [Observation(busy())])], unit="{slot}",
|
||||
description="Занятые consumer-потоки")
|
||||
m.create_observable_gauge("messenger.worker.capacity", [_safe(lambda: [Observation(capacity)])],
|
||||
unit="{slot}", description="WORKER_CONCURRENCY")
|
||||
+417
@@ -0,0 +1,417 @@
|
||||
"""Worker обработки вложений (lab5): python -m src.worker
|
||||
|
||||
Отдельный процесс/контейнер. WORKER_CONCURRENCY потоков; у каждого свой consumer
|
||||
в группе Redis Streams и prefetch = 1 (XREADGROUP COUNT 1), поэтому одновременно
|
||||
в работе не больше WORKER_CONCURRENCY сообщений на процесс.
|
||||
|
||||
Обработка одного сообщения:
|
||||
1. Проверка схемы (contracts/jobs.schema.json). Мусор -> ack + лог (poison).
|
||||
2. claim(): аренда в PostgreSQL (lease_token, JOB_TIMEOUT_SECONDS). Нельзя
|
||||
захватить (готово/упало/устарело/арендовано живым) -> no-op, ack.
|
||||
3. Источник: staging-объект (или уже опубликованный оригинал при повторе);
|
||||
ключи берутся из БД, не из сообщения. Размер <= MAX_UPLOAD_BYTES, SHA-256
|
||||
должен совпасть с метаданными.
|
||||
4. Оригинал -> final/<id>/original (если там ещё нет того же SHA-256).
|
||||
5. Варианты в изолированном подпроцессе (src/imaging.py) -> детерминированные
|
||||
ключи final/<id>/p<версия>/<вариант>.<ext>: повтор перезаписывает те же байты.
|
||||
6. complete(): ОДНА транзакция, fenced по lease_token: варианты + ready.
|
||||
7. XACK — только после COMMIT (consumer ack после устойчивого эффекта).
|
||||
8. Удаление staging (не удалось — уберёт сборщик, src/reconcile.py, проход D).
|
||||
|
||||
Ошибки: временные (S3/БД недоступны, таймаут обработки) -> retry через outbox с
|
||||
backoff+jitter, не больше JOB_MAX_ATTEMPTS; постоянные (битое изображение, лимит
|
||||
памяти, нет источника) -> failed + dead_letters сразу. Если недоступна сама БД,
|
||||
сообщение НЕ подтверждается и вернётся через PEL.
|
||||
|
||||
SIGTERM: потоки перестают брать новые сообщения и дорабатывают текущее
|
||||
(stop_grace_period в compose больше JOB_TIMEOUT_SECONDS).
|
||||
"""
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import psycopg
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from psycopg_pool import PoolTimeout
|
||||
|
||||
from . import telemetry
|
||||
from .broker import Broker, BrokerUnavailable, Delivery
|
||||
from .config import load_settings
|
||||
from .db import create_pool
|
||||
from .imaging import VARIANT_EXT, ImageFailure, ImageTimeout, Sandbox, sha256_hex
|
||||
from .jobs import JOB_TYPE, Claim, JobStore, validate_payload
|
||||
from .objectstore import ObjectMissing, ObjectStore, ObjectStoreUnavailable
|
||||
from .outbox import Relay
|
||||
from .reconcile import gc_loop
|
||||
from .runtime import INSTANCE_ID
|
||||
|
||||
log = logging.getLogger("messenger.worker")
|
||||
|
||||
HEARTBEAT = Path("/tmp/worker-heartbeat")
|
||||
|
||||
|
||||
class Transient(Exception):
|
||||
def __init__(self, code: str) -> None:
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
class Permanent(Exception):
|
||||
def __init__(self, code: str, detail: str = "") -> None:
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
self.detail = detail
|
||||
|
||||
|
||||
class InjectedCrash(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def final_original_key(attachment_id) -> str:
|
||||
return "final/%s/original" % attachment_id
|
||||
|
||||
|
||||
def variant_key(attachment_id, pipeline_version: int, name: str) -> str:
|
||||
return "final/%s/p%d/%s.%s" % (attachment_id, pipeline_version, name, VARIANT_EXT[name])
|
||||
|
||||
|
||||
class Worker:
|
||||
def __init__(self) -> None:
|
||||
self.settings = load_settings()
|
||||
s = self.settings
|
||||
if not s.broker_url:
|
||||
raise RuntimeError("BROKER_URL or BROKER_URL_FILE must be set")
|
||||
if not ObjectStore.configured(s):
|
||||
raise RuntimeError("S3 must be configured for the worker")
|
||||
self.pool = create_pool(s)
|
||||
self.jobs = JobStore(self.pool, s)
|
||||
self.store = ObjectStore(s)
|
||||
self.sandbox = Sandbox(s.worker_image_memory_mb, s.worker_image_timeout_seconds, s.max_image_pixels)
|
||||
self.stop = threading.Event()
|
||||
self.in_flight = 0
|
||||
self.lock = threading.Lock()
|
||||
self.processed = 0
|
||||
self.beats: Dict[int, float] = {}
|
||||
# lab6: DB pool, занятость worker'а (busy/capacity)
|
||||
telemetry.register_pool_metrics(self.pool)
|
||||
telemetry.register_worker_metrics(lambda: self.in_flight, s.worker_concurrency)
|
||||
|
||||
# ---------------------------------------------------------------- loop
|
||||
|
||||
def consumer_name(self, index: int) -> str:
|
||||
# Стабильно при рестарте контейнера (hostname = короткий ID контейнера):
|
||||
# новая инкарнация сразу видит свой неподтверждённый PEL.
|
||||
return "%s-%d" % (INSTANCE_ID, index)
|
||||
|
||||
def run_thread(self, index: int) -> None:
|
||||
name = self.consumer_name(index)
|
||||
broker = Broker(self.settings.broker_url, self.settings.broker_stream, self.settings.broker_group,
|
||||
self.settings.broker_timeout_seconds, block_ms=2000)
|
||||
recovered_own = False
|
||||
last_autoclaim = 0.0
|
||||
while not self.stop.is_set():
|
||||
self.beats[index] = time.monotonic()
|
||||
try:
|
||||
deliveries: List[Delivery] = []
|
||||
if not recovered_own:
|
||||
deliveries = broker.read(name, own_pending=True)
|
||||
if not deliveries:
|
||||
recovered_own = True
|
||||
else:
|
||||
log.info("%s: recovering own unacked message %s", name, deliveries[0].message_id)
|
||||
if not deliveries and time.monotonic() - last_autoclaim > 2:
|
||||
last_autoclaim = time.monotonic()
|
||||
deliveries = broker.autoclaim(name, self.settings.job_claim_idle_seconds * 1000)
|
||||
for d in deliveries:
|
||||
log.info("%s: claimed idle message %s (deliveries=%d)", name, d.message_id, d.deliveries)
|
||||
if not deliveries:
|
||||
deliveries = broker.read(name)
|
||||
for d in deliveries:
|
||||
self.handle(broker, name, d)
|
||||
except BrokerUnavailable:
|
||||
log.warning("%s: broker unavailable, retrying", name)
|
||||
self.stop.wait(1.0)
|
||||
except (psycopg.Error, PoolTimeout) as exc:
|
||||
# БД недоступна до/после обработки: сообщение не подтверждено, вернётся.
|
||||
log.warning("%s: database unavailable (%s); message stays pending", name, type(exc).__name__)
|
||||
self.stop.wait(1.0)
|
||||
except Exception:
|
||||
log.exception("%s: unexpected error in loop", name)
|
||||
self.stop.wait(1.0)
|
||||
broker.close()
|
||||
log.info("%s: stopped", name)
|
||||
|
||||
# ---------------------------------------------------------------- one message
|
||||
|
||||
def handle(self, broker: Broker, consumer: str, d: Delivery) -> None:
|
||||
"""lab6: CONSUMER span — ребёнок PRODUCER span из traceparent сообщения.
|
||||
Внутри него — SQL (psycopg) и S3 (botocore) spans этой попытки; retry
|
||||
пишет в outbox traceparent этого span, следующая попытка — в той же трассе."""
|
||||
raw = d.payload()
|
||||
started = time.monotonic()
|
||||
with telemetry.tracer().start_as_current_span(
|
||||
"%s process" % broker.stream, context=telemetry.extract(raw), kind=SpanKind.CONSUMER,
|
||||
attributes={
|
||||
"messaging.system": "redis",
|
||||
"messaging.operation.type": "process",
|
||||
"messaging.destination.name": broker.stream,
|
||||
"messaging.consumer.group.name": broker.group,
|
||||
"messaging.message.id": d.message_id,
|
||||
"messaging.message.delivery_count": d.deliveries,
|
||||
"messenger.job.type": JOB_TYPE,
|
||||
},
|
||||
) as span:
|
||||
outcome = "error"
|
||||
try:
|
||||
outcome = self._handle(broker, consumer, d, raw, span)
|
||||
finally:
|
||||
kind = outcome.split(":", 1)[0].split(" ", 1)[0]
|
||||
span.set_attribute("messenger.job.outcome", kind)
|
||||
if kind in ("failed", "poison", "error"):
|
||||
span.set_status(Status(StatusCode.ERROR, outcome))
|
||||
attrs = {"job.type": JOB_TYPE, "outcome": kind}
|
||||
telemetry.INSTRUMENTS.jobs_processed.add(1, attrs)
|
||||
if kind not in ("noop", "poison"):
|
||||
telemetry.INSTRUMENTS.job_duration.record(time.monotonic() - started, attrs)
|
||||
log.info("%s: message %s -> %s in %.2fs", consumer, d.message_id, outcome,
|
||||
time.monotonic() - started,
|
||||
extra={"event": "job.processed", "outcome": kind, "job_type": JOB_TYPE,
|
||||
"duration_s": round(time.monotonic() - started, 3)})
|
||||
|
||||
def _handle(self, broker: Broker, consumer: str, d: Delivery, raw, span) -> str:
|
||||
msg = validate_payload(raw)
|
||||
if msg is None:
|
||||
log.error("%s: poison message %s (invalid schema), acking", consumer, d.message_id)
|
||||
broker.ack(d.message_id)
|
||||
return "poison"
|
||||
span.set_attribute("messenger.job.id", str(msg["job_id"]))
|
||||
span.set_attribute("messenger.attachment.id", str(msg["attachment_id"]))
|
||||
span.set_attribute("messenger.job.attempt", msg["attempt"])
|
||||
if d.deliveries > self.settings.job_max_deliveries:
|
||||
# Сообщение многократно «убивало» worker (или его не могли подтвердить).
|
||||
self.jobs.fail_poison(msg["job_id"], "poison_message", consumer)
|
||||
log.error("%s: job %s delivered %d times, marked failed", consumer, msg["job_id"], d.deliveries)
|
||||
broker.ack(d.message_id)
|
||||
return "failed:poison_message"
|
||||
claim, reason = self.jobs.claim(msg, consumer)
|
||||
if claim is None:
|
||||
log.info("%s: job %s attempt %d: no-op (%s), ack", consumer, msg["job_id"], msg["attempt"], reason)
|
||||
broker.ack(d.message_id)
|
||||
return "noop:" + reason
|
||||
with self.lock:
|
||||
self.in_flight += 1
|
||||
started = time.monotonic()
|
||||
try:
|
||||
outcome = self.process(claim, consumer)
|
||||
finally:
|
||||
with self.lock:
|
||||
self.in_flight -= 1
|
||||
# Эффект (ready/retry/failed) уже зафиксирован в БД -> теперь можно ack.
|
||||
if outcome == "ready" and self.settings.worker_faults and self._marker(claim) == "crash-before-ack" \
|
||||
and not self.jobs.has_event(claim.job_id, "fault_injected"):
|
||||
self.jobs.event(claim.job_id, claim.attachment_id, claim.attempt, "fault_injected",
|
||||
"crash after COMMIT before XACK", consumer)
|
||||
log.error("FAULT_INJECTION: worker exits after COMMIT before XACK (job %s)", claim.job_id)
|
||||
os._exit(70)
|
||||
broker.ack(d.message_id)
|
||||
self.processed += 1
|
||||
log.info("%s: job %s attempt %d -> %s in %.2fs", consumer, claim.job_id, claim.attempt, outcome,
|
||||
time.monotonic() - started)
|
||||
if outcome == "ready":
|
||||
self.cleanup_staging(claim)
|
||||
return "succeeded"
|
||||
return outcome
|
||||
|
||||
def process(self, claim: Claim, consumer: str) -> str:
|
||||
try:
|
||||
variants, original_key = self.produce(claim, consumer)
|
||||
age = self.jobs.complete(claim, variants, original_key, consumer)
|
||||
if age is not None:
|
||||
telemetry.INSTRUMENTS.time_to_ready.record(age, {"job.type": JOB_TYPE})
|
||||
else:
|
||||
log.warning("%s: lease lost for job %s; result not published (fenced)", consumer, claim.job_id)
|
||||
self.jobs.event(claim.job_id, claim.attachment_id, claim.attempt, "lease_lost",
|
||||
"result not published: lease token no longer valid", consumer)
|
||||
return "lease_lost"
|
||||
return "ready"
|
||||
except Permanent as exc:
|
||||
self.jobs.fail(claim, exc.code, consumer, exc.detail or None)
|
||||
return "failed:" + exc.code
|
||||
except Transient as exc:
|
||||
result, delay = self.jobs.retry_or_fail(claim, exc.code, consumer)
|
||||
return "retry in %.1fs (%s)" % (delay, exc.code) if result == "retry" else result
|
||||
|
||||
def _marker(self, claim: Claim) -> str:
|
||||
"""Маркеры внедрения отказов в filename — ТОЛЬКО при WORKER_FAULTS=true
|
||||
(compose.faults.yaml). В обычном стенде filename ни на что не влияет."""
|
||||
if not self.settings.worker_faults:
|
||||
return ""
|
||||
src = self.jobs.attachment_source(claim.attachment_id) or {}
|
||||
name = src.get("filename") or ""
|
||||
for marker in ("crash-after-effects", "crash-before-ack", "transient-until-3",
|
||||
"transient-until-replay", "permanent", "slow"):
|
||||
if name.startswith("fault-" + marker):
|
||||
return marker
|
||||
return ""
|
||||
|
||||
def produce(self, claim: Claim, consumer: str):
|
||||
"""Шаги 3–5: прочитать источник, записать оригинал и варианты в final/."""
|
||||
src = self.jobs.attachment_source(claim.attachment_id)
|
||||
if src is None:
|
||||
raise Permanent("processing_failed", "attachment row disappeared")
|
||||
marker = self._marker(claim)
|
||||
if marker == "transient-until-3" and claim.attempt < 3:
|
||||
raise Transient("injected_transient")
|
||||
if marker == "transient-until-replay" and claim.replay_count == 0:
|
||||
raise Transient("injected_transient")
|
||||
if marker == "permanent":
|
||||
raise Permanent("invalid_image", "injected permanent failure")
|
||||
if marker == "slow" and claim.attempt == 1:
|
||||
# «Зависание» заметно дольше аренды + grace + период сборщика (5 с).
|
||||
time.sleep(self.settings.job_timeout_seconds + 20)
|
||||
|
||||
original_key = final_original_key(claim.attachment_id)
|
||||
data = self.read_source(src, original_key)
|
||||
try:
|
||||
# 4. Оригинал в конечный префикс (неизменен: те же байты, тот же SHA-256).
|
||||
try:
|
||||
head = self.store.head(original_key)
|
||||
except ObjectStoreUnavailable:
|
||||
raise Transient("storage_unavailable")
|
||||
if head is None or (head.get("Metadata") or {}).get("sha256") != src["sha256"]:
|
||||
self._put(original_key, data, src["content_type"])
|
||||
# 5. Варианты.
|
||||
try:
|
||||
rendered = self.sandbox.render(data)
|
||||
log.info("job %s: variants rendered, subprocess peak RSS %.0f MiB",
|
||||
claim.job_id, self.sandbox.last_peak_mb)
|
||||
except ImageTimeout:
|
||||
raise Transient("processing_timeout")
|
||||
except ImageFailure as exc:
|
||||
raise Permanent(exc.code if exc.code in ("invalid_image", "resource_limit") else "processing_failed",
|
||||
str(exc))
|
||||
finally:
|
||||
del data
|
||||
variants: List[Dict] = []
|
||||
for name in ("thumbnail", "crop", "optimized"):
|
||||
body, width, height, ctype = rendered[name]
|
||||
key = variant_key(claim.attachment_id, claim.pipeline_version, name)
|
||||
self._put(key, body, ctype)
|
||||
variants.append({"name": name, "content_type": ctype, "size_bytes": len(body), "width": width,
|
||||
"height": height, "sha256": sha256_hex(body), "object_key": key})
|
||||
if marker == "crash-after-effects" and not self.jobs.has_event(claim.job_id, "fault_injected"):
|
||||
self.jobs.event(claim.job_id, claim.attachment_id, claim.attempt, "fault_injected",
|
||||
"crash after S3 writes before COMMIT", consumer)
|
||||
log.error("FAULT_INJECTION: worker exits after S3 writes before COMMIT (job %s)", claim.job_id)
|
||||
os._exit(70)
|
||||
return variants, original_key
|
||||
|
||||
def read_source(self, src: Dict, original_key: str) -> bytes:
|
||||
limit = self.settings.max_upload_bytes
|
||||
keys = [k for k in (src["staging_key"], original_key) if k]
|
||||
for key in keys:
|
||||
try:
|
||||
data = self.store.read_all(key, limit)
|
||||
except ObjectMissing:
|
||||
continue
|
||||
except ValueError:
|
||||
raise Permanent("invalid_image", "source exceeds MAX_UPLOAD_BYTES")
|
||||
except ObjectStoreUnavailable:
|
||||
raise Transient("storage_unavailable")
|
||||
if len(data) != src["size_bytes"] or sha256_hex(data) != src["sha256"]:
|
||||
if key == original_key:
|
||||
continue # частично/чужой объект в final — перезапишем из staging
|
||||
raise Permanent("processing_failed", "staging checksum mismatch")
|
||||
return data
|
||||
raise Permanent("processing_failed", "source object is missing")
|
||||
|
||||
def _put(self, key: str, data: bytes, content_type: str) -> None:
|
||||
try:
|
||||
self.store.put(key, data, content_type, hashlib.md5(data, usedforsecurity=False).digest(),
|
||||
sha256_hex(data))
|
||||
except ObjectStoreUnavailable:
|
||||
raise Transient("storage_unavailable")
|
||||
|
||||
def cleanup_staging(self, claim: Claim) -> None:
|
||||
"""Шаг 8: staging больше не нужен — результат опубликован и подтверждён."""
|
||||
try:
|
||||
src = self.jobs.attachment_source(claim.attachment_id)
|
||||
if not src or src["state"] != "ready" or not src["staging_key"]:
|
||||
return
|
||||
self.store.delete(src["staging_key"])
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s AND state = 'ready' "
|
||||
"AND staging_key = %s", (claim.attachment_id, src["staging_key"]))
|
||||
except Exception as exc:
|
||||
log.warning("staging cleanup deferred to gc: %s", type(exc).__name__)
|
||||
|
||||
# ---------------------------------------------------------------- housekeeping
|
||||
|
||||
def housekeeping(self) -> None:
|
||||
broker = Broker(self.settings.broker_url, self.settings.broker_stream, self.settings.broker_group,
|
||||
self.settings.broker_timeout_seconds)
|
||||
while not self.stop.wait(30):
|
||||
try:
|
||||
trimmed = broker.trim_acknowledged()
|
||||
if trimmed:
|
||||
log.info("broker: trimmed %d acknowledged entries", trimmed)
|
||||
except BrokerUnavailable:
|
||||
pass
|
||||
broker.close()
|
||||
|
||||
def run(self) -> None:
|
||||
s = self.settings
|
||||
log.info("worker %s: concurrency=%d max_attempts=%d lease=%ss claim_idle=%ss image_timeout=%ss "
|
||||
"image_memory=%dMiB retry=%.1f..%.1fs faults=%s",
|
||||
INSTANCE_ID, s.worker_concurrency, s.job_max_attempts, s.job_timeout_seconds,
|
||||
s.job_claim_idle_seconds, s.worker_image_timeout_seconds, s.worker_image_memory_mb,
|
||||
s.job_retry_base_seconds, s.job_retry_max_seconds, "on" if s.worker_faults else "off")
|
||||
# Прогрев forkserver до приёма сообщений.
|
||||
self.sandbox.ctx.Process(target=time.sleep, args=(0,)).start()
|
||||
HEARTBEAT.touch()
|
||||
|
||||
def on_signal(sig, _frame):
|
||||
log.info("signal %s: finishing in-flight jobs (%d) and stopping", sig, self.in_flight)
|
||||
self.stop.set()
|
||||
|
||||
signal.signal(signal.SIGTERM, on_signal)
|
||||
signal.signal(signal.SIGINT, on_signal)
|
||||
|
||||
threads = [threading.Thread(target=self.run_thread, args=(i,), name="consumer-%d" % i)
|
||||
for i in range(s.worker_concurrency)]
|
||||
aux_stop = threading.Event()
|
||||
relay_broker = Broker(s.broker_url, s.broker_stream, s.broker_group, s.broker_timeout_seconds)
|
||||
relay = Relay(self.pool, relay_broker, s, "worker:" + INSTANCE_ID)
|
||||
aux = [threading.Thread(target=relay.loop, args=(aux_stop,), name="outbox-relay", daemon=True),
|
||||
threading.Thread(target=self.housekeeping, name="housekeeping", daemon=True),
|
||||
threading.Thread(target=gc_loop, args=(self.pool, self.store, s, aux_stop), name="attachment-gc",
|
||||
daemon=True)]
|
||||
for t in threads + aux:
|
||||
t.start()
|
||||
# Heartbeat для healthcheck: пишется, только пока КАЖДЫЙ consumer-поток
|
||||
# проходит цикл (зависание дольше аренды -> контейнер unhealthy).
|
||||
limit = s.job_timeout_seconds + 15
|
||||
while any(t.is_alive() for t in threads):
|
||||
now = time.monotonic()
|
||||
if self.beats and all(now - b < limit for b in self.beats.values()):
|
||||
HEARTBEAT.touch()
|
||||
for t in threads:
|
||||
t.join(timeout=2)
|
||||
aux_stop.set()
|
||||
self.pool.close()
|
||||
log.info("worker %s stopped; processed %d messages", INSTANCE_ID, self.processed)
|
||||
telemetry.shutdown()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
telemetry.setup("messenger-worker")
|
||||
Worker().run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user