Files
mtusi2026lab7/app/main.py
T
alenchha 861e7722b5 ЛР 7 — БВТ2301 — Курлова — на 4
Перенесён минимум lab6 (API без изменений), добавлена защита на границах доверия:
- внешний HTTPS; обязательный mTLS LB -> API с allowlist identity и отзывом по serial;
- TLS на всех связях: PostgreSQL verify-full, MinIO HTTPS, RabbitMQ AMQPS mTLS (EXTERNAL),
  Valkey TLS + ACL, Collector и хранилища телеметрии через mTLS с отдельными CA;
- OpenBao в server mode: cert auth, токены 15 мин, политики только на свой путь;
- роли БД, S3-политики и права брокера по принципу минимальных прав;
- Argon2id с замером, rehash, общий лимит входа, бюджет защиты от login DoS;
- hardening всех контейнеров, scan образов с разбором, модель угроз, матрица доверия;
- отрицательные тесты: TLS-аудит (135 проверок), ACL, ротации, sealed OpenBao.
2026-11-16 10:49:13 +03:00

802 lines
34 KiB
Python
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import asyncio
import base64
import binascii
import hashlib
import json
import logging
import os
import re
import resource
import signal
import tempfile
import unicodedata
import sys
import uuid
from contextlib import asynccontextmanager
from typing import List, Optional, Tuple
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse, Response
from starlette.concurrency import iterate_in_threadpool, run_in_threadpool
from starlette.responses import StreamingResponse
from app import config, faults, jobs, openbao, passwords, reconcile, repo, telemetry
from app.cursor import BadCursor, decode_cursor, encode_cursor
from app.db import Database
from app.imagecheck import inspect_image
from app.errors import ApiError, bad, error_body, not_found, unauthorized, unavailable
from app.broker import Publisher
from app.outbox import Relay
from app.s3 import S3Storage
from app.sessions import SessionStore
log = logging.getLogger("messenger")
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
ZERO_WIDTH = str.maketrans({char: None for char in "​‌‍⁠"})
EXEMPT_PATHS = ("/health/live", "/health/ready")
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096
ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"}
PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
JPEG_MAGIC = b"\xff\xd8\xff"
upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY)
background = set()
db = Database()
sessions = SessionStore(db)
s3 = S3Storage()
publisher = Publisher()
relay = Relay(db, publisher)
state = {"draining": False, "ready": True, "s3_ok": False, "inflight": 0, "rejected": {}, "pending_jobs": 0,
"queue": None}
VARIANTS = ("original", "thumbnail", "crop", "optimized")
def rss_mb() -> float:
try:
with open("/proc/self/statm") as handle:
return int(handle.read().split()[1]) * PAGE_SIZE / 1048576
except OSError:
peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
return peak / 1048576 if sys.platform == "darwin" else peak / 1024
def count_rejection(code: str) -> None:
state["rejected"][code] = state["rejected"].get(code, 0) + 1
async def sweeper() -> None:
while True:
await asyncio.sleep(config.SESSION_SWEEP_SECONDS)
try:
removed = await sessions.sweep()
if removed:
log.info("sessions_expired removed=%d", removed)
except Exception:
log.warning("session_sweep_skipped: storage unavailable")
try:
async with db.connection() as conn:
removed = await repo.cleanup_idempotency(conn, config.IDEMPOTENCY_TTL_SECONDS)
if removed:
log.info("idempotency_cleanup removed=%d", removed)
except Exception:
log.warning("idempotency_cleanup_skipped")
try:
removed = await relay.cleanup()
if removed:
log.info("outbox_cleanup removed=%d", removed)
except Exception:
log.warning("outbox_cleanup_skipped")
async def pending_watch() -> None:
"""Admission control input and queue metrics (depth, oldest job age, unpublished outbox), once per second."""
while True:
try:
async with db.connection() as conn:
queue = await jobs.queue_stats(conn)
state["pending_jobs"] = queue["queued"] + queue["processing"]
state["queue"] = queue
except Exception:
state["queue"] = None # unknown is not zero: the gauge disappears instead of lying
await asyncio.sleep(config.PENDING_CHECK_SECONDS)
async def reconciler_loop() -> None:
await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS)
while True:
try:
result = await reconcile.run(db, s3, config.RECONCILE_GRACE_SECONDS, dry_run=False)
if result.get("skipped") or any(v for k, v in result.items() if k.startswith(("deleted", "marked", "removed"))):
log.info("reconcile %s", result)
except Exception:
log.warning("reconcile_skipped: dependency unavailable")
await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS)
async def readiness_watch() -> None:
failures = 0
while True:
if await db.ready():
failures = 0
if not state["ready"]:
state["ready"] = True
log.info("ready_again: database reachable")
else:
failures += 1
if failures >= config.READY_FAILURES and state["ready"]:
state["ready"] = False
log.warning("not_ready: database unreachable, business requests get 503")
s3_ok = await run_in_threadpool(s3.probe)
if s3_ok != state["s3_ok"]:
state["s3_ok"] = s3_ok
log.info("object_storage %s", "reachable" if s3_ok else "UNREACHABLE: uploads/downloads get 503, /health/ready 503")
await asyncio.sleep(config.READY_CHECK_SECONDS)
@asynccontextmanager
async def lifespan(_app: FastAPI):
previous = {}
def on_signal(signum, frame):
if not state["draining"]:
state["draining"] = True
log.info("draining signal=%s: new requests get 503", signal.Signals(signum).name)
handler = previous.get(signum)
if callable(handler):
handler(signum, frame)
for sig in (signal.SIGTERM, signal.SIGINT):
previous[sig] = signal.getsignal(sig)
signal.signal(sig, on_signal)
await db.open()
telemetry.register_pool(db)
telemetry.register_queue(state)
await sessions.open()
await run_in_threadpool(s3.open)
state["s3_ok"] = await run_in_threadpool(s3.probe)
tasks = [asyncio.create_task(sweeper()), asyncio.create_task(readiness_watch()), asyncio.create_task(reconciler_loop()),
asyncio.create_task(relay.loop()), asyncio.create_task(pending_watch())]
health_server = await asyncio.start_server(health_listener, "127.0.0.1", config.HEALTH_LISTEN_PORT)
bao = openbao.client()
if bao is not None:
# Rotation without restart: a new KV version swaps DB and S3 credentials for new connections.
bao.on_change(lambda data: db.rotate(data.get("database_url", "")))
bao.on_change(s3.rotate)
bao.start_refresher(config.BAO_REFRESH_SECONDS)
log.info(
"started: instance=%s session_store=%s session_ttl=%d cache_ttl=%d pool=%d..%d max_inflight=%d max_body=%d idem_ttl=%d s3=%s bucket=%s",
config.INSTANCE_ID, config.SESSION_STORE, config.SESSION_TTL_SECONDS, config.SESSION_CACHE_TTL_SECONDS,
config.DB_POOL_MIN, config.DB_POOL_MAX, config.MAX_INFLIGHT_REQUESTS, config.MAX_BODY_BYTES,
config.IDEMPOTENCY_TTL_SECONDS, config.S3_ENDPOINT, config.S3_BUCKET,
)
log.info("pipeline_config pipeline_version=%d max_attempts=%d max_pending_jobs=%d fault_injection=%s",
config.PIPELINE_VERSION, config.JOB_MAX_ATTEMPTS, config.MAX_PENDING_JOBS, config.FAULT_INJECTION)
yield
for task in tasks:
task.cancel()
health_server.close()
await publisher.close()
await sessions.close()
await db.close()
log.info("stopped: cache=%s rejected=%s", sessions.stats, state["rejected"])
telemetry.shutdown()
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
@app.middleware("http")
async def guard(request: Request, call_next):
rid = uuid.uuid4().hex
request.state.rid = rid
# X-Request-Id <-> trace: the id is a span attribute and is present in every log line of the request.
telemetry.set_attributes(messenger__request_id=rid, messenger__instance_id=config.INSTANCE_ID)
exempt = request.scope["path"] in EXEMPT_PATHS
if state["draining"] and request.scope["path"] != "/health/live":
count_rejection("draining")
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Сервер завершает работу", {"Retry-After": "1", "Connection": "close"})
if not exempt and not state["ready"]:
count_rejection("not_ready")
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Нет связи с базой данных", {"Retry-After": "1"})
if not exempt and state["inflight"] >= config.MAX_INFLIGHT_REQUESTS:
count_rejection("inflight")
return error_body(rid, config.INSTANCE_ID, 429, "rate_limited", "Слишком много одновременных запросов", {"Retry-After": "1"})
state["inflight"] += 1
try:
fault = request.headers.get("x-fault", "") if config.FAULT_INJECTION else ""
if fault:
await inject_http_fault(fault)
response = await asyncio.wait_for(call_next(request), config.REQUEST_TIMEOUT)
except asyncio.TimeoutError:
count_rejection("request_timeout")
log.warning("request_timeout request_id=%s method=%s path=%s", rid, request.method, request.scope["path"])
response = error_body(
rid, config.INSTANCE_ID, 503, "unavailable",
"Запрос не удалось обработать за отведённое время", {"Retry-After": "1"},
)
except Exception as exc:
telemetry.record_exception(exc)
log.exception("unhandled_error request_id=%s", rid)
response = error_body(rid, config.INSTANCE_ID, 500, "internal_error", "Внутренняя ошибка сервера")
finally:
state["inflight"] -= 1
response.headers["X-Request-Id"] = rid
response.headers["X-Instance-Id"] = config.INSTANCE_ID
return response
async def inject_http_fault(fault: str) -> None:
"""Test stand only (FAULT_INJECTION=1): `X-Fault: slow` adds 600 ms inside the handler path, `X-Fault: error`
raises an unexpected exception (-> 500). Used by the track-1 load to create rare slow/failed requests whose
trace survival can be counted (span attribute messenger.fault)."""
telemetry.set_attributes(messenger__fault=fault[:16])
if fault == "slow":
await asyncio.sleep(0.6)
elif fault == "error":
raise RuntimeError("injected handler failure")
@app.exception_handler(ApiError)
async def api_error_handler(request: Request, exc: ApiError):
if exc.status in (413, 429, 503):
count_rejection(exc.code)
return error_body(request.state.rid, config.INSTANCE_ID, exc.status, exc.code, exc.message, exc.headers)
def parse_uuid(value: object) -> Optional[str]:
if not isinstance(value, str):
return None
try:
return str(uuid.UUID(value))
except ValueError:
return None
def non_blank(value: object, max_len: int) -> bool:
if not isinstance(value, str) or not 1 <= len(value) <= max_len:
return False
return value.translate(ZERO_WIDTH).strip() != ""
async def read_json(request: Request, required: Tuple[str, ...], optional: Tuple[str, ...] = ()) -> dict:
if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json":
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
too_large = ApiError(413, "payload_too_large", "Тело запроса слишком большое")
declared = request.headers.get("content-length")
if declared and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES:
raise too_large
raw = bytearray()
async for chunk in request.stream():
raw += chunk
if len(raw) > config.MAX_BODY_BYTES:
raise too_large
try:
body = json.loads(bytes(raw).decode("utf-8"))
except (UnicodeDecodeError, ValueError, RecursionError):
raise bad("Некорректный JSON") from None
if not isinstance(body, dict):
raise bad("Ожидается JSON-объект")
allowed = set(required) | set(optional)
if set(body) - allowed:
raise bad("Лишние поля в запросе")
if set(required) - set(body):
raise bad("Отсутствуют обязательные поля")
return body
def ensure_capacity() -> None:
if config.MAX_RSS_MB and rss_mb() > config.MAX_RSS_MB:
raise unavailable("Достигнут лимит памяти процесса, запись недоступна")
def bearer_token(request: Request) -> str:
scheme, _, value = request.headers.get("authorization", "").partition(" ")
token = value.strip()
if scheme.lower() != "bearer" or not token:
raise unauthorized()
return token
async def actor(request: Request) -> dict:
user = await sessions.resolve(bearer_token(request))
if user is None:
raise unauthorized()
return user
def basic_credentials(request: Request) -> Tuple[str, str]:
scheme, _, value = request.headers.get("authorization", "").partition(" ")
if scheme.lower() != "basic" or not value.strip():
raise unauthorized("Basic")
try:
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
except (binascii.Error, ValueError, UnicodeDecodeError):
raise unauthorized("Basic") from None
username, separator, password = decoded.partition(":")
if not separator:
raise unauthorized("Basic")
return username, password
async def member_chat(user: dict, chat_id: str) -> dict:
cid = parse_uuid(chat_id)
if cid is None:
raise bad("chat_id должен быть UUID")
async with db.connection() as conn:
chat = await repo.chat_for_member(conn, cid, user["id"])
if chat is None:
raise not_found()
return chat
def page_params(request: Request) -> Tuple[int, Optional[str]]:
query = request.query_params
limit = 50
if "limit" in query:
if not re.fullmatch(r"[0-9]{1,3}", query["limit"]) or not 1 <= int(query["limit"]) <= 100:
raise bad("limit должен быть целым 1..100")
limit = int(query["limit"])
cursor = query.get("cursor")
if cursor is not None and cursor == "":
raise bad("Некорректный cursor")
return limit, cursor
def keyset(scope: str, cursor: Optional[str]):
if cursor is None:
return None
try:
return decode_cursor(scope, cursor)
except BadCursor:
raise bad("Некорректный cursor") from None
def page(scope: str, items: List[dict], has_more: bool) -> JSONResponse:
next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if has_more and items else None
return JSONResponse({"items": items, "next_cursor": next_cursor})
def detect_content_type(head: bytes) -> Optional[str]:
if head[:8] == PNG_MAGIC:
return "image/png"
if head[:3] == JPEG_MAGIC:
return "image/jpeg"
return None
def validate_filename(filename: str) -> None:
if not 1 <= len(filename) <= 255 or filename.strip() == "" or filename in (".", ".."):
raise bad("filename: 1..255 символов, не только пробелы")
if "/" in filename or "\\" in filename or any(unicodedata.category(ch) in ("Cc", "Cs", "Zl", "Zp") for ch in filename):
raise bad("filename содержит недопустимые символы")
hash_slots = asyncio.Semaphore(config.PASSWORD_HASH_CONCURRENCY)
async def hash_slot(func, *args):
"""Login DoS budget: at most PASSWORD_HASH_CONCURRENCY Argon2 computations per replica at a time (each ~19 MiB,
measured time in REPORT.md); a request that cannot get a slot quickly gets 503 instead of queueing unbounded."""
try:
await asyncio.wait_for(hash_slots.acquire(), config.PASSWORD_HASH_QUEUE_SECONDS)
except asyncio.TimeoutError:
count_rejection("hash_busy")
raise ApiError(503, "unavailable", "Сервер перегружен проверкой паролей, повторите позже",
{"Retry-After": "1"}) from None
try:
return await run_in_threadpool(func, *args)
finally:
hash_slots.release()
def client_ip(request: Request) -> str:
"""X-Real-IP is set by nginx. It can be trusted: only the LB identity passes the API mTLS check, so a client
cannot reach the API directly and forge it. Fallback: the TCP peer."""
value = request.headers.get("x-real-ip", "")
return value if re.fullmatch(r"[0-9a-fA-F:.]{2,45}", value) else (request.client.host if request.client else "-")
def rate_limited(scope: str, retry_after: int) -> None:
count_rejection("login_rate_" + scope)
log.info("login_rate_limited scope=%s retry_after=%d", scope, retry_after)
raise ApiError(429, "rate_limited", "Слишком много попыток входа, повторите позже",
{"Retry-After": str(int(retry_after))})
@app.get("/health/live")
async def live():
return {"status": "ok"}
async def health_listener(reader, writer) -> None:
"""Loopback-only (127.0.0.1) plain HTTP for the Docker healthcheck: /health/live and /health/ready only.
It is not published and not reachable from the network, so it is no bypass of the mTLS business listener."""
try:
line = (await asyncio.wait_for(reader.readline(), 2)).decode("latin-1")
path = line.split(" ")[1] if line.count(" ") >= 2 else ""
if path == "/health/live":
status = 200
elif path == "/health/ready":
status = 200 if (not state["draining"] and state["s3_ok"] and await db.ready()) else 503
else:
status = 404
body = b'{"status":"ok"}' if status == 200 else b'{"status":"unavailable"}'
writer.write(b"HTTP/1.1 %d X\r\nContent-Type: application/json\r\nContent-Length: %d\r\n"
b"Connection: close\r\n\r\n%s" % (status, len(body), body))
await writer.drain()
except Exception:
pass
finally:
writer.close()
@app.get("/health/ready")
async def ready():
if not await db.ready():
raise unavailable("База данных недоступна")
if not state["s3_ok"]:
raise unavailable("Хранилище файлов недоступно")
return {"status": "ok"}
@app.post("/api/v1/users", status_code=201)
async def create_user(request: Request):
body = await read_json(request, ("username", "display_name", "password"))
if not isinstance(body["username"], str) or not USERNAME_RE.fullmatch(body["username"]):
raise bad("username должен соответствовать [a-z0-9_]{3,32}")
if not non_blank(body["display_name"], 100):
raise bad("display_name: 1..100 символов, не только пробелы")
password = body["password"]
if not isinstance(password, str) or not config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
raise bad("password: %d..%d символов" % (config.PASSWORD_MIN, config.PASSWORD_MAX))
ensure_capacity()
password_hash = await hash_slot(passwords.hash_password, password)
async with db.connection() as conn:
user = await repo.create_user(conn, body["username"], body["display_name"], password_hash)
if user is None:
raise ApiError(409, "conflict", "username уже занят")
log.info("user_created id=%s", user["id"])
return JSONResponse(user, status_code=201)
@app.post("/api/v1/auth/sessions", status_code=201)
async def login(request: Request):
if await request.body():
raise bad("POST /auth/sessions не принимает тело запроса")
username, password = basic_credentials(request)
row = None
if USERNAME_RE.fullmatch(username) and config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
async with db.connection() as conn:
row = await repo.credentials(conn, username)
# lab7: limits BEFORE the expensive Argon2 verification; counters are shared by all replicas (Valkey).
ip_key = "ratelimit:login:ip:" + client_ip(request)
user_key = "ratelimit:login:user:" + (username if USERNAME_RE.fullmatch(username) else "-invalid-")
count, retry = await sessions.hit(ip_key, config.LOGIN_RATE_WINDOW_SECONDS)
if count > config.LOGIN_RATE_LIMIT_PER_IP:
rate_limited("ip", retry)
failures, retry = await sessions.peek(user_key)
if failures >= config.LOGIN_RATE_LIMIT_PER_USER:
rate_limited("user", retry)
stored = row["password_hash"] if row else passwords.DUMMY_HASH
matches = await hash_slot(passwords.verify_password, stored, password)
if row is None or not matches:
await sessions.hit(user_key, config.LOGIN_RATE_WINDOW_SECONDS)
log.info("login_failed username_known=%s", row is not None)
raise unauthorized("Basic")
await sessions.reset(user_key)
if passwords.needs_rehash(stored):
# Old parameters (or another algorithm) -> re-hash with the current Argon2id parameters while we know the
# plaintext; a concurrent login doing the same is harmless (both hashes are valid).
new_hash = await hash_slot(passwords.hash_password, password)
async with db.connection() as conn:
await repo.update_password_hash(conn, row["id"], new_hash)
log.info("password_rehashed user=%s", row["id"])
user = repo.public_user(row)
token, expires_at = await sessions.create(user)
log.info("session_created user=%s", user["id"])
return JSONResponse({"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}, status_code=201)
@app.delete("/api/v1/auth/sessions/current", status_code=204)
async def logout(request: Request):
token = bearer_token(request)
user = await sessions.resolve(token)
if user is None:
raise unauthorized()
await sessions.revoke(token)
log.info("session_revoked user=%s", user["id"])
return Response(status_code=204)
@app.get("/api/v1/users/me")
async def me(request: Request):
return await actor(request)
@app.post("/api/v1/chats", status_code=201)
async def create_chat(request: Request):
user = await actor(request)
body = await read_json(request, ("title", "member_ids"))
if not non_blank(body["title"], 100):
raise bad("title: 1..100 символов, не только пробелы")
raw_ids = body["member_ids"]
if not isinstance(raw_ids, list) or not 1 <= len(raw_ids) <= 100:
raise bad("member_ids: массив из 1..100 UUID")
ids: List[str] = []
for value in raw_ids:
parsed = parse_uuid(value)
if parsed is None:
raise bad("member_ids содержит некорректный UUID")
ids.append(parsed)
if len(set(ids)) != len(ids):
raise bad("member_ids содержит дубликаты")
if user["id"] not in ids:
raise bad("Создатель должен входить в member_ids")
ensure_capacity()
async def insert(conn):
return await repo.create_chat(conn, body["title"], ids)
try:
chat = await db.in_transaction(insert)
except repo.UnknownMember:
raise bad("member_ids содержит неизвестного пользователя") from None
log.info("chat_created id=%s members=%d", chat["id"], len(ids))
return JSONResponse(chat, status_code=201)
@app.get("/api/v1/chats")
async def list_chats(request: Request):
user = await actor(request)
limit, cursor = page_params(request)
scope = "chats:" + user["id"]
after = keyset(scope, cursor)
async with db.connection() as conn:
items, has_more = await repo.list_chats(conn, user["id"], limit, after)
return page(scope, items, has_more)
@app.get("/api/v1/chats/{chat_id}")
async def get_chat(chat_id: str, request: Request):
return await member_chat(await actor(request), chat_id)
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
async def create_message(chat_id: str, request: Request):
user = await actor(request)
cid = parse_uuid(chat_id)
if cid is None:
raise bad("chat_id должен быть UUID")
idem_key = request.headers.get("idempotency-key", "")
if not idem_key or not IDEMPOTENCY_KEY_RE.fullmatch(idem_key):
raise bad("Заголовок Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}")
body = await read_json(request, ("text",), ("attachment_ids",))
text = body["text"]
attachment_ids_raw = body.get("attachment_ids", [])
if not isinstance(attachment_ids_raw, list):
raise bad("attachment_ids должен быть массивом UUID")
if len(attachment_ids_raw) > 10:
raise bad("attachment_ids: максимум 10 вложений")
attachment_ids: List[str] = []
for value in attachment_ids_raw:
parsed = parse_uuid(value)
if parsed is None:
raise bad("attachment_ids содержит некорректный UUID")
attachment_ids.append(parsed)
if len(set(attachment_ids)) != len(attachment_ids):
raise bad("attachment_ids содержит дубликаты")
has_text = isinstance(text, str) and non_blank(text, 4000)
has_attachments = len(attachment_ids) > 0
if not has_text and not has_attachments:
raise bad("text: 1..4000 символов, не только пробелы")
if isinstance(text, str) and len(text) > 4000:
raise bad("text: 1..4000 символов, не только пробелы")
if not isinstance(text, str):
raise bad("text должен быть строкой")
ensure_capacity()
normalized = {"text": text}
if attachment_ids:
normalized["attachment_ids"] = attachment_ids
body_hash = hashlib.sha256(
json.dumps(normalized, sort_keys=True, ensure_ascii=False).encode()
).hexdigest()
async def do(conn):
return await repo.create_message_idempotent(conn, cid, user["id"], text, attachment_ids, idem_key, body_hash)
status, message = await db.in_transaction(do)
if status == "conflict":
raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом запроса")
if status == "not_member":
raise not_found()
if status == "attachment_not_found":
raise not_found()
if status == "attachment_failed":
raise ApiError(409, "conflict", "Вложение в состоянии failed, нельзя прикрепить")
return JSONResponse(message, status_code=201)
@app.get("/api/v1/chats/{chat_id}/messages")
async def list_messages(chat_id: str, request: Request):
chat = await member_chat(await actor(request), chat_id)
limit, cursor = page_params(request)
scope = "messages:" + chat["id"]
after = keyset(scope, cursor)
async with db.connection() as conn:
items, has_more = await repo.list_messages(conn, chat["id"], limit, after)
return page(scope, items, has_more)
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def upload_attachment(chat_id: str, request: Request):
user = await actor(request)
chat = await member_chat(user, chat_id)
filename = request.query_params.get("filename", "")
validate_filename(filename)
declared_ct = request.headers.get("content-type", "").split(";")[0].strip().lower()
if declared_ct not in ALLOWED_CONTENT_TYPES:
raise ApiError(415, "unsupported_media_type", "Допускается только image/png или image/jpeg")
declared_length = request.headers.get("content-length")
too_large = ApiError(413, "payload_too_large", "Файл превышает 10 MiB")
if declared_length and declared_length.isdigit() and int(declared_length) > config.MAX_UPLOAD_BYTES:
raise too_large
ensure_capacity()
# Backpressure: do not accept more work than the pipeline can hold; the client retries later.
if config.MAX_PENDING_JOBS and state["pending_jobs"] >= config.MAX_PENDING_JOBS:
count_rejection("queue_full")
raise ApiError(503, "unavailable", "Очередь обработки переполнена, повторите позже", {"Retry-After": "5"})
sha = hashlib.sha256()
size = 0
with tempfile.SpooledTemporaryFile(max_size=config.UPLOAD_SPOOL_BYTES) as spool:
head = b""
async for chunk in request.stream():
size += len(chunk)
if size > config.MAX_UPLOAD_BYTES:
raise too_large
if len(head) < 8:
head += chunk[: 8 - len(head)]
if len(head) == 8 and detect_content_type(head) is None:
raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG")
sha.update(chunk)
spool.write(chunk)
if size == 0:
raise bad("Пустой файл")
actual_ct = detect_content_type(head)
if actual_ct is None:
raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG")
if actual_ct != declared_ct:
raise ApiError(415, "unsupported_media_type", "Content-Type не соответствует содержимому файла")
async with upload_slots:
await run_in_threadpool(inspect_image, spool, size, actual_ct)
attachment_id = str(uuid.uuid4())
staging_key = "staging/" + attachment_id
object_key = "attachments/%s/original" % attachment_id
async with db.connection() as conn:
await repo.begin_attachment(
conn, attachment_id, chat["id"], user["id"], filename, actual_ct, size, sha.hexdigest(),
staging_key, object_key,
)
try:
async with upload_slots:
await run_in_threadpool(s3.put, staging_key, spool, size, actual_ct)
except BaseException:
defer(compensate(attachment_id, staging_key))
raise
# Durable handoff: staging object exists; one COMMIT makes the attachment 'queued' and writes job + outbox.
try:
att = await db.in_transaction(lambda conn: repo.queue_attachment(
conn, attachment_id, "api:" + config.INSTANCE_ID, config.MAX_PENDING_JOBS))
except BaseException:
await compensate(attachment_id, staging_key)
raise
if att == repo.QUEUE_FULL:
defer(compensate(attachment_id, staging_key))
count_rejection("queue_full")
raise ApiError(503, "unavailable", "Очередь обработки переполнена, повторите позже", {"Retry-After": "5"})
if att is None:
raise unavailable("Загрузка отменена сверкой, повторите запрос")
telemetry.set_attributes(messenger__attachment_id=att["id"])
if faults.active(filename, "api_crash_after_commit"):
faults.crash("API after COMMIT of job/outbox, before publish")
relay.notify()
log.info("attachment_queued id=%s chat=%s size=%d", att["id"], chat["id"], size)
return JSONResponse(att, status_code=202)
def defer(coro) -> None:
task = asyncio.ensure_future(coro)
background.add(task)
task.add_done_callback(background.discard)
async def compensate(attachment_id: str, object_key: str) -> None:
"""Best effort; whatever is left behind is collected by the reconciler after the grace period."""
try:
async with db.connection() as conn:
await repo.abort_attachment(conn, attachment_id)
await run_in_threadpool(s3.delete, object_key)
except BaseException:
log.warning("compensation_incomplete attachment=%s, left for reconciler", attachment_id)
async def accessible_attachment(request: Request, attachment_id: str) -> dict:
user = await actor(request)
aid = parse_uuid(attachment_id)
if aid is None:
raise bad("attachment_id должен быть UUID")
async with db.connection() as conn:
row = await repo.get_attachment(conn, aid)
if row is None:
raise not_found()
chat = await repo.chat_for_member(conn, str(row["chat_id"]), user["id"])
if chat is None:
raise not_found()
return row
@app.get("/api/v1/attachments/{attachment_id}")
async def get_attachment(attachment_id: str, request: Request):
row = await accessible_attachment(request, attachment_id)
variants = []
if row["status"] == "ready":
async with db.connection() as conn:
variants = await repo.get_variants(conn, row["id"], row["pipeline_version"])
return repo._attachment(row, variants)
@app.get("/api/v1/attachments/{attachment_id}/content")
async def download_attachment(attachment_id: str, request: Request):
row = await accessible_attachment(request, attachment_id)
variant = request.query_params.get("variant", "original")
if variant not in VARIANTS:
raise bad("variant: original, thumbnail, crop или optimized")
if row["status"] != "ready":
raise ApiError(409, "conflict", "Вложение ещё не готово" if row["status"] in ("queued", "processing")
else "Обработка вложения завершилась ошибкой")
key, content_type = row["object_key"], row["content_type"]
if variant != "original":
async with db.connection() as conn:
found = [v for v in await repo.get_variants(conn, row["id"], row["pipeline_version"]) if v["name"] == variant]
if not found:
raise not_found()
key, content_type = found[0]["object_key"], found[0]["content_type"]
opened = await run_in_threadpool(s3.open_stream, key)
if opened is None:
log.error("object_missing attachment=%s variant=%s", row["id"], variant)
raise unavailable("Файл временно недоступен")
chunks, length = opened
return StreamingResponse(
iterate_in_threadpool(chunks), media_type=content_type, headers={"Content-Length": str(length)},
)
@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
async def fallback(request: Request):
raise not_found()
# Last statement on purpose: instrument_app() must run AFTER @app.middleware("http") is registered, so the OTel
# ASGI middleware is the outermost user middleware and guard's own 429/503/500 answers are in spans and metrics.
telemetry.setup("messenger-api", app)