перенос: реализация lab4 без изменений (API, миграции 0001-0004, HAProxy, Valkey, SeaweedFS, тесты)

Скопировано из mtusi2026lab4 (ветка solution/bvt2302-bodanyuk, commit 17c33cd) через
git archive: исходники, миграции, собственные тесты, Dockerfile, compose, конфигурации
infra/, скрипты и bench/. README, COURSE, contracts/, tests/ и отчёт lab4 не копировались.
Makefile и .env.example — версии lab4 (skeleton-часть совпадает с lab5, строки broker
добавляются в следующих коммитах).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alexey Bodanyuk
2026-10-31 15:19:00 +03:00
co-authored by Claude Opus 5.5
parent 276dbdad8a
commit 5112be83cc
70 changed files with 10544 additions and 7 deletions
+3
View File
@@ -0,0 +1,3 @@
"""In-memory messenger for lab 1 (MTUSI, «Высоконагруженные приложения»)."""
__version__ = "1.0.0"
+88
View File
@@ -0,0 +1,88 @@
"""Entry point: `python -m messenger`. The HTTP server is the main process (PID 1 in Docker).
Shutdown sequence on SIGTERM/SIGINT (lab 3: the replica stands behind HAProxy):
1. readiness flips to 503 (`app.state.draining = True`), requests are still served;
2. after DRAIN_SECONDS (compose: 3 s > HAProxy detection 2 x 1 s) the load balancer has
marked this replica DOWN and sends nothing new here;
3. uvicorn closes the listening socket; in-flight requests get up to
SHUTDOWN_TIMEOUT_SECONDS (compose: 8 s) to finish;
4. the lifespan closes the DB pool and the Valkey client; the process exits with code 0.
DRAIN + SHUTDOWN (11 s) < compose stop_grace_period (15 s), so Docker never needs SIGKILL.
A second signal during the drain skips the rest of it. Data stays in PostgreSQL and Valkey.
"""
from __future__ import annotations
import logging
import os
import sys
import threading
from types import FrameType
import uvicorn
from . import __version__, logging_setup
from .app import create_app
from .config import ConfigError, Settings
log = logging.getLogger("messenger")
class GracefulServer(uvicorn.Server):
"""uvicorn.Server that drains (readiness 503, still serving) before the normal graceful stop."""
def __init__(self, config: uvicorn.Config, drain_seconds: float):
super().__init__(config)
self.drain_seconds = drain_seconds
self.drain_timer: threading.Timer | None = None
def handle_exit(self, sig: int, frame: FrameType | None) -> None:
self.config.app.state.draining = True # config.app is our Starlette instance
if self.drain_timer is None and self.drain_seconds > 0 and not self.should_exit:
log.info("drain_started", extra={"fields": {"signal": sig, "drain_s": self.drain_seconds,
"timeout_s": self.config.timeout_graceful_shutdown}})
# uvicorn polls should_exit in its main loop, so setting it from a timer thread is safe.
self.drain_timer = threading.Timer(self.drain_seconds, self._stop, args=(sig, frame))
self.drain_timer.daemon = True
self.drain_timer.start()
return
if self.drain_timer is not None:
self.drain_timer.cancel() # second signal: stop draining now
self._stop(sig, frame)
def _stop(self, sig: int, frame: FrameType | None) -> None:
if not self.should_exit:
log.info("shutdown_requested", extra={"fields": {"signal": sig}})
super().handle_exit(sig, frame)
def main() -> int:
try:
settings = Settings.from_env()
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
logging_setup.configure(settings.log_level)
app = create_app(settings)
config = uvicorn.Config(
app,
host=settings.host,
port=settings.port,
log_config=None,
access_log=False,
server_header=False,
proxy_headers=False,
lifespan="on",
timeout_graceful_shutdown=settings.shutdown_timeout,
)
server = GracefulServer(config, settings.drain_seconds)
log.info("starting", extra={"fields": {
"version": __version__, "revision": os.environ.get("APP_REVISION", "dev"), "env": settings.env, "host": settings.host, "port": settings.port,
"python": sys.version.split()[0], "storage": "postgresql", "instance_id": settings.instance_id,
}})
server.run()
log.info("stopped")
return 0
if __name__ == "__main__":
sys.exit(main())
+584
View File
@@ -0,0 +1,584 @@
"""HTTP layer: Starlette routes, request-id/logging middleware and error mapping."""
from __future__ import annotations
import asyncio
import contextlib
import hashlib
import json
import logging
import random
import re
import time
import uuid
from typing import Any
from starlette.applications import Starlette
from starlette.exceptions import HTTPException
from starlette.requests import Request
from starlette.responses import JSONResponse, Response, StreamingResponse
from starlette.routing import Route
from starlette.types import ASGIApp, Message as AsgiMessage, Receive, Scope, Send
from . import attachments, authhdr, cursor, docs, passwords, repo, validation
from .config import Settings
from .db import Database
from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unauthorized, Unavailable
from .sessions import SessionStore, build_store
from .storage import ObjectMissing, Storage
from .timefmt import rfc3339
log = logging.getLogger("messenger")
access_log = logging.getLogger("messenger.access")
REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
# --------------------------------------------------------------------------
# Responses
# --------------------------------------------------------------------------
def error_response(request_id: str, status: int, code: str, message: str, headers: dict[str, str] | None = None) -> JSONResponse:
body = {"error": {"code": code, "message": message, "request_id": request_id}}
return JSONResponse(body, status_code=status, headers={"X-Request-Id": request_id, **(headers or {})})
async def api_error_handler(request: Request, exc: Exception) -> JSONResponse:
assert isinstance(exc, ApiError)
return error_response(request.state.request_id, exc.status, exc.code, exc.message, exc.headers)
async def http_exception_handler(request: Request, exc: Exception) -> JSONResponse:
"""Unknown routes and wrong methods also answer in the contract error format."""
assert isinstance(exc, HTTPException)
if exc.status_code == 404:
return error_response(request.state.request_id, 404, "not_found", "Объект не найден")
if exc.status_code == 405:
return error_response(request.state.request_id, 405, "invalid_request", "Метод не поддерживается", exc.headers)
return error_response(request.state.request_id, exc.status_code, "invalid_request", "Неверный запрос")
# --------------------------------------------------------------------------
# Middleware (pure ASGI, so it also covers 404/405 and unexpected exceptions)
# --------------------------------------------------------------------------
class RequestContextMiddleware:
"""Adds X-Request-Id and X-Instance-Id to EVERY response (errors, health and crash 500/503
included), writes one access-log line, turns crashes into 500."""
def __init__(self, app: ASGIApp, instance_id: str):
self.app = app
self.instance_id = instance_id.encode("ascii")
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] != "http":
await self.app(scope, receive, send)
return
incoming = dict(scope["headers"]).get(b"x-request-id", b"").decode("latin-1")
request_id = incoming if REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex
scope.setdefault("state", {})["request_id"] = request_id
status = 500
started = False
begin = time.perf_counter()
async def send_with_id(message: AsgiMessage) -> None:
nonlocal status, started
if message["type"] == "http.response.start":
started = True
status = message["status"]
headers = [(k, v) for k, v in message.get("headers", [])
if k.lower() not in (b"x-request-id", b"x-instance-id")]
headers.append((b"x-request-id", request_id.encode("latin-1")))
headers.append((b"x-instance-id", self.instance_id))
message = {**message, "headers": headers}
await send(message)
try:
await self.app(scope, receive, send_with_id)
except asyncio.CancelledError:
# uvicorn cancels requests still running when SHUTDOWN_TIMEOUT_SECONDS expires:
# answer 503 (retryable) instead of a generic 500, then let the cancellation proceed.
if not started:
status = 503
response = error_response(request_id, 503, "unavailable", "Сервер завершает работу")
await response(scope, receive, send_with_id)
raise
except Exception:
log.exception("unhandled_error", extra={"fields": {"request_id": request_id}})
if not started:
status = 500
response = error_response(request_id, 500, "internal_error", "Внутренняя ошибка сервера")
await response(scope, receive, send_with_id)
finally:
access_log.info("request", extra={"fields": {
"request_id": request_id,
"method": scope["method"],
"path": scope["path"], # ids only; query (cursor) and bodies are never logged
"status": status,
"instance_id": self.instance_id.decode("ascii"),
"duration_ms": round((time.perf_counter() - begin) * 1000, 3),
}})
# --------------------------------------------------------------------------
# Request helpers
# --------------------------------------------------------------------------
def db_of(request: Request) -> Database:
return request.app.state.db
def sessions_of(request: Request) -> SessionStore:
return request.app.state.sessions
def storage_of(request: Request) -> Storage:
return request.app.state.storage
async def authenticate(request: Request) -> str:
"""User id of a valid Bearer session, otherwise 401. X-User-Id is ignored since lab 2.
Store failures are 503 (Unavailable), never 401 and never anonymous access.
"""
token = authhdr.parse_bearer(request.headers.get("authorization"))
user_id = await sessions_of(request).user_id_for(token)
if user_id is None:
raise Unauthorized("Сессия недействительна, истекла или отозвана")
return user_id
def path_uuid(request: Request, name: str) -> str:
return validation.parse_uuid(request.path_params[name], name)
async def read_json(request: Request) -> dict[str, Any]:
content_type = request.headers.get("content-type", "")
media_type, _, params = content_type.partition(";")
if media_type.strip().lower() != "application/json":
raise InvalidRequest("Content-Type должен быть application/json")
charset = params.strip().lower()
if charset and charset.replace(" ", "") not in {"charset=utf-8", "charset=\"utf-8\""}:
raise InvalidRequest("Поддерживается только кодировка UTF-8")
limit: int = request.app.state.settings.max_body_bytes
declared = request.headers.get("content-length")
if declared is not None and declared.isdigit() and int(declared) > limit:
raise PayloadTooLarge()
chunks: list[bytes] = []
size = 0
async for chunk in request.stream():
size += len(chunk)
if size > limit:
raise PayloadTooLarge()
chunks.append(chunk)
return validation.parse_json_object(b"".join(chunks))
def page_params(request: Request) -> tuple[int, str | None]:
query = request.query_params.multi_items()
limit = validation.parse_limit(validation.single_query_param(query, "limit"))
raw_cursor = validation.single_query_param(query, "cursor")
if raw_cursor is not None and raw_cursor == "":
raise InvalidRequest("cursor не может быть пустым")
return limit, raw_cursor
async def reject_body(request: Request) -> None:
"""POST /auth/sessions takes credentials only from the Basic header; any body is 400."""
if request.headers.get("transfer-encoding") or request.headers.get("content-length", "0") != "0":
raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела")
async for chunk in request.stream():
if chunk:
raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела")
def idempotency_key(request: Request) -> str:
values = request.headers.getlist("idempotency-key")
return validation.idempotency_key(values[0] if len(values) == 1 else None)
def request_fingerprint(text: str, attachment_ids: list[str]) -> bytes:
"""SHA-256 of the canonical body: key order and an absent attachment_ids vs [] do not matter."""
canonical = json.dumps({"attachment_ids": attachment_ids, "text": text},
ensure_ascii=False, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(canonical.encode("utf-8")).digest()
def ok(body: Any, status: int = 200) -> JSONResponse:
return JSONResponse(body, status_code=status)
# --------------------------------------------------------------------------
# Handlers
# --------------------------------------------------------------------------
async def live(request: Request) -> JSONResponse:
return ok({"status": "ok"})
async def ready(request: Request) -> JSONResponse:
"""200 only if the WHOLE mandatory synchronous path can be served: PostgreSQL, Valkey
(SESSION_STORE=valkey) and, since lab 4, the S3 bucket (upload/download of attachments)
answer. 503 while draining after SIGTERM. HAProxy routes to such replicas first."""
if request.app.state.draining:
raise Unavailable("Сервер завершает работу")
db_ok, store_ok, s3_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping(),
storage_of(request).ping())
if not db_ok:
raise Unavailable("PostgreSQL недоступен")
if not store_ok:
raise Unavailable("Хранилище сессий недоступно")
if not s3_ok:
raise Unavailable("Хранилище файлов (S3) недоступно")
return ok({"status": "ok"})
async def ready_core(request: Request) -> JSONResponse:
"""Not in the contract: readiness WITHOUT S3, for the load balancer's degraded mode only.
If S3 is down for every replica, /health/ready is 503 everywhere; HAProxy then routes to
replicas that are at least core-ready, so chats and messages keep working while every
attachment request answers 503 itself (see infra/haproxy/haproxy.cfg)."""
if request.app.state.draining:
raise Unavailable("Сервер завершает работу")
db_ok, store_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping())
if not (db_ok and store_ok):
raise Unavailable("PostgreSQL или хранилище сессий недоступно")
return ok({"status": "ok"})
async def register_user(request: Request) -> JSONResponse:
data = await read_json(request)
validation.check_fields(data, {"username", "display_name", "password"})
username = validation.username_field(data)
display_name = validation.text_field(data, "display_name", 100)
password = validation.password_field(data)
password_hash = await passwords.hash_password(password)
user = await repo.create_user(db_of(request), username, display_name, password_hash)
log.info("user_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}})
return ok(user.to_json(), 201)
async def create_session(request: Request) -> JSONResponse:
"""Login: only `Authorization: Basic`; wrong user and wrong password give the same 401."""
username, password = authhdr.parse_basic(request.headers.get("authorization"))
await reject_body(request)
found = await repo.get_credentials(db_of(request), username)
user, stored_hash = found if found else (None, None)
if not await passwords.verify_password(stored_hash, password) or user is None:
log.info("login_failed", extra={"fields": {"request_id": request.state.request_id}})
raise Unauthorized("Неверное имя пользователя или пароль", challenge=BASIC_CHALLENGE)
ttl = request.app.state.settings.session_ttl_seconds
token, expires_at = await sessions_of(request).create(user.id, ttl)
log.info("session_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}})
return ok({"token": token, "token_type": "Bearer", "expires_at": rfc3339(expires_at), "user": user.to_json()}, 201)
async def revoke_session(request: Request) -> Response:
token = authhdr.parse_bearer(request.headers.get("authorization"))
if not await sessions_of(request).revoke(token):
raise Unauthorized("Сессия недействительна, истекла или отозвана")
log.info("session_revoked", extra={"fields": {"request_id": request.state.request_id}})
return Response(status_code=204)
async def me(request: Request) -> JSONResponse:
user_id = await authenticate(request)
user = await repo.get_user(db_of(request), user_id)
if user is None:
raise Unauthorized("Сессия недействительна, истекла или отозвана")
return ok(user.to_json())
async def create_chat(request: Request) -> JSONResponse:
user_id = await authenticate(request)
data = await read_json(request)
validation.check_fields(data, {"title", "member_ids"})
title = validation.text_field(data, "title", 100)
member_ids = validation.member_ids_field(data)
chat = await repo.create_chat(db_of(request), user_id, title, member_ids)
log.info("chat_created", extra={"fields": {"request_id": request.state.request_id, "chat_id": chat.id, "members": len(member_ids)}})
return ok(chat.to_json(), 201)
async def list_chats(request: Request) -> JSONResponse:
user_id = await authenticate(request)
limit, raw_cursor = page_params(request)
after = cursor.decode(raw_cursor, cursor.CHATS, user_id) if raw_cursor else None
items, has_more = await repo.list_chats(db_of(request), user_id, after, limit)
next_cursor = cursor.encode(cursor.CHATS, user_id, items[-1].key) if has_more else None
return ok({"items": [c.to_json() for c in items], "next_cursor": next_cursor})
async def get_chat(request: Request) -> JSONResponse:
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
return ok((await repo.get_chat(db_of(request), user_id, chat_id)).to_json())
def parse_message(data: dict[str, Any]) -> tuple[str, list[str]]:
"""`text` is required and a string (also with attachments); `attachment_ids`, if present, an
array of 0-10 unique UUIDs. "" or whitespace-only text is allowed only with attachments."""
validation.check_fields(data, {"text"}, frozenset({"attachment_ids"}))
attachment_ids = validation.attachment_ids_field(data)
text = validation.text_field(data, "text", 4000, min_len=0 if attachment_ids else 1,
allow_blank=bool(attachment_ids))
return text, attachment_ids
async def create_message(request: Request) -> JSONResponse:
"""Order of answers for both write paths: 401 -> 400 bad chat id -> 404 outsider -> 400 key/body
-> 409 other body with the same key -> 201 (new or replayed). sender_id comes from the session."""
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
db = db_of(request)
settings = request.app.state.settings
if settings.message_write_path == "multi":
await repo.ensure_member(db, user_id, chat_id) # outsiders get 404 before key/body are looked at
key = idempotency_key(request)
text, attachment_ids = parse_message(await read_json(request))
result = await repo.post_message(db, user_id, chat_id, key, request_fingerprint(text, attachment_ids), text,
settings.idempotency_ttl_seconds, attachment_ids)
else:
try:
key = idempotency_key(request)
text, attachment_ids = parse_message(await read_json(request))
except ApiError:
# Rare error path only: an outsider must still get 404, not a hint about the body.
await repo.ensure_member(db, user_id, chat_id)
raise
# Membership check, key claim, chat clock and message: one SQL call, one transaction.
result = await repo.post_message_single_call(db, user_id, chat_id, key,
request_fingerprint(text, attachment_ids), text,
settings.idempotency_ttl_seconds, attachment_ids)
if result.replayed:
log.info("idempotent_replay", extra={"fields": {"request_id": request.state.request_id,
"message_id": result.message.id}})
return ok(result.message.to_json(), 201)
async def list_messages(request: Request) -> JSONResponse:
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
db = db_of(request)
await repo.ensure_member(db, user_id, chat_id) # 404 for outsiders before cursor details leak
limit, raw_cursor = page_params(request)
after = cursor.decode(raw_cursor, cursor.MESSAGES, chat_id) if raw_cursor else None
items, has_more = await repo.list_messages(db, chat_id, after, limit)
next_cursor = cursor.encode(cursor.MESSAGES, chat_id, items[-1].key) if has_more else None
return ok({"items": [m.to_json() for m in items], "next_cursor": next_cursor})
# ------------------------------------------------------------------ attachments (lab 4)
def fault(request: Request, point: str) -> None:
"""Failure injection for the level 4 experiments (FAULT_POINTS, compose.faults.yaml only)."""
if point in request.app.state.settings.fault_points:
log.warning("fault_injected", extra={"fields": {"request_id": request.state.request_id, "point": point}})
raise Unavailable("Внедрённый отказ: " + point)
async def upload_attachment(request: Request) -> JSONResponse:
"""POST /chats/{chat_id}/attachments?filename=...: raw PNG/JPEG -> 201 ready (attachments.py)."""
settings: Settings = request.app.state.settings
reader = attachments.BodyReader(request)
slots: asyncio.Semaphore = request.app.state.upload_slots
acquired = False
try:
try:
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
db = db_of(request)
await repo.ensure_member(db, user_id, chat_id) # outsiders: 404 before anything about the body
filename = validation.filename_param(validation.query_items(request.scope["query_string"]))
declared = attachments.declared_type(request)
length = attachments.declared_length(request, settings.max_upload_bytes)
try:
await asyncio.wait_for(slots.acquire(), settings.upload_queue_timeout_seconds)
except asyncio.TimeoutError:
raise attachments.Busy("Слишком много одновременных загрузок, повторите позже") from None
acquired = True
upload = await attachments.read_image(reader, settings, declared, request.app.state.verify_slots, length)
except ApiError:
# Early rejection: drop (not store) a bounded rest of the body so the client reads our error.
await reader.discard(settings.max_upload_bytes + (1 << 20), timeout=5.0)
raise
# The slot is held until the buffer is no longer needed (after the S3 PUT), so at most
# UPLOAD_CONCURRENCY bodies of this replica exist in memory at any moment.
return await store_upload(request, db, chat_id, user_id, filename, upload)
finally:
if acquired:
slots.release()
async def store_upload(request: Request, db: Database, chat_id: str, user_id: str, filename: str,
upload: attachments.Upload) -> JSONResponse:
"""intent (pending) -> S3 PUT -> ready -> 201 (order and failure analysis: docs/CONSISTENCY.md)."""
attachment_id = uuid.uuid4()
object_key = attachments.new_object_key()
storage = storage_of(request)
await attachments.create_intent(db, attachment_id, chat_id, user_id, filename, upload.info, len(upload.data),
upload.sha256, object_key)
await storage.put(object_key, upload.data, upload.info.content_type,
{"attachment-id": str(attachment_id), "sha256": upload.sha256.hex()})
fault(request, "after_s3_put")
attachment = await attachments.mark_ready(db, attachment_id)
if attachment is None:
raise Unavailable("Загрузка не завершена, повторите запрос")
fault(request, "after_ready_commit")
log.info("attachment_created", extra={"fields": {"request_id": request.state.request_id,
"attachment_id": attachment.id, "chat_id": chat_id,
"size": attachment.size_bytes, "type": attachment.content_type}})
return ok(attachment.to_json(), 201)
async def get_attachment(request: Request) -> JSONResponse:
user_id = await authenticate(request)
attachment_id = path_uuid(request, "attachment_id")
return ok((await attachments.get_visible(db_of(request), user_id, attachment_id)).to_json())
def content_disposition(filename: str) -> str:
"""RFC 6266/5987: ASCII fallback + UTF-8 name; the stored name itself is never interpreted."""
from urllib.parse import quote
fallback = "".join(c if 32 <= ord(c) < 127 and c not in '"\\%;' else "_" for c in filename)
return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{quote(filename, safe='')}"
async def download_attachment(request: Request) -> Response:
"""Original bytes through the authorized API: no redirect, no public S3 URL. Streamed from
S3 in 64 KiB chunks (memory per download does not depend on the file size)."""
user_id = await authenticate(request)
attachment_id = path_uuid(request, "attachment_id")
query = validation.query_items(request.scope["query_string"])
try:
validation.variant_param(query)
except validation.NotFoundVariant:
await attachments.get_visible(db_of(request), user_id, attachment_id) # 404 for outsiders first
raise NotFound("Такого варианта у вложения нет") from None
attachment = await attachments.get_visible(db_of(request), user_id, attachment_id)
if attachment.lost:
raise NotFound("Оригинал вложения утрачен")
try:
size, chunks = await storage_of(request).open_stream(attachment.object_key)
except ObjectMissing:
# A ready row without its object: data loss, reported by the reconciler (quarantine).
log.error("attachment_object_missing", extra={"fields": {"request_id": request.state.request_id,
"attachment_id": attachment.id}})
raise NotFound("Оригинал вложения утрачен") from None
if size != attachment.size_bytes:
log.error("attachment_size_mismatch", extra={"fields": {"request_id": request.state.request_id,
"attachment_id": attachment.id}})
await chunks.aclose()
raise Unavailable("Оригинал вложения временно недоступен")
headers = {
"Content-Length": str(size),
"Content-Disposition": content_disposition(attachment.filename),
"X-Content-Type-Options": "nosniff",
"Cache-Control": "private, no-store",
"ETag": '"' + attachment.sha256 + '"',
}
return StreamingResponse(chunks, media_type=attachment.content_type, headers=headers)
def create_app(settings: Settings | None = None) -> Starlette:
settings = settings or Settings.from_env()
db = Database(settings)
sessions = build_store(settings, db)
storage = Storage(settings)
async def purge_loop() -> None:
"""Housekeeping: expired Idempotency-Key rows (older than IDEMPOTENCY_TTL_SECONDS)."""
while True:
await asyncio.sleep(600)
try:
deleted = await repo.purge_idempotency_keys(db, settings.idempotency_ttl_seconds)
if deleted:
log.info("idempotency_keys_purged", extra={"fields": {"deleted": deleted}})
except ApiError:
pass # database unavailable right now; next round
async def janitor_loop(every: float) -> None:
"""Level 4 cleanup: intents of failed uploads older than PENDING_GRACE_SECONDS (attachments.py).
Every replica runs it; the advisory lock lets one round run at a time."""
while True:
await asyncio.sleep(every * (0.5 + random.random()))
try:
removed = await attachments.clean_stale_pending(db, storage, settings.pending_grace_seconds)
if removed:
log.info("stale_uploads_removed", extra={"fields": {"count": removed}})
except ApiError:
pass # PostgreSQL or S3 unavailable right now; the rows stay pending for the next round
stats_log = logging.getLogger("messenger.stats")
async def stats_loop(every: float) -> None:
"""Experiments only: DB pool saturation (waiting requests, wait time) and process CPU."""
cpu = time.process_time()
db.pool.pop_stats()
while True:
await asyncio.sleep(every)
stats = db.pool.pop_stats()
now_cpu = time.process_time()
stats_log.info("pool_stats", extra={"fields": {
"interval_s": every, "cpu_s": round(now_cpu - cpu, 4),
**{k: stats.get(k, 0) for k in ("pool_size", "pool_available", "requests_waiting", "requests_num",
"requests_queued", "requests_wait_ms", "usage_ms", "requests_errors")}}})
cpu = now_cpu
@contextlib.asynccontextmanager
async def lifespan(app: Starlette):
passwords.configure(settings.hash_concurrency)
await db.open()
await storage.open()
log.info("dependencies_configured", extra={"fields": {
"session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds,
"db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds,
"instance_id": settings.instance_id, "drain_s": settings.drain_seconds,
"message_write_path": settings.message_write_path, "s3_bucket": settings.s3_bucket,
"upload_concurrency": settings.upload_concurrency, "verify_concurrency": settings.image_verify_concurrency,
"pending_grace_s": settings.pending_grace_seconds,
"fault_points": sorted(settings.fault_points)}})
tasks = [asyncio.create_task(purge_loop())]
if settings.janitor_interval_seconds > 0:
tasks.append(asyncio.create_task(janitor_loop(settings.janitor_interval_seconds)))
if settings.stats_log_seconds > 0:
tasks.append(asyncio.create_task(stats_loop(settings.stats_log_seconds)))
try:
yield
finally:
for task in tasks:
task.cancel()
await sessions.close()
await storage.close()
await db.close()
routes = [
Route("/health/live", live, methods=["GET"]),
Route("/health/ready", ready, methods=["GET"]),
Route("/health/ready/core", ready_core, methods=["GET"]),
Route("/api/v1/users", register_user, methods=["POST"]),
Route("/api/v1/users/me", me, methods=["GET"]),
Route("/api/v1/auth/sessions", create_session, methods=["POST"]),
Route("/api/v1/auth/sessions/current", revoke_session, methods=["DELETE"]),
Route("/api/v1/chats", create_chat, methods=["POST"]),
Route("/api/v1/chats", list_chats, methods=["GET"]),
Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]),
Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]),
Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]),
Route("/api/v1/chats/{chat_id}/attachments", upload_attachment, methods=["POST"]),
Route("/api/v1/attachments/{attachment_id}", get_attachment, methods=["GET"]),
Route("/api/v1/attachments/{attachment_id}/content", download_attachment, methods=["GET"]),
]
if settings.docs_enabled: # Swagger UI for manual experiments, not part of the contract
routes += [
Route("/docs", docs.swagger_page, methods=["GET"]),
Route("/openapi.json", docs.openapi_json, methods=["GET"]),
]
app = Starlette(
routes=routes,
exception_handlers={ApiError: api_error_handler, HTTPException: http_exception_handler},
lifespan=lifespan,
)
app.add_middleware(RequestContextMiddleware, instance_id=settings.instance_id)
app.state.settings = settings
app.state.db = db
app.state.sessions = sessions
app.state.storage = storage
app.state.upload_slots = asyncio.Semaphore(settings.upload_concurrency)
app.state.verify_slots = asyncio.Semaphore(settings.image_verify_concurrency)
app.state.draining = False
return app
+300
View File
@@ -0,0 +1,300 @@
"""Attachments (lab 4): metadata in PostgreSQL, original bytes in the private S3 bucket.
Upload (POST /chats/{id}/attachments, raw PNG/JPEG body), in this order:
auth -> membership (404) -> filename / Content-Type / Content-Length (400/415/413)
-> upload slot (at most UPLOAD_CONCURRENCY bodies per replica, else 503 + Retry-After)
-> body in chunks: size limit, read deadline, format + pixel count from the first bytes
-> full integrity check (images.verify) -> SHA-256 / size known
-> INSERT row status='pending' with a fresh random object_key, COMMIT (intent)
-> PUT object to S3 (Content-MD5)
-> UPDATE status='ready' WHERE status='pending', COMMIT -> 201 ready.
Why 201 is honest: it is sent only after BOTH the object and the ready row are durable.
What a failure between the steps leaves behind, and who removes it (docs/CONSISTENCY.md):
* before the intent COMMIT: nothing (no object was written);
* after the intent, before/while PUT, before the ready COMMIT: a 'pending' row (+ maybe the
object). The client got 503 and may retry (a new attachment). The janitor (below) deletes the
object - idempotent, a missing key is fine - and then the row, once it is older than
PENDING_GRACE_SECONDS. Nothing is compensated in the request path: after a lost answer of
COMMIT we do not know whether the row is ready, and deleting the object could destroy a
ready attachment. Only the janitor decides, under a row lock, from the committed state;
* after the ready COMMIT (answer lost): a correct ready attachment the client does not know.
"""
from __future__ import annotations
import asyncio
import datetime as dt
import hashlib
import logging
import secrets
import uuid
from dataclasses import dataclass
import psycopg
from psycopg import errors as pg_errors
from starlette.requests import Request
from . import images
from .config import Settings
from .db import Database
from .errors import ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType
from .storage import OBJECT_PREFIX, Storage
from .timefmt import rfc3339
log = logging.getLogger("messenger.attachments")
JANITOR_LOCK_ID = 2026_04 # advisory lock: one janitor round at a time across all replicas
@dataclass(frozen=True, slots=True)
class Attachment:
id: str
chat_id: str
owner_id: str
filename: str
content_type: str
size_bytes: int
sha256: str
object_key: str
status: str
created_at: dt.datetime
updated_at: dt.datetime
lost: bool = False
def to_json(self) -> dict:
"""Public representation: no object key, no bucket, no internal state."""
return {"id": self.id, "chat_id": self.chat_id, "owner_id": self.owner_id, "filename": self.filename,
"content_type": self.content_type, "size_bytes": self.size_bytes, "sha256": self.sha256,
"status": "ready", "variants": [], "error_code": None,
"created_at": rfc3339(self.created_at), "updated_at": rfc3339(self.updated_at)}
COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.object_key, "
"a.status, a.created_at, a.updated_at, a.lost_at IS NOT NULL")
def _attachment(row) -> Attachment:
return Attachment(str(row[0]), str(row[1]), str(row[2]), row[3], row[4], row[5], bytes(row[6]).hex(), row[7],
row[8], row[9], row[10], row[11])
def new_object_key() -> str:
"""Server-generated, 128 random bits, unrelated to the attachment id and the filename."""
return OBJECT_PREFIX + secrets.token_hex(16)
# ------------------------------------------------------------------ SQL
async def create_intent(db: Database, attachment_id: uuid.UUID, chat_id: str, owner_id: str, filename: str,
info: images.ImageInfo, size: int, sha256: bytes, object_key: str) -> None:
async def work(conn: psycopg.AsyncConnection) -> None:
await conn.execute(
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"width, height, object_key, status) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending')",
(attachment_id, chat_id, owner_id, filename, info.content_type, size, sha256, info.width, info.height,
object_key))
try:
await db.run(work, transaction=False)
except pg_errors.ForeignKeyViolation:
raise NotFound("Чат не найден") from None
async def mark_ready(db: Database, attachment_id: uuid.UUID) -> Attachment | None:
"""pending -> ready. None if the row is no longer pending (the janitor removed a stale intent)."""
async def work(conn: psycopg.AsyncConnection) -> Attachment | None:
cur = await conn.execute(
"UPDATE attachments a SET status = 'ready', updated_at = clock_timestamp() "
f"WHERE a.id = %s AND a.status = 'pending' RETURNING {COLUMNS}", (attachment_id,))
row = await cur.fetchone()
return _attachment(row) if row else None
return await db.run(work, transaction=False)
async def get_visible(db: Database, user_id: str, attachment_id: str) -> Attachment:
"""A READY attachment of a chat the user is a member of; anything else is 404 (unknown,
foreign chat, unfinished upload) - existence of foreign objects is never revealed."""
async def work(conn: psycopg.AsyncConnection) -> Attachment:
cur = await conn.execute(
f"SELECT {COLUMNS} FROM attachments a "
"JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s "
"WHERE a.id = %s AND a.status = 'ready'", (user_id, attachment_id))
row = await cur.fetchone()
if row is None:
raise NotFound("Вложение не найдено")
return _attachment(row)
return await db.run(work, transaction=False)
async def clean_stale_pending(db: Database, storage: Storage, grace_seconds: int, batch: int = 20) -> int:
"""Janitor round: remove intents of uploads that never became ready.
Under FOR UPDATE: an upload that is (impossibly) still running would block on the row in
mark_ready and then find it gone (-> 503), so a ready attachment can never lose its object.
Order: object first (idempotent), row second; a crash in between leaves the pending row and
the next round repeats. Only status='pending' rows older than the grace period are touched:
fresh uploads and ready/used attachments are never selected.
"""
async def work(conn: psycopg.AsyncConnection) -> int:
cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (JANITOR_LOCK_ID,))
if not (await cur.fetchone())[0]:
return 0
cur = await conn.execute(
"SELECT id, object_key FROM attachments WHERE status = 'pending' "
"AND created_at < clock_timestamp() - make_interval(secs => %s) "
"ORDER BY created_at LIMIT %s FOR UPDATE SKIP LOCKED", (grace_seconds, batch))
rows = await cur.fetchall()
if not rows:
return 0
await asyncio.gather(*(storage.delete(key) for _, key in rows))
await conn.execute("DELETE FROM attachments WHERE id = ANY(%s) AND status = 'pending'",
([r[0] for r in rows],))
return len(rows)
return await db.run(work, deadline=storage.timeout + 5)
# ------------------------------------------------------------------ request body
class Busy(Unavailable):
"""All upload slots of this replica are taken: retry shortly (no body was read)."""
@property
def headers(self) -> dict[str, str]:
return {"Retry-After": "1"}
@dataclass
class Upload:
data: bytearray
info: images.ImageInfo
sha256: bytes
class BodyReader:
"""Reads the raw ASGI body ourselves (not request.stream()), so we always know how much of it
is consumed and can discard the rest after an early rejection."""
def __init__(self, request: Request):
self.receive = request.receive
self.done = False
async def chunk(self, timeout: float) -> bytes | None:
if self.done:
return None
message = await asyncio.wait_for(self.receive(), timeout)
if message["type"] == "http.disconnect":
self.done = True
raise InvalidRequest("Клиент прервал загрузку")
if not message.get("more_body", False):
self.done = True
return message.get("body", b"")
async def discard(self, limit: int, timeout: float) -> None:
"""After an early 4xx: read and DROP up to `limit` more bytes, so an ordinary client that
writes the whole body before reading the answer still gets our JSON error instead of a
connection reset. Nothing is kept in memory; beyond the limit the connection is dropped."""
loop = asyncio.get_running_loop()
deadline = loop.time() + timeout
seen = 0
try:
while not self.done and seen <= limit:
remaining = deadline - loop.time()
if remaining <= 0:
return
seen += len(await self.chunk(remaining) or b"")
except (ApiError, asyncio.TimeoutError):
return
def declared_type(request: Request) -> str:
"""Content-Type of the upload must be image/png or image/jpeg (parameters ignored)."""
media_type = request.headers.get("content-type", "").partition(";")[0].strip().lower()
if media_type not in (images.PNG, images.JPEG):
raise UnsupportedMediaType("Content-Type должен быть image/png или image/jpeg")
return media_type
def declared_length(request: Request, limit: int) -> int | None:
raw = request.headers.get("content-length")
if raw is None:
return None # chunked transfer: the limit is enforced while reading
if not raw.isdigit():
raise InvalidRequest("Неверный Content-Length")
length = int(raw)
if length > limit:
raise PayloadTooLarge(f"Файл больше {limit} байт")
if length == 0:
raise InvalidRequest("Тело запроса пустое: нужны байты изображения")
return length
async def read_image(reader: BodyReader, settings: Settings, declared: str, verify_slots: asyncio.Semaphore,
length: int | None) -> Upload:
"""Body -> verified image. Memory: ONE buffer of at most MAX_UPLOAD_BYTES per upload - exactly
Content-Length bytes allocated once when the length is declared (no growth, no final copy);
it is passed as is to the checks and to the S3 PUT. The format and the pixel count are known
after the first bytes, so a picture that is too big or not a PNG/JPEG is rejected without
reading/decoding the rest."""
loop = asyncio.get_running_loop()
deadline = loop.time() + settings.upload_read_timeout_seconds
limit = settings.max_upload_bytes
buffer = bytearray(length) if length else bytearray()
view = memoryview(buffer) if length else None
filled = 0
digest = hashlib.sha256()
info: images.ImageInfo | None = None
while True:
remaining = deadline - loop.time()
if remaining <= 0:
raise InvalidRequest("Тело запроса не получено вовремя")
try:
chunk = await reader.chunk(remaining)
except asyncio.TimeoutError:
raise InvalidRequest("Тело запроса не получено вовремя") from None
if chunk is None:
break
if filled + len(chunk) > (length or limit):
raise PayloadTooLarge(f"Файл больше {limit} байт")
if view is not None:
view[filled:filled + len(chunk)] = chunk
else:
buffer += chunk
filled += len(chunk)
digest.update(chunk)
if info is None and filled:
info = _sniff(view[:filled] if view is not None else buffer, settings, declared)
if view is not None:
view.release()
if not filled:
raise InvalidRequest("Тело запроса пустое: нужны байты изображения")
if length and filled != length:
raise InvalidRequest("Тело запроса короче Content-Length")
if info is None:
raise UnsupportedMediaType("Файл не является корректным изображением PNG или JPEG")
try:
async with verify_slots:
await images.verify(buffer, info, timeout=settings.image_verify_timeout_seconds,
jpeg_max_memory_mb=settings.jpeg_max_memory_mb)
except images.ImageError as exc:
raise _image_error(exc) from None
except asyncio.TimeoutError:
raise UnsupportedMediaType("Изображение не удалось проверить за отведённое время") from None
except (images.VerifierUnavailable, MemoryError):
log.warning("image_verifier_unavailable", extra={"fields": {"type": info.content_type, "size": filled}})
raise Busy("Не хватило ресурсов для проверки изображения, повторите позже") from None
return Upload(buffer, info, digest.digest())
def _sniff(buffer: bytearray, settings: Settings, declared: str) -> images.ImageInfo | None:
try:
info = images.sniff(buffer, settings.max_image_pixels)
except images.ImageError as exc:
raise _image_error(exc) from None
if info is not None and info.content_type != declared:
# "MIME spoofing": the header says PNG, the bytes are JPEG (or vice versa).
raise UnsupportedMediaType("Содержимое не совпадает с Content-Type")
return info
def _image_error(exc: images.ImageError) -> ApiError:
return PayloadTooLarge(exc.message) if exc.status == 413 else UnsupportedMediaType(exc.message)
+48
View File
@@ -0,0 +1,48 @@
"""Parsing of the Authorization header. Any malformed value is 401, never 500.
Basic (RFC 7617) is accepted only by POST /auth/sessions; everything else needs
Bearer. Basic is an encoding, not encryption: plain HTTP is acceptable only for the
local synthetic lab stand (lab 7 adds TLS).
"""
from __future__ import annotations
import base64
import binascii
import re
from .errors import BASIC_CHALLENGE, BEARER_CHALLENGE, Unauthorized
from .validation import USERNAME_RE
B64_RE = re.compile(r"[A-Za-z0-9+/]+={0,2}")
TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{32,128}") # token_urlsafe(32) gives 43 characters
def _split(header: str | None) -> tuple[str, str]:
if not header:
return "", ""
scheme, _, value = header.strip().partition(" ")
return scheme.lower(), value.strip()
def parse_basic(header: str | None) -> tuple[str, str]:
"""(username, password) from `Basic base64(username:password)`; the password may contain ':'."""
invalid = Unauthorized("Неверное имя пользователя или пароль", challenge=BASIC_CHALLENGE)
scheme, value = _split(header)
if scheme != "basic" or B64_RE.fullmatch(value) is None or len(value) > 1024:
raise invalid
try:
decoded = base64.b64decode(value, validate=True).decode("utf-8")
except (binascii.Error, ValueError, UnicodeDecodeError):
raise invalid from None
username, sep, password = decoded.partition(":") # split at the FIRST colon
if not sep or USERNAME_RE.fullmatch(username) is None or not password:
raise invalid
return username, password
def parse_bearer(header: str | None) -> str:
scheme, value = _split(header)
if scheme != "bearer" or TOKEN_RE.fullmatch(value) is None:
# Missing header, Basic on a business endpoint, garbage token: one answer.
raise Unauthorized("Требуется действительная Bearer-сессия", challenge=BEARER_CHALLENGE)
return value
+236
View File
@@ -0,0 +1,236 @@
"""Runtime configuration read from environment variables (12-factor style).
Non-secret settings have safe defaults. Connection strings with passwords are
read from files (`DATABASE_URL_FILE`, `CACHE_URL_FILE` → Docker secrets in
/run/secrets), so they never appear in `docker inspect`, compose files or logs.
Invalid values stop the process at startup with a clear message.
"""
from __future__ import annotations
import os
import re
import socket
from dataclasses import dataclass
from pathlib import Path
class ConfigError(ValueError):
pass
INSTANCE_ID_RE = re.compile(r"[A-Za-z0-9._-]{1,64}")
def _instance_id() -> str:
"""X-Instance-Id of this process: INSTANCE_ID or the container hostname.
Docker sets the hostname of a container to its short container ID, so the value
is unique per replica, stable for the life of the process and maps 1:1 to a real
container (`docker inspect`), which scripts/stop-one.sh relies on.
"""
value = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname()
if INSTANCE_ID_RE.fullmatch(value) is None:
raise ConfigError("INSTANCE_ID must match [A-Za-z0-9._-]{1,64}")
return value
def _int(name: str, default: int, low: int, high: int) -> int:
raw = os.environ.get(name)
if raw is None or raw.strip() == "":
return default
try:
value = int(raw.strip())
except ValueError:
raise ConfigError(f"{name} must be an integer, got {raw!r}") from None
if not low <= value <= high:
raise ConfigError(f"{name} must be in [{low}, {high}], got {value}")
return value
def _float(name: str, default: float, low: float, high: float) -> float:
raw = os.environ.get(name)
if raw is None or raw.strip() == "":
return default
try:
value = float(raw.strip())
except ValueError:
raise ConfigError(f"{name} must be a number, got {raw!r}") from None
if not low <= value <= high:
raise ConfigError(f"{name} must be in [{low}, {high}], got {value}")
return value
def _bool(name: str, default: bool) -> bool:
raw = os.environ.get(name, "").strip().lower()
if raw == "":
return default
if raw not in {"true", "false", "1", "0"}:
raise ConfigError(f"{name} must be true/false, got {raw!r}")
return raw in {"true", "1"}
def _choice(name: str, default: str, allowed: set[str]) -> str:
value = os.environ.get(name, "").strip().lower() or default
if value not in allowed:
raise ConfigError(f"{name} must be one of {sorted(allowed)}, got {value!r}")
return value
def _secret(name: str, required: bool) -> str:
"""Value of <NAME>_FILE (preferred, Docker secret) or <NAME> (local runs/tests)."""
path = os.environ.get(f"{name}_FILE", "").strip()
if path:
try:
value = Path(path).read_text(encoding="utf-8").strip()
except OSError as exc:
# The message names the file, never its content.
raise ConfigError(f"{name}_FILE={path} cannot be read: {exc.strerror}") from None
else:
value = os.environ.get(name, "").strip()
if required and not value:
raise ConfigError(f"{name}_FILE (or {name}) must be set")
return value
@dataclass(frozen=True)
class Settings:
host: str
port: int
env: str
log_level: str
shutdown_timeout: float
max_body_bytes: int
docs_enabled: bool = True
database_url: str = ""
session_store: str = "postgres"
cache_url: str = ""
session_ttl_seconds: int = 3600
db_pool_min: int = 1
db_pool_max: int = 10
db_timeout_seconds: float = 3.0
cache_timeout_seconds: float = 0.5
hash_concurrency: int = 2
instance_id: str = "local"
drain_seconds: float = 0.0
idempotency_ttl_seconds: int = 86400
stats_log_seconds: float = 0.0
message_write_path: str = "function"
# Lab 4: private S3 bucket and attachment limits (see storage.py, attachments.py).
s3_endpoint: str = "http://s3:8333"
s3_region: str = "us-east-1"
s3_bucket: str = "messenger-attachments"
s3_access_key_id: str = ""
s3_secret_access_key: str = ""
s3_timeout_seconds: float = 3.0
s3_connect_timeout_seconds: float = 1.0
s3_max_connections: int = 20
max_upload_bytes: int = 10 * 1024 * 1024
max_image_pixels: int = 40_000_000
upload_concurrency: int = 4
upload_queue_timeout_seconds: float = 5.0
upload_read_timeout_seconds: float = 30.0
image_verify_timeout_seconds: float = 10.0
image_verify_concurrency: int = 1
jpeg_max_memory_mb: int = 256
pending_grace_seconds: int = 900
janitor_interval_seconds: float = 60.0
fault_points: frozenset[str] = frozenset()
@property
def db_statement_timeout_ms(self) -> int:
# The server stops a statement a bit before the client-side deadline fires.
return max(100, int(self.db_timeout_seconds * 1000) - 500)
@classmethod
def from_env(cls) -> "Settings":
log_level = os.environ.get("LOG_LEVEL", "info").strip().lower() or "info"
if log_level not in {"debug", "info", "warning", "error"}:
raise ConfigError(f"LOG_LEVEL must be debug/info/warning/error, got {log_level!r}")
session_store = os.environ.get("SESSION_STORE", "postgres").strip().lower() or "postgres"
if session_store == "redis":
session_store = "valkey" # same protocol; the course allows Redis OR Valkey
if session_store not in {"postgres", "valkey"}:
raise ConfigError(f"SESSION_STORE must be postgres or valkey, got {session_store!r}")
db_pool_min = _int("DB_POOL_MIN", 1, 0, 50)
db_pool_max = _int("DB_POOL_MAX", 10, 1, 100)
if db_pool_min > db_pool_max:
raise ConfigError("DB_POOL_MIN must not exceed DB_POOL_MAX")
return cls(
host=os.environ.get("APP_HOST", "0.0.0.0").strip() or "0.0.0.0",
port=_int("APP_PORT", 8080, 1, 65535),
env=os.environ.get("APP_ENV", "lab").strip() or "lab",
log_level=log_level,
# DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS must stay below compose stop_grace_period.
shutdown_timeout=_float("SHUTDOWN_TIMEOUT_SECONDS", 8.0, 0.0, 30.0),
max_body_bytes=_int("MAX_BODY_BYTES", 64 * 1024, 1024, 1024 * 1024),
docs_enabled=_bool("DOCS_ENABLED", True),
database_url=_secret("DATABASE_URL", required=True),
session_store=session_store,
cache_url=_secret("CACHE_URL", required=session_store == "valkey"),
# Contract name. Minimum 1 s so the expiry test can use 2 s.
session_ttl_seconds=_int("SESSION_TTL_SECONDS", 3600, 1, 30 * 24 * 3600),
db_pool_min=db_pool_min,
db_pool_max=db_pool_max,
db_timeout_seconds=_float("DB_TIMEOUT_SECONDS", 3.0, 0.5, 30.0),
cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.5, 0.05, 10.0),
hash_concurrency=_int("PASSWORD_HASH_CONCURRENCY", 2, 1, 16),
instance_id=_instance_id(),
# After SIGTERM: readiness is 503 but requests are still served for this long,
# so the load balancer stops routing here before the listener closes.
drain_seconds=_float("DRAIN_SECONDS", 0.0, 0.0, 10.0),
# Contract: a replay within 24 h returns the original result.
idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400, 86400, 30 * 24 * 3600),
# Experiments (bench/track1): log DB pool counters and process CPU every N s; 0 = off.
stats_log_seconds=_float("STATS_LOG_SECONDS", 0.0, 0.0, 60.0),
# Track 1: `function` = one SQL call per send (migration 0003), `multi` = the previous
# multi-statement transaction; both have identical semantics (A/B on one image).
message_write_path=_choice("MESSAGE_WRITE_PATH", "function", {"function", "multi"}),
**s3_settings(),
)
# Contract limits (contracts/README.md): configurable DOWN for experiments, never above them.
CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024
CONTRACT_MAX_IMAGE_PIXELS = 40_000_000
# Failure points for the level 4 experiments only (compose.faults.yaml); empty in normal runs.
FAULT_POINTS = {"after_s3_put", "after_ready_commit"}
def s3_settings(*, credentials_required: bool = True) -> dict:
"""S3 part of Settings; also used by the reconciler/bootstrap CLIs with their own keys."""
faults = frozenset(f.strip() for f in os.environ.get("FAULT_POINTS", "").split(",") if f.strip())
if faults - FAULT_POINTS:
raise ConfigError(f"FAULT_POINTS may contain only {sorted(FAULT_POINTS)}")
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").strip() or "http://s3:8333"
if not re.fullmatch(r"https?://[A-Za-z0-9.\-]+(:[0-9]{1,5})?/?", endpoint):
raise ConfigError(f"S3_ENDPOINT must be http(s)://host[:port], got {endpoint!r}")
bucket = os.environ.get("S3_BUCKET", "messenger-attachments").strip() or "messenger-attachments"
if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket):
raise ConfigError(f"S3_BUCKET is not a valid bucket name: {bucket!r}")
return dict(
s3_endpoint=endpoint,
s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1",
s3_bucket=bucket,
s3_access_key_id=_secret("S3_ACCESS_KEY_ID", required=credentials_required),
s3_secret_access_key=_secret("S3_SECRET_ACCESS_KEY", required=credentials_required),
s3_timeout_seconds=_float("S3_TIMEOUT_SECONDS", 3.0, 0.2, 30.0),
s3_connect_timeout_seconds=_float("S3_CONNECT_TIMEOUT_SECONDS", 1.0, 0.1, 10.0),
s3_max_connections=_int("S3_MAX_CONNECTIONS", 20, 1, 200),
max_upload_bytes=_int("MAX_UPLOAD_BYTES", CONTRACT_MAX_UPLOAD_BYTES, 1024, CONTRACT_MAX_UPLOAD_BYTES),
max_image_pixels=_int("MAX_IMAGE_PIXELS", CONTRACT_MAX_IMAGE_PIXELS, 1, CONTRACT_MAX_IMAGE_PIXELS),
# At most this many request bodies of one replica are buffered/checked at once (RAM bound:
# UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES); further uploads wait, then get 503.
upload_concurrency=_int("UPLOAD_CONCURRENCY", 4, 1, 64),
upload_queue_timeout_seconds=_float("UPLOAD_QUEUE_TIMEOUT_SECONDS", 5.0, 0.0, 60.0),
upload_read_timeout_seconds=_float("UPLOAD_READ_TIMEOUT_SECONDS", 30.0, 1.0, 300.0),
image_verify_timeout_seconds=_float("IMAGE_VERIFY_TIMEOUT_SECONDS", 10.0, 1.0, 120.0),
# Full checks running at once per replica: a strict JPEG decode of a 40 Mpx progressive
# file needs up to ~120-240 MB of coefficient buffers (capped by JPEG_MAX_MEMORY_MB).
image_verify_concurrency=_int("IMAGE_VERIFY_CONCURRENCY", 1, 1, 16),
jpeg_max_memory_mb=_int("JPEG_MAX_MEMORY_MB", 256, 16, 2048),
# An intent row still `pending` after this long belongs to a failed upload: the janitor
# deletes its object and the row. Far above the longest possible upload (~45 s).
pending_grace_seconds=_int("PENDING_GRACE_SECONDS", 900, 1, 7 * 24 * 3600),
janitor_interval_seconds=_float("JANITOR_INTERVAL_SECONDS", 60.0, 0.0, 3600.0),
fault_points=faults,
)
+56
View File
@@ -0,0 +1,56 @@
"""Opaque keyset cursors.
A cursor encodes the sort key `(created_at, id)` of the last item on the page
plus the resource it belongs to, e.g. `v1|m|<chat_id>|<ts_us>|<message_id>`,
packed in URL-safe base64. The client treats it as an opaque string; the server
rejects cursors of another chat/user (400) and any non-canonical encoding.
"""
from __future__ import annotations
import base64
import re
from .errors import InvalidRequest
from .validation import UUID_RE
B64_RE = re.compile(r"[A-Za-z0-9_-]{1,512}")
TS_RE = re.compile(r"[0-9]{1,19}")
# Latest instant both Python datetime and PostgreSQL accept: 9999-12-31T23:59:59.999999Z.
MAX_TS_US = 253402300799999999
MESSAGES = "m"
CHATS = "c"
SortKey = tuple[int, str]
def encode(scope: str, owner_id: str, key: SortKey) -> str:
ts, item_id = key
raw = f"v1|{scope}|{owner_id}|{ts}|{item_id}".encode("ascii")
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
def decode(cursor: str, scope: str, owner_id: str) -> SortKey:
"""Return the sort key stored in the cursor or raise InvalidRequest."""
bad = InvalidRequest("Неверный cursor")
if B64_RE.fullmatch(cursor) is None:
raise bad
try:
raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)).decode("ascii")
except (ValueError, UnicodeDecodeError):
raise bad from None
parts = raw.split("|")
if len(parts) != 5 or parts[0] != "v1":
raise bad
_, got_scope, got_owner, ts, item_id = parts
if TS_RE.fullmatch(ts) is None or UUID_RE.fullmatch(item_id) is None or item_id != item_id.lower():
raise bad
key = (int(ts), item_id)
if key[0] > MAX_TS_US:
raise bad # out-of-range time must be 400, not an overflow (500) in datetime/SQL
if encode(got_scope, got_owner, key) != cursor:
raise bad # only the canonical form produced by the server is accepted
if got_scope != scope or got_owner != owner_id:
raise InvalidRequest("cursor относится к другому ресурсу")
return key
+131
View File
@@ -0,0 +1,131 @@
"""PostgreSQL access: a bounded connection pool and a hard deadline for every unit of work.
Three independent limits keep a request from hanging when the database misbehaves:
* pool `timeout` — waiting for a free connection (pool exhausted);
* server `statement_timeout` / `lock_timeout` — PostgreSQL itself cancels a slow
statement or a statement stuck behind a lock;
* client deadline — covers what the server cannot report, e.g. a
paused/frozen PostgreSQL container or a dead network.
The client deadline needs care. Every unit of work runs as its own task; at the deadline
the REQUEST stops waiting and gets 503 at once, but the task is NOT cancelled: cancelling a
psycopg query makes psycopg send a cancel request, and libpq starts that connection
(PQcancelStart) synchronously in the event-loop thread, resolving the host name first.
When the API has lost its network to PostgreSQL (lab 3 review scenario: `docker network
disconnect`), that blocked the WHOLE replica, /health/live included, for ~1 minute.
The abandoned task instead ends by itself, bounded by:
* TCP keepalives + `tcp_user_timeout` (= the deadline): a dead peer or a vanished network
breaks the socket within seconds, the connection is discarded by the pool;
* `statement_timeout` / `lock_timeout` on a live but slow server;
* a frozen (paused) server keeps the connection until it resumes; meanwhile new work waits
at most the pool timeout and gets 503.
Any failure becomes 503 `unavailable` — never 401, never a hang.
"""
from __future__ import annotations
import asyncio
import logging
from collections.abc import Awaitable, Callable
from typing import TypeVar
import psycopg
from psycopg_pool import AsyncConnectionPool, PoolTimeout
from .config import Settings
from .errors import Unavailable
log = logging.getLogger("messenger.db")
T = TypeVar("T")
Work = Callable[[psycopg.AsyncConnection], Awaitable[T]]
# Errors that mean "the database is unavailable right now", not "the request is wrong".
# QueryCanceled (statement_timeout), LockNotAvailable (lock_timeout) and AdminShutdown
# (server restart) are subclasses of OperationalError.
UNAVAILABLE_ERRORS = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TimeoutError)
def _forget(task: asyncio.Task) -> None:
"""Done-callback for abandoned tasks: retrieve the result so nothing is logged as 'never retrieved'."""
if not task.cancelled():
task.exception()
class Database:
def __init__(self, settings: Settings):
self.deadline = settings.db_timeout_seconds
options = (
f"-c statement_timeout={settings.db_statement_timeout_ms} "
f"-c lock_timeout={max(100, settings.db_statement_timeout_ms // 2)} "
"-c idle_in_transaction_session_timeout=10000"
)
self.pool = AsyncConnectionPool(
conninfo=settings.database_url,
min_size=settings.db_pool_min,
max_size=settings.db_pool_max,
timeout=settings.db_timeout_seconds, # max wait for a free connection
open=False,
name="messenger",
check=AsyncConnectionPool.check_connection, # drop connections broken by a DB restart
kwargs={
"options": options,
"connect_timeout": max(1, int(settings.db_timeout_seconds)),
"application_name": "messenger-api",
# Autocommit connections: a single statement (post_message function, ping) is its
# own transaction with ONE round trip; multi-statement work uses an explicit
# conn.transaction() (BEGIN ... COMMIT), see run().
"autocommit": True,
# Detect a dead peer / lost network without any cancel request (see module doc).
"keepalives": 1,
"keepalives_idle": 5,
"keepalives_interval": 1,
"keepalives_count": 3,
"tcp_user_timeout": int(settings.db_timeout_seconds * 1000),
},
)
async def open(self) -> None:
# wait=False: the API starts (liveness 200) even if PostgreSQL is not up yet;
# readiness stays 503 until the database answers.
await self.pool.open(wait=False)
async def close(self) -> None:
await self.pool.close(timeout=2)
async def run(self, work: Work[T], deadline: float | None = None, *, transaction: bool = True) -> T:
"""Run `work(conn)` in one transaction (COMMIT on success, ROLLBACK on any exception).
transaction=False: `work` issues exactly ONE statement, which autocommit makes atomic.
Domain errors raised by `work` (NotFound, Conflict, ...) propagate unchanged;
database unavailability and the deadline become Unavailable (503).
"""
async def unit() -> T:
async with self.pool.connection() as conn:
if not transaction:
return await work(conn)
async with conn.transaction():
return await work(conn)
task = asyncio.ensure_future(unit())
try:
return await asyncio.wait_for(asyncio.shield(task), deadline or self.deadline)
except UNAVAILABLE_ERRORS as exc:
log.warning("database_unavailable", extra={"fields": {"error": type(exc).__name__}})
raise Unavailable("База данных временно недоступна") from None
finally:
if not task.done():
# Deadline or cancelled request: stop waiting now. The task is left to finish
# on its own (no task.cancel(): that would trigger psycopg's blocking cancel).
task.add_done_callback(_forget)
async def ping(self, timeout: float = 1.0) -> bool:
async def select_one(conn: psycopg.AsyncConnection) -> None:
await conn.execute("SELECT 1")
try:
await self.run(select_one, deadline=timeout, transaction=False)
return True
except Unavailable:
return False
+57
View File
@@ -0,0 +1,57 @@
"""Swagger UI for manual experiments: http://127.0.0.1:8080/docs
The page shows the course contract contracts/openapi.json as is (it is not
generated from our code). Only `servers` is replaced with "/" so that the
"Try it out" requests go to the same host the page was opened from.
Swagger UI itself (JS/CSS) is loaded by the browser from the jsDelivr CDN.
Disable with DOCS_ENABLED=false.
"""
from __future__ import annotations
import json
from functools import lru_cache
from pathlib import Path
from starlette.requests import Request
from starlette.responses import HTMLResponse, JSONResponse
SPEC_PATH = Path(__file__).resolve().parents[2] / "contracts" / "openapi.json"
SWAGGER_UI = "https://cdn.jsdelivr.net/npm/swagger-ui-dist@5.33.0"
PAGE = f"""<!doctype html>
<html lang="ru">
<head>
<meta charset="utf-8">
<title>Messenger lab1 — Swagger UI</title>
<link rel="stylesheet" href="{SWAGGER_UI}/swagger-ui.css">
</head>
<body>
<div id="swagger-ui"></div>
<script src="{SWAGGER_UI}/swagger-ui-bundle.js"></script>
<script>
window.ui = SwaggerUIBundle({{
url: "/openapi.json",
dom_id: "#swagger-ui",
tryItOutEnabled: true, // fields are editable right away
persistAuthorization: true, // X-User-Id from "Authorize" survives page reload
displayRequestDuration: true,
}});
</script>
</body>
</html>
"""
@lru_cache(maxsize=1)
def _spec() -> dict:
spec = json.loads(SPEC_PATH.read_text(encoding="utf-8"))
spec["servers"] = [{"url": "/", "description": "этот сервер"}]
return spec
async def swagger_page(request: Request) -> HTMLResponse:
return HTMLResponse(PAGE)
async def openapi_json(request: Request) -> JSONResponse:
return JSONResponse(_spec())
+71
View File
@@ -0,0 +1,71 @@
"""Domain errors mapped 1:1 to the contract error codes (contracts/README.md)."""
from __future__ import annotations
BEARER_CHALLENGE = 'Bearer realm="messenger"'
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
class ApiError(Exception):
status = 500
code = "internal_error"
default_message = "Внутренняя ошибка сервера"
def __init__(self, message: str | None = None):
self.message = message or self.default_message
super().__init__(self.message)
@property
def headers(self) -> dict[str, str]:
return {}
class InvalidRequest(ApiError):
status = 400
code = "invalid_request"
default_message = "Неверный запрос"
class Unauthorized(ApiError):
"""401 with the challenge of the scheme the endpoint expects (RFC 7235)."""
status = 401
code = "unauthorized"
default_message = "Требуется действительная сессия"
def __init__(self, message: str | None = None, challenge: str = BEARER_CHALLENGE):
super().__init__(message)
self.challenge = challenge
@property
def headers(self) -> dict[str, str]:
return {"WWW-Authenticate": self.challenge}
class NotFound(ApiError):
status = 404
code = "not_found"
default_message = "Объект не найден"
class Conflict(ApiError):
status = 409
code = "conflict"
default_message = "Конфликт"
class PayloadTooLarge(ApiError):
status = 413
code = "payload_too_large"
default_message = "Тело запроса слишком большое"
class UnsupportedMediaType(ApiError):
status = 415
code = "unsupported_media_type"
default_message = "Неподдерживаемый тип содержимого"
class Unavailable(ApiError):
status = 503
code = "unavailable"
default_message = "Сервис временно недоступен"
+19
View File
@@ -0,0 +1,19 @@
"""Docker HEALTHCHECK probe without curl/wget in the image: `python -m messenger.healthcheck`."""
from __future__ import annotations
import os
import sys
import urllib.request
def main() -> int:
port = os.environ.get("APP_PORT", "8080").strip() or "8080"
try:
with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/ready", timeout=2) as response:
return 0 if response.status == 200 else 1
except Exception:
return 1
if __name__ == "__main__":
sys.exit(main())
+328
View File
@@ -0,0 +1,328 @@
"""Content checks of uploaded images (lab 4): what the bytes ARE, not what the client says.
Two stages, both with bounded memory and both BEFORE anything is stored:
1. `sniff(head)` — runs on the first bytes while the body is still arriving. Recognises
PNG/JPEG by the signature and reads the dimensions from the header (PNG IHDR, JPEG SOFn).
A foreign format is rejected (415) and a picture with more than MAX_IMAGE_PIXELS pixels
(413) before the rest of the body is read and long before any pixel is decoded.
2. `verify(data, info)` — full integrity check of the complete file, run in a worker thread:
* PNG: our own streaming validator. Every chunk CRC is checked, the IDAT zlib stream is
inflated in 1 MiB steps and DISCARDED (no raster in memory), its size must be exactly
height x (1 + row bytes) (Adam7 per pass), every row filter byte must be 0..4, IEND
must close the file. A truncated file, a broken IDAT or extra/missing pixel data fail.
Pillow is not used for PNG: it would allocate the whole raster (40 Mpx RGBA = 160 MB).
* JPEG: libjpeg-turbo `djpeg -strict` decodes the whole entropy-coded data at 1/8 scale
(DCT scaling: the output raster is 64 times smaller) in a child process; any decoder
warning is fatal, so a truncated file or a file without scan data after SOS fails.
Every problem of the CONTENT is an ImageError carrying the HTTP status (415 or 413);
nothing here may turn user input into a 500.
"""
from __future__ import annotations
import asyncio
import os
import shutil
import struct
import zlib
from dataclasses import dataclass
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
JPEG_SOI = b"\xff\xd8\xff"
PNG = "image/png"
JPEG = "image/jpeg"
# Bits per pixel channel count by PNG colour type, and the bit depths allowed for it.
PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}}
# Adam7 passes: (x start, y start, x step, y step).
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
# JPEG start-of-frame markers that carry the frame size (not DHT C4, JPG C8, DAC CC).
JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF}
# Standalone markers without a length field.
JPEG_STANDALONE = {0x01, *range(0xD0, 0xD8)}
DJPEG = shutil.which("djpeg") or "djpeg" # libjpeg-turbo-progs, installed in the Dockerfile
INPUT_STEP = 1 << 16 # compressed bytes handed to zlib at once
INFLATE_STEP = 1 << 20 # bytes of raw PNG data produced per zlib call
MAX_DIMENSION = 2**31 - 1
class ImageError(Exception):
"""The uploaded bytes are not an acceptable image. `status`: 415 (format/corrupt) or 413."""
def __init__(self, status: int, message: str):
super().__init__(message)
self.status = status
self.message = message
class VerifierUnavailable(Exception):
"""The checker itself could not run to the end (resources): the API answers 503, retryable."""
def unsupported(message: str ="Файл не является корректным изображением PNG или JPEG") -> ImageError:
return ImageError(415, message)
@dataclass(frozen=True, slots=True)
class ImageInfo:
content_type: str
width: int
height: int
def detect(head: bytes) -> str | None:
"""Format by signature; None while fewer bytes than a signature have arrived."""
if head[:8] == PNG_SIGNATURE:
return PNG
if head[:3] == JPEG_SOI:
return JPEG
probe = min(len(head), 8)
if PNG_SIGNATURE.startswith(head[:probe]) or JPEG_SOI.startswith(head[:min(probe, 3)]):
return None
raise unsupported()
def sniff(head: bytes, max_pixels: int) -> ImageInfo | None:
"""Format and size from the beginning of the file, or None if more bytes are needed.
Raises ImageError: 415 for a foreign/broken header, 413 for too many pixels."""
kind = detect(head)
if kind is None:
return None
info = _png_header(head) if kind == PNG else _jpeg_header(head)
if info is None:
return None
if info.width * info.height > max_pixels:
raise ImageError(413, f"Изображение больше {max_pixels} пикселей")
return info
def _png_header(head: bytes) -> ImageInfo | None:
if len(head) < 33:
return None
length, ctype = struct.unpack(">I4s", head[8:16])
if ctype != b"IHDR" or length != 13:
raise unsupported()
if zlib.crc32(head[12:29]) != struct.unpack(">I", head[29:33])[0]:
raise unsupported()
width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", head[16:29])
if not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION):
raise unsupported()
if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1:
raise unsupported()
return ImageInfo(PNG, width, height)
def _jpeg_header(head: bytes) -> ImageInfo | None:
"""Walk the marker segments up to the first SOFn. None = the SOF has not arrived yet."""
i = 2
n = len(head)
while True:
if i >= n:
return None
if head[i] != 0xFF:
raise unsupported()
while i < n and head[i] == 0xFF: # fill bytes
i += 1
if i >= n:
return None
marker = head[i]
i += 1
if marker in JPEG_STANDALONE:
continue
if marker in (0xD8, 0xD9, 0xDA, 0x00):
# Second SOI, EOI or scan data before any frame header: not a usable JPEG.
raise unsupported()
if i + 2 > n:
return None
length = struct.unpack(">H", head[i:i + 2])[0]
if length < 2:
raise unsupported()
if marker in JPEG_SOF:
if i + 7 > n:
return None
if length < 8:
raise unsupported()
height, width = struct.unpack(">HH", head[i + 3:i + 7])
if width == 0 or height == 0: # height 0 = DNL-defined height: not supported
raise unsupported()
return ImageInfo(JPEG, width, height)
i += length
# ------------------------------------------------------------------ full verification
async def verify(data: bytes | bytearray | memoryview, info: ImageInfo, *, timeout: float = 10.0,
jpeg_max_memory_mb: int = 256) -> None:
"""Complete integrity check of the whole file. PNG: our validator in a worker thread (zlib
releases the GIL); JPEG: a strict libjpeg decoder in a child process."""
if info.content_type == PNG:
await asyncio.wait_for(asyncio.to_thread(verify_png, data), timeout)
else:
await verify_jpeg(data, timeout, jpeg_max_memory_mb)
def _png_segments(width: int, height: int, depth: int, color: int, interlace: int) -> list[tuple[int, int]]:
"""(byte count, row stride) of every pass of the raw (filtered) image data."""
bpp = PNG_CHANNELS[color] * depth
passes = ADAM7 if interlace else ((0, 0, 1, 1),)
segments = []
for xs, ys, dx, dy in passes:
pw = (width - xs + dx - 1) // dx if width > xs else 0
ph = (height - ys + dy - 1) // dy if height > ys else 0
if pw and ph:
stride = 1 + (pw * bpp + 7) // 8
segments.append((ph * stride, stride))
return segments
class _RawChecker:
"""Counts inflated bytes against the expected size and checks the filter byte of every row
using C-speed slicing (`piece[first::stride]`), so even 40 000 000 one-pixel rows are fast."""
VALID_FILTERS = bytes(range(5))
def __init__(self, segments: list[tuple[int, int]]):
self.segments = segments
self.index = 0 # current pass
self.offset = 0 # bytes already seen inside the current pass
self.expected = sum(size for size, _ in segments)
self.total = 0
def feed(self, piece: bytes) -> None:
self.total += len(piece)
if self.total > self.expected:
raise unsupported()
pos = 0
while pos < len(piece):
size, stride = self.segments[self.index]
take = min(size - self.offset, len(piece) - pos)
part = piece[pos:pos + take]
first = (-self.offset) % stride
if part[first::stride].translate(None, self.VALID_FILTERS):
raise unsupported()
self.offset += take
pos += take
if self.offset == size:
self.index += 1
self.offset = 0
def verify_png(data: bytes | bytearray | memoryview) -> None:
view = memoryview(data)
if bytes(view[:8]) != PNG_SIGNATURE:
raise unsupported()
pos = 8
n = len(view)
checker: _RawChecker | None = None
inflater = zlib.decompressobj()
color = None
seen_plte = seen_idat = idat_done = False
first = True
while True:
if pos + 12 > n:
raise unsupported() # truncated: no room for another chunk header + CRC
length, ctype = struct.unpack(">I4s", view[pos:pos + 8])
if length > n - pos - 12:
raise unsupported()
body = view[pos + 8:pos + 8 + length]
crc = struct.unpack(">I", view[pos + 8 + length:pos + 12 + length])[0]
if zlib.crc32(body, zlib.crc32(ctype)) != crc:
raise unsupported()
pos += 12 + length
if not all(65 <= c <= 90 or 97 <= c <= 122 for c in ctype):
raise unsupported()
if first:
if ctype != b"IHDR" or length != 13:
raise unsupported()
width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", body)
if (not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION) or color not in PNG_DEPTHS
or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1):
raise unsupported()
checker = _RawChecker(_png_segments(width, height, depth, color, interlace))
first = False
continue
assert checker is not None
if ctype == b"IHDR":
raise unsupported()
if ctype == b"PLTE":
if seen_plte or seen_idat or color in (0, 4) or length % 3 or not 3 <= length <= 768:
raise unsupported()
seen_plte = True
elif ctype == b"IDAT":
if idat_done:
raise unsupported() # IDAT chunks must be consecutive
if color == 3 and not seen_plte:
raise unsupported()
seen_idat = True
_inflate(inflater, body, checker)
elif ctype == b"IEND":
if length != 0 or not seen_idat:
raise unsupported()
if pos != n:
raise unsupported() # bytes after IEND (polyglot files) are not accepted
break
elif not 97 <= ctype[0] <= 122:
raise unsupported() # unknown CRITICAL chunk: a decoder could not show the image
if ctype != b"IDAT" and seen_idat:
idat_done = True
# The zlib stream must end exactly with the last pixel row, without trailing data.
if not inflater.eof or inflater.unused_data or checker.total != checker.expected:
raise unsupported()
def _inflate(inflater, body: memoryview, checker: _RawChecker) -> None:
if inflater.eof:
if len(body):
raise unsupported() # data after the end of the zlib stream
return
# Input is fed in small slices: zlib's `unconsumed_tail` is a COPY of the unread input, so
# feeding the whole 10 MiB IDAT at once would copy the rest of it after every output step.
try:
for start in range(0, len(body), INPUT_STEP):
pending: bytes | memoryview = body[start:start + INPUT_STEP]
while pending:
if inflater.eof:
raise unsupported() # data after the end of the zlib stream
checker.feed(inflater.decompress(pending, INFLATE_STEP))
pending = inflater.unconsumed_tail
if inflater.eof and inflater.unused_data:
raise unsupported()
except zlib.error:
raise unsupported() from None
async def verify_jpeg(data: bytes | bytearray | memoryview, timeout: float, max_memory_mb: int) -> None:
"""Decode the WHOLE file with the reference decoder (libjpeg-turbo `djpeg`) in strict mode.
`-strict` turns every libjpeg warning into a failure: "premature end of data segment"
(scan data missing or cut before EOI), "premature end of JPEG file", corrupt Huffman data.
Pillow cannot be used for this: it ignores libjpeg warnings and returns a grey picture for
a JPEG whose scan data is missing. `-scale 1/8` decodes every block but writes a 64x
smaller raster to /dev/null; `-maxmemory` caps libjpeg's working memory (coefficient
buffers of progressive files); the separate process is killed at the deadline.
"""
process = await asyncio.create_subprocess_exec(
DJPEG, "-strict", "-scale", "1/8", "-maxmemory", str(max_memory_mb * 1024), "-outfile", os.devnull,
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE)
stderr = b""
try:
_, stderr = await asyncio.wait_for(process.communicate(data), timeout)
except (asyncio.TimeoutError, asyncio.CancelledError):
process.kill()
await process.wait()
raise
except (BrokenPipeError, ConnectionResetError):
await process.wait() # djpeg stopped reading: the file is already known to be broken
if process.returncode is not None and process.returncode < 0:
# Killed by a signal (the cgroup OOM killer under memory pressure): a problem of the
# SERVER, not of the file - never report it as 415.
raise VerifierUnavailable()
if process.returncode != 0:
if b"Insufficient memory" in (stderr or b""):
# A valid but extremely memory-hungry file (progressive, 4:4:4, ~40 Mpx): a limit, not corruption.
raise ImageError(413, "Изображение слишком велико для обработки")
raise unsupported()
+36
View File
@@ -0,0 +1,36 @@
"""JSON-lines logging to stdout. Logs never contain message texts or request bodies."""
from __future__ import annotations
import datetime as dt
import json
import logging
import sys
class JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
entry = {
"ts": dt.datetime.fromtimestamp(record.created, dt.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z"),
"level": record.levelname.lower(),
"logger": record.name,
"event": record.getMessage(),
}
fields = getattr(record, "fields", None)
if fields:
entry.update(fields)
if record.exc_info:
# Stack traces go only to the server log, never to the HTTP client.
entry["exception"] = self.formatException(record.exc_info)
return json.dumps(entry, ensure_ascii=False)
def configure(level: str) -> None:
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
root = logging.getLogger()
root.handlers[:] = [handler]
root.setLevel(level.upper())
# uvicorn's own access log is disabled; our middleware writes one line per request.
for name in ("uvicorn", "uvicorn.error"):
logging.getLogger(name).handlers[:] = []
logging.getLogger(name).propagate = True
+76
View File
@@ -0,0 +1,76 @@
"""Versioned SQL migrations: `python -m messenger.migrate` (compose service `migrate`).
* Files `migrations/NNNN_name.sql` are applied in name order, each in its own
transaction together with its row in `schema_migrations` (DDL is transactional
in PostgreSQL, so a failed migration leaves nothing behind).
* Re-running is safe: applied versions are skipped, so data is never duplicated.
* The SHA-256 of every applied file is stored; if an applied file was edited later,
the run stops instead of silently diverging from the real schema.
* `pg_advisory_lock` serialises concurrent runs (two API replicas starting at once).
"""
from __future__ import annotations
import hashlib
import sys
import time
from pathlib import Path
import psycopg
from .config import ConfigError, _secret
MIGRATIONS_DIR = Path(__file__).resolve().parents[2] / "migrations"
LOCK_ID = 2026_02 # arbitrary application-wide advisory lock id
def say(text: str) -> None:
print(f"migrate: {text}", flush=True)
def connect(url: str, wait_seconds: float = 60) -> psycopg.Connection:
deadline = time.monotonic() + wait_seconds
while True:
try:
return psycopg.connect(url, autocommit=True, connect_timeout=3, application_name="messenger-migrate")
except psycopg.OperationalError as exc:
if time.monotonic() > deadline:
raise SystemExit(f"migrate: database is not reachable: {type(exc).__name__}") from None
time.sleep(1)
def main() -> int:
try:
url = _secret("DATABASE_URL", required=True)
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
files = sorted(MIGRATIONS_DIR.glob("*.sql"))
with connect(url) as conn:
conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,))
try:
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
" version text PRIMARY KEY, checksum text NOT NULL,"
" applied_at timestamptz NOT NULL DEFAULT now())")
applied = dict(conn.execute("SELECT version, checksum FROM schema_migrations").fetchall())
for path in files:
version, sql = path.stem, path.read_bytes()
checksum = hashlib.sha256(sql).hexdigest()
if version in applied:
if applied[version] != checksum:
say(f"{version}: file changed after it was applied; create a new migration instead")
return 1
say(f"{version}: already applied, skipped")
continue
with conn.transaction():
conn.execute(sql.decode("utf-8"))
conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)", (version, checksum))
say(f"{version}: applied")
finally:
conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
say(f"done, {len(files)} migration file(s) checked")
return 0
if __name__ == "__main__":
sys.exit(main())
+53
View File
@@ -0,0 +1,53 @@
"""Password hashing with Argon2id (memory-hard password KDF; also mandatory in lab 7).
* Parameters follow the OWASP minimum for Argon2id: m=19 MiB, t=2, p=1. Unlike bcrypt
there is no 72-byte limit, so any 12-128 code point UTF-8 password is hashed whole.
* Hashing takes tens of milliseconds of CPU and 19 MiB of RAM. It runs in a worker
thread (argon2-cffi releases the GIL), so the event loop keeps serving other
requests, and a semaphore caps parallel hashes (memory/CPU DoS protection).
* An unknown username is checked against a dummy hash, so "no such user" and
"wrong password" take the same time and return the same 401.
"""
from __future__ import annotations
import asyncio
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
_hasher = PasswordHasher(time_cost=2, memory_cost=19 * 1024, parallelism=1, hash_len=32, salt_len=16)
_semaphore: asyncio.Semaphore | None = None
_dummy_hash: str | None = None
def configure(concurrency: int) -> None:
global _semaphore
_semaphore = asyncio.Semaphore(concurrency)
async def _run(func, *args):
if _semaphore is None:
configure(2)
async with _semaphore:
return await asyncio.to_thread(func, *args)
async def hash_password(password: str) -> str:
return await _run(_hasher.hash, password)
def _verify(stored_hash: str, password: str) -> bool:
try:
return _hasher.verify(stored_hash, password)
except (VerifyMismatchError, VerificationError, InvalidHashError):
return False
async def verify_password(stored_hash: str | None, password: str) -> bool:
global _dummy_hash
if stored_hash is None:
if _dummy_hash is None:
_dummy_hash = await hash_password("dummy-password-for-timing")
await _run(_verify, _dummy_hash, password)
return False
return await _run(_verify, stored_hash, password)
+437
View File
@@ -0,0 +1,437 @@
"""Reconciler of the two stores (lab 4, track 2): S3 objects vs attachment rows in PostgreSQL.
docker compose run --rm reconciler # dry-run: report only (default)
docker compose run --rm reconciler --apply # fix what is safe to fix
docker compose run --rm reconciler --verify-checksums # also read every object (SHA-256)
How it walks: both sides are read as key-ordered streams and merge-joined - ListObjectsV2
pages (prefix o/, byte order) against keyset pages of `attachments ORDER BY object_key`
(COLLATE "C" = the same byte order, unique index). Memory does not depend on the bucket size;
cost is ~N/1000 S3 list requests + N/page DB pages (measured: docs/CONSISTENCY.md).
What it finds and what --apply does (nothing else is ever changed):
kind condition --apply
orphan_object object, no row, older than --grace delete the object
stale_intent row 'pending' older than --grace (+/- object) delete object, then row
missing_object row 'ready', object absent, older than --grace quarantine the row
size_mismatch row 'ready', object size != size_bytes quarantine the row
checksum_mismatch row 'ready', SHA-256 differs (--verify-checksums) quarantine the row
restored_object row quarantined, object present and correct again remove the quarantine
(a checksum quarantine only with --verify-checksums)
fresh_* the same, but younger than --grace nothing (in-flight upload)
Safety:
* dry-run is read-only BY CONSTRUCTION: the PostgreSQL session runs with
default_transaction_read_only=on and the S3 wrapper has no write methods, so even a bug in
the planner cannot modify a store;
* ready rows are never deleted - a used (or unused) ready attachment can only be quarantined,
which is reversible; objects that have a row are never deleted unless the row is a stale
intent; anything younger than --grace is never touched;
* every action re-checks its condition right before acting, under a row lock (FOR UPDATE)
where a row exists, so the live API racing with the reconciler cannot be hurt;
* object first, row second: an interrupted run leaves a state the next run repairs;
re-running is idempotent (the second run finds nothing to do);
* safety valve: --apply refuses to delete more than --max-orphan-fraction of the scanned
objects (pointing it at the wrong/empty database would otherwise wipe the bucket).
"""
from __future__ import annotations
import argparse
import asyncio
import datetime as dt
import hashlib
import json
import sys
import time
from collections import Counter
from collections.abc import AsyncIterator
from dataclasses import dataclass, field
import psycopg
from .config import ConfigError, Settings, _float, _secret, s3_settings
from .errors import Unavailable
from .storage import OBJECT_PREFIX, ListedObject, ObjectMissing, Storage
KINDS = ("orphan_object", "stale_intent", "missing_object", "size_mismatch", "checksum_mismatch",
"restored_object", "fresh_orphan_object", "fresh_intent", "fresh_missing_object")
MIN_SAFE_GRACE = 60.0 # > the longest possible upload (queue 5 s + body 30 s + verify 10 s + S3)
ACTION_OF = {"orphan_object": "delete_object", "stale_intent": "delete_intent", "missing_object": "quarantine",
"size_mismatch": "quarantine", "checksum_mismatch": "quarantine", "restored_object": "unquarantine"}
def log(text: str) -> None:
print(f"reconcile: {text}", file=sys.stderr, flush=True)
@dataclass(frozen=True, slots=True)
class Row:
id: str
key: str
status: str
size: int
sha256: bytes
created_at: dt.datetime
updated_at: dt.datetime
lost_reason: str | None
used: bool
@dataclass(slots=True)
class Finding:
kind: str
key: str
attachment_id: str | None = None
used: bool = False
detail: str = ""
result: str = "planned" # planned | applied | skipped:<why> | failed:<why>
@dataclass
class Stats:
objects: int = 0
rows: int = 0
consistent: int = 0
s3_list_requests: int = 0
s3_head_requests: int = 0
s3_get_bytes: int = 0
db_pages: int = 0
class ReadOnlyStorage:
"""The dry-run view of S3: listing and reading only - there is no delete to call."""
def __init__(self, storage: Storage):
self._s = storage
async def list_page(self, *args, **kwargs):
return await self._s.list_page(*args, **kwargs)
async def head(self, key: str):
return await self._s.head(key)
async def read_all(self, key: str, limit: int) -> bytes:
return await self._s.read_all(key, limit)
class Reconciler:
def __init__(self, conn: psycopg.AsyncConnection, storage: Storage | ReadOnlyStorage, *, grace: float,
page_size: int, verify: bool, apply: bool):
self.conn = conn
self.s3 = storage
self.grace = grace
self.page_size = page_size
self.verify = verify
self.apply = apply
self.stats = Stats()
self.findings: list[Finding] = []
# ---------------------------------------------------------- streams
async def objects(self) -> AsyncIterator[ListedObject]:
after = None
while True:
page, more = await self.s3.list_page(OBJECT_PREFIX, after, self.page_size)
self.stats.s3_list_requests += 1
for item in page:
self.stats.objects += 1
yield item
if not more or not page:
return
after = page[-1].key
async def rows(self) -> AsyncIterator[Row]:
after = ""
while True:
cur = await self.conn.execute(
"SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256, a.created_at, a.updated_at, "
"a.lost_reason, EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) "
"FROM attachments a WHERE a.object_key > %s ORDER BY a.object_key LIMIT %s",
(after, self.page_size))
page = await cur.fetchall()
self.stats.db_pages += 1
for r in page:
self.stats.rows += 1
yield Row(str(r[0]), r[1], r[2], r[3], bytes(r[4]), r[5], r[6], r[7], r[8])
if len(page) < self.page_size:
return
after = page[-1][1]
# ---------------------------------------------------------- planning (read-only)
async def scan(self) -> None:
cur = await self.conn.execute("SELECT clock_timestamp()")
self.db_now = (await cur.fetchone())[0]
self.s3_now = time.time()
objects, rows = self.objects(), self.rows()
obj = await anext(objects, None)
row = await anext(rows, None)
while obj is not None or row is not None:
if row is None or (obj is not None and obj.key < row.key):
self.object_only(obj)
obj = await anext(objects, None)
elif obj is None or row.key < obj.key:
self.row_only(row)
row = await anext(rows, None)
else:
await self.both(obj, row)
obj = await anext(objects, None)
row = await anext(rows, None)
def row_old(self, row: Row) -> bool:
return (self.db_now - max(row.created_at, row.updated_at)).total_seconds() > self.grace
def object_only(self, obj: ListedObject) -> None:
old = self.s3_now - obj.last_modified_ts > self.grace
self.add(Finding("orphan_object" if old else "fresh_orphan_object", obj.key))
def row_only(self, row: Row) -> None:
if row.status == "pending":
self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id,
detail="object absent"))
elif row.lost_reason is not None:
self.stats.consistent += 1 # already quarantined, still missing: nothing new
else:
self.add(Finding("missing_object" if self.row_old(row) else "fresh_missing_object", row.key, row.id,
used=row.used))
async def both(self, obj: ListedObject, row: Row) -> None:
if row.status == "pending":
self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id,
detail="object present"))
return
problem = await self.object_problem(row, obj.size)
if problem:
if row.lost_reason is None:
self.add(Finding(problem, row.key, row.id, used=row.used))
else:
self.stats.consistent += 1 # quarantined and still broken
elif row.lost_reason == "checksum_mismatch" and not self.verify:
# Same size, but this pass does not read the bytes: it cannot prove the object is the
# original again, so a checksum quarantine is lifted only by --verify-checksums.
self.stats.consistent += 1
elif row.lost_reason is not None:
self.add(Finding("restored_object", row.key, row.id, used=row.used))
else:
self.stats.consistent += 1
async def object_problem(self, row: Row, size: int) -> str | None:
if size != row.size:
return "size_mismatch"
if self.verify:
data = await self.s3.read_all(row.key, row.size)
self.stats.s3_get_bytes += len(data)
if hashlib.sha256(data).digest() != row.sha256:
return "checksum_mismatch"
return None
def add(self, finding: Finding) -> None:
if finding.kind not in ACTION_OF:
finding.result = "not_actionable"
self.findings.append(finding)
# ---------------------------------------------------------- actions (--apply only)
async def run_actions(self, max_orphan_fraction: float) -> None:
orphans = sum(1 for f in self.findings if f.kind == "orphan_object")
if self.stats.objects and orphans > max_orphan_fraction * self.stats.objects:
raise SafetyValve(f"{orphans} of {self.stats.objects} objects look orphaned "
f"(> {max_orphan_fraction:.0%}); wrong database? nothing was changed")
handlers = {"delete_object": self.delete_object, "delete_intent": self.delete_intent,
"quarantine": self.quarantine, "unquarantine": self.unquarantine}
for finding in self.findings:
action = ACTION_OF.get(finding.kind)
if action is None:
continue
try:
finding.result = await handlers[action](finding)
except (Unavailable, psycopg.OperationalError) as exc:
finding.result = f"failed:{type(exc).__name__}"
raise # a store is down: stop; the next run continues from the committed state
async def delete_object(self, f: Finding) -> str:
cur = await self.conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (f.key,))
if await cur.fetchone():
return "skipped:row_appeared"
try:
head = await self.s3.head(f.key)
self.stats.s3_head_requests += 1
except ObjectMissing:
return "skipped:already_gone"
if time.time() - head.last_modified_ts <= self.grace:
return "skipped:object_rewritten_recently"
# No row names this key and the object is older than the grace period. Rows are always
# written BEFORE their object and keys are random and never reused, so no upload can
# still claim it.
await self.s3.delete(f.key)
return "applied"
async def delete_intent(self, f: Finding) -> str:
async with self.conn.transaction():
cur = await self.conn.execute(
"SELECT 1 FROM attachments WHERE id = %s AND status = 'pending' "
"AND greatest(created_at, updated_at) < clock_timestamp() - make_interval(secs => %s) FOR UPDATE",
(f.attachment_id, self.grace))
if await cur.fetchone() is None:
return "skipped:no_longer_stale_intent"
await self.s3.delete(f.key) # object first (idempotent), then the row
await self.conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'pending'",
(f.attachment_id,))
return "applied"
async def quarantine(self, f: Finding) -> str:
async with self.conn.transaction():
cur = await self.conn.execute(
"SELECT size_bytes, sha256 FROM attachments WHERE id = %s AND status = 'ready' "
"AND lost_at IS NULL FOR UPDATE", (f.attachment_id,))
row = await cur.fetchone()
if row is None:
return "skipped:row_changed"
reason = await self.current_problem(f.key, row[0], bytes(row[1]))
if reason is None:
return "skipped:object_ok_now"
await self.conn.execute(
"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = %s WHERE id = %s",
(reason, f.attachment_id))
return "applied"
async def unquarantine(self, f: Finding) -> str:
async with self.conn.transaction():
cur = await self.conn.execute(
"SELECT size_bytes, sha256, lost_reason FROM attachments WHERE id = %s AND lost_at IS NOT NULL FOR UPDATE",
(f.attachment_id,))
row = await cur.fetchone()
if row is None:
return "skipped:row_changed"
if row[2] == "checksum_mismatch" and not self.verify:
return "skipped:needs_verify_checksums"
if await self.current_problem(f.key, row[0], bytes(row[1])) is not None:
return "skipped:object_still_bad"
await self.conn.execute("UPDATE attachments SET lost_at = NULL, lost_reason = NULL WHERE id = %s",
(f.attachment_id,))
return "applied"
async def current_problem(self, key: str, size: int, sha256: bytes) -> str | None:
"""Fresh look at the object right before changing the row (the listing may be stale)."""
try:
head = await self.s3.head(key)
self.stats.s3_head_requests += 1
except ObjectMissing:
return "missing_object"
if head.size != size:
return "size_mismatch"
if self.verify:
data = await self.s3.read_all(key, size)
self.stats.s3_get_bytes += len(data)
if hashlib.sha256(data).digest() != sha256:
return "checksum_mismatch"
return None
# ---------------------------------------------------------- report
def report(self, mode: str, seconds: float, error: str | None, sample: int) -> dict:
kinds = Counter(f.kind for f in self.findings)
results = Counter(f.result.split(":")[0] for f in self.findings if f.kind in ACTION_OF)
samples = {}
for f in self.findings:
items = samples.setdefault(f.kind, [])
if len(items) < sample:
items.append({"key": f.key, "attachment_id": f.attachment_id, "used": f.used,
"result": f.result, **({"detail": f.detail} if f.detail else {})})
return {
"mode": mode, "grace_seconds": self.grace, "verify_checksums": self.verify,
"duration_seconds": round(seconds, 3), "error": error,
"scanned": {"objects": self.stats.objects, "rows": self.stats.rows, "consistent": self.stats.consistent},
"findings": {k: kinds.get(k, 0) for k in KINDS},
"used_attachments_in_findings": sum(1 for f in self.findings if f.used),
"actions": {"planned": sum(1 for f in self.findings if f.kind in ACTION_OF),
"applied": results.get("applied", 0), "skipped": results.get("skipped", 0),
"failed": results.get("failed", 0)},
"cost": {"s3_list_requests": self.stats.s3_list_requests, "s3_head_requests": self.stats.s3_head_requests,
"s3_get_bytes": self.stats.s3_get_bytes, "db_pages": self.stats.db_pages,
"page_size": self.page_size},
"samples": samples,
}
class SafetyValve(Exception):
pass
def arguments(argv: list[str] | None = None) -> argparse.Namespace:
p = argparse.ArgumentParser(prog="python -m messenger.reconcile", description=__doc__.split("\n")[0])
p.add_argument("--apply", action="store_true", help="fix findings (default: dry-run, read-only)")
p.add_argument("--grace", type=float, default=3600.0,
help="seconds; anything younger is never touched (default 3600, >= PENDING_GRACE_SECONDS)")
p.add_argument("--allow-small-grace", action="store_true",
help="permit --grace below 60 s (tests only: a grace shorter than an upload could remove the "
"intent of an upload that is still running; that upload then fails with 503)")
p.add_argument("--page-size", type=int, default=1000, help="S3 list / DB page size (1..1000)")
p.add_argument("--verify-checksums", action="store_true", help="read every ready object and compare SHA-256")
p.add_argument("--samples", type=int, default=20, help="keys listed per finding kind in the report")
p.add_argument("--max-orphan-fraction", type=float, default=0.5,
help="--apply refuses if more than this share of objects look orphaned (default 0.5)")
args = p.parse_args(argv)
if not 1 <= args.page_size <= 1000:
p.error("--page-size must be 1..1000")
if args.grace < 1 or (args.grace < MIN_SAFE_GRACE and not args.allow_small_grace):
p.error(f"--grace must be >= {MIN_SAFE_GRACE:.0f} s (or >= 1 s with --allow-small-grace)")
if not 0 < args.max_orphan_fraction <= 1:
p.error("--max-orphan-fraction must be in (0, 1]")
return args
async def run(args: argparse.Namespace) -> int:
url = _secret("DATABASE_URL", required=True)
base = s3_settings()
settings = Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024,
database_url=url, **base)
timeout_ms = int(_float("DB_TIMEOUT_SECONDS", 10.0, 1.0, 120.0) * 1000)
options = f"-c statement_timeout={timeout_ms} -c lock_timeout={timeout_ms // 2}"
if not args.apply:
options += " -c default_transaction_read_only=on" # dry-run: PostgreSQL itself refuses writes
mode = "apply" if args.apply else "dry-run"
storage = Storage(settings)
await storage.open()
started = time.perf_counter()
error = None
code = 0
try:
conn = await psycopg.AsyncConnection.connect(url, autocommit=True, connect_timeout=5, options=options,
application_name=f"messenger-reconcile-{mode}")
except psycopg.OperationalError as exc:
await storage.close()
error = f"store_unavailable: database {type(exc).__name__}"
print(json.dumps({"mode": mode, "error": error}, ensure_ascii=False))
log(error)
return 1
rec = Reconciler(conn, storage if args.apply else ReadOnlyStorage(storage), grace=args.grace,
page_size=args.page_size, verify=args.verify_checksums, apply=args.apply)
try:
log(f"{mode}: scanning bucket {settings.s3_bucket} (prefix {OBJECT_PREFIX}) against table attachments")
await rec.scan()
log(f"scan done: {rec.stats.objects} objects, {rec.stats.rows} rows, {len(rec.findings)} findings")
if args.apply:
await rec.run_actions(args.max_orphan_fraction)
except SafetyValve as exc:
error, code = f"safety_valve: {exc}", 3
except (Unavailable, psycopg.OperationalError) as exc:
error, code = f"store_unavailable: {type(exc).__name__}", 1
finally:
await conn.close()
await storage.close()
print(json.dumps(rec.report(mode, time.perf_counter() - started, error, args.samples), ensure_ascii=False,
indent=1))
if error:
log(error)
return code
def main(argv: list[str] | None = None) -> int:
args = arguments(argv)
try:
return asyncio.run(run(args))
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())
+372
View File
@@ -0,0 +1,372 @@
"""SQL for users, chats and messages. All statements are parameterised (%s placeholders):
values travel separately from the SQL text, so input can never change the query.
Free text is stored as UTF-8 BYTEA (PostgreSQL TEXT cannot hold U+0000), so it is
encoded on the way in and decoded on the way out. See docs/DATA_MODEL.md.
"""
from __future__ import annotations
import datetime as dt
import uuid
from dataclasses import dataclass
import psycopg
from psycopg import errors as pg_errors
from .cursor import SortKey
from .db import Database
from .errors import Conflict, InvalidRequest, NotFound, Unavailable
from .timefmt import from_us, rfc3339, to_us
def _b(text: str) -> bytes:
return text.encode("utf-8")
def _s(raw: bytes | memoryview) -> str:
return bytes(raw).decode("utf-8")
@dataclass(slots=True, frozen=True)
class User:
id: str
username: str
display_name: str
created_at: dt.datetime
def to_json(self) -> dict:
return {"id": self.id, "username": self.username, "display_name": self.display_name,
"created_at": rfc3339(self.created_at)}
@dataclass(slots=True, frozen=True)
class Chat:
id: str
title: str
member_ids: tuple[str, ...]
created_at: dt.datetime
@property
def key(self) -> SortKey:
return (to_us(self.created_at), self.id)
def to_json(self) -> dict:
return {"id": self.id, "title": self.title, "member_ids": list(self.member_ids),
"created_at": rfc3339(self.created_at)}
@dataclass(slots=True, frozen=True)
class Message:
id: str
chat_id: str
sender_id: str
text: str
created_at: dt.datetime
attachment_ids: tuple[str, ...] = ()
@property
def key(self) -> SortKey:
return (to_us(self.created_at), self.id)
def to_json(self) -> dict:
return {"id": self.id, "chat_id": self.chat_id, "sender_id": self.sender_id, "text": self.text,
"created_at": rfc3339(self.created_at), "attachment_ids": list(self.attachment_ids)}
def _user(row) -> User:
return User(str(row[0]), row[1], _s(row[2]), row[3])
# ------------------------------------------------------------------ users
async def create_user(db: Database, username: str, display_name: str, password_hash: str) -> User:
async def work(conn: psycopg.AsyncConnection) -> User:
cur = await conn.execute(
"INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s) "
"RETURNING id, username, display_name, created_at",
(uuid.uuid4(), username, _b(display_name), password_hash))
return _user(await cur.fetchone())
try:
return await db.run(work)
except pg_errors.UniqueViolation:
# Two concurrent registrations: the unique index lets exactly one INSERT commit.
raise Conflict("Имя пользователя уже занято") from None
async def get_user(db: Database, user_id: str) -> User | None:
async def work(conn: psycopg.AsyncConnection) -> User | None:
cur = await conn.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s", (user_id,))
row = await cur.fetchone()
return _user(row) if row else None
return await db.run(work)
async def get_credentials(db: Database, username: str) -> tuple[User, str] | None:
async def work(conn: psycopg.AsyncConnection) -> tuple[User, str] | None:
cur = await conn.execute(
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
(username,))
row = await cur.fetchone()
return (_user(row), row[4]) if row else None
return await db.run(work)
# ------------------------------------------------------------------ chats
async def create_chat(db: Database, creator_id: str, title: str, member_ids: list[str]) -> Chat:
"""Atomic: the chat row and all memberships commit together or not at all.
The chat is inserted FIRST; an unknown member then fails the membership INSERT
with a foreign-key violation and the whole transaction (chat row included) is
rolled back. student-tests check that nothing is left behind.
"""
if creator_id not in member_ids:
raise InvalidRequest("member_ids должен включать текущего пользователя")
chat_id = uuid.uuid4()
async def work(conn: psycopg.AsyncConnection) -> dt.datetime:
cur = await conn.execute(
"INSERT INTO chats (id, title, created_by) VALUES (%s, %s, %s) RETURNING created_at",
(chat_id, _b(title), creator_id))
(created_at,) = await cur.fetchone()
await conn.execute(
"INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) "
"SELECT %s, m.user_id, m.ord - 1, %s "
"FROM unnest(%s::uuid[]) WITH ORDINALITY AS m(user_id, ord)",
(chat_id, created_at, member_ids))
return created_at
try:
created_at = await db.run(work)
except pg_errors.ForeignKeyViolation:
raise InvalidRequest("member_ids содержит несуществующих пользователей") from None
return Chat(str(chat_id), title, tuple(member_ids), created_at)
async def _load_chats(conn: psycopg.AsyncConnection, rows) -> list[Chat]:
if not rows:
return []
ids = [r[0] for r in rows]
cur = await conn.execute(
"SELECT chat_id, user_id FROM chat_members WHERE chat_id = ANY(%s) ORDER BY chat_id, position", (ids,))
members: dict[uuid.UUID, list[str]] = {i: [] for i in ids}
for chat_id, user_id in await cur.fetchall():
members[chat_id].append(str(user_id))
return [Chat(str(r[0]), _s(r[1]), tuple(members[r[0]]), r[2]) for r in rows]
async def get_chat(db: Database, user_id: str, chat_id: str) -> Chat:
"""The chat if the user is a member; a missing chat and a foreign chat are both 404."""
async def work(conn: psycopg.AsyncConnection) -> Chat:
cur = await conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s",
(user_id, chat_id))
row = await cur.fetchone()
if row is None:
raise NotFound("Чат не найден")
return (await _load_chats(conn, [row]))[0]
return await db.run(work)
async def _check_member(conn: psycopg.AsyncConnection, user_id: str, chat_id: str) -> None:
cur = await conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id))
if await cur.fetchone() is None:
raise NotFound("Чат не найден")
async def ensure_member(db: Database, user_id: str, chat_id: str) -> None:
async def work(conn: psycopg.AsyncConnection) -> None:
await _check_member(conn, user_id, chat_id)
await db.run(work)
async def list_chats(db: Database, user_id: str, after: SortKey | None, limit: int) -> tuple[list[Chat], bool]:
async def work(conn: psycopg.AsyncConnection) -> tuple[list[Chat], bool]:
if after is None:
cur = await conn.execute(
"SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id "
"WHERE m.user_id = %s ORDER BY m.chat_created_at, m.chat_id LIMIT %s",
(user_id, limit + 1))
else:
cur = await conn.execute(
"SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id "
"WHERE m.user_id = %s AND (m.chat_created_at, m.chat_id) > (%s, %s) "
"ORDER BY m.chat_created_at, m.chat_id LIMIT %s",
(user_id, from_us(after[0]), after[1], limit + 1))
rows = await cur.fetchall()
return await _load_chats(conn, rows[:limit]), len(rows) > limit
return await db.run(work)
# ------------------------------------------------------------------ messages
@dataclass(slots=True, frozen=True)
class PostResult:
message: Message
replayed: bool # True: the key was already used with this body, nothing new was written
async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID, user_id: str, chat_id: str,
text: str, attachment_ids: list[str]) -> dt.datetime:
"""Insert with a per-chat strictly increasing timestamp.
`UPDATE chats ... RETURNING` locks the chat row until COMMIT, so writers of one
chat are serialised: a message with a smaller created_at can never commit after
a message with a bigger one, and a reader paging by cursor cannot skip it.
"""
cur = await conn.execute(
"UPDATE chats SET last_message_at = greatest(clock_timestamp(), "
"last_message_at + interval '1 microsecond') WHERE id = %s RETURNING last_message_at",
(chat_id,))
(created_at,) = await cur.fetchone()
await conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) "
"VALUES (%s, %s, %s, %s, %s, %s::uuid[])",
(message_id, chat_id, user_id, _b(text), created_at, attachment_ids))
if attachment_ids:
await conn.execute(
"INSERT INTO message_attachments (message_id, position, attachment_id) "
"SELECT %s, a.ord - 1, a.id FROM unnest(%s::uuid[]) WITH ORDINALITY AS a(id, ord)",
(message_id, attachment_ids))
return created_at
# Attachments a message may reference: ready, of THIS chat (else 404), not quarantined (else 409).
# FOR SHARE keeps a concurrent quarantine (reconciler) from slipping in before COMMIT.
CHECK_ATTACHMENTS = (
"SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL) FROM ("
"SELECT lost_at FROM attachments WHERE id = ANY(%s::uuid[]) AND chat_id = %s AND status = 'ready' "
"FOR SHARE) a")
async def _check_attachments(conn: psycopg.AsyncConnection, chat_id: str, attachment_ids: list[str]) -> str:
if not attachment_ids:
return "ok"
cur = await conn.execute(CHECK_ATTACHMENTS, (attachment_ids, chat_id))
found, usable = await cur.fetchone()
if found != len(attachment_ids):
return "attachment_not_found"
return "ok" if usable == found else "attachment_conflict"
def _attachment_error(status: str) -> None:
if status == "attachment_not_found":
raise NotFound("Вложение не найдено")
if status == "attachment_conflict":
raise Conflict("Вложение недоступно и не может быть прикреплено")
# Claim the key. A row of a concurrent transaction that is not committed yet makes this
# INSERT wait on the unique index until that transaction ends: after COMMIT it is a conflict
# (the key is taken), after ROLLBACK the INSERT goes through. A key older than the TTL is
# re-used as if it were new (ON CONFLICT ... DO UPDATE ... WHERE expired).
CLAIM_KEY = (
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) "
"VALUES (%s, %s, %s, %s, %s) "
"ON CONFLICT (user_id, chat_id, key) DO UPDATE SET "
"request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp() "
"WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => %s) "
"RETURNING 1")
STORED_RESULT = (
"SELECT k.request_hash, m.id, m.text, m.created_at, m.attachment_ids FROM idempotency_keys k "
"JOIN messages m ON m.id = k.message_id "
"WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s")
async def post_message(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes, text: str,
ttl_seconds: int, attachment_ids: list[str] = ()) -> PostResult:
"""Idempotent send: the key, the message and the stored result commit in ONE transaction.
* first request -> key row + message + chat clock, all or nothing (no "pending" key is
ever left behind: a crash or 5xx rolls everything back and the client may retry);
* same key, same body (any replica, also concurrently) -> the stored Message, 201, replayed;
* same key, other body -> 409; the membership check comes first, so outsiders always get 404;
* attachments (lab 4) are checked before the key is claimed (same order as post_message_v2).
"""
message_id = uuid.uuid4()
attachment_ids = list(attachment_ids)
async def work(conn: psycopg.AsyncConnection) -> PostResult:
await _check_member(conn, user_id, chat_id)
check = await _check_attachments(conn, chat_id, attachment_ids)
if check == "ok":
cur = await conn.execute(CLAIM_KEY, (user_id, chat_id, key, request_hash, message_id, ttl_seconds))
if await cur.fetchone() is not None:
created_at = await _insert_message(conn, message_id, user_id, chat_id, text, attachment_ids)
return PostResult(Message(str(message_id), chat_id, user_id, text, created_at,
tuple(attachment_ids)), False)
cur = await conn.execute(STORED_RESULT, (user_id, chat_id, key))
row = await cur.fetchone()
if row is None:
_attachment_error(check)
# The key row vanished between the two statements (expired-key cleanup): retryable.
raise Unavailable("Повторите запрос с тем же Idempotency-Key")
if bytes(row[0]) != request_hash:
raise Conflict("Idempotency-Key уже использован с другим сообщением")
return PostResult(Message(str(row[1]), chat_id, user_id, _s(row[2]), row[3],
tuple(str(a) for a in row[4])), True)
return await db.run(work)
POST_MESSAGE_CALL = ("SELECT o_status, o_id, o_text, o_created_at, o_attachments "
"FROM post_message_v2(%s, %s, %s, %s, %s, %s, %s, %s::uuid[])")
async def post_message_single_call(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes,
text: str, ttl_seconds: int, attachment_ids: list[str] = ()) -> PostResult:
"""Same contract as post_message, ONE round trip: SQL function post_message_v2 (migration 0004 =
0003 + attachment_ids) in autocommit mode, i.e. the call is the whole transaction."""
message_id = uuid.uuid4()
async def work(conn: psycopg.AsyncConnection) -> tuple:
cur = await conn.execute(POST_MESSAGE_CALL, (user_id, chat_id, key, request_hash, message_id, _b(text),
ttl_seconds, list(attachment_ids)))
return await cur.fetchone()
status, stored_id, stored_text, created_at, stored_attachments = await db.run(work, transaction=False)
if status == "not_found":
raise NotFound("Чат не найден")
_attachment_error(status)
if status == "conflict":
raise Conflict("Idempotency-Key уже использован с другим сообщением")
if status == "retry":
raise Unavailable("Повторите запрос с тем же Idempotency-Key")
return PostResult(Message(str(stored_id), chat_id, user_id, _s(stored_text), created_at,
tuple(str(a) for a in stored_attachments)), status == "replayed")
async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 5000) -> int:
"""Delete expired keys in bounded batches; one replica at a time (advisory xact lock)."""
async def work(conn: psycopg.AsyncConnection) -> int:
cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (2026_03,))
if not (await cur.fetchone())[0]:
return 0
cur = await conn.execute(
"DELETE FROM idempotency_keys WHERE ctid = ANY(ARRAY("
"SELECT ctid FROM idempotency_keys "
"WHERE created_at < clock_timestamp() - make_interval(secs => %s) LIMIT %s))",
(ttl_seconds, batch))
return cur.rowcount
return await db.run(work)
# Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR
# would push the cursor into a Filter and late pages would scan the whole prefix.
# attachment_ids is a column of the message row (migration 0004): still one index range scan.
HISTORY_FIRST = (
"SELECT id, sender_id, text, created_at, attachment_ids FROM messages "
"WHERE chat_id = %s ORDER BY created_at, id LIMIT %s")
HISTORY_AFTER = (
"SELECT id, sender_id, text, created_at, attachment_ids FROM messages "
"WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s")
async def list_messages(db: Database, chat_id: str, after: SortKey | None, limit: int) -> tuple[list[Message], bool]:
async def work(conn: psycopg.AsyncConnection) -> list:
if after is None:
cur = await conn.execute(HISTORY_FIRST, (chat_id, limit + 1))
else:
cur = await conn.execute(HISTORY_AFTER, (chat_id, from_us(after[0]), after[1], limit + 1))
return await cur.fetchall()
rows = await db.run(work)
items = [Message(str(r[0]), chat_id, str(r[1]), _s(r[2]), r[3], tuple(str(a) for a in r[4]))
for r in rows[:limit]]
return items, len(rows) > limit
+151
View File
@@ -0,0 +1,151 @@
"""One-shot S3 bootstrap: `python -m messenger.s3_bootstrap` (compose service `s3-init`).
Idempotent, safe to run on every `docker compose up`:
1. waits until the S3 endpoint answers;
2. creates the bucket with the ADMIN key pair if it does not exist yet (an existing bucket and
its objects are left untouched);
3. writes the sentinel object `health/sentinel` that the API's readiness probe HEADs;
4. proves the access model and FAILS the stand (exit 1) if it is wrong:
* anonymous (unsigned) GET of an object and anonymous LIST of the bucket are refused;
* the API key pair can read/write objects of ITS bucket, but can neither list it, nor create
a bucket, nor see or read another bucket (a canary bucket created here for this check);
* the reconciler key pair can list the bucket but cannot create one.
Only file names and verdicts are printed, never key values.
"""
from __future__ import annotations
import asyncio
import os
import sys
import time
import urllib.error
import urllib.request
from botocore.exceptions import ClientError
from .config import ConfigError, Settings, _secret, s3_settings
from .errors import Unavailable
from .storage import SENTINEL_KEY, Storage
def say(text: str) -> None:
print(f"s3-bootstrap: {text}", flush=True)
def anonymous_status(url: str) -> int:
"""HTTP status of an UNSIGNED request (what anyone on the network could do)."""
try:
with urllib.request.urlopen(url, timeout=3) as response:
return response.status
except urllib.error.HTTPError as exc:
return exc.code
async def error_code(coro) -> str:
"""S3 error code of a call that is expected to be refused ('' if it succeeded)."""
try:
await coro
return ""
except ClientError as exc:
return str(exc.response.get("Error", {}).get("Code", "error"))
def settings_with(keys: tuple[str, str]) -> Settings:
base = s3_settings(credentials_required=False)
base.update(s3_access_key_id=keys[0], s3_secret_access_key=keys[1])
return Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024, **base)
async def run() -> int:
admin = (_secret("S3_ADMIN_ACCESS_KEY_ID", True), _secret("S3_ADMIN_SECRET_ACCESS_KEY", True))
api = (_secret("S3_ACCESS_KEY_ID", True), _secret("S3_SECRET_ACCESS_KEY", True))
reconciler = (_secret("S3_RECONCILER_ACCESS_KEY_ID", True), _secret("S3_RECONCILER_SECRET_ACCESS_KEY", True))
settings = settings_with(admin)
bucket = settings.s3_bucket
admin_s3, api_s3, rec_s3 = Storage(settings), Storage(settings_with(api)), Storage(settings_with(reconciler))
for s in (admin_s3, api_s3, rec_s3):
await s.open()
try:
deadline = time.monotonic() + 60
while True:
try:
await admin_s3._call("list_buckets", admin_s3.client.list_buckets())
break
except Unavailable:
if time.monotonic() > deadline:
say("S3 endpoint does not answer within 60 s")
return 1
await asyncio.sleep(1)
code = await error_code(admin_s3.client.head_bucket(Bucket=bucket))
if code == "":
say(f"bucket {bucket}: exists, kept as is")
elif code in ("404", "NoSuchBucket", "NotFound"):
code = await error_code(admin_s3.client.create_bucket(Bucket=bucket))
if code not in ("", "BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
say(f"bucket {bucket}: create failed ({code})")
return 1
say(f"bucket {bucket}: created")
else:
say(f"bucket {bucket}: head failed ({code})")
return 1
await admin_s3.put(SENTINEL_KEY, b"ok\n", "text/plain", {})
say(f"sentinel {SENTINEL_KEY}: written")
canary = bucket + "-canary"
if await error_code(admin_s3.client.head_bucket(Bucket=canary)):
await admin_s3.client.create_bucket(Bucket=canary)
await admin_s3.client.put_object(Bucket=canary, Key="canary", Body=b"secret of another bucket")
failures = []
base = settings.s3_endpoint.rstrip("/")
for what, url in (("anonymous GET object", f"{base}/{bucket}/{SENTINEL_KEY}"),
("anonymous LIST bucket", f"{base}/{bucket}?list-type=2")):
status = anonymous_status(url)
verdict = "refused" if status in (401, 403) else "ALLOWED"
say(f"{what}: HTTP {status} -> {verdict}")
if verdict != "refused":
failures.append(what)
probe = "health/bootstrap-probe"
checks = [
("api: read object", await error_code(api_s3.client.head_object(Bucket=bucket, Key=SENTINEL_KEY)), True),
("api: write object", await error_code(api_s3.client.put_object(Bucket=bucket, Key=probe, Body=b"p")), True),
("api: delete object", await error_code(api_s3.client.delete_object(Bucket=bucket, Key=probe)), True),
("api: list bucket", await error_code(api_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), False),
("api: create bucket", await error_code(api_s3.client.create_bucket(Bucket=bucket + "-x")), False),
("api: read object of another bucket",
await error_code(api_s3.client.head_object(Bucket=canary, Key="canary")), False),
("reconciler: list bucket", await error_code(rec_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), True),
("reconciler: create bucket", await error_code(rec_s3.client.create_bucket(Bucket=bucket + "-x")), False),
]
for what, code, allowed in checks:
ok = (code == "") == allowed
say(f"{what}: {'allowed' if code == '' else 'refused (' + code + ')'} -> {'OK' if ok else 'WRONG'}")
if not ok:
failures.append(what)
visible = [b["Name"] for b in (await api_s3.client.list_buckets()).get("Buckets", [])]
say(f"api: buckets visible in ListBuckets: {visible} -> {'OK' if canary not in visible else 'WRONG'}")
if canary in visible:
failures.append("api sees another bucket")
if failures:
say("access model is WRONG: " + ", ".join(failures))
return 1
say("done: private bucket, least-privilege keys verified")
return 0
finally:
for s in (admin_s3, api_s3, rec_s3):
await s.close()
def main() -> int:
try:
return asyncio.run(run())
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
os.environ.setdefault("PYTHONUNBUFFERED", "1")
sys.exit(main())
+160
View File
@@ -0,0 +1,160 @@
"""Server-side sessions behind an opaque Bearer token.
* Token: `secrets.token_urlsafe(32)` = 256 random bits. Only SHA-256(token) is
stored, so a leaked database/AOF does not contain usable tokens. A fast hash is
enough for a 256-bit random value; passwords need a slow KDF (passwords.py).
* Expiry is absolute: `expires_at = login time + SESSION_TTL_SECONDS`. Every lookup
compares it with the current time, and a restart never extends it.
Two interchangeable stores (SESSION_STORE):
* `valkey` (compose default): the ONLY copy of a session is a Valkey key with
`PXAT expires_at`, persisted by AOF with `appendfsync always`. There is no second
copy (no cache over PostgreSQL), so a logout cannot be "forgotten" and a revoked
token cannot come back after Valkey fails and recovers. If Valkey is unreachable
a session cannot be checked: protected requests and logout get 503, never access.
* `postgres`: table `sessions` (migrations/0001_init.sql).
"""
from __future__ import annotations
import asyncio
import datetime as dt
import hashlib
import json
import logging
import secrets
from typing import Protocol
from redis import exceptions as redis_errors
from redis.asyncio import Redis
from redis.backoff import NoBackoff
from redis.retry import Retry
from .config import Settings
from .db import Database
from .errors import Unavailable
from .timefmt import UTC, from_us, to_us
log = logging.getLogger("messenger.sessions")
def new_token() -> str:
return secrets.token_urlsafe(32)
def token_hash(token: str) -> bytes:
return hashlib.sha256(token.encode("ascii")).digest()
def now() -> dt.datetime:
return dt.datetime.now(UTC)
class SessionStore(Protocol):
name: str
async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]: ...
async def user_id_for(self, token: str) -> str | None: ...
async def revoke(self, token: str) -> bool: ...
async def ping(self) -> bool: ...
async def close(self) -> None: ...
class PostgresSessionStore:
name = "postgres"
def __init__(self, db: Database):
self.db = db
async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]:
token = new_token()
expires_at = now() + dt.timedelta(seconds=ttl_seconds)
async def work(conn) -> None:
await conn.execute(
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
(token_hash(token), user_id, expires_at))
# Housekeeping: drop this user's expired sessions (bounded by the user's own rows).
await conn.execute("DELETE FROM sessions WHERE user_id = %s AND expires_at <= %s", (user_id, now()))
await self.db.run(work)
return token, expires_at
async def user_id_for(self, token: str) -> str | None:
async def work(conn):
cur = await conn.execute(
"SELECT user_id FROM sessions WHERE token_hash = %s AND expires_at > %s", (token_hash(token), now()))
return await cur.fetchone()
row = await self.db.run(work)
return str(row[0]) if row else None
async def revoke(self, token: str) -> bool:
async def work(conn) -> bool:
cur = await conn.execute(
"DELETE FROM sessions WHERE token_hash = %s AND expires_at > %s", (token_hash(token), now()))
return cur.rowcount == 1
return await self.db.run(work)
async def ping(self) -> bool:
return True # covered by the database readiness check
async def close(self) -> None:
return None
class ValkeySessionStore:
name = "valkey"
PREFIX = "session:"
def __init__(self, settings: Settings):
self.timeout = settings.cache_timeout_seconds
self.client = Redis.from_url(
settings.cache_url,
socket_timeout=self.timeout,
socket_connect_timeout=self.timeout,
retry=Retry(NoBackoff(), 0), # no hidden retries: a failure is reported at once
decode_responses=False,
)
def _key(self, token: str) -> str:
return self.PREFIX + token_hash(token).hex()
async def _call(self, coro):
try:
async with asyncio.timeout(self.timeout * 2):
return await coro
except (redis_errors.ConnectionError, redis_errors.TimeoutError, TimeoutError, OSError) as exc:
log.warning("session_store_unavailable", extra={"fields": {"error": type(exc).__name__}})
raise Unavailable("Хранилище сессий временно недоступно") from None
async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]:
token = new_token()
expires_at = now() + dt.timedelta(seconds=ttl_seconds)
value = json.dumps({"user_id": user_id, "expires_at_us": to_us(expires_at)}).encode()
# PXAT = absolute expiry time: survives restarts (AOF) without being extended.
await self._call(self.client.set(self._key(token), value, pxat=to_us(expires_at) // 1000))
return token, expires_at
async def user_id_for(self, token: str) -> str | None:
raw = await self._call(self.client.get(self._key(token)))
if raw is None:
return None
data = json.loads(raw)
if from_us(data["expires_at_us"]) <= now():
return None # do not rely only on Valkey's lazy expiry
return data["user_id"]
async def revoke(self, token: str) -> bool:
return await self._call(self.client.delete(self._key(token))) == 1
async def ping(self) -> bool:
try:
return bool(await self._call(self.client.ping()))
except Unavailable:
return False
async def close(self) -> None:
await self.client.aclose()
def build_store(settings: Settings, db: Database) -> SessionStore:
return ValkeySessionStore(settings) if settings.session_store == "valkey" else PostgresSessionStore(db)
+177
View File
@@ -0,0 +1,177 @@
"""Private S3 bucket (SeaweedFS in compose) through aiobotocore, with a hard deadline per call.
* Credentials: the API's OWN key pair (S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE ->
Docker secrets). SeaweedFS grants it only Read + Write on this one bucket: no List, no
Admin (cannot create/delete buckets or read other buckets). The admin pair is given only
to the S3 server and the one-shot bootstrap (infra/seaweedfs, messenger.s3_bootstrap).
* Deadlines: botocore connect/read timeouts AND an asyncio deadline around every call, so a
paused or unreachable S3 costs at most S3_TIMEOUT_SECONDS, then 503 `unavailable`. Unlike a
libpq query (db.py), an aiohttp request can be cancelled safely, so the call is cancelled.
* ONE retry inside botocore (total_max_attempts=2), within the same asyncio deadline: after S3
restarts, pooled keep-alive connections to the old process are dead and the first request on
such a connection fails although S3 is up again (found by test_s3_outage_..._stop on a clean
clone). Every call here is idempotent PER KEY (PUT of the same bytes to the same server-chosen
key, GET, HEAD, DELETE), so repeating it cannot create a second attachment.
* Checksums: Content-MD5 is sent with every PUT, so S3 rejects bytes damaged on the way.
botocore's newer default (aws-chunked CRC32 trailers) is switched off ("when_required"):
plain, widely supported S3 requests.
"""
from __future__ import annotations
import asyncio
import base64
import contextlib
import hashlib
import logging
from collections.abc import AsyncIterator
from dataclasses import dataclass
from aiobotocore.config import AioConfig
from aiobotocore.session import get_session
from botocore.exceptions import BotoCoreError, ClientError
from .config import Settings
from .errors import Unavailable
log = logging.getLogger("messenger.storage")
MISSING_CODES = {"NoSuchKey", "404", "NotFound"}
class ObjectMissing(Exception):
"""The key does not exist in the bucket (definitive answer of S3, not an outage)."""
@dataclass(frozen=True, slots=True)
class ObjectHead:
size: int
content_type: str
metadata: dict[str, str]
last_modified_ts: float # POSIX seconds
@dataclass(frozen=True, slots=True)
class ListedObject:
key: str
size: int
last_modified_ts: float # POSIX seconds
class Storage:
def __init__(self, settings: Settings, *, access_key: str | None = None, secret_key: str | None = None):
self.bucket = settings.s3_bucket
self.timeout = settings.s3_timeout_seconds
self._endpoint = settings.s3_endpoint
self._region = settings.s3_region
self._access_key = access_key or settings.s3_access_key_id
self._secret_key = secret_key or settings.s3_secret_access_key
self._config = AioConfig(
connect_timeout=settings.s3_connect_timeout_seconds,
read_timeout=settings.s3_timeout_seconds,
retries={"total_max_attempts": 2, "mode": "standard"},
s3={"addressing_style": "path"},
signature_version="s3v4",
request_checksum_calculation="when_required",
response_checksum_validation="when_required",
max_pool_connections=settings.s3_max_connections,
)
self._stack: contextlib.AsyncExitStack | None = None
self._client = None
async def open(self) -> None:
self._stack = contextlib.AsyncExitStack()
session = get_session()
self._client = await self._stack.enter_async_context(session.create_client(
"s3", endpoint_url=self._endpoint, region_name=self._region,
aws_access_key_id=self._access_key, aws_secret_access_key=self._secret_key, config=self._config))
async def close(self) -> None:
if self._stack is not None:
await self._stack.aclose()
self._stack = None
@property
def client(self):
assert self._client is not None, "Storage.open() was not called"
return self._client
async def _call(self, what: str, coro, deadline: float | None = None):
"""Await one S3 call under the deadline; outages become Unavailable (503)."""
try:
return await asyncio.wait_for(coro, deadline or self.timeout)
except ClientError as exc:
code = str(exc.response.get("Error", {}).get("Code", ""))
if code in MISSING_CODES:
raise ObjectMissing() from None
log.warning("s3_error", extra={"fields": {"op": what, "code": code}})
raise Unavailable("Хранилище файлов временно недоступно") from None
except (asyncio.TimeoutError, BotoCoreError, OSError) as exc:
log.warning("s3_unavailable", extra={"fields": {"op": what, "error": type(exc).__name__}})
raise Unavailable("Хранилище файлов временно недоступно") from None
async def put(self, key: str, data: bytes | bytearray, content_type: str, metadata: dict[str, str]) -> None:
md5 = base64.b64encode(hashlib.md5(data, usedforsecurity=False).digest()).decode("ascii")
# Time budget grows with the size: 10 MiB over a local network takes well under a second.
deadline = self.timeout + len(data) / (8 * 1024 * 1024)
await self._call("put", self.client.put_object(
Bucket=self.bucket, Key=key, Body=data, ContentType=content_type, ContentLength=len(data),
ContentMD5=md5, Metadata=metadata), deadline)
async def head(self, key: str) -> ObjectHead:
response = await self._call("head", self.client.head_object(Bucket=self.bucket, Key=key))
return ObjectHead(int(response["ContentLength"]), response.get("ContentType", ""),
dict(response.get("Metadata", {})), response["LastModified"].timestamp())
async def open_stream(self, key: str) -> tuple[int, AsyncIterator[bytes]]:
"""Start a GET: (Content-Length, async iterator of chunks). The headers arrive under the
deadline; every chunk read afterwards has its own deadline (a stalled S3 aborts the
download instead of hanging the connection)."""
response = await self._call("get", self.client.get_object(Bucket=self.bucket, Key=key))
body = response["Body"]
size = int(response["ContentLength"])
async def chunks() -> AsyncIterator[bytes]:
try:
while True:
chunk = await self._call("get_body", body.read(64 * 1024))
if not chunk:
return
yield chunk
finally:
body.close()
return size, chunks()
async def read_all(self, key: str, limit: int) -> bytes:
"""Whole object (reconciler checksum verification); refuses objects above `limit`."""
size, chunks = await self.open_stream(key)
if size > limit:
raise Unavailable("Объект больше допустимого размера")
parts = [chunk async for chunk in chunks]
return b"".join(parts)
async def delete(self, key: str) -> None:
"""Idempotent: deleting a missing key is a success in S3."""
await self._call("delete", self.client.delete_object(Bucket=self.bucket, Key=key))
async def list_page(self, prefix: str, start_after: str | None, page_size: int) -> tuple[list[ListedObject], bool]:
"""One ListObjectsV2 page in key order (needs the List permission: reconciler only)."""
params = {"Bucket": self.bucket, "Prefix": prefix, "MaxKeys": page_size}
if start_after:
params["StartAfter"] = start_after
response = await self._call("list", self.client.list_objects_v2(**params))
items = [ListedObject(o["Key"], int(o["Size"]), o["LastModified"].timestamp())
for o in response.get("Contents", [])]
return items, bool(response.get("IsTruncated"))
async def ping(self, timeout: float = 1.0) -> bool:
"""Readiness: HEAD of the sentinel object written by the bootstrap. Proves that S3
answers, the bucket exists and the API credentials are accepted (Read permission)."""
try:
await self._call("ping", self.client.head_object(Bucket=self.bucket, Key=SENTINEL_KEY), timeout)
return True
except (Unavailable, ObjectMissing):
return False
SENTINEL_KEY = "health/sentinel"
OBJECT_PREFIX = "o/" # every attachment original lives under o/<random 128-bit hex>
+22
View File
@@ -0,0 +1,22 @@
"""Conversions between PostgreSQL timestamptz values, cursor integers and RFC 3339 strings."""
from __future__ import annotations
import datetime as dt
UTC = dt.timezone.utc
_EPOCH = dt.datetime(1970, 1, 1, tzinfo=UTC)
def to_us(value: dt.datetime) -> int:
"""Microseconds since the Unix epoch (exact: timestamptz has microsecond precision)."""
delta = value - _EPOCH
return (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds
def from_us(us: int) -> dt.datetime:
return _EPOCH + dt.timedelta(microseconds=us)
def rfc3339(value: dt.datetime) -> str:
"""RFC 3339 UTC with fixed microsecond precision, e.g. 2026-10-12T12:00:00.000001Z."""
return value.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+199
View File
@@ -0,0 +1,199 @@
"""Strict request validation according to contracts/openapi.json + contracts/README.md.
Written by hand instead of a schema library so that every rule is visible:
extra fields, wrong JSON types, whitespace-only strings and lengths counted in
Unicode code points all produce 400 invalid_request.
"""
from __future__ import annotations
import json
import re
import unicodedata
from typing import Any
from urllib.parse import parse_qsl
from .errors import InvalidRequest
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
# Canonical textual UUID (8-4-4-4-12 hex). Case-insensitive, normalised to lower case.
UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
LIMIT_RE = re.compile(r"[0-9]{1,3}")
# Idempotency-Key (contract, lab 3): ASCII [A-Za-z0-9._:-]{1,128}.
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
SURROGATES_RE = re.compile("[\ud800-\udfff]")
DEFAULT_LIMIT = 50
MAX_LIMIT = 100
def _reject_constant(name: str) -> Any:
# Python's json accepts NaN/Infinity, which are not valid JSON.
raise ValueError(f"invalid JSON constant {name}")
def _unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
# {"text":"a","text":"b"} is ambiguous (which one is meant?) -> 400, not "last one wins".
result = dict(pairs)
if len(result) != len(pairs):
raise ValueError("duplicate key in JSON object")
return result
def parse_json_object(body: bytes) -> dict[str, Any]:
"""Decode a UTF-8 JSON object or raise InvalidRequest (malformed JSON -> 400)."""
try:
text = body.decode("utf-8")
value = json.loads(text, parse_constant=_reject_constant, object_pairs_hook=_unique_keys)
except (UnicodeDecodeError, ValueError):
raise InvalidRequest("Тело запроса должно быть корректным JSON в UTF-8") from None
if not isinstance(value, dict):
raise InvalidRequest("Тело запроса должно быть JSON-объектом")
return value
def check_fields(data: dict[str, Any], required: set[str], optional: frozenset[str] = frozenset()) -> None:
extra = set(data) - required - optional
if extra:
raise InvalidRequest("Неизвестные поля: " + ", ".join(sorted(extra)))
missing = required - set(data)
if missing:
raise InvalidRequest("Отсутствуют обязательные поля: " + ", ".join(sorted(missing)))
def text_field(data: dict[str, Any], name: str, max_len: int, *, min_len: int = 1, allow_blank: bool = False) -> str:
value = data[name]
if not isinstance(value, str):
raise InvalidRequest(f"Поле {name} должно быть строкой")
if SURROGATES_RE.search(value):
# "\ud800" escapes decode to lone surrogates that cannot be encoded as UTF-8.
raise InvalidRequest(f"Поле {name} содержит недопустимые символы")
# len() of a Python str is the number of Unicode code points, as the contract requires.
if not min_len <= len(value) <= max_len:
raise InvalidRequest(f"Длина поля {name} должна быть от {min_len} до {max_len} символов")
if value.strip() == "" and not allow_blank:
raise InvalidRequest(f"Поле {name} не может состоять только из пробелов")
return value
def username_field(data: dict[str, Any]) -> str:
value = data["username"]
if not isinstance(value, str) or USERNAME_RE.fullmatch(value) is None:
raise InvalidRequest("username должен соответствовать [a-z0-9_]{3,32}")
return value
def password_field(data: dict[str, Any]) -> str:
"""12-128 Unicode code points, not whitespace-only; any other character (':' too) is allowed."""
value = data["password"]
if not isinstance(value, str):
raise InvalidRequest("Поле password должно быть строкой")
if SURROGATES_RE.search(value):
raise InvalidRequest("Поле password содержит недопустимые символы")
if not 12 <= len(value) <= 128:
raise InvalidRequest("Длина пароля должна быть от 12 до 128 символов")
if value.strip() == "":
raise InvalidRequest("Пароль не может состоять только из пробелов")
return value
def parse_uuid(value: Any, what: str = "UUID") -> str:
if not isinstance(value, str) or UUID_RE.fullmatch(value) is None:
raise InvalidRequest(f"Неверный {what}")
return value.lower()
def member_ids_field(data: dict[str, Any]) -> list[str]:
value = data["member_ids"]
if not isinstance(value, list):
raise InvalidRequest("member_ids должен быть массивом")
if not 1 <= len(value) <= 100:
raise InvalidRequest("member_ids должен содержать от 1 до 100 элементов")
ids = [parse_uuid(item, "UUID участника") for item in value]
if len(set(ids)) != len(ids):
raise InvalidRequest("member_ids содержит дубликаты")
return ids
def attachment_ids_field(data: dict[str, Any]) -> list[str]:
"""Optional `attachment_ids` (lab 4+): absent and [] are the same message (contracts/README.md);
null or any other non-array type is 400."""
value = data.get("attachment_ids", [])
if not isinstance(value, list):
raise InvalidRequest("attachment_ids должен быть массивом")
if len(value) > 10:
raise InvalidRequest("attachment_ids содержит больше 10 элементов")
ids = [parse_uuid(item, "UUID вложения") for item in value]
if len(set(ids)) != len(ids):
raise InvalidRequest("attachment_ids содержит дубликаты")
return ids
def idempotency_key(raw: str | None) -> str:
if raw is None or IDEMPOTENCY_KEY_RE.fullmatch(raw) is None:
raise InvalidRequest("Заголовок Idempotency-Key обязателен: [A-Za-z0-9._:-]{1,128}")
return raw
def single_query_param(query: list[tuple[str, str]], name: str) -> str | None:
values = [v for k, v in query if k == name]
if len(values) > 1:
raise InvalidRequest(f"Параметр {name} указан несколько раз")
return values[0] if values else None
def parse_limit(raw: str | None) -> int:
if raw is None:
return DEFAULT_LIMIT
if LIMIT_RE.fullmatch(raw) is None or not 1 <= int(raw) <= MAX_LIMIT:
raise InvalidRequest(f"limit должен быть целым числом от 1 до {MAX_LIMIT}")
return int(raw)
# Lab 4: the display name of an attachment. Never used as a path or an S3 key, but it is shown
# to other users, so characters that change how a name LOOKS are refused: control characters
# (C0, DEL, C1 such as U+0085), line/paragraph separators, unpaired surrogates and bidi controls
# ("photo<U+202E>gnp.exe" would display as "photoexe.png").
BIDI_CONTROLS = frozenset("\u061c\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069")
FORBIDDEN_CATEGORIES = frozenset({"Cc", "Cs", "Zl", "Zp"})
def query_items(raw: bytes) -> list[tuple[str, str]]:
"""Strict query parsing: percent-encoded UTF-8 only (Starlette would replace bad bytes by U+FFFD)."""
try:
return parse_qsl(raw.decode("ascii"), keep_blank_values=True, encoding="utf-8", errors="strict")
except (UnicodeDecodeError, ValueError):
raise InvalidRequest("Параметры запроса должны быть в UTF-8 с percent-encoding") from None
def filename_param(query: list[tuple[str, str]]) -> str:
value = single_query_param(query, "filename")
if value is None:
raise InvalidRequest("Параметр filename обязателен")
if not 1 <= len(value) <= 255:
raise InvalidRequest("Длина filename должна быть от 1 до 255 символов")
if value.strip() == "" or value in {".", ".."}:
raise InvalidRequest("filename не может состоять только из пробелов или точек")
if "/" in value or "\\" in value:
raise InvalidRequest("filename не может содержать / или \\")
for char in value:
if unicodedata.category(char) in FORBIDDEN_CATEGORIES or char in BIDI_CONTROLS:
raise InvalidRequest("filename содержит управляющие символы")
return value
ATTACHMENT_VARIANTS = frozenset({"thumbnail", "crop", "optimized"}) # exist in the schema, not in lab 4
def variant_param(query: list[tuple[str, str]]) -> str:
"""`variant` of GET /attachments/{id}/content: only `original` exists in lab 4. A known variant
name is a missing variant of a ready object (404); anything else is an invalid value (400)."""
value = single_query_param(query, "variant")
if value is None or value == "original":
return "original"
if value in ATTACHMENT_VARIANTS:
raise NotFoundVariant()
raise InvalidRequest("variant должен быть original")
class NotFoundVariant(Exception):
"""Raised for a variant that lab 4 does not produce; mapped to 404 by the handler."""