feat: add private attachments and safe reconciliation experiment

This commit is contained in:
2026-11-06 17:55:51 +03:00
parent ec540a1a72
commit 59b2f031bc
35 changed files with 3341 additions and 20 deletions
+1
View File
@@ -0,0 +1 @@
"""Учебный REST-мессенджер."""
+12
View File
@@ -0,0 +1,12 @@
import os
import uvicorn
uvicorn.run(
"messenger.app:app",
host=os.getenv("APP_HOST", "0.0.0.0"),
port=int(os.getenv("APP_PORT", "8080")),
workers=1,
access_log=False,
timeout_graceful_shutdown=5,
)
+314
View File
@@ -0,0 +1,314 @@
import base64
import binascii
import json
import logging
import os
import re
import time
from contextlib import asynccontextmanager
from threading import BoundedSemaphore
from uuid import uuid4
from fastapi import Depends, FastAPI, Request, Response
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from starlette.exceptions import HTTPException
from .attachments import AttachmentService
from .domain import Failure, boundary, identifier, page
from .models import CreateChat, CreateMessage, CreateUser
from .postgres import PostgresStore
from .storage import ObjectStorage
logger = logging.getLogger("messenger")
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"), format="%(message)s")
INSTANCE_ID = str(uuid4())
CODES = {
400: "invalid_request",
401: "unauthorized",
404: "not_found",
409: "conflict",
413: "payload_too_large",
415: "unsupported_media_type",
429: "rate_limited",
503: "unavailable",
500: "internal_error",
}
@asynccontextmanager
async def lifespan(app):
app.state.store = PostgresStore()
app.state.storage = ObjectStorage()
app.state.attachments = AttachmentService(app.state.store, app.state.storage)
app.state.upload_slots = BoundedSemaphore(1)
logger.info(json.dumps({"event": "started", "storage": "postgres"}))
yield
app.state.storage.close()
app.state.store.close()
logger.info(json.dumps({"event": "stopped"}))
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
def error(request, status, message, headers=None):
return JSONResponse(
{
"error": {
"code": CODES.get(status, "invalid_request"),
"message": message,
"request_id": request.state.request_id,
}
},
status_code=status,
headers=headers,
)
@app.middleware("http")
async def request_log(request: Request, call_next):
request.state.request_id = str(uuid4())
start = time.perf_counter()
try:
if (
request.method == "POST"
and request.url.path == "/api/v1/auth/sessions"
and await request.body()
):
response = error(request, 400, "Вход не принимает тело запроса")
else:
response = await call_next(request)
except Exception:
logger.error(
json.dumps({"event": "unexpected_error", "request_id": request.state.request_id})
)
response = error(request, 500, "Внутренняя ошибка")
response.headers["X-Request-Id"] = request.state.request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
route = request.scope.get("route")
logger.info(
json.dumps(
{
"event": "request",
"request_id": request.state.request_id,
"method": request.method,
"route": getattr(route, "path", "unknown"),
"status": response.status_code,
"duration_ms": round((time.perf_counter() - start) * 1000, 3),
}
)
)
return response
@app.exception_handler(Failure)
async def domain_error(request, exc):
return error(request, exc.status, exc.message, exc.headers)
@app.exception_handler(RequestValidationError)
async def validation_error(request, exc):
return error(request, 400, "Неверный JSON или поля запроса")
@app.exception_handler(HTTPException)
async def http_error(request, exc):
return error(request, exc.status_code, "Неверный запрос", exc.headers)
def bearer_failure():
return Failure(
401, "Требуется действующая сессия", {"WWW-Authenticate": 'Bearer realm="messenger"'}
)
def bearer(request):
parts = request.headers.get("Authorization", "").split()
if len(parts) != 2 or parts[0].lower() != "bearer":
raise bearer_failure()
return parts[1]
def actor(request: Request):
user = request.app.state.store.authenticate(bearer(request))
if user is None:
raise bearer_failure()
return user
@app.post("/api/v1/auth/sessions", status_code=201)
def login(request: Request):
headers = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
parts = request.headers.get("Authorization", "").split()
try:
if len(parts) != 2 or parts[0].lower() != "basic" or len(parts[1]) > 2048:
raise ValueError
decoded = base64.b64decode(parts[1], validate=True).decode("utf-8")
username, password = decoded.split(":", 1)
if not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128:
raise ValueError
except (ValueError, UnicodeError, binascii.Error):
raise Failure(401, "Неверные учётные данные", headers) from None
session = request.app.state.store.login(username, password)
if session is None:
raise Failure(401, "Неверные учётные данные", headers)
return session
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def logout(request: Request, user=Depends(actor)):
if not request.app.state.store.logout(bearer(request)):
raise bearer_failure()
return Response(status_code=204)
def pagination(request, scope):
allowed = {"limit", "cursor"}
if any(k not in allowed for k in request.query_params):
raise Failure(400, "Неизвестный параметр запроса")
if any(len(request.query_params.getlist(k)) > 1 for k in allowed):
raise Failure(400, "Повторный параметр запроса")
value = request.query_params.get("limit", "50")
if len(value) > 3 or not re.fullmatch(r"[0-9]+", value) or not 1 <= int(value) <= 100:
raise Failure(400, "limit должен быть целым от 1 до 100")
return int(value), boundary(scope, request.query_params.get("cursor"))
@app.get("/health/live")
def live():
return {"status": "ok"}
@app.get("/health/ready")
def ready(request: Request):
if not request.app.state.store.ready() or not request.app.state.storage.ready():
raise Failure(503, "Хранилище недоступно")
return {"status": "ok"}
@app.post("/api/v1/users", status_code=201)
def register(data: CreateUser, request: Request):
return request.app.state.store.register(data.model_dump(mode="json"))
@app.get("/api/v1/users/me")
def me(user=Depends(actor)):
return user
@app.post("/api/v1/chats", status_code=201)
def create_chat(data: CreateChat, request: Request, user=Depends(actor)):
return request.app.state.store.create_chat(user["id"], data.model_dump(mode="json"))
@app.get("/api/v1/chats")
def chats(request: Request, user=Depends(actor)):
scope = "chats:" + user["id"]
limit, after = pagination(request, scope)
return page(request.app.state.store.list_chats(user["id"], after, limit), limit, scope)
@app.get("/api/v1/chats/{chat_id}")
def chat(chat_id: str, request: Request, user=Depends(actor)):
return request.app.state.store.chat(user["id"], identifier(chat_id))
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
def send(chat_id: str, data: CreateMessage, request: Request, user=Depends(actor)):
key = request.headers.get("Idempotency-Key", "")
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key):
raise Failure(400, "Требуется корректный Idempotency-Key")
return request.app.state.store.send(
user["id"], identifier(chat_id), data.text, key, [str(a) for a in data.attachment_ids]
)
@app.get("/api/v1/chats/{chat_id}/messages")
def history(chat_id: str, request: Request, user=Depends(actor)):
cid = identifier(chat_id)
scope = "messages:" + cid
limit, after = pagination(request, scope)
return page(request.app.state.store.history(user["id"], cid, after, limit), limit, scope)
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
async def upload_attachment(chat_id: str, request: Request, user=Depends(actor)):
import asyncio
import tempfile
from starlette.concurrency import run_in_threadpool
from .attachments import MAX_BYTES, filename_valid
if (
set(request.query_params) != {"filename"}
or len(request.query_params.getlist("filename")) != 1
):
raise Failure(400, "Требуется одно имя файла")
filename = filename_valid(request.query_params["filename"])
cid = identifier(chat_id)
await run_in_threadpool(request.app.state.store.chat, user["id"], cid)
content_type = request.headers.get("content-type", "").lower()
if content_type not in {"image/png", "image/jpeg"}:
raise Failure(415, "Ожидается PNG или JPEG")
if not request.app.state.upload_slots.acquire(blocking=False):
raise Failure(429, "Все места загрузки заняты", {"Retry-After": "1"})
try:
with tempfile.SpooledTemporaryFile(max_size=256 * 1024) as spool:
size = 0
try:
async with asyncio.timeout(15):
async for chunk in request.stream():
size += len(chunk)
if size > MAX_BYTES:
raise Failure(413, "Файл больше 10 MiB")
spool.write(chunk)
except TimeoutError:
raise Failure(400, "Истекло время загрузки") from None
spool.seek(0)
return await run_in_threadpool(
request.app.state.attachments.upload,
user["id"],
cid,
filename,
content_type,
spool,
size,
)
finally:
request.app.state.upload_slots.release()
@app.get("/api/v1/attachments/{attachment_id}")
def attachment_metadata(attachment_id: str, request: Request, user=Depends(actor)):
return request.app.state.attachments.metadata(user["id"], identifier(attachment_id))
@app.get("/api/v1/attachments/{attachment_id}/content")
def attachment_content(attachment_id: str, request: Request, user=Depends(actor)):
from fastapi.responses import StreamingResponse
if (
any(k != "variant" for k in request.query_params)
or len(request.query_params.getlist("variant")) > 1
):
raise Failure(400, "Неверный вариант")
variant = request.query_params.get("variant", "original")
if variant not in {"original", "thumbnail", "crop", "optimized"}:
raise Failure(400, "Неверный вариант")
obj = request.app.state.attachments.content(user["id"], identifier(attachment_id), variant)
def chunks():
try:
while data := obj["Body"].read(64 * 1024):
yield data
finally:
obj["Body"].close()
return StreamingResponse(
chunks(),
media_type=obj["ContentType"],
headers={"Content-Length": str(obj["ContentLength"])},
)
+154
View File
@@ -0,0 +1,154 @@
"""Synchronous attachment intake: bounded spool, full image validation, durable intent."""
import hashlib
import os
import subprocess
import unicodedata
from uuid import uuid4
from PIL import Image, UnidentifiedImageError
from .domain import Failure, timestamp
from .postgres import public
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
# Check dimensions before decoding. Pillow's own hard bomb threshold remains active.
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
PUBLIC_FIELDS = (
"id",
"chat_id",
"owner_id",
"filename",
"content_type",
"size_bytes",
"sha256",
"status",
"error_code",
"created_at",
"updated_at",
)
def filename_valid(name):
if not isinstance(name, str) or not 1 <= len(name) <= 255 or not name.strip():
raise Failure(400, "Неверное имя файла")
if any(c in "/\\" or unicodedata.category(c) == "Cc" for c in name):
raise Failure(400, "Неверное имя файла")
return name
def validate_image(fileobj, declared_type):
if declared_type not in {"image/png", "image/jpeg"}:
raise Failure(415, "Ожидается PNG или JPEG")
try:
fileobj.seek(0)
with Image.open(fileobj) as image:
width, height = image.size
if width * height > MAX_PIXELS:
raise Failure(413, "Слишком много пикселей")
actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format)
if actual != declared_type:
raise Failure(415, "Формат изображения не соответствует MIME")
image.verify()
# verify() alone does not decode IDAT/JPEG scan data.
fileobj.seek(0)
with Image.open(fileobj) as image:
image.load()
if actual == "image/jpeg":
# libjpeg emits a warning but Pillow accepts e.g. a half scan with a
# forged EOI. djpeg's nonzero warning exit rejects that corruption.
fileobj.seek(0)
try:
decoded = subprocess.run(
["djpeg", "-strict", "-maxmemory", "192m", "-outfile", "/dev/null"],
stdin=fileobj,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
timeout=5,
check=False,
)
except subprocess.TimeoutExpired:
raise Failure(415, "Изображение не прошло ограниченную проверку") from None
if decoded.returncode:
raise Failure(415, "Повреждённое изображение")
fileobj.seek(0)
return actual, width, height
except Image.DecompressionBombError:
raise Failure(413, "Слишком много пикселей") from None
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
raise Failure(415, "Повреждённое изображение") from None
def metadata_public(row):
return {**public({k: row[k] for k in PUBLIC_FIELDS}), "variants": []}
class AttachmentService:
def __init__(self, store, storage):
self.store = store
self.storage = storage
def upload(self, actor, cid, filename, content_type, fileobj, size):
filename_valid(filename)
if not 1 <= size <= MAX_BYTES:
raise Failure(413 if size > MAX_BYTES else 415, "Неверный размер изображения")
validate_image(fileobj, content_type)
digest = hashlib.file_digest(fileobj, "sha256").hexdigest()
aid = str(uuid4())
key = "original/" + aid
now = timestamp()
with self.store.connection() as conn:
self.store._chat(conn, actor, cid)
conn.execute(
"INSERT INTO upload_intents(id,object_key,created_at) VALUES (%s,%s,%s)",
(aid, key, now),
)
# Keep the intent locked throughout PUT/finalization: reconciler skips live
# uploads even if a slow request crosses the grace boundary.
with self.store.connection() as conn, conn.transaction():
intent = conn.execute(
"SELECT id FROM upload_intents WHERE id=%s FOR UPDATE", (aid,)
).fetchone()
if not intent:
raise Failure(503, "Загрузка больше не активна")
self.storage.put(key, fileobj, size, content_type)
if os.getenv("UPLOAD_FAULT") == "after_put":
raise Failure(503, "Учебный сбой после записи файла")
row = conn.execute(
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,"
"status,error_code,created_at,updated_at,object_key) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,'ready',NULL,%s,%s,%s) RETURNING *",
(aid, cid, actor, filename, content_type, size, digest, now, now, key),
).fetchone()
conn.execute("DELETE FROM upload_intents WHERE id=%s", (aid,))
# Never delete S3 on an ambiguous COMMIT: the transaction may have committed.
if os.getenv("UPLOAD_FAULT") == "after_commit":
raise Failure(503, "Учебный сбой после фиксации")
return metadata_public(row)
def metadata(self, actor, aid):
with self.store.connection() as conn:
row = conn.execute(
"SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id "
"WHERE a.id=%s AND m.user_id=%s",
(aid, actor),
).fetchone()
if not row:
raise Failure(404, "Объект не найден")
return metadata_public(row)
def content(self, actor, aid, variant="original"):
with self.store.connection() as conn:
row = conn.execute(
"SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id "
"WHERE a.id=%s AND m.user_id=%s",
(aid, actor),
).fetchone()
if not row:
raise Failure(404, "Объект не найден")
if row["status"] != "ready":
raise Failure(409, "Вложение не готово")
if variant != "original":
raise Failure(404, "Вариант не найден")
return self.storage.get(row["object_key"])
+10
View File
@@ -0,0 +1,10 @@
import os
from pathlib import Path
def secret(name: str) -> str:
filename = os.environ.get(name + "_FILE")
value = Path(filename).read_text().strip() if filename else os.environ.get(name, "")
if not value:
raise RuntimeError("Отсутствует настройка " + name)
return value
+54
View File
@@ -0,0 +1,54 @@
import base64
import json
import re
from datetime import UTC, datetime
from uuid import UUID
class Failure(Exception):
def __init__(self, status: int, message: str, headers: dict | None = None):
self.status = status
self.message = message
self.headers = headers or {}
def timestamp() -> str:
return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
def identifier(value: str) -> str:
try:
return str(UUID(value))
except (ValueError, TypeError, AttributeError):
raise Failure(400, "Некорректный UUID") from None
def boundary(scope: str, cursor: str | None) -> tuple[str, str] | None:
if cursor is None:
return None
try:
if not cursor or len(cursor) > 1024 or not re.fullmatch(r"[A-Za-z0-9_-]+", cursor):
raise ValueError
data = json.loads(base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)))
if not isinstance(data, list) or len(data) != 4 or data[:2] != [1, scope]:
raise ValueError
date, uid = data[2:]
if not isinstance(date, str) or not date.endswith("Z"):
raise ValueError
datetime.fromisoformat(date.replace("Z", "+00:00"))
if not isinstance(uid, str) or str(UUID(uid)) != uid:
raise ValueError
return date, uid
except (ValueError, TypeError, KeyError, UnicodeError):
raise Failure(400, "Некорректный cursor или область запроса") from None
def page(items: list[dict], limit: int, scope: str) -> dict:
more = len(items) > limit
selected = items[:limit]
cursor = None
if more:
last = selected[-1]
payload = json.dumps([1, scope, last["created_at"], last["id"]], separators=(",", ":"))
cursor = base64.urlsafe_b64encode(payload.encode()).decode().rstrip("=")
return {"items": selected, "next_cursor": cursor}
+32
View File
@@ -0,0 +1,32 @@
import hashlib
from pathlib import Path
import psycopg
from .config import secret
def migrate():
with psycopg.connect(secret("DATABASE_URL")) as conn:
# Serialize independent migration processes without changing application data.
conn.execute("SELECT pg_advisory_xact_lock(2301002)")
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations "
"(name text PRIMARY KEY, checksum text NOT NULL)"
)
for path in sorted(Path("migrations").glob("*.sql")):
checksum = hashlib.sha256(path.read_bytes()).hexdigest()
old = conn.execute(
"SELECT checksum FROM schema_migrations WHERE name=%s", (path.name,)
).fetchone()
if old:
if old[0] != checksum:
raise RuntimeError("Изменена применённая миграция: " + path.name)
continue
conn.execute(path.read_text())
conn.execute("INSERT INTO schema_migrations VALUES (%s,%s)", (path.name, checksum))
print("Applied", path.name)
if __name__ == "__main__":
migrate()
+68
View File
@@ -0,0 +1,68 @@
import re
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator, model_validator
class RequestModel(BaseModel):
model_config = ConfigDict(extra="forbid")
@field_validator("*", mode="after")
@classmethod
def nonblank(cls, value):
if isinstance(value, str) and not value.strip():
raise ValueError("Строка не может состоять из пробелов")
return value
class CreateUser(RequestModel):
username: StrictStr = Field(pattern=r"^[a-z0-9_]{3,32}$")
display_name: StrictStr = Field(min_length=1, max_length=100)
password: StrictStr = Field(min_length=12, max_length=128)
@field_validator("username")
@classmethod
def full_username(cls, value):
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
raise ValueError("Неверное имя пользователя")
return value
class CreateChat(RequestModel):
title: StrictStr = Field(min_length=1, max_length=100)
member_ids: list[UUID] = Field(min_length=1, max_length=100)
@field_validator("member_ids", mode="before")
@classmethod
def string_ids(cls, value):
if not isinstance(value, list) or any(not isinstance(x, str) for x in value):
raise ValueError("Ожидается массив UUID-строк")
return value
@field_validator("member_ids")
@classmethod
def unique_ids(cls, value):
if len(set(value)) != len(value):
raise ValueError("Участники не должны повторяться")
return value
class CreateMessage(BaseModel):
model_config = ConfigDict(extra="forbid")
text: StrictStr = Field(max_length=4000)
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
@field_validator("attachment_ids", mode="before")
@classmethod
def string_ids(cls, value):
if not isinstance(value, list) or any(not isinstance(x, str) for x in value):
raise ValueError("Ожидается массив UUID-строк")
return value
@model_validator(mode="after")
def content(self):
if len(set(self.attachment_ids)) != len(self.attachment_ids):
raise ValueError("Вложения не должны повторяться")
if not self.text.strip() and not self.attachment_ids:
raise ValueError("Пустое сообщение без вложений")
return self
+324
View File
@@ -0,0 +1,324 @@
import hashlib
import json
import logging
import os
import secrets
from contextlib import contextmanager
from datetime import UTC, datetime, timedelta
from threading import BoundedSemaphore
from uuid import UUID, uuid4
import psycopg
import redis
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
from .config import secret
from .domain import Failure, timestamp
logger = logging.getLogger("messenger")
KDF_SLOTS = BoundedSemaphore(2)
KDF = PasswordHasher(time_cost=2, memory_cost=65536, parallelism=1)
DUMMY_HASH = KDF.hash("invalid-user-dummy-password")
class DeadlineConnection(psycopg.Connection):
def wait(self, gen, interval=0.1, timeout=None):
try:
return super().wait(gen, interval=interval, timeout=timeout or 3.0)
except psycopg.errors._WaitTimeout:
# A paused server cannot complete cancellation. Discard an uncertain
# connection instead of returning it to the pool or waiting forever.
self.pgconn.finish()
raise
def public(row):
return {
k: (
str(v)
if isinstance(v, UUID)
else v.astimezone(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
if isinstance(v, datetime)
else bytes(v).decode("utf-8")
if isinstance(v, (bytes, memoryview))
else v
)
for k, v in row.items()
if k not in {"password_hash", "token_digest"}
}
class PostgresStore:
def __init__(self):
self.pool = ConnectionPool(
secret("DATABASE_URL"),
connection_class=DeadlineConnection,
min_size=1,
max_size=int(os.getenv("POOL_MAX_SIZE", "8")),
timeout=2,
max_waiting=32,
kwargs={
"autocommit": True,
"row_factory": dict_row,
"connect_timeout": 2,
"options": "-c statement_timeout=2500 -c lock_timeout=1500",
},
check=ConnectionPool.check_connection,
)
self.pool.wait(timeout=15)
self.cache = redis.Redis.from_url(
secret("CACHE_URL"),
socket_connect_timeout=0.2,
socket_timeout=0.2,
decode_responses=True,
)
self.ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
if self.ttl < 1:
raise RuntimeError("SESSION_TTL_SECONDS должен быть положительным")
@contextmanager
def connection(self):
try:
with self.pool.connection() as conn:
yield conn
except (
psycopg.OperationalError,
psycopg.InterfaceError,
PoolTimeout,
TooManyRequests,
psycopg.errors.QueryCanceled,
psycopg.errors.LockNotAvailable,
):
raise Failure(503, "Хранилище временно недоступно") from None
def close(self):
self.cache.close()
self.pool.close()
def ready(self):
try:
with self.connection() as conn:
conn.execute("SELECT 1")
return True
except Failure:
return False
def _cached_user(self, uid):
key = "user:v1:" + uid
try:
value = self.cache.get(key)
if value:
cached = json.loads(value)
if (
isinstance(cached, dict)
and cached.get("id") == uid
and set(cached) == {"id", "username", "display_name", "created_at"}
):
return cached
except (redis.RedisError, ValueError, TypeError):
logger.warning(json.dumps({"event": "cache_bypass"}))
user = self.user(uid)
if user:
try:
self.cache.setex(key, 60, json.dumps(user))
except redis.RedisError:
logger.warning(json.dumps({"event": "cache_bypass"}))
return user
def user(self, uid):
with self.connection() as conn:
row = conn.execute(
"SELECT id,username,display_name,created_at FROM users WHERE id=%s", (uid,)
).fetchone()
return public(row) if row else None
def register(self, data):
with KDF_SLOTS:
hashed = KDF.hash(data.pop("password"))
uid = str(uuid4())
try:
with self.connection() as conn:
row = conn.execute(
"INSERT INTO users VALUES (%s,%s,%s,%s,%s) "
"RETURNING id,username,display_name,created_at",
(
uid,
data["username"],
data["display_name"].encode("utf-8"),
hashed,
timestamp(),
),
).fetchone()
return public(row)
except psycopg.errors.UniqueViolation:
raise Failure(409, "Имя пользователя занято") from None
def login(self, username, password):
with self.connection() as conn:
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
try:
with KDF_SLOTS:
KDF.verify(row["password_hash"] if row else DUMMY_HASH, password)
except VerificationError:
return None
if row is None:
return None
token = secrets.token_urlsafe(32)
digest = hashlib.sha256(token.encode()).hexdigest()
expires = datetime.now(UTC) + timedelta(seconds=self.ttl)
with self.connection() as conn:
conn.execute("INSERT INTO sessions VALUES (%s,%s,%s)", (digest, row["id"], expires))
return {
"token": token,
"token_type": "Bearer",
"expires_at": public({"v": expires})["v"],
"user": public(row),
}
def authenticate(self, token):
if not token or len(token) > 512:
return None
digest = hashlib.sha256(token.encode()).hexdigest()
# Session validity always comes from PostgreSQL, including on a cache hit.
with self.connection() as conn:
row = conn.execute(
"SELECT s.user_id FROM sessions s WHERE s.token_digest=%s AND s.expires_at>now()",
(digest,),
).fetchone()
return self._cached_user(str(row["user_id"])) if row else None
def logout(self, token):
with self.connection() as conn:
row = conn.execute(
"DELETE FROM sessions WHERE token_digest=%s AND expires_at>now() RETURNING user_id",
(hashlib.sha256(token.encode()).hexdigest(),),
).fetchone()
return row is not None
def create_chat(self, actor, data):
if actor not in data["member_ids"]:
raise Failure(400, "Вызывающий должен входить в чат")
cid = str(uuid4())
try:
with self.connection() as conn, conn.transaction():
row = conn.execute(
"INSERT INTO chats VALUES (%s,%s,%s) RETURNING *",
(cid, data["title"].encode("utf-8"), timestamp()),
).fetchone()
for index, uid in enumerate(data["member_ids"]):
conn.execute("INSERT INTO members VALUES (%s,%s,%s)", (cid, uid, index))
return {**public(row), "member_ids": data["member_ids"]}
except psycopg.errors.ForeignKeyViolation:
raise Failure(400, "Неизвестный участник") from None
@staticmethod
def _chat(conn, actor, cid):
row = conn.execute(
"SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id "
"WHERE c.id=%s AND m.user_id=%s",
(cid, actor),
).fetchone()
if row is None:
raise Failure(404, "Объект не найден")
members = conn.execute(
"SELECT user_id FROM members WHERE chat_id=%s ORDER BY position", (cid,)
).fetchall()
return {**public(row), "member_ids": [str(m["user_id"]) for m in members]}
def chat(self, actor, cid):
with self.connection() as conn:
return self._chat(conn, actor, cid)
def list_chats(self, actor, after, limit):
with self.connection() as conn:
sql = "SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s"
params = [actor]
if after:
sql += " AND (c.created_at,c.id)>(%s,%s)"
params.extend(after)
sql += " ORDER BY c.created_at,c.id LIMIT %s"
params.append(limit + 1)
rows = conn.execute(sql, params).fetchall()
return [self._chat(conn, actor, str(row["id"])) for row in rows]
def send(self, actor, cid, text, key, attachment_ids=None):
attachment_ids = attachment_ids or []
body_digest = hashlib.sha256(
json.dumps(
{"text": text, "attachment_ids": attachment_ids},
sort_keys=True,
ensure_ascii=False,
separators=(",", ":"),
).encode()
).hexdigest()
with self.connection() as conn, conn.transaction():
self._chat(conn, actor, cid)
# Serializes exactly this semantic key across all API processes. Hash
# collisions can only serialize unrelated requests, never mix results.
scope = json.dumps([actor, "POST", cid, key], separators=(",", ":"))
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True)
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
old = conn.execute(
"SELECT body_digest,response FROM message_requests "
"WHERE actor_id=%s AND chat_id=%s AND key=%s AND expires_at>now()",
(actor, cid, key),
).fetchone()
if old:
if old["body_digest"] != body_digest:
raise Failure(409, "Ключ уже использован для другого сообщения")
return json.loads(bytes(old["response"]))
conn.execute(
"DELETE FROM message_requests WHERE actor_id=%s AND chat_id=%s AND key=%s",
(actor, cid, key),
)
for aid in attachment_ids:
attachment = conn.execute(
"SELECT id,status FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE",
(aid, cid),
).fetchone()
if not attachment:
raise Failure(404, "Объект не найден")
if attachment["status"] == "failed":
raise Failure(409, "Вложение недоступно")
row = conn.execute(
"INSERT INTO messages VALUES (%s,%s,%s,%s,%s) RETURNING *",
(str(uuid4()), cid, actor, text.encode("utf-8"), timestamp()),
).fetchone()
for position, aid in enumerate(attachment_ids):
conn.execute(
"INSERT INTO message_attachments VALUES (%s,%s,%s)", (row["id"], aid, position)
)
response = {**public(row), "attachment_ids": attachment_ids}
conn.execute(
"INSERT INTO message_requests(actor_id,chat_id,key,body_digest,response,expires_at) "
"VALUES (%s,%s,%s,%s,%s,now()+interval '24 hours')",
(actor, cid, key, body_digest, json.dumps(response).encode("utf-8")),
)
return response
def history(self, actor, cid, after, limit):
with self.connection() as conn:
self._chat(conn, actor, cid)
sql = "SELECT * FROM messages WHERE chat_id=%s"
params = [cid]
if after:
sql += " AND (created_at,id)>(%s,%s)"
params.extend(after)
sql += " ORDER BY created_at,id LIMIT %s"
params.append(limit + 1)
result = []
for row in conn.execute(sql, params).fetchall():
attachments = conn.execute(
"SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position",
(row["id"],),
).fetchall()
result.append(
{
**public(row),
"attachment_ids": [str(a["attachment_id"]) for a in attachments],
}
)
return result
+176
View File
@@ -0,0 +1,176 @@
"""Explicit, repeatable reconciliation. Default mode is strictly read-only."""
import argparse
import json
import time
from datetime import UTC, datetime, timedelta
from .domain import Failure
from .postgres import PostgresStore
from .storage import ObjectStorage
def reconcile(store, storage, *, apply=False, grace_seconds=3600, page_size=100):
if grace_seconds < 1 or not 1 <= page_size <= 1000:
raise ValueError("grace >= 1 second; page size 1..1000")
started = time.perf_counter()
cutoff = datetime.now(UTC) - timedelta(seconds=grace_seconds)
report = {
"mode": "apply" if apply else "dry-run",
"grace_seconds": grace_seconds,
"page_size": page_size,
"intents_scanned": 0,
"objects_scanned": 0,
"metadata_scanned": 0,
"old_intents": 0,
"orphan_objects": 0,
"missing_objects": 0,
"deleted_objects": 0,
"deleted_intents": 0,
"marked_failed": 0,
"fresh_objects_skipped": 0,
"errors": 0,
}
def audit(conn, event, key):
conn.execute(
"INSERT INTO attachment_reconcile_audit(event,object_key) VALUES (%s,%s)", (event, key)
)
# Locked live uploads are skipped. A dead upload's committed intent survives
# rollback and makes both its object and metadata absence discoverable.
after = None
while True:
with store.connection() as conn:
rows = conn.execute(
"SELECT id,object_key,created_at FROM upload_intents WHERE created_at<%s "
+ ("AND (created_at,id)>(%s,%s) " if after else "")
+ "ORDER BY created_at,id LIMIT %s",
[cutoff, *(after or []), page_size],
).fetchall()
if not rows:
break
for candidate in rows:
report["intents_scanned"] += 1
try:
with store.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT * FROM upload_intents WHERE id=%s FOR UPDATE SKIP LOCKED",
(candidate["id"],),
).fetchone()
if not row:
continue
current_object = storage.head(row["object_key"])
if current_object and current_object["LastModified"] >= cutoff:
report["fresh_objects_skipped"] += 1
continue
report["old_intents"] += 1
known = conn.execute(
"SELECT id FROM attachments WHERE object_key=%s", (row["object_key"],)
).fetchone()
if apply:
if not known:
storage.delete(row["object_key"])
report["deleted_objects"] += 1
conn.execute("DELETE FROM upload_intents WHERE id=%s", (row["id"],))
audit(conn, "intent_cleaned", row["object_key"])
report["deleted_intents"] += 1
except Failure:
report["errors"] += 1
after = (rows[-1]["created_at"], rows[-1]["id"])
# Only server-owned original prefix is eligible. Unknown future prefixes are
# deliberately outside this tool's deletion policy.
try:
for obj in storage.objects(page_size):
report["objects_scanned"] += 1
if not obj["Key"].startswith("original/"):
continue
if obj["LastModified"] >= cutoff:
report["fresh_objects_skipped"] += 1
continue
with store.connection() as conn, conn.transaction():
known = conn.execute(
"SELECT 1 FROM attachments WHERE object_key=%s UNION ALL "
"SELECT 1 FROM upload_intents WHERE object_key=%s LIMIT 1",
(obj["Key"], obj["Key"]),
).fetchone()
if known:
continue
current_object = storage.head(obj["Key"])
if current_object and current_object["LastModified"] >= cutoff:
report["fresh_objects_skipped"] += 1
continue
report["orphan_objects"] += 1
if apply:
storage.delete(obj["Key"])
audit(conn, "orphan_deleted", obj["Key"])
report["deleted_objects"] += 1
except Failure:
report["errors"] += 1
# A listing snapshot cannot prove absence. HEAD is checked live under the row
# lock immediately before changing status. Never delete legitimate metadata,
# including attachments referenced by messages; missing originals are reported.
after_id = None
while True:
with store.connection() as conn:
rows = conn.execute(
"SELECT id FROM attachments WHERE created_at<%s AND status='ready' "
+ ("AND id>%s " if after_id else "")
+ "ORDER BY id LIMIT %s",
[cutoff, *([after_id] if after_id else []), page_size],
).fetchall()
if not rows:
break
for candidate in rows:
report["metadata_scanned"] += 1
try:
with store.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT * FROM attachments WHERE id=%s AND status='ready' FOR UPDATE SKIP LOCKED",
(candidate["id"],),
).fetchone()
if not row or storage.exists(row["object_key"]):
continue
report["missing_objects"] += 1
if apply:
conn.execute(
"UPDATE attachments SET status='failed',error_code='storage_missing',"
"updated_at=now() WHERE id=%s",
(row["id"],),
)
audit(conn, "missing_marked_failed", row["object_key"])
report["marked_failed"] += 1
except Failure:
report["errors"] += 1
after_id = rows[-1]["id"]
report["duration_seconds"] = round(time.perf_counter() - started, 6)
return report
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--apply", action="store_true")
parser.add_argument("--grace-seconds", type=int, default=3600)
parser.add_argument("--page-size", type=int, default=100)
args = parser.parse_args()
store, storage = PostgresStore(), ObjectStorage()
try:
result = reconcile(
store,
storage,
apply=args.apply,
grace_seconds=args.grace_seconds,
page_size=args.page_size,
)
print(json.dumps(result, ensure_ascii=False))
if result["errors"]:
raise SystemExit(1)
finally:
storage.close()
store.close()
if __name__ == "__main__":
main()
+96
View File
@@ -0,0 +1,96 @@
"""Private S3 access with bounded network waits; never expose storage keys in API."""
import os
from contextlib import contextmanager
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from .config import secret
from .domain import Failure
class ObjectStorage:
def __init__(self, admin=False):
self.bucket = os.getenv("S3_BUCKET", "messenger")
prefix = "S3_ADMIN" if admin else "S3"
self.client = boto3.client(
"s3",
endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"),
aws_access_key_id=secret(prefix + "_ACCESS_KEY"),
aws_secret_access_key=secret(prefix + "_SECRET_KEY"),
region_name="us-east-1",
config=Config(
connect_timeout=1,
read_timeout=2,
retries={"total_max_attempts": 1},
s3={"addressing_style": "path"},
request_checksum_calculation="when_required",
response_checksum_validation="when_required",
),
)
@contextmanager
def boundary(self):
try:
yield
except (BotoCoreError, ClientError):
raise Failure(503, "Хранилище файлов временно недоступно") from None
def put(self, key, fileobj, size, content_type):
fileobj.seek(0)
with self.boundary():
self.client.put_object(
Bucket=self.bucket,
Key=key,
Body=fileobj,
ContentLength=size,
ContentType=content_type,
)
def get(self, key):
try:
return self.client.get_object(Bucket=self.bucket, Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
raise Failure(404, "Объект не найден") from None
raise Failure(503, "Хранилище файлов временно недоступно") from None
except BotoCoreError:
raise Failure(503, "Хранилище файлов временно недоступно") from None
def head(self, key):
try:
return self.client.head_object(Bucket=self.bucket, Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404", "NotFound"}:
return None
raise Failure(503, "Хранилище файлов временно недоступно") from None
except BotoCoreError:
raise Failure(503, "Хранилище файлов временно недоступно") from None
def exists(self, key):
return self.head(key) is not None
def delete(self, key):
with self.boundary():
self.client.delete_object(Bucket=self.bucket, Key=key)
def objects(self, page_size=100):
with self.boundary():
paginator = self.client.get_paginator("list_objects_v2")
for page in paginator.paginate(
Bucket=self.bucket, PaginationConfig={"PageSize": page_size}
):
yield from page.get("Contents", [])
def ready(self):
try:
with self.boundary():
self.client.head_bucket(Bucket=self.bucket)
return True
except Failure:
return False
def close(self):
self.client.close()
+30
View File
@@ -0,0 +1,30 @@
"""Idempotent private bucket bootstrap; this process alone receives admin keys."""
import time
from botocore.exceptions import BotoCoreError, ClientError
from .storage import ObjectStorage
def main():
storage = ObjectStorage(admin=True)
for attempt in range(30):
try:
try:
storage.client.head_bucket(Bucket=storage.bucket)
except ClientError as exc:
if exc.response["Error"]["Code"] not in {"404", "NoSuchBucket", "NotFound"}:
raise
storage.client.create_bucket(Bucket=storage.bucket)
print("Private bucket ready; anonymous identity is not configured.")
storage.close()
return
except (ClientError, BotoCoreError):
if attempt == 29:
raise RuntimeError("S3 bootstrap timeout") from None
time.sleep(1)
if __name__ == "__main__":
main()