feat: add private attachments and safe reconciliation experiment
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Учебный REST-мессенджер."""
|
||||
@@ -0,0 +1,12 @@
|
||||
import os
|
||||
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(
|
||||
"messenger.app:app",
|
||||
host=os.getenv("APP_HOST", "0.0.0.0"),
|
||||
port=int(os.getenv("APP_PORT", "8080")),
|
||||
workers=1,
|
||||
access_log=False,
|
||||
timeout_graceful_shutdown=5,
|
||||
)
|
||||
@@ -0,0 +1,314 @@
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from threading import BoundedSemaphore
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import Depends, FastAPI, Request, Response
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.exceptions import HTTPException
|
||||
|
||||
from .attachments import AttachmentService
|
||||
from .domain import Failure, boundary, identifier, page
|
||||
from .models import CreateChat, CreateMessage, CreateUser
|
||||
from .postgres import PostgresStore
|
||||
from .storage import ObjectStorage
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"), format="%(message)s")
|
||||
INSTANCE_ID = str(uuid4())
|
||||
|
||||
CODES = {
|
||||
400: "invalid_request",
|
||||
401: "unauthorized",
|
||||
404: "not_found",
|
||||
409: "conflict",
|
||||
413: "payload_too_large",
|
||||
415: "unsupported_media_type",
|
||||
429: "rate_limited",
|
||||
503: "unavailable",
|
||||
500: "internal_error",
|
||||
}
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
app.state.store = PostgresStore()
|
||||
app.state.storage = ObjectStorage()
|
||||
app.state.attachments = AttachmentService(app.state.store, app.state.storage)
|
||||
app.state.upload_slots = BoundedSemaphore(1)
|
||||
logger.info(json.dumps({"event": "started", "storage": "postgres"}))
|
||||
yield
|
||||
app.state.storage.close()
|
||||
app.state.store.close()
|
||||
logger.info(json.dumps({"event": "stopped"}))
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||||
|
||||
|
||||
def error(request, status, message, headers=None):
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": {
|
||||
"code": CODES.get(status, "invalid_request"),
|
||||
"message": message,
|
||||
"request_id": request.state.request_id,
|
||||
}
|
||||
},
|
||||
status_code=status,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_log(request: Request, call_next):
|
||||
request.state.request_id = str(uuid4())
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
if (
|
||||
request.method == "POST"
|
||||
and request.url.path == "/api/v1/auth/sessions"
|
||||
and await request.body()
|
||||
):
|
||||
response = error(request, 400, "Вход не принимает тело запроса")
|
||||
else:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
logger.error(
|
||||
json.dumps({"event": "unexpected_error", "request_id": request.state.request_id})
|
||||
)
|
||||
response = error(request, 500, "Внутренняя ошибка")
|
||||
response.headers["X-Request-Id"] = request.state.request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
route = request.scope.get("route")
|
||||
logger.info(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "request",
|
||||
"request_id": request.state.request_id,
|
||||
"method": request.method,
|
||||
"route": getattr(route, "path", "unknown"),
|
||||
"status": response.status_code,
|
||||
"duration_ms": round((time.perf_counter() - start) * 1000, 3),
|
||||
}
|
||||
)
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(Failure)
|
||||
async def domain_error(request, exc):
|
||||
return error(request, exc.status, exc.message, exc.headers)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request, exc):
|
||||
return error(request, 400, "Неверный JSON или поля запроса")
|
||||
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request, exc):
|
||||
return error(request, exc.status_code, "Неверный запрос", exc.headers)
|
||||
|
||||
|
||||
def bearer_failure():
|
||||
return Failure(
|
||||
401, "Требуется действующая сессия", {"WWW-Authenticate": 'Bearer realm="messenger"'}
|
||||
)
|
||||
|
||||
|
||||
def bearer(request):
|
||||
parts = request.headers.get("Authorization", "").split()
|
||||
if len(parts) != 2 or parts[0].lower() != "bearer":
|
||||
raise bearer_failure()
|
||||
return parts[1]
|
||||
|
||||
|
||||
def actor(request: Request):
|
||||
user = request.app.state.store.authenticate(bearer(request))
|
||||
if user is None:
|
||||
raise bearer_failure()
|
||||
return user
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||
def login(request: Request):
|
||||
headers = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
|
||||
parts = request.headers.get("Authorization", "").split()
|
||||
try:
|
||||
if len(parts) != 2 or parts[0].lower() != "basic" or len(parts[1]) > 2048:
|
||||
raise ValueError
|
||||
decoded = base64.b64decode(parts[1], validate=True).decode("utf-8")
|
||||
username, password = decoded.split(":", 1)
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128:
|
||||
raise ValueError
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
raise Failure(401, "Неверные учётные данные", headers) from None
|
||||
session = request.app.state.store.login(username, password)
|
||||
if session is None:
|
||||
raise Failure(401, "Неверные учётные данные", headers)
|
||||
return session
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def logout(request: Request, user=Depends(actor)):
|
||||
if not request.app.state.store.logout(bearer(request)):
|
||||
raise bearer_failure()
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
def pagination(request, scope):
|
||||
allowed = {"limit", "cursor"}
|
||||
if any(k not in allowed for k in request.query_params):
|
||||
raise Failure(400, "Неизвестный параметр запроса")
|
||||
if any(len(request.query_params.getlist(k)) > 1 for k in allowed):
|
||||
raise Failure(400, "Повторный параметр запроса")
|
||||
value = request.query_params.get("limit", "50")
|
||||
if len(value) > 3 or not re.fullmatch(r"[0-9]+", value) or not 1 <= int(value) <= 100:
|
||||
raise Failure(400, "limit должен быть целым от 1 до 100")
|
||||
return int(value), boundary(scope, request.query_params.get("cursor"))
|
||||
|
||||
|
||||
@app.get("/health/live")
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
def ready(request: Request):
|
||||
if not request.app.state.store.ready() or not request.app.state.storage.ready():
|
||||
raise Failure(503, "Хранилище недоступно")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
def register(data: CreateUser, request: Request):
|
||||
return request.app.state.store.register(data.model_dump(mode="json"))
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
def me(user=Depends(actor)):
|
||||
return user
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
def create_chat(data: CreateChat, request: Request, user=Depends(actor)):
|
||||
return request.app.state.store.create_chat(user["id"], data.model_dump(mode="json"))
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
def chats(request: Request, user=Depends(actor)):
|
||||
scope = "chats:" + user["id"]
|
||||
limit, after = pagination(request, scope)
|
||||
return page(request.app.state.store.list_chats(user["id"], after, limit), limit, scope)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
def chat(chat_id: str, request: Request, user=Depends(actor)):
|
||||
return request.app.state.store.chat(user["id"], identifier(chat_id))
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
def send(chat_id: str, data: CreateMessage, request: Request, user=Depends(actor)):
|
||||
key = request.headers.get("Idempotency-Key", "")
|
||||
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key):
|
||||
raise Failure(400, "Требуется корректный Idempotency-Key")
|
||||
return request.app.state.store.send(
|
||||
user["id"], identifier(chat_id), data.text, key, [str(a) for a in data.attachment_ids]
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
def history(chat_id: str, request: Request, user=Depends(actor)):
|
||||
cid = identifier(chat_id)
|
||||
scope = "messages:" + cid
|
||||
limit, after = pagination(request, scope)
|
||||
return page(request.app.state.store.history(user["id"], cid, after, limit), limit, scope)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
|
||||
async def upload_attachment(chat_id: str, request: Request, user=Depends(actor)):
|
||||
import asyncio
|
||||
import tempfile
|
||||
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from .attachments import MAX_BYTES, filename_valid
|
||||
|
||||
if (
|
||||
set(request.query_params) != {"filename"}
|
||||
or len(request.query_params.getlist("filename")) != 1
|
||||
):
|
||||
raise Failure(400, "Требуется одно имя файла")
|
||||
filename = filename_valid(request.query_params["filename"])
|
||||
cid = identifier(chat_id)
|
||||
await run_in_threadpool(request.app.state.store.chat, user["id"], cid)
|
||||
content_type = request.headers.get("content-type", "").lower()
|
||||
if content_type not in {"image/png", "image/jpeg"}:
|
||||
raise Failure(415, "Ожидается PNG или JPEG")
|
||||
if not request.app.state.upload_slots.acquire(blocking=False):
|
||||
raise Failure(429, "Все места загрузки заняты", {"Retry-After": "1"})
|
||||
try:
|
||||
with tempfile.SpooledTemporaryFile(max_size=256 * 1024) as spool:
|
||||
size = 0
|
||||
try:
|
||||
async with asyncio.timeout(15):
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise Failure(413, "Файл больше 10 MiB")
|
||||
spool.write(chunk)
|
||||
except TimeoutError:
|
||||
raise Failure(400, "Истекло время загрузки") from None
|
||||
spool.seek(0)
|
||||
return await run_in_threadpool(
|
||||
request.app.state.attachments.upload,
|
||||
user["id"],
|
||||
cid,
|
||||
filename,
|
||||
content_type,
|
||||
spool,
|
||||
size,
|
||||
)
|
||||
finally:
|
||||
request.app.state.upload_slots.release()
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}")
|
||||
def attachment_metadata(attachment_id: str, request: Request, user=Depends(actor)):
|
||||
return request.app.state.attachments.metadata(user["id"], identifier(attachment_id))
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def attachment_content(attachment_id: str, request: Request, user=Depends(actor)):
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
if (
|
||||
any(k != "variant" for k in request.query_params)
|
||||
or len(request.query_params.getlist("variant")) > 1
|
||||
):
|
||||
raise Failure(400, "Неверный вариант")
|
||||
variant = request.query_params.get("variant", "original")
|
||||
if variant not in {"original", "thumbnail", "crop", "optimized"}:
|
||||
raise Failure(400, "Неверный вариант")
|
||||
obj = request.app.state.attachments.content(user["id"], identifier(attachment_id), variant)
|
||||
|
||||
def chunks():
|
||||
try:
|
||||
while data := obj["Body"].read(64 * 1024):
|
||||
yield data
|
||||
finally:
|
||||
obj["Body"].close()
|
||||
|
||||
return StreamingResponse(
|
||||
chunks(),
|
||||
media_type=obj["ContentType"],
|
||||
headers={"Content-Length": str(obj["ContentLength"])},
|
||||
)
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Synchronous attachment intake: bounded spool, full image validation, durable intent."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import subprocess
|
||||
import unicodedata
|
||||
from uuid import uuid4
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from .domain import Failure, timestamp
|
||||
from .postgres import public
|
||||
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
# Check dimensions before decoding. Pillow's own hard bomb threshold remains active.
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
PUBLIC_FIELDS = (
|
||||
"id",
|
||||
"chat_id",
|
||||
"owner_id",
|
||||
"filename",
|
||||
"content_type",
|
||||
"size_bytes",
|
||||
"sha256",
|
||||
"status",
|
||||
"error_code",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
)
|
||||
|
||||
|
||||
def filename_valid(name):
|
||||
if not isinstance(name, str) or not 1 <= len(name) <= 255 or not name.strip():
|
||||
raise Failure(400, "Неверное имя файла")
|
||||
if any(c in "/\\" or unicodedata.category(c) == "Cc" for c in name):
|
||||
raise Failure(400, "Неверное имя файла")
|
||||
return name
|
||||
|
||||
|
||||
def validate_image(fileobj, declared_type):
|
||||
if declared_type not in {"image/png", "image/jpeg"}:
|
||||
raise Failure(415, "Ожидается PNG или JPEG")
|
||||
try:
|
||||
fileobj.seek(0)
|
||||
with Image.open(fileobj) as image:
|
||||
width, height = image.size
|
||||
if width * height > MAX_PIXELS:
|
||||
raise Failure(413, "Слишком много пикселей")
|
||||
actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format)
|
||||
if actual != declared_type:
|
||||
raise Failure(415, "Формат изображения не соответствует MIME")
|
||||
image.verify()
|
||||
# verify() alone does not decode IDAT/JPEG scan data.
|
||||
fileobj.seek(0)
|
||||
with Image.open(fileobj) as image:
|
||||
image.load()
|
||||
if actual == "image/jpeg":
|
||||
# libjpeg emits a warning but Pillow accepts e.g. a half scan with a
|
||||
# forged EOI. djpeg's nonzero warning exit rejects that corruption.
|
||||
fileobj.seek(0)
|
||||
try:
|
||||
decoded = subprocess.run(
|
||||
["djpeg", "-strict", "-maxmemory", "192m", "-outfile", "/dev/null"],
|
||||
stdin=fileobj,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=5,
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise Failure(415, "Изображение не прошло ограниченную проверку") from None
|
||||
if decoded.returncode:
|
||||
raise Failure(415, "Повреждённое изображение")
|
||||
fileobj.seek(0)
|
||||
return actual, width, height
|
||||
except Image.DecompressionBombError:
|
||||
raise Failure(413, "Слишком много пикселей") from None
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
|
||||
raise Failure(415, "Повреждённое изображение") from None
|
||||
|
||||
|
||||
def metadata_public(row):
|
||||
return {**public({k: row[k] for k in PUBLIC_FIELDS}), "variants": []}
|
||||
|
||||
|
||||
class AttachmentService:
|
||||
def __init__(self, store, storage):
|
||||
self.store = store
|
||||
self.storage = storage
|
||||
|
||||
def upload(self, actor, cid, filename, content_type, fileobj, size):
|
||||
filename_valid(filename)
|
||||
if not 1 <= size <= MAX_BYTES:
|
||||
raise Failure(413 if size > MAX_BYTES else 415, "Неверный размер изображения")
|
||||
validate_image(fileobj, content_type)
|
||||
digest = hashlib.file_digest(fileobj, "sha256").hexdigest()
|
||||
aid = str(uuid4())
|
||||
key = "original/" + aid
|
||||
now = timestamp()
|
||||
with self.store.connection() as conn:
|
||||
self.store._chat(conn, actor, cid)
|
||||
conn.execute(
|
||||
"INSERT INTO upload_intents(id,object_key,created_at) VALUES (%s,%s,%s)",
|
||||
(aid, key, now),
|
||||
)
|
||||
# Keep the intent locked throughout PUT/finalization: reconciler skips live
|
||||
# uploads even if a slow request crosses the grace boundary.
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
intent = conn.execute(
|
||||
"SELECT id FROM upload_intents WHERE id=%s FOR UPDATE", (aid,)
|
||||
).fetchone()
|
||||
if not intent:
|
||||
raise Failure(503, "Загрузка больше не активна")
|
||||
self.storage.put(key, fileobj, size, content_type)
|
||||
if os.getenv("UPLOAD_FAULT") == "after_put":
|
||||
raise Failure(503, "Учебный сбой после записи файла")
|
||||
row = conn.execute(
|
||||
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,"
|
||||
"status,error_code,created_at,updated_at,object_key) "
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s,'ready',NULL,%s,%s,%s) RETURNING *",
|
||||
(aid, cid, actor, filename, content_type, size, digest, now, now, key),
|
||||
).fetchone()
|
||||
conn.execute("DELETE FROM upload_intents WHERE id=%s", (aid,))
|
||||
# Never delete S3 on an ambiguous COMMIT: the transaction may have committed.
|
||||
if os.getenv("UPLOAD_FAULT") == "after_commit":
|
||||
raise Failure(503, "Учебный сбой после фиксации")
|
||||
return metadata_public(row)
|
||||
|
||||
def metadata(self, actor, aid):
|
||||
with self.store.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id "
|
||||
"WHERE a.id=%s AND m.user_id=%s",
|
||||
(aid, actor),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise Failure(404, "Объект не найден")
|
||||
return metadata_public(row)
|
||||
|
||||
def content(self, actor, aid, variant="original"):
|
||||
with self.store.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id "
|
||||
"WHERE a.id=%s AND m.user_id=%s",
|
||||
(aid, actor),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise Failure(404, "Объект не найден")
|
||||
if row["status"] != "ready":
|
||||
raise Failure(409, "Вложение не готово")
|
||||
if variant != "original":
|
||||
raise Failure(404, "Вариант не найден")
|
||||
return self.storage.get(row["object_key"])
|
||||
@@ -0,0 +1,10 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def secret(name: str) -> str:
|
||||
filename = os.environ.get(name + "_FILE")
|
||||
value = Path(filename).read_text().strip() if filename else os.environ.get(name, "")
|
||||
if not value:
|
||||
raise RuntimeError("Отсутствует настройка " + name)
|
||||
return value
|
||||
@@ -0,0 +1,54 @@
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
|
||||
class Failure(Exception):
|
||||
def __init__(self, status: int, message: str, headers: dict | None = None):
|
||||
self.status = status
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def timestamp() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def identifier(value: str) -> str:
|
||||
try:
|
||||
return str(UUID(value))
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
raise Failure(400, "Некорректный UUID") from None
|
||||
|
||||
|
||||
def boundary(scope: str, cursor: str | None) -> tuple[str, str] | None:
|
||||
if cursor is None:
|
||||
return None
|
||||
try:
|
||||
if not cursor or len(cursor) > 1024 or not re.fullmatch(r"[A-Za-z0-9_-]+", cursor):
|
||||
raise ValueError
|
||||
data = json.loads(base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)))
|
||||
if not isinstance(data, list) or len(data) != 4 or data[:2] != [1, scope]:
|
||||
raise ValueError
|
||||
date, uid = data[2:]
|
||||
if not isinstance(date, str) or not date.endswith("Z"):
|
||||
raise ValueError
|
||||
datetime.fromisoformat(date.replace("Z", "+00:00"))
|
||||
if not isinstance(uid, str) or str(UUID(uid)) != uid:
|
||||
raise ValueError
|
||||
return date, uid
|
||||
except (ValueError, TypeError, KeyError, UnicodeError):
|
||||
raise Failure(400, "Некорректный cursor или область запроса") from None
|
||||
|
||||
|
||||
def page(items: list[dict], limit: int, scope: str) -> dict:
|
||||
more = len(items) > limit
|
||||
selected = items[:limit]
|
||||
cursor = None
|
||||
if more:
|
||||
last = selected[-1]
|
||||
payload = json.dumps([1, scope, last["created_at"], last["id"]], separators=(",", ":"))
|
||||
cursor = base64.urlsafe_b64encode(payload.encode()).decode().rstrip("=")
|
||||
return {"items": selected, "next_cursor": cursor}
|
||||
@@ -0,0 +1,32 @@
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
from .config import secret
|
||||
|
||||
|
||||
def migrate():
|
||||
with psycopg.connect(secret("DATABASE_URL")) as conn:
|
||||
# Serialize independent migration processes without changing application data.
|
||||
conn.execute("SELECT pg_advisory_xact_lock(2301002)")
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations "
|
||||
"(name text PRIMARY KEY, checksum text NOT NULL)"
|
||||
)
|
||||
for path in sorted(Path("migrations").glob("*.sql")):
|
||||
checksum = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
old = conn.execute(
|
||||
"SELECT checksum FROM schema_migrations WHERE name=%s", (path.name,)
|
||||
).fetchone()
|
||||
if old:
|
||||
if old[0] != checksum:
|
||||
raise RuntimeError("Изменена применённая миграция: " + path.name)
|
||||
continue
|
||||
conn.execute(path.read_text())
|
||||
conn.execute("INSERT INTO schema_migrations VALUES (%s,%s)", (path.name, checksum))
|
||||
print("Applied", path.name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
migrate()
|
||||
@@ -0,0 +1,68 @@
|
||||
import re
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator, model_validator
|
||||
|
||||
|
||||
class RequestModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@field_validator("*", mode="after")
|
||||
@classmethod
|
||||
def nonblank(cls, value):
|
||||
if isinstance(value, str) and not value.strip():
|
||||
raise ValueError("Строка не может состоять из пробелов")
|
||||
return value
|
||||
|
||||
|
||||
class CreateUser(RequestModel):
|
||||
username: StrictStr = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
||||
display_name: StrictStr = Field(min_length=1, max_length=100)
|
||||
password: StrictStr = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def full_username(cls, value):
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
|
||||
raise ValueError("Неверное имя пользователя")
|
||||
return value
|
||||
|
||||
|
||||
class CreateChat(RequestModel):
|
||||
title: StrictStr = Field(min_length=1, max_length=100)
|
||||
member_ids: list[UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("member_ids", mode="before")
|
||||
@classmethod
|
||||
def string_ids(cls, value):
|
||||
if not isinstance(value, list) or any(not isinstance(x, str) for x in value):
|
||||
raise ValueError("Ожидается массив UUID-строк")
|
||||
return value
|
||||
|
||||
@field_validator("member_ids")
|
||||
@classmethod
|
||||
def unique_ids(cls, value):
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError("Участники не должны повторяться")
|
||||
return value
|
||||
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
text: StrictStr = Field(max_length=4000)
|
||||
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("attachment_ids", mode="before")
|
||||
@classmethod
|
||||
def string_ids(cls, value):
|
||||
if not isinstance(value, list) or any(not isinstance(x, str) for x in value):
|
||||
raise ValueError("Ожидается массив UUID-строк")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def content(self):
|
||||
if len(set(self.attachment_ids)) != len(self.attachment_ids):
|
||||
raise ValueError("Вложения не должны повторяться")
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError("Пустое сообщение без вложений")
|
||||
return self
|
||||
@@ -0,0 +1,324 @@
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from threading import BoundedSemaphore
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import psycopg
|
||||
import redis
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
|
||||
|
||||
from .config import secret
|
||||
from .domain import Failure, timestamp
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
KDF_SLOTS = BoundedSemaphore(2)
|
||||
KDF = PasswordHasher(time_cost=2, memory_cost=65536, parallelism=1)
|
||||
DUMMY_HASH = KDF.hash("invalid-user-dummy-password")
|
||||
|
||||
|
||||
class DeadlineConnection(psycopg.Connection):
|
||||
def wait(self, gen, interval=0.1, timeout=None):
|
||||
try:
|
||||
return super().wait(gen, interval=interval, timeout=timeout or 3.0)
|
||||
except psycopg.errors._WaitTimeout:
|
||||
# A paused server cannot complete cancellation. Discard an uncertain
|
||||
# connection instead of returning it to the pool or waiting forever.
|
||||
self.pgconn.finish()
|
||||
raise
|
||||
|
||||
|
||||
def public(row):
|
||||
return {
|
||||
k: (
|
||||
str(v)
|
||||
if isinstance(v, UUID)
|
||||
else v.astimezone(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
if isinstance(v, datetime)
|
||||
else bytes(v).decode("utf-8")
|
||||
if isinstance(v, (bytes, memoryview))
|
||||
else v
|
||||
)
|
||||
for k, v in row.items()
|
||||
if k not in {"password_hash", "token_digest"}
|
||||
}
|
||||
|
||||
|
||||
class PostgresStore:
|
||||
def __init__(self):
|
||||
self.pool = ConnectionPool(
|
||||
secret("DATABASE_URL"),
|
||||
connection_class=DeadlineConnection,
|
||||
min_size=1,
|
||||
max_size=int(os.getenv("POOL_MAX_SIZE", "8")),
|
||||
timeout=2,
|
||||
max_waiting=32,
|
||||
kwargs={
|
||||
"autocommit": True,
|
||||
"row_factory": dict_row,
|
||||
"connect_timeout": 2,
|
||||
"options": "-c statement_timeout=2500 -c lock_timeout=1500",
|
||||
},
|
||||
check=ConnectionPool.check_connection,
|
||||
)
|
||||
self.pool.wait(timeout=15)
|
||||
self.cache = redis.Redis.from_url(
|
||||
secret("CACHE_URL"),
|
||||
socket_connect_timeout=0.2,
|
||||
socket_timeout=0.2,
|
||||
decode_responses=True,
|
||||
)
|
||||
self.ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
if self.ttl < 1:
|
||||
raise RuntimeError("SESSION_TTL_SECONDS должен быть положительным")
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
yield conn
|
||||
except (
|
||||
psycopg.OperationalError,
|
||||
psycopg.InterfaceError,
|
||||
PoolTimeout,
|
||||
TooManyRequests,
|
||||
psycopg.errors.QueryCanceled,
|
||||
psycopg.errors.LockNotAvailable,
|
||||
):
|
||||
raise Failure(503, "Хранилище временно недоступно") from None
|
||||
|
||||
def close(self):
|
||||
self.cache.close()
|
||||
self.pool.close()
|
||||
|
||||
def ready(self):
|
||||
try:
|
||||
with self.connection() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
return True
|
||||
except Failure:
|
||||
return False
|
||||
|
||||
def _cached_user(self, uid):
|
||||
key = "user:v1:" + uid
|
||||
try:
|
||||
value = self.cache.get(key)
|
||||
if value:
|
||||
cached = json.loads(value)
|
||||
if (
|
||||
isinstance(cached, dict)
|
||||
and cached.get("id") == uid
|
||||
and set(cached) == {"id", "username", "display_name", "created_at"}
|
||||
):
|
||||
return cached
|
||||
except (redis.RedisError, ValueError, TypeError):
|
||||
logger.warning(json.dumps({"event": "cache_bypass"}))
|
||||
user = self.user(uid)
|
||||
if user:
|
||||
try:
|
||||
self.cache.setex(key, 60, json.dumps(user))
|
||||
except redis.RedisError:
|
||||
logger.warning(json.dumps({"event": "cache_bypass"}))
|
||||
return user
|
||||
|
||||
def user(self, uid):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id,username,display_name,created_at FROM users WHERE id=%s", (uid,)
|
||||
).fetchone()
|
||||
return public(row) if row else None
|
||||
|
||||
def register(self, data):
|
||||
with KDF_SLOTS:
|
||||
hashed = KDF.hash(data.pop("password"))
|
||||
uid = str(uuid4())
|
||||
try:
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO users VALUES (%s,%s,%s,%s,%s) "
|
||||
"RETURNING id,username,display_name,created_at",
|
||||
(
|
||||
uid,
|
||||
data["username"],
|
||||
data["display_name"].encode("utf-8"),
|
||||
hashed,
|
||||
timestamp(),
|
||||
),
|
||||
).fetchone()
|
||||
return public(row)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise Failure(409, "Имя пользователя занято") from None
|
||||
|
||||
def login(self, username, password):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
|
||||
try:
|
||||
with KDF_SLOTS:
|
||||
KDF.verify(row["password_hash"] if row else DUMMY_HASH, password)
|
||||
except VerificationError:
|
||||
return None
|
||||
if row is None:
|
||||
return None
|
||||
token = secrets.token_urlsafe(32)
|
||||
digest = hashlib.sha256(token.encode()).hexdigest()
|
||||
expires = datetime.now(UTC) + timedelta(seconds=self.ttl)
|
||||
with self.connection() as conn:
|
||||
conn.execute("INSERT INTO sessions VALUES (%s,%s,%s)", (digest, row["id"], expires))
|
||||
return {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": public({"v": expires})["v"],
|
||||
"user": public(row),
|
||||
}
|
||||
|
||||
def authenticate(self, token):
|
||||
if not token or len(token) > 512:
|
||||
return None
|
||||
digest = hashlib.sha256(token.encode()).hexdigest()
|
||||
# Session validity always comes from PostgreSQL, including on a cache hit.
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT s.user_id FROM sessions s WHERE s.token_digest=%s AND s.expires_at>now()",
|
||||
(digest,),
|
||||
).fetchone()
|
||||
return self._cached_user(str(row["user_id"])) if row else None
|
||||
|
||||
def logout(self, token):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"DELETE FROM sessions WHERE token_digest=%s AND expires_at>now() RETURNING user_id",
|
||||
(hashlib.sha256(token.encode()).hexdigest(),),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
def create_chat(self, actor, data):
|
||||
if actor not in data["member_ids"]:
|
||||
raise Failure(400, "Вызывающий должен входить в чат")
|
||||
cid = str(uuid4())
|
||||
try:
|
||||
with self.connection() as conn, conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO chats VALUES (%s,%s,%s) RETURNING *",
|
||||
(cid, data["title"].encode("utf-8"), timestamp()),
|
||||
).fetchone()
|
||||
for index, uid in enumerate(data["member_ids"]):
|
||||
conn.execute("INSERT INTO members VALUES (%s,%s,%s)", (cid, uid, index))
|
||||
return {**public(row), "member_ids": data["member_ids"]}
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
raise Failure(400, "Неизвестный участник") from None
|
||||
|
||||
@staticmethod
|
||||
def _chat(conn, actor, cid):
|
||||
row = conn.execute(
|
||||
"SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id "
|
||||
"WHERE c.id=%s AND m.user_id=%s",
|
||||
(cid, actor),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise Failure(404, "Объект не найден")
|
||||
members = conn.execute(
|
||||
"SELECT user_id FROM members WHERE chat_id=%s ORDER BY position", (cid,)
|
||||
).fetchall()
|
||||
return {**public(row), "member_ids": [str(m["user_id"]) for m in members]}
|
||||
|
||||
def chat(self, actor, cid):
|
||||
with self.connection() as conn:
|
||||
return self._chat(conn, actor, cid)
|
||||
|
||||
def list_chats(self, actor, after, limit):
|
||||
with self.connection() as conn:
|
||||
sql = "SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s"
|
||||
params = [actor]
|
||||
if after:
|
||||
sql += " AND (c.created_at,c.id)>(%s,%s)"
|
||||
params.extend(after)
|
||||
sql += " ORDER BY c.created_at,c.id LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
rows = conn.execute(sql, params).fetchall()
|
||||
return [self._chat(conn, actor, str(row["id"])) for row in rows]
|
||||
|
||||
def send(self, actor, cid, text, key, attachment_ids=None):
|
||||
attachment_ids = attachment_ids or []
|
||||
body_digest = hashlib.sha256(
|
||||
json.dumps(
|
||||
{"text": text, "attachment_ids": attachment_ids},
|
||||
sort_keys=True,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
).hexdigest()
|
||||
with self.connection() as conn, conn.transaction():
|
||||
self._chat(conn, actor, cid)
|
||||
# Serializes exactly this semantic key across all API processes. Hash
|
||||
# collisions can only serialize unrelated requests, never mix results.
|
||||
scope = json.dumps([actor, "POST", cid, key], separators=(",", ":"))
|
||||
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True)
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
|
||||
old = conn.execute(
|
||||
"SELECT body_digest,response FROM message_requests "
|
||||
"WHERE actor_id=%s AND chat_id=%s AND key=%s AND expires_at>now()",
|
||||
(actor, cid, key),
|
||||
).fetchone()
|
||||
if old:
|
||||
if old["body_digest"] != body_digest:
|
||||
raise Failure(409, "Ключ уже использован для другого сообщения")
|
||||
return json.loads(bytes(old["response"]))
|
||||
conn.execute(
|
||||
"DELETE FROM message_requests WHERE actor_id=%s AND chat_id=%s AND key=%s",
|
||||
(actor, cid, key),
|
||||
)
|
||||
for aid in attachment_ids:
|
||||
attachment = conn.execute(
|
||||
"SELECT id,status FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE",
|
||||
(aid, cid),
|
||||
).fetchone()
|
||||
if not attachment:
|
||||
raise Failure(404, "Объект не найден")
|
||||
if attachment["status"] == "failed":
|
||||
raise Failure(409, "Вложение недоступно")
|
||||
row = conn.execute(
|
||||
"INSERT INTO messages VALUES (%s,%s,%s,%s,%s) RETURNING *",
|
||||
(str(uuid4()), cid, actor, text.encode("utf-8"), timestamp()),
|
||||
).fetchone()
|
||||
for position, aid in enumerate(attachment_ids):
|
||||
conn.execute(
|
||||
"INSERT INTO message_attachments VALUES (%s,%s,%s)", (row["id"], aid, position)
|
||||
)
|
||||
response = {**public(row), "attachment_ids": attachment_ids}
|
||||
conn.execute(
|
||||
"INSERT INTO message_requests(actor_id,chat_id,key,body_digest,response,expires_at) "
|
||||
"VALUES (%s,%s,%s,%s,%s,now()+interval '24 hours')",
|
||||
(actor, cid, key, body_digest, json.dumps(response).encode("utf-8")),
|
||||
)
|
||||
return response
|
||||
|
||||
def history(self, actor, cid, after, limit):
|
||||
with self.connection() as conn:
|
||||
self._chat(conn, actor, cid)
|
||||
sql = "SELECT * FROM messages WHERE chat_id=%s"
|
||||
params = [cid]
|
||||
if after:
|
||||
sql += " AND (created_at,id)>(%s,%s)"
|
||||
params.extend(after)
|
||||
sql += " ORDER BY created_at,id LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
result = []
|
||||
for row in conn.execute(sql, params).fetchall():
|
||||
attachments = conn.execute(
|
||||
"SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position",
|
||||
(row["id"],),
|
||||
).fetchall()
|
||||
result.append(
|
||||
{
|
||||
**public(row),
|
||||
"attachment_ids": [str(a["attachment_id"]) for a in attachments],
|
||||
}
|
||||
)
|
||||
return result
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Explicit, repeatable reconciliation. Default mode is strictly read-only."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from .domain import Failure
|
||||
from .postgres import PostgresStore
|
||||
from .storage import ObjectStorage
|
||||
|
||||
|
||||
def reconcile(store, storage, *, apply=False, grace_seconds=3600, page_size=100):
|
||||
if grace_seconds < 1 or not 1 <= page_size <= 1000:
|
||||
raise ValueError("grace >= 1 second; page size 1..1000")
|
||||
started = time.perf_counter()
|
||||
cutoff = datetime.now(UTC) - timedelta(seconds=grace_seconds)
|
||||
report = {
|
||||
"mode": "apply" if apply else "dry-run",
|
||||
"grace_seconds": grace_seconds,
|
||||
"page_size": page_size,
|
||||
"intents_scanned": 0,
|
||||
"objects_scanned": 0,
|
||||
"metadata_scanned": 0,
|
||||
"old_intents": 0,
|
||||
"orphan_objects": 0,
|
||||
"missing_objects": 0,
|
||||
"deleted_objects": 0,
|
||||
"deleted_intents": 0,
|
||||
"marked_failed": 0,
|
||||
"fresh_objects_skipped": 0,
|
||||
"errors": 0,
|
||||
}
|
||||
|
||||
def audit(conn, event, key):
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_reconcile_audit(event,object_key) VALUES (%s,%s)", (event, key)
|
||||
)
|
||||
|
||||
# Locked live uploads are skipped. A dead upload's committed intent survives
|
||||
# rollback and makes both its object and metadata absence discoverable.
|
||||
after = None
|
||||
while True:
|
||||
with store.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id,object_key,created_at FROM upload_intents WHERE created_at<%s "
|
||||
+ ("AND (created_at,id)>(%s,%s) " if after else "")
|
||||
+ "ORDER BY created_at,id LIMIT %s",
|
||||
[cutoff, *(after or []), page_size],
|
||||
).fetchall()
|
||||
if not rows:
|
||||
break
|
||||
for candidate in rows:
|
||||
report["intents_scanned"] += 1
|
||||
try:
|
||||
with store.connection() as conn, conn.transaction():
|
||||
row = conn.execute(
|
||||
"SELECT * FROM upload_intents WHERE id=%s FOR UPDATE SKIP LOCKED",
|
||||
(candidate["id"],),
|
||||
).fetchone()
|
||||
if not row:
|
||||
continue
|
||||
current_object = storage.head(row["object_key"])
|
||||
if current_object and current_object["LastModified"] >= cutoff:
|
||||
report["fresh_objects_skipped"] += 1
|
||||
continue
|
||||
report["old_intents"] += 1
|
||||
known = conn.execute(
|
||||
"SELECT id FROM attachments WHERE object_key=%s", (row["object_key"],)
|
||||
).fetchone()
|
||||
if apply:
|
||||
if not known:
|
||||
storage.delete(row["object_key"])
|
||||
report["deleted_objects"] += 1
|
||||
conn.execute("DELETE FROM upload_intents WHERE id=%s", (row["id"],))
|
||||
audit(conn, "intent_cleaned", row["object_key"])
|
||||
report["deleted_intents"] += 1
|
||||
except Failure:
|
||||
report["errors"] += 1
|
||||
after = (rows[-1]["created_at"], rows[-1]["id"])
|
||||
|
||||
# Only server-owned original prefix is eligible. Unknown future prefixes are
|
||||
# deliberately outside this tool's deletion policy.
|
||||
try:
|
||||
for obj in storage.objects(page_size):
|
||||
report["objects_scanned"] += 1
|
||||
if not obj["Key"].startswith("original/"):
|
||||
continue
|
||||
if obj["LastModified"] >= cutoff:
|
||||
report["fresh_objects_skipped"] += 1
|
||||
continue
|
||||
with store.connection() as conn, conn.transaction():
|
||||
known = conn.execute(
|
||||
"SELECT 1 FROM attachments WHERE object_key=%s UNION ALL "
|
||||
"SELECT 1 FROM upload_intents WHERE object_key=%s LIMIT 1",
|
||||
(obj["Key"], obj["Key"]),
|
||||
).fetchone()
|
||||
if known:
|
||||
continue
|
||||
current_object = storage.head(obj["Key"])
|
||||
if current_object and current_object["LastModified"] >= cutoff:
|
||||
report["fresh_objects_skipped"] += 1
|
||||
continue
|
||||
report["orphan_objects"] += 1
|
||||
if apply:
|
||||
storage.delete(obj["Key"])
|
||||
audit(conn, "orphan_deleted", obj["Key"])
|
||||
report["deleted_objects"] += 1
|
||||
except Failure:
|
||||
report["errors"] += 1
|
||||
|
||||
# A listing snapshot cannot prove absence. HEAD is checked live under the row
|
||||
# lock immediately before changing status. Never delete legitimate metadata,
|
||||
# including attachments referenced by messages; missing originals are reported.
|
||||
after_id = None
|
||||
while True:
|
||||
with store.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id FROM attachments WHERE created_at<%s AND status='ready' "
|
||||
+ ("AND id>%s " if after_id else "")
|
||||
+ "ORDER BY id LIMIT %s",
|
||||
[cutoff, *([after_id] if after_id else []), page_size],
|
||||
).fetchall()
|
||||
if not rows:
|
||||
break
|
||||
for candidate in rows:
|
||||
report["metadata_scanned"] += 1
|
||||
try:
|
||||
with store.connection() as conn, conn.transaction():
|
||||
row = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s AND status='ready' FOR UPDATE SKIP LOCKED",
|
||||
(candidate["id"],),
|
||||
).fetchone()
|
||||
if not row or storage.exists(row["object_key"]):
|
||||
continue
|
||||
report["missing_objects"] += 1
|
||||
if apply:
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status='failed',error_code='storage_missing',"
|
||||
"updated_at=now() WHERE id=%s",
|
||||
(row["id"],),
|
||||
)
|
||||
audit(conn, "missing_marked_failed", row["object_key"])
|
||||
report["marked_failed"] += 1
|
||||
except Failure:
|
||||
report["errors"] += 1
|
||||
after_id = rows[-1]["id"]
|
||||
report["duration_seconds"] = round(time.perf_counter() - started, 6)
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
parser.add_argument("--grace-seconds", type=int, default=3600)
|
||||
parser.add_argument("--page-size", type=int, default=100)
|
||||
args = parser.parse_args()
|
||||
store, storage = PostgresStore(), ObjectStorage()
|
||||
try:
|
||||
result = reconcile(
|
||||
store,
|
||||
storage,
|
||||
apply=args.apply,
|
||||
grace_seconds=args.grace_seconds,
|
||||
page_size=args.page_size,
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
if result["errors"]:
|
||||
raise SystemExit(1)
|
||||
finally:
|
||||
storage.close()
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Private S3 access with bounded network waits; never expose storage keys in API."""
|
||||
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .config import secret
|
||||
from .domain import Failure
|
||||
|
||||
|
||||
class ObjectStorage:
|
||||
def __init__(self, admin=False):
|
||||
self.bucket = os.getenv("S3_BUCKET", "messenger")
|
||||
prefix = "S3_ADMIN" if admin else "S3"
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"),
|
||||
aws_access_key_id=secret(prefix + "_ACCESS_KEY"),
|
||||
aws_secret_access_key=secret(prefix + "_SECRET_KEY"),
|
||||
region_name="us-east-1",
|
||||
config=Config(
|
||||
connect_timeout=1,
|
||||
read_timeout=2,
|
||||
retries={"total_max_attempts": 1},
|
||||
s3={"addressing_style": "path"},
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
),
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def boundary(self):
|
||||
try:
|
||||
yield
|
||||
except (BotoCoreError, ClientError):
|
||||
raise Failure(503, "Хранилище файлов временно недоступно") from None
|
||||
|
||||
def put(self, key, fileobj, size, content_type):
|
||||
fileobj.seek(0)
|
||||
with self.boundary():
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=fileobj,
|
||||
ContentLength=size,
|
||||
ContentType=content_type,
|
||||
)
|
||||
|
||||
def get(self, key):
|
||||
try:
|
||||
return self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
|
||||
raise Failure(404, "Объект не найден") from None
|
||||
raise Failure(503, "Хранилище файлов временно недоступно") from None
|
||||
except BotoCoreError:
|
||||
raise Failure(503, "Хранилище файлов временно недоступно") from None
|
||||
|
||||
def head(self, key):
|
||||
try:
|
||||
return self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404", "NotFound"}:
|
||||
return None
|
||||
raise Failure(503, "Хранилище файлов временно недоступно") from None
|
||||
except BotoCoreError:
|
||||
raise Failure(503, "Хранилище файлов временно недоступно") from None
|
||||
|
||||
def exists(self, key):
|
||||
return self.head(key) is not None
|
||||
|
||||
def delete(self, key):
|
||||
with self.boundary():
|
||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
||||
|
||||
def objects(self, page_size=100):
|
||||
with self.boundary():
|
||||
paginator = self.client.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(
|
||||
Bucket=self.bucket, PaginationConfig={"PageSize": page_size}
|
||||
):
|
||||
yield from page.get("Contents", [])
|
||||
|
||||
def ready(self):
|
||||
try:
|
||||
with self.boundary():
|
||||
self.client.head_bucket(Bucket=self.bucket)
|
||||
return True
|
||||
except Failure:
|
||||
return False
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Idempotent private bucket bootstrap; this process alone receives admin keys."""
|
||||
|
||||
import time
|
||||
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .storage import ObjectStorage
|
||||
|
||||
|
||||
def main():
|
||||
storage = ObjectStorage(admin=True)
|
||||
for attempt in range(30):
|
||||
try:
|
||||
try:
|
||||
storage.client.head_bucket(Bucket=storage.bucket)
|
||||
except ClientError as exc:
|
||||
if exc.response["Error"]["Code"] not in {"404", "NoSuchBucket", "NotFound"}:
|
||||
raise
|
||||
storage.client.create_bucket(Bucket=storage.bucket)
|
||||
print("Private bucket ready; anonymous identity is not configured.")
|
||||
storage.close()
|
||||
return
|
||||
except (ClientError, BotoCoreError):
|
||||
if attempt == 29:
|
||||
raise RuntimeError("S3 bootstrap timeout") from None
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user