forked from mtusi2026/mtusi2026lab4
lab4: вложения в приватном S3 (SeaweedFS), метаданные в PostgreSQL
- compose: сервис s3 (SeaweedFS 4.46 по digest, volume s3data, внутренняя сеть storage,
web-интерфейсы только на 127.0.0.1 внутри контейнера), одноразовый s3-init;
- три пары ключей S3: admin (только bootstrap), api (Read+Write своего bucket),
reconciler (+List); bootstrap идемпотентен и проверяет приватность и права;
- миграция 0004: attachments (pending -> ready, случайный object_key), message_attachments,
post_message_v2 с проверкой вложений в той же транзакции;
- POST /chats/{id}/attachments: лимиты до чтения/декодирования, потоковая проверка PNG
(CRC + zlib до точного размера), JPEG через djpeg -strict, семафор загрузок;
- GET метаданных и /content (стриминг из S3, без redirect), readiness с S3,
HAProxy: деградированный режим без S3 через /health/ready/core;
- janitor незавершённых загрузок с grace period.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
d916d09f7b
commit
335df4b2b2
+160
-12
@@ -6,6 +6,7 @@ import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
@@ -14,15 +15,16 @@ from typing import Any
|
||||
from starlette.applications import Starlette
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import JSONResponse, Response
|
||||
from starlette.responses import JSONResponse, Response, StreamingResponse
|
||||
from starlette.routing import Route
|
||||
from starlette.types import ASGIApp, Message as AsgiMessage, Receive, Scope, Send
|
||||
|
||||
from . import authhdr, cursor, docs, passwords, repo, validation
|
||||
from . import attachments, authhdr, cursor, docs, passwords, repo, validation
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unauthorized, Unavailable
|
||||
from .sessions import SessionStore, build_store
|
||||
from .storage import ObjectMissing, Storage
|
||||
from .timefmt import rfc3339
|
||||
|
||||
log = logging.getLogger("messenger")
|
||||
@@ -126,6 +128,10 @@ def sessions_of(request: Request) -> SessionStore:
|
||||
return request.app.state.sessions
|
||||
|
||||
|
||||
def storage_of(request: Request) -> Storage:
|
||||
return request.app.state.storage
|
||||
|
||||
|
||||
async def authenticate(request: Request) -> str:
|
||||
"""User id of a valid Bearer session, otherwise 401. X-User-Id is ignored since lab 2.
|
||||
|
||||
@@ -206,15 +212,32 @@ async def live(request: Request) -> JSONResponse:
|
||||
|
||||
|
||||
async def ready(request: Request) -> JSONResponse:
|
||||
"""200 only if the mandatory synchronous path can be served: PostgreSQL and,
|
||||
with SESSION_STORE=valkey, Valkey answer. 503 while draining after SIGTERM."""
|
||||
"""200 only if the WHOLE mandatory synchronous path can be served: PostgreSQL, Valkey
|
||||
(SESSION_STORE=valkey) and, since lab 4, the S3 bucket (upload/download of attachments)
|
||||
answer. 503 while draining after SIGTERM. HAProxy routes to such replicas first."""
|
||||
if request.app.state.draining:
|
||||
raise Unavailable("Сервер завершает работу")
|
||||
db_ok, store_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping())
|
||||
db_ok, store_ok, s3_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping(),
|
||||
storage_of(request).ping())
|
||||
if not db_ok:
|
||||
raise Unavailable("PostgreSQL недоступен")
|
||||
if not store_ok:
|
||||
raise Unavailable("Хранилище сессий недоступно")
|
||||
if not s3_ok:
|
||||
raise Unavailable("Хранилище файлов (S3) недоступно")
|
||||
return ok({"status": "ok"})
|
||||
|
||||
|
||||
async def ready_core(request: Request) -> JSONResponse:
|
||||
"""Not in the contract: readiness WITHOUT S3, for the load balancer's degraded mode only.
|
||||
If S3 is down for every replica, /health/ready is 503 everywhere; HAProxy then routes to
|
||||
replicas that are at least core-ready, so chats and messages keep working while every
|
||||
attachment request answers 503 itself (see infra/haproxy/haproxy.cfg)."""
|
||||
if request.app.state.draining:
|
||||
raise Unavailable("Сервер завершает работу")
|
||||
db_ok, store_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping())
|
||||
if not (db_ok and store_ok):
|
||||
raise Unavailable("PostgreSQL или хранилище сессий недоступно")
|
||||
return ok({"status": "ok"})
|
||||
|
||||
|
||||
@@ -288,12 +311,12 @@ async def get_chat(request: Request) -> JSONResponse:
|
||||
|
||||
|
||||
def parse_message(data: dict[str, Any]) -> tuple[str, list[str]]:
|
||||
"""`text` is required and a string (also with attachments); `attachment_ids`, if present, an
|
||||
array of 0-10 unique UUIDs. "" or whitespace-only text is allowed only with attachments."""
|
||||
validation.check_fields(data, {"text"}, frozenset({"attachment_ids"}))
|
||||
text = validation.text_field(data, "text", 4000)
|
||||
attachment_ids = validation.attachment_ids_field(data)
|
||||
if attachment_ids:
|
||||
# Lab 3 has no attachments: every referenced ID is unknown (lab 4 semantics: 404).
|
||||
raise NotFound("Вложение не найдено")
|
||||
text = validation.text_field(data, "text", 4000, min_len=0 if attachment_ids else 1,
|
||||
allow_blank=bool(attachment_ids))
|
||||
return text, attachment_ids
|
||||
|
||||
|
||||
@@ -309,7 +332,7 @@ async def create_message(request: Request) -> JSONResponse:
|
||||
key = idempotency_key(request)
|
||||
text, attachment_ids = parse_message(await read_json(request))
|
||||
result = await repo.post_message(db, user_id, chat_id, key, request_fingerprint(text, attachment_ids), text,
|
||||
settings.idempotency_ttl_seconds)
|
||||
settings.idempotency_ttl_seconds, attachment_ids)
|
||||
else:
|
||||
try:
|
||||
key = idempotency_key(request)
|
||||
@@ -321,7 +344,7 @@ async def create_message(request: Request) -> JSONResponse:
|
||||
# Membership check, key claim, chat clock and message: one SQL call, one transaction.
|
||||
result = await repo.post_message_single_call(db, user_id, chat_id, key,
|
||||
request_fingerprint(text, attachment_ids), text,
|
||||
settings.idempotency_ttl_seconds)
|
||||
settings.idempotency_ttl_seconds, attachment_ids)
|
||||
if result.replayed:
|
||||
log.info("idempotent_replay", extra={"fields": {"request_id": request.state.request_id,
|
||||
"message_id": result.message.id}})
|
||||
@@ -340,10 +363,111 @@ async def list_messages(request: Request) -> JSONResponse:
|
||||
return ok({"items": [m.to_json() for m in items], "next_cursor": next_cursor})
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ attachments (lab 4)
|
||||
def fault(request: Request, point: str) -> None:
|
||||
"""Failure injection for the level 4 experiments (FAULT_POINTS, compose.faults.yaml only)."""
|
||||
if point in request.app.state.settings.fault_points:
|
||||
log.warning("fault_injected", extra={"fields": {"request_id": request.state.request_id, "point": point}})
|
||||
raise Unavailable("Внедрённый отказ: " + point)
|
||||
|
||||
|
||||
async def upload_attachment(request: Request) -> JSONResponse:
|
||||
"""POST /chats/{chat_id}/attachments?filename=...: raw PNG/JPEG -> 201 ready (attachments.py)."""
|
||||
settings: Settings = request.app.state.settings
|
||||
reader = attachments.BodyReader(request)
|
||||
try:
|
||||
user_id = await authenticate(request)
|
||||
chat_id = path_uuid(request, "chat_id")
|
||||
db = db_of(request)
|
||||
await repo.ensure_member(db, user_id, chat_id) # outsiders: 404 before anything about the body
|
||||
filename = validation.filename_param(validation.query_items(request.scope["query_string"]))
|
||||
declared = attachments.declared_type(request)
|
||||
attachments.declared_length(request, settings.max_upload_bytes)
|
||||
slots: asyncio.Semaphore = request.app.state.upload_slots
|
||||
try:
|
||||
await asyncio.wait_for(slots.acquire(), settings.upload_queue_timeout_seconds)
|
||||
except asyncio.TimeoutError:
|
||||
raise attachments.Busy("Слишком много одновременных загрузок, повторите позже") from None
|
||||
try:
|
||||
upload = await attachments.read_image(reader, settings, declared)
|
||||
finally:
|
||||
slots.release()
|
||||
except ApiError:
|
||||
# Early rejection: drop (not store) a bounded rest of the body so the client reads our error.
|
||||
await reader.discard(settings.max_upload_bytes + (1 << 20), timeout=5.0)
|
||||
raise
|
||||
attachment_id = uuid.uuid4()
|
||||
object_key = attachments.new_object_key()
|
||||
storage = storage_of(request)
|
||||
await attachments.create_intent(db, attachment_id, chat_id, user_id, filename, upload.info, len(upload.data),
|
||||
upload.sha256, object_key)
|
||||
await storage.put(object_key, upload.data, upload.info.content_type,
|
||||
{"attachment-id": str(attachment_id), "sha256": upload.sha256.hex()})
|
||||
fault(request, "after_s3_put")
|
||||
attachment = await attachments.mark_ready(db, attachment_id)
|
||||
if attachment is None:
|
||||
raise Unavailable("Загрузка не завершена, повторите запрос")
|
||||
fault(request, "after_ready_commit")
|
||||
log.info("attachment_created", extra={"fields": {"request_id": request.state.request_id,
|
||||
"attachment_id": attachment.id, "chat_id": chat_id,
|
||||
"size": attachment.size_bytes, "type": attachment.content_type}})
|
||||
return ok(attachment.to_json(), 201)
|
||||
|
||||
|
||||
async def get_attachment(request: Request) -> JSONResponse:
|
||||
user_id = await authenticate(request)
|
||||
attachment_id = path_uuid(request, "attachment_id")
|
||||
return ok((await attachments.get_visible(db_of(request), user_id, attachment_id)).to_json())
|
||||
|
||||
|
||||
def content_disposition(filename: str) -> str:
|
||||
"""RFC 6266/5987: ASCII fallback + UTF-8 name; the stored name itself is never interpreted."""
|
||||
from urllib.parse import quote
|
||||
fallback = "".join(c if 32 <= ord(c) < 127 and c not in '"\\%;' else "_" for c in filename)
|
||||
return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{quote(filename, safe='')}"
|
||||
|
||||
|
||||
async def download_attachment(request: Request) -> Response:
|
||||
"""Original bytes through the authorized API: no redirect, no public S3 URL. Streamed from
|
||||
S3 in 64 KiB chunks (memory per download does not depend on the file size)."""
|
||||
user_id = await authenticate(request)
|
||||
attachment_id = path_uuid(request, "attachment_id")
|
||||
query = validation.query_items(request.scope["query_string"])
|
||||
try:
|
||||
validation.variant_param(query)
|
||||
except validation.NotFoundVariant:
|
||||
await attachments.get_visible(db_of(request), user_id, attachment_id) # 404 for outsiders first
|
||||
raise NotFound("Такого варианта у вложения нет") from None
|
||||
attachment = await attachments.get_visible(db_of(request), user_id, attachment_id)
|
||||
if attachment.lost:
|
||||
raise NotFound("Оригинал вложения утрачен")
|
||||
try:
|
||||
size, chunks = await storage_of(request).open_stream(attachment.object_key)
|
||||
except ObjectMissing:
|
||||
# A ready row without its object: data loss, reported by the reconciler (quarantine).
|
||||
log.error("attachment_object_missing", extra={"fields": {"request_id": request.state.request_id,
|
||||
"attachment_id": attachment.id}})
|
||||
raise NotFound("Оригинал вложения утрачен") from None
|
||||
if size != attachment.size_bytes:
|
||||
log.error("attachment_size_mismatch", extra={"fields": {"request_id": request.state.request_id,
|
||||
"attachment_id": attachment.id}})
|
||||
await chunks.aclose()
|
||||
raise Unavailable("Оригинал вложения временно недоступен")
|
||||
headers = {
|
||||
"Content-Length": str(size),
|
||||
"Content-Disposition": content_disposition(attachment.filename),
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"Cache-Control": "private, no-store",
|
||||
"ETag": '"' + attachment.sha256 + '"',
|
||||
}
|
||||
return StreamingResponse(chunks, media_type=attachment.content_type, headers=headers)
|
||||
|
||||
|
||||
def create_app(settings: Settings | None = None) -> Starlette:
|
||||
settings = settings or Settings.from_env()
|
||||
db = Database(settings)
|
||||
sessions = build_store(settings, db)
|
||||
storage = Storage(settings)
|
||||
|
||||
async def purge_loop() -> None:
|
||||
"""Housekeeping: expired Idempotency-Key rows (older than IDEMPOTENCY_TTL_SECONDS)."""
|
||||
@@ -356,6 +480,18 @@ def create_app(settings: Settings | None = None) -> Starlette:
|
||||
except ApiError:
|
||||
pass # database unavailable right now; next round
|
||||
|
||||
async def janitor_loop(every: float) -> None:
|
||||
"""Level 4 cleanup: intents of failed uploads older than PENDING_GRACE_SECONDS (attachments.py).
|
||||
Every replica runs it; the advisory lock lets one round run at a time."""
|
||||
while True:
|
||||
await asyncio.sleep(every * (0.5 + random.random()))
|
||||
try:
|
||||
removed = await attachments.clean_stale_pending(db, storage, settings.pending_grace_seconds)
|
||||
if removed:
|
||||
log.info("stale_uploads_removed", extra={"fields": {"count": removed}})
|
||||
except ApiError:
|
||||
pass # PostgreSQL or S3 unavailable right now; the rows stay pending for the next round
|
||||
|
||||
stats_log = logging.getLogger("messenger.stats")
|
||||
|
||||
async def stats_loop(every: float) -> None:
|
||||
@@ -376,12 +512,17 @@ def create_app(settings: Settings | None = None) -> Starlette:
|
||||
async def lifespan(app: Starlette):
|
||||
passwords.configure(settings.hash_concurrency)
|
||||
await db.open()
|
||||
await storage.open()
|
||||
log.info("dependencies_configured", extra={"fields": {
|
||||
"session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds,
|
||||
"db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds,
|
||||
"instance_id": settings.instance_id, "drain_s": settings.drain_seconds,
|
||||
"message_write_path": settings.message_write_path}})
|
||||
"message_write_path": settings.message_write_path, "s3_bucket": settings.s3_bucket,
|
||||
"upload_concurrency": settings.upload_concurrency, "pending_grace_s": settings.pending_grace_seconds,
|
||||
"fault_points": sorted(settings.fault_points)}})
|
||||
tasks = [asyncio.create_task(purge_loop())]
|
||||
if settings.janitor_interval_seconds > 0:
|
||||
tasks.append(asyncio.create_task(janitor_loop(settings.janitor_interval_seconds)))
|
||||
if settings.stats_log_seconds > 0:
|
||||
tasks.append(asyncio.create_task(stats_loop(settings.stats_log_seconds)))
|
||||
try:
|
||||
@@ -390,11 +531,13 @@ def create_app(settings: Settings | None = None) -> Starlette:
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await sessions.close()
|
||||
await storage.close()
|
||||
await db.close()
|
||||
|
||||
routes = [
|
||||
Route("/health/live", live, methods=["GET"]),
|
||||
Route("/health/ready", ready, methods=["GET"]),
|
||||
Route("/health/ready/core", ready_core, methods=["GET"]),
|
||||
Route("/api/v1/users", register_user, methods=["POST"]),
|
||||
Route("/api/v1/users/me", me, methods=["GET"]),
|
||||
Route("/api/v1/auth/sessions", create_session, methods=["POST"]),
|
||||
@@ -404,6 +547,9 @@ def create_app(settings: Settings | None = None) -> Starlette:
|
||||
Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]),
|
||||
Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]),
|
||||
Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]),
|
||||
Route("/api/v1/chats/{chat_id}/attachments", upload_attachment, methods=["POST"]),
|
||||
Route("/api/v1/attachments/{attachment_id}", get_attachment, methods=["GET"]),
|
||||
Route("/api/v1/attachments/{attachment_id}/content", download_attachment, methods=["GET"]),
|
||||
]
|
||||
if settings.docs_enabled: # Swagger UI for manual experiments, not part of the contract
|
||||
routes += [
|
||||
@@ -419,5 +565,7 @@ def create_app(settings: Settings | None = None) -> Starlette:
|
||||
app.state.settings = settings
|
||||
app.state.db = db
|
||||
app.state.sessions = sessions
|
||||
app.state.storage = storage
|
||||
app.state.upload_slots = asyncio.Semaphore(settings.upload_concurrency)
|
||||
app.state.draining = False
|
||||
return app
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
"""Attachments (lab 4): metadata in PostgreSQL, original bytes in the private S3 bucket.
|
||||
|
||||
Upload (POST /chats/{id}/attachments, raw PNG/JPEG body), in this order:
|
||||
|
||||
auth -> membership (404) -> filename / Content-Type / Content-Length (400/415/413)
|
||||
-> upload slot (at most UPLOAD_CONCURRENCY bodies per replica, else 503 + Retry-After)
|
||||
-> body in chunks: size limit, read deadline, format + pixel count from the first bytes
|
||||
-> full integrity check (images.verify) -> SHA-256 / size known
|
||||
-> INSERT row status='pending' with a fresh random object_key, COMMIT (intent)
|
||||
-> PUT object to S3 (Content-MD5)
|
||||
-> UPDATE status='ready' WHERE status='pending', COMMIT -> 201 ready.
|
||||
|
||||
Why 201 is honest: it is sent only after BOTH the object and the ready row are durable.
|
||||
What a failure between the steps leaves behind, and who removes it (docs/CONSISTENCY.md):
|
||||
|
||||
* before the intent COMMIT: nothing (no object was written);
|
||||
* after the intent, before/while PUT, before the ready COMMIT: a 'pending' row (+ maybe the
|
||||
object). The client got 503 and may retry (a new attachment). The janitor (below) deletes the
|
||||
object - idempotent, a missing key is fine - and then the row, once it is older than
|
||||
PENDING_GRACE_SECONDS. Nothing is compensated in the request path: after a lost answer of
|
||||
COMMIT we do not know whether the row is ready, and deleting the object could destroy a
|
||||
ready attachment. Only the janitor decides, under a row lock, from the committed state;
|
||||
* after the ready COMMIT (answer lost): a correct ready attachment the client does not know.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import logging
|
||||
import secrets
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
import psycopg
|
||||
from psycopg import errors as pg_errors
|
||||
from starlette.requests import Request
|
||||
|
||||
from . import images
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
from .errors import ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType
|
||||
from .storage import OBJECT_PREFIX, Storage
|
||||
from .timefmt import rfc3339
|
||||
|
||||
log = logging.getLogger("messenger.attachments")
|
||||
|
||||
JANITOR_LOCK_ID = 2026_04 # advisory lock: one janitor round at a time across all replicas
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Attachment:
|
||||
id: str
|
||||
chat_id: str
|
||||
owner_id: str
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
object_key: str
|
||||
status: str
|
||||
created_at: dt.datetime
|
||||
updated_at: dt.datetime
|
||||
lost: bool = False
|
||||
|
||||
def to_json(self) -> dict:
|
||||
"""Public representation: no object key, no bucket, no internal state."""
|
||||
return {"id": self.id, "chat_id": self.chat_id, "owner_id": self.owner_id, "filename": self.filename,
|
||||
"content_type": self.content_type, "size_bytes": self.size_bytes, "sha256": self.sha256,
|
||||
"status": "ready", "variants": [], "error_code": None,
|
||||
"created_at": rfc3339(self.created_at), "updated_at": rfc3339(self.updated_at)}
|
||||
|
||||
|
||||
COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.object_key, "
|
||||
"a.status, a.created_at, a.updated_at, a.lost_at IS NOT NULL")
|
||||
|
||||
|
||||
def _attachment(row) -> Attachment:
|
||||
return Attachment(str(row[0]), str(row[1]), str(row[2]), row[3], row[4], row[5], bytes(row[6]).hex(), row[7],
|
||||
row[8], row[9], row[10], row[11])
|
||||
|
||||
|
||||
def new_object_key() -> str:
|
||||
"""Server-generated, 128 random bits, unrelated to the attachment id and the filename."""
|
||||
return OBJECT_PREFIX + secrets.token_hex(16)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ SQL
|
||||
async def create_intent(db: Database, attachment_id: uuid.UUID, chat_id: str, owner_id: str, filename: str,
|
||||
info: images.ImageInfo, size: int, sha256: bytes, object_key: str) -> None:
|
||||
async def work(conn: psycopg.AsyncConnection) -> None:
|
||||
await conn.execute(
|
||||
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
"width, height, object_key, status) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending')",
|
||||
(attachment_id, chat_id, owner_id, filename, info.content_type, size, sha256, info.width, info.height,
|
||||
object_key))
|
||||
try:
|
||||
await db.run(work, transaction=False)
|
||||
except pg_errors.ForeignKeyViolation:
|
||||
raise NotFound("Чат не найден") from None
|
||||
|
||||
|
||||
async def mark_ready(db: Database, attachment_id: uuid.UUID) -> Attachment | None:
|
||||
"""pending -> ready. None if the row is no longer pending (the janitor removed a stale intent)."""
|
||||
async def work(conn: psycopg.AsyncConnection) -> Attachment | None:
|
||||
cur = await conn.execute(
|
||||
"UPDATE attachments a SET status = 'ready', updated_at = clock_timestamp() "
|
||||
f"WHERE a.id = %s AND a.status = 'pending' RETURNING {COLUMNS}", (attachment_id,))
|
||||
row = await cur.fetchone()
|
||||
return _attachment(row) if row else None
|
||||
return await db.run(work, transaction=False)
|
||||
|
||||
|
||||
async def get_visible(db: Database, user_id: str, attachment_id: str) -> Attachment:
|
||||
"""A READY attachment of a chat the user is a member of; anything else is 404 (unknown,
|
||||
foreign chat, unfinished upload) - existence of foreign objects is never revealed."""
|
||||
async def work(conn: psycopg.AsyncConnection) -> Attachment:
|
||||
cur = await conn.execute(
|
||||
f"SELECT {COLUMNS} FROM attachments a "
|
||||
"JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s "
|
||||
"WHERE a.id = %s AND a.status = 'ready'", (user_id, attachment_id))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
raise NotFound("Вложение не найдено")
|
||||
return _attachment(row)
|
||||
return await db.run(work, transaction=False)
|
||||
|
||||
|
||||
async def clean_stale_pending(db: Database, storage: Storage, grace_seconds: int, batch: int = 20) -> int:
|
||||
"""Janitor round: remove intents of uploads that never became ready.
|
||||
|
||||
Under FOR UPDATE: an upload that is (impossibly) still running would block on the row in
|
||||
mark_ready and then find it gone (-> 503), so a ready attachment can never lose its object.
|
||||
Order: object first (idempotent), row second; a crash in between leaves the pending row and
|
||||
the next round repeats. Only status='pending' rows older than the grace period are touched:
|
||||
fresh uploads and ready/used attachments are never selected.
|
||||
"""
|
||||
async def work(conn: psycopg.AsyncConnection) -> int:
|
||||
cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (JANITOR_LOCK_ID,))
|
||||
if not (await cur.fetchone())[0]:
|
||||
return 0
|
||||
cur = await conn.execute(
|
||||
"SELECT id, object_key FROM attachments WHERE status = 'pending' "
|
||||
"AND created_at < clock_timestamp() - make_interval(secs => %s) "
|
||||
"ORDER BY created_at LIMIT %s FOR UPDATE SKIP LOCKED", (grace_seconds, batch))
|
||||
rows = await cur.fetchall()
|
||||
if not rows:
|
||||
return 0
|
||||
await asyncio.gather(*(storage.delete(key) for _, key in rows))
|
||||
await conn.execute("DELETE FROM attachments WHERE id = ANY(%s) AND status = 'pending'",
|
||||
([r[0] for r in rows],))
|
||||
return len(rows)
|
||||
return await db.run(work, deadline=storage.timeout + 5)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ request body
|
||||
class Busy(Unavailable):
|
||||
"""All upload slots of this replica are taken: retry shortly (no body was read)."""
|
||||
|
||||
@property
|
||||
def headers(self) -> dict[str, str]:
|
||||
return {"Retry-After": "1"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Upload:
|
||||
data: bytes
|
||||
info: images.ImageInfo
|
||||
sha256: bytes
|
||||
|
||||
|
||||
class BodyReader:
|
||||
"""Reads the raw ASGI body ourselves (not request.stream()), so we always know how much of it
|
||||
is consumed and can discard the rest after an early rejection."""
|
||||
|
||||
def __init__(self, request: Request):
|
||||
self.receive = request.receive
|
||||
self.done = False
|
||||
|
||||
async def chunk(self, timeout: float) -> bytes | None:
|
||||
if self.done:
|
||||
return None
|
||||
message = await asyncio.wait_for(self.receive(), timeout)
|
||||
if message["type"] == "http.disconnect":
|
||||
self.done = True
|
||||
raise InvalidRequest("Клиент прервал загрузку")
|
||||
if not message.get("more_body", False):
|
||||
self.done = True
|
||||
return message.get("body", b"")
|
||||
|
||||
async def discard(self, limit: int, timeout: float) -> None:
|
||||
"""After an early 4xx: read and DROP up to `limit` more bytes, so an ordinary client that
|
||||
writes the whole body before reading the answer still gets our JSON error instead of a
|
||||
connection reset. Nothing is kept in memory; beyond the limit the connection is dropped."""
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + timeout
|
||||
seen = 0
|
||||
try:
|
||||
while not self.done and seen <= limit:
|
||||
remaining = deadline - loop.time()
|
||||
if remaining <= 0:
|
||||
return
|
||||
seen += len(await self.chunk(remaining) or b"")
|
||||
except (ApiError, asyncio.TimeoutError):
|
||||
return
|
||||
|
||||
|
||||
def declared_type(request: Request) -> str:
|
||||
"""Content-Type of the upload must be image/png or image/jpeg (parameters ignored)."""
|
||||
media_type = request.headers.get("content-type", "").partition(";")[0].strip().lower()
|
||||
if media_type not in (images.PNG, images.JPEG):
|
||||
raise UnsupportedMediaType("Content-Type должен быть image/png или image/jpeg")
|
||||
return media_type
|
||||
|
||||
|
||||
def declared_length(request: Request, limit: int) -> int | None:
|
||||
raw = request.headers.get("content-length")
|
||||
if raw is None:
|
||||
return None # chunked transfer: the limit is enforced while reading
|
||||
if not raw.isdigit():
|
||||
raise InvalidRequest("Неверный Content-Length")
|
||||
length = int(raw)
|
||||
if length > limit:
|
||||
raise PayloadTooLarge(f"Файл больше {limit} байт")
|
||||
if length == 0:
|
||||
raise InvalidRequest("Тело запроса пустое: нужны байты изображения")
|
||||
return length
|
||||
|
||||
|
||||
async def read_image(reader: BodyReader, settings: Settings, declared: str) -> Upload:
|
||||
"""Body -> verified image. Memory: at most MAX_UPLOAD_BYTES per upload (bytearray); the format
|
||||
and the pixel count are known after the first bytes, so a picture that is too big or not a
|
||||
PNG/JPEG is rejected without reading/decoding the rest."""
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + settings.upload_read_timeout_seconds
|
||||
limit = settings.max_upload_bytes
|
||||
buffer = bytearray()
|
||||
digest = hashlib.sha256()
|
||||
info: images.ImageInfo | None = None
|
||||
while True:
|
||||
remaining = deadline - loop.time()
|
||||
if remaining <= 0:
|
||||
raise InvalidRequest("Тело запроса не получено вовремя")
|
||||
try:
|
||||
chunk = await reader.chunk(remaining)
|
||||
except asyncio.TimeoutError:
|
||||
raise InvalidRequest("Тело запроса не получено вовремя") from None
|
||||
if chunk is None:
|
||||
break
|
||||
if len(buffer) + len(chunk) > limit:
|
||||
raise PayloadTooLarge(f"Файл больше {limit} байт")
|
||||
buffer += chunk
|
||||
digest.update(chunk)
|
||||
if info is None and buffer:
|
||||
info = _sniff(buffer, settings, declared)
|
||||
if not buffer:
|
||||
raise InvalidRequest("Тело запроса пустое: нужны байты изображения")
|
||||
if info is None:
|
||||
raise UnsupportedMediaType("Файл не является корректным изображением PNG или JPEG")
|
||||
data = bytes(buffer)
|
||||
del buffer
|
||||
try:
|
||||
await images.verify(data, info, timeout=settings.image_verify_timeout_seconds,
|
||||
jpeg_max_memory_mb=settings.jpeg_max_memory_mb)
|
||||
except images.ImageError as exc:
|
||||
raise _image_error(exc) from None
|
||||
except asyncio.TimeoutError:
|
||||
raise UnsupportedMediaType("Изображение не удалось проверить за отведённое время") from None
|
||||
return Upload(data, info, digest.digest())
|
||||
|
||||
|
||||
def _sniff(buffer: bytearray, settings: Settings, declared: str) -> images.ImageInfo | None:
|
||||
try:
|
||||
info = images.sniff(buffer, settings.max_image_pixels)
|
||||
except images.ImageError as exc:
|
||||
raise _image_error(exc) from None
|
||||
if info is not None and info.content_type != declared:
|
||||
# "MIME spoofing": the header says PNG, the bytes are JPEG (or vice versa).
|
||||
raise UnsupportedMediaType("Содержимое не совпадает с Content-Type")
|
||||
return info
|
||||
|
||||
|
||||
def _image_error(exc: images.ImageError) -> ApiError:
|
||||
return PayloadTooLarge(exc.message) if exc.status == 413 else UnsupportedMediaType(exc.message)
|
||||
|
||||
@@ -115,6 +115,25 @@ class Settings:
|
||||
idempotency_ttl_seconds: int = 86400
|
||||
stats_log_seconds: float = 0.0
|
||||
message_write_path: str = "function"
|
||||
# Lab 4: private S3 bucket and attachment limits (see storage.py, attachments.py).
|
||||
s3_endpoint: str = "http://s3:8333"
|
||||
s3_region: str = "us-east-1"
|
||||
s3_bucket: str = "messenger-attachments"
|
||||
s3_access_key_id: str = ""
|
||||
s3_secret_access_key: str = ""
|
||||
s3_timeout_seconds: float = 3.0
|
||||
s3_connect_timeout_seconds: float = 1.0
|
||||
s3_max_connections: int = 20
|
||||
max_upload_bytes: int = 10 * 1024 * 1024
|
||||
max_image_pixels: int = 40_000_000
|
||||
upload_concurrency: int = 4
|
||||
upload_queue_timeout_seconds: float = 5.0
|
||||
upload_read_timeout_seconds: float = 30.0
|
||||
image_verify_timeout_seconds: float = 10.0
|
||||
jpeg_max_memory_mb: int = 256
|
||||
pending_grace_seconds: int = 900
|
||||
janitor_interval_seconds: float = 60.0
|
||||
fault_points: frozenset[str] = frozenset()
|
||||
|
||||
@property
|
||||
def db_statement_timeout_ms(self) -> int:
|
||||
@@ -165,4 +184,49 @@ class Settings:
|
||||
# Track 1: `function` = one SQL call per send (migration 0003), `multi` = the previous
|
||||
# multi-statement transaction; both have identical semantics (A/B on one image).
|
||||
message_write_path=_choice("MESSAGE_WRITE_PATH", "function", {"function", "multi"}),
|
||||
**s3_settings(),
|
||||
)
|
||||
|
||||
|
||||
# Contract limits (contracts/README.md): configurable DOWN for experiments, never above them.
|
||||
CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024
|
||||
CONTRACT_MAX_IMAGE_PIXELS = 40_000_000
|
||||
# Failure points for the level 4 experiments only (compose.faults.yaml); empty in normal runs.
|
||||
FAULT_POINTS = {"after_s3_put", "after_ready_commit"}
|
||||
|
||||
|
||||
def s3_settings(*, credentials_required: bool = True) -> dict:
|
||||
"""S3 part of Settings; also used by the reconciler/bootstrap CLIs with their own keys."""
|
||||
faults = frozenset(f.strip() for f in os.environ.get("FAULT_POINTS", "").split(",") if f.strip())
|
||||
if faults - FAULT_POINTS:
|
||||
raise ConfigError(f"FAULT_POINTS may contain only {sorted(FAULT_POINTS)}")
|
||||
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").strip() or "http://s3:8333"
|
||||
if not re.fullmatch(r"https?://[A-Za-z0-9.\-]+(:[0-9]{1,5})?/?", endpoint):
|
||||
raise ConfigError(f"S3_ENDPOINT must be http(s)://host[:port], got {endpoint!r}")
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments").strip() or "messenger-attachments"
|
||||
if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket):
|
||||
raise ConfigError(f"S3_BUCKET is not a valid bucket name: {bucket!r}")
|
||||
return dict(
|
||||
s3_endpoint=endpoint,
|
||||
s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1",
|
||||
s3_bucket=bucket,
|
||||
s3_access_key_id=_secret("S3_ACCESS_KEY_ID", required=credentials_required),
|
||||
s3_secret_access_key=_secret("S3_SECRET_ACCESS_KEY", required=credentials_required),
|
||||
s3_timeout_seconds=_float("S3_TIMEOUT_SECONDS", 3.0, 0.2, 30.0),
|
||||
s3_connect_timeout_seconds=_float("S3_CONNECT_TIMEOUT_SECONDS", 1.0, 0.1, 10.0),
|
||||
s3_max_connections=_int("S3_MAX_CONNECTIONS", 20, 1, 200),
|
||||
max_upload_bytes=_int("MAX_UPLOAD_BYTES", CONTRACT_MAX_UPLOAD_BYTES, 1024, CONTRACT_MAX_UPLOAD_BYTES),
|
||||
max_image_pixels=_int("MAX_IMAGE_PIXELS", CONTRACT_MAX_IMAGE_PIXELS, 1, CONTRACT_MAX_IMAGE_PIXELS),
|
||||
# At most this many request bodies of one replica are buffered/checked at once (RAM bound:
|
||||
# UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES); further uploads wait, then get 503.
|
||||
upload_concurrency=_int("UPLOAD_CONCURRENCY", 4, 1, 64),
|
||||
upload_queue_timeout_seconds=_float("UPLOAD_QUEUE_TIMEOUT_SECONDS", 5.0, 0.0, 60.0),
|
||||
upload_read_timeout_seconds=_float("UPLOAD_READ_TIMEOUT_SECONDS", 30.0, 1.0, 300.0),
|
||||
image_verify_timeout_seconds=_float("IMAGE_VERIFY_TIMEOUT_SECONDS", 10.0, 1.0, 120.0),
|
||||
jpeg_max_memory_mb=_int("JPEG_MAX_MEMORY_MB", 256, 16, 2048),
|
||||
# An intent row still `pending` after this long belongs to a failed upload: the janitor
|
||||
# deletes its object and the row. Far above the longest possible upload (~45 s).
|
||||
pending_grace_seconds=_int("PENDING_GRACE_SECONDS", 900, 1, 7 * 24 * 3600),
|
||||
janitor_interval_seconds=_float("JANITOR_INTERVAL_SECONDS", 60.0, 0.0, 3600.0),
|
||||
fault_points=faults,
|
||||
)
|
||||
|
||||
@@ -59,6 +59,12 @@ class PayloadTooLarge(ApiError):
|
||||
default_message = "Тело запроса слишком большое"
|
||||
|
||||
|
||||
class UnsupportedMediaType(ApiError):
|
||||
status = 415
|
||||
code = "unsupported_media_type"
|
||||
default_message = "Неподдерживаемый тип содержимого"
|
||||
|
||||
|
||||
class Unavailable(ApiError):
|
||||
status = 503
|
||||
code = "unavailable"
|
||||
|
||||
@@ -0,0 +1,317 @@
|
||||
"""Content checks of uploaded images (lab 4): what the bytes ARE, not what the client says.
|
||||
|
||||
Two stages, both with bounded memory and both BEFORE anything is stored:
|
||||
|
||||
1. `sniff(head)` — runs on the first bytes while the body is still arriving. Recognises
|
||||
PNG/JPEG by the signature and reads the dimensions from the header (PNG IHDR, JPEG SOFn).
|
||||
A foreign format is rejected (415) and a picture with more than MAX_IMAGE_PIXELS pixels
|
||||
(413) before the rest of the body is read and long before any pixel is decoded.
|
||||
2. `verify(data, info)` — full integrity check of the complete file, run in a worker thread:
|
||||
* PNG: our own streaming validator. Every chunk CRC is checked, the IDAT zlib stream is
|
||||
inflated in 1 MiB steps and DISCARDED (no raster in memory), its size must be exactly
|
||||
height x (1 + row bytes) (Adam7 per pass), every row filter byte must be 0..4, IEND
|
||||
must close the file. A truncated file, a broken IDAT or extra/missing pixel data fail.
|
||||
Pillow is not used for PNG: it would allocate the whole raster (40 Mpx RGBA = 160 MB).
|
||||
* JPEG: libjpeg-turbo `djpeg -strict` decodes the whole entropy-coded data at 1/8 scale
|
||||
(DCT scaling: the output raster is 64 times smaller) in a child process; any decoder
|
||||
warning is fatal, so a truncated file or a file without scan data after SOS fails.
|
||||
|
||||
Every problem of the CONTENT is an ImageError carrying the HTTP status (415 or 413);
|
||||
nothing here may turn user input into a 500.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import zlib
|
||||
from dataclasses import dataclass
|
||||
|
||||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
JPEG_SOI = b"\xff\xd8\xff"
|
||||
|
||||
PNG = "image/png"
|
||||
JPEG = "image/jpeg"
|
||||
|
||||
# Bits per pixel channel count by PNG colour type, and the bit depths allowed for it.
|
||||
PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
|
||||
PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}}
|
||||
# Adam7 passes: (x start, y start, x step, y step).
|
||||
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
||||
# JPEG start-of-frame markers that carry the frame size (not DHT C4, JPG C8, DAC CC).
|
||||
JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF}
|
||||
# Standalone markers without a length field.
|
||||
JPEG_STANDALONE = {0x01, *range(0xD0, 0xD8)}
|
||||
|
||||
DJPEG = shutil.which("djpeg") or "djpeg" # libjpeg-turbo-progs, installed in the Dockerfile
|
||||
INFLATE_STEP = 1 << 20 # bytes of raw PNG data produced per zlib call
|
||||
MAX_DIMENSION = 2**31 - 1
|
||||
|
||||
|
||||
class ImageError(Exception):
|
||||
"""The uploaded bytes are not an acceptable image. `status`: 415 (format/corrupt) or 413."""
|
||||
|
||||
def __init__(self, status: int, message: str):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.message = message
|
||||
|
||||
|
||||
def unsupported(message: str = "Файл не является корректным изображением PNG или JPEG") -> ImageError:
|
||||
return ImageError(415, message)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ImageInfo:
|
||||
content_type: str
|
||||
width: int
|
||||
height: int
|
||||
|
||||
|
||||
def detect(head: bytes) -> str | None:
|
||||
"""Format by signature; None while fewer bytes than a signature have arrived."""
|
||||
if head[:8] == PNG_SIGNATURE:
|
||||
return PNG
|
||||
if head[:3] == JPEG_SOI:
|
||||
return JPEG
|
||||
probe = min(len(head), 8)
|
||||
if PNG_SIGNATURE.startswith(head[:probe]) or JPEG_SOI.startswith(head[:min(probe, 3)]):
|
||||
return None
|
||||
raise unsupported()
|
||||
|
||||
|
||||
def sniff(head: bytes, max_pixels: int) -> ImageInfo | None:
|
||||
"""Format and size from the beginning of the file, or None if more bytes are needed.
|
||||
|
||||
Raises ImageError: 415 for a foreign/broken header, 413 for too many pixels."""
|
||||
kind = detect(head)
|
||||
if kind is None:
|
||||
return None
|
||||
info = _png_header(head) if kind == PNG else _jpeg_header(head)
|
||||
if info is None:
|
||||
return None
|
||||
if info.width * info.height > max_pixels:
|
||||
raise ImageError(413, f"Изображение больше {max_pixels} пикселей")
|
||||
return info
|
||||
|
||||
|
||||
def _png_header(head: bytes) -> ImageInfo | None:
|
||||
if len(head) < 33:
|
||||
return None
|
||||
length, ctype = struct.unpack(">I4s", head[8:16])
|
||||
if ctype != b"IHDR" or length != 13:
|
||||
raise unsupported()
|
||||
if zlib.crc32(head[12:29]) != struct.unpack(">I", head[29:33])[0]:
|
||||
raise unsupported()
|
||||
width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", head[16:29])
|
||||
if not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION):
|
||||
raise unsupported()
|
||||
if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1:
|
||||
raise unsupported()
|
||||
return ImageInfo(PNG, width, height)
|
||||
|
||||
|
||||
def _jpeg_header(head: bytes) -> ImageInfo | None:
|
||||
"""Walk the marker segments up to the first SOFn. None = the SOF has not arrived yet."""
|
||||
i = 2
|
||||
n = len(head)
|
||||
while True:
|
||||
if i >= n:
|
||||
return None
|
||||
if head[i] != 0xFF:
|
||||
raise unsupported()
|
||||
while i < n and head[i] == 0xFF: # fill bytes
|
||||
i += 1
|
||||
if i >= n:
|
||||
return None
|
||||
marker = head[i]
|
||||
i += 1
|
||||
if marker in JPEG_STANDALONE:
|
||||
continue
|
||||
if marker in (0xD8, 0xD9, 0xDA, 0x00):
|
||||
# Second SOI, EOI or scan data before any frame header: not a usable JPEG.
|
||||
raise unsupported()
|
||||
if i + 2 > n:
|
||||
return None
|
||||
length = struct.unpack(">H", head[i:i + 2])[0]
|
||||
if length < 2:
|
||||
raise unsupported()
|
||||
if marker in JPEG_SOF:
|
||||
if i + 7 > n:
|
||||
return None
|
||||
if length < 8:
|
||||
raise unsupported()
|
||||
height, width = struct.unpack(">HH", head[i + 3:i + 7])
|
||||
if width == 0 or height == 0: # height 0 = DNL-defined height: not supported
|
||||
raise unsupported()
|
||||
return ImageInfo(JPEG, width, height)
|
||||
i += length
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ full verification
|
||||
async def verify(data: bytes | bytearray | memoryview, info: ImageInfo, *, timeout: float = 10.0,
|
||||
jpeg_max_memory_mb: int = 256) -> None:
|
||||
"""Complete integrity check of the whole file. PNG: our validator in a worker thread (zlib
|
||||
releases the GIL); JPEG: a strict libjpeg decoder in a child process."""
|
||||
if info.content_type == PNG:
|
||||
await asyncio.wait_for(asyncio.to_thread(verify_png, data), timeout)
|
||||
else:
|
||||
await verify_jpeg(data, timeout, jpeg_max_memory_mb)
|
||||
|
||||
|
||||
def _png_segments(width: int, height: int, depth: int, color: int, interlace: int) -> list[tuple[int, int]]:
|
||||
"""(byte count, row stride) of every pass of the raw (filtered) image data."""
|
||||
bpp = PNG_CHANNELS[color] * depth
|
||||
passes = ADAM7 if interlace else ((0, 0, 1, 1),)
|
||||
segments = []
|
||||
for xs, ys, dx, dy in passes:
|
||||
pw = (width - xs + dx - 1) // dx if width > xs else 0
|
||||
ph = (height - ys + dy - 1) // dy if height > ys else 0
|
||||
if pw and ph:
|
||||
stride = 1 + (pw * bpp + 7) // 8
|
||||
segments.append((ph * stride, stride))
|
||||
return segments
|
||||
|
||||
|
||||
class _RawChecker:
|
||||
"""Counts inflated bytes against the expected size and checks the filter byte of every row
|
||||
using C-speed slicing (`piece[first::stride]`), so even 40 000 000 one-pixel rows are fast."""
|
||||
|
||||
VALID_FILTERS = bytes(range(5))
|
||||
|
||||
def __init__(self, segments: list[tuple[int, int]]):
|
||||
self.segments = segments
|
||||
self.index = 0 # current pass
|
||||
self.offset = 0 # bytes already seen inside the current pass
|
||||
self.expected = sum(size for size, _ in segments)
|
||||
self.total = 0
|
||||
|
||||
def feed(self, piece: bytes) -> None:
|
||||
self.total += len(piece)
|
||||
if self.total > self.expected:
|
||||
raise unsupported()
|
||||
pos = 0
|
||||
while pos < len(piece):
|
||||
size, stride = self.segments[self.index]
|
||||
take = min(size - self.offset, len(piece) - pos)
|
||||
part = piece[pos:pos + take]
|
||||
first = (-self.offset) % stride
|
||||
if part[first::stride].translate(None, self.VALID_FILTERS):
|
||||
raise unsupported()
|
||||
self.offset += take
|
||||
pos += take
|
||||
if self.offset == size:
|
||||
self.index += 1
|
||||
self.offset = 0
|
||||
|
||||
|
||||
def verify_png(data: bytes | bytearray | memoryview) -> None:
|
||||
view = memoryview(data)
|
||||
if bytes(view[:8]) != PNG_SIGNATURE:
|
||||
raise unsupported()
|
||||
pos = 8
|
||||
n = len(view)
|
||||
checker: _RawChecker | None = None
|
||||
inflater = zlib.decompressobj()
|
||||
color = None
|
||||
seen_plte = seen_idat = idat_done = False
|
||||
first = True
|
||||
while True:
|
||||
if pos + 12 > n:
|
||||
raise unsupported() # truncated: no room for another chunk header + CRC
|
||||
length, ctype = struct.unpack(">I4s", view[pos:pos + 8])
|
||||
if length > n - pos - 12:
|
||||
raise unsupported()
|
||||
body = view[pos + 8:pos + 8 + length]
|
||||
crc = struct.unpack(">I", view[pos + 8 + length:pos + 12 + length])[0]
|
||||
if zlib.crc32(body, zlib.crc32(ctype)) != crc:
|
||||
raise unsupported()
|
||||
pos += 12 + length
|
||||
if not all(65 <= c <= 90 or 97 <= c <= 122 for c in ctype):
|
||||
raise unsupported()
|
||||
if first:
|
||||
if ctype != b"IHDR" or length != 13:
|
||||
raise unsupported()
|
||||
width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", body)
|
||||
if (not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION) or color not in PNG_DEPTHS
|
||||
or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1):
|
||||
raise unsupported()
|
||||
checker = _RawChecker(_png_segments(width, height, depth, color, interlace))
|
||||
first = False
|
||||
continue
|
||||
assert checker is not None
|
||||
if ctype == b"IHDR":
|
||||
raise unsupported()
|
||||
if ctype == b"PLTE":
|
||||
if seen_plte or seen_idat or color in (0, 4) or length % 3 or not 3 <= length <= 768:
|
||||
raise unsupported()
|
||||
seen_plte = True
|
||||
elif ctype == b"IDAT":
|
||||
if idat_done:
|
||||
raise unsupported() # IDAT chunks must be consecutive
|
||||
if color == 3 and not seen_plte:
|
||||
raise unsupported()
|
||||
seen_idat = True
|
||||
_inflate(inflater, body, checker)
|
||||
elif ctype == b"IEND":
|
||||
if length != 0 or not seen_idat:
|
||||
raise unsupported()
|
||||
if pos != n:
|
||||
raise unsupported() # bytes after IEND (polyglot files) are not accepted
|
||||
break
|
||||
elif not 97 <= ctype[0] <= 122:
|
||||
raise unsupported() # unknown CRITICAL chunk: a decoder could not show the image
|
||||
if ctype != b"IDAT" and seen_idat:
|
||||
idat_done = True
|
||||
# The zlib stream must end exactly with the last pixel row, without trailing data.
|
||||
if not inflater.eof or inflater.unused_data or checker.total != checker.expected:
|
||||
raise unsupported()
|
||||
|
||||
|
||||
def _inflate(inflater, body: memoryview, checker: _RawChecker) -> None:
|
||||
if inflater.eof:
|
||||
if len(body):
|
||||
raise unsupported() # data after the end of the zlib stream
|
||||
return
|
||||
pending: bytes | memoryview = body
|
||||
try:
|
||||
while pending:
|
||||
piece = inflater.decompress(pending, INFLATE_STEP)
|
||||
checker.feed(piece)
|
||||
pending = inflater.unconsumed_tail
|
||||
if inflater.eof:
|
||||
if pending or inflater.unused_data:
|
||||
raise unsupported()
|
||||
break
|
||||
except zlib.error:
|
||||
raise unsupported() from None
|
||||
|
||||
|
||||
async def verify_jpeg(data: bytes | bytearray | memoryview, timeout: float, max_memory_mb: int) -> None:
|
||||
"""Decode the WHOLE file with the reference decoder (libjpeg-turbo `djpeg`) in strict mode.
|
||||
|
||||
`-strict` turns every libjpeg warning into a failure: "premature end of data segment"
|
||||
(scan data missing or cut before EOI), "premature end of JPEG file", corrupt Huffman data.
|
||||
Pillow cannot be used for this: it ignores libjpeg warnings and returns a grey picture for
|
||||
a JPEG whose scan data is missing. `-scale 1/8` decodes every block but writes a 64x
|
||||
smaller raster to /dev/null; `-maxmemory` caps libjpeg's working memory (coefficient
|
||||
buffers of progressive files); the separate process is killed at the deadline.
|
||||
"""
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
DJPEG, "-strict", "-scale", "1/8", "-maxmemory", str(max_memory_mb * 1024), "-outfile", os.devnull,
|
||||
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE)
|
||||
stderr = b""
|
||||
try:
|
||||
_, stderr = await asyncio.wait_for(process.communicate(bytes(data)), timeout)
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
process.kill()
|
||||
await process.wait()
|
||||
raise
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
await process.wait() # djpeg stopped reading: the file is already known to be broken
|
||||
if process.returncode != 0:
|
||||
if b"Insufficient memory" in (stderr or b""):
|
||||
# A valid but extremely memory-hungry file (progressive, 4:4:4, ~40 Mpx): a limit, not corruption.
|
||||
raise ImageError(413, "Изображение слишком велико для обработки")
|
||||
raise unsupported()
|
||||
+71
-25
@@ -62,6 +62,7 @@ class Message:
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: dt.datetime
|
||||
attachment_ids: tuple[str, ...] = ()
|
||||
|
||||
@property
|
||||
def key(self) -> SortKey:
|
||||
@@ -69,7 +70,7 @@ class Message:
|
||||
|
||||
def to_json(self) -> dict:
|
||||
return {"id": self.id, "chat_id": self.chat_id, "sender_id": self.sender_id, "text": self.text,
|
||||
"created_at": rfc3339(self.created_at)}
|
||||
"created_at": rfc3339(self.created_at), "attachment_ids": list(self.attachment_ids)}
|
||||
|
||||
|
||||
def _user(row) -> User:
|
||||
@@ -205,7 +206,7 @@ class PostResult:
|
||||
|
||||
|
||||
async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID, user_id: str, chat_id: str,
|
||||
text: str) -> dt.datetime:
|
||||
text: str, attachment_ids: list[str]) -> dt.datetime:
|
||||
"""Insert with a per-chat strictly increasing timestamp.
|
||||
|
||||
`UPDATE chats ... RETURNING` locks the chat row until COMMIT, so writers of one
|
||||
@@ -220,9 +221,39 @@ async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID,
|
||||
await conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)",
|
||||
(message_id, chat_id, user_id, _b(text), created_at))
|
||||
if attachment_ids:
|
||||
await conn.execute(
|
||||
"INSERT INTO message_attachments (message_id, position, attachment_id) "
|
||||
"SELECT %s, a.ord - 1, a.id FROM unnest(%s::uuid[]) WITH ORDINALITY AS a(id, ord)",
|
||||
(message_id, attachment_ids))
|
||||
return created_at
|
||||
|
||||
|
||||
# Attachments a message may reference: ready, of THIS chat (else 404), not quarantined (else 409).
|
||||
# FOR SHARE keeps a concurrent quarantine (reconciler) from slipping in before COMMIT.
|
||||
CHECK_ATTACHMENTS = (
|
||||
"SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL) FROM ("
|
||||
"SELECT lost_at FROM attachments WHERE id = ANY(%s::uuid[]) AND chat_id = %s AND status = 'ready' "
|
||||
"FOR SHARE) a")
|
||||
|
||||
|
||||
async def _check_attachments(conn: psycopg.AsyncConnection, chat_id: str, attachment_ids: list[str]) -> str:
|
||||
if not attachment_ids:
|
||||
return "ok"
|
||||
cur = await conn.execute(CHECK_ATTACHMENTS, (attachment_ids, chat_id))
|
||||
found, usable = await cur.fetchone()
|
||||
if found != len(attachment_ids):
|
||||
return "attachment_not_found"
|
||||
return "ok" if usable == found else "attachment_conflict"
|
||||
|
||||
|
||||
def _attachment_error(status: str) -> None:
|
||||
if status == "attachment_not_found":
|
||||
raise NotFound("Вложение не найдено")
|
||||
if status == "attachment_conflict":
|
||||
raise Conflict("Вложение недоступно и не может быть прикреплено")
|
||||
|
||||
|
||||
# Claim the key. A row of a concurrent transaction that is not committed yet makes this
|
||||
# INSERT wait on the unique index until that transaction ends: after COMMIT it is a conflict
|
||||
# (the key is taken), after ROLLBACK the INSERT goes through. A key older than the TTL is
|
||||
@@ -235,60 +266,72 @@ CLAIM_KEY = (
|
||||
"WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => %s) "
|
||||
"RETURNING 1")
|
||||
STORED_RESULT = (
|
||||
"SELECT k.request_hash, m.id, m.text, m.created_at FROM idempotency_keys k "
|
||||
"SELECT k.request_hash, m.id, m.text, m.created_at, "
|
||||
"ARRAY(SELECT ma.attachment_id FROM message_attachments ma WHERE ma.message_id = m.id ORDER BY ma.position) "
|
||||
"FROM idempotency_keys k "
|
||||
"JOIN messages m ON m.id = k.message_id "
|
||||
"WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s")
|
||||
|
||||
|
||||
async def post_message(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes, text: str,
|
||||
ttl_seconds: int) -> PostResult:
|
||||
ttl_seconds: int, attachment_ids: list[str] = ()) -> PostResult:
|
||||
"""Idempotent send: the key, the message and the stored result commit in ONE transaction.
|
||||
|
||||
* first request -> key row + message + chat clock, all or nothing (no "pending" key is
|
||||
ever left behind: a crash or 5xx rolls everything back and the client may retry);
|
||||
* same key, same body (any replica, also concurrently) -> the stored Message, 201, replayed;
|
||||
* same key, other body -> 409; the membership check comes first, so outsiders always get 404.
|
||||
* same key, other body -> 409; the membership check comes first, so outsiders always get 404;
|
||||
* attachments (lab 4) are checked before the key is claimed (same order as post_message_v2).
|
||||
"""
|
||||
message_id = uuid.uuid4()
|
||||
attachment_ids = list(attachment_ids)
|
||||
|
||||
async def work(conn: psycopg.AsyncConnection) -> PostResult:
|
||||
await _check_member(conn, user_id, chat_id)
|
||||
cur = await conn.execute(CLAIM_KEY, (user_id, chat_id, key, request_hash, message_id, ttl_seconds))
|
||||
if await cur.fetchone() is not None:
|
||||
created_at = await _insert_message(conn, message_id, user_id, chat_id, text)
|
||||
return PostResult(Message(str(message_id), chat_id, user_id, text, created_at), False)
|
||||
check = await _check_attachments(conn, chat_id, attachment_ids)
|
||||
if check == "ok":
|
||||
cur = await conn.execute(CLAIM_KEY, (user_id, chat_id, key, request_hash, message_id, ttl_seconds))
|
||||
if await cur.fetchone() is not None:
|
||||
created_at = await _insert_message(conn, message_id, user_id, chat_id, text, attachment_ids)
|
||||
return PostResult(Message(str(message_id), chat_id, user_id, text, created_at,
|
||||
tuple(attachment_ids)), False)
|
||||
cur = await conn.execute(STORED_RESULT, (user_id, chat_id, key))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
_attachment_error(check)
|
||||
# The key row vanished between the two statements (expired-key cleanup): retryable.
|
||||
raise Unavailable("Повторите запрос с тем же Idempotency-Key")
|
||||
if bytes(row[0]) != request_hash:
|
||||
raise Conflict("Idempotency-Key уже использован с другим сообщением")
|
||||
return PostResult(Message(str(row[1]), chat_id, user_id, _s(row[2]), row[3]), True)
|
||||
return PostResult(Message(str(row[1]), chat_id, user_id, _s(row[2]), row[3],
|
||||
tuple(str(a) for a in row[4])), True)
|
||||
return await db.run(work)
|
||||
|
||||
|
||||
POST_MESSAGE_CALL = "SELECT o_status, o_id, o_text, o_created_at FROM post_message(%s, %s, %s, %s, %s, %s, %s)"
|
||||
POST_MESSAGE_CALL = ("SELECT o_status, o_id, o_text, o_created_at, o_attachments "
|
||||
"FROM post_message_v2(%s, %s, %s, %s, %s, %s, %s, %s::uuid[])")
|
||||
|
||||
|
||||
async def post_message_single_call(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes,
|
||||
text: str, ttl_seconds: int) -> PostResult:
|
||||
"""Same contract as post_message, ONE round trip: SQL function post_message (migration 0003)
|
||||
in autocommit mode, i.e. the call is the whole transaction. Track 1 improvement."""
|
||||
text: str, ttl_seconds: int, attachment_ids: list[str] = ()) -> PostResult:
|
||||
"""Same contract as post_message, ONE round trip: SQL function post_message_v2 (migration 0004 =
|
||||
0003 + attachment_ids) in autocommit mode, i.e. the call is the whole transaction."""
|
||||
message_id = uuid.uuid4()
|
||||
|
||||
async def work(conn: psycopg.AsyncConnection) -> tuple:
|
||||
cur = await conn.execute(POST_MESSAGE_CALL,
|
||||
(user_id, chat_id, key, request_hash, message_id, _b(text), ttl_seconds))
|
||||
cur = await conn.execute(POST_MESSAGE_CALL, (user_id, chat_id, key, request_hash, message_id, _b(text),
|
||||
ttl_seconds, list(attachment_ids)))
|
||||
return await cur.fetchone()
|
||||
status, stored_id, stored_text, created_at = await db.run(work, transaction=False)
|
||||
status, stored_id, stored_text, created_at, stored_attachments = await db.run(work, transaction=False)
|
||||
if status == "not_found":
|
||||
raise NotFound("Чат не найден")
|
||||
_attachment_error(status)
|
||||
if status == "conflict":
|
||||
raise Conflict("Idempotency-Key уже использован с другим сообщением")
|
||||
if status == "retry":
|
||||
raise Unavailable("Повторите запрос с тем же Idempotency-Key")
|
||||
return PostResult(Message(str(stored_id), chat_id, user_id, _s(stored_text), created_at), status == "replayed")
|
||||
return PostResult(Message(str(stored_id), chat_id, user_id, _s(stored_text), created_at,
|
||||
tuple(str(a) for a in stored_attachments)), status == "replayed")
|
||||
|
||||
|
||||
async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 5000) -> int:
|
||||
@@ -308,12 +351,14 @@ async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 50
|
||||
|
||||
# Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR
|
||||
# would push the cursor into a Filter and late pages would scan the whole prefix.
|
||||
HISTORY_FIRST = (
|
||||
"SELECT id, sender_id, text, created_at FROM messages "
|
||||
"WHERE chat_id = %s ORDER BY created_at, id LIMIT %s")
|
||||
HISTORY_AFTER = (
|
||||
"SELECT id, sender_id, text, created_at FROM messages "
|
||||
"WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s")
|
||||
# attachment_ids: a correlated sub-select per returned row (at most limit + 1 = 101 lookups on the
|
||||
# primary key of message_attachments), so the keyset range scan itself stays unchanged.
|
||||
HISTORY_COLUMNS = ("SELECT m.id, m.sender_id, m.text, m.created_at, "
|
||||
"ARRAY(SELECT ma.attachment_id FROM message_attachments ma "
|
||||
"WHERE ma.message_id = m.id ORDER BY ma.position) FROM messages m ")
|
||||
HISTORY_FIRST = HISTORY_COLUMNS + "WHERE m.chat_id = %s ORDER BY m.created_at, m.id LIMIT %s"
|
||||
HISTORY_AFTER = (HISTORY_COLUMNS +
|
||||
"WHERE m.chat_id = %s AND (m.created_at, m.id) > (%s, %s) ORDER BY m.created_at, m.id LIMIT %s")
|
||||
|
||||
|
||||
async def list_messages(db: Database, chat_id: str, after: SortKey | None, limit: int) -> tuple[list[Message], bool]:
|
||||
@@ -324,5 +369,6 @@ async def list_messages(db: Database, chat_id: str, after: SortKey | None, limit
|
||||
cur = await conn.execute(HISTORY_AFTER, (chat_id, from_us(after[0]), after[1], limit + 1))
|
||||
return await cur.fetchall()
|
||||
rows = await db.run(work)
|
||||
items = [Message(str(r[0]), chat_id, str(r[1]), _s(r[2]), r[3]) for r in rows[:limit]]
|
||||
items = [Message(str(r[0]), chat_id, str(r[1]), _s(r[2]), r[3], tuple(str(a) for a in r[4]))
|
||||
for r in rows[:limit]]
|
||||
return items, len(rows) > limit
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
"""One-shot S3 bootstrap: `python -m messenger.s3_bootstrap` (compose service `s3-init`).
|
||||
|
||||
Idempotent, safe to run on every `docker compose up`:
|
||||
|
||||
1. waits until the S3 endpoint answers;
|
||||
2. creates the bucket with the ADMIN key pair if it does not exist yet (an existing bucket and
|
||||
its objects are left untouched);
|
||||
3. writes the sentinel object `health/sentinel` that the API's readiness probe HEADs;
|
||||
4. proves the access model and FAILS the stand (exit 1) if it is wrong:
|
||||
* anonymous (unsigned) GET of an object and anonymous LIST of the bucket are refused;
|
||||
* the API key pair can read/write objects of ITS bucket, but can neither list it, nor create
|
||||
a bucket, nor see or read another bucket (a canary bucket created here for this check);
|
||||
* the reconciler key pair can list the bucket but cannot create one.
|
||||
Only file names and verdicts are printed, never key values.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from .config import ConfigError, Settings, _secret, s3_settings
|
||||
from .errors import Unavailable
|
||||
from .storage import SENTINEL_KEY, Storage
|
||||
|
||||
|
||||
def say(text: str) -> None:
|
||||
print(f"s3-bootstrap: {text}", flush=True)
|
||||
|
||||
|
||||
def anonymous_status(url: str) -> int:
|
||||
"""HTTP status of an UNSIGNED request (what anyone on the network could do)."""
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=3) as response:
|
||||
return response.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code
|
||||
|
||||
|
||||
async def error_code(coro) -> str:
|
||||
"""S3 error code of a call that is expected to be refused ('' if it succeeded)."""
|
||||
try:
|
||||
await coro
|
||||
return ""
|
||||
except ClientError as exc:
|
||||
return str(exc.response.get("Error", {}).get("Code", "error"))
|
||||
|
||||
|
||||
def settings_with(keys: tuple[str, str]) -> Settings:
|
||||
base = s3_settings(credentials_required=False)
|
||||
base.update(s3_access_key_id=keys[0], s3_secret_access_key=keys[1])
|
||||
return Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024, **base)
|
||||
|
||||
|
||||
async def run() -> int:
|
||||
admin = (_secret("S3_ADMIN_ACCESS_KEY_ID", True), _secret("S3_ADMIN_SECRET_ACCESS_KEY", True))
|
||||
api = (_secret("S3_ACCESS_KEY_ID", True), _secret("S3_SECRET_ACCESS_KEY", True))
|
||||
reconciler = (_secret("S3_RECONCILER_ACCESS_KEY_ID", True), _secret("S3_RECONCILER_SECRET_ACCESS_KEY", True))
|
||||
settings = settings_with(admin)
|
||||
bucket = settings.s3_bucket
|
||||
admin_s3, api_s3, rec_s3 = Storage(settings), Storage(settings_with(api)), Storage(settings_with(reconciler))
|
||||
for s in (admin_s3, api_s3, rec_s3):
|
||||
await s.open()
|
||||
try:
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
await admin_s3._call("list_buckets", admin_s3.client.list_buckets())
|
||||
break
|
||||
except Unavailable:
|
||||
if time.monotonic() > deadline:
|
||||
say("S3 endpoint does not answer within 60 s")
|
||||
return 1
|
||||
await asyncio.sleep(1)
|
||||
|
||||
code = await error_code(admin_s3.client.head_bucket(Bucket=bucket))
|
||||
if code == "":
|
||||
say(f"bucket {bucket}: exists, kept as is")
|
||||
elif code in ("404", "NoSuchBucket", "NotFound"):
|
||||
code = await error_code(admin_s3.client.create_bucket(Bucket=bucket))
|
||||
if code not in ("", "BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
|
||||
say(f"bucket {bucket}: create failed ({code})")
|
||||
return 1
|
||||
say(f"bucket {bucket}: created")
|
||||
else:
|
||||
say(f"bucket {bucket}: head failed ({code})")
|
||||
return 1
|
||||
await admin_s3.put(SENTINEL_KEY, b"ok\n", "text/plain", {})
|
||||
say(f"sentinel {SENTINEL_KEY}: written")
|
||||
canary = bucket + "-canary"
|
||||
if await error_code(admin_s3.client.head_bucket(Bucket=canary)):
|
||||
await admin_s3.client.create_bucket(Bucket=canary)
|
||||
await admin_s3.client.put_object(Bucket=canary, Key="canary", Body=b"secret of another bucket")
|
||||
|
||||
failures = []
|
||||
base = settings.s3_endpoint.rstrip("/")
|
||||
for what, url in (("anonymous GET object", f"{base}/{bucket}/{SENTINEL_KEY}"),
|
||||
("anonymous LIST bucket", f"{base}/{bucket}?list-type=2")):
|
||||
status = anonymous_status(url)
|
||||
verdict = "refused" if status in (401, 403) else "ALLOWED"
|
||||
say(f"{what}: HTTP {status} -> {verdict}")
|
||||
if verdict != "refused":
|
||||
failures.append(what)
|
||||
|
||||
probe = "health/bootstrap-probe"
|
||||
checks = [
|
||||
("api: read object", await error_code(api_s3.client.head_object(Bucket=bucket, Key=SENTINEL_KEY)), True),
|
||||
("api: write object", await error_code(api_s3.client.put_object(Bucket=bucket, Key=probe, Body=b"p")), True),
|
||||
("api: delete object", await error_code(api_s3.client.delete_object(Bucket=bucket, Key=probe)), True),
|
||||
("api: list bucket", await error_code(api_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), False),
|
||||
("api: create bucket", await error_code(api_s3.client.create_bucket(Bucket=bucket + "-x")), False),
|
||||
("api: read object of another bucket",
|
||||
await error_code(api_s3.client.head_object(Bucket=canary, Key="canary")), False),
|
||||
("reconciler: list bucket", await error_code(rec_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), True),
|
||||
("reconciler: create bucket", await error_code(rec_s3.client.create_bucket(Bucket=bucket + "-x")), False),
|
||||
]
|
||||
for what, code, allowed in checks:
|
||||
ok = (code == "") == allowed
|
||||
say(f"{what}: {'allowed' if code == '' else 'refused (' + code + ')'} -> {'OK' if ok else 'WRONG'}")
|
||||
if not ok:
|
||||
failures.append(what)
|
||||
visible = [b["Name"] for b in (await api_s3.client.list_buckets()).get("Buckets", [])]
|
||||
say(f"api: buckets visible in ListBuckets: {visible} -> {'OK' if canary not in visible else 'WRONG'}")
|
||||
if canary in visible:
|
||||
failures.append("api sees another bucket")
|
||||
if failures:
|
||||
say("access model is WRONG: " + ", ".join(failures))
|
||||
return 1
|
||||
say("done: private bucket, least-privilege keys verified")
|
||||
return 0
|
||||
finally:
|
||||
for s in (admin_s3, api_s3, rec_s3):
|
||||
await s.close()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
return asyncio.run(run())
|
||||
except ConfigError as exc:
|
||||
print(f"configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.environ.setdefault("PYTHONUNBUFFERED", "1")
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Private S3 bucket (SeaweedFS in compose) through aiobotocore, with a hard deadline per call.
|
||||
|
||||
* Credentials: the API's OWN key pair (S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE ->
|
||||
Docker secrets). SeaweedFS grants it only Read + Write on this one bucket: no List, no
|
||||
Admin (cannot create/delete buckets or read other buckets). The admin pair is given only
|
||||
to the S3 server and the one-shot bootstrap (infra/seaweedfs, messenger.s3_bootstrap).
|
||||
* Deadlines: botocore connect/read timeouts AND an asyncio deadline around every call, so a
|
||||
paused or unreachable S3 costs at most S3_TIMEOUT_SECONDS, then 503 `unavailable`. Unlike a
|
||||
libpq query (db.py), an aiohttp request can be cancelled safely, so the call is cancelled.
|
||||
* No retries inside botocore (max_attempts=1): the request handler decides; an upload is not
|
||||
retried behind the client's back (contract: a retry may create another object).
|
||||
* Checksums: Content-MD5 is sent with every PUT, so S3 rejects bytes damaged on the way.
|
||||
botocore's newer default (aws-chunked CRC32 trailers) is switched off ("when_required"):
|
||||
plain, widely supported S3 requests.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import contextlib
|
||||
import hashlib
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
|
||||
from aiobotocore.config import AioConfig
|
||||
from aiobotocore.session import get_session
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .config import Settings
|
||||
from .errors import Unavailable
|
||||
|
||||
log = logging.getLogger("messenger.storage")
|
||||
|
||||
MISSING_CODES = {"NoSuchKey", "404", "NotFound"}
|
||||
|
||||
|
||||
class ObjectMissing(Exception):
|
||||
"""The key does not exist in the bucket (definitive answer of S3, not an outage)."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObjectHead:
|
||||
size: int
|
||||
content_type: str
|
||||
metadata: dict[str, str]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ListedObject:
|
||||
key: str
|
||||
size: int
|
||||
last_modified_ts: float # POSIX seconds
|
||||
|
||||
|
||||
class Storage:
|
||||
def __init__(self, settings: Settings, *, access_key: str | None = None, secret_key: str | None = None):
|
||||
self.bucket = settings.s3_bucket
|
||||
self.timeout = settings.s3_timeout_seconds
|
||||
self._endpoint = settings.s3_endpoint
|
||||
self._region = settings.s3_region
|
||||
self._access_key = access_key or settings.s3_access_key_id
|
||||
self._secret_key = secret_key or settings.s3_secret_access_key
|
||||
self._config = AioConfig(
|
||||
connect_timeout=settings.s3_connect_timeout_seconds,
|
||||
read_timeout=settings.s3_timeout_seconds,
|
||||
retries={"total_max_attempts": 1, "mode": "standard"},
|
||||
s3={"addressing_style": "path"},
|
||||
signature_version="s3v4",
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
max_pool_connections=settings.s3_max_connections,
|
||||
)
|
||||
self._stack: contextlib.AsyncExitStack | None = None
|
||||
self._client = None
|
||||
|
||||
async def open(self) -> None:
|
||||
self._stack = contextlib.AsyncExitStack()
|
||||
session = get_session()
|
||||
self._client = await self._stack.enter_async_context(session.create_client(
|
||||
"s3", endpoint_url=self._endpoint, region_name=self._region,
|
||||
aws_access_key_id=self._access_key, aws_secret_access_key=self._secret_key, config=self._config))
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._stack is not None:
|
||||
await self._stack.aclose()
|
||||
self._stack = None
|
||||
|
||||
@property
|
||||
def client(self):
|
||||
assert self._client is not None, "Storage.open() was not called"
|
||||
return self._client
|
||||
|
||||
async def _call(self, what: str, coro, deadline: float | None = None):
|
||||
"""Await one S3 call under the deadline; outages become Unavailable (503)."""
|
||||
try:
|
||||
return await asyncio.wait_for(coro, deadline or self.timeout)
|
||||
except ClientError as exc:
|
||||
code = str(exc.response.get("Error", {}).get("Code", ""))
|
||||
if code in MISSING_CODES:
|
||||
raise ObjectMissing() from None
|
||||
log.warning("s3_error", extra={"fields": {"op": what, "code": code}})
|
||||
raise Unavailable("Хранилище файлов временно недоступно") from None
|
||||
except (asyncio.TimeoutError, BotoCoreError, OSError) as exc:
|
||||
log.warning("s3_unavailable", extra={"fields": {"op": what, "error": type(exc).__name__}})
|
||||
raise Unavailable("Хранилище файлов временно недоступно") from None
|
||||
|
||||
async def put(self, key: str, data: bytes | bytearray, content_type: str, metadata: dict[str, str]) -> None:
|
||||
md5 = base64.b64encode(hashlib.md5(data, usedforsecurity=False).digest()).decode("ascii")
|
||||
# Time budget grows with the size: 10 MiB over a local network takes well under a second.
|
||||
deadline = self.timeout + len(data) / (8 * 1024 * 1024)
|
||||
await self._call("put", self.client.put_object(
|
||||
Bucket=self.bucket, Key=key, Body=bytes(data), ContentType=content_type, ContentLength=len(data),
|
||||
ContentMD5=md5, Metadata=metadata), deadline)
|
||||
|
||||
async def head(self, key: str) -> ObjectHead:
|
||||
response = await self._call("head", self.client.head_object(Bucket=self.bucket, Key=key))
|
||||
return ObjectHead(int(response["ContentLength"]), response.get("ContentType", ""),
|
||||
dict(response.get("Metadata", {})))
|
||||
|
||||
async def open_stream(self, key: str) -> tuple[int, AsyncIterator[bytes]]:
|
||||
"""Start a GET: (Content-Length, async iterator of chunks). The headers arrive under the
|
||||
deadline; every chunk read afterwards has its own deadline (a stalled S3 aborts the
|
||||
download instead of hanging the connection)."""
|
||||
response = await self._call("get", self.client.get_object(Bucket=self.bucket, Key=key))
|
||||
body = response["Body"]
|
||||
size = int(response["ContentLength"])
|
||||
|
||||
async def chunks() -> AsyncIterator[bytes]:
|
||||
try:
|
||||
while True:
|
||||
chunk = await self._call("get_body", body.read(64 * 1024))
|
||||
if not chunk:
|
||||
return
|
||||
yield chunk
|
||||
finally:
|
||||
body.close()
|
||||
return size, chunks()
|
||||
|
||||
async def read_all(self, key: str, limit: int) -> bytes:
|
||||
"""Whole object (reconciler checksum verification); refuses objects above `limit`."""
|
||||
size, chunks = await self.open_stream(key)
|
||||
if size > limit:
|
||||
raise Unavailable("Объект больше допустимого размера")
|
||||
parts = [chunk async for chunk in chunks]
|
||||
return b"".join(parts)
|
||||
|
||||
async def delete(self, key: str) -> None:
|
||||
"""Idempotent: deleting a missing key is a success in S3."""
|
||||
await self._call("delete", self.client.delete_object(Bucket=self.bucket, Key=key))
|
||||
|
||||
async def list_page(self, prefix: str, start_after: str | None, page_size: int) -> tuple[list[ListedObject], bool]:
|
||||
"""One ListObjectsV2 page in key order (needs the List permission: reconciler only)."""
|
||||
params = {"Bucket": self.bucket, "Prefix": prefix, "MaxKeys": page_size}
|
||||
if start_after:
|
||||
params["StartAfter"] = start_after
|
||||
response = await self._call("list", self.client.list_objects_v2(**params))
|
||||
items = [ListedObject(o["Key"], int(o["Size"]), o["LastModified"].timestamp())
|
||||
for o in response.get("Contents", [])]
|
||||
return items, bool(response.get("IsTruncated"))
|
||||
|
||||
async def ping(self, timeout: float = 1.0) -> bool:
|
||||
"""Readiness: HEAD of the sentinel object written by the bootstrap. Proves that S3
|
||||
answers, the bucket exists and the API credentials are accepted (Read permission)."""
|
||||
try:
|
||||
await self._call("ping", self.client.head_object(Bucket=self.bucket, Key=SENTINEL_KEY), timeout)
|
||||
return True
|
||||
except (Unavailable, ObjectMissing):
|
||||
return False
|
||||
|
||||
|
||||
SENTINEL_KEY = "health/sentinel"
|
||||
OBJECT_PREFIX = "o/" # every attachment original lives under o/<random 128-bit hex>
|
||||
@@ -8,7 +8,9 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
from .errors import InvalidRequest
|
||||
|
||||
@@ -58,7 +60,7 @@ def check_fields(data: dict[str, Any], required: set[str], optional: frozenset[s
|
||||
raise InvalidRequest("Отсутствуют обязательные поля: " + ", ".join(sorted(missing)))
|
||||
|
||||
|
||||
def text_field(data: dict[str, Any], name: str, max_len: int, *, min_len: int = 1) -> str:
|
||||
def text_field(data: dict[str, Any], name: str, max_len: int, *, min_len: int = 1, allow_blank: bool = False) -> str:
|
||||
value = data[name]
|
||||
if not isinstance(value, str):
|
||||
raise InvalidRequest(f"Поле {name} должно быть строкой")
|
||||
@@ -68,7 +70,7 @@ def text_field(data: dict[str, Any], name: str, max_len: int, *, min_len: int =
|
||||
# len() of a Python str is the number of Unicode code points, as the contract requires.
|
||||
if not min_len <= len(value) <= max_len:
|
||||
raise InvalidRequest(f"Длина поля {name} должна быть от {min_len} до {max_len} символов")
|
||||
if value.strip() == "":
|
||||
if value.strip() == "" and not allow_blank:
|
||||
raise InvalidRequest(f"Поле {name} не может состоять только из пробелов")
|
||||
return value
|
||||
|
||||
@@ -113,8 +115,8 @@ def member_ids_field(data: dict[str, Any]) -> list[str]:
|
||||
|
||||
|
||||
def attachment_ids_field(data: dict[str, Any]) -> list[str]:
|
||||
"""Optional `attachment_ids` (lab 4+). Absent and [] are the same message (contracts/README.md);
|
||||
lab 3 has no attachments yet, so a non-empty list can only reference unknown objects."""
|
||||
"""Optional `attachment_ids` (lab 4+): absent and [] are the same message (contracts/README.md);
|
||||
null or any other non-array type is 400."""
|
||||
value = data.get("attachment_ids", [])
|
||||
if not isinstance(value, list):
|
||||
raise InvalidRequest("attachment_ids должен быть массивом")
|
||||
@@ -145,3 +147,53 @@ def parse_limit(raw: str | None) -> int:
|
||||
if LIMIT_RE.fullmatch(raw) is None or not 1 <= int(raw) <= MAX_LIMIT:
|
||||
raise InvalidRequest(f"limit должен быть целым числом от 1 до {MAX_LIMIT}")
|
||||
return int(raw)
|
||||
|
||||
|
||||
# Lab 4: the display name of an attachment. Never used as a path or an S3 key, but it is shown
|
||||
# to other users, so characters that change how a name LOOKS are refused: control characters
|
||||
# (C0, DEL, C1 such as U+0085), line/paragraph separators, unpaired surrogates and bidi controls
|
||||
# ("photo<U+202E>gnp.exe" would display as "photoexe.png").
|
||||
BIDI_CONTROLS = frozenset("\u061c\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069")
|
||||
FORBIDDEN_CATEGORIES = frozenset({"Cc", "Cs", "Zl", "Zp"})
|
||||
|
||||
|
||||
def query_items(raw: bytes) -> list[tuple[str, str]]:
|
||||
"""Strict query parsing: percent-encoded UTF-8 only (Starlette would replace bad bytes by U+FFFD)."""
|
||||
try:
|
||||
return parse_qsl(raw.decode("ascii"), keep_blank_values=True, encoding="utf-8", errors="strict")
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
raise InvalidRequest("Параметры запроса должны быть в UTF-8 с percent-encoding") from None
|
||||
|
||||
|
||||
def filename_param(query: list[tuple[str, str]]) -> str:
|
||||
value = single_query_param(query, "filename")
|
||||
if value is None:
|
||||
raise InvalidRequest("Параметр filename обязателен")
|
||||
if not 1 <= len(value) <= 255:
|
||||
raise InvalidRequest("Длина filename должна быть от 1 до 255 символов")
|
||||
if value.strip() == "" or value in {".", ".."}:
|
||||
raise InvalidRequest("filename не может состоять только из пробелов или точек")
|
||||
if "/" in value or "\\" in value:
|
||||
raise InvalidRequest("filename не может содержать / или \\")
|
||||
for char in value:
|
||||
if unicodedata.category(char) in FORBIDDEN_CATEGORIES or char in BIDI_CONTROLS:
|
||||
raise InvalidRequest("filename содержит управляющие символы")
|
||||
return value
|
||||
|
||||
|
||||
ATTACHMENT_VARIANTS = frozenset({"thumbnail", "crop", "optimized"}) # exist in the schema, not in lab 4
|
||||
|
||||
|
||||
def variant_param(query: list[tuple[str, str]]) -> str:
|
||||
"""`variant` of GET /attachments/{id}/content: only `original` exists in lab 4. A known variant
|
||||
name is a missing variant of a ready object (404); anything else is an invalid value (400)."""
|
||||
value = single_query_param(query, "variant")
|
||||
if value is None or value == "original":
|
||||
return "original"
|
||||
if value in ATTACHMENT_VARIANTS:
|
||||
raise NotFoundVariant()
|
||||
raise InvalidRequest("variant должен быть original")
|
||||
|
||||
|
||||
class NotFoundVariant(Exception):
|
||||
"""Raised for a variant that lab 4 does not produce; mapped to 404 by the handler."""
|
||||
|
||||
Reference in New Issue
Block a user