Files
mtusi2026lab6/src/attachments.py
T

183 lines
7.1 KiB
Python

"""Private S3 access shared by attachment routes and maintenance jobs."""
import asyncio
import hashlib
import logging
import os
import unicodedata
from functools import lru_cache
from pathlib import Path
from tempfile import SpooledTemporaryFile
from uuid import UUID, uuid4
from fastapi import HTTPException, Request
from minio import Minio
from minio.error import S3Error
from opentelemetry import trace
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from PIL import Image, UnidentifiedImageError
from urllib3 import PoolManager, Retry, Timeout
from .pipeline_jobs import queue_attachment, renew_upload, reserve_upload
from .store import connection, store, timestamp
BUCKET = "messenger-attachments"
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
logger = logging.getLogger(__name__)
tracer = trace.get_tracer(__name__)
trace_context = TraceContextTextMapPropagator()
def secret(name: str) -> str:
return Path(os.environ[f"{name}_FILE"]).read_text(encoding="utf-8").strip()
@lru_cache(maxsize=1)
def storage() -> Minio:
return Minio(
os.environ.get("S3_ENDPOINT", "s3:9000"),
access_key=secret("S3_ACCESS_KEY_ID"),
secret_key=secret("S3_SECRET_ACCESS_KEY"),
secure=False,
region="us-east-1",
http_client=PoolManager(timeout=Timeout(connect=2, read=10), retries=Retry(total=0)),
)
def storage_ready() -> None:
try:
storage().stat_object(BUCKET, "health/ready-probe")
except S3Error as exc:
if exc.code != "NoSuchKey":
raise HTTPException(503) from None
except Exception:
raise HTTPException(503) from None
def attachment_view(row: dict) -> dict:
return {
"id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]),
"filename": row["filename"], "content_type": row["content_type"],
"size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"],
"variants": [], "error_code": row["error_code"],
"created_at": timestamp(row["created_at"]), "updated_at": timestamp(row["updated_at"]),
}
def validate_filename(filename: str) -> None:
if not 1 <= len(filename) <= 255 or filename in {".", ".."}:
raise HTTPException(400)
if any(char in "/\\" or unicodedata.category(char) == "Cc" for char in filename):
raise HTTPException(400)
def validate_image(stream, declared_type: str) -> str:
try:
with Image.open(stream) as image:
actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format)
if actual_type is None or actual_type != declared_type:
raise HTTPException(415)
if image.width * image.height > MAX_PIXELS:
raise HTTPException(413)
image.verify()
stream.seek(0)
with Image.open(stream) as image:
image.load()
return actual_type
except Image.DecompressionBombError:
raise HTTPException(413) from None
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
raise HTTPException(415) from None
def verify_staging(client: Minio, key: str, size: int, sha256: str, content_type: str) -> None:
"""Confirm complete durable bytes; an S3 ETag is not a SHA-256 checksum."""
info = client.stat_object(BUCKET, key)
if info.size != size or info.content_type.split(";", 1)[0] != content_type:
raise ValueError("staging metadata mismatch")
obj = client.get_object(BUCKET, key)
try:
digest = hashlib.sha256()
received = 0
for chunk in obj.stream(64 * 1024):
received += len(chunk)
if received > size:
raise ValueError("staging size mismatch")
digest.update(chunk)
if received != size or digest.hexdigest() != sha256:
raise ValueError("staging checksum mismatch")
finally:
obj.close()
obj.release_conn()
def persist_attachment(chat_id: str, actor_id: str, filename: str, content_type: str, size: int, sha256: str, stream) -> dict:
attachment_id = uuid4()
staging_key = f"staging/{attachment_id}"
object_key = f"originals/{attachment_id}"
with connection() as conn:
reserve_upload(conn, attachment_id, staging_key)
try:
stream.seek(0)
client = storage()
with tracer.start_as_current_span("s3.stage_upload", record_exception=False, set_status_on_exception=False):
client.put_object(BUCKET, staging_key, stream, size, content_type=content_type)
with connection() as conn:
if not renew_upload(conn, attachment_id):
raise HTTPException(503)
with tracer.start_as_current_span("s3.verify_staging", record_exception=False, set_status_on_exception=False):
verify_staging(client, staging_key, size, sha256, content_type)
carrier = {}
trace_context.inject(carrier)
with connection() as conn:
with tracer.start_as_current_span("db.queue_attachment", record_exception=False,
set_status_on_exception=False):
row, _job_id = queue_attachment(
conn, attachment_id=attachment_id, chat_id=UUID(chat_id), owner_id=UUID(actor_id),
filename=filename, staging_key=staging_key, object_key=object_key,
content_type=content_type, size_bytes=size, sha256=sha256,
traceparent=carrier.get("traceparent"), tracestate=carrier.get("tracestate"),
)
except HTTPException:
raise
except Exception as exc:
# A lost COMMIT acknowledgement may leave a valid queued job. Keep the
# staging object; cleanup must check PostgreSQL before deleting it.
logger.warning("Attachment handoff failed for %s (%s)", attachment_id, type(exc).__name__)
raise HTTPException(503) from None
return attachment_view(row)
async def upload(request: Request, chat_id: str, actor_id: str, filename: str) -> dict:
await asyncio.to_thread(store.chat, actor_id, chat_id)
validate_filename(filename)
declared_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if declared_type not in {"image/png", "image/jpeg"}:
raise HTTPException(415)
length = request.headers.get("content-length")
if length is not None:
try:
if int(length) > MAX_BYTES:
raise HTTPException(413)
except ValueError:
raise HTTPException(400) from None
with SpooledTemporaryFile(max_size=1024 * 1024) as stream:
digest = hashlib.sha256()
size = 0
async for chunk in request.stream():
size += len(chunk)
if size > MAX_BYTES:
raise HTTPException(413)
stream.write(chunk)
digest.update(chunk)
stream.seek(0)
validate_image(stream, declared_type)
return await asyncio.to_thread(
persist_attachment, chat_id, actor_id, filename, declared_type, size, digest.hexdigest(), stream
)