Files
mtusi2026lab6/app/attachments.py
T

241 lines
11 KiB
Python

"""Lab5 attachments: synchronous durable admission, asynchronous image pipeline.
202 means verified bytes were staged to private S3 and metadata + a job + an
outbox event were committed in ONE PostgreSQL transaction. No broker availability
is required for admission; the dispatcher replays unsent intents.
"""
import hashlib
import os
import tempfile
import threading
import uuid
from contextlib import closing
import boto3
from boto3.s3.transfer import TransferConfig
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError
from fastapi import APIRouter, Query, Request
from fastapi.responses import StreamingResponse
from PIL import Image, ImageFile, UnidentifiedImageError
from starlette.concurrency import run_in_threadpool
from .auth import current_user
from .core import APIError, invalid, iso, parse_uuid
from .storage import db
from . import telemetry as ot
router = APIRouter()
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
SPOOL_MEMORY_BYTES = 1024 * 1024
S3_BUFFER_BYTES = 256 * 1024
DECODE_LIMIT = threading.BoundedSemaphore(1)
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
ImageFile.LOAD_TRUNCATED_IMAGES = False
def s3_client():
return ot.S3Tracing(boto3.client(
"s3", endpoint_url=os.environ["S3_ENDPOINT"],
aws_access_key_id=os.environ["S3_ACCESS_KEY"],
aws_secret_access_key=os.environ["S3_SECRET_KEY"],
region_name="us-east-1", use_ssl=False,
config=Config(s3={"addressing_style": "path"}, connect_timeout=3,
read_timeout=10, retries={"max_attempts": 1}),
))
def s3_error(exc):
return APIError(503, "unavailable", "Object storage unavailable")
def format_attachment(row, variants=()):
return {
"id": str(row["id"]), "chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]), "filename": row["filename"],
"content_type": row["content_type"], "size_bytes": row["size_bytes"],
"sha256": row["sha256"], "status": row["status"],
"variants": [{k: item[k] for k in ("name", "content_type", "size_bytes", "width", "height", "sha256")}
for item in variants] if row["status"] == "ready" else [],
"error_code": row["error_code"] if row["status"] == "failed" else None,
"created_at": iso(row["created_at"]), "updated_at": iso(row["updated_at"]),
}
def require_member(conn, chat_id, user_id):
if not conn.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user_id)).fetchone():
raise APIError(404, "not_found", "Object not found")
def attachment_row(conn, attachment_id, user_id):
row = conn.execute("""
SELECT a.* FROM attachments a
JOIN chat_members cm ON cm.chat_id=a.chat_id
WHERE a.id=%s AND cm.user_id=%s
""", (attachment_id, user_id)).fetchone()
if not row:
raise APIError(404, "not_found", "Object not found")
return row
def validate_filename(filename: str):
if not (1 <= len(filename) <= 255) or any(c in "/\\" or ord(c) < 32 or ord(c) == 127 for c in filename):
raise invalid("Invalid filename")
def validate_image(fileobj, mime: str):
"""Validate MIME, magic, dimensions and complete decode; worker does the resize."""
try:
fileobj.seek(0)
with Image.open(fileobj) as im:
if im.format not in ("PNG", "JPEG"):
raise APIError(415, "unsupported_media_type", "Unsupported image encoding")
actual = "image/png" if im.format == "PNG" else "image/jpeg"
if actual != mime:
raise APIError(415, "unsupported_media_type", "Content-Type does not match image")
if im.width * im.height > MAX_PIXELS:
raise APIError(413, "payload_too_large", "Image dimensions too large")
im.verify()
fileobj.seek(0)
with DECODE_LIMIT:
with Image.open(fileobj) as im:
im.load()
except APIError:
raise
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
raise APIError(413, "payload_too_large", "Image dimensions too large") from None
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
raise APIError(415, "unsupported_media_type", "Invalid image") from None
finally:
fileobj.seek(0)
def _admit_upload(spool, *, attachment_id, chat_id, user_id, filename, mime, size, digest):
from .jobs import job_payload
stage_key = f"staging/{attachment_id}/original"
key = f"attachments/{attachment_id}/original"
backup = f"recovery/{attachment_id}/original"
job_id = str(uuid.uuid4())
outbox_id = str(uuid.uuid4())
client = s3_client()
try:
spool.seek(0)
client.upload_fileobj(spool, BUCKET, stage_key,
ExtraArgs={"ContentType": mime},
Config=TransferConfig(use_threads=False, multipart_threshold=16*1024*1024,
io_chunksize=S3_BUFFER_BYTES))
except (BotoCoreError, ClientError, EndpointConnectionError, OSError) as exc:
raise s3_error(exc) from exc
# Atomic DB metadata + intent. If DB commit is ambiguous, do not delete the
# staged object: an accepted task may already exist; operator cleanup handles it.
with db() as conn:
with conn.transaction():
require_member(conn, chat_id, user_id)
# Small educational admission bound; pre-existing queued tasks remain safe.
pending = conn.execute("SELECT count(*) AS c FROM processing_jobs WHERE status IN ('queued','processing')").fetchone()["c"]
if pending >= int(os.getenv("JOB_BACKLOG_LIMIT", "1000")):
raise APIError(429, "rate_limited", "Processing queue is full")
row = conn.execute("""
INSERT INTO attachments(id,chat_id,owner_id,object_key,backup_key,filename,
content_type,size_bytes,sha256,status,staging_key,pipeline_version)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,1) RETURNING *
""", (attachment_id, chat_id, user_id, key, backup, filename, mime, size, digest, stage_key)).fetchone()
conn.execute("""
INSERT INTO processing_jobs(id,attachment_id,pipeline_version,status,attempt)
VALUES (%s,%s,1,'queued',1)
""", (job_id, attachment_id))
payload = job_payload(job_id, attachment_id, 1, 1)
conn.execute("""
INSERT INTO job_outbox(id,job_id,payload) VALUES (%s,%s,%s::jsonb)
""", (outbox_id, job_id, __import__('json').dumps(payload)))
ot.event("upload.accepted", job_id=job_id)
return format_attachment(row) # snapshot = queued, even if worker is very fast
@router.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def upload_attachment(req: Request, chat_id: str, filename: str = Query(...)):
from .api import actor, require_chat
who = actor(req)["id"]
chat_id = parse_uuid(chat_id)
validate_filename(filename)
mime = req.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if mime not in ("image/png", "image/jpeg"):
raise APIError(415, "unsupported_media_type", "Only PNG and JPEG are supported")
with db() as conn:
require_chat(conn, chat_id, who)
announced = req.headers.get("content-length")
if announced:
try:
if int(announced) > MAX_BYTES:
raise APIError(413, "payload_too_large", "Image too large")
if int(announced) < 0:
raise invalid("Invalid Content-Length")
except ValueError:
raise invalid("Invalid Content-Length") from None
size = 0
digest = hashlib.sha256()
with tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_BYTES, mode="w+b") as spool:
async for chunk in req.stream():
size += len(chunk)
if size > MAX_BYTES:
raise APIError(413, "payload_too_large", "Image too large")
digest.update(chunk)
spool.write(chunk)
if not size:
raise APIError(415, "unsupported_media_type", "Empty image")
await run_in_threadpool(validate_image, spool, mime)
return await run_in_threadpool(_admit_upload, spool, attachment_id=str(uuid.uuid4()),
chat_id=chat_id, user_id=who, filename=filename,
mime=mime, size=size, digest=digest.hexdigest())
@router.get("/api/v1/attachments/{attachment_id}")
def get_attachment(req: Request, attachment_id: str):
who = getattr(req.state, "actor", None) or current_user(req)
with db() as conn:
row = attachment_row(conn, parse_uuid(attachment_id), who["id"])
variants = conn.execute("""
SELECT name,content_type,size_bytes,width,height,sha256
FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s
ORDER BY CASE name WHEN 'thumbnail' THEN 0 WHEN 'crop' THEN 1 ELSE 2 END
""", (row["id"], row["pipeline_version"])).fetchall() if row["status"] == "ready" else []
return format_attachment(row, variants)
@router.get("/api/v1/attachments/{attachment_id}/content")
def download_attachment(req: Request, attachment_id: str, variant: str = "original"):
who = getattr(req.state, "actor", None) or current_user(req)
with db() as conn:
row = attachment_row(conn, parse_uuid(attachment_id), who["id"])
if row["status"] != "ready":
raise APIError(409, "conflict", "Attachment is not ready")
if variant == "original":
key, mime, size = row["object_key"], row["content_type"], row["size_bytes"]
else:
v = conn.execute("""
SELECT * FROM attachment_variants
WHERE attachment_id=%s AND pipeline_version=%s AND name=%s
""", (row["id"], row["pipeline_version"], variant)).fetchone()
if not v:
raise APIError(404, "not_found", "Object not found")
key, mime, size = v["object_key"], v["content_type"], v["size_bytes"]
client = s3_client()
try:
response = client.get_object(Bucket=BUCKET, Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404", "NotFound"):
raise APIError(503, "unavailable", "Stored object missing") from exc
raise s3_error(exc) from exc
except BotoCoreError as exc:
raise s3_error(exc) from exc
def iterator():
with closing(response["Body"]) as body:
while chunk := body.read(S3_BUFFER_BYTES):
yield chunk
return StreamingResponse(iterator(), media_type=mime, headers={"Content-Length": str(size)})