Files

242 lines
9.9 KiB
Python

"""Lab4 synchronous attachment API: streamed ingress, S3 storage and per-chat ACL.
The spool is temporary request workspace, not an authoritative file store. No
original survives a completed request on either API replica's local disk.
"""
import hashlib
import io
import os
import tempfile
import threading
import uuid
from contextlib import closing
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError
from fastapi import APIRouter, Query, Request
from fastapi.responses import StreamingResponse
from PIL import Image, ImageFile, UnidentifiedImageError
from starlette.concurrency import run_in_threadpool
from .auth import current_user
from .core import APIError, invalid, iso, parse_uuid
from .storage import db
router = APIRouter()
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
SPOOL_MEMORY_BYTES = 1024 * 1024
S3_BUFFER_BYTES = 256 * 1024
DECODE_LIMIT = threading.BoundedSemaphore(1) # bound expensive Pillow decoding per replica
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
ImageFile.LOAD_TRUNCATED_IMAGES = False
def s3_client():
return boto3.client(
"s3", endpoint_url=os.environ["S3_ENDPOINT"],
aws_access_key_id=os.environ["S3_ACCESS_KEY"],
aws_secret_access_key=os.environ["S3_SECRET_KEY"],
region_name="us-east-1", use_ssl=False,
config=Config(s3={"addressing_style": "path"},
connect_timeout=3, read_timeout=10, retries={"max_attempts": 1}),
)
def s3_error(exc):
return APIError(503, "unavailable", "Object storage unavailable")
def format_attachment(row):
return {
"id": str(row["id"]), "chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]), "filename": row["filename"],
"content_type": row["content_type"], "size_bytes": row["size_bytes"],
"sha256": row["sha256"], "status": "ready", "variants": [],
"error_code": None, "created_at": iso(row["created_at"]),
"updated_at": iso(row["updated_at"]),
}
def require_member(conn, chat_id, user_id):
row = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user_id)
).fetchone()
if not row:
raise APIError(404, "not_found", "Object not found")
def attachment_row(conn, attachment_id, user_id):
row = conn.execute("""
SELECT a.* FROM attachments a
JOIN chat_members cm ON cm.chat_id=a.chat_id
WHERE a.id=%s AND cm.user_id=%s
""", (attachment_id, user_id)).fetchone()
if not row:
raise APIError(404, "not_found", "Object not found")
return row
def validate_filename(filename: str):
if not (1 <= len(filename) <= 255) or any(
c in "/\\" or ord(c) < 32 or ord(c) == 127 for c in filename
):
raise invalid("Invalid filename")
def validate_image(fileobj, mime: str):
"""Check full decode, dimensions, magic and MIME before writing to S3."""
try:
fileobj.seek(0)
with Image.open(fileobj) as im:
fmt = im.format
if fmt not in ("PNG", "JPEG"):
raise APIError(415, "unsupported_media_type", "Unsupported image encoding")
actual_mime = "image/png" if fmt == "PNG" else "image/jpeg"
if actual_mime != mime:
raise APIError(415, "unsupported_media_type", "Content-Type does not match image")
if im.width * im.height > MAX_PIXELS:
raise APIError(413, "payload_too_large", "Image dimensions too large")
im.verify()
fileobj.seek(0)
with DECODE_LIMIT:
with Image.open(fileobj) as im:
im.load() # reject corrupt/truncated pixel data, not just headers
except APIError:
raise
except Image.DecompressionBombError:
raise APIError(413, "payload_too_large", "Image dimensions too large") from None
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
raise APIError(415, "unsupported_media_type", "Invalid image") from None
except Image.DecompressionBombWarning:
raise APIError(413, "payload_too_large", "Image dimensions too large") from None
finally:
fileobj.seek(0)
def _upload_checked(spool, *, attachment_id, chat_id, user_id, filename, mime, size, digest):
"""Blocking S3 and DB steps run outside the event loop."""
key = f"attachments/{attachment_id}/original"
backup = f"recovery/{attachment_id}/original"
client = s3_client()
wrote_original = False
wrote_backup = False
committed = False
try:
# A recovery copy enables non-destructive repair if the primary key is lost.
spool.seek(0)
client.upload_fileobj(spool, BUCKET, key, ExtraArgs={"ContentType": mime},
Config=__import__('boto3.s3.transfer', fromlist=['TransferConfig']).TransferConfig(
use_threads=False, multipart_threshold=16*1024*1024,
io_chunksize=S3_BUFFER_BYTES))
wrote_original = True
client.copy_object(Bucket=BUCKET, Key=backup,
CopySource={"Bucket": BUCKET, "Key": key}, ContentType=mime,
MetadataDirective="REPLACE")
wrote_backup = True
with db() as conn:
with conn.transaction():
require_member(conn, chat_id, user_id)
row = conn.execute("""
INSERT INTO attachments(id,chat_id,owner_id,object_key,backup_key,filename,
content_type,size_bytes,sha256)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *
""", (attachment_id, chat_id, user_id, key, backup, filename, mime, size, digest)).fetchone()
committed = True
return format_attachment(row)
except (BotoCoreError, ClientError, EndpointConnectionError) as exc:
raise s3_error(exc) from exc
finally:
# Never delete after an ambiguous COMMIT unless a fresh read proves
# metadata is absent. A lost COMMIT acknowledgement can still mean
# PostgreSQL committed successfully. Reconciler handles uncertainty.
if not committed:
confirmed_absent = False
try:
with db() as conn:
confirmed_absent = conn.execute(
"SELECT 1 FROM attachments WHERE id=%s", (attachment_id,)
).fetchone() is None
except Exception:
pass
if confirmed_absent:
for written, object_key in ((wrote_backup, backup), (wrote_original, key)):
if written:
try:
client.delete_object(Bucket=BUCKET, Key=object_key)
except (BotoCoreError, ClientError, EndpointConnectionError):
pass
@router.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
async def upload_attachment(req: Request, chat_id: str, filename: str = Query(...)):
from .api import actor, require_chat
who = actor(req)["id"]
chat_id = parse_uuid(chat_id)
validate_filename(filename)
mime = req.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if mime not in ("image/png", "image/jpeg"):
raise APIError(415, "unsupported_media_type", "Only PNG and JPEG are supported")
with db() as conn:
require_chat(conn, chat_id, who)
announced_size = req.headers.get("content-length")
if announced_size:
try:
if int(announced_size) > MAX_BYTES:
raise APIError(413, "payload_too_large", "Image too large")
except ValueError:
raise invalid("Invalid Content-Length") from None
size = 0
digest = hashlib.sha256()
with tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_BYTES, mode="w+b") as spool:
async for chunk in req.stream():
size += len(chunk)
if size > MAX_BYTES:
raise APIError(413, "payload_too_large", "Image too large")
digest.update(chunk)
spool.write(chunk)
if size == 0:
raise APIError(415, "unsupported_media_type", "Empty image")
await run_in_threadpool(validate_image, spool, mime)
return await run_in_threadpool(_upload_checked, spool,
attachment_id=str(uuid.uuid4()), chat_id=chat_id, user_id=who,
filename=filename, mime=mime, size=size, digest=digest.hexdigest())
@router.get("/api/v1/attachments/{attachment_id}")
def get_attachment(req: Request, attachment_id: str):
who = getattr(req.state, "actor", None) or current_user(req)
with db() as conn:
return format_attachment(attachment_row(conn, parse_uuid(attachment_id), who["id"]))
@router.get("/api/v1/attachments/{attachment_id}/content")
def download_attachment(req: Request, attachment_id: str, variant: str = "original"):
who = getattr(req.state, "actor", None) or current_user(req)
with db() as conn:
row = attachment_row(conn, parse_uuid(attachment_id), who["id"])
if variant != "original":
raise APIError(404, "not_found", "Object not found")
client = s3_client()
try:
response = client.get_object(Bucket=BUCKET, Key=row["object_key"])
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404", "NotFound"):
raise APIError(503, "unavailable", "Stored object missing") from exc
raise s3_error(exc) from exc
except BotoCoreError as exc:
raise s3_error(exc) from exc
def iterator():
with closing(response["Body"]) as body:
while True:
chunk = body.read(S3_BUFFER_BYTES)
if not chunk:
break
yield chunk
return StreamingResponse(iterator(), media_type=row["content_type"],
headers={"Content-Length": str(row["size_bytes"])})