forked from mtusi2026/mtusi2026lab4
242 lines
9.9 KiB
Python
242 lines
9.9 KiB
Python
"""Lab4 synchronous attachment API: streamed ingress, S3 storage and per-chat ACL.
|
|
|
|
The spool is temporary request workspace, not an authoritative file store. No
|
|
original survives a completed request on either API replica's local disk.
|
|
"""
|
|
import hashlib
|
|
import io
|
|
import os
|
|
import tempfile
|
|
import threading
|
|
import uuid
|
|
from contextlib import closing
|
|
|
|
import boto3
|
|
from botocore.config import Config
|
|
from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError
|
|
from fastapi import APIRouter, Query, Request
|
|
from fastapi.responses import StreamingResponse
|
|
from PIL import Image, ImageFile, UnidentifiedImageError
|
|
from starlette.concurrency import run_in_threadpool
|
|
|
|
from .auth import current_user
|
|
from .core import APIError, invalid, iso, parse_uuid
|
|
from .storage import db
|
|
|
|
router = APIRouter()
|
|
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
|
MAX_BYTES = 10 * 1024 * 1024
|
|
MAX_PIXELS = 40_000_000
|
|
SPOOL_MEMORY_BYTES = 1024 * 1024
|
|
S3_BUFFER_BYTES = 256 * 1024
|
|
DECODE_LIMIT = threading.BoundedSemaphore(1) # bound expensive Pillow decoding per replica
|
|
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
|
ImageFile.LOAD_TRUNCATED_IMAGES = False
|
|
|
|
|
|
def s3_client():
|
|
return boto3.client(
|
|
"s3", endpoint_url=os.environ["S3_ENDPOINT"],
|
|
aws_access_key_id=os.environ["S3_ACCESS_KEY"],
|
|
aws_secret_access_key=os.environ["S3_SECRET_KEY"],
|
|
region_name="us-east-1", use_ssl=False,
|
|
config=Config(s3={"addressing_style": "path"},
|
|
connect_timeout=3, read_timeout=10, retries={"max_attempts": 1}),
|
|
)
|
|
|
|
|
|
def s3_error(exc):
|
|
return APIError(503, "unavailable", "Object storage unavailable")
|
|
|
|
|
|
def format_attachment(row):
|
|
return {
|
|
"id": str(row["id"]), "chat_id": str(row["chat_id"]),
|
|
"owner_id": str(row["owner_id"]), "filename": row["filename"],
|
|
"content_type": row["content_type"], "size_bytes": row["size_bytes"],
|
|
"sha256": row["sha256"], "status": "ready", "variants": [],
|
|
"error_code": None, "created_at": iso(row["created_at"]),
|
|
"updated_at": iso(row["updated_at"]),
|
|
}
|
|
|
|
|
|
def require_member(conn, chat_id, user_id):
|
|
row = conn.execute(
|
|
"SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user_id)
|
|
).fetchone()
|
|
if not row:
|
|
raise APIError(404, "not_found", "Object not found")
|
|
|
|
|
|
def attachment_row(conn, attachment_id, user_id):
|
|
row = conn.execute("""
|
|
SELECT a.* FROM attachments a
|
|
JOIN chat_members cm ON cm.chat_id=a.chat_id
|
|
WHERE a.id=%s AND cm.user_id=%s
|
|
""", (attachment_id, user_id)).fetchone()
|
|
if not row:
|
|
raise APIError(404, "not_found", "Object not found")
|
|
return row
|
|
|
|
|
|
def validate_filename(filename: str):
|
|
if not (1 <= len(filename) <= 255) or any(
|
|
c in "/\\" or ord(c) < 32 or ord(c) == 127 for c in filename
|
|
):
|
|
raise invalid("Invalid filename")
|
|
|
|
|
|
def validate_image(fileobj, mime: str):
|
|
"""Check full decode, dimensions, magic and MIME before writing to S3."""
|
|
try:
|
|
fileobj.seek(0)
|
|
with Image.open(fileobj) as im:
|
|
fmt = im.format
|
|
if fmt not in ("PNG", "JPEG"):
|
|
raise APIError(415, "unsupported_media_type", "Unsupported image encoding")
|
|
actual_mime = "image/png" if fmt == "PNG" else "image/jpeg"
|
|
if actual_mime != mime:
|
|
raise APIError(415, "unsupported_media_type", "Content-Type does not match image")
|
|
if im.width * im.height > MAX_PIXELS:
|
|
raise APIError(413, "payload_too_large", "Image dimensions too large")
|
|
im.verify()
|
|
fileobj.seek(0)
|
|
with DECODE_LIMIT:
|
|
with Image.open(fileobj) as im:
|
|
im.load() # reject corrupt/truncated pixel data, not just headers
|
|
except APIError:
|
|
raise
|
|
except Image.DecompressionBombError:
|
|
raise APIError(413, "payload_too_large", "Image dimensions too large") from None
|
|
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
|
|
raise APIError(415, "unsupported_media_type", "Invalid image") from None
|
|
except Image.DecompressionBombWarning:
|
|
raise APIError(413, "payload_too_large", "Image dimensions too large") from None
|
|
finally:
|
|
fileobj.seek(0)
|
|
|
|
|
|
def _upload_checked(spool, *, attachment_id, chat_id, user_id, filename, mime, size, digest):
|
|
"""Blocking S3 and DB steps run outside the event loop."""
|
|
key = f"attachments/{attachment_id}/original"
|
|
backup = f"recovery/{attachment_id}/original"
|
|
client = s3_client()
|
|
wrote_original = False
|
|
wrote_backup = False
|
|
committed = False
|
|
try:
|
|
# A recovery copy enables non-destructive repair if the primary key is lost.
|
|
spool.seek(0)
|
|
client.upload_fileobj(spool, BUCKET, key, ExtraArgs={"ContentType": mime},
|
|
Config=__import__('boto3.s3.transfer', fromlist=['TransferConfig']).TransferConfig(
|
|
use_threads=False, multipart_threshold=16*1024*1024,
|
|
io_chunksize=S3_BUFFER_BYTES))
|
|
wrote_original = True
|
|
client.copy_object(Bucket=BUCKET, Key=backup,
|
|
CopySource={"Bucket": BUCKET, "Key": key}, ContentType=mime,
|
|
MetadataDirective="REPLACE")
|
|
wrote_backup = True
|
|
with db() as conn:
|
|
with conn.transaction():
|
|
require_member(conn, chat_id, user_id)
|
|
row = conn.execute("""
|
|
INSERT INTO attachments(id,chat_id,owner_id,object_key,backup_key,filename,
|
|
content_type,size_bytes,sha256)
|
|
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *
|
|
""", (attachment_id, chat_id, user_id, key, backup, filename, mime, size, digest)).fetchone()
|
|
committed = True
|
|
return format_attachment(row)
|
|
except (BotoCoreError, ClientError, EndpointConnectionError) as exc:
|
|
raise s3_error(exc) from exc
|
|
finally:
|
|
# Never delete after an ambiguous COMMIT unless a fresh read proves
|
|
# metadata is absent. A lost COMMIT acknowledgement can still mean
|
|
# PostgreSQL committed successfully. Reconciler handles uncertainty.
|
|
if not committed:
|
|
confirmed_absent = False
|
|
try:
|
|
with db() as conn:
|
|
confirmed_absent = conn.execute(
|
|
"SELECT 1 FROM attachments WHERE id=%s", (attachment_id,)
|
|
).fetchone() is None
|
|
except Exception:
|
|
pass
|
|
if confirmed_absent:
|
|
for written, object_key in ((wrote_backup, backup), (wrote_original, key)):
|
|
if written:
|
|
try:
|
|
client.delete_object(Bucket=BUCKET, Key=object_key)
|
|
except (BotoCoreError, ClientError, EndpointConnectionError):
|
|
pass
|
|
|
|
|
|
@router.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
|
|
async def upload_attachment(req: Request, chat_id: str, filename: str = Query(...)):
|
|
from .api import actor, require_chat
|
|
who = actor(req)["id"]
|
|
chat_id = parse_uuid(chat_id)
|
|
validate_filename(filename)
|
|
mime = req.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
|
if mime not in ("image/png", "image/jpeg"):
|
|
raise APIError(415, "unsupported_media_type", "Only PNG and JPEG are supported")
|
|
with db() as conn:
|
|
require_chat(conn, chat_id, who)
|
|
announced_size = req.headers.get("content-length")
|
|
if announced_size:
|
|
try:
|
|
if int(announced_size) > MAX_BYTES:
|
|
raise APIError(413, "payload_too_large", "Image too large")
|
|
except ValueError:
|
|
raise invalid("Invalid Content-Length") from None
|
|
size = 0
|
|
digest = hashlib.sha256()
|
|
with tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_BYTES, mode="w+b") as spool:
|
|
async for chunk in req.stream():
|
|
size += len(chunk)
|
|
if size > MAX_BYTES:
|
|
raise APIError(413, "payload_too_large", "Image too large")
|
|
digest.update(chunk)
|
|
spool.write(chunk)
|
|
if size == 0:
|
|
raise APIError(415, "unsupported_media_type", "Empty image")
|
|
await run_in_threadpool(validate_image, spool, mime)
|
|
return await run_in_threadpool(_upload_checked, spool,
|
|
attachment_id=str(uuid.uuid4()), chat_id=chat_id, user_id=who,
|
|
filename=filename, mime=mime, size=size, digest=digest.hexdigest())
|
|
|
|
|
|
@router.get("/api/v1/attachments/{attachment_id}")
|
|
def get_attachment(req: Request, attachment_id: str):
|
|
who = getattr(req.state, "actor", None) or current_user(req)
|
|
with db() as conn:
|
|
return format_attachment(attachment_row(conn, parse_uuid(attachment_id), who["id"]))
|
|
|
|
|
|
@router.get("/api/v1/attachments/{attachment_id}/content")
|
|
def download_attachment(req: Request, attachment_id: str, variant: str = "original"):
|
|
who = getattr(req.state, "actor", None) or current_user(req)
|
|
with db() as conn:
|
|
row = attachment_row(conn, parse_uuid(attachment_id), who["id"])
|
|
if variant != "original":
|
|
raise APIError(404, "not_found", "Object not found")
|
|
client = s3_client()
|
|
try:
|
|
response = client.get_object(Bucket=BUCKET, Key=row["object_key"])
|
|
except ClientError as exc:
|
|
if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404", "NotFound"):
|
|
raise APIError(503, "unavailable", "Stored object missing") from exc
|
|
raise s3_error(exc) from exc
|
|
except BotoCoreError as exc:
|
|
raise s3_error(exc) from exc
|
|
|
|
def iterator():
|
|
with closing(response["Body"]) as body:
|
|
while True:
|
|
chunk = body.read(S3_BUFFER_BYTES)
|
|
if not chunk:
|
|
break
|
|
yield chunk
|
|
|
|
return StreamingResponse(iterator(), media_type=row["content_type"],
|
|
headers={"Content-Length": str(row["size_bytes"])})
|