Files
mtusi2026lab6/app/api/attachments.py

646 lines
18 KiB
Python

import hashlib
import tempfile
import uuid
from datetime import datetime, timezone, timedelta
from urllib.parse import quote
from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, Response
from fastapi.responses import StreamingResponse
from sqlalchemy import select, func, text
from sqlalchemy.orm import Session
from botocore.exceptions import BotoCoreError, ClientError
from opentelemetry.propagate import inject
from app.api.deps import get_db, get_current_user
from app.core.config import settings
from app.models import CreateAttachmentUpload
from app.db.models import (
Attachment,
AttachmentUpload,
AttachmentVariant,
ProcessingJob,
OutboxEvent,
User,
)
from app.services.object_storage import client, create_presigned_upload_url
from app.services.image_validation import MAX_BYTES, validate_image_file
from app.repositories.chat_repository import ChatRepository
router = APIRouter(prefix="/api/v1", tags=["Attachments"])
def iso_utc(value):
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def attachment_json(item, db):
variants = db.scalars(
select(AttachmentVariant)
.where(AttachmentVariant.attachment_id == item.id)
.order_by(AttachmentVariant.variant)
).all()
return {
"id": str(item.id),
"chat_id": str(item.chat_id),
"owner_id": str(item.owner_id),
"filename": item.filename,
"content_type": item.content_type,
"size_bytes": item.size_bytes,
"sha256": item.sha256,
"status": item.status,
"variants": [
{
"name": variant.variant,
"content_type": variant.content_type,
"size_bytes": variant.size_bytes,
"width": variant.width,
"height": variant.height,
"sha256": variant.sha256,
}
for variant in variants
],
"error_code": item.error_code,
"created_at": iso_utc(item.created_at),
"updated_at": iso_utc(item.updated_at),
}
def member_attachment(db, attachment_id, user_id):
item = db.get(Attachment, attachment_id)
if item is None or not ChatRepository(db).is_member(item.chat_id, user_id):
raise HTTPException(404, "Attachment not found")
return item
async def read_limited(request: Request):
declared = request.headers.get("content-length")
if declared and declared.isdigit() and int(declared) > MAX_BYTES:
raise HTTPException(413, "Image exceeds 10 MiB")
size = 0
# Keep at most 1 MiB in RAM; larger bodies spill to a temporary file.
spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b")
try:
async for chunk in request.stream():
size += len(chunk)
if size > MAX_BYTES:
raise HTTPException(413, "Image exceeds 10 MiB")
spool.write(chunk)
spool.seek(0)
return spool, size
except Exception:
spool.close()
raise
def sha256_file(fileobj):
digest = hashlib.sha256()
fileobj.seek(0)
while True:
chunk = fileobj.read(1024 * 1024)
if not chunk:
break
digest.update(chunk)
fileobj.seek(0)
return digest.hexdigest()
@router.post("/chats/{chat_id}/attachments/presigned", status_code=201)
def create_attachment_upload(
chat_id: str,
payload: CreateAttachmentUpload,
db: Session = Depends(get_db),
user: User = Depends(get_current_user),
):
try:
chat_uuid = uuid.UUID(chat_id)
except ValueError:
raise HTTPException(400, "Invalid chat UUID") from None
if not ChatRepository(db).is_member(chat_uuid, user.id):
raise HTTPException(404, "Chat not found")
if payload.size_bytes > settings.max_upload_bytes:
raise HTTPException(413, "Image exceeds configured upload limit")
upload_id = uuid.uuid4()
object_key = f"uploads/{chat_uuid}/{upload_id}"
expires_at = datetime.now(timezone.utc) + timedelta(
seconds=settings.presigned_upload_ttl_seconds
)
try:
upload_url = create_presigned_upload_url(
object_key=object_key,
content_type=payload.content_type,
)
except (BotoCoreError, ClientError):
raise HTTPException(503, "Object storage unavailable") from None
upload = AttachmentUpload(
id=upload_id,
chat_id=chat_uuid,
owner_id=user.id,
filename=payload.filename,
content_type=payload.content_type,
expected_size_bytes=payload.size_bytes,
expected_sha256=payload.sha256.lower(),
object_key=object_key,
status="pending",
expires_at=expires_at,
)
try:
db.add(upload)
db.commit()
except Exception:
db.rollback()
raise HTTPException(503, "Could not save upload metadata") from None
return {
"upload_id": str(upload_id),
"upload_url": upload_url,
"method": "PUT",
"headers": {
"Content-Type": payload.content_type,
},
"expires_in": settings.presigned_upload_ttl_seconds,
"expires_at": iso_utc(expires_at),
}
@router.post("/attachments/uploads/{upload_id}/complete", status_code=200)
def complete_attachment_upload(
upload_id: str,
db: Session = Depends(get_db),
user: User = Depends(get_current_user),
):
try:
parsed_id = uuid.UUID(upload_id)
except ValueError:
raise HTTPException(400, "Invalid upload UUID") from None
# Lock the upload row to prevent concurrent completion.
upload = db.scalar(
select(AttachmentUpload)
.where(AttachmentUpload.id == parsed_id)
.with_for_update()
)
if upload is None:
raise HTTPException(404, "Upload not found")
if (
upload.owner_id != user.id
or not ChatRepository(db).is_member(upload.chat_id, user.id)
):
raise HTTPException(404, "Upload not found")
# Idempotent completion: return the existing attachment.
if upload.status == "completed":
item = db.scalar(
select(Attachment).where(
Attachment.object_key == upload.object_key
)
)
if item is None:
raise HTTPException(409, "Completed attachment not found")
return attachment_json(item, db)
if upload.status != "pending":
raise HTTPException(409, "Upload is not pending")
now = datetime.now(timezone.utc)
if upload.expires_at <= now:
upload.status = "expired"
db.commit()
try:
client().delete_object(
Bucket=settings.s3_bucket,
Key=upload.object_key,
)
except Exception:
pass
raise HTTPException(409, "Upload has expired")
s3 = client()
# Read the object from S3.
try:
result = s3.get_object(
Bucket=settings.s3_bucket,
Key=upload.object_key,
)
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code", "")
if code in {"NoSuchKey", "404", "NotFound"}:
raise HTTPException(409, "Uploaded object not found") from None
raise HTTPException(503, "Object storage unavailable") from None
except BotoCoreError:
raise HTTPException(503, "Object storage unavailable") from None
body = result["Body"]
spool = tempfile.SpooledTemporaryFile(
max_size=1024 * 1024,
mode="w+b",
)
def reject_upload(status_code: int, detail: str):
upload.status = "failed"
db.commit()
try:
s3.delete_object(
Bucket=settings.s3_bucket,
Key=upload.object_key,
)
except Exception:
pass
raise HTTPException(status_code=status_code, detail=detail)
try:
actual_size = result.get("ContentLength")
if actual_size != upload.expected_size_bytes:
reject_upload(409, "Uploaded file size mismatch")
if actual_size > settings.max_upload_bytes:
reject_upload(413, "Image exceeds configured upload limit")
total = 0
digest = hashlib.sha256()
try:
while True:
chunk = body.read(64 * 1024)
if not chunk:
break
total += len(chunk)
if total > settings.max_upload_bytes:
reject_upload(413, "Image exceeds configured upload limit")
if total > upload.expected_size_bytes:
reject_upload(409, "Uploaded file size mismatch")
digest.update(chunk)
spool.write(chunk)
finally:
body.close()
if total != upload.expected_size_bytes:
reject_upload(409, "Uploaded file size mismatch")
if digest.hexdigest() != upload.expected_sha256:
reject_upload(409, "Uploaded file checksum mismatch")
# Validate the actual image content, not only S3 metadata.
try:
actual_type, _, _ = validate_image_file(
spool,
upload.content_type,
)
except HTTPException as exc:
reject_upload(exc.status_code, str(exc.detail))
attachment = Attachment(
id=uuid.uuid4(),
chat_id=upload.chat_id,
owner_id=upload.owner_id,
filename=upload.filename,
content_type=actual_type,
size_bytes=total,
sha256=digest.hexdigest(),
object_key=upload.object_key,
status="ready",
error_code=None,
)
upload.status = "completed"
upload.completed_at = now
db.add(attachment)
db.commit()
db.refresh(attachment)
return attachment_json(attachment, db)
except HTTPException:
raise
except (BotoCoreError, ClientError):
db.rollback()
raise HTTPException(503, "Object storage unavailable") from None
except Exception:
db.rollback()
raise
finally:
spool.close()
@router.post("/chats/{chat_id}/attachments", status_code=202)
async def upload_attachment(
chat_id: str,
request: Request,
filename: str = Query(..., min_length=1, max_length=255),
db: Session = Depends(get_db),
user: User = Depends(get_current_user),
):
try:
chat_uuid = uuid.UUID(chat_id)
except ValueError:
raise HTTPException(400, "Invalid chat UUID") from None
if not ChatRepository(db).is_member(chat_uuid, user.id):
raise HTTPException(404, "Chat not found")
if (
not filename.strip()
or any(c in filename for c in ("/", "\\", "\x00"))
or any(ord(c) < 32 or ord(c) == 127 for c in filename)
):
raise HTTPException(400, "Unsafe filename")
content_type = (
request.headers.get("content-type", "")
.split(";", 1)[0]
.strip()
.lower()
)
if content_type not in {"image/png", "image/jpeg"}:
raise HTTPException(415, "Only PNG and JPEG are supported")
spool, size = await read_limited(request)
attachment_id = uuid.uuid4()
job_id = uuid.uuid4()
staging_key = f"staging/{chat_uuid}/{attachment_id}"
object_key = f"original/{chat_uuid}/{attachment_id}"
staged = False
try:
actual_type, _, _ = validate_image_file(spool, content_type)
digest = sha256_file(spool)
# Serialize admission checks across all API instances.
db.execute(
text("SELECT pg_advisory_xact_lock(:lock_id)"),
{"lock_id": 72420261110},
)
active_jobs = db.scalar(
select(func.count())
.select_from(ProcessingJob)
.where(ProcessingJob.status.in_(("queued", "processing")))
) or 0
if active_jobs >= settings.max_active_processing_jobs:
raise HTTPException(
status_code=429,
detail="Processing queue is full; retry later",
headers={"Retry-After": "5"},
)
s3 = client()
try:
spool.seek(0)
s3.put_object(
Bucket=settings.s3_bucket,
Key=staging_key,
Body=spool,
ContentType=actual_type,
)
staged = True
except (BotoCoreError, ClientError):
raise HTTPException(
503, "Object storage unavailable"
) from None
now = datetime.now(timezone.utc)
attachment = Attachment(
id=attachment_id,
chat_id=chat_uuid,
owner_id=user.id,
filename=filename,
content_type=actual_type,
size_bytes=size,
sha256=digest,
object_key=object_key,
staging_key=staging_key,
status="queued",
error_code=None,
pipeline_version=1,
)
job = ProcessingJob(
id=job_id,
attachment_id=attachment_id,
pipeline_version=1,
attempt=0,
status="queued",
available_at=now,
)
# Preserve the current HTTP trace context in the durable outbox event.
trace_context = {}
inject(trace_context)
event = OutboxEvent(
event_type="attachment.process.v1",
aggregate_id=attachment_id,
payload={
"schema_version": 1,
"job_id": str(job_id),
"type": "attachment.process.v1",
"attachment_id": str(attachment_id),
"pipeline_version": 1,
"attempt": 1,
"created_at": now.isoformat(),
"trace_context": trace_context,
},
status="pending",
)
db.add_all([attachment, job, event])
db.commit()
db.refresh(attachment)
return attachment_json(attachment, db)
except HTTPException:
db.rollback()
if staged:
try:
client().delete_object(
Bucket=settings.s3_bucket,
Key=staging_key,
)
except Exception:
pass
raise
except Exception:
db.rollback()
if staged:
try:
client().delete_object(
Bucket=settings.s3_bucket,
Key=staging_key,
)
except Exception:
pass
raise HTTPException(
503, "Could not save attachment processing job"
) from None
finally:
spool.close()
@router.get("/attachments/{attachment_id}")
def get_attachment(
attachment_id: str,
db: Session = Depends(get_db),
user: User = Depends(get_current_user),
):
try:
parsed = uuid.UUID(attachment_id)
except ValueError:
raise HTTPException(400, "Invalid attachment UUID") from None
item = member_attachment(db, parsed, user.id)
return attachment_json(item, db)
@router.get("/attachments/{attachment_id}/content")
def download_attachment(
attachment_id: str,
variant: str = "original",
db: Session = Depends(get_db),
user: User = Depends(get_current_user),
):
try:
parsed = uuid.UUID(attachment_id)
except ValueError:
raise HTTPException(400, "Invalid attachment UUID") from None
item = member_attachment(db, parsed, user.id)
if item.status != "ready":
raise HTTPException(409, "Attachment is not ready")
# Select the original file or a generated variant.
if variant == "original":
object_key = item.object_key
expected_size = item.size_bytes
expected_sha256 = item.sha256
content_type = item.content_type
else:
variant_item = db.scalar(
select(AttachmentVariant).where(
AttachmentVariant.attachment_id == item.id,
AttachmentVariant.variant == variant,
)
)
if variant_item is None:
raise HTTPException(404, "Variant not found")
object_key = variant_item.object_key
expected_size = variant_item.size_bytes
expected_sha256 = variant_item.sha256
content_type = variant_item.content_type
spool = tempfile.SpooledTemporaryFile(
max_size=1024 * 1024,
mode="w+b",
)
try:
result = client().get_object(
Bucket=settings.s3_bucket,
Key=object_key,
)
body = result["Body"]
total = 0
digest = hashlib.sha256()
try:
while True:
chunk = body.read(64 * 1024)
if not chunk:
break
total += len(chunk)
if total > MAX_BYTES:
raise HTTPException(
503,
"Stored object exceeds configured limit",
)
digest.update(chunk)
spool.write(chunk)
finally:
body.close()
# Verify stored object integrity.
if (
total != expected_size
or digest.hexdigest() != expected_sha256
):
raise HTTPException(
503,
"Stored object checksum mismatch",
)
spool.seek(0)
except (BotoCoreError, ClientError):
spool.close()
raise HTTPException(
503,
"Object storage unavailable",
) from None
except Exception:
spool.close()
raise
def stream_verified_file():
try:
while True:
chunk = spool.read(64 * 1024)
if not chunk:
break
yield chunk
finally:
spool.close()
return StreamingResponse(
stream_verified_file(),
media_type=content_type,
headers={
"Content-Disposition": (
f"inline; filename*=UTF-8''{quote(item.filename)}"
)
},
)