forked from mtusi2026/mtusi2026lab6
646 lines
18 KiB
Python
646 lines
18 KiB
Python
import hashlib
|
|
import tempfile
|
|
import uuid
|
|
from datetime import datetime, timezone, timedelta
|
|
from urllib.parse import quote
|
|
|
|
from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, Response
|
|
from fastapi.responses import StreamingResponse
|
|
from sqlalchemy import select, func, text
|
|
from sqlalchemy.orm import Session
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
from opentelemetry.propagate import inject
|
|
|
|
from app.api.deps import get_db, get_current_user
|
|
from app.core.config import settings
|
|
from app.models import CreateAttachmentUpload
|
|
from app.db.models import (
|
|
Attachment,
|
|
AttachmentUpload,
|
|
AttachmentVariant,
|
|
ProcessingJob,
|
|
OutboxEvent,
|
|
User,
|
|
)
|
|
from app.services.object_storage import client, create_presigned_upload_url
|
|
from app.services.image_validation import MAX_BYTES, validate_image_file
|
|
from app.repositories.chat_repository import ChatRepository
|
|
|
|
router = APIRouter(prefix="/api/v1", tags=["Attachments"])
|
|
|
|
|
|
def iso_utc(value):
|
|
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
|
|
|
|
|
def attachment_json(item, db):
|
|
variants = db.scalars(
|
|
select(AttachmentVariant)
|
|
.where(AttachmentVariant.attachment_id == item.id)
|
|
.order_by(AttachmentVariant.variant)
|
|
).all()
|
|
|
|
return {
|
|
"id": str(item.id),
|
|
"chat_id": str(item.chat_id),
|
|
"owner_id": str(item.owner_id),
|
|
"filename": item.filename,
|
|
"content_type": item.content_type,
|
|
"size_bytes": item.size_bytes,
|
|
"sha256": item.sha256,
|
|
"status": item.status,
|
|
"variants": [
|
|
{
|
|
"name": variant.variant,
|
|
"content_type": variant.content_type,
|
|
"size_bytes": variant.size_bytes,
|
|
"width": variant.width,
|
|
"height": variant.height,
|
|
"sha256": variant.sha256,
|
|
}
|
|
for variant in variants
|
|
],
|
|
"error_code": item.error_code,
|
|
"created_at": iso_utc(item.created_at),
|
|
"updated_at": iso_utc(item.updated_at),
|
|
}
|
|
|
|
|
|
def member_attachment(db, attachment_id, user_id):
|
|
item = db.get(Attachment, attachment_id)
|
|
if item is None or not ChatRepository(db).is_member(item.chat_id, user_id):
|
|
raise HTTPException(404, "Attachment not found")
|
|
return item
|
|
|
|
|
|
async def read_limited(request: Request):
|
|
declared = request.headers.get("content-length")
|
|
if declared and declared.isdigit() and int(declared) > MAX_BYTES:
|
|
raise HTTPException(413, "Image exceeds 10 MiB")
|
|
size = 0
|
|
# Keep at most 1 MiB in RAM; larger bodies spill to a temporary file.
|
|
spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b")
|
|
try:
|
|
async for chunk in request.stream():
|
|
size += len(chunk)
|
|
if size > MAX_BYTES:
|
|
raise HTTPException(413, "Image exceeds 10 MiB")
|
|
spool.write(chunk)
|
|
spool.seek(0)
|
|
return spool, size
|
|
except Exception:
|
|
spool.close()
|
|
raise
|
|
|
|
|
|
def sha256_file(fileobj):
|
|
digest = hashlib.sha256()
|
|
fileobj.seek(0)
|
|
while True:
|
|
chunk = fileobj.read(1024 * 1024)
|
|
if not chunk:
|
|
break
|
|
digest.update(chunk)
|
|
fileobj.seek(0)
|
|
return digest.hexdigest()
|
|
|
|
|
|
@router.post("/chats/{chat_id}/attachments/presigned", status_code=201)
|
|
def create_attachment_upload(
|
|
chat_id: str,
|
|
payload: CreateAttachmentUpload,
|
|
db: Session = Depends(get_db),
|
|
user: User = Depends(get_current_user),
|
|
):
|
|
try:
|
|
chat_uuid = uuid.UUID(chat_id)
|
|
except ValueError:
|
|
raise HTTPException(400, "Invalid chat UUID") from None
|
|
|
|
if not ChatRepository(db).is_member(chat_uuid, user.id):
|
|
raise HTTPException(404, "Chat not found")
|
|
|
|
if payload.size_bytes > settings.max_upload_bytes:
|
|
raise HTTPException(413, "Image exceeds configured upload limit")
|
|
|
|
upload_id = uuid.uuid4()
|
|
object_key = f"uploads/{chat_uuid}/{upload_id}"
|
|
expires_at = datetime.now(timezone.utc) + timedelta(
|
|
seconds=settings.presigned_upload_ttl_seconds
|
|
)
|
|
|
|
try:
|
|
upload_url = create_presigned_upload_url(
|
|
object_key=object_key,
|
|
content_type=payload.content_type,
|
|
)
|
|
except (BotoCoreError, ClientError):
|
|
raise HTTPException(503, "Object storage unavailable") from None
|
|
|
|
upload = AttachmentUpload(
|
|
id=upload_id,
|
|
chat_id=chat_uuid,
|
|
owner_id=user.id,
|
|
filename=payload.filename,
|
|
content_type=payload.content_type,
|
|
expected_size_bytes=payload.size_bytes,
|
|
expected_sha256=payload.sha256.lower(),
|
|
object_key=object_key,
|
|
status="pending",
|
|
expires_at=expires_at,
|
|
)
|
|
|
|
try:
|
|
db.add(upload)
|
|
db.commit()
|
|
except Exception:
|
|
db.rollback()
|
|
raise HTTPException(503, "Could not save upload metadata") from None
|
|
|
|
return {
|
|
"upload_id": str(upload_id),
|
|
"upload_url": upload_url,
|
|
"method": "PUT",
|
|
"headers": {
|
|
"Content-Type": payload.content_type,
|
|
},
|
|
"expires_in": settings.presigned_upload_ttl_seconds,
|
|
"expires_at": iso_utc(expires_at),
|
|
}
|
|
|
|
|
|
@router.post("/attachments/uploads/{upload_id}/complete", status_code=200)
|
|
def complete_attachment_upload(
|
|
upload_id: str,
|
|
db: Session = Depends(get_db),
|
|
user: User = Depends(get_current_user),
|
|
):
|
|
try:
|
|
parsed_id = uuid.UUID(upload_id)
|
|
except ValueError:
|
|
raise HTTPException(400, "Invalid upload UUID") from None
|
|
|
|
# Lock the upload row to prevent concurrent completion.
|
|
upload = db.scalar(
|
|
select(AttachmentUpload)
|
|
.where(AttachmentUpload.id == parsed_id)
|
|
.with_for_update()
|
|
)
|
|
|
|
if upload is None:
|
|
raise HTTPException(404, "Upload not found")
|
|
|
|
if (
|
|
upload.owner_id != user.id
|
|
or not ChatRepository(db).is_member(upload.chat_id, user.id)
|
|
):
|
|
raise HTTPException(404, "Upload not found")
|
|
|
|
# Idempotent completion: return the existing attachment.
|
|
if upload.status == "completed":
|
|
item = db.scalar(
|
|
select(Attachment).where(
|
|
Attachment.object_key == upload.object_key
|
|
)
|
|
)
|
|
if item is None:
|
|
raise HTTPException(409, "Completed attachment not found")
|
|
return attachment_json(item, db)
|
|
|
|
if upload.status != "pending":
|
|
raise HTTPException(409, "Upload is not pending")
|
|
|
|
now = datetime.now(timezone.utc)
|
|
|
|
if upload.expires_at <= now:
|
|
upload.status = "expired"
|
|
db.commit()
|
|
|
|
try:
|
|
client().delete_object(
|
|
Bucket=settings.s3_bucket,
|
|
Key=upload.object_key,
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
raise HTTPException(409, "Upload has expired")
|
|
|
|
s3 = client()
|
|
|
|
# Read the object from S3.
|
|
try:
|
|
result = s3.get_object(
|
|
Bucket=settings.s3_bucket,
|
|
Key=upload.object_key,
|
|
)
|
|
except ClientError as exc:
|
|
code = exc.response.get("Error", {}).get("Code", "")
|
|
|
|
if code in {"NoSuchKey", "404", "NotFound"}:
|
|
raise HTTPException(409, "Uploaded object not found") from None
|
|
|
|
raise HTTPException(503, "Object storage unavailable") from None
|
|
except BotoCoreError:
|
|
raise HTTPException(503, "Object storage unavailable") from None
|
|
|
|
body = result["Body"]
|
|
spool = tempfile.SpooledTemporaryFile(
|
|
max_size=1024 * 1024,
|
|
mode="w+b",
|
|
)
|
|
|
|
def reject_upload(status_code: int, detail: str):
|
|
upload.status = "failed"
|
|
db.commit()
|
|
|
|
try:
|
|
s3.delete_object(
|
|
Bucket=settings.s3_bucket,
|
|
Key=upload.object_key,
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
raise HTTPException(status_code=status_code, detail=detail)
|
|
|
|
try:
|
|
actual_size = result.get("ContentLength")
|
|
|
|
if actual_size != upload.expected_size_bytes:
|
|
reject_upload(409, "Uploaded file size mismatch")
|
|
|
|
if actual_size > settings.max_upload_bytes:
|
|
reject_upload(413, "Image exceeds configured upload limit")
|
|
|
|
total = 0
|
|
digest = hashlib.sha256()
|
|
|
|
try:
|
|
while True:
|
|
chunk = body.read(64 * 1024)
|
|
if not chunk:
|
|
break
|
|
|
|
total += len(chunk)
|
|
|
|
if total > settings.max_upload_bytes:
|
|
reject_upload(413, "Image exceeds configured upload limit")
|
|
|
|
if total > upload.expected_size_bytes:
|
|
reject_upload(409, "Uploaded file size mismatch")
|
|
|
|
digest.update(chunk)
|
|
spool.write(chunk)
|
|
finally:
|
|
body.close()
|
|
|
|
if total != upload.expected_size_bytes:
|
|
reject_upload(409, "Uploaded file size mismatch")
|
|
|
|
if digest.hexdigest() != upload.expected_sha256:
|
|
reject_upload(409, "Uploaded file checksum mismatch")
|
|
|
|
# Validate the actual image content, not only S3 metadata.
|
|
try:
|
|
actual_type, _, _ = validate_image_file(
|
|
spool,
|
|
upload.content_type,
|
|
)
|
|
except HTTPException as exc:
|
|
reject_upload(exc.status_code, str(exc.detail))
|
|
|
|
attachment = Attachment(
|
|
id=uuid.uuid4(),
|
|
chat_id=upload.chat_id,
|
|
owner_id=upload.owner_id,
|
|
filename=upload.filename,
|
|
content_type=actual_type,
|
|
size_bytes=total,
|
|
sha256=digest.hexdigest(),
|
|
object_key=upload.object_key,
|
|
status="ready",
|
|
error_code=None,
|
|
)
|
|
|
|
upload.status = "completed"
|
|
upload.completed_at = now
|
|
|
|
db.add(attachment)
|
|
db.commit()
|
|
db.refresh(attachment)
|
|
|
|
return attachment_json(attachment, db)
|
|
|
|
except HTTPException:
|
|
raise
|
|
except (BotoCoreError, ClientError):
|
|
db.rollback()
|
|
raise HTTPException(503, "Object storage unavailable") from None
|
|
except Exception:
|
|
db.rollback()
|
|
raise
|
|
finally:
|
|
spool.close()
|
|
|
|
|
|
@router.post("/chats/{chat_id}/attachments", status_code=202)
|
|
async def upload_attachment(
|
|
chat_id: str,
|
|
request: Request,
|
|
filename: str = Query(..., min_length=1, max_length=255),
|
|
db: Session = Depends(get_db),
|
|
user: User = Depends(get_current_user),
|
|
):
|
|
try:
|
|
chat_uuid = uuid.UUID(chat_id)
|
|
except ValueError:
|
|
raise HTTPException(400, "Invalid chat UUID") from None
|
|
|
|
if not ChatRepository(db).is_member(chat_uuid, user.id):
|
|
raise HTTPException(404, "Chat not found")
|
|
|
|
if (
|
|
not filename.strip()
|
|
or any(c in filename for c in ("/", "\\", "\x00"))
|
|
or any(ord(c) < 32 or ord(c) == 127 for c in filename)
|
|
):
|
|
raise HTTPException(400, "Unsafe filename")
|
|
|
|
content_type = (
|
|
request.headers.get("content-type", "")
|
|
.split(";", 1)[0]
|
|
.strip()
|
|
.lower()
|
|
)
|
|
|
|
if content_type not in {"image/png", "image/jpeg"}:
|
|
raise HTTPException(415, "Only PNG and JPEG are supported")
|
|
|
|
spool, size = await read_limited(request)
|
|
|
|
attachment_id = uuid.uuid4()
|
|
job_id = uuid.uuid4()
|
|
|
|
staging_key = f"staging/{chat_uuid}/{attachment_id}"
|
|
object_key = f"original/{chat_uuid}/{attachment_id}"
|
|
|
|
staged = False
|
|
|
|
try:
|
|
actual_type, _, _ = validate_image_file(spool, content_type)
|
|
digest = sha256_file(spool)
|
|
|
|
# Serialize admission checks across all API instances.
|
|
db.execute(
|
|
text("SELECT pg_advisory_xact_lock(:lock_id)"),
|
|
{"lock_id": 72420261110},
|
|
)
|
|
|
|
active_jobs = db.scalar(
|
|
select(func.count())
|
|
.select_from(ProcessingJob)
|
|
.where(ProcessingJob.status.in_(("queued", "processing")))
|
|
) or 0
|
|
|
|
if active_jobs >= settings.max_active_processing_jobs:
|
|
raise HTTPException(
|
|
status_code=429,
|
|
detail="Processing queue is full; retry later",
|
|
headers={"Retry-After": "5"},
|
|
)
|
|
|
|
s3 = client()
|
|
|
|
try:
|
|
spool.seek(0)
|
|
s3.put_object(
|
|
Bucket=settings.s3_bucket,
|
|
Key=staging_key,
|
|
Body=spool,
|
|
ContentType=actual_type,
|
|
)
|
|
staged = True
|
|
except (BotoCoreError, ClientError):
|
|
raise HTTPException(
|
|
503, "Object storage unavailable"
|
|
) from None
|
|
|
|
now = datetime.now(timezone.utc)
|
|
|
|
attachment = Attachment(
|
|
id=attachment_id,
|
|
chat_id=chat_uuid,
|
|
owner_id=user.id,
|
|
filename=filename,
|
|
content_type=actual_type,
|
|
size_bytes=size,
|
|
sha256=digest,
|
|
object_key=object_key,
|
|
staging_key=staging_key,
|
|
status="queued",
|
|
error_code=None,
|
|
pipeline_version=1,
|
|
)
|
|
|
|
job = ProcessingJob(
|
|
id=job_id,
|
|
attachment_id=attachment_id,
|
|
pipeline_version=1,
|
|
attempt=0,
|
|
status="queued",
|
|
available_at=now,
|
|
)
|
|
|
|
# Preserve the current HTTP trace context in the durable outbox event.
|
|
trace_context = {}
|
|
inject(trace_context)
|
|
|
|
event = OutboxEvent(
|
|
event_type="attachment.process.v1",
|
|
aggregate_id=attachment_id,
|
|
payload={
|
|
"schema_version": 1,
|
|
"job_id": str(job_id),
|
|
"type": "attachment.process.v1",
|
|
"attachment_id": str(attachment_id),
|
|
"pipeline_version": 1,
|
|
"attempt": 1,
|
|
"created_at": now.isoformat(),
|
|
"trace_context": trace_context,
|
|
},
|
|
status="pending",
|
|
)
|
|
|
|
db.add_all([attachment, job, event])
|
|
db.commit()
|
|
db.refresh(attachment)
|
|
|
|
return attachment_json(attachment, db)
|
|
|
|
except HTTPException:
|
|
db.rollback()
|
|
if staged:
|
|
try:
|
|
client().delete_object(
|
|
Bucket=settings.s3_bucket,
|
|
Key=staging_key,
|
|
)
|
|
except Exception:
|
|
pass
|
|
raise
|
|
|
|
except Exception:
|
|
db.rollback()
|
|
|
|
if staged:
|
|
try:
|
|
client().delete_object(
|
|
Bucket=settings.s3_bucket,
|
|
Key=staging_key,
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
raise HTTPException(
|
|
503, "Could not save attachment processing job"
|
|
) from None
|
|
|
|
finally:
|
|
spool.close()
|
|
|
|
|
|
@router.get("/attachments/{attachment_id}")
|
|
def get_attachment(
|
|
attachment_id: str,
|
|
db: Session = Depends(get_db),
|
|
user: User = Depends(get_current_user),
|
|
):
|
|
try:
|
|
parsed = uuid.UUID(attachment_id)
|
|
except ValueError:
|
|
raise HTTPException(400, "Invalid attachment UUID") from None
|
|
|
|
item = member_attachment(db, parsed, user.id)
|
|
return attachment_json(item, db)
|
|
|
|
|
|
@router.get("/attachments/{attachment_id}/content")
|
|
def download_attachment(
|
|
attachment_id: str,
|
|
variant: str = "original",
|
|
db: Session = Depends(get_db),
|
|
user: User = Depends(get_current_user),
|
|
):
|
|
try:
|
|
parsed = uuid.UUID(attachment_id)
|
|
except ValueError:
|
|
raise HTTPException(400, "Invalid attachment UUID") from None
|
|
|
|
item = member_attachment(db, parsed, user.id)
|
|
|
|
if item.status != "ready":
|
|
raise HTTPException(409, "Attachment is not ready")
|
|
|
|
# Select the original file or a generated variant.
|
|
if variant == "original":
|
|
object_key = item.object_key
|
|
expected_size = item.size_bytes
|
|
expected_sha256 = item.sha256
|
|
content_type = item.content_type
|
|
else:
|
|
variant_item = db.scalar(
|
|
select(AttachmentVariant).where(
|
|
AttachmentVariant.attachment_id == item.id,
|
|
AttachmentVariant.variant == variant,
|
|
)
|
|
)
|
|
|
|
if variant_item is None:
|
|
raise HTTPException(404, "Variant not found")
|
|
|
|
object_key = variant_item.object_key
|
|
expected_size = variant_item.size_bytes
|
|
expected_sha256 = variant_item.sha256
|
|
content_type = variant_item.content_type
|
|
|
|
spool = tempfile.SpooledTemporaryFile(
|
|
max_size=1024 * 1024,
|
|
mode="w+b",
|
|
)
|
|
|
|
try:
|
|
result = client().get_object(
|
|
Bucket=settings.s3_bucket,
|
|
Key=object_key,
|
|
)
|
|
|
|
body = result["Body"]
|
|
total = 0
|
|
digest = hashlib.sha256()
|
|
|
|
try:
|
|
while True:
|
|
chunk = body.read(64 * 1024)
|
|
|
|
if not chunk:
|
|
break
|
|
|
|
total += len(chunk)
|
|
|
|
if total > MAX_BYTES:
|
|
raise HTTPException(
|
|
503,
|
|
"Stored object exceeds configured limit",
|
|
)
|
|
|
|
digest.update(chunk)
|
|
spool.write(chunk)
|
|
finally:
|
|
body.close()
|
|
|
|
# Verify stored object integrity.
|
|
if (
|
|
total != expected_size
|
|
or digest.hexdigest() != expected_sha256
|
|
):
|
|
raise HTTPException(
|
|
503,
|
|
"Stored object checksum mismatch",
|
|
)
|
|
|
|
spool.seek(0)
|
|
|
|
except (BotoCoreError, ClientError):
|
|
spool.close()
|
|
raise HTTPException(
|
|
503,
|
|
"Object storage unavailable",
|
|
) from None
|
|
|
|
except Exception:
|
|
spool.close()
|
|
raise
|
|
|
|
def stream_verified_file():
|
|
try:
|
|
while True:
|
|
chunk = spool.read(64 * 1024)
|
|
|
|
if not chunk:
|
|
break
|
|
|
|
yield chunk
|
|
finally:
|
|
spool.close()
|
|
|
|
return StreamingResponse(
|
|
stream_verified_file(),
|
|
media_type=content_type,
|
|
headers={
|
|
"Content-Disposition": (
|
|
f"inline; filename*=UTF-8''{quote(item.filename)}"
|
|
)
|
|
},
|
|
)
|