Files
mtusi2026lab4-fork/src/attachments.py
T

165 lines
6.9 KiB
Python

import hashlib
import os
import re
import time
import unicodedata
from tempfile import TemporaryFile
from threading import BoundedSemaphore
from uuid import uuid4
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from PIL import Image
from config import positive_int, secret
from storage import unavailable
from validation import ApiError, invalid, not_found
MIME_FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"}
S3_ERRORS = (BotoCoreError, ClientError)
CHUNK = 64 * 1024
class Attachments:
def __init__(self, store):
self.store = store
self.bucket = os.environ["S3_BUCKET"]
self.max_bytes = min(positive_int("MAX_UPLOAD_BYTES", 10485760), 10485760)
self.max_pixels = min(positive_int("MAX_IMAGE_PIXELS", 40000000), 40000000)
self.slots = BoundedSemaphore(2)
self.s3 = boto3.client(
"s3", endpoint_url=os.environ["S3_ENDPOINT"], region_name=os.environ["S3_REGION"],
aws_access_key_id=secret("S3_ACCESS_KEY_ID"), aws_secret_access_key=secret("S3_SECRET_ACCESS_KEY"),
config=Config(connect_timeout=1, read_timeout=3, retries={"total_max_attempts": 1},
request_checksum_calculation="when_required", s3={"addressing_style": "path"}),
)
def ready(self):
try:
self.s3.head_bucket(Bucket=self.bucket)
except S3_ERRORS:
raise unavailable() from None
@staticmethod
def key(attachment_id):
return f"attachments/{attachment_id}"
def upload(self, request, actor, chat_id, filename):
if (not 1 <= len(filename) <= 255 or filename in (".", "..")
or any(c in "/\\" or unicodedata.category(c) in ("Cc", "Cs") for c in filename)):
raise invalid("Invalid filename")
content_type = request.headers.get_content_type()
if content_type not in MIME_FORMATS:
raise ApiError(415, "unsupported_media_type", "Expected PNG or JPEG")
lengths = request.headers.get_all("Content-Length", [])
if (request.headers.get("Transfer-Encoding") or len(lengths) != 1
or not re.fullmatch(r"[0-9]{1,19}", lengths[0])):
raise invalid("Content-Length required")
length = int(lengths[0])
if length > self.max_bytes:
raise ApiError(413, "payload_too_large", "Attachment is too large")
if not self.slots.acquire(timeout=2):
raise unavailable()
try:
with TemporaryFile() as file:
checksum = self.receive(request, file, length)
self.validate(file, content_type)
attachment_id = uuid4()
self.reserve(attachment_id, chat_id, actor, filename, content_type, length, checksum)
try:
self.s3.put_object(Bucket=self.bucket, Key=self.key(attachment_id), Body=file,
ContentLength=length, ContentType=content_type)
except S3_ERRORS:
raise unavailable() from None
with self.store.pool.connection() as db:
row = db.execute("""
UPDATE attachments SET ready=true,updated_at=clock_timestamp()
WHERE id=%s AND NOT ready RETURNING *
""", (attachment_id,)).fetchone()
if row is None:
raise unavailable()
return self.public(row)
finally:
self.slots.release()
@staticmethod
def receive(request, file, length):
checksum = hashlib.sha256()
deadline = time.monotonic() + 10
remaining = length
while remaining:
if time.monotonic() > deadline:
raise ApiError(408, "invalid_request", "Upload timed out")
chunk = request.rfile.read1(min(CHUNK, remaining))
if not chunk:
raise invalid("Incomplete attachment")
file.write(chunk)
checksum.update(chunk)
remaining -= len(chunk)
file.seek(0)
return checksum.hexdigest()
def validate(self, file, content_type):
try:
with Image.open(file, formats=[MIME_FORMATS[content_type]]) as image:
if image.width * image.height > self.max_pixels:
raise ApiError(413, "payload_too_large", "Image pixel limit exceeded")
image.verify()
file.seek(0)
with Image.open(file, formats=[MIME_FORMATS[content_type]]) as image:
image.load()
except Image.DecompressionBombError:
raise ApiError(413, "payload_too_large", "Image pixel limit exceeded") from None
except (OSError, ValueError, SyntaxError, EOFError):
raise ApiError(415, "unsupported_media_type", "Invalid image") from None
file.seek(0)
def reserve(self, attachment_id, chat_id, actor, filename, content_type, length, checksum):
with self.store.pool.connection() as db:
db.execute("""
INSERT INTO attachments (id,chat_id,owner_id,filename,content_type,size_bytes,sha256)
VALUES (%s,%s,%s,%s,%s,%s,%s)
""", (attachment_id, chat_id, actor, filename, content_type, length, checksum))
@staticmethod
def public(row):
return {**{name: value for name, value in row.items() if name != "ready"},
"status": "ready", "variants": [], "error_code": None}
def get(self, actor, attachment_id):
with self.store.pool.connection() as db:
row = db.execute("""
SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id
WHERE a.id=%s AND a.ready AND m.user_id=%s
""", (attachment_id, actor)).fetchone()
if row is None:
raise not_found()
return self.public(row)
def download(self, attachment):
try:
result = self.s3.get_object(Bucket=self.bucket, Key=self.key(attachment["id"]))
except S3_ERRORS:
raise unavailable() from None
if result["ContentLength"] != attachment["size_bytes"]:
result["Body"].close()
raise unavailable()
return result["Body"]
def cleanup(self):
count = 0
while True:
with self.store.pool.connection() as db:
row = db.execute("""
SELECT id FROM attachments
WHERE NOT ready AND created_at < now() - interval '1 hour'
ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED
""").fetchone()
if row is None:
return count
attachment_id = row["id"]
self.s3.delete_object(Bucket=self.bucket, Key=self.key(attachment_id))
db.execute("DELETE FROM attachments WHERE id=%s", (attachment_id,))
count += 1