forked from mtusi2026/mtusi2026lab4
165 lines
6.9 KiB
Python
165 lines
6.9 KiB
Python
import hashlib
|
|
import os
|
|
import re
|
|
import time
|
|
import unicodedata
|
|
from tempfile import TemporaryFile
|
|
from threading import BoundedSemaphore
|
|
from uuid import uuid4
|
|
|
|
import boto3
|
|
from botocore.config import Config
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
from PIL import Image
|
|
|
|
from config import positive_int, secret
|
|
from storage import unavailable
|
|
from validation import ApiError, invalid, not_found
|
|
|
|
MIME_FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"}
|
|
S3_ERRORS = (BotoCoreError, ClientError)
|
|
CHUNK = 64 * 1024
|
|
|
|
|
|
class Attachments:
|
|
def __init__(self, store):
|
|
self.store = store
|
|
self.bucket = os.environ["S3_BUCKET"]
|
|
self.max_bytes = min(positive_int("MAX_UPLOAD_BYTES", 10485760), 10485760)
|
|
self.max_pixels = min(positive_int("MAX_IMAGE_PIXELS", 40000000), 40000000)
|
|
self.slots = BoundedSemaphore(2)
|
|
self.s3 = boto3.client(
|
|
"s3", endpoint_url=os.environ["S3_ENDPOINT"], region_name=os.environ["S3_REGION"],
|
|
aws_access_key_id=secret("S3_ACCESS_KEY_ID"), aws_secret_access_key=secret("S3_SECRET_ACCESS_KEY"),
|
|
config=Config(connect_timeout=1, read_timeout=3, retries={"total_max_attempts": 1},
|
|
request_checksum_calculation="when_required", s3={"addressing_style": "path"}),
|
|
)
|
|
|
|
def ready(self):
|
|
try:
|
|
self.s3.head_bucket(Bucket=self.bucket)
|
|
except S3_ERRORS:
|
|
raise unavailable() from None
|
|
|
|
@staticmethod
|
|
def key(attachment_id):
|
|
return f"attachments/{attachment_id}"
|
|
|
|
def upload(self, request, actor, chat_id, filename):
|
|
if (not 1 <= len(filename) <= 255 or filename in (".", "..")
|
|
or any(c in "/\\" or unicodedata.category(c) in ("Cc", "Cs") for c in filename)):
|
|
raise invalid("Invalid filename")
|
|
content_type = request.headers.get_content_type()
|
|
if content_type not in MIME_FORMATS:
|
|
raise ApiError(415, "unsupported_media_type", "Expected PNG or JPEG")
|
|
lengths = request.headers.get_all("Content-Length", [])
|
|
if (request.headers.get("Transfer-Encoding") or len(lengths) != 1
|
|
or not re.fullmatch(r"[0-9]{1,19}", lengths[0])):
|
|
raise invalid("Content-Length required")
|
|
length = int(lengths[0])
|
|
if length > self.max_bytes:
|
|
raise ApiError(413, "payload_too_large", "Attachment is too large")
|
|
if not self.slots.acquire(timeout=2):
|
|
raise unavailable()
|
|
try:
|
|
with TemporaryFile() as file:
|
|
checksum = self.receive(request, file, length)
|
|
self.validate(file, content_type)
|
|
attachment_id = uuid4()
|
|
self.reserve(attachment_id, chat_id, actor, filename, content_type, length, checksum)
|
|
try:
|
|
self.s3.put_object(Bucket=self.bucket, Key=self.key(attachment_id), Body=file,
|
|
ContentLength=length, ContentType=content_type)
|
|
except S3_ERRORS:
|
|
raise unavailable() from None
|
|
with self.store.pool.connection() as db:
|
|
row = db.execute("""
|
|
UPDATE attachments SET ready=true,updated_at=clock_timestamp()
|
|
WHERE id=%s AND NOT ready RETURNING *
|
|
""", (attachment_id,)).fetchone()
|
|
if row is None:
|
|
raise unavailable()
|
|
return self.public(row)
|
|
finally:
|
|
self.slots.release()
|
|
|
|
@staticmethod
|
|
def receive(request, file, length):
|
|
checksum = hashlib.sha256()
|
|
deadline = time.monotonic() + 10
|
|
remaining = length
|
|
while remaining:
|
|
if time.monotonic() > deadline:
|
|
raise ApiError(408, "invalid_request", "Upload timed out")
|
|
chunk = request.rfile.read1(min(CHUNK, remaining))
|
|
if not chunk:
|
|
raise invalid("Incomplete attachment")
|
|
file.write(chunk)
|
|
checksum.update(chunk)
|
|
remaining -= len(chunk)
|
|
file.seek(0)
|
|
return checksum.hexdigest()
|
|
|
|
def validate(self, file, content_type):
|
|
try:
|
|
with Image.open(file, formats=[MIME_FORMATS[content_type]]) as image:
|
|
if image.width * image.height > self.max_pixels:
|
|
raise ApiError(413, "payload_too_large", "Image pixel limit exceeded")
|
|
image.verify()
|
|
file.seek(0)
|
|
with Image.open(file, formats=[MIME_FORMATS[content_type]]) as image:
|
|
image.load()
|
|
except Image.DecompressionBombError:
|
|
raise ApiError(413, "payload_too_large", "Image pixel limit exceeded") from None
|
|
except (OSError, ValueError, SyntaxError, EOFError):
|
|
raise ApiError(415, "unsupported_media_type", "Invalid image") from None
|
|
file.seek(0)
|
|
|
|
def reserve(self, attachment_id, chat_id, actor, filename, content_type, length, checksum):
|
|
with self.store.pool.connection() as db:
|
|
db.execute("""
|
|
INSERT INTO attachments (id,chat_id,owner_id,filename,content_type,size_bytes,sha256)
|
|
VALUES (%s,%s,%s,%s,%s,%s,%s)
|
|
""", (attachment_id, chat_id, actor, filename, content_type, length, checksum))
|
|
|
|
@staticmethod
|
|
def public(row):
|
|
return {**{name: value for name, value in row.items() if name != "ready"},
|
|
"status": "ready", "variants": [], "error_code": None}
|
|
|
|
def get(self, actor, attachment_id):
|
|
with self.store.pool.connection() as db:
|
|
row = db.execute("""
|
|
SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id
|
|
WHERE a.id=%s AND a.ready AND m.user_id=%s
|
|
""", (attachment_id, actor)).fetchone()
|
|
if row is None:
|
|
raise not_found()
|
|
return self.public(row)
|
|
|
|
def download(self, attachment):
|
|
try:
|
|
result = self.s3.get_object(Bucket=self.bucket, Key=self.key(attachment["id"]))
|
|
except S3_ERRORS:
|
|
raise unavailable() from None
|
|
if result["ContentLength"] != attachment["size_bytes"]:
|
|
result["Body"].close()
|
|
raise unavailable()
|
|
return result["Body"]
|
|
|
|
def cleanup(self):
|
|
count = 0
|
|
while True:
|
|
with self.store.pool.connection() as db:
|
|
row = db.execute("""
|
|
SELECT id FROM attachments
|
|
WHERE NOT ready AND created_at < now() - interval '1 hour'
|
|
ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED
|
|
""").fetchone()
|
|
if row is None:
|
|
return count
|
|
attachment_id = row["id"]
|
|
self.s3.delete_object(Bucket=self.bucket, Key=self.key(attachment_id))
|
|
db.execute("DELETE FROM attachments WHERE id=%s", (attachment_id,))
|
|
count += 1
|