157 lines
6.2 KiB
Python
157 lines
6.2 KiB
Python
"""Bounded S3 operations. No root credentials or filename-derived object keys."""
|
|
import contextlib
|
|
import hashlib
|
|
import os
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unicodedata
|
|
import warnings
|
|
|
|
import boto3
|
|
from botocore.config import Config
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
from PIL import Image, PngImagePlugin, UnidentifiedImageError
|
|
|
|
from common import APIError, invalid
|
|
|
|
MAX_BYTES = 10 * 1024 * 1024
|
|
MAX_PIXELS = 40_000_000
|
|
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
|
PngImagePlugin.MAX_TEXT_CHUNK = 1024 * 1024
|
|
PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024
|
|
warnings.filterwarnings('error', category=Image.DecompressionBombWarning)
|
|
UPLOAD_SLOTS = threading.BoundedSemaphore(8)
|
|
DECODE_SLOTS = threading.BoundedSemaphore(1)
|
|
|
|
|
|
def unavailable():
|
|
raise APIError(503, 'unavailable', 'Объектное хранилище временно недоступно')
|
|
|
|
|
|
def filename(value):
|
|
if not isinstance(value, str) or not 1 <= len(value) <= 255 or not value.strip():
|
|
invalid()
|
|
if any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in value):
|
|
invalid()
|
|
try:
|
|
value.encode('utf-8')
|
|
except UnicodeError:
|
|
invalid()
|
|
return value
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def bounded_upload():
|
|
if not UPLOAD_SLOTS.acquire(blocking=False):
|
|
raise APIError(503, 'unavailable', 'Все слоты загрузки заняты')
|
|
try:
|
|
yield
|
|
finally:
|
|
UPLOAD_SLOTS.release()
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def receive(handler):
|
|
lengths = handler.headers.get_all('Content-Length', [])
|
|
if len(lengths) != 1 or handler.headers.get('Transfer-Encoding'):
|
|
invalid()
|
|
try:
|
|
length = int(lengths[0])
|
|
except ValueError:
|
|
invalid()
|
|
if length > MAX_BYTES:
|
|
raise APIError(413, 'payload_too_large', 'Максимум 10 MiB')
|
|
if length <= 0:
|
|
invalid()
|
|
mime = handler.headers.get_content_type()
|
|
if mime not in ('image/png', 'image/jpeg'):
|
|
raise APIError(415, 'unsupported_media_type', 'Требуется PNG или JPEG')
|
|
deadline = time.monotonic() + 15
|
|
digest = hashlib.sha256()
|
|
# Ephemeral bounded spool is never a durable source of a confirmed attachment.
|
|
with tempfile.TemporaryFile(mode='w+b') as source:
|
|
remaining = length
|
|
while remaining:
|
|
handler.connection.settimeout(min(3, max(0.01, deadline - time.monotonic())))
|
|
if time.monotonic() >= deadline:
|
|
invalid()
|
|
try:
|
|
chunk = handler.rfile.read1(min(64 * 1024, remaining))
|
|
except (OSError, TimeoutError):
|
|
invalid()
|
|
if not chunk:
|
|
invalid()
|
|
digest.update(chunk)
|
|
source.write(chunk)
|
|
remaining -= len(chunk)
|
|
if not DECODE_SLOTS.acquire(timeout=10):
|
|
raise APIError(503, 'unavailable', 'Проверка изображений перегружена')
|
|
try:
|
|
source.flush()
|
|
for decode in (False,):
|
|
source.seek(0)
|
|
# Pillow.close releases pixel buffers and may close its input file.
|
|
# A duplicate descriptor preserves the original for the S3 PUT.
|
|
with os.fdopen(os.dup(source.fileno()), 'rb') as image_file:
|
|
image = Image.open(image_file, formats=['PNG', 'JPEG'])
|
|
try:
|
|
if image.width * image.height > MAX_PIXELS:
|
|
raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей')
|
|
if Image.MIME[image.format] != mime:
|
|
raise APIError(415, 'unsupported_media_type', 'MIME не соответствует изображению')
|
|
if decode:
|
|
image.load()
|
|
else:
|
|
image.verify()
|
|
finally:
|
|
image.close() # Free pixels BEFORE releasing the decoder semaphore.
|
|
except (Image.DecompressionBombWarning, Image.DecompressionBombError):
|
|
raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей') from None
|
|
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
|
|
raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from None
|
|
finally:
|
|
DECODE_SLOTS.release()
|
|
source.seek(0)
|
|
yield source, length, mime, digest.hexdigest()
|
|
|
|
|
|
class Objects:
|
|
def __init__(self, setting):
|
|
self.bucket = os.environ.get('S3_BUCKET', 'messenger-attachments')
|
|
self.client = boto3.client('s3', endpoint_url=os.environ.get('S3_ENDPOINT', 'http://s3:8333'), region_name='us-east-1', aws_access_key_id=setting('S3_ACCESS_KEY_ID'), aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'), config=Config(signature_version='s3v4', s3={'addressing_style': 'path'}, connect_timeout=1, read_timeout=3, max_pool_connections=16, retries={'total_max_attempts': 1}, request_checksum_calculation='when_required', response_checksum_validation='when_required'))
|
|
|
|
def head(self, key):
|
|
try:
|
|
return self.client.head_object(Bucket=self.bucket, Key=key)
|
|
except ClientError as exc:
|
|
if exc.response['Error']['Code'] in ('404', 'NoSuchKey', 'NotFound'):
|
|
return None
|
|
unavailable()
|
|
except BotoCoreError:
|
|
unavailable()
|
|
|
|
def ready(self):
|
|
try:
|
|
self.client.head_bucket(Bucket=self.bucket)
|
|
except (ClientError, BotoCoreError):
|
|
unavailable()
|
|
|
|
def put(self, key, source, length, mime, digest):
|
|
try:
|
|
self.client.put_object(Bucket=self.bucket, Key=key, Body=source, ContentLength=length, ContentType=mime, Metadata={'sha256': digest})
|
|
except (ClientError, BotoCoreError):
|
|
unavailable()
|
|
|
|
def get(self, key):
|
|
try:
|
|
return self.client.get_object(Bucket=self.bucket, Key=key)
|
|
except (ClientError, BotoCoreError):
|
|
unavailable()
|
|
|
|
def delete(self, key):
|
|
try:
|
|
self.client.delete_object(Bucket=self.bucket, Key=key)
|
|
except (ClientError, BotoCoreError):
|
|
unavailable()
|