"""Bounded S3 operations. No root credentials or filename-derived object keys.""" import contextlib import hashlib import os import tempfile import threading import time import unicodedata import warnings import boto3 from botocore.config import Config from botocore.exceptions import BotoCoreError, ClientError from PIL import Image, PngImagePlugin, UnidentifiedImageError from common import APIError, invalid MAX_BYTES = 10 * 1024 * 1024 MAX_PIXELS = 40_000_000 Image.MAX_IMAGE_PIXELS = MAX_PIXELS PngImagePlugin.MAX_TEXT_CHUNK = 1024 * 1024 PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024 warnings.filterwarnings('error', category=Image.DecompressionBombWarning) UPLOAD_SLOTS = threading.BoundedSemaphore(8) DECODE_SLOTS = threading.BoundedSemaphore(1) def unavailable(): raise APIError(503, 'unavailable', 'Объектное хранилище временно недоступно') def filename(value): if not isinstance(value, str) or not 1 <= len(value) <= 255 or not value.strip(): invalid() if any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in value): invalid() try: value.encode('utf-8') except UnicodeError: invalid() return value @contextlib.contextmanager def bounded_upload(): if not UPLOAD_SLOTS.acquire(blocking=False): raise APIError(503, 'unavailable', 'Все слоты загрузки заняты') try: yield finally: UPLOAD_SLOTS.release() @contextlib.contextmanager def receive(handler): lengths = handler.headers.get_all('Content-Length', []) if len(lengths) != 1 or handler.headers.get('Transfer-Encoding'): invalid() try: length = int(lengths[0]) except ValueError: invalid() if length > MAX_BYTES: raise APIError(413, 'payload_too_large', 'Максимум 10 MiB') if length <= 0: invalid() mime = handler.headers.get_content_type() if mime not in ('image/png', 'image/jpeg'): raise APIError(415, 'unsupported_media_type', 'Требуется PNG или JPEG') deadline = time.monotonic() + 15 digest = hashlib.sha256() # Ephemeral bounded spool is never a durable source of a confirmed attachment. with tempfile.TemporaryFile(mode='w+b') as source: remaining = length while remaining: handler.connection.settimeout(min(3, max(0.01, deadline - time.monotonic()))) if time.monotonic() >= deadline: invalid() try: chunk = handler.rfile.read1(min(64 * 1024, remaining)) except (OSError, TimeoutError): invalid() if not chunk: invalid() digest.update(chunk) source.write(chunk) remaining -= len(chunk) if not DECODE_SLOTS.acquire(timeout=10): raise APIError(503, 'unavailable', 'Проверка изображений перегружена') try: source.flush() for decode in (False,): source.seek(0) # Pillow.close releases pixel buffers and may close its input file. # A duplicate descriptor preserves the original for the S3 PUT. with os.fdopen(os.dup(source.fileno()), 'rb') as image_file: image = Image.open(image_file, formats=['PNG', 'JPEG']) try: if image.width * image.height > MAX_PIXELS: raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей') if Image.MIME[image.format] != mime: raise APIError(415, 'unsupported_media_type', 'MIME не соответствует изображению') if decode: image.load() else: image.verify() finally: image.close() # Free pixels BEFORE releasing the decoder semaphore. except (Image.DecompressionBombWarning, Image.DecompressionBombError): raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей') from None except (UnidentifiedImageError, OSError, ValueError, SyntaxError): raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from None finally: DECODE_SLOTS.release() source.seek(0) yield source, length, mime, digest.hexdigest() class Objects: def __init__(self, setting): self.bucket = os.environ.get('S3_BUCKET', 'messenger-attachments') self.client = boto3.client('s3', endpoint_url=os.environ.get('S3_ENDPOINT', 'http://s3:8333'), region_name='us-east-1', aws_access_key_id=setting('S3_ACCESS_KEY_ID'), aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'), config=Config(signature_version='s3v4', s3={'addressing_style': 'path'}, connect_timeout=1, read_timeout=3, max_pool_connections=16, retries={'total_max_attempts': 1}, request_checksum_calculation='when_required', response_checksum_validation='when_required')) def head(self, key): try: return self.client.head_object(Bucket=self.bucket, Key=key) except ClientError as exc: if exc.response['Error']['Code'] in ('404', 'NoSuchKey', 'NotFound'): return None unavailable() except BotoCoreError: unavailable() def ready(self): try: self.client.head_bucket(Bucket=self.bucket) except (ClientError, BotoCoreError): unavailable() def put(self, key, source, length, mime, digest): try: self.client.put_object(Bucket=self.bucket, Key=key, Body=source, ContentLength=length, ContentType=mime, Metadata={'sha256': digest}) except (ClientError, BotoCoreError): unavailable() def get(self, key): try: return self.client.get_object(Bucket=self.bucket, Key=key) except (ClientError, BotoCoreError): unavailable() def delete(self, key): try: self.client.delete_object(Bucket=self.bucket, Key=key) except (ClientError, BotoCoreError): unavailable()