From 959eee7f9d4821fc606f737fd079b161aaf0cf56 Mon Sep 17 00:00:00 2001 From: da1zy3r Date: Fri, 30 Oct 2026 20:08:17 +0300 Subject: [PATCH] Grade 4 --- .env.example | 28 +++-------- Makefile | 5 +- compose.yaml | 63 +++++++++++++++++++++++ infra/nginx.conf | 13 +++-- infra/readiness.sh | 12 +++-- infra/s3-bootstrap.sh | 9 ++++ infra/s3-policy.json | 15 ++++++ migrations/004_attachments.sql | 18 +++++++ requirements.txt | 2 + scripts/restart.py | 4 +- scripts/upload_load.py | 71 ++++++++++++++++++++++++++ src/attachments.py | 82 ++++++++++++++++++++++++++++++ src/cache.py | 2 +- src/cleanup.py | 61 ++++++++++++++++++++++ src/limits.py | 4 ++ src/main.py | 5 +- src/middleware.py | 6 +++ src/models.py | 18 ++++++- src/object_storage.py | 23 +++++++++ src/routers/attachments.py | 84 +++++++++++++++++++++++++++++++ src/routers/chats.py | 2 +- src/routers/health.py | 2 + src/schemas/message.py | 24 ++++++--- src/store.py | 16 ++++-- student-tests/test_attachments.py | 39 ++++++++++++++ 25 files changed, 562 insertions(+), 46 deletions(-) create mode 100644 infra/s3-bootstrap.sh create mode 100644 infra/s3-policy.json create mode 100644 migrations/004_attachments.sql create mode 100644 scripts/upload_load.py create mode 100644 src/attachments.py create mode 100644 src/cleanup.py create mode 100644 src/object_storage.py create mode 100644 src/routers/attachments.py create mode 100644 student-tests/test_attachments.py diff --git a/.env.example b/.env.example index 0fe1de5..d2f1aef 100644 --- a/.env.example +++ b/.env.example @@ -1,29 +1,17 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 +# Copy to .env and choose local credentials. Never commit .env. APP_PORT=8080 -APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Settings used by the inherited lab3 application and compose.yaml. POSTGRES_DB=messenger POSTGRES_USER=messenger POSTGRES_PASSWORD=replace-with-local-password +SESSION_TTL_SECONDS=3600 CACHE_TTL_SECONDS=30 MAX_REQUEST_BYTES=65536 MAX_INFLIGHT=32 -# Compose sets POSTGRES_HOST=postgres and REDIS_URL=redis://redis:6379/0. -# S3 settings below belong to the next implementation stage. -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +MINIO_ROOT_USER=local-admin +MINIO_ROOT_PASSWORD=replace-with-admin-password +S3_ACCESS_KEY_ID=messenger-api +S3_SECRET_ACCESS_KEY=replace-with-api-password MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +# Compose sets internal PostgreSQL, Redis and S3 addresses. +# S3 bucket: messenger-attachments. Root credentials go only to s3/s3-init. diff --git a/Makefile b/Makefile index 5d86d17..fe46f96 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load upload-load test-persistence test-failover help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -21,6 +21,9 @@ test: load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) +upload-load: + $(PYTHON) scripts/upload_load.py --base-url "$(BASE_URL)" + test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/compose.yaml b/compose.yaml index 0f38835..60daad1 100644 --- a/compose.yaml +++ b/compose.yaml @@ -16,6 +16,13 @@ x-api: &api CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-30} MAX_REQUEST_BYTES: ${MAX_REQUEST_BYTES:-65536} MAX_INFLIGHT: ${MAX_INFLIGHT:-32} + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} networks: [backend] cpus: 1.0 mem_limit: 512m @@ -31,6 +38,8 @@ services: condition: service_completed_successfully redis: condition: service_healthy + s3-init: + condition: service_completed_successfully healthcheck: test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1).read()"] interval: 2s @@ -48,6 +57,59 @@ services: postgres: condition: service_healthy + s3: + # RELEASE.2025-09-07T16-13-09Z, pinned by immutable image digest. + image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + command: [server, /data, --console-address, '127.0.0.1:9001'] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env} + MINIO_BROWSER: 'off' + volumes: + - s3_data:/data + networks: [backend] + healthcheck: + test: [CMD, curl, -f, http://127.0.0.1:9000/minio/health/live] + interval: 2s + timeout: 2s + retries: 30 + cpus: 1.0 + mem_limit: 512m + + s3-init: + # The pinned server image also contains mc; no second image is needed. + image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + entrypoint: [sh, /bootstrap.sh] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} + volumes: + - ./infra/s3-bootstrap.sh:/bootstrap.sh:ro + - ./infra/s3-policy.json:/policy.json:ro + depends_on: + s3: + condition: service_healthy + networks: [backend] + restart: 'no' + cpus: 0.25 + mem_limit: 128m + + cleanup: + <<: *api + command: [python, -m, src.cleanup] + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + restart: unless-stopped + cpus: 0.25 + mem_limit: 128m + nginx: image: nginx:1.28.0-alpine entrypoint: [sh, /etc/nginx/readiness.sh] @@ -110,3 +172,4 @@ networks: volumes: postgres_data: redis_data: + s3_data: diff --git a/infra/nginx.conf b/infra/nginx.conf index 4dee29a..1cc308b 100644 --- a/infra/nginx.conf +++ b/infra/nginx.conf @@ -11,19 +11,24 @@ http { access_log /var/log/nginx/access.log upstream; server { listen 8080; - client_max_body_size 64k; + # API enforces the byte limit and returns the contracted JSON error. + client_max_body_size 0; client_body_timeout 5s; client_header_timeout 5s; keepalive_timeout 5s; location / { proxy_pass http://messenger; proxy_http_version 1.1; + proxy_request_buffering off; + proxy_buffering off; proxy_set_header Host $host; proxy_set_header Connection close; proxy_connect_timeout 1s; - proxy_send_timeout 6s; - proxy_read_timeout 6s; - proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_send_timeout 35s; + proxy_read_timeout 120s; + # A 503 is an API response (for example, S3 is unavailable), so + # preserve it for the client instead of marking every replica down. + proxy_next_upstream error timeout http_502 http_504; proxy_next_upstream_tries 2; proxy_next_upstream_timeout 8s; # Never transparently retry a POST already sent to an API. diff --git a/infra/readiness.sh b/infra/readiness.sh index 5bb2563..9a2e627 100644 --- a/infra/readiness.sh +++ b/infra/readiness.sh @@ -5,12 +5,18 @@ update_backends() { : > "$next" for address in $(getent ahostsv4 api | awk '{print $1}' | sort -u); do if wget -q -T 1 -O /dev/null "http://$address:8080/health/ready"; then - echo "server $address:8080 max_fails=1 fail_timeout=2s;" >> "$next" + echo "server $address:8080;" >> "$next" fi done - # An empty upstream is invalid; a down placeholder returns 502 without - # sending business requests to an unready API. + # If every running API temporarily loses a shared dependency (for example + # S3), retain the last addresses. The API can then return the contracted + # 503 instead of Nginx replacing it with a 502. At initial startup there + # is no previous healthy set, so use an inert placeholder. if [ ! -s "$next" ]; then + if [ -s /etc/nginx/ready-backends.conf ] && ! grep -q '127.0.0.1:9 down' /etc/nginx/ready-backends.conf; then + rm -f "$next" + return + fi echo 'server 127.0.0.1:9 down;' > "$next" fi if ! cmp -s "$next" /etc/nginx/ready-backends.conf; then diff --git a/infra/s3-bootstrap.sh b/infra/s3-bootstrap.sh new file mode 100644 index 0000000..c0baf62 --- /dev/null +++ b/infra/s3-bootstrap.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +test "$MINIO_ROOT_USER" != "$S3_ACCESS_KEY_ID" +mc alias set local http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null +mc mb --ignore-existing local/messenger-attachments +mc anonymous set none local/messenger-attachments +mc admin user add local "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null +mc admin policy info local attachments >/dev/null 2>&1 || mc admin policy create local attachments /policy.json +mc admin policy attach local attachments --user "$S3_ACCESS_KEY_ID" diff --git a/infra/s3-policy.json b/infra/s3-policy.json new file mode 100644 index 0000000..41bcff8 --- /dev/null +++ b/infra/s3-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:ListBucket", "s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/originals/*"] + } + ] +} diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 0000000..77317cf --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,18 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users(id), + object_key text NOT NULL UNIQUE, + filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255), + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX attachments_chat_idx ON attachments(chat_id); + +ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}'; +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CHECK (char_length(text) <= 4000); +ALTER TABLE messages ADD CHECK (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0); +ALTER TABLE messages ADD CHECK (cardinality(attachment_ids) <= 10); diff --git a/requirements.txt b/requirements.txt index 7702e0b..86eff95 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,5 @@ psycopg[binary]==3.2.6 SQLAlchemy==2.0.41 redis==5.2.1 argon2-cffi==23.1.0 +boto3==1.40.70 +Pillow==12.0.0 diff --git a/scripts/restart.py b/scripts/restart.py index 8933193..14932b1 100644 --- a/scripts/restart.py +++ b/scripts/restart.py @@ -15,12 +15,12 @@ def snapshot(): def main(): before = snapshot() - subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis', 'nginx'], + subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis', 's3', 'cleanup', 'nginx'], cwd=ROOT, check=True, timeout=60) after = snapshot() (ROOT / 'evidence').mkdir(exist_ok=True) (ROOT / 'evidence' / 'restart.txt').write_text( - 'Command: docker compose restart api postgres redis nginx\nBefore (UTC):\n' + before + 'Command: docker compose restart api postgres redis s3 cleanup nginx\nBefore (UTC):\n' + before + 'After (UTC):\n' + after, encoding='utf-8') diff --git a/scripts/upload_load.py b/scripts/upload_load.py new file mode 100644 index 0000000..98ae701 --- /dev/null +++ b/scripts/upload_load.py @@ -0,0 +1,71 @@ +"""Upload one fixture concurrently and record the peak RSS of API containers.""" +import argparse +import json +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + + +def api_memory_bytes(): + text = subprocess.check_output( + ['docker', 'compose', 'stats', '--no-stream', '--format', '{{.Name}} {{.MemUsage}}', 'api'], + cwd=ROOT, text=True, + ) + values = [] + for line in text.splitlines(): + amount = line.split()[1] + number, unit = float(amount[:-3]), amount[-3:].lower() + values.append(int(number * {'b': 1, 'kib': 1024, 'mib': 1024**2, 'gib': 1024**3}[unit])) + return sum(values) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--concurrency', type=int, default=10) + parser.add_argument('--output', default='evidence/upload-load.json') + args = parser.parse_args() + check(args.concurrency >= 10, 'concurrency must be at least 10') + client = Client(args.base_url) + alice, _, _, chat = client.bootstrap() + content = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes() + start = threading.Event() + done = threading.Event() + peak = [0] + + def monitor(): + while not done.is_set(): + peak[0] = max(peak[0], api_memory_bytes()) + time.sleep(.2) + + def upload(index): + start.wait() + return Client(args.base_url, timeout=60).call( + 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=load-%d.png' % index, + 201, actor=alice, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment') + + watcher = threading.Thread(target=monitor, daemon=True) + watcher.start() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + jobs = [pool.submit(upload, item) for item in range(args.concurrency)] + start.set() + results = [job.result() for job in jobs] + done.set() + watcher.join(timeout=1) + report = {'concurrency': args.concurrency, 'fixture_bytes': len(content), + 'successful_uploads': len(results), 'peak_api_memory_bytes': peak[0]} + destination = ROOT / args.output + destination.parent.mkdir(exist_ok=True) + destination.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + print(json.dumps(report, ensure_ascii=False)) + + +if __name__ == '__main__': + main() diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 0000000..9f86243 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,82 @@ +import os +import unicodedata +from uuid import uuid4 + +from PIL import Image, UnidentifiedImageError +from sqlalchemy import func, select + +from .errors import APIError +from .models import Attachment, ChatMember +from .object_storage import BUCKET, PREFIX, s3_client +from .store import ensure_uuid, serialize, store + +MAX_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760) +MAX_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000) +if MAX_BYTES < 1 or MAX_PIXELS < 1: + raise ValueError('Upload limits must be positive') + + +def validate_filename(filename): + if (not filename or not filename.strip() or len(filename) > 255 + or any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in filename)): + raise APIError(400, 'invalid_request', 'Неверное имя файла') + + +def validate_image(source, content_type): + try: + source.seek(0) + with Image.open(source, formats=['PNG', 'JPEG']) as picture: + if picture.width * picture.height > MAX_PIXELS: + raise APIError(413, 'payload_too_large', 'Слишком много пикселей') + if Image.MIME[picture.format] != content_type: + raise APIError(415, 'unsupported_media_type', 'Тип изображения не совпадает с Content-Type') + picture.verify() + source.seek(0) + with Image.open(source, formats=['PNG', 'JPEG']) as picture: + picture.load() # verify alone does not decode JPEG or detect truncated pixel data. + except Image.DecompressionBombError as exc: + raise APIError(413, 'payload_too_large', 'Слишком много пикселей') from exc + except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc: + raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from exc + source.seek(0) + + +def lock_object(session, key): + # Shared by upload and cleanup: no deletion while a COMMIT is in flight. + session.execute(select(func.pg_advisory_xact_lock(func.hashtextextended(key, 0)))) + + +def metadata(attachment): + result = serialize(attachment, 'id', 'chat_id', 'owner_id', 'filename', + 'content_type', 'size_bytes', 'sha256', 'created_at') + return {**result, 'updated_at': result['created_at'], 'status': 'ready', + 'variants': [], 'error_code': None} + + +def save_attachment(source, chat_id, actor_id, filename, content_type, size, digest): + validate_image(source, content_type) + attachment_id = str(uuid4()) + key = PREFIX + attachment_id + with store.sessions.begin() as session: + store._chat(session, chat_id, actor_id) + lock_object(session, key) + s3_client().put_object(Bucket=BUCKET, Key=key, Body=source, + ContentLength=size, ContentType=content_type) + attachment = Attachment(id=attachment_id, chat_id=chat_id, owner_id=actor_id, + object_key=key, filename=filename, content_type=content_type, + size_bytes=size, sha256=digest) + session.add(attachment) + session.flush() + result = metadata(attachment) + # An uncertain COMMIT is not followed by DELETE: cleanup checks the DB later. + return result + + +def get_attachment(attachment_id, actor_id): + with store.sessions() as session: + attachment = session.scalar(select(Attachment).join( + ChatMember, ChatMember.chat_id == Attachment.chat_id).where( + Attachment.id == ensure_uuid(attachment_id), ChatMember.user_id == actor_id)) + if attachment is None: + raise APIError(404, 'not_found', 'Объект не найден') + return attachment diff --git a/src/cache.py b/src/cache.py index af851c2..bde2077 100644 --- a/src/cache.py +++ b/src/cache.py @@ -37,4 +37,4 @@ class HistoryCache: return page -history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30'))) +history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30')), prefix='history-v4') diff --git a/src/cleanup.py b/src/cleanup.py new file mode 100644 index 0000000..ac1297e --- /dev/null +++ b/src/cleanup.py @@ -0,0 +1,61 @@ +"""Remove old S3 objects without committed metadata. Never delete on a DB error.""" +import argparse +import logging +import time +from datetime import datetime, timedelta, timezone + +from sqlalchemy import select +from sqlalchemy.orm import sessionmaker + +from .attachments import lock_object +from .database import create_db_engine +from .models import Attachment +from .object_storage import BUCKET, PREFIX, s3_client + +GRACE = timedelta(hours=1) + + +def cleanup(sessions, client, now=None): + cutoff = (now or datetime.now(timezone.utc)) - GRACE + deleted = 0 + for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX): + for obj in page.get('Contents', []): + if obj['LastModified'] >= cutoff: + continue + key = obj['Key'] + with sessions.begin() as session: + lock_object(session, key) + if session.scalar(select(Attachment.id).where(Attachment.object_key == key)): + continue + # Recheck age after taking the same lock as the uploader. + if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff: + continue + client.delete_object(Bucket=BUCKET, Key=key) + deleted += 1 + return deleted + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--once', action='store_true') + args = parser.parse_args() + logging.basicConfig(level=logging.INFO) + engine = create_db_engine() + try: + while True: + try: + count = cleanup(sessionmaker(engine), s3_client()) + logging.info('Cleanup completed: deleted=%d', count) + except Exception as exc: + logging.error('Cleanup deferred: %s', type(exc).__name__) + if args.once: + raise SystemExit(1) from None + if args.once: + break + time.sleep(3600) + finally: + engine.dispose() + + +if __name__ == '__main__': + main() diff --git a/src/limits.py b/src/limits.py index e4f3b8a..facdf11 100644 --- a/src/limits.py +++ b/src/limits.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from starlette.requests import Request from .errors import error_response +from .middleware import is_upload @dataclass(frozen=True) @@ -54,6 +55,9 @@ class ResourceLimitMiddleware: return await reject(503, 'unavailable', 'Слишком много одновременных запросов') self.active += 1 try: + # The upload endpoint authenticates and bounds its stream itself. + if is_upload(scope['method'], scope['path']): + return await self.app(scope, receive, send) length = request.headers.get('content-length') if length is not None: try: diff --git a/src/main.py b/src/main.py index 298c7cc..fc96b18 100644 --- a/src/main.py +++ b/src/main.py @@ -5,6 +5,7 @@ from contextlib import asynccontextmanager from sqlalchemy.exc import OperationalError, InterfaceError, TimeoutError as PoolTimeout from sqlalchemy.orm import sessionmaker from redis.exceptions import RedisError +from botocore.exceptions import BotoCoreError, ClientError from .errors import APIError, api_error_handler, validation_error_handler, http_error_handler, internal_error_handler, dependency_error_handler from .database import create_db_engine @@ -16,6 +17,7 @@ from .routers.health import router as health_router from .routers.users import router as users_router from .routers.chats import router as chats_router from .routers.auth import router as auth_router +from .routers.attachments import router as attachments_router @asynccontextmanager @@ -36,10 +38,11 @@ app.add_exception_handler(APIError, api_error_handler) app.add_exception_handler(RequestValidationError, validation_error_handler) app.add_exception_handler(HTTPException, http_error_handler) app.add_exception_handler(Exception, internal_error_handler) -for error in (OperationalError, InterfaceError, PoolTimeout, RedisError): +for error in (OperationalError, InterfaceError, PoolTimeout, RedisError, BotoCoreError, ClientError): app.add_exception_handler(error, dependency_error_handler) app.include_router(health_router) app.include_router(users_router) app.include_router(chats_router) app.include_router(auth_router) +app.include_router(attachments_router) diff --git a/src/middleware.py b/src/middleware.py index 530eeb9..c2b73a1 100644 --- a/src/middleware.py +++ b/src/middleware.py @@ -3,6 +3,11 @@ from socket import gethostname from starlette.responses import JSONResponse from starlette.middleware.base import BaseHTTPMiddleware from starlette.requests import Request +import re + + +def is_upload(method, path): + return method == 'POST' and re.fullmatch(r'/api/v1/chats/[^/]+/attachments', path) is not None INSTANCE_ID = gethostname() @@ -14,6 +19,7 @@ class RequestIdMiddleware(BaseHTTPMiddleware): if (request.method in {"POST", "PUT", "PATCH"} and request.url.path.startswith("/api/") + and not is_upload(request.method, request.url.path) and request.url.path != '/api/v1/auth/sessions'): content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() if content_type != "application/json": diff --git a/src/models.py b/src/models.py index 45b3266..7e1582b 100644 --- a/src/models.py +++ b/src/models.py @@ -3,6 +3,7 @@ from uuid import uuid4 from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, func from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column +from sqlalchemy.dialects.postgresql import ARRAY class Base(DeclarativeBase): @@ -44,13 +45,15 @@ class ChatMember(Base): class Message(Base): __tablename__ = 'messages' __table_args__ = ( - CheckConstraint('char_length(text) BETWEEN 1 AND 4000'), + CheckConstraint('char_length(text) <= 4000'), + CheckConstraint("length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0"), Index('messages_history_idx', 'chat_id', 'created_at', 'id'), ) id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE')) sender_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id')) text: Mapped[str] = mapped_column(Text) + attachment_ids: Mapped[list[str]] = mapped_column(ARRAY(Uuid(as_uuid=False)), default=list, server_default='{}') created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) @@ -60,6 +63,19 @@ class SchemaMigration(Base): applied_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=True) +class Attachment(Base): + __tablename__ = 'attachments' + id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True) + chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE')) + owner_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id')) + object_key: Mapped[str] = mapped_column(Text, unique=True) + filename: Mapped[str] = mapped_column(Text) + content_type: Mapped[str] = mapped_column(Text) + size_bytes: Mapped[int] = mapped_column(BigInteger) + sha256: Mapped[str] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + + class IdempotencyRecord(Base): __tablename__ = 'message_idempotency' actor_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True) diff --git a/src/object_storage.py b/src/object_storage.py new file mode 100644 index 0000000..d3cb252 --- /dev/null +++ b/src/object_storage.py @@ -0,0 +1,23 @@ +import os +from functools import lru_cache + +import boto3 +from botocore.config import Config + +BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments') +PREFIX = 'originals/' + + +@lru_cache(maxsize=1) +def s3_client(): + return boto3.client( + 's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'), + region_name=os.getenv('S3_REGION', 'us-east-1'), + aws_access_key_id=os.environ['S3_ACCESS_KEY_ID'], + aws_secret_access_key=os.environ['S3_SECRET_ACCESS_KEY'], + config=Config(connect_timeout=2, read_timeout=5, + retries={'total_max_attempts': 1}, + s3={'addressing_style': 'path'}, + request_checksum_calculation='when_required', + response_checksum_validation='when_required'), + ) diff --git a/src/routers/attachments.py b/src/routers/attachments.py new file mode 100644 index 0000000..57007b6 --- /dev/null +++ b/src/routers/attachments.py @@ -0,0 +1,84 @@ +import asyncio +import hashlib +from tempfile import TemporaryFile +from urllib.parse import quote + +from fastapi import APIRouter, Depends, Query, Request +from starlette.background import BackgroundTask +from starlette.concurrency import run_in_threadpool +from starlette.requests import ClientDisconnect +from starlette.responses import StreamingResponse + +from ..attachments import MAX_BYTES, get_attachment, metadata, save_attachment, validate_filename +from ..dependencies import current_user +from ..errors import APIError +from ..object_storage import BUCKET, s3_client +from ..store import ensure_uuid, store + +router = APIRouter(prefix='/api/v1', tags=['attachments']) +# At most one decoded image (<=40M pixels) per API process; incoming bodies use disk. +decode_slot = asyncio.Semaphore(1) + + +@router.post('/chats/{chat_id}/attachments', status_code=201) +async def upload(chat_id: str, request: Request, filename: str = Query(...), + user: dict = Depends(current_user)): + chat_id = ensure_uuid(chat_id) + await run_in_threadpool(store.get_chat_for_user, chat_id, user['id']) + validate_filename(filename) + content_type = request.headers.get('content-type', '').split(';', 1)[0].strip().lower() + if content_type not in ('image/png', 'image/jpeg'): + raise APIError(415, 'unsupported_media_type', 'Ожидается PNG или JPEG') + length = request.headers.get('content-length') + if length is not None: + try: + length = int(length) + if length < 0: + raise ValueError() + except ValueError as exc: + raise APIError(400, 'invalid_request', 'Неверный Content-Length') from exc + if length > MAX_BYTES: + raise APIError(413, 'payload_too_large', 'Файл слишком большой') + size = 0 + digest = hashlib.sha256() + with TemporaryFile() as source: + try: + async with asyncio.timeout(30): + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BYTES: + raise APIError(413, 'payload_too_large', 'Файл слишком большой') + digest.update(chunk) + await run_in_threadpool(source.write, chunk) + except (TimeoutError, ClientDisconnect) as exc: + raise APIError(400, 'invalid_request', 'Загрузка прервана') from exc + if length is not None and size != length: + raise APIError(400, 'invalid_request', 'Неполное тело запроса') + async with decode_slot: + return await run_in_threadpool(save_attachment, source, chat_id, user['id'], + filename, content_type, size, digest.hexdigest()) + + +@router.get('/attachments/{attachment_id}') +def attachment_info(attachment_id: str, user: dict = Depends(current_user)): + return metadata(get_attachment(attachment_id, user['id'])) + + +@router.get('/attachments/{attachment_id}/content') +def download(attachment_id: str, variant: str = 'original', user: dict = Depends(current_user)): + attachment = get_attachment(attachment_id, user['id']) + if variant != 'original': + raise APIError(404, 'not_found', 'Вариант не найден') + body = s3_client().get_object(Bucket=BUCKET, Key=attachment.object_key)['Body'] + + def chunks(): + try: + yield from body.iter_chunks(chunk_size=64 * 1024) + finally: + body.close() + + return StreamingResponse(chunks(), media_type=attachment.content_type, + headers={'Content-Length': str(attachment.size_bytes), + 'Content-Disposition': "attachment; filename*=UTF-8''" + quote(attachment.filename, safe=''), + 'X-Content-Type-Options': 'nosniff'}, + background=BackgroundTask(body.close)) diff --git a/src/routers/chats.py b/src/routers/chats.py index 9604e94..58aa1c7 100644 --- a/src/routers/chats.py +++ b/src/routers/chats.py @@ -36,7 +36,7 @@ def create_message(chat_id: str, payload: CreateMessage, user: dict = Depends(cu idempotency_key: Optional[str] = Header(default=None)): if not idempotency_key or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', idempotency_key): raise APIError(400, 'invalid_request', 'Требуется корректный Idempotency-Key') - return store.create_message(chat_id, user["id"], payload.text, idempotency_key) + return store.create_message(chat_id, user["id"], payload.text, idempotency_key, payload.attachment_ids) @router.get("/{chat_id}/messages", response_model=MessagePage) diff --git a/src/routers/health.py b/src/routers/health.py index b0871bd..f2015d3 100644 --- a/src/routers/health.py +++ b/src/routers/health.py @@ -4,6 +4,7 @@ from sqlalchemy import select from ..schemas.common import Health from ..auth import sessions from ..store import store +from ..object_storage import BUCKET, s3_client router = APIRouter(prefix='/health') @@ -18,4 +19,5 @@ def health_ready(): with store.sessions() as session: session.execute(select(1)) sessions.client.ping() + s3_client().head_bucket(Bucket=BUCKET) return {"status": "ok"} diff --git a/src/schemas/message.py b/src/schemas/message.py index 3c75502..75268c5 100644 --- a/src/schemas/message.py +++ b/src/schemas/message.py @@ -1,19 +1,28 @@ from typing import List, Optional -from pydantic import Field, StrictStr, field_validator +from pydantic import Field, StrictStr, field_validator, model_validator +from uuid import UUID from .common import ContractModel class CreateMessage(ContractModel): - text: StrictStr = Field(min_length=1, max_length=4000) + text: StrictStr = Field(max_length=4000) + attachment_ids: List[StrictStr] = Field(default_factory=list, max_length=10) - @field_validator("text") - def validate_text(cls, value): - if not value.strip(): - raise ValueError("blank text") - return value + @field_validator('attachment_ids') + def validate_ids(cls, value): + ids = [str(UUID(item)) for item in value] + if len(ids) != len(set(ids)): + raise ValueError('duplicate attachments') + return ids + + @model_validator(mode='after') + def validate_content(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError('empty message') + return self class Message(ContractModel): @@ -21,6 +30,7 @@ class Message(ContractModel): chat_id: str sender_id: str text: str + attachment_ids: List[str] created_at: str diff --git a/src/store.py b/src/store.py index 01a7e33..48d2a22 100644 --- a/src/store.py +++ b/src/store.py @@ -10,7 +10,7 @@ from sqlalchemy.exc import IntegrityError from .auth import password_hasher, verify_password, unauthorized from .errors import APIError from .cache import history_cache -from .models import User, Chat, ChatMember, Message, IdempotencyRecord +from .models import User, Chat, ChatMember, Message, IdempotencyRecord, Attachment def ensure_uuid(value): @@ -36,7 +36,7 @@ def serialize(model, *fields): USER_FIELDS = ('id', 'username', 'display_name', 'created_at') CHAT_FIELDS = ('id', 'title', 'created_at') -MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at') +MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at', 'attachment_ids') def after_condition(model, after): @@ -130,10 +130,16 @@ class PostgresStore: items = [self._members(session, chat) for chat in chats[:limit]] return self._page(items, len(chats) > limit, 'chats', actor_id) - def create_message(self, chat_id, actor_id, text, idempotency_key=None): + def create_message(self, chat_id, actor_id, text, idempotency_key=None, attachment_ids=None): chat_id = ensure_uuid(chat_id) + attachment_ids = [ensure_uuid(item) for item in (attachment_ids or [])] with self.sessions.begin() as session: self._chat(session, chat_id, actor_id) + if attachment_ids: + found = list(session.scalars(select(Attachment.id).where( + Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id))) + if len(found) != len(attachment_ids): + raise APIError(404, 'not_found', 'Объект не найден') if idempotency_key is not None: # Serialize writers to this chat before looking up the key. # The same row is updated below for history cache invalidation. @@ -141,12 +147,12 @@ class PostgresStore: record = session.get(IdempotencyRecord, (actor_id, chat_id, idempotency_key)) if record is not None: previous = session.get(Message, record.message_id) - if previous.text != text: + if previous.text != text or previous.attachment_ids != attachment_ids: raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения') return serialize(previous, *MESSAGE_FIELDS) session.execute(update(Chat).where(Chat.id == chat_id).values( history_version=Chat.history_version + 1)) - message = Message(chat_id=chat_id, sender_id=actor_id, text=text) + message = Message(chat_id=chat_id, sender_id=actor_id, text=text, attachment_ids=attachment_ids) session.add(message) session.flush() if idempotency_key is not None: diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..c23cf96 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,39 @@ +"""Attachment negative cases. Run against the Compose API at localhost:8080.""" +import sys +from pathlib import Path +from uuid import uuid4 + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client, check + + +def main(): + client = Client() + alice, bob, eve, chat = client.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png' + image = (Path(__file__).resolve().parents[1] / 'tests' / 'fixtures' / 'sample.png').read_bytes() + + client.call('POST', path, 415, actor=alice, raw=b'not an image', + headers={'Content-Type': 'image/png'}) + client.call('POST', path, 415, actor=alice, raw=image, + headers={'Content-Type': 'image/jpeg'}) + client.call('POST', path, 413, actor=alice, raw=b'x' * (10 * 1024 * 1024 + 1), + headers={'Content-Type': 'image/png'}) + client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=../sample.png', + 400, actor=alice, raw=image, headers={'Content-Type': 'image/png'}) + + attachment = client.call('POST', path, 201, actor=alice, raw=image, + headers={'Content-Type': 'image/png'}, schema='Attachment')['json'] + client.call('GET', '/api/v1/attachments/' + attachment['id'], 404, actor=eve) + client.call('GET', '/api/v1/attachments/' + str(uuid4()), 404, actor=alice) + + other = client.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'Other chat', 'member_ids': [alice['id']]}, schema='Chat')['json'] + client.call('POST', '/api/v1/chats/' + other['id'] + '/messages', 404, actor=alice, + body={'text': '', 'attachment_ids': [attachment['id']]}, + headers={'Idempotency-Key': uuid4().hex}) + + +if __name__ == '__main__': + main() + print('PASS: attachment validation and access checks')