diff --git a/.env.example b/.env.example index 119384b5..8fcd21a4 100644 --- a/.env.example +++ b/.env.example @@ -11,15 +11,16 @@ SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://minio:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_ACCESS_KEY_ID=messenger-api +S3_SECRET_ACCESS_KEY=change-me-local-only +BROKER_URL=redis://broker:6379/0 +MAX_JOB_ATTEMPTS=5 +PROCESSING_LEASE_SECONDS=30 +ACK_DELAY_SECONDS=0 +MINIO_ROOT_USER=minioadmin +MINIO_ROOT_PASSWORD=change-me-minio-local-only MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 -JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a4..d59fb999 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Борзов Никита, БВТ2304 +Целевая оценка (3/4/5): 4 +Трек 5, если заявлен: не заявлен -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: проект ЛР4 перенесён с сохранением API, авторизации, PostgreSQL, Redis-сессий, MinIO, Nginx, четырёх API-реплик, идемпотентности и failover. Добавлены Redis Streams broker, durable S3 staging, transactional outbox, отдельный worker, queued/processing/ready/failed, thumbnail/crop/optimized, retry с backoff, lease и replay. +Команды запуска из чистого clone: `cp .env.example .env`, `docker compose config --quiet`, `docker compose build`, `docker compose up -d --wait`. +Результаты проверок и ссылка на REPORT.md: `make check`, `make validate`, `make test`, варианты, outbox, worker recovery, persistence и failover прошли; подробности в `REPORT.md`. +Ограничения / известные ошибки: PostgreSQL, Redis, broker, MinIO и Nginx остаются единичными точками отказа; exactly-once доставка не заявляется. +Источники и использование ИИ: skeleton и контракты Gitea, документация Redis Streams, boto3, Pillow, FastAPI, Docker Compose; ИИ использовался как помощник, результаты проверены локально. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; результаты приведены в REPORT.md. +- [x] Сценарии отказов и безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..9c100b47 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY app ./app +COPY migrations ./migrations +COPY scripts ./scripts +RUN useradd --create-home --uid 10001 appuser && chown -R appuser:appuser /app +USER appuser +EXPOSE 8080 +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..ed3dd8e5 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,107 @@ -# Отчёт — лабораторная 5 +# Отчёт по лабораторной работе №5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Борзов Никита, БВТ2304 +- Целевая оценка: 4 +- Commit/PR: `3537111`, [MR !25](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/25) +- Основа: проект ЛР4 с сохранением PostgreSQL, Redis-сессий, MinIO, Nginx, четырёх API-реплик, авторизации, идемпотентности и сценариев отказа +- Новая задача: принять изображение отдельно от его фоновой обработки ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Desktop с Compose v2 и Python 3.9+. Все образы имеют фиксированные версии или digest. Локальные пароли находятся только в `.env`; `.env.example` содержит учебные значения для изолированного стенда. -## Архитектура и решения +```sh +cp .env.example .env +docker compose config --quiet +docker compose build +docker compose up -d --wait +make check +make validate PYTHON=.venv/bin/python +make test BASE_URL=http://localhost:8080 +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Без OpenAPI-проверки достаточно `make check` и `make test`. Для полной валидации сначала создаётся окружение: -## Доказательства критериев +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +``` -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Сервисы Compose: -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Роль | Сервис | Данные/порты | +| --- | --- | --- | +| API | `api-1` ... `api-4` | общий код приложения, порты доступны только внутри сети | +| Балансировщик | `lb` | единственный внешний `localhost:8080` | +| База | `postgres` | `postgres_data` volume | +| Сессии/кэш | `redis` | `redis_data` volume, AOF | +| Broker | `broker` | Redis Streams, `broker_data` volume, AOF | +| Файлы | `minio` | приватный S3 bucket, `s3_data` volume | +| Обработка | `worker` | отдельный процесс, resize и фиксация результата | -## Эксперименты и отказы +`minio-init` и `migrate` являются одноразовыми bootstrap-сервисами. API-реплики не публикуются наружу. `docker compose down` сохраняет volumes, а `docker compose down -v` намеренно удаляет учебные данные. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## Поток загрузки -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +В ЛР4 upload возвращал `201 ready`. В ЛР5 endpoint возвращает `202 queued`, потому что обработка вынесена из API: -## Ограничения и следующие шаги +```text +клиент -> Nginx -> API + | проверка и S3 staging + | PostgreSQL: attachment + outbox job + v + Redis Streams broker -> worker + | original + | thumbnail 256x256 + | crop 128x128 + | optimized WebP + v + S3 final prefix + PostgreSQL metadata +``` -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +До ответа `202` проверенный оригинал записывается в приватный prefix `staging/`, а в одной транзакции PostgreSQL создаются строка вложения и запись `attachment_jobs`. В broker передаются только UUID и служебные поля job; байты, credentials и клиентский filename в сообщение не попадают. + +Клиент читает `GET /api/v1/attachments/{id}` и ждёт `status=ready`. До готовности download возвращает `409`. После готовности original отдаётся побайтно, а варианты доступны через `?variant=thumbnail`, `crop` или `optimized`. + +Worker применяет EXIF orientation, ограничивает число пикселей и создаёт варианты без EXIF. Thumbnail вписывается в 256×256 без увеличения маленького изображения; fixture 320×200 даёт 256×160. Crop — центрированный JPEG 128×128. Optimized — WebP, вписанный в 1280×1280, качество 82. + +## Надёжность и уровень 4 + +Redis Streams работает как отдельный durable broker с AOF и volume. Доставка at-least-once: worker подтверждает сообщение только после устойчивой фиксации результата. Повторная job использует тот же `job_id`, а ключи объектов и `(attachment_id, pipeline_version, variant)` делают обработку идемпотентной. Для уже готового вложения повтор является no-op. + +Transactional outbox закрывает разрыв между PostgreSQL и broker. API сначала коммитит staging, metadata и `attachment_jobs`. Worker отдельным циклом публикует неподтверждённые outbox-записи. Если worker остановлен после commit, запись остаётся в PostgreSQL и публикуется после восстановления; ручное создание нового задания не требуется. + +У processing есть lease 30 секунд. После смерти worker просроченный lease переводит вложение обратно в `queued`, а тот же job снова появляется в outbox. Временные ошибки получают ограниченные повторы до 5 попыток с exponential backoff и jitter. Постоянная ошибка изображения становится `failed` с безопасным `error_code`; для явного операторского повтора используется: + +```sh +docker compose exec worker python scripts/replay_failed.py ATTACHMENT_UUID +``` + +Варианты записываются с `ON CONFLICT` по имени, поэтому повторная доставка не создаёт второй набор вариантов и старая job не перезаписывает новый результат. + +Для проверки окна «результат уже записан, но ack ещё не отправлен» worker поддерживает локальный тестовый `ACK_DELAY_SECONDS`; в обычном `.env` он равен нулю и не влияет на обработку. + +## Проверки и наблюдаемые результаты + +| Сценарий | Команда | Результат | +| --- | --- | --- | +| Контракт и синтаксис | `make check` | PASS | +| OpenAPI и JSON Schema | `make validate PYTHON=.venv/bin/python` | PASS | +| Публичный сценарий | `make test BASE_URL=http://localhost:8080` | `PASS: lab5 public API scenario` | +| MIME/размер/доступ | `python3 student-tests/attachments.py` | PASS; 415/413/404 | +| Все варианты | `python3 student-tests/variants.py` | thumbnail, crop и optimized PASS | +| Идемпотентность сообщений | `python3 student-tests/idempotency.py` | один Message ID при 8 повторах | +| 10 конкурентных upload | `python3 student-tests/upload_load.py` | все 10 получили `202` | +| Worker остановлен | `python3 student-tests/worker_recovery.py` | upload 202 сохраняется и после старта worker становится ready | +| Outbox | `python3 student-tests/outbox_recovery.py` | commit-задание переживает остановку worker и доходит до ready | +| Crash после результата до ack | `python3 student-tests/worker_ack_recovery.py` | redelivery оставляет один набор вариантов | +| Persistence | `make test-persistence BASE_URL=http://localhost:8080 ACTION_SCRIPT=scripts/restart.sh` | данные, сессия и вложение сохранены | +| API failover | `make test-failover BASE_URL=http://localhost:8080 ACTION_SCRIPT=scripts/stop-one.sh` | записи продолжаются через другие реплики | + +Во время локальной проверки было получено: smoke PASS, `observed_instances_before=4`, `failed_attempts=0`, recovery около 1 секунды, все варианты прошли проверку checksum и MIME. + +## Ограничения + +Один PostgreSQL, один Redis-сервис сессий, один Redis broker, один MinIO и один Nginx остаются единичными точками отказа. Стенд проверяет отказ worker, повторную доставку, сохранность volumes и отказ одной API-реплики; потеря Docker-host и удаление volumes в модель не входят. Redis Streams использует at-least-once доставку, поэтому exactly-once доставка не заявляется: ровно один эффект достигается идемпотентной записью результата. + +Все пользователи и изображения синтетические. ИИ использовался как помощник при разборе skeleton и подготовке кода; команды и результаты проверены на локальном Compose-стенде. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..596be505 --- /dev/null +++ b/app/main.py @@ -0,0 +1,792 @@ +import base64 +import io +import json +import os +import re +import secrets +import threading +import time +import uuid +import hashlib +from datetime import datetime, timedelta, timezone + +import redis +import boto3 +from botocore.config import Config +from argon2 import PasswordHasher +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from PIL import Image, UnidentifiedImageError +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +DATABASE_URL = os.getenv('DATABASE_URL', 'postgresql://messenger:messenger@postgres:5432/messenger') +REDIS_URL = os.getenv('REDIS_URL', 'redis://redis:6379/0') +BROKER_URL = os.getenv('BROKER_URL', 'redis://broker:6379/0') +SESSION_TTL_SECONDS = int(os.getenv('SESSION_TTL_SECONDS', '3600')) +INSTANCE_ID = os.getenv('INSTANCE_ID', uuid.uuid4().hex[:12]) +DB_POOL_SIZE = int(os.getenv('DB_POOL_SIZE', '10')) +S3_ENDPOINT = os.getenv('S3_ENDPOINT', 'http://minio:9000') +S3_REGION = os.getenv('S3_REGION', 'us-east-1') +S3_BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments') +S3_ACCESS_KEY_ID = os.getenv('S3_ACCESS_KEY_ID', 'messenger-api') +S3_SECRET_ACCESS_KEY = os.getenv('S3_SECRET_ACCESS_KEY', 'messenger-api-secret-123') +MAX_UPLOAD_BYTES = int(os.getenv('MAX_UPLOAD_BYTES', '10485760')) +MAX_IMAGE_PIXELS = int(os.getenv('MAX_IMAGE_PIXELS', '40000000')) +MAX_JOB_ATTEMPTS = int(os.getenv('MAX_JOB_ATTEMPTS', '5')) +PROCESSING_LEASE_SECONDS = int(os.getenv('PROCESSING_LEASE_SECONDS', '30')) + +password_hasher = PasswordHasher() +db_pool = ConnectionPool( + DATABASE_URL, min_size=1, max_size=DB_POOL_SIZE, timeout=5, + kwargs={'row_factory': dict_row}, open=False, +) +redis_client = redis.Redis.from_url( + REDIS_URL, + decode_responses=True, + socket_connect_timeout=1, + socket_timeout=1, + retry_on_timeout=False, +) +broker_client = redis.Redis.from_url( + BROKER_URL, decode_responses=True, socket_connect_timeout=1, + socket_timeout=2, retry_on_timeout=False, +) +s3_client = boto3.client( + 's3', endpoint_url=S3_ENDPOINT, region_name=S3_REGION, + aws_access_key_id=S3_ACCESS_KEY_ID, aws_secret_access_key=S3_SECRET_ACCESS_KEY, + config=Config(connect_timeout=2, read_timeout=5, retries={'max_attempts': 1}), +) +chat_cache_lock = threading.Lock() +chat_cache_sql_reads = 0 +invalidated_chat_caches = set() +app = FastAPI(title='MTUSI Messenger Lab 5', version='1.5.0') + + +class IdempotencyConflict(Exception): + pass + + +class AttachmentConflict(Exception): + pass + + +def utc_now(): + return datetime.now(timezone.utc) + + +def format_time(value): + return value.astimezone(timezone.utc).isoformat().replace('+00:00', 'Z') + + +def parse_uuid(value): + try: + return uuid.UUID(str(value)) + except (TypeError, ValueError) as exc: + raise ValueError('invalid uuid') from exc + + +def user_response(row): + return { + 'id': str(row['id']), 'username': row['username'], + 'display_name': row['display_name'], 'created_at': format_time(row['created_at']), + } + + +def chat_response(row, member_ids): + return { + 'id': str(row['id']), 'title': row['title'], + 'member_ids': [str(item) for item in member_ids], + 'created_at': format_time(row['created_at']), + } + + +def message_response(row, attachment_ids=None): + return { + 'id': str(row['id']), 'chat_id': str(row['chat_id']), + 'sender_id': str(row['sender_id']), 'text': row['text'], + 'created_at': format_time(row['created_at']), + 'attachment_ids': [str(item) for item in (attachment_ids or [])], + } + + +def attachment_response(connection, row): + variants = connection.execute( + 'SELECT name, content_type, size_bytes, width, height, sha256 ' + 'FROM attachment_variants WHERE attachment_id=%s ORDER BY name', (row['id'],) + ).fetchall() + return { + 'id': str(row['id']), 'chat_id': str(row['chat_id']), + 'owner_id': str(row['owner_id']), 'filename': row['filename'], + 'content_type': row['content_type'], 'size_bytes': row['size_bytes'], + 'sha256': row['sha256'], 'status': row['status'], + 'variants': [dict(item) for item in variants], + 'error_code': row['error_code'], 'created_at': format_time(row['created_at']), + 'updated_at': format_time(row['updated_at']), + } + + +def attachment_ids_for(connection, message_id): + rows = connection.execute( + 'SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY attachment_id', + (message_id,), + ).fetchall() + return [row['attachment_id'] for row in rows] + + +def safe_filename(value): + return ( + isinstance(value, str) and 1 <= len(value) <= 255 and + not any(ord(char) < 32 for char in value) and '/' not in value and '\\' not in value + ) + + +async def read_upload(request): + chunks, total = [], 0 + async for chunk in request.stream(): + total += len(chunk) + if total > MAX_UPLOAD_BYTES: + return None + chunks.append(chunk) + return b''.join(chunks) + + +def inspect_image(data, content_type): + try: + with Image.open(io.BytesIO(data)) as image: + actual = 'image/png' if image.format == 'PNG' else 'image/jpeg' if image.format == 'JPEG' else None + if actual != content_type: + raise ValueError('content type does not match image') + width, height = image.size + if width * height > MAX_IMAGE_PIXELS: + raise OverflowError('too many pixels') + image.verify() + except Image.DecompressionBombError as exc: + raise OverflowError('too many pixels') from exc + except (UnidentifiedImageError, OSError, ValueError) as exc: + raise ValueError('invalid image') from exc + + +def error_response(request, status_code, code, message, extra_headers=None): + request_id = getattr(request.state, 'request_id', uuid.uuid4().hex) + headers = {'X-Request-Id': request_id, 'X-Instance-Id': INSTANCE_ID} + if extra_headers: + headers.update(extra_headers) + return JSONResponse( + status_code=status_code, + content={'error': {'code': code, 'message': message, 'request_id': request_id}}, + headers=headers, + ) + + +@app.middleware('http') +async def request_context(request: Request, call_next): + request.state.request_id = request.headers.get('X-Request-Id') or uuid.uuid4().hex + try: + response = await call_next(request) + except Exception: + return error_response(request, 500, 'internal_error', 'Внутренняя ошибка') + response.headers['X-Request-Id'] = request.state.request_id + response.headers['X-Instance-Id'] = INSTANCE_ID + return response + + +@app.on_event('startup') +def start_application(): + db_pool.open(wait=True) + with db_pool.connection() as connection: + connection.execute('SELECT 1') + + +@app.on_event('shutdown') +def stop_application(): + db_pool.close() + + +async def read_json_body(request): + try: + value = await request.json() + except Exception as exc: + raise ValueError('malformed json') from exc + if not isinstance(value, dict): + raise ValueError('request body must be an object') + return value + + +def require_fields(data, required): + if set(data) != required: + raise ValueError('unexpected or missing fields') + + +def get_bearer_session(request): + authorization = request.headers.get('Authorization', '') + if not authorization.startswith('Bearer '): + return None + token = authorization.removeprefix('Bearer ') + if len(token) < 40: + return None + try: + value = redis_client.get('session:' + token) + return json.loads(value) if value else None + except Exception: + return None + + +def get_current_user(request): + session = get_bearer_session(request) + if session is None: + return None + with db_pool.connection() as connection: + return connection.execute( + 'SELECT * FROM users WHERE id=%s', (session['user_id'],) + ).fetchone() + + +def require_current_user(request): + user = get_current_user(request) + if user is None: + return None, error_response( + request, 401, 'unauthorized', 'Требуется действующая Bearer-сессия', + {'WWW-Authenticate': 'Bearer realm="messenger"'}, + ) + return user, None + + +def is_chat_member(connection, chat_id, user_id): + row = connection.execute( + 'SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', + (chat_id, user_id), + ).fetchone() + return row is not None + + +def make_cursor(scope, owner_id, row): + created_at = row['created_at'] + if isinstance(created_at, datetime): + created_at = format_time(created_at) + data = [scope, owner_id, created_at, str(row['id'])] + return base64.urlsafe_b64encode(json.dumps(data).encode()).decode().rstrip('=') + + +def read_cursor(cursor, expected_scope, expected_owner): + try: + padding = '=' * ((4 - len(cursor) % 4) % 4) + value = json.loads(base64.urlsafe_b64decode((cursor + padding).encode())) + if value[0] != expected_scope or value[1] != expected_owner: + raise ValueError + timestamp = datetime.fromisoformat(value[2].replace('Z', '+00:00')) + return timestamp, parse_uuid(value[3]) + except (IndexError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise ValueError('invalid cursor') from exc + + +def page_chats(items, limit, cursor_data, owner_id): + if cursor_data: + items = [ + item for item in items + if ( + datetime.fromisoformat(item['created_at'].replace('Z', '+00:00')), + uuid.UUID(item['id']), + ) > cursor_data + ] + page = items[:limit] + next_cursor = make_cursor('chats', owner_id, page[-1]) if len(items) > limit and page else None + return {'items': page, 'next_cursor': next_cursor} + + +def parse_limit(value): + try: + limit = int(value) + except (TypeError, ValueError) as exc: + raise ValueError('invalid limit') from exc + if not 1 <= limit <= 100: + raise ValueError('invalid limit') + return limit + + +@app.get('/health/live') +def health_live(): + return {'status': 'ok'} + + +@app.get('/health/ready') +def health_ready(request: Request): + try: + with db_pool.connection() as connection: + connection.execute('SELECT 1') + redis_client.ping() + s3_client.head_bucket(Bucket=S3_BUCKET) + except Exception: + return error_response(request, 503, 'unavailable', 'Зависимость недоступна') + return {'status': 'ok'} + + +@app.get('/debug/cache-stats') +def cache_stats(request: Request): + if os.getenv('ENABLE_DEBUG_METRICS') != '1': + return error_response(request, 404, 'not_found', 'Ресурс не найден') + return {'chat_list_sql_reads': chat_cache_sql_reads} + + +@app.post('/api/v1/users', status_code=201) +async def create_user(request: Request): + try: + data = await read_json_body(request) + require_fields(data, {'username', 'display_name', 'password'}) + username, display_name, password = data['username'], data['display_name'], data['password'] + if not isinstance(username, str) or not re.fullmatch(r'[a-z0-9_]{3,32}', username): + raise ValueError('invalid username') + if not isinstance(display_name, str) or not display_name.strip() or not 1 <= len(display_name) <= 100: + raise ValueError('invalid display name') + if not isinstance(password, str) or not 12 <= len(password) <= 128: + raise ValueError('invalid password') + + user_id, created_at = uuid.uuid4(), utc_now() + password_hash = password_hasher.hash(password) + with db_pool.connection() as connection: + try: + connection.execute( + 'INSERT INTO users VALUES (%s,%s,%s,%s,%s)', + (user_id, username, display_name, password_hash, created_at), + ) + connection.commit() + except Exception as exc: + connection.rollback() + if 'unique' in str(exc).lower(): + return error_response(request, 409, 'conflict', 'Имя пользователя уже занято') + raise + return user_response({'id': user_id, 'username': username, 'display_name': display_name, 'created_at': created_at}) + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверные поля запроса') + + +@app.post('/api/v1/auth/sessions', status_code=201) +def create_session(request: Request): + authorization = request.headers.get('Authorization', '') + try: + if not authorization.startswith('Basic '): + raise ValueError + credentials = base64.b64decode(authorization.removeprefix('Basic '), validate=True).decode() + username, password = credentials.split(':', 1) + with db_pool.connection() as connection: + user = connection.execute('SELECT * FROM users WHERE username=%s', (username,)).fetchone() + if user is None: + raise ValueError + password_hasher.verify(user['password_hash'], password) + except Exception: + return error_response( + request, 401, 'unauthorized', 'Неверные учётные данные', + {'WWW-Authenticate': 'Basic realm="messenger", charset="UTF-8"'}, + ) + + token = secrets.token_urlsafe(32) + expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS) + redis_client.setex( + 'session:' + token, + SESSION_TTL_SECONDS, + json.dumps({'user_id': str(user['id']), 'expires_at': format_time(expires_at)}), + ) + return { + 'token': token, 'token_type': 'Bearer', + 'expires_at': format_time(expires_at), 'user': user_response(user), + } + + +@app.delete('/api/v1/auth/sessions/current', status_code=204) +def logout(request: Request): + if get_bearer_session(request) is None: + return error_response( + request, 401, 'unauthorized', 'Сессия недействительна', + {'WWW-Authenticate': 'Bearer realm="messenger"'}, + ) + token = request.headers['Authorization'].removeprefix('Bearer ') + redis_client.delete('session:' + token) + return Response(status_code=204) + + +@app.get('/api/v1/users/me') +def get_me(request: Request): + user, failure = require_current_user(request) + return failure if failure else user_response(user) + + +@app.post('/api/v1/chats', status_code=201) +async def create_chat(request: Request): + user, failure = require_current_user(request) + if failure: + return failure + try: + data = await read_json_body(request) + require_fields(data, {'title', 'member_ids'}) + title, member_values = data['title'], data['member_ids'] + if not isinstance(title, str) or not title.strip() or not 1 <= len(title) <= 100: + raise ValueError('invalid title') + if not isinstance(member_values, list) or not 1 <= len(member_values) <= 100: + raise ValueError('invalid members') + if len(set(member_values)) != len(member_values): + raise ValueError('duplicate members') + member_ids = [parse_uuid(value) for value in member_values] + if user['id'] not in member_ids: + raise ValueError('current user must be a member') + + chat_id, created_at = uuid.uuid4(), utc_now() + with db_pool.connection() as connection: + with connection.transaction(): + count = connection.execute( + 'SELECT count(*) AS n FROM users WHERE id = ANY(%s)', (member_ids,) + ).fetchone()['n'] + if count != len(member_ids): + raise ValueError('unknown member') + connection.execute('INSERT INTO chats VALUES (%s,%s,%s)', (chat_id, title, created_at)) + for member_id in member_ids: + connection.execute('INSERT INTO chat_members VALUES (%s,%s)', (chat_id, member_id)) + for member_id in member_ids: + try: + redis_client.delete('chats:' + str(member_id)) + except Exception: + with chat_cache_lock: + invalidated_chat_caches.add('chats:' + str(member_id)) + return chat_response({'id': chat_id, 'title': title, 'created_at': created_at}, member_ids) + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверные данные чата') + + +@app.get('/api/v1/chats') +def list_chats(request: Request, limit: str = '50', cursor: str | None = None): + global chat_cache_sql_reads + user, failure = require_current_user(request) + if failure: + return failure + try: + limit = parse_limit(limit) + except ValueError: + return error_response(request, 400, 'invalid_request', 'limit должен быть от 1 до 100') + + cursor_data = None + if cursor is not None: + if not cursor: + return error_response(request, 400, 'invalid_request', 'Неверный cursor') + try: + cursor_data = read_cursor(cursor, 'chats', str(user['id'])) + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверный cursor') + + cache_key = 'chats:' + str(user['id']) + cache_needs_refresh = cache_key in invalidated_chat_caches + if not cache_needs_refresh: + try: + cached = redis_client.get(cache_key) + if cached: + return page_chats(json.loads(cached), limit, cursor_data, str(user['id'])) + except Exception: + pass + + with chat_cache_lock: + try: + cached = None if cache_needs_refresh else redis_client.get(cache_key) + except Exception: + cached = None + if cached: + chats = json.loads(cached) + else: + with db_pool.connection() as connection: + rows = connection.execute( + 'SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id ' + 'WHERE m.user_id=%s ORDER BY c.created_at,c.id', (user['id'],) + ).fetchall() + chats = [] + for row in rows: + members = connection.execute( + 'SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (row['id'],) + ).fetchall() + chats.append(chat_response(row, [item['user_id'] for item in members])) + chat_cache_sql_reads += 1 + try: + redis_client.setex(cache_key, 30, json.dumps(chats)) + invalidated_chat_caches.discard(cache_key) + except Exception: + pass + return page_chats(chats, limit, cursor_data, str(user['id'])) + + +@app.get('/api/v1/chats/{chat_id}') +def get_chat(chat_id: str, request: Request): + user, failure = require_current_user(request) + if failure: + return failure + try: + parsed_chat_id = parse_uuid(chat_id) + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверный UUID') + with db_pool.connection() as connection: + chat = connection.execute('SELECT * FROM chats WHERE id=%s', (parsed_chat_id,)).fetchone() + if chat is None or not is_chat_member(connection, parsed_chat_id, user['id']): + return error_response(request, 404, 'not_found', 'Чат не найден') + members = connection.execute('SELECT user_id FROM chat_members WHERE chat_id=%s', (parsed_chat_id,)).fetchall() + return chat_response(chat, [item['user_id'] for item in members]) + + +@app.post('/api/v1/chats/{chat_id}/attachments', status_code=202) +async def upload_attachment(chat_id: str, request: Request): + user, failure = require_current_user(request) + if failure: + return failure + try: + parsed_chat_id = parse_uuid(chat_id) + filename = request.query_params.get('filename', '') + if not safe_filename(filename): + raise ValueError('invalid filename') + content_type = request.headers.get('Content-Type', '').split(';', 1)[0].strip().lower() + if content_type not in {'image/png', 'image/jpeg'}: + return error_response(request, 415, 'unsupported_media_type', 'Поддерживаются только PNG и JPEG') + data = await read_upload(request) + if data is None: + return error_response(request, 413, 'payload_too_large', 'Файл слишком большой') + if not data: + raise ValueError('empty file') + except OverflowError: + return error_response(request, 413, 'payload_too_large', 'Слишком много пикселей') + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверные данные вложения') + try: + inspect_image(data, content_type) + except OverflowError: + return error_response(request, 413, 'payload_too_large', 'Слишком много пикселей') + except ValueError: + return error_response(request, 415, 'unsupported_media_type', 'Файл не является корректным PNG или JPEG') + + attachment_id = uuid.uuid4() + staging_key = 'staging/' + str(attachment_id) + '/original' + object_key = 'attachments/' + str(attachment_id) + '/original' + job_id = uuid.uuid4() + now = utc_now() + digest = hashlib.sha256(data).hexdigest() + with db_pool.connection() as connection: + if not is_chat_member(connection, parsed_chat_id, user['id']): + return error_response(request, 404, 'not_found', 'Чат не найден') + try: + s3_client.put_object( + Bucket=S3_BUCKET, Key=staging_key, Body=data, + ContentType=content_type, ContentLength=len(data), + ) + with connection.transaction(): + row = connection.execute( + 'INSERT INTO attachments ' + '(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,staging_key,object_key,status,error_code,created_at,updated_at) ' + 'VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *', + (attachment_id, parsed_chat_id, user['id'], filename, content_type, + len(data), digest, staging_key, object_key, 'queued', None, now, now), + ).fetchone() + connection.execute( + 'INSERT INTO attachment_jobs (job_id,attachment_id,pipeline_version,attempt) ' + 'VALUES (%s,%s,%s,%s)', (job_id, attachment_id, 1, 1), + ) + except Exception: + try: + s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key) + except Exception: + pass + return error_response(request, 503, 'unavailable', 'Хранилище вложений недоступно') + with db_pool.connection() as connection: + return attachment_response(connection, row) + + +def get_attachment_for_user(connection, attachment_id, user_id): + row = connection.execute( + 'SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id ' + 'WHERE a.id=%s AND m.user_id=%s', (attachment_id, user_id), + ).fetchone() + return row + + +@app.get('/api/v1/attachments/{attachment_id}') +def get_attachment(attachment_id: str, request: Request): + user, failure = require_current_user(request) + if failure: + return failure + try: + parsed_id = parse_uuid(attachment_id) + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверный UUID') + with db_pool.connection() as connection: + row = get_attachment_for_user(connection, parsed_id, user['id']) + if row is None: + return error_response(request, 404, 'not_found', 'Вложение не найдено') + with db_pool.connection() as connection: + return attachment_response(connection, row) + + +@app.get('/api/v1/attachments/{attachment_id}/content') +def download_attachment(attachment_id: str, request: Request, variant: str = 'original'): + user, failure = require_current_user(request) + if failure: + return failure + if variant not in {'original', 'thumbnail', 'crop', 'optimized'}: + return error_response(request, 404, 'not_found', 'Вариант не найден') + try: + parsed_id = parse_uuid(attachment_id) + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверный UUID') + with db_pool.connection() as connection: + row = get_attachment_for_user(connection, parsed_id, user['id']) + if row is not None and row['status'] != 'ready': + return error_response(request, 409, 'conflict', 'Вложение ещё не готово') + variant_row = None + if row is not None and variant != 'original': + variant_row = connection.execute( + 'SELECT * FROM attachment_variants WHERE attachment_id=%s AND name=%s', + (parsed_id, variant), + ).fetchone() + if row is None: + return error_response(request, 404, 'not_found', 'Вложение не найдено') + if variant != 'original' and variant_row is None: + return error_response(request, 404, 'not_found', 'Вариант не найден') + object_key = row['object_key'] if variant == 'original' else variant_row['object_key'] + content_type = row['content_type'] if variant == 'original' else variant_row['content_type'] + try: + value = s3_client.get_object(Bucket=S3_BUCKET, Key=object_key) + data = value['Body'].read(MAX_UPLOAD_BYTES + 1) + if len(data) > MAX_UPLOAD_BYTES: + return error_response(request, 503, 'unavailable', 'Некорректный объект в хранилище') + except Exception: + return error_response(request, 503, 'unavailable', 'Хранилище вложений недоступно') + return Response(content=data, media_type=content_type) + + +@app.post('/api/v1/chats/{chat_id}/messages', status_code=201) +async def create_message(chat_id: str, request: Request): + user, failure = require_current_user(request) + if failure: + return failure + try: + parsed_chat_id = parse_uuid(chat_id) + idempotency_key = request.headers.get('Idempotency-Key', '') + if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', idempotency_key): + raise ValueError('invalid idempotency key') + data = await read_json_body(request) + require_fields(data, {'text', 'attachment_ids'}) if 'attachment_ids' in data else require_fields(data, {'text'}) + text = data['text'] + attachment_values = data.get('attachment_ids', []) + if not isinstance(attachment_values, list) or len(attachment_values) > 10: + raise ValueError('invalid attachments') + attachment_ids = [parse_uuid(value) for value in attachment_values] + if len(set(attachment_ids)) != len(attachment_ids): + raise ValueError('duplicate attachments') + if not isinstance(text, str) or len(text) > 4000 or (not text.strip() and not attachment_ids): + raise ValueError('invalid text') + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверные данные сообщения') + + body_hash = hashlib.sha256(json.dumps( + {'text': text, 'attachment_ids': sorted(str(value) for value in attachment_ids)}, + sort_keys=True, + ).encode()).hexdigest() + with db_pool.connection() as connection: + if not is_chat_member(connection, parsed_chat_id, user['id']): + return error_response(request, 404, 'not_found', 'Чат не найден') + try: + with connection.transaction(): + idempotency = connection.execute( + 'INSERT INTO message_idempotency ' + '(user_id, chat_id, idempotency_key, body_hash) ' + 'VALUES (%s,%s,%s,%s) ' + 'ON CONFLICT (user_id, chat_id, idempotency_key) DO NOTHING ' + 'RETURNING message_id', + (user['id'], parsed_chat_id, idempotency_key, body_hash), + ).fetchone() + if idempotency is None: + saved = connection.execute( + 'SELECT message_id, body_hash FROM message_idempotency ' + 'WHERE user_id=%s AND chat_id=%s AND idempotency_key=%s', + (user['id'], parsed_chat_id, idempotency_key), + ).fetchone() + if saved['body_hash'] != body_hash: + raise IdempotencyConflict + row = connection.execute( + 'SELECT * FROM messages WHERE id=%s', (saved['message_id'],) + ).fetchone() + return message_response(row, attachment_ids_for(connection, row['id'])) + + if attachment_ids: + rows = connection.execute( + 'SELECT id, status FROM attachments WHERE id = ANY(%s) AND chat_id=%s', + (attachment_ids, parsed_chat_id), + ).fetchall() + if len(rows) != len(attachment_ids): + raise ValueError('attachment is not available in this chat') + if any(item['status'] == 'failed' for item in rows): + raise AttachmentConflict + + message_id, created_at = uuid.uuid4(), utc_now() + connection.execute( + 'INSERT INTO messages VALUES (%s,%s,%s,%s,%s)', + (message_id, parsed_chat_id, user['id'], text, created_at), + ) + connection.execute( + 'UPDATE message_idempotency SET message_id=%s ' + 'WHERE user_id=%s AND chat_id=%s AND idempotency_key=%s', + (message_id, user['id'], parsed_chat_id, idempotency_key), + ) + for attachment_id in attachment_ids: + connection.execute( + 'INSERT INTO message_attachments (message_id, attachment_id) VALUES (%s,%s)', + (message_id, attachment_id), + ) + row = { + 'id': message_id, 'chat_id': parsed_chat_id, + 'sender_id': user['id'], 'text': text, 'created_at': created_at, + } + except IdempotencyConflict: + return error_response(request, 409, 'conflict', 'Ключ уже использован для другого сообщения') + except AttachmentConflict: + return error_response(request, 409, 'conflict', 'Вложение ещё не готово') + except ValueError: + return error_response(request, 404, 'not_found', 'Вложение не найдено') + return message_response(row, attachment_ids) + + +@app.get('/api/v1/chats/{chat_id}/messages') +def list_messages(chat_id: str, request: Request, limit: str = '50', cursor: str | None = None): + user, failure = require_current_user(request) + if failure: + return failure + try: + limit = parse_limit(limit) + except ValueError: + return error_response(request, 400, 'invalid_request', 'limit должен быть от 1 до 100') + try: + parsed_chat_id = parse_uuid(chat_id) + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверный UUID') + + cursor_data = None + if cursor is not None: + if not cursor: + return error_response(request, 400, 'invalid_request', 'Неверный cursor') + try: + cursor_data = read_cursor(cursor, 'messages', str(parsed_chat_id)) + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверный cursor') + + with db_pool.connection() as connection: + if not is_chat_member(connection, parsed_chat_id, user['id']): + return error_response(request, 404, 'not_found', 'Чат не найден') + if cursor_data: + rows = connection.execute( + 'SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ' + 'ORDER BY created_at,id LIMIT %s', + (parsed_chat_id, cursor_data[0], cursor_data[1], limit + 1), + ).fetchall() + else: + rows = connection.execute( + 'SELECT * FROM messages WHERE chat_id=%s ORDER BY created_at,id LIMIT %s', + (parsed_chat_id, limit + 1), + ).fetchall() + items = [message_response(row, attachment_ids_for(connection, row['id'])) for row in rows[:limit]] + + rows_to_return = rows[:limit] + next_cursor = make_cursor('messages', str(parsed_chat_id), rows_to_return[-1]) if len(rows) > limit else None + return { + 'items': items, + 'next_cursor': next_cursor, + } diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..906afc9b --- /dev/null +++ b/compose.yaml @@ -0,0 +1,247 @@ +services: + minio: + image: cgr.dev/chainguard/minio@sha256:0dab2f9fee1c1139ae8753b5e35bb4461af2a276cfe0923a2504c75a6170bc64 + command: server /data --console-address ':9001' + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadminpassword} + volumes: + - s3_data:/data + healthcheck: + test: ["CMD-SHELL", "test -f /data/.minio.sys/format.json"] + interval: 3s + timeout: 3s + retries: 20 + + minio-init: + build: + context: ./infra + dockerfile: minio-init.Dockerfile + image: mtusi-minio-init:1.0 + depends_on: + minio: {condition: service_healthy} + environment: + MINIO_ENDPOINT: http://minio:9000 + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadminpassword} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-messenger-api-secret-123} + + postgres: + image: postgres:16.4-alpine + environment: + POSTGRES_DB: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-local-only-password} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-messenger} -d ${POSTGRES_DB:-messenger}"] + interval: 3s + timeout: 3s + retries: 20 + mem_limit: 512m + cpus: 1.0 + + redis: + image: redis:7.4.2-alpine + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"] + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 3s + retries: 20 + mem_limit: 256m + cpus: 0.5 + + broker: + image: redis:7.4.2-alpine + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"] + volumes: + - broker_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 3s + retries: 20 + mem_limit: 256m + cpus: 0.5 + + migrate: + build: . + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:-local-only-password}@postgres:5432/${POSTGRES_DB:-messenger} + command: ["python", "scripts/migrate.py"] + depends_on: + postgres: {condition: service_healthy} + + worker: + build: . + command: ["python", "scripts/worker.py"] + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:-local-only-password}@postgres:5432/${POSTGRES_DB:-messenger} + BROKER_URL: redis://broker:6379/0 + S3_ENDPOINT: http://minio:9000 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-messenger-api-secret-123} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + MAX_JOB_ATTEMPTS: ${MAX_JOB_ATTEMPTS:-5} + PROCESSING_LEASE_SECONDS: ${PROCESSING_LEASE_SECONDS:-30} + ACK_DELAY_SECONDS: ${ACK_DELAY_SECONDS:-0} + depends_on: + migrate: {condition: service_completed_successfully} + postgres: {condition: service_healthy} + broker: {condition: service_healthy} + minio-init: {condition: service_completed_successfully} + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "kill -0 1"] + interval: 5s + timeout: 3s + retries: 5 + mem_limit: 512m + cpus: 1.0 + + api-1: + build: . + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:-local-only-password}@postgres:5432/${POSTGRES_DB:-messenger} + REDIS_URL: redis://redis:6379/0 + S3_ENDPOINT: http://minio:9000 + S3_REGION: us-east-1 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-messenger-api-secret-123} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + INSTANCE_ID: api-1 + ENABLE_DEBUG_METRICS: ${ENABLE_DEBUG_METRICS:-1} + expose: ["8080"] + depends_on: + migrate: {condition: service_completed_successfully} + minio-init: {condition: service_completed_successfully} + postgres: {condition: service_healthy} + redis: {condition: service_healthy} + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"] + interval: 3s + timeout: 3s + retries: 20 + mem_limit: 256m + cpus: 0.5 + + api-2: + build: . + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:-local-only-password}@postgres:5432/${POSTGRES_DB:-messenger} + REDIS_URL: redis://redis:6379/0 + S3_ENDPOINT: http://minio:9000 + S3_REGION: us-east-1 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-messenger-api-secret-123} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + INSTANCE_ID: api-2 + ENABLE_DEBUG_METRICS: ${ENABLE_DEBUG_METRICS:-1} + expose: ["8080"] + depends_on: + migrate: {condition: service_completed_successfully} + minio-init: {condition: service_completed_successfully} + postgres: {condition: service_healthy} + redis: {condition: service_healthy} + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"] + interval: 3s + timeout: 3s + retries: 20 + mem_limit: 256m + cpus: 0.5 + + api-3: + build: . + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:-local-only-password}@postgres:5432/${POSTGRES_DB:-messenger} + REDIS_URL: redis://redis:6379/0 + S3_ENDPOINT: http://minio:9000 + S3_REGION: us-east-1 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-messenger-api-secret-123} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + INSTANCE_ID: api-3 + ENABLE_DEBUG_METRICS: ${ENABLE_DEBUG_METRICS:-1} + expose: ["8080"] + depends_on: + migrate: {condition: service_completed_successfully} + minio-init: {condition: service_completed_successfully} + postgres: {condition: service_healthy} + redis: {condition: service_healthy} + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"] + interval: 3s + timeout: 3s + retries: 20 + mem_limit: 256m + cpus: 0.5 + + api-4: + build: . + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:-local-only-password}@postgres:5432/${POSTGRES_DB:-messenger} + REDIS_URL: redis://redis:6379/0 + S3_ENDPOINT: http://minio:9000 + S3_REGION: us-east-1 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-messenger-api-secret-123} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + INSTANCE_ID: api-4 + ENABLE_DEBUG_METRICS: ${ENABLE_DEBUG_METRICS:-1} + expose: ["8080"] + depends_on: + migrate: {condition: service_completed_successfully} + minio-init: {condition: service_completed_successfully} + postgres: {condition: service_healthy} + redis: {condition: service_healthy} + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"] + interval: 3s + timeout: 3s + retries: 20 + mem_limit: 256m + cpus: 0.5 + + lb: + image: nginx:1.27.2-alpine + ports: ["8080:8080"] + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + api-1: {condition: service_healthy} + api-2: {condition: service_healthy} + api-3: {condition: service_healthy} + api-4: {condition: service_healthy} + healthcheck: + test: ["CMD-SHELL", "wget -q -O - http://127.0.0.1:8080/health/live >/dev/null"] + interval: 3s + timeout: 3s + retries: 20 + mem_limit: 128m + cpus: 0.25 + +volumes: + postgres_data: + redis_data: + broker_data: + s3_data: diff --git a/infra/minio-init.Dockerfile b/infra/minio-init.Dockerfile new file mode 100644 index 00000000..4d7e8901 --- /dev/null +++ b/infra/minio-init.Dockerfile @@ -0,0 +1,7 @@ +FROM cgr.dev/chainguard/minio-client@sha256:4c288033f700b35846037c53ce39ba287f4bed0a571518c1f4d8d2788beb5cb5 AS client +FROM alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc +COPY --from=client /usr/bin/mc /usr/bin/mc +COPY minio-init.sh /init.sh +COPY s3-policy.json /policy.json +RUN chmod 755 /init.sh +ENTRYPOINT ["/init.sh"] diff --git a/infra/minio-init.sh b/infra/minio-init.sh new file mode 100644 index 00000000..b7fdf493 --- /dev/null +++ b/infra/minio-init.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu + +mc alias set local "${MINIO_ENDPOINT:-http://minio:9000}" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null +mc mb --ignore-existing local/"$S3_BUCKET" >/dev/null +sed "s/messenger-attachments/$S3_BUCKET/g" /policy.json > /tmp/policy.json +POLICY_NAME="messenger-api-policy-$S3_BUCKET" +if ! mc admin policy info local "$POLICY_NAME" >/dev/null 2>&1; then + mc admin policy create local "$POLICY_NAME" /tmp/policy.json >/dev/null +fi +if ! mc admin user info local "$S3_ACCESS_KEY_ID" >/dev/null 2>&1; then + mc admin user add local "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null +fi +mc admin policy attach local "$POLICY_NAME" --user "$S3_ACCESS_KEY_ID" >/dev/null diff --git a/infra/s3-policy.json b/infra/s3-policy.json new file mode 100644 index 00000000..ac67271a --- /dev/null +++ b/infra/s3-policy.json @@ -0,0 +1,7 @@ +{ + "Version": "2012-10-17", + "Statement": [ + {"Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], "Resource": ["arn:aws:s3:::messenger-attachments/*"]}, + {"Effect": "Allow", "Action": ["s3:ListBucket"], "Resource": ["arn:aws:s3:::messenger-attachments"]} + ] +} diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..09678c0e --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, username VARCHAR(32) UNIQUE NOT NULL, + display_name VARCHAR(100) NOT NULL, password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, title VARCHAR(100) NOT NULL, created_at TIMESTAMPTZ NOT NULL +); +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id), text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); +CREATE INDEX IF NOT EXISTS messages_history_idx ON messages(chat_id, created_at, id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..888fba19 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS message_idempotency ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + idempotency_key VARCHAR(128) NOT NULL, + body_hash CHAR(64) NOT NULL, + message_id UUID REFERENCES messages(id) ON DELETE CASCADE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, chat_id, idempotency_key) +); + +CREATE INDEX IF NOT EXISTS message_idempotency_created_idx + ON message_idempotency(created_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..fec805e6 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,65 @@ +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + filename VARCHAR(255) NOT NULL, + content_type VARCHAR(32) NOT NULL, + size_bytes INTEGER NOT NULL, + sha256 CHAR(64) NOT NULL, + staging_key VARCHAR(255) NOT NULL UNIQUE, + object_key VARCHAR(255) NOT NULL UNIQUE, + status VARCHAR(16) NOT NULL DEFAULT 'queued', + error_code VARCHAR(64), + pipeline_version INTEGER NOT NULL DEFAULT 1, + attempt INTEGER NOT NULL DEFAULT 0, + lease_until TIMESTAMPTZ, + next_attempt_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + CHECK (content_type IN ('image/png', 'image/jpeg')), + CHECK (size_bytes > 0 AND size_bytes <= 10485760), + CHECK (status IN ('queued', 'processing', 'ready', 'failed')) +); + +CREATE INDEX IF NOT EXISTS attachments_chat_idx ON attachments(chat_id, created_at, id); +CREATE INDEX IF NOT EXISTS attachments_processing_idx ON attachments(status, lease_until); + +CREATE TABLE IF NOT EXISTS attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + name VARCHAR(16) NOT NULL, + content_type VARCHAR(32) NOT NULL, + object_key VARCHAR(255) NOT NULL UNIQUE, + size_bytes INTEGER NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + sha256 CHAR(64) NOT NULL, + PRIMARY KEY (attachment_id, name), + CHECK (name IN ('thumbnail', 'crop', 'optimized')) +); + +CREATE TABLE IF NOT EXISTS attachment_jobs ( + job_id UUID PRIMARY KEY, + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL, + attempt INTEGER NOT NULL DEFAULT 1, + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), + published_at TIMESTAMPTZ, + last_error VARCHAR(64), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (attachment_id, pipeline_version) +); + +CREATE INDEX IF NOT EXISTS attachment_jobs_outbox_idx + ON attachment_jobs(published_at, next_attempt_at); + +CREATE TABLE IF NOT EXISTS attachment_replays ( + replay_id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT, + PRIMARY KEY (message_id, attachment_id) +); diff --git a/migrations/004_message_attachments.sql b/migrations/004_message_attachments.sql new file mode 100644 index 00000000..cac40596 --- /dev/null +++ b/migrations/004_message_attachments.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT, + PRIMARY KEY (message_id, attachment_id) +); diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 00000000..acab891a --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,25 @@ +events {} + +http { + upstream api_backend { + server api-1:8080 max_fails=2 fail_timeout=3s; + server api-2:8080 max_fails=2 fail_timeout=3s; + server api-3:8080 max_fails=2 fail_timeout=3s; + server api-4:8080 max_fails=2 fail_timeout=3s; + } + + server { + listen 8080; + client_max_body_size 11m; + + location / { + proxy_pass http://api_backend; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_connect_timeout 1s; + proxy_read_timeout 5s; + proxy_send_timeout 5s; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + } +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..8004b3be --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.115.6 +uvicorn[standard]==0.34.0 +psycopg[binary,pool]==3.2.3 +redis==5.2.1 +argon2-cffi==23.1.0 +boto3==1.35.36 +Pillow==11.1.0 diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100755 index 00000000..07a01aa8 --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,10 @@ +import os +import psycopg +from pathlib import Path + +url = os.getenv("DATABASE_URL", "postgresql://messenger:messenger@postgres:5432/messenger") +with psycopg.connect(url) as conn: + for path in sorted(Path("migrations").glob("*.sql")): + conn.execute(path.read_text()) + conn.commit() +print("migrations applied") diff --git a/scripts/replay_failed.py b/scripts/replay_failed.py new file mode 100644 index 00000000..1987e36f --- /dev/null +++ b/scripts/replay_failed.py @@ -0,0 +1,44 @@ +"""Explicit operator replay for one failed attachment job.""" +import argparse +import json +import os +import uuid + +import psycopg + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('attachment_id', type=uuid.UUID) + args = parser.parse_args() + database_url = os.getenv('DATABASE_URL', 'postgresql://messenger:messenger@postgres:5432/messenger') + with psycopg.connect(database_url) as connection: + with connection.transaction(): + row = connection.execute( + "SELECT a.status, j.job_id FROM attachments a " + "JOIN attachment_jobs j ON j.attachment_id=a.id " + "WHERE a.id=%s FOR UPDATE", + (args.attachment_id,), + ).fetchone() + if row is None: + raise SystemExit('attachment not found') + if row[0] != 'failed': + raise SystemExit('attachment is not failed') + replay_id = uuid.uuid4() + connection.execute( + 'INSERT INTO attachment_replays (replay_id,job_id) VALUES (%s,%s)', + (replay_id, row[1]), + ) + connection.execute( + "UPDATE attachments SET status='queued', attempt=0, error_code=NULL, lease_until=NULL, updated_at=now() WHERE id=%s", + (args.attachment_id,), + ) + connection.execute( + "UPDATE attachment_jobs SET attempt=1, published_at=NULL, next_attempt_at=now(), last_error=NULL WHERE job_id=%s", + (row[1],), + ) + print(json.dumps({'attachment_id': str(args.attachment_id), 'job_id': str(row[1]), 'replay_id': str(replay_id)})) + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..df094389 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +# Перезапускаем стенд, не удаляя volumes. +# Балансировщик запускаем после API, чтобы DNS имён реплик уже был доступен. +docker compose up -d minio postgres redis broker minio-init migrate api-1 api-2 api-3 api-4 worker +docker compose restart minio postgres redis broker api-1 api-2 api-3 api-4 worker +docker compose up -d --force-recreate lb +docker compose ps diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..d6020346 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu + +TARGET_INSTANCE_ID=${TARGET_INSTANCE_ID:-api-1} +case "$TARGET_INSTANCE_ID" in + api-1) service=api-1 ;; + api-2) service=api-2 ;; + api-3) service=api-3 ;; + api-4) service=api-4 ;; + *) echo "unknown TARGET_INSTANCE_ID: $TARGET_INSTANCE_ID" >&2; exit 2 ;; +esac + +docker compose stop "$service" diff --git a/scripts/worker.py b/scripts/worker.py new file mode 100644 index 00000000..b17af374 --- /dev/null +++ b/scripts/worker.py @@ -0,0 +1,270 @@ +"""Redis Streams worker for durable attachment processing.""" +import hashlib +import io +import os +import random +import time +import uuid +from datetime import datetime, timedelta, timezone + +import boto3 +import psycopg +import redis +from botocore.config import Config +from PIL import Image, ImageOps, UnidentifiedImageError +from psycopg.rows import dict_row + +DATABASE_URL = os.getenv('DATABASE_URL', 'postgresql://messenger:messenger@postgres:5432/messenger') +BROKER_URL = os.getenv('BROKER_URL', 'redis://broker:6379/0') +S3_ENDPOINT = os.getenv('S3_ENDPOINT', 'http://minio:9000') +S3_BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments') +S3_ACCESS_KEY_ID = os.getenv('S3_ACCESS_KEY_ID', 'messenger-api') +S3_SECRET_ACCESS_KEY = os.getenv('S3_SECRET_ACCESS_KEY', 'messenger-api-secret-123') +MAX_IMAGE_PIXELS = int(os.getenv('MAX_IMAGE_PIXELS', '40000000')) +MAX_JOB_ATTEMPTS = int(os.getenv('MAX_JOB_ATTEMPTS', '5')) +PROCESSING_LEASE_SECONDS = int(os.getenv('PROCESSING_LEASE_SECONDS', '30')) +ACK_DELAY_SECONDS = int(os.getenv('ACK_DELAY_SECONDS', '0')) +STREAM = 'attachment-jobs' +GROUP = 'attachment-workers' +CONSUMER = os.getenv('WORKER_ID', 'worker-' + uuid.uuid4().hex[:8]) + +broker = redis.Redis.from_url( + BROKER_URL, decode_responses=True, socket_connect_timeout=2, + socket_timeout=3, retry_on_timeout=False, +) +s3 = boto3.client( + 's3', endpoint_url=S3_ENDPOINT, region_name='us-east-1', + aws_access_key_id=S3_ACCESS_KEY_ID, aws_secret_access_key=S3_SECRET_ACCESS_KEY, + config=Config(connect_timeout=2, read_timeout=10, retries={'max_attempts': 1}), +) + + +def now(): + return datetime.now(timezone.utc) + + +def ensure_group(): + try: + broker.xgroup_create(STREAM, GROUP, id='0-0', mkstream=True) + except redis.ResponseError as exc: + if 'BUSYGROUP' not in str(exc): + raise + + +def recover_expired_leases(): + with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection: + with connection.transaction(): + rows = connection.execute( + "SELECT id FROM attachments WHERE status='processing' AND lease_until < now()" + ).fetchall() + for item in rows: + attachment_id = item['id'] + connection.execute( + "UPDATE attachments SET status='queued', lease_until=NULL, next_attempt_at=now(), " + "updated_at=now(), error_code=NULL WHERE id=%s", + (attachment_id,), + ) + connection.execute( + "UPDATE attachment_jobs SET published_at=NULL, next_attempt_at=now(), " + "last_error='worker_lease_expired' WHERE attachment_id=%s", + (attachment_id,), + ) + + +def publish_outbox(): + with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection: + row = connection.execute( + "SELECT job_id, attachment_id, pipeline_version, attempt " + "FROM attachment_jobs WHERE published_at IS NULL AND next_attempt_at <= now() " + "ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED" + ).fetchone() + if row is None: + return False + payload = { + 'schema_version': '1', 'job_id': str(row['job_id']), + 'type': 'attachment.process.v1', 'attachment_id': str(row['attachment_id']), + 'pipeline_version': str(row['pipeline_version']), 'attempt': str(row['attempt']), + 'created_at': now().isoformat().replace('+00:00', 'Z'), + } + broker.xadd(STREAM, payload, maxlen=10000, approximate=True) + connection.execute( + 'UPDATE attachment_jobs SET published_at=now() WHERE job_id=%s', (row['job_id'],) + ) + connection.commit() + return True + + +def white_background(image): + image = ImageOps.exif_transpose(image) + if image.mode in ('RGBA', 'LA') or 'transparency' in image.info: + background = Image.new('RGB', image.size, 'white') + alpha = image.convert('RGBA') + background.paste(alpha, mask=alpha.getchannel('A')) + return background + return image.convert('RGB') + + +def jpeg_bytes(image, size, crop=False): + image = image.copy() + if crop: + side = min(image.size) + left = (image.width - side) // 2 + top = (image.height - side) // 2 + image = image.crop((left, top, left + side, top + side)) + if crop: + image = image.resize((size, size), Image.Resampling.LANCZOS) + else: + image.thumbnail((size, size), Image.Resampling.LANCZOS) + output = io.BytesIO() + image.save(output, format='JPEG', quality=85, optimize=True) + return output.getvalue(), image.size + + +def webp_bytes(image): + image = image.copy() + image.thumbnail((1280, 1280), Image.Resampling.LANCZOS) + output = io.BytesIO() + image.save(output, format='WEBP', quality=82, method=6) + return output.getvalue(), image.size + + +def get_attachment(attachment_id): + with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection: + return connection.execute('SELECT * FROM attachments WHERE id=%s', (attachment_id,)).fetchone() + + +def claim(attachment_id, message_attempt): + with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection: + with connection.transaction(): + row = connection.execute( + 'SELECT * FROM attachments WHERE id=%s FOR UPDATE', (attachment_id,) + ).fetchone() + if row is None or row['status'] in ('ready', 'failed'): + return row, False + if row['status'] == 'processing' and row['lease_until'] and row['lease_until'] > now(): + return row, False + attempt = max(row['attempt'] + 1, message_attempt) + connection.execute( + "UPDATE attachments SET status='processing', attempt=%s, lease_until=%s, " + "updated_at=now(), error_code=NULL WHERE id=%s", + (attempt, now() + timedelta(seconds=PROCESSING_LEASE_SECONDS), attachment_id), + ) + row['attempt'] = attempt + row['status'] = 'processing' + return row, True + + +def mark_retry(attachment_id, attempt, error_code, permanent=False): + with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection: + with connection.transaction(): + if permanent or attempt >= MAX_JOB_ATTEMPTS: + connection.execute( + "UPDATE attachments SET status='failed', lease_until=NULL, error_code=%s, updated_at=now() WHERE id=%s", + ('invalid_image' if permanent else 'retry_exhausted', attachment_id), + ) + connection.execute( + "UPDATE attachment_jobs SET published_at=now(), last_error=%s WHERE attachment_id=%s", + ('invalid_image' if permanent else 'retry_exhausted', attachment_id), + ) + else: + delay = min(30, 2 ** max(0, attempt - 1)) + random.uniform(0, 1) + connection.execute( + "UPDATE attachments SET status='queued', lease_until=NULL, next_attempt_at=now(), " + "error_code=%s, updated_at=now() WHERE id=%s", + (error_code, attachment_id), + ) + connection.execute( + "UPDATE attachment_jobs SET attempt=%s, published_at=NULL, next_attempt_at=now()+%s * interval '1 second', last_error=%s " + "WHERE attachment_id=%s", + (attempt + 1, delay, error_code, attachment_id), + ) + + +def process(attachment_id, message_attempt): + row, owned = claim(attachment_id, message_attempt) + if not owned: + return + try: + value = s3.get_object(Bucket=S3_BUCKET, Key=row['staging_key']) + original = value['Body'].read() + with Image.open(io.BytesIO(original)) as source: + if source.format not in ('PNG', 'JPEG') or source.width * source.height > MAX_IMAGE_PIXELS: + raise ValueError('invalid image') + image = white_background(source) + thumbnail, thumb_size = jpeg_bytes(image, 256) + crop, crop_size = jpeg_bytes(image, 128, crop=True) + optimized, optimized_size = webp_bytes(image) + + base = 'attachments/' + str(attachment_id) + '/' + objects = [ + ('original', original, row['content_type'], image.size), + ('thumbnail', thumbnail, 'image/jpeg', thumb_size), + ('crop', crop, 'image/jpeg', crop_size), + ('optimized', optimized, 'image/webp', optimized_size), + ] + for name, data, content_type, _size in objects: + key = base + name + ('.jpg' if content_type == 'image/jpeg' else '.webp' if content_type == 'image/webp' else '') + s3.put_object(Bucket=S3_BUCKET, Key=key, Body=data, ContentType=content_type, ContentLength=len(data)) + + with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection: + with connection.transaction(): + connection.execute( + "UPDATE attachments SET status='ready', lease_until=NULL, error_code=NULL, updated_at=now() WHERE id=%s", + (attachment_id,), + ) + for name, data, content_type, dimensions in objects[1:]: + key = base + name + ('.jpg' if content_type == 'image/jpeg' else '.webp') + connection.execute( + 'INSERT INTO attachment_variants ' + '(attachment_id,name,content_type,object_key,size_bytes,width,height,sha256) ' + 'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) ' + 'ON CONFLICT (attachment_id,name) DO UPDATE SET ' + 'content_type=EXCLUDED.content_type,object_key=EXCLUDED.object_key,size_bytes=EXCLUDED.size_bytes,' + 'width=EXCLUDED.width,height=EXCLUDED.height,sha256=EXCLUDED.sha256', + (attachment_id, name, content_type, key, len(data), dimensions[0], dimensions[1], hashlib.sha256(data).hexdigest()), + ) + try: + s3.delete_object(Bucket=S3_BUCKET, Key=row['staging_key']) + except Exception: + pass + if ACK_DELAY_SECONDS: + time.sleep(ACK_DELAY_SECONDS) + except (UnidentifiedImageError, ValueError, OSError): + mark_retry(attachment_id, row['attempt'], 'invalid_image', permanent=True) + except Exception: + mark_retry(attachment_id, row['attempt'], 'processing_failed') + + +def handle_message(message_id, fields): + process(uuid.UUID(fields['attachment_id']), int(fields.get('attempt', '1'))) + broker.xack(STREAM, GROUP, message_id) + + +def reclaim_pending(): + try: + result = broker.xautoclaim(STREAM, GROUP, CONSUMER, min_idle_time=1000, start_id='0-0', count=10) + for message_id, fields in result[1]: + handle_message(message_id, fields) + except (redis.ResponseError, AttributeError): + pass + + +def main(): + ensure_group() + print('worker started', CONSUMER, flush=True) + while True: + try: + recover_expired_leases() + publish_outbox() + reclaim_pending() + messages = broker.xreadgroup(GROUP, CONSUMER, {STREAM: '>'}, count=1, block=1000) + for _stream, items in messages: + for message_id, fields in items: + handle_message(message_id, fields) + except Exception as exc: + print('worker loop error:', type(exc).__name__, flush=True) + time.sleep(1) + + +if __name__ == '__main__': + main() diff --git a/student-tests/attachments.py b/student-tests/attachments.py new file mode 100644 index 00000000..dae5bef5 --- /dev/null +++ b/student-tests/attachments.py @@ -0,0 +1,51 @@ +"""Negative checks for upload validation and chat-scoped access.""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[1] / 'tests')) +from client import Client, check + + +def main(): + client = Client('http://localhost:8080') + alice, bob, eve, chat = client.bootstrap() + sample = (Path(__file__).parents[1] / 'tests/fixtures/sample.png').read_bytes() + attachment = client.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', + 202, actor=alice, raw=sample, + headers={'Content-Type': 'image/png'}, schema='Attachment', + )['json'] + attachment_id = attachment['id'] + client.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=broken.png', + 415, actor=alice, raw=b'not an image', + headers={'Content-Type': 'image/png'}, + ) + client.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=wrong.jpg', + 415, actor=alice, raw=sample, + headers={'Content-Type': 'image/jpeg'}, + ) + client.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=large.png', + 413, actor=alice, raw=b'0' * (10 * 1024 * 1024 + 1), + headers={'Content-Type': 'image/png'}, + ) + client.call('GET', '/api/v1/attachments/' + attachment_id, 404, actor=eve) + client.call('GET', '/api/v1/attachments/' + attachment_id + '/content', 404, actor=eve) + other_chat = client.call( + 'POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'Другой чат', 'member_ids': [alice['id'], eve['id']]}, schema='Chat', + )['json'] + client.message(alice, other_chat, 'без чужого вложения') + response = client.call( + 'POST', '/api/v1/chats/' + other_chat['id'] + '/messages', 404, actor=alice, + body={'text': '', 'attachment_ids': [attachment_id]}, + headers={'Idempotency-Key': 'foreign-attachment-check'}, + ) + check(response['status'] == 404, 'foreign attachment was accepted') + print('PASS: attachment validation and chat-scoped access') + + +if __name__ == '__main__': + main() diff --git a/student-tests/idempotency.py b/student-tests/idempotency.py new file mode 100755 index 00000000..b7ad5e30 --- /dev/null +++ b/student-tests/idempotency.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Проверка общего Idempotency-Key через обе API-реплики.""" +import sys +import uuid +from concurrent.futures import ThreadPoolExecutor + +sys.path.insert(0, 'tests') +from client import Client, check + + +def main(): + client = Client('http://localhost:8080') + alice, _, _, chat = client.bootstrap() + key = 'student-' + uuid.uuid4().hex + body = {'text': 'одно сообщение при восьми повторах'} + + def send(_): + return client.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/messages', 201, + actor=alice, body=body, + headers={'Idempotency-Key': key}, schema='Message', + ) + + with ThreadPoolExecutor(max_workers=8) as pool: + responses = list(pool.map(send, range(8))) + ids = {item['json']['id'] for item in responses} + check(len(ids) == 1, 'concurrent retries created more than one message') + client.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, + actor=alice, body={'text': 'другой текст'}, + headers={'Idempotency-Key': key}, + ) + print('PASS: concurrent idempotency, one message id:', next(iter(ids))) + + +if __name__ == '__main__': + main() diff --git a/student-tests/outbox_recovery.py b/student-tests/outbox_recovery.py new file mode 100644 index 00000000..1561d71d --- /dev/null +++ b/student-tests/outbox_recovery.py @@ -0,0 +1,44 @@ +"""Show that a committed job survives while the worker/broker publisher is stopped.""" +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[1] / 'tests')) +from client import Client, check + + +def compose(*args): + subprocess.run(['docker', 'compose', *args], check=True, stdout=subprocess.DEVNULL) + + +def outbox_is_pending(attachment_id): + code = """import os,sys,psycopg +with psycopg.connect(os.environ['DATABASE_URL']) as c: + print(c.execute("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.id=%s AND j.published_at IS NULL", (sys.argv[1],)).fetchone()[0])""" + result = subprocess.run( + ['docker', 'compose', 'exec', '-T', 'api-1', 'python', '-c', code, str(attachment_id)], + check=True, text=True, capture_output=True, + ) + return result.stdout.strip().endswith('1') + + +def main(): + client = Client('http://localhost:8080') + alice, bob, _, chat = client.bootstrap() + data = (Path(__file__).parents[1] / 'tests/fixtures/sample.png').read_bytes() + compose('stop', 'worker') + try: + attachment = client.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=outbox.png', + 202, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment', + )['json'] + check(outbox_is_pending(attachment['id']), 'outbox intent was not durable') + finally: + compose('up', '-d', 'worker') + ready = client.wait_ready(bob, attachment) + check(ready['status'] == 'ready', 'outbox job was not published after worker restart') + print('PASS: committed outbox intent survived worker stop and reached ready') + + +if __name__ == '__main__': + main() diff --git a/student-tests/upload_load.py b/student-tests/upload_load.py new file mode 100644 index 00000000..aefbc6f7 --- /dev/null +++ b/student-tests/upload_load.py @@ -0,0 +1,30 @@ +"""Ten concurrent bounded uploads for a small local memory check.""" +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +import sys +import uuid + +sys.path.insert(0, str(Path(__file__).parents[1] / 'tests')) +from client import Client, check + + +def main(): + client = Client('http://localhost:8080') + alice, _, _, chat = client.bootstrap() + data = (Path(__file__).parents[1] / 'tests/fixtures/sample.png').read_bytes() + + def upload(number): + return client.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=load-' + str(number) + '.png', + 202, actor=alice, raw=data, + headers={'Content-Type': 'image/png'}, schema='Attachment', + ) + + with ThreadPoolExecutor(max_workers=10) as pool: + results = list(pool.map(upload, range(10))) + check(len(results) == 10, 'not all uploads completed') + print('PASS: 10 concurrent uploads, bytes per upload:', len(data)) + + +if __name__ == '__main__': + main() diff --git a/student-tests/variants.py b/student-tests/variants.py new file mode 100644 index 00000000..ebd65e3e --- /dev/null +++ b/student-tests/variants.py @@ -0,0 +1,36 @@ +"""Check all level-4 image variants and their metadata.""" +import hashlib +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[1] / 'tests')) +from client import Client, check + + +def main(): + client = Client('http://localhost:8080') + alice, bob, _, chat = client.bootstrap() + data = (Path(__file__).parents[1] / 'tests/fixtures/sample.png').read_bytes() + attachment = client.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=variants.png', + 202, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment', + )['json'] + ready = client.wait_ready(bob, attachment) + variants = {item['name']: item for item in ready['variants']} + check(set(variants) == {'thumbnail', 'crop', 'optimized'}, 'not all variants are ready') + check((variants['thumbnail']['width'], variants['thumbnail']['height']) == (256, 160), 'thumbnail size') + check((variants['crop']['width'], variants['crop']['height']) == (128, 128), 'crop size') + check(variants['optimized']['content_type'] == 'image/webp', 'optimized format') + for name, meta in variants.items(): + response = client.call( + 'GET', '/api/v1/attachments/' + attachment['id'] + '/content?variant=' + name, + 200, actor=bob, + ) + check(response['headers'].get_content_type() == meta['content_type'], name + ' content type') + check(len(response['bytes']) == meta['size_bytes'], name + ' size') + check(hashlib.sha256(response['bytes']).hexdigest() == meta['sha256'], name + ' checksum') + print('PASS: thumbnail, crop and optimized variants') + + +if __name__ == '__main__': + main() diff --git a/student-tests/worker_ack_recovery.py b/student-tests/worker_ack_recovery.py new file mode 100644 index 00000000..12a4ddcf --- /dev/null +++ b/student-tests/worker_ack_recovery.py @@ -0,0 +1,55 @@ +"""Inject a pause after DB ready and before broker ack, then restart the worker.""" +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[1] / 'tests')) +from client import Client, check + + +TEST_CONTAINER = 'mtusi2026lab5-ack-test' + + +def run(*args, capture=False): + return subprocess.run(['docker', 'compose', *args], check=True, text=True, + capture_output=capture) + + +def cleanup_test_container(): + subprocess.run(['docker', 'rm', '-f', TEST_CONTAINER], check=False, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def main(): + client = Client('http://localhost:8080') + alice, bob, _, chat = client.bootstrap() + data = (Path(__file__).parents[1] / 'tests/fixtures/sample.png').read_bytes() + run('stop', 'worker') + cleanup_test_container() + run('run', '--no-deps', '-d', '--name', TEST_CONTAINER, '-e', 'ACK_DELAY_SECONDS=60', 'worker') + try: + attachment = client.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=ack-window.png', + 202, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment', + )['json'] + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + state = client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=bob, schema='Attachment')['json'] + if state['status'] == 'ready': + break + time.sleep(0.2) + else: + raise AssertionError('attachment did not reach ready during ack pause') + time.sleep(2) + subprocess.run(['docker', 'stop', TEST_CONTAINER], check=True, stdout=subprocess.DEVNULL) + finally: + cleanup_test_container() + run('up', '-d', 'worker') + ready = client.wait_ready(bob, attachment) + check(len(ready['variants']) == 3, 'redelivery changed the variant set') + print('PASS: redelivery after ready-before-ack kept one result set') + + +if __name__ == '__main__': + main() diff --git a/student-tests/worker_recovery.py b/student-tests/worker_recovery.py new file mode 100644 index 00000000..5e055936 --- /dev/null +++ b/student-tests/worker_recovery.py @@ -0,0 +1,44 @@ +"""Worker stop/start scenario for the durable staging and broker path.""" +import subprocess +import sys +import time +import uuid +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[1] / 'tests')) +from client import Client, check + + +def compose(*args): + subprocess.run(['docker', 'compose', *args], check=True, stdout=subprocess.DEVNULL) + + +def main(): + client = Client('http://localhost:8080') + alice, bob, _, chat = client.bootstrap() + data = (Path(__file__).parents[1] / 'tests/fixtures/sample.png').read_bytes() + compose('stop', 'worker') + try: + attachment = client.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=stopped.png', + 202, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment', + )['json'] + client.message(alice, chat, '', attachments=[attachment['id']]) + client.call( + 'GET', '/api/v1/attachments/' + attachment['id'] + '/content', + 409, actor=bob, + ) + finally: + compose('up', '-d', 'worker') + ready = client.wait_ready(bob, attachment) + check(ready['status'] == 'ready', 'worker did not finish queued job') + content = client.call( + 'GET', '/api/v1/attachments/' + attachment['id'] + '/content?variant=thumbnail', + 200, actor=bob, + ) + check(content['headers'].get_content_type() == 'image/jpeg', 'thumbnail is not JPEG') + print('PASS: upload survived stopped worker and reached ready after restart') + + +if __name__ == '__main__': + main()