From 8c36a874ceb9a84aed596d456ff3629f9f9e5225 Mon Sep 17 00:00:00 2001 From: Otto <151371198+SaitWors@users.noreply.github.com> Date: Tue, 17 Nov 2026 03:37:05 +0300 Subject: [PATCH] =?UTF-8?q?=D1=80=D0=B5=D1=81=D1=82=D0=B5=D1=83=20=D0=B1?= =?UTF-8?q?=D0=B2=D1=822302=20=D0=BB=D0=B0=D0=B1=D0=B05?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 32 +- Dockerfile | 29 ++ REPORT.md | 43 +-- compose.yaml | 171 +++++++++ infra/generate-s3-config.py | 30 ++ migrations/01_initial.sql | 30 ++ migrations/02_chat_order.sql | 1 + migrations/03_idempotency.sql | 11 + migrations/04_attachments.sql | 22 ++ migrations/05_jobs.sql | 31 ++ nginx/default.conf | 27 ++ nginx/nginx.conf | 7 + requirements.txt | 20 + scripts/replay_failed.py | 28 ++ src/__init__.py | 0 src/db.py | 29 ++ src/errors.py | 14 + src/jobs.py | 268 ++++++++++++++ src/main.py | 663 ++++++++++++++++++++++++++++++++++ src/migrate.py | 36 ++ src/pagination.py | 47 +++ src/reconcile.py | 57 +++ src/run.py | 23 ++ src/sessions.py | 33 ++ src/storage.py | 43 +++ student-tests/test_images.py | 33 ++ 26 files changed, 1680 insertions(+), 48 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 infra/generate-s3-config.py create mode 100644 migrations/01_initial.sql create mode 100644 migrations/02_chat_order.sql create mode 100644 migrations/03_idempotency.sql create mode 100644 migrations/04_attachments.sql create mode 100644 migrations/05_jobs.sql create mode 100644 nginx/default.conf create mode 100644 nginx/nginx.conf create mode 100644 requirements.txt create mode 100644 scripts/replay_failed.py create mode 100644 src/__init__.py create mode 100644 src/db.py create mode 100644 src/errors.py create mode 100644 src/jobs.py create mode 100644 src/main.py create mode 100644 src/migrate.py create mode 100644 src/pagination.py create mode 100644 src/reconcile.py create mode 100644 src/run.py create mode 100644 src/sessions.py create mode 100644 src/storage.py create mode 100644 student-tests/test_images.py diff --git a/.env.example b/.env.example index 119384b5..17ca9c6f 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,17 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +# Local synthetic lab credentials only. Change all passwords before use. +POSTGRES_DB=messenger +POSTGRES_USER=messenger +POSTGRES_PASSWORD=change-this-for-local-demo SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 +S3_ROOT_USER=lab4admin +S3_ROOT_PASSWORD=change-minio-admin-password-123 +S3_ACCESS_KEY_ID=lab4api +S3_SECRET_ACCESS_KEY=change-minio-api-password-123 +# Policy in infra/s3-policy.json is scoped to this bucket; keep unchanged. S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 + +BROKER_HOST=broker JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 +JOB_LEASE_SECONDS=90 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..43c768b0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.14.8-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab + +WORKDIR /app +COPY requirements.txt ./ +RUN python -m pip install --no-cache-dir -r requirements.txt +RUN groupadd --gid 10001 appuser \ + && useradd --uid 10001 --gid appuser --no-create-home --shell /usr/sbin/nologin appuser + +COPY src ./src +COPY migrations ./migrations +COPY tests ./tests +COPY student-tests ./student-tests +COPY scripts ./scripts +COPY contracts ./contracts +COPY lab.json ./ + +USER 10001:10001 +EXPOSE 8080 +STOPSIGNAL SIGTERM +HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \ + CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready', timeout=2).close()"] + +CMD ["python", "-m", "src.run"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..61e7e3c7 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,24 @@ -# Отчёт — лабораторная 5 +# Отчёт ЛР5 — Рестеу Д. А., БВТ2302 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +**Целевая оценка:** 4. -## Запуск из чистого clone +## Архитектура -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Существующие API1/API2 + Nginx + PostgreSQL + Redis сессий + SeaweedFS сохранены из ЛР4. Добавлены отдельный broker (Redis Streams с AOF), dispatcher и worker. HTTP upload принимает полные bytes, проверяет формат/размер, кладёт оригинал в приватный `staging/` и атомарно сохраняет queued и outbox в PostgreSQL. Только после этого возвращает 202. Отдельный dispatcher публикует IDs заданий в durable stream. Worker читает stream в consumer group и подтверждает сообщения после устойчивого эффекта. -## Архитектура и решения +Оригинал: побайтная копия из staging в originals. Thumbnail — JPEG, до 256×256, без увеличения. Crop — JPEG, центрированный квадрат 128×128. Optimized — WebP, до 1280×1280 без увеличения. EXIF orientation применяется, EXIF не сохраняется в производных файлах; прозрачность компонуется на белый. JPEG quality=85; WebP quality=80/method=4. -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Состояния: queued → processing → ready/failed, с повтором queued при временном отказе. Ограничение 5 попыток, backoff с jitter, 90-секундный lease, выделенный worker с 384 MiB и 0.75 CPU, обработки по одному заданию. Повторная доставка уже ready — no-op. Эффекты S3 имеют детерминированные ключи, метаданные вариантов записываются атомарно. -## Доказательства критериев +## Реальные проверки (заполнить после запуска) -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +- `docker compose config --quiet`: НЕ ПРОВЕРЕНО. +- `python tests/smoke.py`: НЕ ПРОВЕРЕНО. +- Остановка worker → upload 202 → restart → ready: НЕ ПРОВЕРЕНО. +- COMMIT до publish → восстановление dispatcher: НЕ ПРОВЕРЕНО. +- Crash после S3 до БД/ack → восстановление lease: НЕ ПРОВЕРЕНО. +- failed → явный replay: НЕ ПРОВЕРЕНО. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Ограничения -## Эксперименты и отказы - -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | - -## Ограничения и следующие шаги - -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Единственная PostgreSQL, S3 и единый broker остаются точками отказа. Отказ worker не блокирует HTTP upload, если staging и БД исправны. Accepted означает сохранённые bytes и намерение обработки, но не готовое изображение. Отсутствие именно однократной доставки компенсируется идемпотентной записью эффектов. Staging удаляется только после коммита ready, при ошибках может оставаться; для production требуется дополнительная периодическая очистка бесхозных объектов с retention-политикой. Текущий worker использует ограниченный по времени lease, но жёсткого прерывания отдельной долгой декодировки нет; ресурсный лимит Docker ограничивает память. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..747a4b38 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,171 @@ +services: + postgres: + image: postgres:17.11-bookworm + environment: + POSTGRES_DB: ${POSTGRES_DB:?Set POSTGRES_DB in .env} + POSTGRES_USER: ${POSTGRES_USER:?Set POSTGRES_USER in .env} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + volumes: [postgres_data:/var/lib/postgresql/data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + + redis: + image: redis:8.2.10-bookworm + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"] + volumes: [redis_data:/data] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + + s3_config: + image: python:3.14.8-slim-bookworm + command: ["python", "/bootstrap/generate-s3-config.py"] + environment: + S3_ROOT_USER: ${S3_ROOT_USER:?Set S3_ROOT_USER} + S3_ROOT_PASSWORD: ${S3_ROOT_PASSWORD:?Set S3_ROOT_PASSWORD} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + volumes: + - ./infra/generate-s3-config.py:/bootstrap/generate-s3-config.py:ro + - s3_config_data:/s3-config + restart: "no" + + s3: + image: chrislusf/seaweedfs:4.47 + command: ["mini", "-dir=/data", "-s3.config=/s3-config/s3.json", "-bucket=messenger-attachments"] + volumes: + - s3_data:/data + - s3_config_data:/s3-config:ro + depends_on: + s3_config: {condition: service_completed_successfully} + # Nothing is published: API and Admin UI exist only inside Compose network. + restart: unless-stopped + + migrate: + build: . + command: ["python", "-m", "src.migrate"] + environment: &db_env + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + depends_on: + postgres: + condition: service_healthy + restart: "no" + + api1: + build: . + environment: &api_env + <<: *db_env + INSTANCE_ID: api1 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_REGION: us-east-1 + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + depends_on: &api_depends + postgres: {condition: service_healthy} + redis: {condition: service_healthy} + migrate: {condition: service_completed_successfully} + s3: {condition: service_started} + healthcheck: &api_health + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).close()"] + interval: 5s + timeout: 5s + retries: 4 + start_period: 15s + cpus: 0.50 + mem_limit: 256m + stop_grace_period: 15s + restart: unless-stopped + + api2: + build: . + environment: + <<: *api_env + INSTANCE_ID: api2 + depends_on: *api_depends + healthcheck: *api_health + cpus: 0.50 + mem_limit: 256m + stop_grace_period: 15s + restart: unless-stopped + + broker: + image: redis:8.2.10-bookworm + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"] + volumes: [broker_data:/data] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + + dispatcher: + build: . + command: ["python", "-m", "src.jobs", "dispatch"] + healthcheck: + disable: true + environment: &job_env + <<: *api_env + BROKER_HOST: broker + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90} + depends_on: + migrate: {condition: service_completed_successfully} + broker: {condition: service_healthy} + s3: {condition: service_started} + restart: unless-stopped + mem_limit: 256m + + worker: + build: . + command: ["python", "-m", "src.jobs", "worker"] + healthcheck: + disable: true + environment: + <<: *job_env + LAB5_CRASH_AFTER_S3: ${LAB5_CRASH_AFTER_S3:-0} + depends_on: + migrate: {condition: service_completed_successfully} + broker: {condition: service_healthy} + s3: {condition: service_started} + restart: unless-stopped + mem_limit: 384m + cpus: 0.75 + stop_grace_period: 15s + + lb: + image: nginx:1.28.0-alpine + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./nginx/default.conf:/etc/nginx/conf.d/default.conf:ro + ports: ["127.0.0.1:8080:8080"] + depends_on: + api1: {condition: service_healthy} + api2: {condition: service_healthy} + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"] + interval: 5s + timeout: 4s + retries: 3 + restart: unless-stopped + +volumes: + broker_data: + postgres_data: + redis_data: + s3_data: + s3_config_data: diff --git a/infra/generate-s3-config.py b/infra/generate-s3-config.py new file mode 100644 index 00000000..2e319f4a --- /dev/null +++ b/infra/generate-s3-config.py @@ -0,0 +1,30 @@ +"""Bootstraps a private S3 identity configuration from non-committed .env. + +The server reads it from a shared internal-only configuration volume. +""" +import json +import os +from pathlib import Path + +bucket = os.environ["S3_BUCKET"] +assert bucket == "messenger-attachments", "Fixed bucket name; change policy and config deliberately" +config = { + "identities": [ + { + "name": "lab4-admin", + "credentials": [{"accessKey": os.environ["S3_ROOT_USER"], + "secretKey": os.environ["S3_ROOT_PASSWORD"]}], + "actions": ["Admin", "Read", "Write", "List", "Tagging"], + }, + { + "name": "lab4-api", + "credentials": [{"accessKey": os.environ["S3_ACCESS_KEY_ID"], + "secretKey": os.environ["S3_SECRET_ACCESS_KEY"]}], + "actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"], + }, + ], +} +target = Path("/s3-config/s3.json") +target.write_text(json.dumps(config, indent=2), encoding="utf-8") +target.chmod(0o644) # Only mounted into S3 service; SeaweedFS runs non-root on some images. +print("PASS: private S3 identity config generated; credentials not logged") diff --git a/migrations/01_initial.sql b/migrations/01_initial.sql new file mode 100644 index 00000000..c26001b5 --- /dev/null +++ b/migrations/01_initial.sql @@ -0,0 +1,30 @@ +CREATE TABLE users ( + id UUID PRIMARY KEY, + username VARCHAR(32) NOT NULL UNIQUE, + display_name VARCHAR(100) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id), + user_id UUID NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + sender_id UUID NOT NULL REFERENCES users(id), + text VARCHAR(4000) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/migrations/02_chat_order.sql b/migrations/02_chat_order.sql new file mode 100644 index 00000000..0922abd8 --- /dev/null +++ b/migrations/02_chat_order.sql @@ -0,0 +1 @@ +CREATE INDEX chats_order_idx ON chats (created_at, id); diff --git a/migrations/03_idempotency.sql b/migrations/03_idempotency.sql new file mode 100644 index 00000000..ac3f7681 --- /dev/null +++ b/migrations/03_idempotency.sql @@ -0,0 +1,11 @@ +CREATE TABLE message_requests ( + sender_id UUID NOT NULL REFERENCES users(id), + chat_id UUID NOT NULL REFERENCES chats(id), + request_key VARCHAR(128) NOT NULL, + request_hash CHAR(64) NOT NULL, + message_id UUID REFERENCES messages(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (sender_id, chat_id, request_key) +); + +CREATE INDEX message_requests_created_idx ON message_requests (created_at); diff --git a/migrations/04_attachments.sql b/migrations/04_attachments.sql new file mode 100644 index 00000000..fc54bb5c --- /dev/null +++ b/migrations/04_attachments.sql @@ -0,0 +1,22 @@ +-- Object bytes are deliberately NOT stored in the database. +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + owner_id UUID NOT NULL REFERENCES users(id), + filename VARCHAR(255) NOT NULL, + content_type VARCHAR(10) NOT NULL CHECK (content_type IN ('image/png','image/jpeg')), + size_bytes INTEGER NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 CHAR(64) NOT NULL, + status VARCHAR(12) NOT NULL DEFAULT 'ready' CHECK (status = 'ready'), + object_key TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX attachments_chat_idx ON attachments (chat_id); +CREATE TABLE message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id), + attachment_id UUID NOT NULL REFERENCES attachments(id), + position INTEGER NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position) +); diff --git a/migrations/05_jobs.sql b/migrations/05_jobs.sql new file mode 100644 index 00000000..1153e6cf --- /dev/null +++ b/migrations/05_jobs.sql @@ -0,0 +1,31 @@ +ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status_check; +ALTER TABLE attachments ADD CONSTRAINT attachments_status_check CHECK (status IN ('queued','processing','ready','failed')); +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS error_code TEXT; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS staging_key TEXT; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS attempts INTEGER NOT NULL DEFAULT 0; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS lease_token UUID; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS lease_until TIMESTAMPTZ; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(); +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS pipeline_version INTEGER NOT NULL DEFAULT 1; +CREATE TABLE IF NOT EXISTS attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL DEFAULT 1, + name TEXT NOT NULL CHECK (name IN ('thumbnail','crop','optimized')), + object_key TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + sha256 CHAR(64) NOT NULL, + PRIMARY KEY (attachment_id,pipeline_version,name) +); +CREATE TABLE IF NOT EXISTS attachment_outbox ( + id BIGSERIAL PRIMARY KEY, + attachment_id UUID NOT NULL REFERENCES attachments(id), + pipeline_version INTEGER NOT NULL DEFAULT 1, + available_at TIMESTAMPTZ NOT NULL DEFAULT now(), + published_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS attachment_outbox_pending ON attachment_outbox(available_at,id) WHERE published_at IS NULL; +CREATE INDEX IF NOT EXISTS attachment_jobs_lease ON attachments(status,lease_until); diff --git a/nginx/default.conf b/nginx/default.conf new file mode 100644 index 00000000..976cb965 --- /dev/null +++ b/nginx/default.conf @@ -0,0 +1,27 @@ +upstream api_backends { + zone api_backends 64k; + resolver 127.0.0.11 valid=2s ipv6=off; + server api1:8080 resolve max_fails=1 fail_timeout=2s; + server api2:8080 resolve max_fails=1 fail_timeout=2s; +} + +server { + listen 8080; + server_name _; + client_max_body_size 0; # API enforces 10 MiB during streamed reads (uniform JSON errors). + + location / { + proxy_pass http://api_backends; + proxy_request_buffering off; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Connection ""; + proxy_connect_timeout 1s; + proxy_read_timeout 45s; + proxy_send_timeout 45s; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + } +} diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 00000000..3d9e6a54 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,7 @@ +worker_processes auto; +events { worker_connections 1024; } +http { + access_log /dev/stdout; + error_log /dev/stderr warn; + include /etc/nginx/conf.d/*.conf; +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..c99d3530 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,20 @@ +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +click==8.5.0 +fastapi==0.143.0 +h11==0.16.0 +idna==3.20 +opentelemetry-api==1.45.1 +pydantic==2.14.1 +pydantic_core==2.50.1 +starlette==1.7.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +uvicorn==0.54.0 +psycopg[binary]==3.3.6 +psycopg-pool==3.3.3 +argon2-cffi==25.1.0 +redis==8.1.0 +boto3==1.40.52 +Pillow==12.0.0 diff --git a/scripts/replay_failed.py b/scripts/replay_failed.py new file mode 100644 index 00000000..ace07d90 --- /dev/null +++ b/scripts/replay_failed.py @@ -0,0 +1,28 @@ +import argparse +from uuid import UUID + +from src.db import pool + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('attachment_id') + args = parser.parse_args() + identifier = UUID(args.attachment_id) + pool.open(wait=True) + with pool.connection() as conn: + with conn.transaction(): + row = conn.execute('SELECT status,pipeline_version FROM attachments WHERE id=%s FOR UPDATE', + (identifier,)).fetchone() + if row is None or row['status'] != 'failed': + raise SystemExit('Только failed-вложение можно повторно отправить в обработку') + conn.execute('''UPDATE attachments SET status='queued',attempts=0,error_code=NULL, + lease_token=NULL,lease_until=NULL,next_attempt_at=now(),updated_at=now() + WHERE id=%s''', (identifier,)) + conn.execute('INSERT INTO attachment_outbox(attachment_id,pipeline_version) VALUES (%s,%s)', + (identifier,row['pipeline_version'])) + print('QUEUED:',identifier) + + +if __name__=='__main__': + main() diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..fbb7bde3 --- /dev/null +++ b/src/db.py @@ -0,0 +1,29 @@ +import os + +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + + +def connection_kwargs(): + return { + "host": os.getenv("POSTGRES_HOST", "postgres"), + "port": int(os.getenv("POSTGRES_PORT", "5432")), + "dbname": os.environ["POSTGRES_DB"], + "user": os.environ["POSTGRES_USER"], + "password": os.environ["POSTGRES_PASSWORD"], + "connect_timeout": 5, + "options": "-c statement_timeout=5000 -c lock_timeout=3000", + "autocommit": True, + "row_factory": dict_row, + } + + +pool = ConnectionPool( + kwargs=connection_kwargs(), + min_size=1, + max_size=5, + max_waiting=20, + timeout=3, + check=ConnectionPool.check_connection, + open=False, +) diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..73e36a37 --- /dev/null +++ b/src/errors.py @@ -0,0 +1,14 @@ +class ApiError(Exception): + def __init__(self, status, code, message): + super().__init__(message) + self.status = status + self.code = code + self.message = message + + +def invalid(message="Неверный запрос"): + raise ApiError(400, "invalid_request", message) + + +def unauthorized(): + raise ApiError(401, "unauthorized", "Неверные данные авторизации") diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 00000000..2f1ceb7a --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,268 @@ +import hashlib +import io +import os +import random +import time +from datetime import datetime, timedelta, timezone +from uuid import UUID, uuid4 + +import redis +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, ImageOps, UnidentifiedImageError + +from src.db import pool +from src.storage import bucket, s3, MAX_IMAGE_PIXELS + +STREAM = 'lab5:image-jobs' +GROUP = 'lab5-workers' +ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5')) +LEASE_SECONDS = int(os.getenv('JOB_LEASE_SECONDS', '90')) +WORKER_MAX_PIXELS = int(os.getenv('MAX_IMAGE_PIXELS', '40000000')) +client = redis.Redis(host=os.getenv('BROKER_HOST', 'broker'), port=6379, decode_responses=True, + socket_connect_timeout=3, socket_timeout=3) +Image.MAX_IMAGE_PIXELS = WORKER_MAX_PIXELS + + +def stage_upload(file, key, size, content_type): + file.seek(0) + s3.put_object(Bucket=bucket, Key=key, Body=file, ContentLength=size, ContentType=content_type) + + +def queue_attachment(attachment_id, chat_id, owner_id, filename, content_type, size, digest, stage_key): + with pool.connection() as conn: + with conn.transaction(): + membership = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', + (chat_id, owner_id)).fetchone() + if not membership: + return None + item = conn.execute('''INSERT INTO attachments + (id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,object_key,staging_key) + VALUES (%s,%s,%s,%s,%s,%s,%s,'queued',%s,%s) RETURNING *''', + (attachment_id, chat_id, owner_id, filename, content_type, size, digest, + 'originals/'+attachment_id.hex, stage_key)).fetchone() + conn.execute('INSERT INTO attachment_outbox (attachment_id) VALUES (%s)', (attachment_id,)) + return item + + +def variants_for(attachment_id, pipeline_version=1): + with pool.connection() as conn: + rows = conn.execute('''SELECT name,content_type,size_bytes,width,height,sha256 + FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name''', + (attachment_id, pipeline_version)).fetchall() + return [dict(r) for r in rows] + + +def fetch_variant(attachment_id, name, pipeline_version=1): + with pool.connection() as conn: + return conn.execute('''SELECT * FROM attachment_variants + WHERE attachment_id=%s AND pipeline_version=%s AND name=%s''', + (attachment_id, pipeline_version, name)).fetchone() + + +def _make_images(data): + if len(data) > 10485760: + raise ValueError('size_limit') + with Image.open(io.BytesIO(data)) as probe: + w,h = probe.size + if w*h > WORKER_MAX_PIXELS: + raise ValueError('pixel_limit') + image = ImageOps.exif_transpose(probe) + image.load() + if image.mode == 'RGB': + image = image.copy() + else: + rgba = image.convert('RGBA') + background = Image.new('RGB', rgba.size, 'white') + background.paste(rgba, mask=rgba.getchannel('A')) + image = background + variants = [] + thumb = image.copy() + thumb.thumbnail((256,256), Image.Resampling.LANCZOS) + variants.append(('thumbnail',thumb,'JPEG',{'quality':85,'optimize':True})) + crop = ImageOps.fit(image,(128,128),Image.Resampling.LANCZOS,centering=(0.5,0.5)) + variants.append(('crop',crop,'JPEG',{'quality':85,'optimize':True})) + opt = image.copy() + opt.thumbnail((1280,1280),Image.Resampling.LANCZOS) + variants.append(('optimized',opt,'WEBP',{'quality':80,'method':4})) + out = [] + for name,im,fmt,kwargs in variants: + buf=io.BytesIO() + im.save(buf, format=fmt, **kwargs) + raw=buf.getvalue() + out.append((name,raw,'image/webp' if fmt=='WEBP' else 'image/jpeg',im.width,im.height,hashlib.sha256(raw).hexdigest())) + return out + + +def _publish_pending(): + with pool.connection() as conn: + with conn.transaction(): + rows=conn.execute('''SELECT o.id,o.attachment_id,o.pipeline_version + FROM attachment_outbox o JOIN attachments a ON a.id=o.attachment_id + WHERE o.published_at IS NULL AND o.available_at<=now() AND a.status='queued' + ORDER BY o.id LIMIT 25 FOR UPDATE OF o SKIP LOCKED''').fetchall() + for row in rows: + client.xadd(STREAM,{'attachment_id':str(row['attachment_id']), + 'job_id':str(row['attachment_id']), + 'pipeline_version':str(row['pipeline_version'])}) + conn.execute('UPDATE attachment_outbox SET published_at=now() WHERE id=%s',(row['id'],)) + + +def _recover(): + with pool.connection() as conn: + with conn.transaction(): + expired=conn.execute('''UPDATE attachments SET status='queued',lease_token=NULL, + lease_until=NULL,next_attempt_at=now(),updated_at=now() + WHERE status='processing' AND lease_untilnow()-interval '30 seconds') LIMIT 50''').fetchall() + for row in retry: + conn.execute('INSERT INTO attachment_outbox(attachment_id,pipeline_version) VALUES (%s,%s)', + (row['id'],row['pipeline_version'])) + + +def dispatcher(): + pool.open(wait=True) + while True: + try: + _recover() + _publish_pending() + except Exception as exc: + print('dispatcher:',type(exc).__name__,flush=True) + time.sleep(2) + + +def _claim(attachment_id, version): + token=uuid4() + with pool.connection() as conn: + with conn.transaction(): + row=conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(attachment_id,)).fetchone() + if not row or row['pipeline_version']!=version or row['status'] in ('ready','failed'): + return None + if row['status']=='processing' or row['next_attempt_at']>datetime.now(timezone.utc): + return None + conn.execute('''UPDATE attachments SET status='processing',attempts=attempts+1, + lease_token=%s,lease_until=now()+(%s * interval '1 second'),updated_at=now() + WHERE id=%s''',(token,LEASE_SECONDS,attachment_id)) + return (row,token) + + +def _finish(row,token,outputs): + with pool.connection() as conn: + with conn.transaction(): + locked=conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(row['id'],)).fetchone() + if not locked or locked['status']!='processing' or locked['lease_token']!=token: + return False + for name,data,ct,w,h,sha in outputs: + key=f"variants/{row['id'].hex}/v{row['pipeline_version']}/{name}" + conn.execute('''INSERT INTO attachment_variants + (attachment_id,pipeline_version,name,object_key,content_type,size_bytes,width,height,sha256) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) + ON CONFLICT (attachment_id,pipeline_version,name) DO UPDATE SET + object_key=EXCLUDED.object_key,content_type=EXCLUDED.content_type, + size_bytes=EXCLUDED.size_bytes,width=EXCLUDED.width,height=EXCLUDED.height, + sha256=EXCLUDED.sha256''', + (row['id'],row['pipeline_version'],name,key,ct,len(data),w,h,sha)) + conn.execute('''UPDATE attachments SET status='ready',error_code=NULL, + lease_token=NULL,lease_until=NULL,updated_at=now() WHERE id=%s''',(row['id'],)) + return True + + +def _failure(row,token,exc): + permanent = isinstance(exc,(ValueError,UnidentifiedImageError,Image.DecompressionBombError)) + with pool.connection() as conn: + with conn.transaction(): + current=conn.execute('SELECT attempts,lease_token,status FROM attachments WHERE id=%s FOR UPDATE', + (row['id'],)).fetchone() + if not current or current['status']!='processing' or current['lease_token']!=token: + return + attempts=current['attempts'] + if permanent or attempts>=ATTEMPTS: + code='invalid_image' if permanent else 'retry_exhausted' + conn.execute('''UPDATE attachments SET status='failed',error_code=%s, + lease_token=NULL,lease_until=NULL,updated_at=now() WHERE id=%s''',(code,row['id'])) + else: + delay=min(60, 2**min(attempts,5))+random.uniform(0,1.5) + conn.execute('''UPDATE attachments SET status='queued',error_code=NULL, + lease_token=NULL,lease_until=NULL, + next_attempt_at=now()+(%s * interval '1 second'),updated_at=now() + WHERE id=%s''',(delay,row['id'])) + conn.execute('''INSERT INTO attachment_outbox(attachment_id,pipeline_version,available_at) + VALUES (%s,%s,now()+(%s * interval '1 second'))''', + (row['id'],row['pipeline_version'],delay)) + + +def _process(attachment_id,version): + claim=_claim(attachment_id,version) + if not claim: + return + row,token=claim + try: + response=s3.get_object(Bucket=bucket,Key=row['staging_key']) + data=response['Body'].read(10485761) + response['Body'].close() + if hashlib.sha256(data).hexdigest()!=row['sha256']: + raise ValueError('checksum_mismatch') + outputs=_make_images(data) + s3.copy_object(Bucket=bucket,CopySource={'Bucket':bucket,'Key':row['staging_key']}, + Key=row['object_key'],ContentType=row['content_type'],MetadataDirective='REPLACE') + for name,content,ct,w,h,sha in outputs: + s3.put_object(Bucket=bucket, + Key=f"variants/{row['id'].hex}/v{version}/{name}", + Body=content,ContentType=ct) + if os.getenv('LAB5_CRASH_AFTER_S3')=='1': + os._exit(91) + if _finish(row,token,outputs): + try: + s3.delete_object(Bucket=bucket,Key=row['staging_key']) + except Exception: + pass + except Exception as exc: + _failure(row,token,exc) + + +def worker(): + pool.open(wait=True) + consumer=os.getenv('HOSTNAME','worker')+'-'+uuid4().hex[:6] + try: + client.xgroup_create(STREAM,GROUP,id='0',mkstream=True) + except redis.exceptions.ResponseError as e: + if 'BUSYGROUP' not in str(e): + raise + last_scan=0 + while True: + try: + if time.monotonic()-last_scan>5: + pending=client.xautoclaim(STREAM,GROUP,consumer,min_idle_time=LEASE_SECONDS*1000, + start_id='0-0',count=10) + for msgid,fields in pending[1]: + try: + _process(UUID(fields['attachment_id']),int(fields.get('pipeline_version','1'))) + finally: + client.xack(STREAM,GROUP,msgid) + last_scan=time.monotonic() + entries=client.xreadgroup(GROUP,consumer,{STREAM:'>'},count=1,block=2000) + for _,messages in entries: + for msgid,fields in messages: + try: + _process(UUID(fields['attachment_id']),int(fields.get('pipeline_version','1'))) + except Exception as e: + print('worker:',type(e).__name__,flush=True) + else: + client.xack(STREAM,GROUP,msgid) + except Exception as e: + print('broker:',type(e).__name__,flush=True) + time.sleep(1) + + +if __name__=='__main__': + import sys + if len(sys.argv)>1 and sys.argv[1]=='dispatch': + dispatcher() + else: + worker() diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..d7e1b22a --- /dev/null +++ b/src/main.py @@ -0,0 +1,663 @@ +import base64 +import binascii +import hashlib +import json +import os +import logging +import re +import time +import tempfile +import threading +import unicodedata +from io import BytesIO +from contextlib import asynccontextmanager +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError +from fastapi import FastAPI, Header, Query, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, Response, StreamingResponse +from psycopg import InterfaceError, OperationalError +from psycopg.errors import ForeignKeyViolation, UniqueViolation +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from redis.exceptions import RedisError +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException + +from src.db import pool +from src.jobs import stage_upload, queue_attachment, variants_for, fetch_variant +from src.errors import ApiError, invalid, unauthorized +from src.pagination import encode_cursor, parse_page +from src.sessions import issue_session, redis_client +from src.storage import s3, bucket, check_storage, upload_original, download_original, delete_original, StorageFailure, MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, UnidentifiedImageError +Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS + + +log = logging.getLogger("messenger") +password_hasher = PasswordHasher() +INSTANCE_ID = os.getenv("INSTANCE_ID", uuid4().hex) +ready = False +IMAGE_DECODE_SLOT = threading.BoundedSemaphore(1) # prevent concurrent decoded-image RAM spikes + + +def parse_uuid(value): + try: + return UUID(value) + except (ValueError, TypeError, AttributeError): + invalid("Некорректный UUID") + + +def row_response(row): + result = {} + for key, value in row.items(): + if isinstance(value, UUID): + value = str(value) + elif isinstance(value, datetime): + value = value.astimezone(timezone.utc).isoformat( + timespec="microseconds" + ).replace("+00:00", "Z") + elif key in ("member_ids", "attachment_ids"): + value = [str(member) for member in value] + result[key] = value + return result + + +class InputModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + @field_validator("*", check_fields=False) + @classmethod + def reject_blank(cls, value): + if isinstance(value, str) and not value.strip(): + raise ValueError("Строка не должна быть пустой") + return value + + @field_validator("display_name", "title", "text", check_fields=False) + @classmethod + def reject_nul(cls, value): + if "\u0000" in value: + raise ValueError("Нулевой символ недопустим в текстовом поле") + return value + + +class CreateUser(InputModel): + username: str = Field(min_length=3, max_length=32) + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("username") + @classmethod + def validate_username(cls, value): + if re.fullmatch(r"[a-z0-9_]{3,32}", value) is None: + raise ValueError("Некорректный username") + return value + + +class CreateChat(InputModel): + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + text: str = Field(min_length=0, max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + @model_validator(mode="after") + def validate_message(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError("Нужен текст или вложение") + if "\x00" in self.text: + raise ValueError("Недопустимый текст") + if len(set(self.attachment_ids)) != len(self.attachment_ids): + raise ValueError("Повторные attachment_ids") + for value in self.attachment_ids: + try: + UUID(value) + except (ValueError, TypeError): + raise ValueError("Некорректный attachment_id") from None + return self + + +def bearer_token(value): + scheme, _, token = (value or "").partition(" ") + if scheme.lower() != "bearer" or not token: + unauthorized() + return token + + +def current_user(value): + token = bearer_token(value) + user_id = redis_client.get("session:" + token) + if user_id is None: + unauthorized() + with pool.connection() as conn: + user = conn.execute( + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", + (UUID(user_id),), + ).fetchone() + if user is None: + unauthorized() + return row_response(user) + + +@asynccontextmanager +async def lifespan(app): + global ready + try: + await run_in_threadpool(pool.open, wait=True, timeout=10) + await run_in_threadpool(redis_client.ping) + # Container may have started before the S3 service finished bootstrap. + storage_ok = False + for _ in range(30): + try: + await run_in_threadpool(check_storage) + storage_ok = True + break + except (BotoCoreError, ClientError, StorageFailure): + import asyncio + await asyncio.sleep(2) + if not storage_ok: + raise RuntimeError("S3 not ready") + ready = True + log.info("startup ready=true") + yield + finally: + ready = False + redis_client.close() + pool.close() + log.info("shutdown complete") + + +app = FastAPI(title="Lab 4 Messenger", lifespan=lifespan) + + +def error_response(request, status, code, message): + request_id = request.state.request_id + headers = {"X-Request-Id": request_id} + if status == 401: + if request.method == "POST" and request.url.path == "/api/v1/auth/sessions": + headers["WWW-Authenticate"] = 'Basic realm="messenger", charset="UTF-8"' + else: + headers["WWW-Authenticate"] = 'Bearer realm="messenger"' + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + headers=headers, + ) + + +@app.exception_handler(ApiError) +async def api_error(request, exc): + return error_response(request, exc.status, exc.code, exc.message) + + +@app.exception_handler(RequestValidationError) +async def validation_error(request, exc): + return error_response(request, 400, "invalid_request", "Некорректные поля или JSON") + + +@app.exception_handler((StorageFailure)) +async def storage_error(request, exc): + return error_response(request, 503, "unavailable", "Хранилище недоступно") + +@app.exception_handler(ClientError) +async def boto_error(request, exc): + return error_response(request, 503, "unavailable", "Хранилище недоступно") + +@app.exception_handler(StarletteHTTPException) +async def http_error(request, exc): + code = "not_found" if exc.status_code == 404 else "invalid_request" + return error_response(request, exc.status_code, code, "Некорректный запрос") + + +@app.middleware("http") +async def process_request(request: Request, call_next): + request.state.request_id = str(uuid4()) + started = time.perf_counter() + try: + is_login = request.method == "POST" and request.url.path == "/api/v1/auth/sessions" + public = request.method == "POST" and request.url.path in ( + "/api/v1/users", "/api/v1/auth/sessions" + ) + if request.url.path.startswith("/api/v1/") and not public: + request.state.user = await run_in_threadpool( + current_user, request.headers.get("Authorization") + ) + if is_login: + if await request.body(): + invalid("Для входа тело запроса должно быть пустым") + elif request.method == "POST" and request.url.path.startswith("/api/v1/"): + media_type = request.headers.get("content-type", "").split(";")[0].strip().lower() + if re.fullmatch(r"/api/v1/chats/[^/]+/attachments", request.url.path): + if media_type not in ("image/png", "image/jpeg"): + raise ApiError(415, "unsupported_media_type", "Только PNG/JPEG") + elif media_type != "application/json": + raise ApiError(415, "unsupported_media_type", "Нужен application/json") + response = await call_next(request) + except ApiError as exc: + response = error_response(request, exc.status, exc.code, exc.message) + except (OperationalError, InterfaceError, RedisError, StorageFailure, BotoCoreError, ClientError): + response = error_response(request, 503, "unavailable", "Хранилище недоступно") + except Exception as exc: + log.error("request_failed type=%s request_id=%s", + type(exc).__name__, request.state.request_id) + response = error_response(request, 500, "internal_error", "Внутренняя ошибка") + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + route = request.scope.get("route") + log.info("request method=%s route=%s status=%s duration_ms=%.2f request_id=%s", + request.method, getattr(route, "path", "unmatched"), response.status_code, + (time.perf_counter() - started) * 1000, request.state.request_id) + return response + + +@app.get("/health/live") +def live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def readiness(): + if not ready: + raise ApiError(503, "unavailable", "Приложение не готово") + with pool.connection() as conn: + conn.execute("SELECT 1") + redis_client.ping() + check_storage() + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +def create_user(body: CreateUser): + password_hash = password_hasher.hash(body.password) + try: + with pool.connection() as conn: + user = conn.execute( + "INSERT INTO users (id, username, display_name, password_hash) " + "VALUES (%s, %s, %s, %s) RETURNING id, username, display_name, created_at", + (uuid4(), body.username, body.display_name, password_hash), + ).fetchone() + except UniqueViolation as exc: + if exc.diag.constraint_name == "users_username_key": + raise ApiError(409, "conflict", "Username занят") from None + raise + return row_response(user) + + +@app.post("/api/v1/auth/sessions", status_code=201) +def login(authorization: str | None = Header(default=None)): + scheme, _, encoded = (authorization or "").partition(" ") + if scheme.lower() != "basic" or not encoded: + unauthorized() + try: + decoded = base64.b64decode(encoded, validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + except (ValueError, binascii.Error): + unauthorized() + if re.fullmatch(r"[a-z0-9_]{3,32}", username) is None or not 12 <= len(password) <= 128: + unauthorized() + with pool.connection() as conn: + user = conn.execute( + "SELECT id, username, display_name, password_hash, created_at " + "FROM users WHERE username = %s", (username,), + ).fetchone() + if user is None: + unauthorized() + try: + password_hasher.verify(user["password_hash"], password) + except VerifyMismatchError: + unauthorized() + token, expires_at = issue_session(user["id"]) + del user["password_hash"] + return {"token": token, "token_type": "Bearer", "expires_at": expires_at, + "user": row_response(user)} + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def logout(request: Request): + token = bearer_token(request.headers.get("Authorization")) + redis_client.delete("session:" + token) + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +def get_me(request: Request): + return request.state.user + + +def require_member(conn, chat_id, user_id): + found = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, UUID(user_id)), + ).fetchone() + if found is None: + raise ApiError(404, "not_found", "Чат не найден") + + +CHAT_SELECT = """ + SELECT c.id, c.title, c.created_at, + ARRAY(SELECT user_id FROM chat_members + WHERE chat_id = c.id ORDER BY user_id) AS member_ids + FROM chats c + JOIN chat_members mine ON mine.chat_id = c.id + WHERE mine.user_id = %s +""" + +MESSAGE_SELECT = """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT ma.attachment_id FROM message_attachments ma + WHERE ma.message_id=m.id ORDER BY ma.position)::uuid[] AS attachment_ids + FROM messages m +""" + +HISTORY_FIRST_SQL = MESSAGE_SELECT + """ + WHERE m.chat_id = %s ORDER BY m.created_at, m.id LIMIT %s +""" + +HISTORY_AFTER_SQL = MESSAGE_SELECT + """ + WHERE m.chat_id = %s AND (m.created_at, m.id) > (%s, %s) + ORDER BY m.created_at, m.id LIMIT %s +""" + + +def page_response(rows, size, scope): + items = [row_response(row) for row in rows[:size]] + cursor = encode_cursor(scope, items[-1]) if len(rows) > size else None + return {"items": items, "next_cursor": cursor} + + +@app.post("/api/v1/chats", status_code=201) +def create_chat(body: CreateChat, request: Request): + members = sorted(parse_uuid(value) for value in body.member_ids) + if len(members) != len(set(members)): + invalid("Участники повторяются") + if UUID(request.state.user["id"]) not in members: + invalid("Вызывающий должен входить в member_ids") + try: + with pool.connection() as conn: + with conn.transaction(): + chat = conn.execute( + "INSERT INTO chats (id, title) VALUES (%s, %s) " + "RETURNING id, title, created_at", (uuid4(), body.title), + ).fetchone() + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + [(chat["id"], member) for member in members], + ) + except ForeignKeyViolation: + invalid("Неизвестный участник") + chat["member_ids"] = members + return row_response(chat) + + +@app.get("/api/v1/chats") +def list_chats(request: Request, limit: str = Query(default="50"), + cursor: str | None = Query(default=None)): + user_id = request.state.user["id"] + scope = "chats:" + user_id + size, after = parse_page(limit, cursor, scope) + query = CHAT_SELECT + params = [UUID(user_id)] + if after is not None: + query += " AND (c.created_at, c.id) > (%s, %s)" + params.extend(after) + query += " ORDER BY c.created_at, c.id LIMIT %s" + params.append(size + 1) + with pool.connection() as conn: + rows = conn.execute(query, params).fetchall() + return page_response(rows, size, scope) + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: str, request: Request): + with pool.connection() as conn: + chat = conn.execute( + CHAT_SELECT + " AND c.id = %s", + (UUID(request.state.user["id"]), parse_uuid(chat_id)), + ).fetchone() + if chat is None: + raise ApiError(404, "not_found", "Чат не найден") + return row_response(chat) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message( + chat_id: str, + body: CreateMessage, + request: Request, + idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"), +): + if idempotency_key is None or re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key) is None: + invalid("Некорректный Idempotency-Key") + + chat_uuid = parse_uuid(chat_id) + user_id = UUID(request.state.user["id"]) + digest = hashlib.sha256( + json.dumps({"text": body.text, "attachment_ids": body.attachment_ids}, ensure_ascii=False, sort_keys=True).encode("utf-8") + ).hexdigest() + + with pool.connection() as conn: + with conn.transaction(): + require_member(conn, chat_uuid, str(user_id)) + if body.attachment_ids: + attachment_uuids = [UUID(v) for v in body.attachment_ids] + found = conn.execute( + "SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s)", + (attachment_uuids,), + ).fetchall() + if len(found) != len(attachment_uuids) or any(row["chat_id"] != chat_uuid for row in found): + raise ApiError(404, "not_found", "Вложение не найдено") + if any(row["status"] == "failed" for row in found): + raise ApiError(409, "conflict", "Вложение не готово") + conn.execute( + "DELETE FROM message_requests WHERE sender_id = %s AND chat_id = %s " + "AND request_key = %s AND created_at < now() - interval '24 hours'", + (user_id, chat_uuid, idempotency_key), + ) + created = conn.execute( + "INSERT INTO message_requests (sender_id, chat_id, request_key, request_hash) " + "VALUES (%s, %s, %s, %s) ON CONFLICT DO NOTHING RETURNING request_key", + (user_id, chat_uuid, idempotency_key, digest), + ).fetchone() + if created is None: + existing = conn.execute( + "SELECT request_hash, message_id FROM message_requests " + "WHERE sender_id = %s AND chat_id = %s AND request_key = %s", + (user_id, chat_uuid, idempotency_key), + ).fetchone() + if existing is None: + raise ApiError(503, "unavailable", "Повторите запрос") + if existing["request_hash"] != digest: + raise ApiError(409, "conflict", "Ключ уже использован для другого сообщения") + message = conn.execute( + MESSAGE_SELECT + " WHERE m.id = %s", + (existing["message_id"],), + ).fetchone() + else: + message = conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text) " + "VALUES (%s, %s, %s, %s) " + "RETURNING id, chat_id, sender_id, text, created_at", + (uuid4(), chat_uuid, user_id, body.text), + ).fetchone() + for position, attach in enumerate(body.attachment_ids): + conn.execute( + "INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)", + (message["id"], UUID(attach), position), + ) + conn.execute( + "UPDATE message_requests SET message_id = %s " + "WHERE sender_id = %s AND chat_id = %s AND request_key = %s", + (message["id"], user_id, chat_uuid, idempotency_key), + ) + if "attachment_ids" not in message: + message["attachment_ids"] = [UUID(v) for v in body.attachment_ids] + return row_response(message) + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(chat_id: str, request: Request, limit: str = Query(default="50"), + cursor: str | None = Query(default=None)): + chat_uuid = parse_uuid(chat_id) + scope = "messages:" + str(chat_uuid) + size, after = parse_page(limit, cursor, scope) + with pool.connection() as conn: + require_member(conn, chat_uuid, request.state.user["id"]) + if after is None: + rows = conn.execute(HISTORY_FIRST_SQL, (chat_uuid, size + 1)).fetchall() + else: + rows = conn.execute( + HISTORY_AFTER_SQL, (chat_uuid, after[0], after[1], size + 1) + ).fetchall() + return page_response(rows, size, scope) + + +# Lab 4: binary media are kept in private S3; PostgreSQL stores only metadata. +# Raw request bodies are streamed in bounded chunks to a temporary spool file. +def validate_image(stream, advertised_type): + stream.seek(0) + try: + with IMAGE_DECODE_SLOT, Image.open(stream) as probe: + actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(probe.format) + width, height = probe.size + if width * height > MAX_IMAGE_PIXELS: + raise ApiError(413, "payload_too_large", "Слишком много пикселей") + if actual != advertised_type: + raise ApiError(415, "unsupported_media_type", "Тип изображения не совпадает с содержимым") + probe.verify() + stream.seek(0) + # Verifying full decode rejects truncated/corrupt image streams. + with IMAGE_DECODE_SLOT, Image.open(stream) as image: + image.load() + except ApiError: + raise + except (UnidentifiedImageError, OSError, ValueError, SyntaxError, Image.DecompressionBombError): + raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from None + stream.seek(0) + + +def attachment_response(row): + result = row_response(row) + for key in ("object_key", "staging_key", "attempts", "lease_token", "lease_until", "next_attempt_at", "pipeline_version"): + result.pop(key, None) + result["variants"] = variants_for(row["id"]) if row["status"] == "ready" else [] + return result + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def create_attachment(chat_id: str, request: Request, filename: str = Query(...)): + chat_uuid = parse_uuid(chat_id) + user_id = UUID(request.state.user["id"]) + if not 1 <= len(filename) <= 255 or any(c in filename for c in "/\\\x00") or any(unicodedata.category(c) == "Cc" for c in filename): + invalid("Опасное имя файла") + content_type = request.headers.get("content-type", "").split(";")[0].strip().lower() + length_header = request.headers.get("content-length") + if length_header and length_header.isdigit() and int(length_header) > MAX_UPLOAD_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше 10 MiB") + # ACL before touching storage. + await run_in_threadpool(check_membership, chat_uuid, user_id) + file = tempfile.SpooledTemporaryFile(max_size=256 * 1024) + digest = hashlib.sha256() + size = 0 + try: + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_UPLOAD_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше 10 MiB") + digest.update(chunk) + file.write(chunk) + if size == 0: + raise ApiError(415, "unsupported_media_type", "Пустой файл") + await run_in_threadpool(validate_image, file, content_type) + attachment_id = uuid4() + object_key = f"staging/{attachment_id.hex}" + file.seek(0) + # One S3 PUT; no multipart uploads left by a client disconnect. + try: + await run_in_threadpool(stage_upload, file, object_key, size, content_type) + except (StorageFailure, BotoCoreError, ClientError): + raise ApiError(503, "unavailable", "Объектное хранилище недоступно") from None + try: + result = await run_in_threadpool(queue_attachment, attachment_id, chat_uuid, user_id, filename, + content_type, size, digest.hexdigest(), object_key) + if result is None: + raise ApiError(404, "not_found", "Чат не найден") + except Exception: + # Best effort compensation; orphan reconciler handles crash between S3 PUT and DB COMMIT. + try: + await run_in_threadpool(delete_original, object_key) + except Exception: + log.warning("upload compensation failed for %s", object_key) + raise + return attachment_response(result) + finally: + file.close() + + +def check_membership(chat_uuid, user_id): + with pool.connection() as conn: + require_member(conn, chat_uuid, str(user_id)) + + +def save_attachment(attachment_id, chat_uuid, user_id, filename, content_type, size, digest, key): + with pool.connection() as conn: + with conn.transaction(): + require_member(conn, chat_uuid, str(user_id)) + return conn.execute( + "INSERT INTO attachments (id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,object_key) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,'ready',%s) RETURNING *", + (attachment_id, chat_uuid, user_id, filename, content_type, size, digest, key) + ).fetchone() + + +def owned_attachment(attachment_id, user_id): + with pool.connection() as conn: + row = conn.execute( + "SELECT a.* FROM attachments a JOIN chat_members cm ON cm.chat_id = a.chat_id " + "WHERE a.id = %s AND cm.user_id = %s", + (attachment_id, UUID(user_id)), + ).fetchone() + if row is None: + raise ApiError(404, "not_found", "Вложение не найдено") + return row + + +@app.get("/api/v1/attachments/{attachment_id}") +def attachment_metadata(attachment_id: str, request: Request): + return attachment_response(owned_attachment(parse_uuid(attachment_id), request.state.user["id"])) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def attachment_content(attachment_id: str, request: Request, variant: str = Query(default="original")): + row = owned_attachment(parse_uuid(attachment_id), request.state.user["id"]) + if row["status"] != "ready": + raise ApiError(409, "conflict", "Вложение не готово") + item = None + if variant != "original": + if variant not in ("thumbnail", "crop", "optimized"): + raise ApiError(404, "not_found", "Вариант не найден") + item = fetch_variant(row["id"], variant) + if item is None: + raise ApiError(404, "not_found", "Вариант не найден") + key = row["object_key"] if item is None else item["object_key"] + mime = row["content_type"] if item is None else item["content_type"] + size = row["size_bytes"] if item is None else item["size_bytes"] + try: + s3_body = download_original(key) + except (BotoCoreError, ClientError, StorageFailure): + raise ApiError(503, "unavailable", "Объектное хранилище недоступно") from None + def chunks(): + try: + for data in s3_body.iter_chunks(chunk_size=64 * 1024): + if data: + yield data + finally: + s3_body.close() + return StreamingResponse(chunks(), media_type=mime, + headers={"Content-Length": str(size), "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff"}) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..25bd7d36 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,36 @@ +from pathlib import Path + +import psycopg + +from src.db import connection_kwargs + + +def main(): + files = sorted((Path(__file__).resolve().parents[1] / "migrations").glob("*.sql")) + if not files: + raise RuntimeError("Миграции не найдены") + with psycopg.connect(**connection_kwargs()) as conn: + conn.execute(""" + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """) + for file in files: + applied = False + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(202602)") + exists = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", (file.name,) + ).fetchone() + if exists is None: + conn.execute(file.read_text(encoding="utf-8")) + conn.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", (file.name,) + ) + applied = True + print(("APPLIED " if applied else "SKIP ") + file.name) + + +if __name__ == "__main__": + main() diff --git a/src/pagination.py b/src/pagination.py new file mode 100644 index 00000000..03615854 --- /dev/null +++ b/src/pagination.py @@ -0,0 +1,47 @@ +import base64 +import binascii +import json +import re +from datetime import datetime +from uuid import UUID + +from src.errors import invalid + + +def encode_cursor(scope, item): + data = {"scope": scope, "created_at": item["created_at"], "id": item["id"]} + raw = json.dumps(data, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def parse_page(limit, cursor, scope): + if len(limit) > 10 or re.fullmatch(r"[0-9]+", limit) is None: + invalid("limit должен быть целым числом от 1 до 100") + size = int(limit) + if not 1 <= size <= 100: + invalid("limit должен быть от 1 до 100") + if cursor is None: + return size, None + if not cursor or len(cursor) > 2048: + invalid("Некорректный cursor") + try: + raw = base64.b64decode( + cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True + ) + data = json.loads(raw) + if not isinstance(data, dict) or set(data) != {"scope", "created_at", "id"}: + raise ValueError + if data["scope"] != scope or not isinstance(data["created_at"], str): + raise ValueError + if re.fullmatch( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,6})?Z", + data["created_at"], + ) is None: + raise ValueError + created_at = datetime.fromisoformat(data["created_at"][:-1] + "+00:00") + if created_at.tzinfo is None: + raise ValueError + item_id = UUID(data["id"]) + except (ValueError, TypeError, AttributeError, binascii.Error): + invalid("Некорректный cursor или cursor от другого ресурса") + return size, (created_at, item_id) diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..afd3c4d0 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,57 @@ +"""Dry-run by default: find unused S3 originals older than safe grace period. + +Usage: + docker compose exec -T api1 python -m src.reconcile + docker compose exec -T api1 python -m src.reconcile --apply --min-age-seconds 3600 + +No database objects or live originals are ever deleted. Only orphans, and only +under the server-owned originals/ prefix, are eligible for cleanup. +""" +import argparse +import datetime as dt +import json +import logging +from src.db import connection_kwargs +from src.storage import s3, bucket +import psycopg + +log = logging.getLogger("attachment-reconcile") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--apply", action="store_true", help="Delete eligible orphan objects") + parser.add_argument("--min-age-seconds", type=int, default=3600) + args = parser.parse_args() + if args.min_age_seconds < 3600: + parser.error("Safety grace period must be at least 3600s") + now = dt.datetime.now(dt.timezone.utc) + checked = 0 + orphan = 0 + removed = 0 + with psycopg.connect(**connection_kwargs()) as conn: + for page in s3.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix="originals/"): + for item in page.get("Contents", []): + checked += 1 + key = item["Key"] + age = (now - item["LastModified"]).total_seconds() + if age < args.min_age_seconds: + continue + exists = conn.execute("SELECT 1 FROM attachments WHERE object_key=%s", (key,)).fetchone() + if exists: + continue + orphan += 1 + if args.apply: + # Recheck immediately before deletion; do not delete newly referenced keys. + exists = conn.execute("SELECT 1 FROM attachments WHERE object_key=%s", (key,)).fetchone() + if exists: + continue + s3.delete_object(Bucket=bucket, Key=key) + removed += 1 + print(json.dumps({"mode": "apply" if args.apply else "dry-run", "scanned": checked, + "eligible_orphans": orphan, "removed": removed, + "min_age_seconds": args.min_age_seconds}, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/src/run.py b/src/run.py new file mode 100644 index 00000000..d01a0b04 --- /dev/null +++ b/src/run.py @@ -0,0 +1,23 @@ +import logging +import os + +import uvicorn + + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + logging.getLogger("messenger").info( + "config app_env=%s", os.getenv("APP_ENV", "lab") + ) + uvicorn.run( + "src.main:app", + host=os.getenv("APP_HOST", "0.0.0.0"), + port=int(os.getenv("APP_PORT", "8080")), + workers=1, + timeout_graceful_shutdown=5, + timeout_keep_alive=1, + access_log=False, + ) \ No newline at end of file diff --git a/src/sessions.py b/src/sessions.py new file mode 100644 index 00000000..82854049 --- /dev/null +++ b/src/sessions.py @@ -0,0 +1,33 @@ +import os +import secrets +import time +from datetime import datetime, timezone + +from redis import Redis +from redis.backoff import NoBackoff +from redis.retry import Retry + + +redis_client = Redis( + host=os.getenv("REDIS_HOST", "redis"), + port=int(os.getenv("REDIS_PORT", "6379")), + decode_responses=True, + socket_connect_timeout=2, + socket_timeout=2, + max_connections=10, + retry=Retry(NoBackoff(), 0), +) + +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +if SESSION_TTL_SECONDS <= 0: + raise RuntimeError("SESSION_TTL_SECONDS должен быть положительным") + + +def issue_session(user_id): + token = secrets.token_urlsafe(32) + expires_ms = int(time.time() * 1000) + SESSION_TTL_SECONDS * 1000 + redis_client.set("session:" + token, str(user_id), pxat=expires_ms) + expires_at = datetime.fromtimestamp( + expires_ms / 1000, timezone.utc + ).isoformat(timespec="milliseconds").replace("+00:00", "Z") + return token, expires_at diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..45ba7d89 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,43 @@ +"""S3-compatible private storage. Only API-scoped credentials are available here.""" +import os +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError, BotoCoreError + +MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) +bucket = os.getenv("S3_BUCKET", "messenger-attachments") +s3 = boto3.client( + "s3", endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"), + region_name=os.getenv("S3_REGION", "us-east-1"), + aws_access_key_id=os.environ.get("S3_ACCESS_KEY_ID", "unset"), + aws_secret_access_key=os.environ.get("S3_SECRET_ACCESS_KEY", "unset"), + config=Config(s3={"addressing_style": "path"}, connect_timeout=2, read_timeout=30, + retries={"mode": "standard", "max_attempts": 2}), +) + +class StorageFailure(Exception): + pass + + +def check_storage(): + # Scoped user has ListBucket but no privileges outside this bucket. + s3.list_objects_v2(Bucket=bucket, MaxKeys=1) + + +def upload_original(file, key, size, content_type): + file.seek(0) + s3.put_object(Bucket=bucket, Key=key, Body=file, ContentLength=size, ContentType=content_type) + + +def delete_original(key): + s3.delete_object(Bucket=bucket, Key=key) + + +def download_original(key): + try: + return s3.get_object(Bucket=bucket, Key=key)["Body"] + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): + raise StorageFailure("Object missing") from None + raise diff --git a/student-tests/test_images.py b/student-tests/test_images.py new file mode 100644 index 00000000..e0de17e0 --- /dev/null +++ b/student-tests/test_images.py @@ -0,0 +1,33 @@ +"""Unit-level image checks; run in built api1 container.""" +import tempfile +import unittest +from pathlib import Path +from src.main import validate_image +from src.errors import ApiError + +PNG = (Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes() + + +class ImageValidationTest(unittest.TestCase): + def check(self, content, content_type): + with tempfile.SpooledTemporaryFile() as stream: + stream.write(content) + validate_image(stream, content_type) + + def test_fixture(self): + self.check(PNG,'image/png') + + def test_corrupt(self): + with self.assertRaises(ApiError): + self.check(PNG[:70], 'image/png') + + def test_spoof_mime(self): + with self.assertRaises(ApiError): + self.check(PNG,'image/jpeg') + + def test_random_bytes(self): + with self.assertRaises(ApiError): + self.check(b'unrelated bytes', 'image/png') + +if __name__=='__main__': + unittest.main() -- 2.54.0