diff --git a/.dockerignore b/.dockerignore index bc16e556..036eee9e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,10 @@ __pycache__ node_modules evidence .DS_Store +.git +.env +.venv +__pycache__ +.pytest_cache +*.pyc +evidence \ No newline at end of file diff --git a/.env.example b/.env.example index 119384b5..7eeadbf2 100644 --- a/.env.example +++ b/.env.example @@ -23,3 +23,7 @@ WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 +RABBITMQ_USER=messenger +RABBITMQ_PASSWORD=change-me +RABBITMQ_QUEUE=image_jobs +IMAGE_MAX_ATTEMPTS=5 \ No newline at end of file diff --git a/.gitignore b/.gitignore index b5a71896..702888b4 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,8 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +.env +.venv/ +__pycache__/ +*.pyc +evidence/*.tmp \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..66851c47 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.12-slim + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +COPY src/ ./src/ +COPY migrations/ ./migrations/ + +ENV HOST=0.0.0.0 +ENV PORT=8080 + +EXPOSE 8080 + +CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..021c351d 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,131 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- **Автор, группа:** Матвиенко, БПИ2304 +- **Целевая оценка:** 3 +- **Commit/PR:** добавить после публикации +- **Использованные источники и библиотеки:** материалы лабораторной, OpenAPI-контракт, Python 3.12, FastAPI, PostgreSQL 16, RustFS 1.0.3, RabbitMQ 4.1.8, Pika 1.3.2, boto3, Pillow, Nginx 1.29, Docker Compose. При разработке и отладке использовалась помощь ИИ. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Необходимы Git, Docker Desktop с Docker Compose, Python 3 и Make. + +Создать `.env` по образцу `.env.example`, указав собственные локальные секреты: + +```dotenv +POSTGRES_PASSWORD=<локальный_пароль> +SESSION_TTL_SECONDS=3600 +S3_ACCESS_KEY=<локальный_ключ> +S3_SECRET_KEY=<локальный_секрет> +S3_BUCKET=messenger-attachments +RABBITMQ_USER=messenger +RABBITMQ_PASSWORD=<локальный_пароль_брокера> +RABBITMQ_QUEUE=image_jobs +IMAGE_MAX_ATTEMPTS=5 +``` + +Сборка, применение миграций, создание приватного bucket и запуск: + +```bash +docker compose up -d --build +docker compose ps -a +curl http://127.0.0.1:8080/health/ready +``` + +Проверки: + +```bash +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 scripts/test-worker-recovery.py +bash scripts/test-redelivery.sh +``` + +Остановка с сохранением данных: + +```bash +docker compose down +``` + +Намеренное удаление учебных данных и volumes: + +```bash +docker compose down -v +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +**Схема:** клиент → Nginx → две API-реплики → PostgreSQL + RustFS → dispatcher → RabbitMQ → worker → RustFS + PostgreSQL. + +Из lab4 сохранены авторизация, чаты, сообщения, балансировка API, идемпотентность и приватное S3-хранилище. + +В lab5 реализована асинхронная обработка изображений: + +- API сохраняет исходный файл в устойчивом S3 staging и создаёт задание в PostgreSQL. +- Upload возвращает HTTP 202 со статусом `queued`. +- Dispatcher публикует идентификаторы заданий в устойчивую очередь RabbitMQ с подтверждением публикации. +- Отдельный worker обрабатывает файл и создаёт JPEG thumbnail в пределах 256×256 с сохранением пропорций. +- Оригинал сохраняется без изменений. +- Статусы `queued`, `processing`, `ready`, `failed` доступны через API. +- Реализованы ограниченные повторы обработки — максимум 5 попыток. +- Подтверждение сообщения RabbitMQ выполняется после завершения транзакции PostgreSQL. + +Байты изображений и учётные данные в сообщениях брокера не передаются. PostgreSQL хранит метаданные и задания, RustFS — staging и конечные файлы. Стабильные object keys и проверка состояния задания защищают от повторной обработки завершённой работы. + +Между S3 и PostgreSQL нет общей распределённой транзакции; при сбоях используются повторные попытки и идемпотентная обработка. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий | Команда или сценарий | Результат | Доказательство | +|---|---|---|---| +| Публичный smoke | `make test` | PASS | `evidence/smoke.txt` | +| Сохранность данных | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS | `evidence/persistence.txt` | +| Отказ API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS, 121 запись, 0 ошибок | `evidence/failover.txt` | +| Восстановление после отказа | Failover-сценарий | 0,783 секунды | `evidence/failover.txt` | +| Остановленный worker и рестарт | `python3 scripts/test-worker-recovery.py` | PASS | `evidence/worker-recovery.txt` | +| Повторная доставка задания | `bash scripts/test-redelivery.sh` | PASS, результат не изменился | `evidence/redelivery.txt` | +| Подтверждение работы worker | Журнал worker | Два повторных задания распознаны как завершённые | `evidence/redelivery-worker.txt` | +| Состояние инфраструктуры | `docker compose ps -a` | API и PostgreSQL healthy, остальные рабочие сервисы запущены | `evidence/containers.txt` | +| Миграции | `docker compose run --rm --no-deps migrate` | Все 5 миграций уже применены | `evidence/migrations.txt` | +| Перезапуск инфраструктуры | Сравнение времени запуска контейнеров | Сохранены данные до и после действия | `evidence/restart-before.txt`, `evidence/restart-after.txt` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Гипотеза:** принятое изображение не потеряется при остановке worker и перезапуске API или брокера. Повторная доставка завершённого задания не изменит результат. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Испытания выполнены на локальном стенде Windows / WSL Ubuntu / Docker Desktop, с RabbitMQ 4.1.8, PostgreSQL 16, RustFS 1.0.3 и двумя API-репликами. + +**Проверка восстановления задания:** + +1. Worker остановлен. +2. Изображение загружено с результатом HTTP 202, статус `queued`. +3. Создано сообщение с ожидающим вложением. +4. Перезапущены RabbitMQ и API. +5. Состояние `queued` сохранилось. +6. После запуска worker вложение перешло в `ready`. +7. Оригинал проверен по SHA-256, thumbnail создан как JPEG 256×160. + +Результат: `PASS: durable worker recovery`. + +**Проверка повторной доставки:** + +Одно завершённое задание повторно опубликовано дважды. До и после проверки состояние осталось `ready`, число попыток — `1`, число вариантов — `1`, SHA-256 thumbnail не изменился. Worker дважды зарегистрировал получение уже завершённого задания. + +Результат: `PASS: duplicate delivery did not change completed job`. + +| Воздействие | Ожидание | Наблюдение | Результат | +|---|---|---|---| +| Остановка API-реплики | Запись через оставшуюся реплику | 121 успешная запись, 0 ошибок | Восстановление 0,783 с | +| Перезапуск хранилищ и брокера | Сохранность данных и сессии | Persistence PASS | Потерь не выявлено | +| Остановка worker и рестарт | Задание сохранится | `queued → ready`, оригинал и thumbnail доступны | PASS | +| Две повторные доставки | Отсутствие повторной обработки | Статус, checksum и количество вариантов неизменны | PASS | + +Отдельные измерения RPS, p50/p95/p99, CPU/RAM и длительные нагрузочные испытания не выполнялись. Проверка повторной доставки проведена для завершённого задания; аварийное завершение worker непосредственно перед ACK отдельно не моделировалось. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +PostgreSQL, RustFS и RabbitMQ работают в одиночных экземплярах и остаются точками отказа. При намеренном удалении volumes данные теряются. + +В перспективе можно добавить резервирование хранилищ, мониторинг очереди, очистку неиспользуемых staging-объектов и испытания с несколькими worker. + +**Вывод:** асинхронная обработка изображений реализована. Публичный smoke, persistence, failover, восстановление заданий и повторная доставка успешно проверены. Принятое задание пережило остановку worker и перезапуск RabbitMQ и API, а повторная доставка не изменила готовый результат. \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..61864d2d --- /dev/null +++ b/compose.yaml @@ -0,0 +1,175 @@ + +services: + rabbitmq: + image: rabbitmq:4.1.8-management + hostname: rabbitmq + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:?Set RABBITMQ_USER in .env} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?Set RABBITMQ_PASSWORD in .env} + volumes: + - rabbitmq_data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 10s + timeout: 5s + retries: 15 + start_period: 15s + restart: unless-stopped + + db: + image: postgres:16-alpine + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 20 + restart: unless-stopped + + minio: + image: rustfs/rustfs:1.0.3 + command: ["/data"] + environment: + RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY in .env} + RUSTFS_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY in .env} + RUSTFS_ADDRESS: ":9000" + volumes: + - minio_data:/data + restart: unless-stopped + + s3-init: + build: . + command: ["python", "-m", "src.s3_setup"] + environment: &s3_env + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + depends_on: + minio: + condition: service_started + restart: "no" + + migrate: + build: . + command: ["python", "-m", "src.migrate"] + environment: + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger + depends_on: + db: + condition: service_healthy + restart: "no" + + api1: + build: . + environment: + RABBITMQ_HOST: rabbitmq + RABBITMQ_USER: ${RABBITMQ_USER} + RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD} + RABBITMQ_QUEUE: ${RABBITMQ_QUEUE:-image_jobs} + IMAGE_MAX_ATTEMPTS: ${IMAGE_MAX_ATTEMPTS:-5} + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + <<: *s3_env + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"] + interval: 5s + timeout: 3s + retries: 20 + stop_grace_period: 15s + restart: unless-stopped + + api2: + build: . + environment: + RABBITMQ_HOST: rabbitmq + RABBITMQ_USER: ${RABBITMQ_USER} + RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD} + RABBITMQ_QUEUE: ${RABBITMQ_QUEUE:-image_jobs} + IMAGE_MAX_ATTEMPTS: ${IMAGE_MAX_ATTEMPTS:-5} + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + <<: *s3_env + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"] + interval: 5s + timeout: 3s + retries: 20 + stop_grace_period: 15s + restart: unless-stopped + + + dispatcher: + build: . + command: ["python", "-m", "src.dispatcher"] + environment: + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger + RABBITMQ_HOST: rabbitmq + RABBITMQ_USER: ${RABBITMQ_USER} + RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD} + RABBITMQ_QUEUE: ${RABBITMQ_QUEUE:-image_jobs} + depends_on: + rabbitmq: + condition: service_healthy + migrate: + condition: service_completed_successfully + restart: unless-stopped + stop_grace_period: 15s + + + worker: + build: . + command: ["python", "-m", "src.worker"] + environment: + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + RABBITMQ_HOST: rabbitmq + RABBITMQ_USER: ${RABBITMQ_USER} + RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD} + RABBITMQ_QUEUE: ${RABBITMQ_QUEUE:-image_jobs} + IMAGE_MAX_ATTEMPTS: ${IMAGE_MAX_ATTEMPTS:-5} + depends_on: + rabbitmq: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + restart: unless-stopped + stop_grace_period: 20s + + nginx: + image: nginx:1.29-alpine + ports: + - "127.0.0.1:8080:80" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + api1: + condition: service_healthy + api2: + condition: service_healthy + restart: unless-stopped + +volumes: + postgres_data: + minio_data: + rabbitmq_data: diff --git a/evidence/containers.txt b/evidence/containers.txt new file mode 100644 index 00000000..78abea2a --- /dev/null +++ b/evidence/containers.txt @@ -0,0 +1,11 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab5-api1-1 mtusi2026lab5-api1 "uvicorn src.main:ap…" api1 10 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi2026lab5-api2-1 mtusi2026lab5-api2 "uvicorn src.main:ap…" api2 10 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi2026lab5-db-1 postgres:16-alpine "docker-entrypoint.s…" db 42 minutes ago Up 6 minutes (healthy) 5432/tcp +mtusi2026lab5-dispatcher-1 mtusi2026lab5-dispatcher "python -m src.dispa…" dispatcher 10 minutes ago Up 6 minutes 8080/tcp +mtusi2026lab5-migrate-1 mtusi2026lab5-migrate "python -m src.migra…" migrate 10 minutes ago Exited (0) 2 minutes ago +mtusi2026lab5-minio-1 rustfs/rustfs:1.0.3 "/entrypoint.sh /data" minio 42 minutes ago Up 6 minutes 9000-9001/tcp +mtusi2026lab5-nginx-1 nginx:1.29-alpine "/docker-entrypoint.…" nginx 10 minutes ago Up 10 minutes 127.0.0.1:8080->80/tcp +mtusi2026lab5-rabbitmq-1 rabbitmq:4.1.8-management "docker-entrypoint.s…" rabbitmq 45 minutes ago Up 3 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15671-15672/tcp, 15691-15692/tcp, 25672/tcp +mtusi2026lab5-s3-init-1 mtusi2026lab5-s3-init "python -m src.s3_se…" s3-init 10 minutes ago Exited (0) 2 minutes ago +mtusi2026lab5-worker-1 mtusi2026lab5-worker "python -m src.worker" worker 10 minutes ago Up 2 minutes 8080/tcp diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..3fcab7a5 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 121, "failed_attempts": 0, "first_recovery_seconds": 0.783} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/migrations.txt b/evidence/migrations.txt new file mode 100644 index 00000000..d12e96ed --- /dev/null +++ b/evidence/migrations.txt @@ -0,0 +1,6 @@ +Already applied: 001_initial.sql +Already applied: 002_idempotency.sql +Already applied: 003_attachments.sql +Already applied: 004_allow_empty_message_text.sql +Already applied: 005_async_jobs.sql +Migrations completed diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..793b001d --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/redelivery-worker.txt b/evidence/redelivery-worker.txt new file mode 100644 index 00000000..5ee5a1b1 --- /dev/null +++ b/evidence/redelivery-worker.txt @@ -0,0 +1,31 @@ +worker-1 | 2026-11-17 16:12:19,058 INFO Pika version 1.3.2 connecting to ('172.19.0.2', 5672) +worker-1 | 2026-11-17 16:12:19,058 INFO Socket connected: +worker-1 | 2026-11-17 16:12:19,059 INFO Streaming transport linked up: (, _StreamingProtocolShim: params=>). +worker-1 | 2026-11-17 16:12:19,060 INFO AMQPConnector - reporting success: params=> +worker-1 | 2026-11-17 16:12:19,061 INFO AMQPConnectionWorkflow - reporting success: params=> +worker-1 | 2026-11-17 16:12:19,061 INFO Connection workflow succeeded: params=> +worker-1 | 2026-11-17 16:12:19,061 INFO Created channel=1 +worker-1 | 2026-11-17 16:12:19,065 INFO Worker listening on image_jobs +worker-1 | 2026-11-17 16:13:54,771 INFO Job da3f295c-1991-4fe3-96a1-4bed377479f5 ready, thumbnail 256x160 +worker-1 | 2026-11-17 16:14:27,063 INFO Job 37c1126c-dc6c-49d7-91bd-fd88f50883b4 ready, thumbnail 256x160 +worker-1 | 2026-11-17 16:15:06,446 INFO Job 7f37da8b-1f55-4b81-8cd3-aa5627756ecd ready, thumbnail 256x160 +worker-1 | 2026-11-17 16:15:36,525 INFO Pika version 1.3.2 connecting to ('172.19.0.2', 5672) +worker-1 | 2026-11-17 16:15:36,525 INFO Socket connected: +worker-1 | 2026-11-17 16:15:36,525 INFO Streaming transport linked up: (, _StreamingProtocolShim: params=>). +worker-1 | 2026-11-17 16:15:36,532 INFO AMQPConnector - reporting success: params=> +worker-1 | 2026-11-17 16:15:36,532 INFO AMQPConnectionWorkflow - reporting success: params=> +worker-1 | 2026-11-17 16:15:36,532 INFO Connection workflow succeeded: params=> +worker-1 | 2026-11-17 16:15:36,532 INFO Created channel=1 +worker-1 | 2026-11-17 16:15:36,537 INFO Worker listening on image_jobs +worker-1 | 2026-11-17 16:16:17,793 INFO Job cd52e633-e071-4966-93ff-6d9712c43c41 ready, thumbnail 256x160 +worker-1 | 2026-11-17 16:19:32,768 INFO Pika version 1.3.2 connecting to ('172.19.0.2', 5672) +worker-1 | 2026-11-17 16:19:32,768 INFO Socket connected: +worker-1 | 2026-11-17 16:19:32,768 INFO Streaming transport linked up: (, _StreamingProtocolShim: params=>). +worker-1 | 2026-11-17 16:19:32,770 INFO AMQPConnector - reporting success: params=> +worker-1 | 2026-11-17 16:19:32,770 INFO AMQPConnectionWorkflow - reporting success: params=> +worker-1 | 2026-11-17 16:19:32,770 INFO Connection workflow succeeded: params=> +worker-1 | 2026-11-17 16:19:32,770 INFO Created channel=1 +worker-1 | 2026-11-17 16:19:32,774 INFO Worker listening on image_jobs +worker-1 | 2026-11-17 16:19:32,819 INFO Job 2d2befcd-5472-44c0-aaf8-4272a173fde6 ready, thumbnail 256x160 +worker-1 | 2026-11-17 16:21:50,498 INFO Job 2d2befcd-5472-44c0-aaf8-4272a173fde6 already completed +worker-1 | 2026-11-17 16:21:50,517 INFO Job 2d2befcd-5472-44c0-aaf8-4272a173fde6 already completed diff --git a/evidence/redelivery.txt b/evidence/redelivery.txt new file mode 100644 index 00000000..52ca4976 --- /dev/null +++ b/evidence/redelivery.txt @@ -0,0 +1,8 @@ +Finding completed image job... +Job: 2d2befcd-5472-44c0-aaf8-4272a173fde6 +Before: ready|1|f9e3cf1ed71334dfb7824260b8777b7cc9dddcbc1a9c1d1059d22ab81932fc34|1 +Publishing the same job twice... +Two duplicate messages published +After: ready|1|f9e3cf1ed71334dfb7824260b8777b7cc9dddcbc1a9c1d1059d22ab81932fc34|1 +PASS: duplicate delivery did not change completed job +PASS: thumbnail checksum and variant count preserved diff --git a/evidence/restart-after.txt b/evidence/restart-after.txt new file mode 100644 index 00000000..bb395a0b --- /dev/null +++ b/evidence/restart-after.txt @@ -0,0 +1,5 @@ +/mtusi2026lab5-api1-1 2026-11-17T16:15:37.499975362Z +/mtusi2026lab5-api2-1 2026-11-17T16:15:37.498279295Z +/mtusi2026lab5-db-1 2026-11-17T16:15:27.473397186Z +/mtusi2026lab5-minio-1 2026-11-17T16:15:27.347848806Z +/mtusi2026lab5-rabbitmq-1 2026-11-17T16:15:28.371689705Z diff --git a/evidence/restart-before.txt b/evidence/restart-before.txt new file mode 100644 index 00000000..ef1bc527 --- /dev/null +++ b/evidence/restart-before.txt @@ -0,0 +1,5 @@ +/mtusi2026lab5-api1-1 2026-11-17T16:12:18.462950541Z +/mtusi2026lab5-api2-1 2026-11-17T16:12:18.455718969Z +/mtusi2026lab5-db-1 2026-11-17T15:39:33.580121321Z +/mtusi2026lab5-minio-1 2026-11-17T15:39:33.581952593Z +/mtusi2026lab5-rabbitmq-1 2026-11-17T15:36:42.420152689Z diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..df05fe26 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/worker-recovery.txt b/evidence/worker-recovery.txt new file mode 100644 index 00000000..a334e543 --- /dev/null +++ b/evidence/worker-recovery.txt @@ -0,0 +1,59 @@ +Stopping worker... + Container mtusi2026lab5-worker-1 Stopping + Container mtusi2026lab5-worker-1 Stopped +Upload: 202 queued +Message created while worker is stopped +Restarting RabbitMQ and API... + Container mtusi2026lab5-rabbitmq-1 Restarting + Container mtusi2026lab5-rabbitmq-1 Started + Container mtusi2026lab5-rabbitmq-1 Running + Container mtusi2026lab5-rabbitmq-1 Waiting + Container mtusi2026lab5-rabbitmq-1 Healthy + Container mtusi2026lab5-api1-1 Restarting + Container mtusi2026lab5-api2-1 Restarting + Container mtusi2026lab5-api2-1 Started + Container mtusi2026lab5-api1-1 Started +Queued attachment survived restart +Starting worker... + Container mtusi2026lab5-db-1 Running + Container mtusi2026lab5-rabbitmq-1 Running + Container mtusi2026lab5-migrate-1 Created + Container mtusi2026lab5-minio-1 Running + Container mtusi2026lab5-s3-init-1 Created + Container mtusi2026lab5-worker-1 Created + Container mtusi2026lab5-s3-init-1 Starting + Container mtusi2026lab5-db-1 Waiting + Container mtusi2026lab5-s3-init-1 Started + Container mtusi2026lab5-db-1 Healthy + Container mtusi2026lab5-migrate-1 Starting + Container mtusi2026lab5-migrate-1 Started + Container mtusi2026lab5-rabbitmq-1 Waiting + Container mtusi2026lab5-migrate-1 Waiting + Container mtusi2026lab5-s3-init-1 Waiting + Container mtusi2026lab5-s3-init-1 Exited + Container mtusi2026lab5-migrate-1 Exited + Container mtusi2026lab5-rabbitmq-1 Healthy + Container mtusi2026lab5-worker-1 Starting + Container mtusi2026lab5-worker-1 Started +Attachment: ready +Original: SHA-256 verified +Thumbnail: JPEG 256x160 verified +PASS: durable worker recovery + Container mtusi2026lab5-minio-1 Running + Container mtusi2026lab5-db-1 Running + Container mtusi2026lab5-rabbitmq-1 Running + Container mtusi2026lab5-s3-init-1 Created + Container mtusi2026lab5-migrate-1 Created + Container mtusi2026lab5-worker-1 Running + Container mtusi2026lab5-s3-init-1 Starting + Container mtusi2026lab5-db-1 Waiting + Container mtusi2026lab5-s3-init-1 Started + Container mtusi2026lab5-db-1 Healthy + Container mtusi2026lab5-migrate-1 Starting + Container mtusi2026lab5-migrate-1 Started + Container mtusi2026lab5-s3-init-1 Waiting + Container mtusi2026lab5-rabbitmq-1 Waiting + Container mtusi2026lab5-migrate-1 Waiting + Container mtusi2026lab5-rabbitmq-1 Healthy + Container mtusi2026lab5-migrate-1 Exited + Container mtusi2026lab5-s3-init-1 Exited diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..0e4d5231 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,74 @@ + +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username VARCHAR(32) NOT NULL UNIQUE, + display_name VARCHAR(100) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + CONSTRAINT username_format + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT display_name_length + CHECK (char_length(display_name) BETWEEN 1 AND 100), + CONSTRAINT display_name_not_blank + CHECK (length(btrim(display_name)) > 0) +); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + CONSTRAINT title_not_blank + CHECK (length(btrim(title)) > 0) +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL + REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL + REFERENCES users(id), + position INTEGER NOT NULL, + + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL + REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL + REFERENCES users(id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + CONSTRAINT message_length + CHECK (char_length(text) BETWEEN 1 AND 4000), + CONSTRAINT message_not_blank + CHECK (length(btrim(text)) > 0) +); + +CREATE TABLE IF NOT EXISTS sessions ( + token_hash CHAR(64) PRIMARY KEY, + user_id UUID NOT NULL + REFERENCES users(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL, + revoked BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX IF NOT EXISTS idx_chat_members_user + ON chat_members(user_id, chat_id); + +CREATE INDEX IF NOT EXISTS idx_chats_order + ON chats(created_at, id); + +CREATE INDEX IF NOT EXISTS idx_messages_history + ON messages(chat_id, created_at, id); + +CREATE INDEX IF NOT EXISTS idx_sessions_user + ON sessions(user_id); + +CREATE INDEX IF NOT EXISTS idx_sessions_expiry + ON sessions(expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..663c6b6c --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,14 @@ + +CREATE TABLE IF NOT EXISTS message_idempotency ( + user_id UUID NOT NULL REFERENCES users(id), + chat_id UUID NOT NULL REFERENCES chats(id), + idempotency_key VARCHAR(128) NOT NULL, + request_hash CHAR(64) NOT NULL, + message_id UUID NOT NULL REFERENCES messages(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + PRIMARY KEY (user_id, chat_id, idempotency_key) +); + +CREATE INDEX IF NOT EXISTS idx_idempotency_created +ON message_idempotency(created_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..49e9f33e --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,34 @@ + +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + uploader_id UUID NOT NULL REFERENCES users(id), + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 CHAR(64) NOT NULL, + object_key TEXT NOT NULL UNIQUE, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + status TEXT NOT NULL DEFAULT 'ready', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT attachment_size_valid + CHECK (size_bytes BETWEEN 1 AND 10485760), + + CONSTRAINT attachment_dimensions_valid + CHECK (width > 0 AND height > 0), + + CONSTRAINT attachment_status_valid + CHECK (status IN ('ready', 'failed')) +); + +CREATE INDEX IF NOT EXISTS idx_attachments_chat +ON attachments(chat_id); + +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id), + attachment_id UUID NOT NULL REFERENCES attachments(id), + + PRIMARY KEY (message_id, attachment_id) +); diff --git a/migrations/004_allow_empty_message_text.sql b/migrations/004_allow_empty_message_text.sql new file mode 100644 index 00000000..9b721535 --- /dev/null +++ b/migrations/004_allow_empty_message_text.sql @@ -0,0 +1,13 @@ + +-- Lab4: messages with attachments may have empty text. +-- The API checks that a message contains text or attachments. + +ALTER TABLE messages +DROP CONSTRAINT IF EXISTS message_length; + +ALTER TABLE messages +DROP CONSTRAINT IF EXISTS message_not_blank; + +ALTER TABLE messages +ADD CONSTRAINT message_length +CHECK (char_length(text) <= 4000); diff --git a/migrations/005_async_jobs.sql b/migrations/005_async_jobs.sql new file mode 100644 index 00000000..9417b500 --- /dev/null +++ b/migrations/005_async_jobs.sql @@ -0,0 +1,95 @@ +-- Allow asynchronous attachment states. +ALTER TABLE attachments +DROP CONSTRAINT IF EXISTS attachment_status_valid; + +ALTER TABLE attachments +ADD CONSTRAINT attachment_status_valid +CHECK ( + status IN ( + 'queued', + 'processing', + 'ready', + 'failed' + ) +); + +-- Staging object stored in the private S3 bucket. +ALTER TABLE attachments +ADD COLUMN IF NOT EXISTS staging_key TEXT; + +-- Machine-readable final error. +ALTER TABLE attachments +ADD COLUMN IF NOT EXISTS error_code TEXT; + +-- Last metadata update timestamp. +ALTER TABLE attachments +ADD COLUMN IF NOT EXISTS updated_at +TIMESTAMPTZ NOT NULL DEFAULT NOW(); + +-- Worker jobs and durable publication intent. +CREATE TABLE IF NOT EXISTS attachment_jobs ( + id UUID PRIMARY KEY, + attachment_id UUID NOT NULL UNIQUE + REFERENCES attachments(id), + + status TEXT NOT NULL DEFAULT 'queued', + attempts INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 5, + + published_at TIMESTAMPTZ, + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT attachment_job_status_valid + CHECK ( + status IN ( + 'queued', + 'processing', + 'ready', + 'failed' + ) + ), + + CONSTRAINT attachment_job_attempts_valid + CHECK ( + attempts >= 0 + AND max_attempts BETWEEN 1 AND 20 + ) +); + +CREATE INDEX IF NOT EXISTS idx_attachment_jobs_pending +ON attachment_jobs(status, next_attempt_at); + +-- Image variants created by the worker. +CREATE TABLE IF NOT EXISTS attachment_variants ( + attachment_id UUID NOT NULL + REFERENCES attachments(id), + + variant_name TEXT NOT NULL, + object_key TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 CHAR(64) NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + PRIMARY KEY (attachment_id, variant_name), + + CONSTRAINT variant_name_valid + CHECK (variant_name IN ('thumbnail')), + + CONSTRAINT variant_size_valid + CHECK (size_bytes > 0), + + CONSTRAINT variant_dimensions_valid + CHECK (width > 0 AND height > 0) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_attachment_variants_key +ON attachment_variants(object_key); diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 00000000..810ed1c8 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,37 @@ +events { + worker_connections 1024; +} + +http { + resolver 127.0.0.11 valid=2s ipv6=off; + resolver_timeout 2s; + + upstream messenger_api { + zone messenger_api 64k; + + server api1:8080 resolve max_fails=1 fail_timeout=2s; + server api2:8080 resolve max_fails=1 fail_timeout=2s; + } + + server { + client_max_body_size 11m; + listen 80; + + location / { + proxy_pass http://messenger_api; + + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_connect_timeout 2s; + proxy_read_timeout 30s; + proxy_send_timeout 30s; + + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + } + } +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..6a109100 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.115.12 +uvicorn==0.34.2 +psycopg[binary]==3.2.6 +argon2-cffi==23.1.0 +boto3==1.40.70 +Pillow==11.2.1 +pika==1.3.2 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..b2c5a845 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +echo "Stopping API and image processing..." +docker compose stop api1 api2 worker dispatcher + +echo "Restarting persistent storage and broker..." +docker compose restart db minio rabbitmq + +echo "Waiting for PostgreSQL and RabbitMQ..." +docker compose up -d --wait db rabbitmq + +echo "Checking S3 bucket..." +docker compose run --rm --no-deps s3-init + +echo "Starting dispatcher and worker..." +docker compose up -d dispatcher worker + +echo "Starting API replicas..." +docker compose up -d --wait api1 api2 + +echo "Restart complete. Persistent volumes preserved." diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..36b95cd6 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +if [[ -z "${TARGET_INSTANCE_ID:-}" ]]; then + echo "TARGET_INSTANCE_ID is not set" >&2 + exit 1 +fi + +for service in api1 api2; do + instance_id=$( + docker compose exec -T "$service" python -c ' +import urllib.request + +response = urllib.request.urlopen( + "http://127.0.0.1:8080/health/live", + timeout=2 +) + +print(response.headers.get("X-Instance-Id", "")) +' 2>/dev/null + ) || continue + + if [[ "$instance_id" == "$TARGET_INSTANCE_ID" ]]; then + echo "Stopping $service" + docker compose stop -t 2 "$service" + exit 0 + fi +done + +echo "Target API instance not found" >&2 +exit 1 diff --git a/scripts/test-redelivery.sh b/scripts/test-redelivery.sh new file mode 100644 index 00000000..c02c0c8c --- /dev/null +++ b/scripts/test-redelivery.sh @@ -0,0 +1,77 @@ + +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +sql() { + docker compose exec -T db \ + psql -U messenger -d messenger -At -c "$1" +} + +echo "Finding completed image job..." + +JOB_ID=$(sql " + SELECT j.id + FROM attachment_jobs j + JOIN attachment_variants v + ON v.attachment_id = j.attachment_id + WHERE j.status = 'ready' + AND v.variant_name = 'thumbnail' + ORDER BY j.created_at DESC + LIMIT 1; +") + +if [[ -z "$JOB_ID" ]]; then + echo "FAIL: no completed image job found" + exit 1 +fi + +echo "Job: $JOB_ID" + +STATE_QUERY=" + SELECT + j.status || '|' || + j.attempts || '|' || + v.sha256 || '|' || + ( + SELECT COUNT(*) + FROM attachment_variants + WHERE attachment_id = j.attachment_id + ) + FROM attachment_jobs j + JOIN attachment_variants v + ON v.attachment_id = j.attachment_id + WHERE j.id = '$JOB_ID' + AND v.variant_name = 'thumbnail'; +" + +BEFORE=$(sql "$STATE_QUERY") + +echo "Before: $BEFORE" +echo "Publishing the same job twice..." + +docker compose run --rm --no-deps api1 \ + python -c ' +import sys +from src.queue import publish_job + +job_id = sys.argv[1] +publish_job(job_id) +publish_job(job_id) +print("Two duplicate messages published") +' "$JOB_ID" + +sleep 5 + +AFTER=$(sql "$STATE_QUERY") + +echo "After: $AFTER" + +if [[ "$BEFORE" != "$AFTER" ]]; then + echo "FAIL: completed job changed after redelivery" + exit 1 +fi + +echo "PASS: duplicate delivery did not change completed job" +echo "PASS: thumbnail checksum and variant count preserved" diff --git a/scripts/test-worker-recovery.py b/scripts/test-worker-recovery.py new file mode 100644 index 00000000..b82df2e7 --- /dev/null +++ b/scripts/test-worker-recovery.py @@ -0,0 +1,246 @@ + +import base64 +import hashlib +import json +import subprocess +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +import zlib +import struct + + +BASE = "http://127.0.0.1:8080" + + +def docker(*args): + subprocess.run( + ["docker", "compose", *args], + check=True, + ) + + +def api(method, path, token=None, body=None, headers=None): + request_headers = dict(headers or {}) + + if token: + request_headers["Authorization"] = f"Bearer {token}" + + if isinstance(body, dict): + body = json.dumps(body).encode("utf-8") + request_headers["Content-Type"] = "application/json" + + request = urllib.request.Request( + BASE + path, + data=body, + headers=request_headers, + method=method, + ) + + with urllib.request.urlopen(request, timeout=10) as response: + data = response.read() + return ( + response.status, + json.loads(data) if data and + "application/json" in response.headers.get( + "Content-Type", "" + ) else data, + ) + + +def png_chunk(name, data): + value = name + data + return ( + struct.pack(">I", len(data)) + + value + + struct.pack(">I", zlib.crc32(value) & 0xFFFFFFFF) + ) + + +def make_png(): + width, height = 320, 200 + scanline = b"\x00" + bytes([220, 80, 40]) * width + + return ( + b"\x89PNG\r\n\x1a\n" + + png_chunk( + b"IHDR", + struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0), + ) + + png_chunk( + b"IDAT", zlib.compress(scanline * height) + ) + + png_chunk(b"IEND", b"") + ) + + +def wait_ready(): + for _ in range(90): + try: + status, data = api("GET", "/health/ready") + if status == 200 and data.get("status") == "ok": + return + except Exception: + pass + time.sleep(1) + + raise RuntimeError("API did not become ready") + + +def main(): + suffix = uuid.uuid4().hex[:12] + username = "lab5_" + suffix + password = "Lab5TestPassword_" + suffix + + print("Stopping worker...", flush=True) + docker("stop", "worker") + wait_ready() + + status, user = api( + "POST", + "/api/v1/users", + body={ + "username": username, + "display_name": "Lab5 recovery test", + "password": password, + }, + ) + assert status == 201 + + credentials = base64.b64encode( + f"{username}:{password}".encode() + ).decode() + + status, session = api( + "POST", + "/api/v1/auth/sessions", + headers={"Authorization": f"Basic {credentials}"}, + body=b"", + ) + assert status == 201 + token = session["token"] + + status, chat = api( + "POST", + "/api/v1/chats", + token, + { + "title": "Worker recovery", + "member_ids": [user["id"]], + }, + ) + assert status == 201 + + original = make_png() + path = ( + f"/api/v1/chats/{chat['id']}/attachments" + "?filename=recovery.png" + ) + + status, attachment = api( + "POST", + path, + token, + original, + {"Content-Type": "image/png"}, + ) + + assert status == 202, f"Expected 202, got {status}" + assert attachment["status"] == "queued" + attachment_id = attachment["id"] + print("Upload: 202 queued", flush=True) + + status, message = api( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + token, + {"text": "", "attachment_ids": [attachment_id]}, + {"Idempotency-Key": "recovery-" + suffix}, + ) + assert status == 201 + assert attachment_id in message["attachment_ids"] + print("Message created while worker is stopped", flush=True) + + status, metadata = api( + "GET", f"/api/v1/attachments/{attachment_id}", token + ) + assert status == 200 and metadata["status"] == "queued" + + print("Restarting RabbitMQ and API...", flush=True) + docker("restart", "rabbitmq") + docker("up", "-d", "--wait", "rabbitmq") + docker("restart", "api1", "api2") + wait_ready() + + status, metadata = api( + "GET", f"/api/v1/attachments/{attachment_id}", token + ) + assert metadata["status"] == "queued" + print("Queued attachment survived restart", flush=True) + + print("Starting worker...", flush=True) + docker("up", "-d", "worker") + + deadline = time.monotonic() + 90 + + while time.monotonic() < deadline: + status, metadata = api( + "GET", f"/api/v1/attachments/{attachment_id}", token + ) + + if metadata["status"] == "ready": + break + + if metadata["status"] == "failed": + raise RuntimeError( + f"Processing failed: {metadata.get('error_code')}" + ) + + time.sleep(1) + else: + raise RuntimeError("Attachment did not become ready") + + status, downloaded = api( + "GET", + f"/api/v1/attachments/{attachment_id}/content", + token, + ) + assert status == 200 + assert downloaded == original + assert hashlib.sha256(downloaded).hexdigest() == ( + hashlib.sha256(original).hexdigest() + ) + + status, thumbnail = api( + "GET", + f"/api/v1/attachments/{attachment_id}/content" + "?variant=thumbnail", + token, + ) + + assert status == 200 + assert thumbnail.startswith(b"\xff\xd8") + assert thumbnail.endswith(b"\xff\xd9") + + variants = metadata["variants"] + assert any( + v["name"] == "thumbnail" + and v["width"] == 256 + and v["height"] == 160 + for v in variants + ) + + print("Attachment: ready", flush=True) + print("Original: SHA-256 verified", flush=True) + print("Thumbnail: JPEG 256x160 verified", flush=True) + print("PASS: durable worker recovery", flush=True) + + +if __name__ == "__main__": + try: + main() + finally: + # Restore the worker even when the test fails. + docker("up", "-d", "worker") diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 00000000..59652d25 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,108 @@ + +import io +import re +import uuid +import hashlib + +from PIL import Image, UnidentifiedImageError + + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 + + +def valid_filename(filename): + if not isinstance(filename, str): + return False + + if not 1 <= len(filename) <= 255: + return False + + if filename.strip() == "": + return False + + if "/" in filename or "\\" in filename: + return False + + if any(ord(ch) < 32 or ord(ch) == 127 for ch in filename): + return False + + return True + + +def inspect_image(data, declared_mime): + if len(data) > MAX_BYTES: + raise ValueError("too_large") + + if not data: + raise ValueError("invalid_image") + + if declared_mime not in ("image/png", "image/jpeg"): + raise ValueError("invalid_image") + + try: + with Image.open(io.BytesIO(data)) as image: + image_format = image.format + + width, height = image.size + + if width * height > MAX_PIXELS: + raise ValueError("too_many_pixels") + + if image_format not in ("PNG", "JPEG"): + raise ValueError("invalid_image") + + actual_mime = ( + "image/png" + if image_format == "PNG" + else "image/jpeg" + ) + + if declared_mime != actual_mime: + raise ValueError("invalid_image") + + image.verify() + + except Image.DecompressionBombError: + raise ValueError("too_many_pixels") + except ( + UnidentifiedImageError, + OSError, + SyntaxError, + EOFError, + ): + raise ValueError("invalid_image") + + return { + "content_type": actual_mime, + "width": width, + "height": height, + "size_bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + } + +def attachment_json(row, variants=None): + from datetime import timezone + + def timestamp(value): + return value.astimezone(timezone.utc).isoformat( + timespec="microseconds" + ).replace("+00:00", "Z") + + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["uploader_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": variants if variants is not None else [], + "error_code": row["error_code"], + "created_at": timestamp(row["created_at"]), + "updated_at": timestamp(row["updated_at"]), + } + +def object_key(): + return f"originals/{uuid.uuid4()}" diff --git a/src/dispatcher.py b/src/dispatcher.py new file mode 100644 index 00000000..31c8cf37 --- /dev/null +++ b/src/dispatcher.py @@ -0,0 +1,84 @@ + +import logging +import os +import time + +import psycopg +from psycopg.rows import dict_row + +from src.queue import publish_job + + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", +) + +logger = logging.getLogger("image-dispatcher") + +DATABASE_URL = os.environ["DATABASE_URL"] + + +def dispatch_one(): + with psycopg.connect( + DATABASE_URL, + row_factory=dict_row, + ) as conn: + + # Only publish due, unpublished jobs. + # The row lock also prevents two dispatchers + # from selecting the same job simultaneously. + job = conn.execute( + """ + SELECT id + FROM attachment_jobs + WHERE status = 'queued' + AND published_at IS NULL + AND next_attempt_at <= NOW() + ORDER BY created_at, id + LIMIT 1 + FOR UPDATE SKIP LOCKED + """, + ).fetchone() + + if job is None: + return False + + job_id = str(job["id"]) + + # RabbitMQ publisher confirms are enabled. + # An exception rolls back this DB transaction. + publish_job(job_id) + + conn.execute( + """ + UPDATE attachment_jobs + SET published_at = NOW(), + updated_at = NOW() + WHERE id = %s + """, + (job_id,), + ) + + logger.info("Published job %s", job_id) + + return True + + +def main(): + logger.info("Dispatcher started") + + while True: + try: + if not dispatch_one(): + time.sleep(1) + + except Exception: + logger.exception( + "Dispatch failed; retrying" + ) + time.sleep(3) + + +if __name__ == "__main__": + main() diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..dea9ee68 --- /dev/null +++ b/src/main.py @@ -0,0 +1,1272 @@ + +import base64 +import hashlib +import io +import json +import os +import re +import secrets +import uuid + +from datetime import datetime, timedelta, timezone + +import psycopg +from psycopg.rows import dict_row +from psycopg.errors import UniqueViolation + +from argon2 import PasswordHasher +from argon2.exceptions import ( + VerifyMismatchError, + VerificationError, + InvalidHashError, +) + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from fastapi.exceptions import RequestValidationError +from starlette.exceptions import HTTPException as StarletteHTTPException + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, UnidentifiedImageError + +app = FastAPI(title="MTUSI Messenger Lab5") + +INSTANCE_ID = str(uuid.uuid4()) + +DATABASE_URL = os.environ["DATABASE_URL"] +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) + +IMAGE_MAX_ATTEMPTS = int(os.getenv("IMAGE_MAX_ATTEMPTS", "5")) +S3_BUCKET = os.environ["S3_BUCKET"] +MAX_IMAGE_BYTES = 10 * 1024 * 1024 +MAX_IMAGE_PIXELS = 40_000_000 + +s3 = boto3.client( + "s3", + endpoint_url=os.environ["S3_ENDPOINT"], + aws_access_key_id=os.environ["S3_ACCESS_KEY"], + aws_secret_access_key=os.environ["S3_SECRET_KEY"], + region_name="us-east-1", + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=3, + read_timeout=10, + retries={"max_attempts": 1}, + ), +) + +password_hasher = PasswordHasher() + + +# ---------- Helpers ---------- + +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def fail(status, code, message, headers=None): + raise ApiError(status, code, message, headers) + + +def db(): + return psycopg.connect(DATABASE_URL, row_factory=dict_row) + + +def utc_string(value): + return value.astimezone(timezone.utc).isoformat( + timespec="microseconds" + ).replace("+00:00", "Z") + + +def user_json(row): + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": utc_string(row["created_at"]), + } + + + +def message_json(row, conn=None): + attachment_ids = [] + + if conn is not None: + attachments = conn.execute( + """ + SELECT attachment_id + FROM message_attachments + WHERE message_id = %s + ORDER BY attachment_id + """, + (row["id"],), + ).fetchall() + + attachment_ids = [ + str(item["attachment_id"]) + for item in attachments + ] + + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "attachment_ids": attachment_ids, + "created_at": utc_string(row["created_at"]), + } + + + +def chat_json(conn, row): + members = conn.execute( + """ + SELECT user_id FROM chat_members + WHERE chat_id = %s + ORDER BY position + """, + (row["id"],) + ).fetchall() + + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(m["user_id"]) for m in members], + "created_at": utc_string(row["created_at"]), + } + + +def valid_uuid(value): + if not isinstance(value, str): + return False + try: + return str(uuid.UUID(value)) == value + except ValueError: + return False + + +def valid_text(value, maximum): + return ( + isinstance(value, str) + and 1 <= len(value) <= maximum + and bool(value.strip()) + ) + + +def validate_limit(request): + value = request.query_params.get("limit", "50") + + if not re.fullmatch(r"[0-9]+", value): + fail(400, "invalid_request", "Invalid limit") + + number = int(value) + + if not 1 <= number <= 100: + fail(400, "invalid_request", "Invalid limit") + + return number + + +def make_cursor(resource, row): + payload = { + "resource": resource, + "created_at": utc_string(row["created_at"]), + "id": str(row["id"]), + } + + raw = json.dumps(payload).encode("utf-8") + + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def read_cursor(cursor, resource): + if not isinstance(cursor, str) or not cursor: + fail(400, "invalid_request", "Invalid cursor") + + try: + raw = base64.urlsafe_b64decode( + cursor + "=" * (-len(cursor) % 4) + ) + value = json.loads(raw) + + if value["resource"] != resource: + raise ValueError() + + if not valid_uuid(value["id"]): + raise ValueError() + + timestamp = datetime.fromisoformat( + value["created_at"].replace("Z", "+00:00") + ) + + if timestamp.tzinfo is None: + raise ValueError() + + return timestamp, value["id"] + + except (ValueError, KeyError, TypeError, UnicodeDecodeError): + fail(400, "invalid_request", "Invalid cursor") + + +async def json_body(request, required, allowed): + content_type = request.headers.get("content-type", "") + media_type = content_type.split(";")[0].strip().lower() + + if media_type != "application/json": + fail(415, "unsupported_media_type", "Expected JSON") + + try: + data = await request.json() + except Exception: + fail(400, "invalid_request", "Invalid JSON") + + if not isinstance(data, dict): + fail(400, "invalid_request", "Expected JSON object") + + if not set(required).issubset(data): + fail(400, "invalid_request", "Missing fields") + + if not set(data).issubset(allowed): + fail(400, "invalid_request", "Unexpected fields") + + return data + + +def unauthorized(kind="Bearer"): + if kind == "Basic": + header = 'Basic realm="messenger", charset="UTF-8"' + else: + header = 'Bearer realm="messenger"' + + fail( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": header}, + ) + + +def current_user(request): + authorization = request.headers.get("Authorization", "") + + if not authorization.startswith("Bearer "): + unauthorized() + + token = authorization[7:] + + if not token: + unauthorized() + + token_hash = hashlib.sha256(token.encode()).hexdigest() + + with db() as conn: + row = conn.execute( + """ + SELECT u.id, u.username, u.display_name, u.created_at + FROM sessions s + JOIN users u ON u.id = s.user_id + WHERE s.token_hash = %s + AND s.revoked = FALSE + AND s.expires_at > clock_timestamp() + """, + (token_hash,) + ).fetchone() + + if row is None: + unauthorized() + + return row + + +def accessible_chat(conn, chat_id, user_id): + if not valid_uuid(chat_id): + fail(400, "invalid_request", "Invalid chat ID") + + row = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE c.id = %s AND m.user_id = %s + """, + (chat_id, user_id) + ).fetchone() + + if row is None: + fail(404, "not_found", "Chat not found") + + return row + + +# ---------- Request IDs and errors ---------- + +def error_response(request, status, code, message, headers=None): + request_id = request.state.request_id + + response_headers = { + "X-Request-Id": request_id, + **(headers or {}), + } + + return JSONResponse( + status_code=status, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers=response_headers, + ) + + +@app.middleware("http") +async def add_request_id(request, call_next): + request.state.request_id = str(uuid.uuid4()) + + try: + response = await call_next(request) + except psycopg.OperationalError: + response = error_response( + request, 503, "unavailable", "Database unavailable" + ) + except Exception: + response = error_response( + request, 500, "internal_error", "Internal server error" + ) + + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(ApiError) +async def api_error_handler(request, exc): + return error_response( + request, exc.status, exc.code, exc.message, exc.headers + ) + + +@app.exception_handler(StarletteHTTPException) +async def http_error_handler(request, exc): + return error_response( + request, exc.status_code, + "not_found" if exc.status_code == 404 else "invalid_request", + "Resource not found" if exc.status_code == 404 else "Invalid request", + ) + + +@app.exception_handler(RequestValidationError) +async def validation_error_handler(request, exc): + return error_response( + request, 400, "invalid_request", "Invalid request" + ) + + +# ---------- Health ---------- + +@app.get("/health/live") +def health_live(): + return {"status": "ok"} + + + +@app.get("/health/ready") +def health_ready(): + try: + with db() as conn: + conn.execute("SELECT 1") + + s3.head_bucket(Bucket=S3_BUCKET) + + return {"status": "ok"} + + except ( + psycopg.Error, + BotoCoreError, + ClientError, + OSError, + ): + fail( + 503, + "unavailable", + "Storage unavailable", + ) + + +# ---------- Users ---------- + +@app.post("/api/v1/users", status_code=201) +async def create_user(request: Request): + data = await json_body( + request, + required=["username", "display_name", "password"], + allowed=["username", "display_name", "password"], + ) + + username = data["username"] + display_name = data["display_name"] + password = data["password"] + + if ( + not isinstance(username, str) + or not re.fullmatch(r"[a-z0-9_]{3,32}", username) + or not valid_text(display_name, 100) + or not isinstance(password, str) + or not 12 <= len(password) <= 128 + ): + fail(400, "invalid_request", "Invalid user fields") + + password_hash = password_hasher.hash(password) + user_id = str(uuid.uuid4()) + + try: + with db() as conn: + row = conn.execute( + """ + INSERT INTO users + (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (user_id, username, display_name, password_hash) + ).fetchone() + + return user_json(row) + + except UniqueViolation: + fail(409, "conflict", "Username already exists") + + +@app.get("/api/v1/users/me") +def get_me(request: Request): + return user_json(current_user(request)) + + +# ---------- Sessions ---------- + +@app.post("/api/v1/auth/sessions", status_code=201) +async def create_session(request: Request): + authorization = request.headers.get("Authorization", "") + + if not authorization.startswith("Basic "): + unauthorized("Basic") + + try: + encoded = authorization[6:] + decoded = base64.b64decode(encoded, validate=True).decode("utf-8") + + if ":" not in decoded: + raise ValueError() + + username, password = decoded.split(":", 1) + + except (ValueError, UnicodeDecodeError): + unauthorized("Basic") + + if not re.fullmatch(r"[a-z0-9_]{3,32}", username): + unauthorized("Basic") + + with db() as conn: + row = conn.execute( + "SELECT * FROM users WHERE username = %s", + (username,) + ).fetchone() + + if row is None: + unauthorized("Basic") + + try: + password_hasher.verify(row["password_hash"], password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + unauthorized("Basic") + + token = secrets.token_urlsafe(32) + token_hash = hashlib.sha256(token.encode()).hexdigest() + + expires_at = datetime.now(timezone.utc) + timedelta( + seconds=SESSION_TTL_SECONDS + ) + + with db() as conn: + conn.execute( + """ + INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES (%s, %s, %s) + """, + (token_hash, row["id"], expires_at) + ) + + return { + "token": token, + "token_type": "Bearer", + "expires_at": utc_string(expires_at), + "user": user_json(row), + } + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def logout(request: Request): + current_user(request) + + token = request.headers["Authorization"][7:] + token_hash = hashlib.sha256(token.encode()).hexdigest() + + with db() as conn: + conn.execute( + "UPDATE sessions SET revoked = TRUE WHERE token_hash = %s", + (token_hash,) + ) + + return Response(status_code=204) + + +# ---------- Chats ---------- + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + actor = current_user(request) + + data = await json_body( + request, + required=["title", "member_ids"], + allowed=["title", "member_ids"], + ) + + title = data["title"] + member_ids = data["member_ids"] + + if not valid_text(title, 100): + fail(400, "invalid_request", "Invalid title") + + if ( + not isinstance(member_ids, list) + or not 1 <= len(member_ids) <= 100 + or not all(valid_uuid(x) for x in member_ids) + or len(member_ids) != len(set(member_ids)) + or str(actor["id"]) not in member_ids + ): + fail(400, "invalid_request", "Invalid members") + + chat_id = str(uuid.uuid4()) + + with db() as conn: + # All operations use one transaction. + existing = conn.execute( + """ + SELECT id FROM users + WHERE id = ANY(%s::uuid[]) + FOR KEY SHARE + """, + (member_ids,) + ).fetchall() + + if len(existing) != len(member_ids): + fail(400, "invalid_request", "Unknown member") + + row = conn.execute( + """ + INSERT INTO chats (id, title) + VALUES (%s, %s) + RETURNING id, title, created_at + """, + (chat_id, title) + ).fetchone() + + for position, member_id in enumerate(member_ids): + conn.execute( + """ + INSERT INTO chat_members (chat_id, user_id, position) + VALUES (%s, %s, %s) + """, + (chat_id, member_id, position) + ) + + result = chat_json(conn, row) + + return result + + +@app.get("/api/v1/chats") +def list_chats(request: Request): + actor = current_user(request) + limit = validate_limit(request) + cursor = request.query_params.get("cursor") + + resource = "chats:" + str(actor["id"]) + + with db() as conn: + if cursor is None: + rows = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = %s + ORDER BY c.created_at, c.id + LIMIT %s + """, + (actor["id"], limit + 1) + ).fetchall() + else: + timestamp, last_id = read_cursor(cursor, resource) + + rows = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = %s + AND (c.created_at, c.id) > (%s, %s) + ORDER BY c.created_at, c.id + LIMIT %s + """, + (actor["id"], timestamp, last_id, limit + 1) + ).fetchall() + + has_more = len(rows) > limit + page = rows[:limit] + + items = [chat_json(conn, row) for row in page] + + next_cursor = ( + make_cursor(resource, page[-1]) + if has_more else None + ) + + return {"items": items, "next_cursor": next_cursor} + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: str, request: Request): + actor = current_user(request) + + with db() as conn: + row = accessible_chat(conn, chat_id, actor["id"]) + result = chat_json(conn, row) + + return result + + +# ---------- Attachments ---------- + +from src.attachments import ( + MAX_BYTES, + valid_filename, + inspect_image, + attachment_json, + object_key, +) + + +def attachment_for_user(conn, attachment_id, user_id): + if not valid_uuid(attachment_id): + fail(400, "invalid_request", "Invalid attachment ID") + + row = conn.execute( + """ + SELECT a.* + FROM attachments a + JOIN chat_members cm + ON cm.chat_id = a.chat_id + WHERE a.id = %s + AND cm.user_id = %s + """, + (attachment_id, user_id), + ).fetchone() + + if row is None: + fail(404, "not_found", "Attachment not found") + + return row + + +@app.post( + "/api/v1/chats/{chat_id}/attachments", + status_code=202, +) +async def upload_attachment(chat_id: str, request: Request): + actor = current_user(request) + + with db() as conn: + accessible_chat(conn, chat_id, actor["id"]) + + filename = request.query_params.get("filename") + + if not valid_filename(filename): + fail(400, "invalid_request", "Invalid filename") + + content_type = request.headers.get( + "content-type", "" + ).split(";")[0].strip().lower() + + if content_type not in ("image/png", "image/jpeg"): + fail( + 415, + "unsupported_media_type", + "Expected PNG or JPEG", + ) + + content_length = request.headers.get("content-length") + + if content_length is not None: + try: + declared_size = int(content_length) + except ValueError: + fail( + 400, + "invalid_request", + "Invalid Content-Length", + ) + + if declared_size > MAX_BYTES: + fail( + 413, + "payload_too_large", + "Image exceeds 10 MiB", + ) + + # Receive the HTTP body with a strict byte limit. + data = bytearray() + + async for chunk in request.stream(): + if len(data) + len(chunk) > MAX_BYTES: + fail( + 413, + "payload_too_large", + "Image exceeds 10 MiB", + ) + + data.extend(chunk) + + data = bytes(data) + + try: + metadata = inspect_image(data, content_type) + except ValueError as exc: + if str(exc) in ("too_large", "too_many_pixels"): + fail( + 413, + "payload_too_large", + "Image exceeds limits", + ) + + fail( + 415, + "unsupported_media_type", + "Invalid PNG or JPEG", + ) + + attachment_id = str(uuid.uuid4()) + job_id = str(uuid.uuid4()) + + # Server-generated keys, independent of filename. + staging_key = f"staging/{attachment_id}" + original_key = f"originals/{attachment_id}" + + # First persist the accepted original in shared S3. + try: + s3.put_object( + Bucket=S3_BUCKET, + Key=staging_key, + Body=data, + ContentType=metadata["content_type"], + ) + except (BotoCoreError, ClientError, OSError): + fail( + 503, + "unavailable", + "Storage unavailable", + ) + + try: + with db() as conn: + accessible_chat(conn, chat_id, actor["id"]) + + row = conn.execute( + """ + INSERT INTO attachments + ( + id, + chat_id, + uploader_id, + filename, + content_type, + size_bytes, + sha256, + object_key, + staging_key, + width, + height, + status, + error_code + ) + VALUES ( + %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, + %s, 'queued', NULL + ) + RETURNING * + """, + ( + attachment_id, + chat_id, + actor["id"], + filename, + metadata["content_type"], + metadata["size_bytes"], + metadata["sha256"], + original_key, + staging_key, + metadata["width"], + metadata["height"], + ), + ).fetchone() + + # Durable job record in the same DB transaction. + conn.execute( + """ + INSERT INTO attachment_jobs + ( + id, + attachment_id, + status, + attempts, + max_attempts, + next_attempt_at + ) + VALUES ( + %s, %s, 'queued', + 0, %s, NOW() + ) + """, + ( + job_id, + attachment_id, + IMAGE_MAX_ATTEMPTS, + ), + ) + + except Exception: + # The database transaction was not confirmed. + # Try to clean up the unreferenced staging object. + try: + s3.delete_object( + Bucket=S3_BUCKET, + Key=staging_key, + ) + except Exception: + pass + + raise + + # Do not perform resizing or require the worker here. + # The dispatcher will publish this durable DB job. + return attachment_json(row) + + + +@app.get("/api/v1/attachments/{attachment_id}") +def get_attachment(attachment_id: str, request: Request): + actor = current_user(request) + + with db() as conn: + row = attachment_for_user( + conn, attachment_id, actor["id"] + ) + + variant_rows = conn.execute( + """ + SELECT variant_name, content_type, + size_bytes, sha256, width, height + FROM attachment_variants + WHERE attachment_id = %s + ORDER BY variant_name + """, + (attachment_id,), + ).fetchall() + + variants = [ + { + "name": v["variant_name"], + "content_type": v["content_type"], + "size_bytes": v["size_bytes"], + "sha256": v["sha256"], + "width": v["width"], + "height": v["height"], + } + for v in variant_rows + ] + + return attachment_json(row, variants) + + + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment( + attachment_id: str, + request: Request, +): + actor = current_user(request) + + variant = request.query_params.get( + "variant", "original" + ) + + with db() as conn: + row = attachment_for_user( + conn, attachment_id, actor["id"] + ) + + if row["status"] != "ready": + fail( + 409, + "conflict", + "Attachment is not ready", + ) + + if variant == "original": + key = row["object_key"] + content_type = row["content_type"] + expected_size = row["size_bytes"] + expected_sha256 = row["sha256"] + + elif variant == "thumbnail": + variant_row = conn.execute( + """ + SELECT object_key, content_type, + size_bytes, sha256 + FROM attachment_variants + WHERE attachment_id = %s + AND variant_name = 'thumbnail' + """, + (attachment_id,), + ).fetchone() + + if variant_row is None: + fail( + 404, + "not_found", + "Variant not found", + ) + + key = variant_row["object_key"] + content_type = variant_row["content_type"] + expected_size = variant_row["size_bytes"] + expected_sha256 = variant_row["sha256"] + + else: + fail( + 404, + "not_found", + "Variant not found", + ) + + try: + result = s3.get_object( + Bucket=S3_BUCKET, + Key=key, + ) + + body_stream = result["Body"] + + try: + data = body_stream.read(MAX_BYTES + 1) + finally: + body_stream.close() + + except (BotoCoreError, ClientError, OSError): + fail( + 503, + "unavailable", + "Storage unavailable", + ) + + if ( + len(data) != expected_size + or hashlib.sha256(data).hexdigest() + != expected_sha256 + ): + fail( + 503, + "unavailable", + "Stored object is unavailable", + ) + + return Response( + content=data, + media_type=content_type, + headers={ + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + }, + ) + +def get_idempotency_key(request): + key = request.headers.get("Idempotency-Key") + + if ( + key is None + or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key) + ): + fail( + 400, + "invalid_request", + "Invalid or missing Idempotency-Key" + ) + + return key + + +def request_fingerprint(data): + normalized = { + "text": data["text"], + "attachment_ids": data.get("attachment_ids", []), + } + + encoded = json.dumps( + normalized, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + return hashlib.sha256(encoded).hexdigest() + + + +def idempotency_lock_id(user_id, chat_id, key): + scope = f"{user_id}:{chat_id}:{key}" + + digest = hashlib.sha256( + scope.encode("utf-8") + ).digest() + + return int.from_bytes( + digest[:8], + byteorder="big", + signed=True + ) + + +# ---------- Messages ---------- + + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def send_message(chat_id: str, request: Request): + actor = current_user(request) + + data = await json_body( + request, + required=["text"], + allowed=["text", "attachment_ids"], + ) + + message_text = data["text"] + attachment_ids = data.get("attachment_ids", []) + + if ( + not isinstance(message_text, str) + or len(message_text) > 4000 + ): + fail(400, "invalid_request", "Invalid message text") + + if ( + not isinstance(attachment_ids, list) + or len(attachment_ids) > 10 + or not all(valid_uuid(x) for x in attachment_ids) + or len(attachment_ids) != len(set(attachment_ids)) + ): + fail(400, "invalid_request", "Invalid attachment_ids") + + if not message_text.strip() and not attachment_ids: + fail(400, "invalid_request", "Empty message") + + key = get_idempotency_key(request) + fingerprint = request_fingerprint(data) + user_id = actor["id"] + + with db() as conn: + accessible_chat(conn, chat_id, user_id) + + lock_id = idempotency_lock_id( + user_id, chat_id, key + ) + + conn.execute( + "SELECT pg_advisory_xact_lock(%s)", + (lock_id,), + ) + + conn.execute( + """ + DELETE FROM message_idempotency + WHERE user_id = %s + AND chat_id = %s + AND idempotency_key = %s + AND created_at < + clock_timestamp() - INTERVAL '24 hours' + """, + (user_id, chat_id, key), + ) + + previous = conn.execute( + """ + SELECT i.request_hash, m.* + FROM message_idempotency i + JOIN messages m ON m.id = i.message_id + WHERE i.user_id = %s + AND i.chat_id = %s + AND i.idempotency_key = %s + """, + (user_id, chat_id, key), + ).fetchone() + + if previous is not None: + if previous["request_hash"] != fingerprint: + fail( + 409, + "conflict", + "Idempotency-Key already used " + "with different content", + ) + + return message_json(previous, conn) + + for attachment_id in attachment_ids: + attachment = conn.execute( + """ + SELECT id, status + FROM attachments + WHERE id = %s + AND chat_id = %s + FOR KEY SHARE + """, + (attachment_id, chat_id), + ).fetchone() + + if attachment is None: + fail( + 404, + "not_found", + "Attachment not found", + ) + + if attachment["status"] == "failed": + fail( + 409, + "conflict", + "Attachment processing failed", + ) + + if attachment["status"] not in ( + "queued", + "processing", + "ready", + ): + fail( + 409, + "conflict", + "Invalid attachment status", + ) + + message_id = str(uuid.uuid4()) + + row = conn.execute( + """ + INSERT INTO messages + (id, chat_id, sender_id, text) + VALUES (%s, %s, %s, %s) + RETURNING id, chat_id, sender_id, + text, created_at + """, + (message_id, chat_id, user_id, message_text), + ).fetchone() + + for attachment_id in attachment_ids: + conn.execute( + """ + INSERT INTO message_attachments + (message_id, attachment_id) + VALUES (%s, %s) + """, + (message_id, attachment_id), + ) + + conn.execute( + """ + INSERT INTO message_idempotency + ( + user_id, chat_id, idempotency_key, + request_hash, message_id + ) + VALUES (%s, %s, %s, %s, %s) + """, + ( + user_id, chat_id, key, + fingerprint, message_id, + ), + ) + + return message_json(row, conn) + + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(chat_id: str, request: Request): + actor = current_user(request) + limit = validate_limit(request) + cursor = request.query_params.get("cursor") + + resource = "messages:" + chat_id + + with db() as conn: + accessible_chat(conn, chat_id, actor["id"]) + + if cursor is None: + rows = conn.execute( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s + ORDER BY created_at, id + LIMIT %s + """, + (chat_id, limit + 1) + ).fetchall() + else: + timestamp, last_id = read_cursor(cursor, resource) + + rows = conn.execute( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s + AND (created_at, id) > (%s, %s) + ORDER BY created_at, id + LIMIT %s + """, + (chat_id, timestamp, last_id, limit + 1) + ).fetchall() + + has_more = len(rows) > limit + page = rows[:limit] + + next_cursor = ( + make_cursor(resource, page[-1]) + if has_more else None + ) + + items = [message_json(row, conn) for row in page] + + return {"items": items, "next_cursor": next_cursor} diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..c13aa99c --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,53 @@ + +import os +from pathlib import Path + +import psycopg + + +DATABASE_URL = os.environ["DATABASE_URL"] +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" + + +def run_migrations(): + with psycopg.connect(DATABASE_URL) as conn: + with conn.cursor() as cur: + # Prevent simultaneous migration runs. + cur.execute("SELECT pg_advisory_xact_lock(2026002)") + + cur.execute(""" + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL + DEFAULT clock_timestamp() + ) + """) + + for file in sorted(MIGRATIONS_DIR.glob("*.sql")): + cur.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (file.name,) + ) + + if cur.fetchone(): + print(f"Already applied: {file.name}") + continue + + sql = file.read_text(encoding="utf-8") + + for statement in sql.split(";"): + if statement.strip(): + cur.execute(statement) + + cur.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", + (file.name,) + ) + + print(f"Applied migration: {file.name}") + + print("Migrations completed") + + +if __name__ == "__main__": + run_migrations() diff --git a/src/queue.py b/src/queue.py new file mode 100644 index 00000000..125b5173 --- /dev/null +++ b/src/queue.py @@ -0,0 +1,89 @@ + +import json +import os +import uuid + +import pika + + +RABBITMQ_HOST = os.getenv("RABBITMQ_HOST", "rabbitmq") +RABBITMQ_USER = os.environ["RABBITMQ_USER"] +RABBITMQ_PASSWORD = os.environ["RABBITMQ_PASSWORD"] +RABBITMQ_QUEUE = os.getenv( + "RABBITMQ_QUEUE", + "image_jobs", +) + + +def connect(): + credentials = pika.PlainCredentials( + RABBITMQ_USER, + RABBITMQ_PASSWORD, + ) + + parameters = pika.ConnectionParameters( + host=RABBITMQ_HOST, + port=5672, + credentials=credentials, + heartbeat=30, + blocked_connection_timeout=15, + connection_attempts=3, + retry_delay=2, + ) + + return pika.BlockingConnection(parameters) + + +def declare_queue(channel): + channel.queue_declare( + queue=RABBITMQ_QUEUE, + durable=True, + arguments={ + "x-queue-type": "quorum", + }, + ) + + +def publish_job(job_id): + connection = connect() + + try: + channel = connection.channel() + declare_queue(channel) + + # Wait for confirmation from RabbitMQ. + channel.confirm_delivery() + + payload = json.dumps({ + "job_id": str(uuid.UUID(str(job_id))), + }).encode("utf-8") + + channel.basic_publish( + exchange="", + routing_key=RABBITMQ_QUEUE, + body=payload, + properties=pika.BasicProperties( + delivery_mode=2, + content_type="application/json", + message_id=str(job_id), + ), + mandatory=True, + ) + + finally: + if connection.is_open: + connection.close() + + +def parse_job(body): + payload = json.loads(body) + + if not isinstance(payload, dict): + raise ValueError("Invalid job payload") + + job_id = payload.get("job_id") + + if not isinstance(job_id, str): + raise ValueError("Missing job_id") + + return str(uuid.UUID(job_id)) diff --git a/src/s3_setup.py b/src/s3_setup.py new file mode 100644 index 00000000..7798af60 --- /dev/null +++ b/src/s3_setup.py @@ -0,0 +1,60 @@ + +import os +import time + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + + +def main(): + bucket = os.environ["S3_BUCKET"] + + client = boto3.client( + "s3", + endpoint_url=os.environ["S3_ENDPOINT"], + aws_access_key_id=os.environ["S3_ACCESS_KEY"], + aws_secret_access_key=os.environ["S3_SECRET_KEY"], + region_name="us-east-1", + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + ), + ) + + for attempt in range(30): + try: + client.head_bucket(Bucket=bucket) + print(f"Bucket already exists: {bucket}", flush=True) + break + + except ClientError as exc: + code = exc.response["Error"]["Code"] + + if code in ("404", "NoSuchBucket"): + try: + client.create_bucket(Bucket=bucket) + print(f"Bucket created: {bucket}", flush=True) + break + except Exception as create_error: + print( + f"Waiting for S3: {create_error}", + flush=True, + ) + else: + print(f"Waiting for S3: {exc}", flush=True) + + except Exception as exc: + print(f"Waiting for S3: {exc}", flush=True) + + if attempt == 29: + raise RuntimeError("Cannot initialize S3 bucket") + + time.sleep(2) + + # No public bucket policy is configured. + print(f"Private S3 bucket is ready: {bucket}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 00000000..fe4cd599 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,456 @@ + +import hashlib +import io +import logging +import os +import time + +import boto3 +import pika +import psycopg +from botocore.config import Config +from PIL import Image, ImageOps +from psycopg.rows import dict_row + +from src.queue import ( + RABBITMQ_QUEUE, + connect, + declare_queue, + parse_job, +) + + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", +) + +logger = logging.getLogger("image-worker") + +DATABASE_URL = os.environ["DATABASE_URL"] +S3_BUCKET = os.environ["S3_BUCKET"] +MAX_BYTES = 10 * 1024 * 1024 + +s3 = boto3.client( + "s3", + endpoint_url=os.environ["S3_ENDPOINT"], + aws_access_key_id=os.environ["S3_ACCESS_KEY"], + aws_secret_access_key=os.environ["S3_SECRET_KEY"], + region_name="us-east-1", + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=3, + read_timeout=15, + retries={"max_attempts": 1}, + ), +) + + +class InvalidImage(Exception): + pass + + +def make_thumbnail(original): + try: + with Image.open(io.BytesIO(original)) as image: + if image.format not in ("PNG", "JPEG"): + raise InvalidImage("Unsupported image") + + if image.width * image.height > 40_000_000: + raise InvalidImage("Too many pixels") + + image = ImageOps.exif_transpose(image) + image.thumbnail( + (256, 256), + Image.Resampling.LANCZOS, + ) + + # JPEG does not support transparency. + rgba = image.convert("RGBA") + + background = Image.new( + "RGB", + rgba.size, + (255, 255, 255), + ) + + background.paste( + rgba, + mask=rgba.getchannel("A"), + ) + + output = io.BytesIO() + + background.save( + output, + format="JPEG", + quality=85, + optimize=True, + ) + + return ( + output.getvalue(), + background.width, + background.height, + ) + + except (OSError, SyntaxError, EOFError) as exc: + raise InvalidImage("Invalid image") from exc + + +def get_staging_object(key): + response = s3.get_object( + Bucket=S3_BUCKET, + Key=key, + ) + + stream = response["Body"] + + try: + data = stream.read(MAX_BYTES + 1) + finally: + stream.close() + + if not data or len(data) > MAX_BYTES: + raise InvalidImage("Invalid staging size") + + return data + + +def process_job(job_id): + """ + Use a PostgreSQL row lock to prevent concurrent + processing of the same logical job. + + The transaction commits before RabbitMQ ACK. + """ + + with psycopg.connect( + DATABASE_URL, + row_factory=dict_row, + ) as conn: + + job = conn.execute( + """ + SELECT j.*, a.staging_key, + a.object_key, a.sha256, + a.size_bytes, a.status AS attachment_status + FROM attachment_jobs j + JOIN attachments a + ON a.id = j.attachment_id + WHERE j.id = %s + FOR UPDATE OF j, a + """, + (job_id,), + ).fetchone() + + if job is None: + logger.info("Unknown job %s, ignoring", job_id) + return + + if job["status"] == "ready": + logger.info("Job %s already completed", job_id) + return + + if job["status"] == "failed": + logger.info("Job %s already failed", job_id) + return + + if job["next_attempt_at"] is not None: + from datetime import datetime, timezone + + if job["next_attempt_at"] > datetime.now(timezone.utc): + logger.info("Job %s is not due yet", job_id) + return + + attempts = job["attempts"] + 1 + + if attempts > job["max_attempts"]: + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', + completed_at = NOW(), + updated_at = NOW() + WHERE id = %s + """, + (job_id,), + ) + + conn.execute( + """ + UPDATE attachments + SET status = 'failed', + error_code = 'processing_failed', + updated_at = NOW() + WHERE id = %s + """, + (job["attachment_id"],), + ) + + return + + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'processing', + attempts = %s, + started_at = NOW(), + updated_at = NOW() + WHERE id = %s + """, + (attempts, job_id), + ) + + conn.execute( + """ + UPDATE attachments + SET status = 'processing', + updated_at = NOW() + WHERE id = %s + """, + (job["attachment_id"],), + ) + + try: + original = get_staging_object( + job["staging_key"] + ) + + if ( + len(original) != job["size_bytes"] + or hashlib.sha256(original).hexdigest() + != job["sha256"] + ): + raise InvalidImage("Staging checksum mismatch") + + thumbnail, width, height = make_thumbnail( + original + ) + + attachment_id = str(job["attachment_id"]) + + # Stable keys make duplicate delivery safe. + original_key = job["object_key"] + thumb_key = ( + f"thumbnails/{attachment_id}.jpg" + ) + + s3.put_object( + Bucket=S3_BUCKET, + Key=original_key, + Body=original, + ContentType="image/png" + if original[:8] == b"\x89PNG\r\n\x1a\n" + else "image/jpeg", + ) + + s3.put_object( + Bucket=S3_BUCKET, + Key=thumb_key, + Body=thumbnail, + ContentType="image/jpeg", + ) + + thumb_sha256 = hashlib.sha256( + thumbnail + ).hexdigest() + + conn.execute( + """ + INSERT INTO attachment_variants + ( + attachment_id, + variant_name, + object_key, + content_type, + size_bytes, + sha256, + width, + height + ) + VALUES ( + %s, 'thumbnail', %s, 'image/jpeg', + %s, %s, %s, %s + ) + ON CONFLICT ( + attachment_id, variant_name + ) + DO UPDATE SET + object_key = EXCLUDED.object_key, + size_bytes = EXCLUDED.size_bytes, + sha256 = EXCLUDED.sha256, + width = EXCLUDED.width, + height = EXCLUDED.height + """, + ( + job["attachment_id"], + thumb_key, + len(thumbnail), + thumb_sha256, + width, + height, + ), + ) + + conn.execute( + """ + UPDATE attachments + SET status = 'ready', + error_code = NULL, + updated_at = NOW() + WHERE id = %s + """, + (job["attachment_id"],), + ) + + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'ready', + completed_at = NOW(), + updated_at = NOW() + WHERE id = %s + """, + (job_id,), + ) + + logger.info( + "Job %s ready, thumbnail %sx%s", + job_id, + width, + height, + ) + + except Exception as exc: + logger.exception( + "Processing failed for job %s: %s", + job_id, + type(exc).__name__, + ) + + if ( + attempts >= job["max_attempts"] + or isinstance(exc, InvalidImage) + ): + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', + completed_at = NOW(), + updated_at = NOW() + WHERE id = %s + """, + (job_id,), + ) + + conn.execute( + """ + UPDATE attachments + SET status = 'failed', + error_code = 'processing_failed', + updated_at = NOW() + WHERE id = %s + """, + (job["attachment_id"],), + ) + + else: + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', + next_attempt_at = + NOW() + INTERVAL '5 seconds', + published_at = NULL, + updated_at = NOW() + WHERE id = %s + """, + (job_id,), + ) + + conn.execute( + """ + UPDATE attachments + SET status = 'queued', + updated_at = NOW() + WHERE id = %s + """, + (job["attachment_id"],), + ) + + +def on_message(channel, method, properties, body): + try: + job_id = parse_job(body) + except (ValueError, TypeError, KeyError): + logger.warning("Invalid queue message") + channel.basic_ack( + delivery_tag=method.delivery_tag + ) + return + + try: + process_job(job_id) + + # ACK only after PostgreSQL transaction commits. + channel.basic_ack( + delivery_tag=method.delivery_tag + ) + + except Exception: + logger.exception( + "Unexpected failure for job %s", + job_id, + ) + + channel.basic_nack( + delivery_tag=method.delivery_tag, + requeue=True, + ) + + time.sleep(2) + + +def main(): + while True: + try: + connection = connect() + channel = connection.channel() + + declare_queue(channel) + + channel.basic_qos( + prefetch_count=1 + ) + + channel.basic_consume( + queue=RABBITMQ_QUEUE, + on_message_callback=on_message, + auto_ack=False, + ) + + logger.info( + "Worker listening on %s", + RABBITMQ_QUEUE, + ) + + channel.start_consuming() + + except Exception: + logger.exception( + "Worker disconnected, reconnecting" + ) + time.sleep(3) + + finally: + if ( + "connection" in locals() + and connection.is_open + ): + connection.close() + + +if __name__ == "__main__": + main()