diff --git a/.dockerignore b/.dockerignore index bc16e55..036eee9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,10 @@ __pycache__ node_modules evidence .DS_Store +.git +.env +.venv +__pycache__ +.pytest_cache +*.pyc +evidence \ No newline at end of file diff --git a/.env.example b/.env.example index 2b32655..451b3be 100644 --- a/.env.example +++ b/.env.example @@ -23,8 +23,8 @@ WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. -OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +RABBITMQ_USER=messenger +RABBITMQ_PASSWORD=change-me +RABBITMQ_QUEUE=image_jobs +IMAGE_MAX_ATTEMPTS=5 +GRAFANA_ADMIN_PASSWORD=replace_me \ No newline at end of file diff --git a/.gitignore b/.gitignore index b5a7189..702888b 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,8 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +.env +.venv/ +__pycache__/ +*.pyc +evidence/*.tmp \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a1420a1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.12-slim + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +COPY src/ ./src/ +COPY migrations/ ./migrations/ + +ENV HOST=0.0.0.0 +ENV PORT=8080 + +EXPOSE 8080 + +CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1"] diff --git a/REPORT.md b/REPORT.md index b97ae42..2060fac 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,329 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- **Автор, группа:** Матвиенко Павел, БПИ2304 +- **Целевая оценка; выбранный трек 5, если нужен:** 3. +- **Commit/PR:** TODO после публикации. +- **Использованные источники, библиотеки и помощь ИИ:** задание и контракты лабораторной №6, документация OpenTelemetry, Prometheus, Loki, Tempo, Grafana, FastAPI, Python SDK; при разработке и отладке использовалась помощь ИИ. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Среда:** Windows с WSL Ubuntu и Docker Desktop с поддержкой Docker Compose. + +**Необходимое ПО:** Git, Docker Engine / Docker Desktop, Docker Compose, Python 3, GNU Make. + +**Основные версии компонентов:** + +| Компонент | Версия | +| --- | --- | +| PostgreSQL | 16-alpine | +| RustFS | 1.0.3 | +| RabbitMQ | 4.1.8-management | +| Nginx | 1.29-alpine | +| OpenTelemetry Collector Contrib | 0.136.0 | +| OpenTelemetry Python SDK | 1.38.0 | +| OpenTelemetry FastAPI instrumentation | 0.59b0 | +| Prometheus | 3.5.0 | +| Loki | 3.5.5 | +| Tempo | 2.8.2 | +| Grafana | 12.1.1 | + +**Подготовка окружения:** + +```bash +cp .env.example .env +``` + +В `.env` необходимо установить собственные локальные значения паролей PostgreSQL, RabbitMQ, администратора Grafana и ключей S3. Для остальных параметров использовать значения и комментарии из `.env.example`. + +Файл `.env` не должен попадать в Git. Секреты в отчёте не публикуются. + +**Сборка и запуск:** + +```bash +docker compose up -d --build +``` + +При запуске через Compose выполняются предусмотренные конфигурацией миграции PostgreSQL и инициализация приватного S3 bucket. + +Проверить состояние: + +```bash +docker compose ps -a +``` + +Проверить готовность бизнес-API: + +```bash +curl http://127.0.0.1:8080/health/ready +``` + +Ожидаемый результат: + +```json +{"status":"ok"} +``` + +Проверить публичный контракт: + +```bash +make test +``` + +Проверить корреляцию логов и трасс: + +```bash +docker compose exec -T api1 python - < scripts/test-trace-correlation.py +``` + +**Доступ к Grafana:** `http://localhost:3000`. Логин администратора — `admin`; пароль задаётся через `GRAFANA_ADMIN_PASSWORD`. + +Dashboard автоматически импортируется из файла `observability/grafana/dashboards/messenger-lab6.json`. + +**Безопасная остановка с сохранением данных:** + +```bash +docker compose down +``` + +**Намеренное удаление всех учебных данных и volumes:** + +```bash +docker compose down -v +``` + +Последняя команда используется только при осознанном сбросе стенда, поскольку удаляет сохранённые данные приложений и подсистемы наблюдаемости. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +**Схема процессов и хранилищ:** + +```text +Client + | +Nginx + | + +------ API1 / API2 ------ PostgreSQL + | | + | +---------- RustFS (S3) + | +Dispatcher ---- RabbitMQ ---- Worker + | | + +------------+-------------+ + | + OTLP / HTTP + | + OpenTelemetry Collector + | + +-------+-------+ + | | | + Prometheus Loki Tempo + | | | + +-------+-------+ + | + Grafana +``` + +**Что перенесено из lab5:** + +- Две реплики FastAPI за Nginx. +- Авторизация, пользователи, чаты, сообщения и вложения. +- PostgreSQL для транзакционных данных, сессий и заданий. +- RustFS как S3-совместимое хранилище изображений. +- RabbitMQ с устойчивой очередью `image_jobs`. +- Отдельные процессы dispatcher и worker. +- Асинхронная обработка PNG/JPEG и создание thumbnail. +- Механизмы восстановления заданий, обработки повторных доставок и миграции БД. + +**Что добавлено в lab6:** + +- OpenTelemetry SDK для API и worker. +- Автоматические HTTP server spans FastAPI. +- Consumer spans обработки изображений. +- Счётчики HTTP-запросов и обработки заданий. +- Гистограммы длительности HTTP-запросов и работы worker. +- Структурированные логи с `trace_id`, `span_id` и `service.name`. +- Отдельный OpenTelemetry Collector. +- Prometheus, Loki, Tempo и Grafana. +- Provisioning источников данных и импортируемого dashboard. +- Скрипты проверки корреляции и контролируемых инцидентов. + +**Назначение хранилищ и источник истины:** + +PostgreSQL остаётся источником истины для пользователей, сообщений, метаданных и состояний заданий. RustFS хранит бинарные объекты изображений. RabbitMQ доставляет задания worker. + +Prometheus хранит временные ряды метрик; Loki — структурированные логи; Tempo — трассы. Grafana обеспечивает отображение и поиск данных, но не заменяет эти хранилища. Collector принимает, обрабатывает и экспортирует телеметрию, не являясь долговременной базой данных. + +**Транзакционные границы:** + +Принятие изображения фиксируется сохранением исходных байтов в RustFS и записи задания в PostgreSQL. Передача задания в RabbitMQ допускает повторную доставку. Worker проверяет состояние задания и сохраняет результат до подтверждения обработки сообщения брокеру. + +Между PostgreSQL и S3 отсутствует единая распределённая транзакция. Используются повторные попытки и идемпотентная обработка. + +**Ресурсы и конфигурация OpenTelemetry:** + +Для API применяется `service.name=messenger-api`, для worker — `service.name=messenger-worker`. Используются также `service.version`, `service.instance.id` и `deployment.environment.name=lab`. + +API и worker отправляют OTLP/HTTP в Collector на `otel-collector:4318`. Collector экспортирует метрики в Prometheus-compatible endpoint, логи через OTLP/HTTP в Loki, трассы через OTLP/gRPC в Tempo. + +Пароли, Authorization, session token, тексты сообщений и байты изображений не включаются в специально создаваемые события телеметрии. Пользовательские UUID не используются как labels метрик. Для HTTP-маршрутов используются шаблоны, а не фактические URL с идентификаторами. + +**Фактические имена метрик:** + +| Логический сигнал | Имя в Prometheus | Единица | +| --- | --- | --- | +| HTTP requests | `messenger_messenger_http_requests_total` | запросы | +| HTTP duration | `messenger_http_server_duration_milliseconds_bucket` | миллисекунды | +| Jobs processed / failed | `messenger_messenger_worker_jobs_processed_total` | попытки | +| Job processing duration | `messenger_messenger_worker_job_duration_seconds_bucket` | секунды | + +Для API error ratio числитель — HTTP 5xx, знаменатель — все учитываемые бизнес-запросы, без health checks. Значения p95 рассчитываются из гистограмм с агрегацией buckets. Миллисекунды HTTP duration преобразуются в секунды. + +**Ограничения архитектуры:** + +Сквозной trace context между HTTP upload и обработкой задания worker не реализован. Логи API и worker коррелируют со своими реальными трассами, но не образуют единую parent-child трассу через RabbitMQ. + +Возможности дополнительных метрик очереди, насыщения ресурсов и alert rules относятся к более высокому уровню и в текущей реализации не заявлены. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke | `make test` | PASS, бизнес-API работает с включённой телеметрией | `evidence/smoke.txt` | +| Smoke после инцидентов | `make test` | PASS после восстановления PostgreSQL и worker | `evidence/smoke-after-incidents.txt` | +| Collector и хранилища трёх сигналов | `docker compose ps -a` | Collector healthy; Prometheus, Loki, Tempo и Grafana запущены | `evidence/containers.txt` | +| HTTP request counter | Запрос Prometheus API | Найдена `messenger_messenger_http_requests_total` | `evidence/prometheus-metrics.json` | +| HTTP duration histogram | Запрос Prometheus API | Найдены buckets, sum и count HTTP duration | `evidence/prometheus-metrics.json` | +| Worker jobs counter | Выполнение `make test` и запрос Prometheus API | Найден счётчик заданий с outcome | `evidence/prometheus-metrics.json` | +| Worker duration histogram | Выполнение `make test` и запрос Prometheus API | Найдена гистограмма длительности worker | `evidence/prometheus-metrics.json` | +| Структурированные логи API | Запрос Loki / Grafana Explore | Найдены HTTP-события с техническими атрибутами | `evidence/grafana-dashboard.png` | +| Структурированные логи worker | Журналы worker и запрос Loki | Найдено событие `Image job handled` | `evidence/worker-telemetry.txt` | +| Реальная трасса API | Loki → Tempo по trace ID | PASS, 4 span, включая server span | `evidence/trace-correlation.txt` | +| Реальная трасса worker | Loki → Tempo по trace ID | PASS, consumer span `image_job.process` | `evidence/trace-correlation.txt` | +| Импортируемый dashboard | Provisioning Grafana | Dashboard загружен, 8 панелей | `observability/grafana/dashboards/messenger-lab6.json` | +| Контролируемый отказ worker | Остановка worker, upload, восстановление | PASS; задание сохранено, затем обработано | `evidence/incident-worker.txt` | +| Контролируемый отказ PostgreSQL | `python3 scripts/test-db-incident.py` | 5/5 неудачных запросов, восстановление HTTP 201 | `evidence/incident-db.txt` | +| Критерии уровня 4 | Не заявлены | Дополнительные метрики и alerts не реализованы | Не применяется | +| Трек уровня 5 | Не заявлен | Не выполнялся | Не применяется | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +**Подтверждённая корреляция API:** + +- Service: `messenger-api` +- Event: `HTTP request completed` +- Trace ID: `c3ea28d96d7c536e222d8e4c7cba2727` +- Span ID: `3be2550ec51156e2` +- Результат: найден реальный HTTP server span `POST /api/v1/auth/sessions`. + +**Подтверждённая корреляция worker:** + +- Service: `messenger-worker` +- Event: `Image job handled` +- Trace ID: `af16dbcb7b7ca25b06d39902dc48afd8` +- Span ID: `644d3cda73387e3f` +- Результат: найден consumer span `image_job.process`. + +Обе проверки завершились `PASS: API and worker trace correlation`. + +Dashboard и конфигурации provisioning сохранены в репозитории, поэтому после чистого запуска они загружаются без ручной настройки через браузер. + +Проверки инфраструктуры подтверждаются состоянием контейнеров и результатами отдельных запросов к Prometheus, Loki и Tempo; один только публичный smoke не считается доказательством работающей наблюдаемости. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Гипотезы:** + +1. При остановке worker API сохранит возможность принимать изображения, а принятые задания будут обработаны после восстановления worker. +2. При остановке PostgreSQL запросы, требующие доступа к БД, начнут завершаться ошибками; после восстановления БД авторизация снова заработает. +3. Метрики, логи и трассы позволят различить отсутствие фоновой обработки и недоступность базы данных. + +**Условия проведения:** + +- Среда: локальный Docker Desktop с WSL Ubuntu. +- Контейнеры: две API-реплики, PostgreSQL, RustFS, RabbitMQ, dispatcher, worker, Collector, Prometheus, Loki, Tempo, Grafana. +- Нагрузка: синтетические операции публичного smoke и специальные последовательные запросы скриптов инцидентов. +- Concurrency: в сценарии отказа БД проверочные запросы выполнялись последовательно. +- Прогрев: перед отказом выполнялись успешные запросы к API. +- Инцидент worker: принудительная остановка worker и дополнительная пауза 60 секунд. +- Инцидент PostgreSQL: остановка контейнера БД, пять проверочных запросов и восстановление. +- Аппаратная конфигурация, выделенные CPU/RAM и лимиты Docker: отдельно не фиксировались. +- Продолжительная нагрузка с фиксированным arrival rate и несколькими уровнями concurrency: не выполнялась. +- Повторяемость: приведены результаты одного зафиксированного прогона каждого контролируемого инцидента. + +**Зафиксированные измерения:** + +Для инцидента PostgreSQL: + +| Запрос | Результат | Длительность | +| --- | --- | --- | +| До отказа | HTTP 201 | Не зафиксирована в выводе | +| Probe 1 | TimeoutError | 5,005 с | +| Probe 2 | HTTP 503 | 0,046 с | +| Probe 3 | HTTP 503 | 4,071 с | +| Probe 4 | HTTP 503 | 0,038 с | +| Probe 5 | HTTP 503 | 0,076 с | +| После восстановления | HTTP 201 | 0,058 с | + +Во время отказа БД все пять запросов завершились неуспешно: четыре HTTP 503 и один клиентский тайм-аут. + +В Grafana зафиксирован рост доли HTTP 5xx примерно до 67% в двухминутном окне, а также изменение p95 длительности запросов. + +Отдельные нагрузочные измерения устойчивого RPS, p50/p95/p99 по серии прогонов, CPU/RAM и распределения задержек не собирались. Точки на dashboard характеризуют конкретный наблюдаемый период, но не являются результатом полноценного нагрузочного теста. + +**Таблица инцидентов:** | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановка worker | API принимает upload; задания остаются в `queued` | HTTP 202, сообщение создано, после запуска worker статус `ready` | Пауза 60 секунд; потеря принятого изображения не обнаружена | +| Перезапуск RabbitMQ и API при остановленном worker | Принятое задание не теряется | Статус `queued` сохранился после перезапуска | После возобновления worker задание обработано | +| Повторное включение worker | Появляется обработка и thumbnail | Создан JPEG 256×160, оригинал проверен SHA-256 | Точное время перехода в `ready` отдельно не измерялось | +| Остановка PostgreSQL | Авторизация, зависящая от БД, перестаёт работать | 4 HTTP 503 и 1 TimeoutError | После восстановления БД — HTTP 201 за 0,058 с | +| Повторный smoke после отказов | Вся бизнес-функциональность восстановлена | `make test` — PASS | Потерь подтверждённых данных в проверенных сценариях не обнаружено | + +**Анализ инцидента worker:** + +На dashboard в период остановки сохранялись успешные HTTP-запросы, но новые успешные обработки worker не появлялись. После восстановления worker был зафиксирован новый успешный job и данные о длительности обработки. + +Причина: отсутствие активного фонового обработчика, а не отказ API. Это подтверждено контролируемой остановкой контейнера и результатом `PASS: durable worker recovery`. + +Доказательства: `evidence/incident-worker.txt`, `evidence/incident-worker-dashboard.png`, `evidence/incident-worker-recovered.png`. + +**Анализ инцидента PostgreSQL:** + +Во время остановки БД запросы авторизации завершались HTTP 503 или тайм-аутом. График 5xx в Grafana показал рост серверных ошибок. После восстановления PostgreSQL авторизация вернулась к HTTP 201. + +Причина: недоступность PostgreSQL. Она подтверждается характером запросов, контролируемой остановкой контейнера, журналами и восстановлением после запуска БД. + +Отдельные DB client spans отсутствуют, поэтому точная локализация по одной только трассе API ограничена. + +Доказательства: `evidence/incident-db.txt`, `evidence/incident-db-dashboard.png`, `evidence/incident-db-api-logs.txt`, `evidence/incident-db-postgres-logs.txt`. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +**Какие отказы выдерживаются:** + +- Остановка worker не приводит к потере принятого задания при сохранности PostgreSQL и RustFS. +- Перезапуск RabbitMQ и API не нарушает сохранность зафиксированного задания. +- После восстановления PostgreSQL приложение возвращается к нормальной работе. +- Публичный smoke проходит при включённой телеметрии. + +**Какие отказы и ограничения остаются:** + +- PostgreSQL, RustFS и RabbitMQ развёрнуты одиночными экземплярами и остаются точками отказа. +- При удалении volumes учебные данные и история наблюдаемости теряются. +- Трассы API и worker не связаны единым W3C-контекстом через брокер. +- Нет отдельной SQL-инструментации с DB client spans. +- Метрики глубины и возраста очереди, насыщения ресурсов и end-to-end времени до `ready` не реализованы. +- Alert rules и автоматическая проверка их переходов в firing/resolved не реализованы. +- Не проводилось отдельное испытание недоступности backend-телеметрии и измерение потерь. +- Метрики worker могут сбрасываться при перезапуске процесса; для анализа необходимы временные ряды и события восстановления. +- Не выполнялись длительные нагрузочные испытания и измерения производительности с фиксированными hardware limits. + +**Следующие шаги:** + +1. Добавить передачу W3C trace context через RabbitMQ. +2. Инструментировать запросы к PostgreSQL и S3 отдельными spans. +3. Добавить метрики очереди и насыщения ресурсов. +4. Добавить alert rules с порогами, окнами и runbook. +5. Провести повторяемые нагрузочные испытания с фиксацией CPU/RAM, RPS и p50/p95/p99. +6. Протестировать частичную недоступность Loki, Tempo и Collector и измерить потери телеметрии. + +**Вывод:** реализована работающая подсистема наблюдаемости на основе OpenTelemetry, Prometheus, Loki, Tempo и Grafana. Подтверждён экспорт трёх сигналов, обнаружение реальных трасс по связанным логам, работоспособность dashboard и восстановление системы после двух контролируемых отказов. Заявленный уровень — оценка 3. \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..e510e6a --- /dev/null +++ b/compose.yaml @@ -0,0 +1,260 @@ + +services: + + prometheus: + image: prom/prometheus:v3.5.0 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + volumes: + - ./observability/prometheus.yaml:/etc/prometheus/prometheus.yml:ro + - prometheus_data:/prometheus + ports: + - "127.0.0.1:9090:9090" + restart: unless-stopped + + loki: + image: grafana/loki:3.5.5 + command: + - -config.file=/etc/loki/local-config.yaml + volumes: + - ./observability/loki.yaml:/etc/loki/local-config.yaml:ro + - loki_data:/loki + user: "0" + restart: unless-stopped + + tempo: + image: grafana/tempo:2.8.2 + command: + - -config.file=/etc/tempo.yaml + volumes: + - ./observability/tempo.yaml:/etc/tempo.yaml:ro + - tempo_data:/var/tempo + user: "0" + restart: unless-stopped + + grafana: + image: grafana/grafana:12.1.1 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env} + GF_USERS_ALLOW_SIGN_UP: "false" + volumes: + - grafana_data:/var/lib/grafana + - ./observability/grafana/provisioning:/etc/grafana/provisioning:ro + - ./observability/grafana/dashboards:/var/lib/grafana/dashboards:ro + ports: + - "127.0.0.1:3000:3000" + depends_on: + - prometheus + - loki + - tempo + restart: unless-stopped + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.136.0 + command: + - --config=/etc/otelcol-contrib/config.yaml + volumes: + - ./observability/otel-collector.yaml:/etc/otelcol-contrib/config.yaml:ro + restart: unless-stopped + ports: + - "127.0.0.1:14317:4317" + - "127.0.0.1:14318:4318" + healthcheck: + test: + - CMD + - /otelcol-contrib + - components + interval: 30s + timeout: 10s + retries: 3 + + rabbitmq: + image: rabbitmq:4.1.8-management + hostname: rabbitmq + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:?Set RABBITMQ_USER in .env} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?Set RABBITMQ_PASSWORD in .env} + volumes: + - rabbitmq_data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 10s + timeout: 5s + retries: 15 + start_period: 15s + restart: unless-stopped + + db: + image: postgres:16-alpine + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 20 + restart: unless-stopped + + minio: + image: rustfs/rustfs:1.0.3 + command: ["/data"] + environment: + RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY in .env} + RUSTFS_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY in .env} + RUSTFS_ADDRESS: ":9000" + volumes: + - minio_data:/data + restart: unless-stopped + + s3-init: + build: . + command: ["python", "-m", "src.s3_setup"] + environment: &s3_env + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + depends_on: + minio: + condition: service_started + restart: "no" + + migrate: + build: . + command: ["python", "-m", "src.migrate"] + environment: + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger + depends_on: + db: + condition: service_healthy + restart: "no" + + api1: + build: . + environment: + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_SERVICE_NAME: messenger-api + OTEL_PYTHON_FASTAPI_EXCLUDED_URLS: "/health/.*" + RABBITMQ_HOST: rabbitmq + RABBITMQ_USER: ${RABBITMQ_USER} + RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD} + RABBITMQ_QUEUE: ${RABBITMQ_QUEUE:-image_jobs} + IMAGE_MAX_ATTEMPTS: ${IMAGE_MAX_ATTEMPTS:-5} + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + <<: *s3_env + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"] + interval: 5s + timeout: 3s + retries: 20 + stop_grace_period: 15s + restart: unless-stopped + + api2: + build: . + environment: + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_SERVICE_NAME: messenger-api + OTEL_PYTHON_FASTAPI_EXCLUDED_URLS: "/health/.*" + RABBITMQ_HOST: rabbitmq + RABBITMQ_USER: ${RABBITMQ_USER} + RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD} + RABBITMQ_QUEUE: ${RABBITMQ_QUEUE:-image_jobs} + IMAGE_MAX_ATTEMPTS: ${IMAGE_MAX_ATTEMPTS:-5} + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + <<: *s3_env + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"] + interval: 5s + timeout: 3s + retries: 20 + stop_grace_period: 15s + restart: unless-stopped + + + dispatcher: + build: . + command: ["python", "-m", "src.dispatcher"] + environment: + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_SERVICE_NAME: messenger-dispatcher + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger + RABBITMQ_HOST: rabbitmq + RABBITMQ_USER: ${RABBITMQ_USER} + RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD} + RABBITMQ_QUEUE: ${RABBITMQ_QUEUE:-image_jobs} + depends_on: + rabbitmq: + condition: service_healthy + migrate: + condition: service_completed_successfully + restart: unless-stopped + stop_grace_period: 15s + + + worker: + build: . + command: ["python", "-m", "src.worker"] + environment: + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_SERVICE_NAME: messenger-worker + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + RABBITMQ_HOST: rabbitmq + RABBITMQ_USER: ${RABBITMQ_USER} + RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD} + RABBITMQ_QUEUE: ${RABBITMQ_QUEUE:-image_jobs} + IMAGE_MAX_ATTEMPTS: ${IMAGE_MAX_ATTEMPTS:-5} + depends_on: + rabbitmq: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + restart: unless-stopped + stop_grace_period: 20s + + nginx: + image: nginx:1.29-alpine + ports: + - "127.0.0.1:8080:80" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + api1: + condition: service_healthy + api2: + condition: service_healthy + restart: unless-stopped + +volumes: + postgres_data: + minio_data: + rabbitmq_data: + prometheus_data: + loki_data: + tempo_data: + grafana_data: diff --git a/evidence/containers.txt b/evidence/containers.txt new file mode 100644 index 0000000..7fff491 --- /dev/null +++ b/evidence/containers.txt @@ -0,0 +1,16 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab6-api1-1 mtusi2026lab6-api1 "uvicorn src.main:ap…" api1 22 minutes ago Up 22 minutes (healthy) 8080/tcp +mtusi2026lab6-api2-1 mtusi2026lab6-api2 "uvicorn src.main:ap…" api2 22 minutes ago Up 22 minutes (healthy) 8080/tcp +mtusi2026lab6-db-1 postgres:16-alpine "docker-entrypoint.s…" db 27 minutes ago Up 27 minutes (healthy) 5432/tcp +mtusi2026lab6-dispatcher-1 mtusi2026lab6-dispatcher "python -m src.dispa…" dispatcher 27 minutes ago Up 27 minutes 8080/tcp +mtusi2026lab6-grafana-1 grafana/grafana:12.1.1 "/run.sh" grafana 9 minutes ago Up 15 seconds 127.0.0.1:3000->3000/tcp +mtusi2026lab6-loki-1 grafana/loki:3.5.5 "/usr/bin/loki -conf…" loki 44 minutes ago Up 44 minutes 3100/tcp +mtusi2026lab6-migrate-1 mtusi2026lab6-migrate "python -m src.migra…" migrate 16 minutes ago Exited (0) 16 minutes ago +mtusi2026lab6-minio-1 rustfs/rustfs:1.0.3 "/entrypoint.sh /data" minio 27 minutes ago Up 27 minutes 9000-9001/tcp +mtusi2026lab6-nginx-1 nginx:1.29-alpine "/docker-entrypoint.…" nginx 27 minutes ago Up 27 minutes 127.0.0.1:8080->80/tcp +mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.136.0 "/otelcol-contrib --…" otel-collector 41 minutes ago Up 41 minutes (healthy) 55679/tcp, 127.0.0.1:14317->4317/tcp, 127.0.0.1:14318->4318/tcp +mtusi2026lab6-prometheus-1 prom/prometheus:v3.5.0 "/bin/prometheus --c…" prometheus 44 minutes ago Up 44 minutes 127.0.0.1:9090->9090/tcp +mtusi2026lab6-rabbitmq-1 rabbitmq:4.1.8-management "docker-entrypoint.s…" rabbitmq 27 minutes ago Up 27 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15671-15672/tcp, 15691-15692/tcp, 25672/tcp +mtusi2026lab6-s3-init-1 mtusi2026lab6-s3-init "python -m src.s3_se…" s3-init 16 minutes ago Exited (0) 16 minutes ago +mtusi2026lab6-tempo-1 grafana/tempo:2.8.2 "/tempo -config.file…" tempo 44 minutes ago Up 44 minutes +mtusi2026lab6-worker-1 mtusi2026lab6-worker "python -m src.worker" worker 16 minutes ago Up 16 minutes 8080/tcp diff --git a/evidence/grafana-dashboard.png b/evidence/grafana-dashboard.png new file mode 100644 index 0000000..5768ec0 Binary files /dev/null and b/evidence/grafana-dashboard.png differ diff --git a/evidence/incident-db-api-logs.txt b/evidence/incident-db-api-logs.txt new file mode 100644 index 0000000..fd59cf2 --- /dev/null +++ b/evidence/incident-db-api-logs.txt @@ -0,0 +1,200 @@ +api2-1 | INFO: 172.20.0.14:37470 - "POST /api/v1/chats HTTP/1.1" 201 Created +api2-1 | 2026-11-21 16:13:27,824 INFO event=http_request_completed trace_id=1889717945c3eddd806a8e37da7368d7 span_id=778a902e8dfd6c54 HTTP request completed +api2-1 | INFO: 172.20.0.14:37470 - "POST /api/v1/chats/4b9eb2b4-44e4-4bb3-863b-4943b9c37c70/messages HTTP/1.1" 201 Created +api2-1 | INFO: 127.0.0.1:54924 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: Shutting down +api2-1 | INFO: Waiting for application shutdown. +api2-1 | INFO: Application shutdown complete. +api2-1 | INFO: Finished server process [1] +api2-1 | INFO: Started server process [1] +api2-1 | INFO: Waiting for application startup. +api2-1 | INFO: Application startup complete. +api2-1 | INFO: Uvicorn running on http://0.0.0.0:8080 (Press CTRL+C to quit) +api2-1 | INFO: 127.0.0.1:45734 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:44306 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:44322 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:37222 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:37234 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:59306 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:59346 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:46812 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:46838 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:57764 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:57780 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:60662 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | 2026-11-21 16:14:40,820 INFO event=http_request_completed trace_id=853e8a1ac0c8acd7182a241ca8c17990 span_id=05519f4c759960a2 HTTP request completed +api2-1 | INFO: 172.20.0.14:33638 - "GET /api/v1/attachments/b98c6927-b6d5-4b9b-b0c1-7caf0d5a4c07 HTTP/1.1" 200 OK +api2-1 | 2026-11-21 16:14:41,864 INFO event=http_request_completed trace_id=3bb041b87253ffd0bc2b24360061cb7e span_id=3815f86eb8e0f940 HTTP request completed +api2-1 | INFO: 172.20.0.14:33638 - "GET /api/v1/attachments/b98c6927-b6d5-4b9b-b0c1-7caf0d5a4c07/content HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:60712 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:41942 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:41974 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:50386 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:50402 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:39396 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:39416 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:33268 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:33276 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:57776 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:57788 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:47430 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:47450 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:42750 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:42762 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:49882 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:40420 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:40426 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:59716 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:59740 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:49390 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:49396 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:52874 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:52888 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:34432 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:34450 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:50824 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:50842 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:44010 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:44022 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:42162 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:42192 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:52500 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:52514 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:43736 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:43756 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:34588 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:34616 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:57026 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:57052 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:60044 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:60050 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:59624 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:59646 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:41186 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | 2026-11-21 16:18:29,812 INFO event=http_request_completed trace_id=1756efab020974bc3f9555ce40099e7d span_id=30bfd1f2baa385c0 HTTP request completed +api2-1 | INFO: 172.20.0.14:56222 - "POST /api/v1/users HTTP/1.1" 201 Created +api2-1 | 2026-11-21 16:18:35,246 INFO event=http_request_completed trace_id=8fdf394cb3bda0228a70e17cddcaa622 span_id=3da8e0fea00100e0 HTTP request completed +api2-1 | INFO: 172.20.0.14:56222 - "POST /api/v1/auth/sessions HTTP/1.1" 503 Service Unavailable +api2-1 | 2026-11-21 16:18:43,293 INFO event=http_request_completed trace_id=60c1a9ff611169aa6530d289c1cf7075 span_id=c5c5694d6f107123 HTTP request completed +api2-1 | INFO: 172.20.0.14:43672 - "POST /api/v1/auth/sessions HTTP/1.1" 503 Service Unavailable +api2-1 | 2026-11-21 16:18:47,408 INFO event=http_request_completed trace_id=e79c6299dbfb2c31e3d1bbd58c7a253e span_id=0f6ee54ea39b9385 HTTP request completed +api2-1 | INFO: 172.20.0.14:43672 - "POST /api/v1/auth/sessions HTTP/1.1" 503 Service Unavailable +api2-1 | INFO: 127.0.0.1:33068 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:33090 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:38568 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:38598 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:44470 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:44482 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:39106 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:39116 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:49932 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:49962 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:55630 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:55634 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:54914 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: Shutting down +api1-1 | INFO: Waiting for application shutdown. +api1-1 | INFO: Application shutdown complete. +api1-1 | INFO: Finished server process [1] +api1-1 | INFO: Started server process [1] +api1-1 | INFO: Waiting for application startup. +api1-1 | INFO: Application startup complete. +api1-1 | INFO: Uvicorn running on http://0.0.0.0:8080 (Press CTRL+C to quit) +api1-1 | INFO: 172.20.0.14:39646 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | 2026-11-21 16:13:39,393 INFO event=http_request_completed trace_id=22e54b0e6d69077f18bfdb7c4e02661d span_id=45ddb3c5048edc9f HTTP request completed +api1-1 | INFO: 172.20.0.14:39646 - "GET /api/v1/attachments/b98c6927-b6d5-4b9b-b0c1-7caf0d5a4c07 HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:45730 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:44304 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:44318 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:37220 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:37226 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:59320 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:59332 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:46808 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:46822 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:57752 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:57796 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:60666 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | 2026-11-21 16:14:41,842 INFO event=http_request_completed trace_id=c58c902e3393684ca4f780fd7f70e520 span_id=a07ee23af91c0bf7 HTTP request completed +api1-1 | INFO: 172.20.0.14:40864 - "GET /api/v1/attachments/b98c6927-b6d5-4b9b-b0c1-7caf0d5a4c07 HTTP/1.1" 200 OK +api1-1 | 2026-11-21 16:14:41,907 INFO event=http_request_completed trace_id=4bcc0c6f19ddf75a5fd7c5e364d9e32b span_id=99fff1d1ee4ace8a HTTP request completed +api1-1 | INFO: 172.20.0.14:40864 - "GET /api/v1/attachments/b98c6927-b6d5-4b9b-b0c1-7caf0d5a4c07/content?variant=thumbnail HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:60724 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:41948 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:41962 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:50380 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:50396 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:38278 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:38280 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:38540 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:38560 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:32862 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:32890 - "GET /health/ready HTTP/1.1" 200 OK +api2-1 | INFO: 127.0.0.1:49622 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:39382 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:39402 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:33254 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:33272 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:57766 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:57780 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:47440 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:47448 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:42758 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:42770 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:49880 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:40414 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:40426 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:59714 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:59724 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:49378 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:49410 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:52872 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:52904 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:34444 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:34450 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:50832 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:50840 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:44004 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:44036 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:42176 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:42178 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:52498 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:52518 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:43746 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:43758 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:34594 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:34600 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:57012 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:57038 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:60030 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:60054 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:59630 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:59654 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:41172 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | 2026-11-21 16:18:29,864 INFO event=http_request_completed trace_id=951f6fcc6a32d838b2dc5fa1e2d7343a span_id=776bbf9a8e2b986a HTTP request completed +api1-1 | INFO: 172.20.0.14:48674 - "POST /api/v1/auth/sessions HTTP/1.1" 201 Created +api1-1 | 2026-11-21 16:18:37,223 INFO event=http_request_completed trace_id=1f8dc7db901d925f6b15fb2348b1a048 span_id=d2e85dfd82bc72b7 HTTP request completed +api1-1 | INFO: 172.20.0.14:33554 - "POST /api/v1/auth/sessions HTTP/1.1" 503 Service Unavailable +api1-1 | 2026-11-21 16:18:45,331 INFO event=http_request_completed trace_id=2afc8659a56936ef96018bc515cd6e61 span_id=88f8963e2289dcfc HTTP request completed +api1-1 | INFO: 172.20.0.14:50422 - "POST /api/v1/auth/sessions HTTP/1.1" 503 Service Unavailable +api1-1 | 2026-11-21 16:18:55,183 INFO event=http_request_completed trace_id=3d6852b9fb0e7047f085b8c239515e9c span_id=2802fa3fff95e680 HTTP request completed +api1-1 | INFO: 172.20.0.14:39552 - "POST /api/v1/auth/sessions HTTP/1.1" 201 Created +api1-1 | INFO: 127.0.0.1:33084 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:33098 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:38580 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:38592 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:44456 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:44476 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:39102 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:39112 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:49946 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:49978 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:55624 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:55638 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:38268 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:38290 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:38552 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:38574 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:32866 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:32876 - "GET /health/ready HTTP/1.1" 200 OK +api1-1 | INFO: 127.0.0.1:49606 - "GET /health/ready HTTP/1.1" 200 OK diff --git a/evidence/incident-db-containers.txt b/evidence/incident-db-containers.txt new file mode 100644 index 0000000..f3b0a07 --- /dev/null +++ b/evidence/incident-db-containers.txt @@ -0,0 +1,16 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab6-api1-1 mtusi2026lab6-api1 "uvicorn src.main:ap…" api1 2 days ago Up 6 minutes (healthy) 8080/tcp +mtusi2026lab6-api2-1 mtusi2026lab6-api2 "uvicorn src.main:ap…" api2 2 days ago Up 6 minutes (healthy) 8080/tcp +mtusi2026lab6-db-1 postgres:16-alpine "docker-entrypoint.s…" db 2 days ago Up About a minute (healthy) 5432/tcp +mtusi2026lab6-dispatcher-1 mtusi2026lab6-dispatcher "python -m src.dispa…" dispatcher 2 days ago Up 2 days 8080/tcp +mtusi2026lab6-grafana-1 grafana/grafana:12.1.1 "/run.sh" grafana 2 days ago Up 2 days 127.0.0.1:3000->3000/tcp +mtusi2026lab6-loki-1 grafana/loki:3.5.5 "/usr/bin/loki -conf…" loki 2 days ago Up 2 days 3100/tcp +mtusi2026lab6-migrate-1 mtusi2026lab6-migrate "python -m src.migra…" migrate 2 days ago Exited (0) 5 minutes ago +mtusi2026lab6-minio-1 rustfs/rustfs:1.0.3 "/entrypoint.sh /data" minio 2 days ago Up 2 days 9000-9001/tcp +mtusi2026lab6-nginx-1 nginx:1.29-alpine "/docker-entrypoint.…" nginx 2 days ago Up 2 days 127.0.0.1:8080->80/tcp +mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.136.0 "/otelcol-contrib --…" otel-collector 2 days ago Up 2 days (healthy) 55679/tcp, 127.0.0.1:14317->4317/tcp, 127.0.0.1:14318->4318/tcp +mtusi2026lab6-prometheus-1 prom/prometheus:v3.5.0 "/bin/prometheus --c…" prometheus 2 days ago Up 2 days 127.0.0.1:9090->9090/tcp +mtusi2026lab6-rabbitmq-1 rabbitmq:4.1.8-management "docker-entrypoint.s…" rabbitmq 2 days ago Up 7 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15671-15672/tcp, 15691-15692/tcp, 25672/tcp +mtusi2026lab6-s3-init-1 mtusi2026lab6-s3-init "python -m src.s3_se…" s3-init 2 days ago Exited (0) 5 minutes ago +mtusi2026lab6-tempo-1 grafana/tempo:2.8.2 "/tempo -config.file…" tempo 2 days ago Up 2 days +mtusi2026lab6-worker-1 mtusi2026lab6-worker "python -m src.worker" worker 2 days ago Up 5 minutes 8080/tcp diff --git a/evidence/incident-db-dashboard.png b/evidence/incident-db-dashboard.png new file mode 100644 index 0000000..e3e097d Binary files /dev/null and b/evidence/incident-db-dashboard.png differ diff --git a/evidence/incident-db-postgres-logs.txt b/evidence/incident-db-postgres-logs.txt new file mode 100644 index 0000000..5f23bb2 --- /dev/null +++ b/evidence/incident-db-postgres-logs.txt @@ -0,0 +1,80 @@ +db-1 | 2026-11-19 10:59:29.991 UTC [1] LOG: listening on IPv6 address "::", port 5432 +db-1 | 2026-11-19 10:59:29.995 UTC [1] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432" +db-1 | 2026-11-19 10:59:30.000 UTC [57] LOG: database system was shut down at 2026-11-19 10:59:29 UTC +db-1 | 2026-11-19 10:59:30.004 UTC [1] LOG: database system is ready to accept connections +db-1 | 2026-11-19 11:01:46.203 UTC [451] ERROR: duplicate key value violates unique constraint "users_username_key" +db-1 | 2026-11-19 11:01:46.203 UTC [451] DETAIL: Key (username)=(u_alice_f0123aa15331) already exists. +db-1 | 2026-11-19 11:01:46.203 UTC [451] STATEMENT: +db-1 | INSERT INTO users +db-1 | (id, username, display_name, password_hash) +db-1 | VALUES ($1, $2, $3, $4) +db-1 | RETURNING id, username, display_name, created_at +db-1 | +db-1 | 2026-11-19 11:04:31.001 UTC [55] LOG: checkpoint starting: time +db-1 | 2026-11-19 11:04:48.288 UTC [55] LOG: checkpoint complete: wrote 175 buffers (1.1%); 0 WAL file(s) added, 0 removed, 0 recycled; write=17.233 s, sync=0.045 s, total=17.288 s; sync files=104, longest=0.002 s, average=0.001 s; distance=803 kB, estimate=803 kB; lsn=0/19E7700, redo lsn=0/19E76C8 +db-1 | 2026-11-19 11:04:50.938 UTC [988] ERROR: duplicate key value violates unique constraint "users_username_key" +db-1 | 2026-11-19 11:04:50.938 UTC [988] DETAIL: Key (username)=(u_alice_dff3cf8f9360) already exists. +db-1 | 2026-11-19 11:04:50.938 UTC [988] STATEMENT: +db-1 | INSERT INTO users +db-1 | (id, username, display_name, password_hash) +db-1 | VALUES ($1, $2, $3, $4) +db-1 | RETURNING id, username, display_name, created_at +db-1 | +db-1 | 2026-11-19 11:09:31.281 UTC [55] LOG: checkpoint starting: time +db-1 | 2026-11-19 11:09:37.044 UTC [55] LOG: checkpoint complete: wrote 58 buffers (0.4%); 0 WAL file(s) added, 0 removed, 0 recycled; write=5.713 s, sync=0.021 s, total=5.763 s; sync files=43, longest=0.002 s, average=0.001 s; distance=14 kB, estimate=724 kB; lsn=0/19EB220, redo lsn=0/19EB1E8 +db-1 | 2026-11-19 11:11:03.824 UTC [2110] ERROR: duplicate key value violates unique constraint "users_username_key" +db-1 | 2026-11-19 11:11:03.824 UTC [2110] DETAIL: Key (username)=(u_alice_a072f61fcaf9) already exists. +db-1 | 2026-11-19 11:11:03.824 UTC [2110] STATEMENT: +db-1 | INSERT INTO users +db-1 | (id, username, display_name, password_hash) +db-1 | VALUES ($1, $2, $3, $4) +db-1 | RETURNING id, username, display_name, created_at +db-1 | +db-1 | 2026-11-19 11:11:24.481 UTC [2231] ERROR: duplicate key value violates unique constraint "users_username_key" +db-1 | 2026-11-19 11:11:24.481 UTC [2231] DETAIL: Key (username)=(u_alice_ed51a8803fc1) already exists. +db-1 | 2026-11-19 11:11:24.481 UTC [2231] STATEMENT: +db-1 | INSERT INTO users +db-1 | (id, username, display_name, password_hash) +db-1 | VALUES ($1, $2, $3, $4) +db-1 | RETURNING id, username, display_name, created_at +db-1 | +db-1 | 2026-11-19 11:14:31.133 UTC [55] LOG: checkpoint starting: time +db-1 | 2026-11-19 11:14:34.469 UTC [55] LOG: checkpoint complete: wrote 34 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.309 s, sync=0.019 s, total=3.336 s; sync files=34, longest=0.003 s, average=0.001 s; distance=30 kB, estimate=655 kB; lsn=0/19F2DB0, redo lsn=0/19F2D78 +db-1 | 2026-11-19 11:27:17.811 UTC [4991] ERROR: duplicate key value violates unique constraint "users_username_key" +db-1 | 2026-11-19 11:27:17.811 UTC [4991] DETAIL: Key (username)=(u_alice_5c60fc1beaf6) already exists. +db-1 | 2026-11-19 11:27:17.811 UTC [4991] STATEMENT: +db-1 | INSERT INTO users +db-1 | (id, username, display_name, password_hash) +db-1 | VALUES ($1, $2, $3, $4) +db-1 | RETURNING id, username, display_name, created_at +db-1 | +db-1 | 2026-11-19 11:29:31.636 UTC [55] LOG: checkpoint starting: time +db-1 | 2026-11-19 11:29:34.995 UTC [55] LOG: checkpoint complete: wrote 34 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.309 s, sync=0.022 s, total=3.359 s; sync files=34, longest=0.003 s, average=0.001 s; distance=32 kB, estimate=592 kB; lsn=0/19FB020, redo lsn=0/19FAFE8 +db-1 | 2026-11-19 11:31:47.459 UTC [5815] ERROR: duplicate key value violates unique constraint "users_username_key" +db-1 | 2026-11-19 11:31:47.459 UTC [5815] DETAIL: Key (username)=(u_alice_afb8c8b7058a) already exists. +db-1 | 2026-11-19 11:31:47.459 UTC [5815] STATEMENT: +db-1 | INSERT INTO users +db-1 | (id, username, display_name, password_hash) +db-1 | VALUES ($1, $2, $3, $4) +db-1 | RETURNING id, username, display_name, created_at +db-1 | +db-1 | 2026-11-19 11:34:32.002 UTC [55] LOG: checkpoint starting: time +db-1 | 2026-11-19 11:34:35.335 UTC [55] LOG: checkpoint complete: wrote 34 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.309 s, sync=0.016 s, total=3.334 s; sync files=34, longest=0.002 s, average=0.001 s; distance=38 kB, estimate=537 kB; lsn=0/1A04AD0, redo lsn=0/1A04A98 +db-1 | 2026-11-21 16:15:11.638 UTC [55] LOG: checkpoint starting: time +db-1 | 2026-11-21 16:15:14.988 UTC [55] LOG: checkpoint complete: wrote 34 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.319 s, sync=0.016 s, total=3.351 s; sync files=34, longest=0.002 s, average=0.001 s; distance=40 kB, estimate=487 kB; lsn=0/1A0EB68, redo lsn=0/1A0EB30 +db-1 | 2026-11-21 16:18:29.948 UTC [1] LOG: received fast shutdown request +db-1 | 2026-11-21 16:18:29.950 UTC [1] LOG: aborting any active transactions +db-1 | 2026-11-21 16:18:29.956 UTC [1] LOG: background worker "logical replication launcher" (PID 60) exited with exit code 1 +db-1 | 2026-11-21 16:18:29.957 UTC [55] LOG: shutting down +db-1 | 2026-11-21 16:18:29.958 UTC [55] LOG: checkpoint starting: shutdown immediate +db-1 | 2026-11-21 16:18:29.972 UTC [55] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.002 s, sync=0.005 s, total=0.016 s; sync files=8, longest=0.002 s, average=0.001 s; distance=13 kB, estimate=440 kB; lsn=0/1A12070, redo lsn=0/1A12070 +db-1 | 2026-11-21 16:18:29.979 UTC [1] LOG: database system is shut down +db-1 | +db-1 | PostgreSQL Database directory appears to contain a database; Skipping initialization +db-1 | +db-1 | 2026-11-21 16:18:49.717 UTC [1] LOG: starting PostgreSQL 16.14 on x86_64-pc-linux-musl, compiled by gcc (Alpine 15.2.0) 15.2.0, 64-bit +db-1 | 2026-11-21 16:18:49.718 UTC [1] LOG: listening on IPv4 address "0.0.0.0", port 5432 +db-1 | 2026-11-21 16:18:49.718 UTC [1] LOG: listening on IPv6 address "::", port 5432 +db-1 | 2026-11-21 16:18:49.722 UTC [1] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432" +db-1 | 2026-11-21 16:18:49.728 UTC [29] LOG: database system was shut down at 2026-11-21 16:18:29 UTC +db-1 | 2026-11-21 16:18:49.735 UTC [1] LOG: database system is ready to accept connections diff --git a/evidence/incident-db.txt b/evidence/incident-db.txt new file mode 100644 index 0000000..0150383 --- /dev/null +++ b/evidence/incident-db.txt @@ -0,0 +1,18 @@ +Baseline: authentication HTTP 201 +INCIDENT START: stopping PostgreSQL + Container mtusi2026lab6-db-1 Stopping + Container mtusi2026lab6-db-1 Stopped +Probe 1: status=TimeoutError, duration=5.005s +Probe 2: status=503, duration=0.046s +Probe 3: status=503, duration=4.071s +Probe 4: status=503, duration=0.038s +Probe 5: status=503, duration=0.076s +INCIDENT END: restoring PostgreSQL + Container mtusi2026lab6-db-1 Created + Container mtusi2026lab6-db-1 Starting + Container mtusi2026lab6-db-1 Started + Container mtusi2026lab6-db-1 Waiting + Container mtusi2026lab6-db-1 Healthy +Recovery: HTTP 201, 0.058s +Failed requests during incident: 5/5 +PASS: database failure observed and recovered diff --git a/evidence/incident-dispatcher-logs.txt b/evidence/incident-dispatcher-logs.txt new file mode 100644 index 0000000..425b58d --- /dev/null +++ b/evidence/incident-dispatcher-logs.txt @@ -0,0 +1,50 @@ +dispatcher-1 | 2026-11-19 11:27:18,594 INFO Closing connection (200): Normal shutdown +dispatcher-1 | 2026-11-19 11:27:18,594 INFO Closing channel (200): 'Normal shutdown' on params=>> +dispatcher-1 | 2026-11-19 11:27:18,594 INFO Received on params=>> +dispatcher-1 | 2026-11-19 11:27:18,594 INFO Closing connection (200): 'Normal shutdown' +dispatcher-1 | 2026-11-19 11:27:18,594 INFO Aborting transport connection: state=1; +dispatcher-1 | 2026-11-19 11:27:18,594 INFO _AsyncTransportBase._initate_abort(): Initiating abrupt asynchronous transport shutdown: state=1; error=None; +dispatcher-1 | 2026-11-19 11:27:18,594 INFO Deactivating transport: state=1; +dispatcher-1 | 2026-11-19 11:27:18,594 INFO AMQP stack terminated, failed to connect, or aborted: opened=True, error-arg=None; pending-error=ConnectionClosedByClient: (200) 'Normal shutdown' +dispatcher-1 | 2026-11-19 11:27:18,594 INFO Stack terminated due to ConnectionClosedByClient: (200) 'Normal shutdown' +dispatcher-1 | 2026-11-19 11:27:18,595 INFO Closing transport socket and unlinking: state=3; +dispatcher-1 | 2026-11-19 11:27:18,595 INFO User-initiated close: result=BlockingConnection__OnClosedArgs(connection=>, error=ConnectionClosedByClient: (200) 'Normal shutdown') +dispatcher-1 | 2026-11-19 11:27:18,595 INFO Published job 6d77b086-944f-447d-8156-81995b5c562b +dispatcher-1 | 2026-11-19 11:31:47,858 INFO Pika version 1.3.2 connecting to ('172.20.0.7', 5672) +dispatcher-1 | 2026-11-19 11:31:47,859 INFO Socket connected: +dispatcher-1 | 2026-11-19 11:31:47,859 INFO Streaming transport linked up: (, _StreamingProtocolShim: params=>). +dispatcher-1 | 2026-11-19 11:31:47,861 INFO AMQPConnector - reporting success: params=> +dispatcher-1 | 2026-11-19 11:31:47,861 INFO AMQPConnectionWorkflow - reporting success: params=> +dispatcher-1 | 2026-11-19 11:31:47,861 INFO Connection workflow succeeded: params=> +dispatcher-1 | 2026-11-19 11:31:47,861 INFO Created channel=1 +dispatcher-1 | 2026-11-19 11:31:47,867 INFO Closing connection (200): Normal shutdown +dispatcher-1 | 2026-11-19 11:31:47,867 INFO Closing channel (200): 'Normal shutdown' on params=>> +dispatcher-1 | 2026-11-19 11:31:47,868 INFO Received on params=>> +dispatcher-1 | 2026-11-19 11:31:47,868 INFO Closing connection (200): 'Normal shutdown' +dispatcher-1 | 2026-11-19 11:31:47,868 INFO Aborting transport connection: state=1; +dispatcher-1 | 2026-11-19 11:31:47,868 INFO _AsyncTransportBase._initate_abort(): Initiating abrupt asynchronous transport shutdown: state=1; error=None; +dispatcher-1 | 2026-11-19 11:31:47,868 INFO Deactivating transport: state=1; +dispatcher-1 | 2026-11-19 11:31:47,868 INFO AMQP stack terminated, failed to connect, or aborted: opened=True, error-arg=None; pending-error=ConnectionClosedByClient: (200) 'Normal shutdown' +dispatcher-1 | 2026-11-19 11:31:47,868 INFO Stack terminated due to ConnectionClosedByClient: (200) 'Normal shutdown' +dispatcher-1 | 2026-11-19 11:31:47,868 INFO Closing transport socket and unlinking: state=3; +dispatcher-1 | 2026-11-19 11:31:47,868 INFO User-initiated close: result=BlockingConnection__OnClosedArgs(connection=>, error=ConnectionClosedByClient: (200) 'Normal shutdown') +dispatcher-1 | 2026-11-19 11:31:47,869 INFO Published job c9c31ec9-a3cf-4f9c-8f5c-387e1f3c81e7 +dispatcher-1 | 2026-11-21 16:13:27,871 INFO Pika version 1.3.2 connecting to ('172.20.0.7', 5672) +dispatcher-1 | 2026-11-21 16:13:27,875 INFO Socket connected: +dispatcher-1 | 2026-11-21 16:13:27,878 INFO Streaming transport linked up: (, _StreamingProtocolShim: params=>). +dispatcher-1 | 2026-11-21 16:13:27,893 INFO AMQPConnector - reporting success: params=> +dispatcher-1 | 2026-11-21 16:13:27,894 INFO AMQPConnectionWorkflow - reporting success: params=> +dispatcher-1 | 2026-11-21 16:13:27,895 INFO Connection workflow succeeded: params=> +dispatcher-1 | 2026-11-21 16:13:27,899 INFO Created channel=1 +dispatcher-1 | 2026-11-21 16:13:27,918 INFO Closing connection (200): Normal shutdown +dispatcher-1 | 2026-11-21 16:13:27,918 INFO Closing channel (200): 'Normal shutdown' on params=>> +dispatcher-1 | 2026-11-21 16:13:27,920 INFO Received on params=>> +dispatcher-1 | 2026-11-21 16:13:27,920 INFO Closing connection (200): 'Normal shutdown' +dispatcher-1 | 2026-11-21 16:13:27,921 INFO Aborting transport connection: state=1; +dispatcher-1 | 2026-11-21 16:13:27,921 INFO _AsyncTransportBase._initate_abort(): Initiating abrupt asynchronous transport shutdown: state=1; error=None; +dispatcher-1 | 2026-11-21 16:13:27,921 INFO Deactivating transport: state=1; +dispatcher-1 | 2026-11-21 16:13:27,921 INFO AMQP stack terminated, failed to connect, or aborted: opened=True, error-arg=None; pending-error=ConnectionClosedByClient: (200) 'Normal shutdown' +dispatcher-1 | 2026-11-21 16:13:27,921 INFO Stack terminated due to ConnectionClosedByClient: (200) 'Normal shutdown' +dispatcher-1 | 2026-11-21 16:13:27,921 INFO Closing transport socket and unlinking: state=3; +dispatcher-1 | 2026-11-21 16:13:27,922 INFO User-initiated close: result=BlockingConnection__OnClosedArgs(connection=>, error=ConnectionClosedByClient: (200) 'Normal shutdown') +dispatcher-1 | 2026-11-21 16:13:27,923 INFO Published job e687ea16-ef44-416e-a96f-15dfac07230e diff --git a/evidence/incident-worker-containers.txt b/evidence/incident-worker-containers.txt new file mode 100644 index 0000000..34c4e31 --- /dev/null +++ b/evidence/incident-worker-containers.txt @@ -0,0 +1,16 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab6-api1-1 mtusi2026lab6-api1 "uvicorn src.main:ap…" api1 2 days ago Up 2 minutes (healthy) 8080/tcp +mtusi2026lab6-api2-1 mtusi2026lab6-api2 "uvicorn src.main:ap…" api2 2 days ago Up 2 minutes (healthy) 8080/tcp +mtusi2026lab6-db-1 postgres:16-alpine "docker-entrypoint.s…" db 2 days ago Up 2 days (healthy) 5432/tcp +mtusi2026lab6-dispatcher-1 mtusi2026lab6-dispatcher "python -m src.dispa…" dispatcher 2 days ago Up 2 days 8080/tcp +mtusi2026lab6-grafana-1 grafana/grafana:12.1.1 "/run.sh" grafana 2 days ago Up 2 days 127.0.0.1:3000->3000/tcp +mtusi2026lab6-loki-1 grafana/loki:3.5.5 "/usr/bin/loki -conf…" loki 2 days ago Up 2 days 3100/tcp +mtusi2026lab6-migrate-1 mtusi2026lab6-migrate "python -m src.migra…" migrate 2 days ago Exited (0) About a minute ago +mtusi2026lab6-minio-1 rustfs/rustfs:1.0.3 "/entrypoint.sh /data" minio 2 days ago Up 2 days 9000-9001/tcp +mtusi2026lab6-nginx-1 nginx:1.29-alpine "/docker-entrypoint.…" nginx 2 days ago Up 2 days 127.0.0.1:8080->80/tcp +mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.136.0 "/otelcol-contrib --…" otel-collector 2 days ago Up 2 days (healthy) 55679/tcp, 127.0.0.1:14317->4317/tcp, 127.0.0.1:14318->4318/tcp +mtusi2026lab6-prometheus-1 prom/prometheus:v3.5.0 "/bin/prometheus --c…" prometheus 2 days ago Up 2 days 127.0.0.1:9090->9090/tcp +mtusi2026lab6-rabbitmq-1 rabbitmq:4.1.8-management "docker-entrypoint.s…" rabbitmq 2 days ago Up 2 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15671-15672/tcp, 15691-15692/tcp, 25672/tcp +mtusi2026lab6-s3-init-1 mtusi2026lab6-s3-init "python -m src.s3_se…" s3-init 2 days ago Exited (0) About a minute ago +mtusi2026lab6-tempo-1 grafana/tempo:2.8.2 "/tempo -config.file…" tempo 2 days ago Up 2 days +mtusi2026lab6-worker-1 mtusi2026lab6-worker "python -m src.worker" worker 2 days ago Up About a minute 8080/tcp diff --git a/evidence/incident-worker-dashboard.png b/evidence/incident-worker-dashboard.png new file mode 100644 index 0000000..e162d18 Binary files /dev/null and b/evidence/incident-worker-dashboard.png differ diff --git a/evidence/incident-worker-logs.txt b/evidence/incident-worker-logs.txt new file mode 100644 index 0000000..dbcab16 --- /dev/null +++ b/evidence/incident-worker-logs.txt @@ -0,0 +1,26 @@ +worker-1 | 2026-11-19 11:10:44,315 INFO Pika version 1.3.2 connecting to ('172.20.0.7', 5672) +worker-1 | 2026-11-19 11:10:44,315 INFO Socket connected: +worker-1 | 2026-11-19 11:10:44,316 INFO Streaming transport linked up: (, _StreamingProtocolShim: params=>). +worker-1 | 2026-11-19 11:10:44,318 INFO AMQPConnector - reporting success: params=> +worker-1 | 2026-11-19 11:10:44,318 INFO AMQPConnectionWorkflow - reporting success: params=> +worker-1 | 2026-11-19 11:10:44,318 INFO Connection workflow succeeded: params=> +worker-1 | 2026-11-19 11:10:44,318 INFO Created channel=1 +worker-1 | 2026-11-19 11:10:44,342 INFO Worker listening on image_jobs +worker-1 | 2026-11-19 11:11:04,434 INFO Job a95f84e1-cd98-4fcd-ae9c-a5782b252fe3 ready, thumbnail 256x160 +worker-1 | 2026-11-19 11:11:04,441 INFO event=image_job_handled trace_id=af16dbcb7b7ca25b06d39902dc48afd8 span_id=644d3cda73387e3f Image job handled +worker-1 | 2026-11-19 11:11:25,616 INFO Job 1d011797-2efd-4bf7-88b9-c2c0e455119c ready, thumbnail 256x160 +worker-1 | 2026-11-19 11:11:25,623 INFO event=image_job_handled trace_id=53f30d09a4f2c578521c8bf075e35b88 span_id=93d63eec0255aa53 Image job handled +worker-1 | 2026-11-19 11:27:18,636 INFO Job 6d77b086-944f-447d-8156-81995b5c562b ready, thumbnail 256x160 +worker-1 | 2026-11-19 11:27:18,643 INFO event=image_job_handled trace_id=6d39def6a9347b9766c7d9820480cbea span_id=91110edf3e0e5f53 Image job handled +worker-1 | 2026-11-19 11:31:47,910 INFO Job c9c31ec9-a3cf-4f9c-8f5c-387e1f3c81e7 ready, thumbnail 256x160 +worker-1 | 2026-11-19 11:31:47,918 INFO event=image_job_handled trace_id=5a9611a8564da15253517786b3b87c35 span_id=692dee1a1ba56089 Image job handled +worker-1 | 2026-11-21 16:14:41,241 INFO Pika version 1.3.2 connecting to ('172.20.0.7', 5672) +worker-1 | 2026-11-21 16:14:41,242 INFO Socket connected: +worker-1 | 2026-11-21 16:14:41,242 INFO Streaming transport linked up: (, _StreamingProtocolShim: params=>). +worker-1 | 2026-11-21 16:14:41,249 INFO AMQPConnector - reporting success: params=> +worker-1 | 2026-11-21 16:14:41,249 INFO AMQPConnectionWorkflow - reporting success: params=> +worker-1 | 2026-11-21 16:14:41,249 INFO Connection workflow succeeded: params=> +worker-1 | 2026-11-21 16:14:41,250 INFO Created channel=1 +worker-1 | 2026-11-21 16:14:41,253 INFO Worker listening on image_jobs +worker-1 | 2026-11-21 16:14:41,323 INFO Job e687ea16-ef44-416e-a96f-15dfac07230e ready, thumbnail 256x160 +worker-1 | 2026-11-21 16:14:41,331 INFO event=image_job_handled trace_id=b768e258011bde0b8084a6665f1067d1 span_id=bee607ec28cf87ab Image job handled diff --git a/evidence/incident-worker-recovered.png b/evidence/incident-worker-recovered.png new file mode 100644 index 0000000..6a49d78 Binary files /dev/null and b/evidence/incident-worker-recovered.png differ diff --git a/evidence/incident-worker.txt b/evidence/incident-worker.txt new file mode 100644 index 0000000..9d9703b --- /dev/null +++ b/evidence/incident-worker.txt @@ -0,0 +1,60 @@ +Stopping worker... + Container mtusi2026lab6-worker-1 Stopping + Container mtusi2026lab6-worker-1 Stopped +Upload: 202 queued +Message created while worker is stopped +Restarting RabbitMQ and API... + Container mtusi2026lab6-rabbitmq-1 Restarting + Container mtusi2026lab6-rabbitmq-1 Started + Container mtusi2026lab6-rabbitmq-1 Running + Container mtusi2026lab6-rabbitmq-1 Waiting + Container mtusi2026lab6-rabbitmq-1 Healthy + Container mtusi2026lab6-api2-1 Restarting + Container mtusi2026lab6-api1-1 Restarting + Container mtusi2026lab6-api1-1 Started + Container mtusi2026lab6-api2-1 Started +Queued attachment survived restart +INCIDENT: worker stopped for 60 seconds +Starting worker... + Container mtusi2026lab6-db-1 Running + Container mtusi2026lab6-rabbitmq-1 Running + Container mtusi2026lab6-minio-1 Running + Container mtusi2026lab6-migrate-1 Created + Container mtusi2026lab6-s3-init-1 Created + Container mtusi2026lab6-worker-1 Created + Container mtusi2026lab6-s3-init-1 Starting + Container mtusi2026lab6-db-1 Waiting + Container mtusi2026lab6-s3-init-1 Started + Container mtusi2026lab6-db-1 Healthy + Container mtusi2026lab6-migrate-1 Starting + Container mtusi2026lab6-migrate-1 Started + Container mtusi2026lab6-rabbitmq-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-rabbitmq-1 Healthy + Container mtusi2026lab6-worker-1 Starting + Container mtusi2026lab6-worker-1 Started +Attachment: ready +Original: SHA-256 verified +Thumbnail: JPEG 256x160 verified +PASS: durable worker recovery + Container mtusi2026lab6-db-1 Running + Container mtusi2026lab6-migrate-1 Created + Container mtusi2026lab6-minio-1 Running + Container mtusi2026lab6-rabbitmq-1 Running + Container mtusi2026lab6-s3-init-1 Created + Container mtusi2026lab6-worker-1 Running + Container mtusi2026lab6-db-1 Waiting + Container mtusi2026lab6-s3-init-1 Starting + Container mtusi2026lab6-s3-init-1 Started + Container mtusi2026lab6-db-1 Healthy + Container mtusi2026lab6-migrate-1 Starting + Container mtusi2026lab6-migrate-1 Started + Container mtusi2026lab6-rabbitmq-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-rabbitmq-1 Healthy + Container mtusi2026lab6-migrate-1 Exited diff --git a/evidence/prometheus-metrics.json b/evidence/prometheus-metrics.json new file mode 100644 index 0000000..35c7355 --- /dev/null +++ b/evidence/prometheus-metrics.json @@ -0,0 +1,26 @@ +{ + "status": "success", + "data": [ + "messenger_http_server_active_requests", + "messenger_http_server_duration_milliseconds_bucket", + "messenger_http_server_duration_milliseconds_count", + "messenger_http_server_duration_milliseconds_sum", + "messenger_http_server_request_size_bytes_bucket", + "messenger_http_server_request_size_bytes_count", + "messenger_http_server_request_size_bytes_sum", + "messenger_http_server_response_size_bytes_bucket", + "messenger_http_server_response_size_bytes_count", + "messenger_http_server_response_size_bytes_sum", + "messenger_messenger_http_requests_total", + "messenger_messenger_worker_job_duration_seconds_bucket", + "messenger_messenger_worker_job_duration_seconds_count", + "messenger_messenger_worker_job_duration_seconds_sum", + "messenger_messenger_worker_jobs_processed_total", + "messenger_target_info", + "scrape_duration_seconds", + "scrape_samples_post_metric_relabeling", + "scrape_samples_scraped", + "scrape_series_added", + "up" + ] +} diff --git a/evidence/smoke-after-incidents.txt b/evidence/smoke-after-incidents.txt new file mode 100644 index 0000000..7e5c8e2 --- /dev/null +++ b/evidence/smoke-after-incidents.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..7e5c8e2 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/trace-correlation.txt b/evidence/trace-correlation.txt new file mode 100644 index 0000000..df4be5e --- /dev/null +++ b/evidence/trace-correlation.txt @@ -0,0 +1,20 @@ +Service: messenger-api +Event: HTTP request completed +Trace ID: c3ea28d96d7c536e222d8e4c7cba2727 +Span ID: 3be2550ec51156e2 +Tempo spans: 4 + Span: POST /api/v1/auth/sessions http send kind: SPAN_KIND_INTERNAL + Span: POST /api/v1/auth/sessions http send kind: SPAN_KIND_INTERNAL + Span: POST /api/v1/auth/sessions http send kind: SPAN_KIND_INTERNAL + Span: POST /api/v1/auth/sessions kind: SPAN_KIND_SERVER +PASS: Loki log linked to Tempo trace + +Service: messenger-worker +Event: Image job handled +Trace ID: af16dbcb7b7ca25b06d39902dc48afd8 +Span ID: 644d3cda73387e3f +Tempo spans: 1 + Span: image_job.process kind: SPAN_KIND_CONSUMER +PASS: Loki log linked to Tempo trace + +PASS: API and worker trace correlation diff --git a/evidence/worker-telemetry.txt b/evidence/worker-telemetry.txt new file mode 100644 index 0000000..b9d90a0 --- /dev/null +++ b/evidence/worker-telemetry.txt @@ -0,0 +1,12 @@ +worker-1 | 2026-11-19 11:10:44,315 INFO Pika version 1.3.2 connecting to ('172.20.0.7', 5672) +worker-1 | 2026-11-19 11:10:44,315 INFO Socket connected: +worker-1 | 2026-11-19 11:10:44,316 INFO Streaming transport linked up: (, _StreamingProtocolShim: params=>). +worker-1 | 2026-11-19 11:10:44,318 INFO AMQPConnector - reporting success: params=> +worker-1 | 2026-11-19 11:10:44,318 INFO AMQPConnectionWorkflow - reporting success: params=> +worker-1 | 2026-11-19 11:10:44,318 INFO Connection workflow succeeded: params=> +worker-1 | 2026-11-19 11:10:44,318 INFO Created channel=1 +worker-1 | 2026-11-19 11:10:44,342 INFO Worker listening on image_jobs +worker-1 | 2026-11-19 11:11:04,434 INFO Job a95f84e1-cd98-4fcd-ae9c-a5782b252fe3 ready, thumbnail 256x160 +worker-1 | 2026-11-19 11:11:04,441 INFO event=image_job_handled trace_id=af16dbcb7b7ca25b06d39902dc48afd8 span_id=644d3cda73387e3f Image job handled +worker-1 | 2026-11-19 11:11:25,616 INFO Job 1d011797-2efd-4bf7-88b9-c2c0e455119c ready, thumbnail 256x160 +worker-1 | 2026-11-19 11:11:25,623 INFO event=image_job_handled trace_id=53f30d09a4f2c578521c8bf075e35b88 span_id=93d63eec0255aa53 Image job handled diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 0000000..0e4d523 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,74 @@ + +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username VARCHAR(32) NOT NULL UNIQUE, + display_name VARCHAR(100) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + CONSTRAINT username_format + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT display_name_length + CHECK (char_length(display_name) BETWEEN 1 AND 100), + CONSTRAINT display_name_not_blank + CHECK (length(btrim(display_name)) > 0) +); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + CONSTRAINT title_not_blank + CHECK (length(btrim(title)) > 0) +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL + REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL + REFERENCES users(id), + position INTEGER NOT NULL, + + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL + REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL + REFERENCES users(id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + CONSTRAINT message_length + CHECK (char_length(text) BETWEEN 1 AND 4000), + CONSTRAINT message_not_blank + CHECK (length(btrim(text)) > 0) +); + +CREATE TABLE IF NOT EXISTS sessions ( + token_hash CHAR(64) PRIMARY KEY, + user_id UUID NOT NULL + REFERENCES users(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL, + revoked BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX IF NOT EXISTS idx_chat_members_user + ON chat_members(user_id, chat_id); + +CREATE INDEX IF NOT EXISTS idx_chats_order + ON chats(created_at, id); + +CREATE INDEX IF NOT EXISTS idx_messages_history + ON messages(chat_id, created_at, id); + +CREATE INDEX IF NOT EXISTS idx_sessions_user + ON sessions(user_id); + +CREATE INDEX IF NOT EXISTS idx_sessions_expiry + ON sessions(expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 0000000..663c6b6 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,14 @@ + +CREATE TABLE IF NOT EXISTS message_idempotency ( + user_id UUID NOT NULL REFERENCES users(id), + chat_id UUID NOT NULL REFERENCES chats(id), + idempotency_key VARCHAR(128) NOT NULL, + request_hash CHAR(64) NOT NULL, + message_id UUID NOT NULL REFERENCES messages(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + PRIMARY KEY (user_id, chat_id, idempotency_key) +); + +CREATE INDEX IF NOT EXISTS idx_idempotency_created +ON message_idempotency(created_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 0000000..49e9f33 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,34 @@ + +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + uploader_id UUID NOT NULL REFERENCES users(id), + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 CHAR(64) NOT NULL, + object_key TEXT NOT NULL UNIQUE, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + status TEXT NOT NULL DEFAULT 'ready', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT attachment_size_valid + CHECK (size_bytes BETWEEN 1 AND 10485760), + + CONSTRAINT attachment_dimensions_valid + CHECK (width > 0 AND height > 0), + + CONSTRAINT attachment_status_valid + CHECK (status IN ('ready', 'failed')) +); + +CREATE INDEX IF NOT EXISTS idx_attachments_chat +ON attachments(chat_id); + +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id), + attachment_id UUID NOT NULL REFERENCES attachments(id), + + PRIMARY KEY (message_id, attachment_id) +); diff --git a/migrations/004_allow_empty_message_text.sql b/migrations/004_allow_empty_message_text.sql new file mode 100644 index 0000000..9b72153 --- /dev/null +++ b/migrations/004_allow_empty_message_text.sql @@ -0,0 +1,13 @@ + +-- Lab4: messages with attachments may have empty text. +-- The API checks that a message contains text or attachments. + +ALTER TABLE messages +DROP CONSTRAINT IF EXISTS message_length; + +ALTER TABLE messages +DROP CONSTRAINT IF EXISTS message_not_blank; + +ALTER TABLE messages +ADD CONSTRAINT message_length +CHECK (char_length(text) <= 4000); diff --git a/migrations/005_async_jobs.sql b/migrations/005_async_jobs.sql new file mode 100644 index 0000000..9417b50 --- /dev/null +++ b/migrations/005_async_jobs.sql @@ -0,0 +1,95 @@ +-- Allow asynchronous attachment states. +ALTER TABLE attachments +DROP CONSTRAINT IF EXISTS attachment_status_valid; + +ALTER TABLE attachments +ADD CONSTRAINT attachment_status_valid +CHECK ( + status IN ( + 'queued', + 'processing', + 'ready', + 'failed' + ) +); + +-- Staging object stored in the private S3 bucket. +ALTER TABLE attachments +ADD COLUMN IF NOT EXISTS staging_key TEXT; + +-- Machine-readable final error. +ALTER TABLE attachments +ADD COLUMN IF NOT EXISTS error_code TEXT; + +-- Last metadata update timestamp. +ALTER TABLE attachments +ADD COLUMN IF NOT EXISTS updated_at +TIMESTAMPTZ NOT NULL DEFAULT NOW(); + +-- Worker jobs and durable publication intent. +CREATE TABLE IF NOT EXISTS attachment_jobs ( + id UUID PRIMARY KEY, + attachment_id UUID NOT NULL UNIQUE + REFERENCES attachments(id), + + status TEXT NOT NULL DEFAULT 'queued', + attempts INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 5, + + published_at TIMESTAMPTZ, + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT attachment_job_status_valid + CHECK ( + status IN ( + 'queued', + 'processing', + 'ready', + 'failed' + ) + ), + + CONSTRAINT attachment_job_attempts_valid + CHECK ( + attempts >= 0 + AND max_attempts BETWEEN 1 AND 20 + ) +); + +CREATE INDEX IF NOT EXISTS idx_attachment_jobs_pending +ON attachment_jobs(status, next_attempt_at); + +-- Image variants created by the worker. +CREATE TABLE IF NOT EXISTS attachment_variants ( + attachment_id UUID NOT NULL + REFERENCES attachments(id), + + variant_name TEXT NOT NULL, + object_key TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 CHAR(64) NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + PRIMARY KEY (attachment_id, variant_name), + + CONSTRAINT variant_name_valid + CHECK (variant_name IN ('thumbnail')), + + CONSTRAINT variant_size_valid + CHECK (size_bytes > 0), + + CONSTRAINT variant_dimensions_valid + CHECK (width > 0 AND height > 0) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_attachment_variants_key +ON attachment_variants(object_key); diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..810ed1c --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,37 @@ +events { + worker_connections 1024; +} + +http { + resolver 127.0.0.11 valid=2s ipv6=off; + resolver_timeout 2s; + + upstream messenger_api { + zone messenger_api 64k; + + server api1:8080 resolve max_fails=1 fail_timeout=2s; + server api2:8080 resolve max_fails=1 fail_timeout=2s; + } + + server { + client_max_body_size 11m; + listen 80; + + location / { + proxy_pass http://messenger_api; + + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_connect_timeout 2s; + proxy_read_timeout 30s; + proxy_send_timeout 30s; + + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + } + } +} diff --git a/observability/grafana/dashboards/messenger-lab6.json b/observability/grafana/dashboards/messenger-lab6.json new file mode 100644 index 0000000..296727a --- /dev/null +++ b/observability/grafana/dashboards/messenger-lab6.json @@ -0,0 +1,245 @@ +{ + "uid": "messenger-lab6", + "title": "Messenger Lab6 - Observability", + "tags": ["lab6", "messenger", "opentelemetry"], + "timezone": "browser", + "schemaVersion": 41, + "version": 1, + "refresh": "5s", + "time": { + "from": "now-30m", + "to": "now" + }, + "annotations": { + "list": [] + }, + "templating": { + "list": [] + }, + "panels": [ + { + "id": 1, + "title": "API - Requests per second", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, "y": 0, "w": 12, "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(messenger_messenger_http_requests_total[2m]))", + "legendFormat": "All business requests" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 2, + "title": "API - HTTP 5xx error ratio", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, "y": 0, "w": 12, "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "(100 * sum(rate(messenger_messenger_http_requests_total{http_status_class=\"5xx\"}[2m])) / clamp_min(sum(rate(messenger_messenger_http_requests_total[2m])), 0.000001)) or vector(0)", + "legendFormat": "5xx / all business requests" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + } + }, + { + "id": 3, + "title": "API - HTTP p95 duration", + "description": "OTel FastAPI duration histogram is in milliseconds; result converted to seconds.", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, "y": 8, "w": 12, "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_server_duration_milliseconds_bucket[5m]))) / 1000", + "legendFormat": "p95 HTTP duration" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + } + }, + { + "id": 4, + "title": "Worker - Jobs by outcome", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, "y": 8, "w": 12, "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (outcome) (increase(messenger_messenger_worker_jobs_processed_total[5m]))", + "legendFormat": "{{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + } + }, + { + "id": 5, + "title": "Worker - Processing p95 duration", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, "y": 16, "w": 12, "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_messenger_worker_job_duration_seconds_bucket[5m])))", + "legendFormat": "p95 job duration" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + } + }, + { + "id": 6, + "title": "Worker - Completed job attempts", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, "y": 16, "w": 12, "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(messenger_messenger_worker_jobs_processed_total)", + "legendFormat": "Processing attempts" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": ["lastNotNull"] + } + } + }, + { + "id": 7, + "title": "API - Structured logs", + "type": "logs", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "x": 0, "y": 24, "w": 24, "h": 9 + }, + "targets": [ + { + "refId": "A", + "expr": "{service_name=\"messenger-api\"}", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": true, + "showCommonLabels": false, + "enableLogDetails": true, + "wrapLogMessage": true, + "sortOrder": "Descending" + } + }, + { + "id": 8, + "title": "Worker - Structured logs", + "type": "logs", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "x": 0, "y": 33, "w": 24, "h": 9 + }, + "targets": [ + { + "refId": "A", + "expr": "{service_name=\"messenger-worker\"}", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": true, + "showCommonLabels": false, + "enableLogDetails": true, + "wrapLogMessage": true, + "sortOrder": "Descending" + } + } + ] +} diff --git a/observability/grafana/provisioning/dashboards/dashboard.yaml b/observability/grafana/provisioning/dashboards/dashboard.yaml new file mode 100644 index 0000000..2a29efd --- /dev/null +++ b/observability/grafana/provisioning/dashboards/dashboard.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: Messenger Lab6 + orgId: 1 + folder: Messenger Lab6 + type: file + disableDeletion: false + editable: false + options: + path: /var/lib/grafana/dashboards diff --git a/observability/grafana/provisioning/datasources/datasources.yaml b/observability/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..e78e8d6 --- /dev/null +++ b/observability/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,38 @@ + +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + derivedFields: + - name: TraceID + matcherRegex: 'trace_id=([a-f0-9]{32})' + datasourceUid: tempo + url: '$${__value.raw}' + + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + editable: false + jsonData: + tracesToLogsV2: + datasourceUid: loki + filterByTraceID: true + filterBySpanID: false + nodeGraph: + enabled: true diff --git a/observability/loki.yaml b/observability/loki.yaml new file mode 100644 index 0000000..615e6ef --- /dev/null +++ b/observability/loki.yaml @@ -0,0 +1,36 @@ + +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + filesystem: + directory: /loki/chunks + + tsdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/index_cache + +limits_config: + allow_structured_metadata: true + +analytics: + reporting_enabled: false diff --git a/observability/otel-collector.yaml b/observability/otel-collector.yaml new file mode 100644 index 0000000..ea505d5 --- /dev/null +++ b/observability/otel-collector.yaml @@ -0,0 +1,67 @@ + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 64 + + batch: + timeout: 5s + send_batch_size: 512 + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + namespace: messenger + resource_to_telemetry_conversion: + enabled: true + + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + sending_queue: + enabled: true + queue_size: 1000 + retry_on_failure: + enabled: true + + otlphttp/loki: + endpoint: http://loki:3100/otlp + sending_queue: + enabled: true + queue_size: 1000 + retry_on_failure: + enabled: true + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +service: + extensions: + - health_check + + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/tempo] + + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] diff --git a/observability/prometheus.yaml b/observability/prometheus.yaml new file mode 100644 index 0000000..4ad0cba --- /dev/null +++ b/observability/prometheus.yaml @@ -0,0 +1,9 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + - job_name: otel-collector + static_configs: + - targets: + - otel-collector:8889 diff --git a/observability/tempo.yaml b/observability/tempo.yaml new file mode 100644 index 0000000..d4f9281 --- /dev/null +++ b/observability/tempo.yaml @@ -0,0 +1,27 @@ + +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +ingester: + trace_idle_period: 10s + +compactor: + compaction: + block_retention: 48h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/traces diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d767a6f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +fastapi==0.115.12 +uvicorn==0.34.2 +psycopg[binary]==3.2.6 +argon2-cffi==23.1.0 +boto3==1.40.70 +Pillow==11.2.1 +pika==1.3.2 +opentelemetry-api==1.38.0 +opentelemetry-sdk==1.38.0 +opentelemetry-exporter-otlp-proto-http==1.38.0 +opentelemetry-instrumentation-fastapi==0.59b0 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..b2c5a84 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +echo "Stopping API and image processing..." +docker compose stop api1 api2 worker dispatcher + +echo "Restarting persistent storage and broker..." +docker compose restart db minio rabbitmq + +echo "Waiting for PostgreSQL and RabbitMQ..." +docker compose up -d --wait db rabbitmq + +echo "Checking S3 bucket..." +docker compose run --rm --no-deps s3-init + +echo "Starting dispatcher and worker..." +docker compose up -d dispatcher worker + +echo "Starting API replicas..." +docker compose up -d --wait api1 api2 + +echo "Restart complete. Persistent volumes preserved." diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..36b95cd --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +if [[ -z "${TARGET_INSTANCE_ID:-}" ]]; then + echo "TARGET_INSTANCE_ID is not set" >&2 + exit 1 +fi + +for service in api1 api2; do + instance_id=$( + docker compose exec -T "$service" python -c ' +import urllib.request + +response = urllib.request.urlopen( + "http://127.0.0.1:8080/health/live", + timeout=2 +) + +print(response.headers.get("X-Instance-Id", "")) +' 2>/dev/null + ) || continue + + if [[ "$instance_id" == "$TARGET_INSTANCE_ID" ]]; then + echo "Stopping $service" + docker compose stop -t 2 "$service" + exit 0 + fi +done + +echo "Target API instance not found" >&2 +exit 1 diff --git a/scripts/test-db-incident.py b/scripts/test-db-incident.py new file mode 100644 index 0000000..8a16533 --- /dev/null +++ b/scripts/test-db-incident.py @@ -0,0 +1,175 @@ + +import base64 +import json +import subprocess +import time +import urllib.error +import urllib.request +import uuid + + +BASE_URL = "http://127.0.0.1:8080" + + +def docker(*args): + subprocess.run( + ["docker", "compose", *args], + check=True, + ) + + +def request(method, path, body=None, headers=None): + request_headers = dict(headers or {}) + + if isinstance(body, dict): + body = json.dumps(body).encode("utf-8") + request_headers["Content-Type"] = "application/json" + + req = urllib.request.Request( + BASE_URL + path, + data=body, + headers=request_headers, + method=method, + ) + + started = time.monotonic() + + try: + with urllib.request.urlopen(req, timeout=5) as response: + response.read() + return response.status, time.monotonic() - started + + except urllib.error.HTTPError as exc: + exc.read() + return exc.code, time.monotonic() - started + + except (urllib.error.URLError, TimeoutError) as exc: + return type(exc).__name__, time.monotonic() - started + + +def login_header(username, password): + encoded = base64.b64encode( + f"{username}:{password}".encode("utf-8") + ).decode("ascii") + + return { + "Authorization": f"Basic {encoded}" + } + + +def wait_for_recovery(headers): + deadline = time.monotonic() + 90 + + while time.monotonic() < deadline: + status, duration = request( + "POST", + "/api/v1/auth/sessions", + body=b"", + headers=headers, + ) + + if status == 201: + print( + f"Recovery: HTTP 201, {duration:.3f}s", + flush=True, + ) + return + + time.sleep(2) + + raise RuntimeError( + "Authentication did not recover after DB restart" + ) + + +def main(): + suffix = uuid.uuid4().hex[:12] + + username = "lab6_db_" + suffix + password = "IncidentTest_" + suffix + + status, duration = request( + "POST", + "/api/v1/users", + body={ + "username": username, + "display_name": "DB incident test", + "password": password, + }, + ) + + if status != 201: + raise RuntimeError( + f"Cannot create test user: HTTP {status}" + ) + + headers = login_header(username, password) + + status, _ = request( + "POST", + "/api/v1/auth/sessions", + body=b"", + headers=headers, + ) + + if status != 201: + raise RuntimeError( + f"Baseline login failed: HTTP {status}" + ) + + print("Baseline: authentication HTTP 201", flush=True) + + observed_failures = 0 + + print("INCIDENT START: stopping PostgreSQL", flush=True) + + try: + docker("stop", "db") + + for attempt in range(1, 6): + status, duration = request( + "POST", + "/api/v1/auth/sessions", + body=b"", + headers=headers, + ) + + print( + f"Probe {attempt}: status={status}, " + f"duration={duration:.3f}s", + flush=True, + ) + + if status != 201: + observed_failures += 1 + + time.sleep(2) + + finally: + print( + "INCIDENT END: restoring PostgreSQL", + flush=True, + ) + docker("up", "-d", "--wait", "db") + + wait_for_recovery(headers) + + print( + f"Failed requests during incident: " + f"{observed_failures}/5", + flush=True, + ) + + if observed_failures == 0: + raise RuntimeError( + "The database incident did not affect requests" + ) + + print( + "PASS: database failure observed and recovered", + flush=True, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/test-redelivery.sh b/scripts/test-redelivery.sh new file mode 100644 index 0000000..c02c0c8 --- /dev/null +++ b/scripts/test-redelivery.sh @@ -0,0 +1,77 @@ + +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +sql() { + docker compose exec -T db \ + psql -U messenger -d messenger -At -c "$1" +} + +echo "Finding completed image job..." + +JOB_ID=$(sql " + SELECT j.id + FROM attachment_jobs j + JOIN attachment_variants v + ON v.attachment_id = j.attachment_id + WHERE j.status = 'ready' + AND v.variant_name = 'thumbnail' + ORDER BY j.created_at DESC + LIMIT 1; +") + +if [[ -z "$JOB_ID" ]]; then + echo "FAIL: no completed image job found" + exit 1 +fi + +echo "Job: $JOB_ID" + +STATE_QUERY=" + SELECT + j.status || '|' || + j.attempts || '|' || + v.sha256 || '|' || + ( + SELECT COUNT(*) + FROM attachment_variants + WHERE attachment_id = j.attachment_id + ) + FROM attachment_jobs j + JOIN attachment_variants v + ON v.attachment_id = j.attachment_id + WHERE j.id = '$JOB_ID' + AND v.variant_name = 'thumbnail'; +" + +BEFORE=$(sql "$STATE_QUERY") + +echo "Before: $BEFORE" +echo "Publishing the same job twice..." + +docker compose run --rm --no-deps api1 \ + python -c ' +import sys +from src.queue import publish_job + +job_id = sys.argv[1] +publish_job(job_id) +publish_job(job_id) +print("Two duplicate messages published") +' "$JOB_ID" + +sleep 5 + +AFTER=$(sql "$STATE_QUERY") + +echo "After: $AFTER" + +if [[ "$BEFORE" != "$AFTER" ]]; then + echo "FAIL: completed job changed after redelivery" + exit 1 +fi + +echo "PASS: duplicate delivery did not change completed job" +echo "PASS: thumbnail checksum and variant count preserved" diff --git a/scripts/test-trace-correlation.py b/scripts/test-trace-correlation.py new file mode 100644 index 0000000..723cd0c --- /dev/null +++ b/scripts/test-trace-correlation.py @@ -0,0 +1,154 @@ + +import json +import re +import time +import urllib.parse +import urllib.request + + +LOKI = "http://loki:3100" +TEMPO = "http://tempo:3200" + +TRACE_PATTERN = re.compile(r"^[0-9a-f]{32}$") +SPAN_PATTERN = re.compile(r"^[0-9a-f]{16}$") + + +def get_json(url): + with urllib.request.urlopen(url, timeout=15) as response: + return json.load(response) + + +def find_log(service, event): + params = urllib.parse.urlencode({ + "query": f'{{service_name="{service}"}}', + "start": str(time.time_ns() - 6 * 3600 * 10**9), + "end": str(time.time_ns()), + "limit": 500, + "direction": "BACKWARD", + }) + + payload = get_json( + f"{LOKI}/loki/api/v1/query_range?{params}" + ) + + streams = payload.get("data", {}).get("result", []) + + for stream in streams: + labels = stream.get("stream", {}) + + for entry in stream.get("values", []): + if len(entry) < 2: + continue + + message = entry[1] + metadata = entry[2] if len(entry) >= 3 else {} + + if event not in message and ( + metadata.get("event") != event + ): + continue + + trace_id = ( + metadata.get("trace_id") + or labels.get("trace_id") + ) + + span_id = ( + metadata.get("span_id") + or labels.get("span_id") + ) + + if not trace_id or not TRACE_PATTERN.fullmatch( + trace_id + ): + continue + + if not span_id or not SPAN_PATTERN.fullmatch( + span_id + ): + continue + + return { + "service": service, + "event": event, + "message": message, + "trace_id": trace_id, + "span_id": span_id, + } + + return None + + +def verify_trace(log, expected_kind): + trace_id = log["trace_id"] + + payload = get_json( + f"{TEMPO}/api/traces/{trace_id}" + ) + + batches = payload.get("batches", []) + spans = [] + + for batch in batches: + for scope in batch.get("scopeSpans", []): + spans.extend(scope.get("spans", [])) + + if not spans: + raise AssertionError( + f"No spans found for {trace_id}" + ) + + kinds = [str(span.get("kind", "")) for span in spans] + + if not any(expected_kind in kind for kind in kinds): + raise AssertionError( + f"Expected span kind {expected_kind}; got {kinds}" + ) + + print(f"Service: {log['service']}") + print(f"Event: {log['event']}") + print(f"Trace ID: {trace_id}") + print(f"Span ID: {log['span_id']}") + print(f"Tempo spans: {len(spans)}") + + for span in spans: + print( + " Span:", + span.get("name"), + "kind:", + span.get("kind"), + ) + + print("PASS: Loki log linked to Tempo trace") + print() + + +def main(): + cases = [ + ( + "messenger-api", + "HTTP request completed", + "SERVER", + ), + ( + "messenger-worker", + "Image job handled", + "CONSUMER", + ), + ] + + for service, event, kind in cases: + log = find_log(service, event) + + if log is None: + raise AssertionError( + f"No correlated log found for {service}" + ) + + verify_trace(log, kind) + + print("PASS: API and worker trace correlation") + + +if __name__ == "__main__": + main() diff --git a/scripts/test-worker-recovery.py b/scripts/test-worker-recovery.py new file mode 100644 index 0000000..4d920f6 --- /dev/null +++ b/scripts/test-worker-recovery.py @@ -0,0 +1,256 @@ +import os +import base64 +import hashlib +import json +import subprocess +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +import zlib +import struct + + +BASE = "http://127.0.0.1:8080" + + +def docker(*args): + subprocess.run( + ["docker", "compose", *args], + check=True, + ) + + +def api(method, path, token=None, body=None, headers=None): + request_headers = dict(headers or {}) + + if token: + request_headers["Authorization"] = f"Bearer {token}" + + if isinstance(body, dict): + body = json.dumps(body).encode("utf-8") + request_headers["Content-Type"] = "application/json" + + request = urllib.request.Request( + BASE + path, + data=body, + headers=request_headers, + method=method, + ) + + with urllib.request.urlopen(request, timeout=10) as response: + data = response.read() + return ( + response.status, + json.loads(data) if data and + "application/json" in response.headers.get( + "Content-Type", "" + ) else data, + ) + + +def png_chunk(name, data): + value = name + data + return ( + struct.pack(">I", len(data)) + + value + + struct.pack(">I", zlib.crc32(value) & 0xFFFFFFFF) + ) + + +def make_png(): + width, height = 320, 200 + scanline = b"\x00" + bytes([220, 80, 40]) * width + + return ( + b"\x89PNG\r\n\x1a\n" + + png_chunk( + b"IHDR", + struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0), + ) + + png_chunk( + b"IDAT", zlib.compress(scanline * height) + ) + + png_chunk(b"IEND", b"") + ) + + +def wait_ready(): + for _ in range(90): + try: + status, data = api("GET", "/health/ready") + if status == 200 and data.get("status") == "ok": + return + except Exception: + pass + time.sleep(1) + + raise RuntimeError("API did not become ready") + + +def main(): + suffix = uuid.uuid4().hex[:12] + username = "lab5_" + suffix + password = "Lab5TestPassword_" + suffix + + print("Stopping worker...", flush=True) + docker("stop", "worker") + wait_ready() + + status, user = api( + "POST", + "/api/v1/users", + body={ + "username": username, + "display_name": "Lab5 recovery test", + "password": password, + }, + ) + assert status == 201 + + credentials = base64.b64encode( + f"{username}:{password}".encode() + ).decode() + + status, session = api( + "POST", + "/api/v1/auth/sessions", + headers={"Authorization": f"Basic {credentials}"}, + body=b"", + ) + assert status == 201 + token = session["token"] + + status, chat = api( + "POST", + "/api/v1/chats", + token, + { + "title": "Worker recovery", + "member_ids": [user["id"]], + }, + ) + assert status == 201 + + original = make_png() + path = ( + f"/api/v1/chats/{chat['id']}/attachments" + "?filename=recovery.png" + ) + + status, attachment = api( + "POST", + path, + token, + original, + {"Content-Type": "image/png"}, + ) + + assert status == 202, f"Expected 202, got {status}" + assert attachment["status"] == "queued" + attachment_id = attachment["id"] + print("Upload: 202 queued", flush=True) + + status, message = api( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + token, + {"text": "", "attachment_ids": [attachment_id]}, + {"Idempotency-Key": "recovery-" + suffix}, + ) + assert status == 201 + assert attachment_id in message["attachment_ids"] + print("Message created while worker is stopped", flush=True) + + status, metadata = api( + "GET", f"/api/v1/attachments/{attachment_id}", token + ) + assert status == 200 and metadata["status"] == "queued" + + print("Restarting RabbitMQ and API...", flush=True) + docker("restart", "rabbitmq") + docker("up", "-d", "--wait", "rabbitmq") + docker("restart", "api1", "api2") + wait_ready() + + status, metadata = api( + "GET", f"/api/v1/attachments/{attachment_id}", token + ) + assert metadata["status"] == "queued" + print("Queued attachment survived restart", flush=True) + + pause_seconds = int( + os.getenv("LAB6_WORKER_PAUSE_SECONDS", "60") + ) + + print( + f"INCIDENT: worker stopped for {pause_seconds} seconds", + flush=True, + ) + time.sleep(pause_seconds) + + print("Starting worker...", flush=True) + docker("up", "-d", "worker") + + deadline = time.monotonic() + 90 + + while time.monotonic() < deadline: + status, metadata = api( + "GET", f"/api/v1/attachments/{attachment_id}", token + ) + + if metadata["status"] == "ready": + break + + if metadata["status"] == "failed": + raise RuntimeError( + f"Processing failed: {metadata.get('error_code')}" + ) + + time.sleep(1) + else: + raise RuntimeError("Attachment did not become ready") + + status, downloaded = api( + "GET", + f"/api/v1/attachments/{attachment_id}/content", + token, + ) + assert status == 200 + assert downloaded == original + assert hashlib.sha256(downloaded).hexdigest() == ( + hashlib.sha256(original).hexdigest() + ) + + status, thumbnail = api( + "GET", + f"/api/v1/attachments/{attachment_id}/content" + "?variant=thumbnail", + token, + ) + + assert status == 200 + assert thumbnail.startswith(b"\xff\xd8") + assert thumbnail.endswith(b"\xff\xd9") + + variants = metadata["variants"] + assert any( + v["name"] == "thumbnail" + and v["width"] == 256 + and v["height"] == 160 + for v in variants + ) + + print("Attachment: ready", flush=True) + print("Original: SHA-256 verified", flush=True) + print("Thumbnail: JPEG 256x160 verified", flush=True) + print("PASS: durable worker recovery", flush=True) + + +if __name__ == "__main__": + try: + main() + finally: + # Restore the worker even when the test fails. + docker("up", "-d", "worker") diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/api_observability.py b/src/api_observability.py new file mode 100644 index 0000000..25eb529 --- /dev/null +++ b/src/api_observability.py @@ -0,0 +1,127 @@ + +import time +import uuid + +from fastapi import Request +from opentelemetry import trace +from opentelemetry.instrumentation.fastapi import ( + FastAPIInstrumentor, +) +from starlette.middleware.base import BaseHTTPMiddleware + +from src.telemetry import setup_telemetry + + +def configure_api_observability(app): + tracer, meter, logger = setup_telemetry( + "messenger-api" + ) + + request_counter = meter.create_counter( + name="messenger_http_requests", + unit="{request}", + description="Completed business HTTP requests", + ) + + class ObservabilityMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + # Health checks should not distort business RPS. + if request.url.path.startswith("/health/"): + return await call_next(request) + + + request_id = uuid.uuid4().hex + started = time.monotonic() + status_code = 500 + + try: + response = await call_next(request) + status_code = response.status_code + + # Preserve the request ID produced by the existing + # API error handler. It must match the JSON body. + existing_id = response.headers.get("X-Request-Id") + + if existing_id: + request_id = existing_id + else: + response.headers["X-Request-Id"] = request_id + + return response + + + finally: + duration = time.monotonic() - started + + # FastAPI provides a route template after routing. + route = request.scope.get("route") + + if route is not None: + route_template = getattr( + route, "path", None + ) + else: + route_template = None + + # Do not use raw paths or dynamic IDs as labels. + if not route_template: + route_template = "unmatched" + + method = request.method + + if method not in ( + "GET", "POST", "PUT", + "PATCH", "DELETE", "HEAD", "OPTIONS" + ): + method = "OTHER" + + status_class = f"{status_code // 100}xx" + + attributes = { + "http.request.method": method, + "http.route": route_template, + "http.status_class": status_class, + } + + request_counter.add( + 1, + attributes=attributes, + ) + + # The span is created by FastAPIInstrumentor. + span = trace.get_current_span() + + if span.is_recording(): + span.set_attribute( + "messenger.request_id", + request_id, + ) + span.set_attribute( + "messenger.status_class", + status_class, + ) + + # No Authorization header, body, filename, + # chat ID, user ID or raw URL in telemetry. + logger.info( + "HTTP request completed", + extra={ + "event": "http_request_completed", + "request_id": request_id, + "http_method": method, + "http_route": route_template, + "status_code": status_code, + "duration_ms": round( + duration * 1000, 3 + ), + }, + ) + + # Add our middleware first so it executes within + # the span created by the OTel ASGI middleware. + app.add_middleware(ObservabilityMiddleware) + + FastAPIInstrumentor.instrument_app( + app, + excluded_urls=r".*/health/.*", + ) diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 0000000..59652d2 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,108 @@ + +import io +import re +import uuid +import hashlib + +from PIL import Image, UnidentifiedImageError + + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 + + +def valid_filename(filename): + if not isinstance(filename, str): + return False + + if not 1 <= len(filename) <= 255: + return False + + if filename.strip() == "": + return False + + if "/" in filename or "\\" in filename: + return False + + if any(ord(ch) < 32 or ord(ch) == 127 for ch in filename): + return False + + return True + + +def inspect_image(data, declared_mime): + if len(data) > MAX_BYTES: + raise ValueError("too_large") + + if not data: + raise ValueError("invalid_image") + + if declared_mime not in ("image/png", "image/jpeg"): + raise ValueError("invalid_image") + + try: + with Image.open(io.BytesIO(data)) as image: + image_format = image.format + + width, height = image.size + + if width * height > MAX_PIXELS: + raise ValueError("too_many_pixels") + + if image_format not in ("PNG", "JPEG"): + raise ValueError("invalid_image") + + actual_mime = ( + "image/png" + if image_format == "PNG" + else "image/jpeg" + ) + + if declared_mime != actual_mime: + raise ValueError("invalid_image") + + image.verify() + + except Image.DecompressionBombError: + raise ValueError("too_many_pixels") + except ( + UnidentifiedImageError, + OSError, + SyntaxError, + EOFError, + ): + raise ValueError("invalid_image") + + return { + "content_type": actual_mime, + "width": width, + "height": height, + "size_bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + } + +def attachment_json(row, variants=None): + from datetime import timezone + + def timestamp(value): + return value.astimezone(timezone.utc).isoformat( + timespec="microseconds" + ).replace("+00:00", "Z") + + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["uploader_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": variants if variants is not None else [], + "error_code": row["error_code"], + "created_at": timestamp(row["created_at"]), + "updated_at": timestamp(row["updated_at"]), + } + +def object_key(): + return f"originals/{uuid.uuid4()}" diff --git a/src/dispatcher.py b/src/dispatcher.py new file mode 100644 index 0000000..31c8cf3 --- /dev/null +++ b/src/dispatcher.py @@ -0,0 +1,84 @@ + +import logging +import os +import time + +import psycopg +from psycopg.rows import dict_row + +from src.queue import publish_job + + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", +) + +logger = logging.getLogger("image-dispatcher") + +DATABASE_URL = os.environ["DATABASE_URL"] + + +def dispatch_one(): + with psycopg.connect( + DATABASE_URL, + row_factory=dict_row, + ) as conn: + + # Only publish due, unpublished jobs. + # The row lock also prevents two dispatchers + # from selecting the same job simultaneously. + job = conn.execute( + """ + SELECT id + FROM attachment_jobs + WHERE status = 'queued' + AND published_at IS NULL + AND next_attempt_at <= NOW() + ORDER BY created_at, id + LIMIT 1 + FOR UPDATE SKIP LOCKED + """, + ).fetchone() + + if job is None: + return False + + job_id = str(job["id"]) + + # RabbitMQ publisher confirms are enabled. + # An exception rolls back this DB transaction. + publish_job(job_id) + + conn.execute( + """ + UPDATE attachment_jobs + SET published_at = NOW(), + updated_at = NOW() + WHERE id = %s + """, + (job_id,), + ) + + logger.info("Published job %s", job_id) + + return True + + +def main(): + logger.info("Dispatcher started") + + while True: + try: + if not dispatch_one(): + time.sleep(1) + + except Exception: + logger.exception( + "Dispatch failed; retrying" + ) + time.sleep(3) + + +if __name__ == "__main__": + main() diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..a77a0dc --- /dev/null +++ b/src/main.py @@ -0,0 +1,1277 @@ + +import base64 +import hashlib +import io +import json +import os +import re +import secrets +import uuid + +from src.api_observability import configure_api_observability + +from datetime import datetime, timedelta, timezone + +import psycopg +from psycopg.rows import dict_row +from psycopg.errors import UniqueViolation + +from argon2 import PasswordHasher +from argon2.exceptions import ( + VerifyMismatchError, + VerificationError, + InvalidHashError, +) + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from fastapi.exceptions import RequestValidationError +from starlette.exceptions import HTTPException as StarletteHTTPException + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, UnidentifiedImageError + +app = FastAPI(title="MTUSI Messenger Lab5") + +INSTANCE_ID = str(uuid.uuid4()) + +DATABASE_URL = os.environ["DATABASE_URL"] +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) + +IMAGE_MAX_ATTEMPTS = int(os.getenv("IMAGE_MAX_ATTEMPTS", "5")) +S3_BUCKET = os.environ["S3_BUCKET"] +MAX_IMAGE_BYTES = 10 * 1024 * 1024 +MAX_IMAGE_PIXELS = 40_000_000 + +s3 = boto3.client( + "s3", + endpoint_url=os.environ["S3_ENDPOINT"], + aws_access_key_id=os.environ["S3_ACCESS_KEY"], + aws_secret_access_key=os.environ["S3_SECRET_KEY"], + region_name="us-east-1", + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=3, + read_timeout=10, + retries={"max_attempts": 1}, + ), +) + +password_hasher = PasswordHasher() + + +# ---------- Helpers ---------- + +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def fail(status, code, message, headers=None): + raise ApiError(status, code, message, headers) + + +def db(): + return psycopg.connect(DATABASE_URL, row_factory=dict_row) + + +def utc_string(value): + return value.astimezone(timezone.utc).isoformat( + timespec="microseconds" + ).replace("+00:00", "Z") + + +def user_json(row): + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": utc_string(row["created_at"]), + } + + + +def message_json(row, conn=None): + attachment_ids = [] + + if conn is not None: + attachments = conn.execute( + """ + SELECT attachment_id + FROM message_attachments + WHERE message_id = %s + ORDER BY attachment_id + """, + (row["id"],), + ).fetchall() + + attachment_ids = [ + str(item["attachment_id"]) + for item in attachments + ] + + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "attachment_ids": attachment_ids, + "created_at": utc_string(row["created_at"]), + } + + + +def chat_json(conn, row): + members = conn.execute( + """ + SELECT user_id FROM chat_members + WHERE chat_id = %s + ORDER BY position + """, + (row["id"],) + ).fetchall() + + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(m["user_id"]) for m in members], + "created_at": utc_string(row["created_at"]), + } + + +def valid_uuid(value): + if not isinstance(value, str): + return False + try: + return str(uuid.UUID(value)) == value + except ValueError: + return False + + +def valid_text(value, maximum): + return ( + isinstance(value, str) + and 1 <= len(value) <= maximum + and bool(value.strip()) + ) + + +def validate_limit(request): + value = request.query_params.get("limit", "50") + + if not re.fullmatch(r"[0-9]+", value): + fail(400, "invalid_request", "Invalid limit") + + number = int(value) + + if not 1 <= number <= 100: + fail(400, "invalid_request", "Invalid limit") + + return number + + +def make_cursor(resource, row): + payload = { + "resource": resource, + "created_at": utc_string(row["created_at"]), + "id": str(row["id"]), + } + + raw = json.dumps(payload).encode("utf-8") + + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def read_cursor(cursor, resource): + if not isinstance(cursor, str) or not cursor: + fail(400, "invalid_request", "Invalid cursor") + + try: + raw = base64.urlsafe_b64decode( + cursor + "=" * (-len(cursor) % 4) + ) + value = json.loads(raw) + + if value["resource"] != resource: + raise ValueError() + + if not valid_uuid(value["id"]): + raise ValueError() + + timestamp = datetime.fromisoformat( + value["created_at"].replace("Z", "+00:00") + ) + + if timestamp.tzinfo is None: + raise ValueError() + + return timestamp, value["id"] + + except (ValueError, KeyError, TypeError, UnicodeDecodeError): + fail(400, "invalid_request", "Invalid cursor") + + +async def json_body(request, required, allowed): + content_type = request.headers.get("content-type", "") + media_type = content_type.split(";")[0].strip().lower() + + if media_type != "application/json": + fail(415, "unsupported_media_type", "Expected JSON") + + try: + data = await request.json() + except Exception: + fail(400, "invalid_request", "Invalid JSON") + + if not isinstance(data, dict): + fail(400, "invalid_request", "Expected JSON object") + + if not set(required).issubset(data): + fail(400, "invalid_request", "Missing fields") + + if not set(data).issubset(allowed): + fail(400, "invalid_request", "Unexpected fields") + + return data + + +def unauthorized(kind="Bearer"): + if kind == "Basic": + header = 'Basic realm="messenger", charset="UTF-8"' + else: + header = 'Bearer realm="messenger"' + + fail( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": header}, + ) + + +def current_user(request): + authorization = request.headers.get("Authorization", "") + + if not authorization.startswith("Bearer "): + unauthorized() + + token = authorization[7:] + + if not token: + unauthorized() + + token_hash = hashlib.sha256(token.encode()).hexdigest() + + with db() as conn: + row = conn.execute( + """ + SELECT u.id, u.username, u.display_name, u.created_at + FROM sessions s + JOIN users u ON u.id = s.user_id + WHERE s.token_hash = %s + AND s.revoked = FALSE + AND s.expires_at > clock_timestamp() + """, + (token_hash,) + ).fetchone() + + if row is None: + unauthorized() + + return row + + +def accessible_chat(conn, chat_id, user_id): + if not valid_uuid(chat_id): + fail(400, "invalid_request", "Invalid chat ID") + + row = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE c.id = %s AND m.user_id = %s + """, + (chat_id, user_id) + ).fetchone() + + if row is None: + fail(404, "not_found", "Chat not found") + + return row + + +# ---------- Request IDs and errors ---------- + +def error_response(request, status, code, message, headers=None): + request_id = request.state.request_id + + response_headers = { + "X-Request-Id": request_id, + **(headers or {}), + } + + return JSONResponse( + status_code=status, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers=response_headers, + ) + + +@app.middleware("http") +async def add_request_id(request, call_next): + request.state.request_id = str(uuid.uuid4()) + + try: + response = await call_next(request) + except psycopg.OperationalError: + response = error_response( + request, 503, "unavailable", "Database unavailable" + ) + except Exception: + response = error_response( + request, 500, "internal_error", "Internal server error" + ) + + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(ApiError) +async def api_error_handler(request, exc): + return error_response( + request, exc.status, exc.code, exc.message, exc.headers + ) + + +@app.exception_handler(StarletteHTTPException) +async def http_error_handler(request, exc): + return error_response( + request, exc.status_code, + "not_found" if exc.status_code == 404 else "invalid_request", + "Resource not found" if exc.status_code == 404 else "Invalid request", + ) + + +@app.exception_handler(RequestValidationError) +async def validation_error_handler(request, exc): + return error_response( + request, 400, "invalid_request", "Invalid request" + ) + + +# ---------- Health ---------- + +@app.get("/health/live") +def health_live(): + return {"status": "ok"} + + + +@app.get("/health/ready") +def health_ready(): + try: + with db() as conn: + conn.execute("SELECT 1") + + s3.head_bucket(Bucket=S3_BUCKET) + + return {"status": "ok"} + + except ( + psycopg.Error, + BotoCoreError, + ClientError, + OSError, + ): + fail( + 503, + "unavailable", + "Storage unavailable", + ) + + +# ---------- Users ---------- + +@app.post("/api/v1/users", status_code=201) +async def create_user(request: Request): + data = await json_body( + request, + required=["username", "display_name", "password"], + allowed=["username", "display_name", "password"], + ) + + username = data["username"] + display_name = data["display_name"] + password = data["password"] + + if ( + not isinstance(username, str) + or not re.fullmatch(r"[a-z0-9_]{3,32}", username) + or not valid_text(display_name, 100) + or not isinstance(password, str) + or not 12 <= len(password) <= 128 + ): + fail(400, "invalid_request", "Invalid user fields") + + password_hash = password_hasher.hash(password) + user_id = str(uuid.uuid4()) + + try: + with db() as conn: + row = conn.execute( + """ + INSERT INTO users + (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (user_id, username, display_name, password_hash) + ).fetchone() + + return user_json(row) + + except UniqueViolation: + fail(409, "conflict", "Username already exists") + + +@app.get("/api/v1/users/me") +def get_me(request: Request): + return user_json(current_user(request)) + + +# ---------- Sessions ---------- + +@app.post("/api/v1/auth/sessions", status_code=201) +async def create_session(request: Request): + authorization = request.headers.get("Authorization", "") + + if not authorization.startswith("Basic "): + unauthorized("Basic") + + try: + encoded = authorization[6:] + decoded = base64.b64decode(encoded, validate=True).decode("utf-8") + + if ":" not in decoded: + raise ValueError() + + username, password = decoded.split(":", 1) + + except (ValueError, UnicodeDecodeError): + unauthorized("Basic") + + if not re.fullmatch(r"[a-z0-9_]{3,32}", username): + unauthorized("Basic") + + with db() as conn: + row = conn.execute( + "SELECT * FROM users WHERE username = %s", + (username,) + ).fetchone() + + if row is None: + unauthorized("Basic") + + try: + password_hasher.verify(row["password_hash"], password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + unauthorized("Basic") + + token = secrets.token_urlsafe(32) + token_hash = hashlib.sha256(token.encode()).hexdigest() + + expires_at = datetime.now(timezone.utc) + timedelta( + seconds=SESSION_TTL_SECONDS + ) + + with db() as conn: + conn.execute( + """ + INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES (%s, %s, %s) + """, + (token_hash, row["id"], expires_at) + ) + + return { + "token": token, + "token_type": "Bearer", + "expires_at": utc_string(expires_at), + "user": user_json(row), + } + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def logout(request: Request): + current_user(request) + + token = request.headers["Authorization"][7:] + token_hash = hashlib.sha256(token.encode()).hexdigest() + + with db() as conn: + conn.execute( + "UPDATE sessions SET revoked = TRUE WHERE token_hash = %s", + (token_hash,) + ) + + return Response(status_code=204) + + +# ---------- Chats ---------- + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + actor = current_user(request) + + data = await json_body( + request, + required=["title", "member_ids"], + allowed=["title", "member_ids"], + ) + + title = data["title"] + member_ids = data["member_ids"] + + if not valid_text(title, 100): + fail(400, "invalid_request", "Invalid title") + + if ( + not isinstance(member_ids, list) + or not 1 <= len(member_ids) <= 100 + or not all(valid_uuid(x) for x in member_ids) + or len(member_ids) != len(set(member_ids)) + or str(actor["id"]) not in member_ids + ): + fail(400, "invalid_request", "Invalid members") + + chat_id = str(uuid.uuid4()) + + with db() as conn: + # All operations use one transaction. + existing = conn.execute( + """ + SELECT id FROM users + WHERE id = ANY(%s::uuid[]) + FOR KEY SHARE + """, + (member_ids,) + ).fetchall() + + if len(existing) != len(member_ids): + fail(400, "invalid_request", "Unknown member") + + row = conn.execute( + """ + INSERT INTO chats (id, title) + VALUES (%s, %s) + RETURNING id, title, created_at + """, + (chat_id, title) + ).fetchone() + + for position, member_id in enumerate(member_ids): + conn.execute( + """ + INSERT INTO chat_members (chat_id, user_id, position) + VALUES (%s, %s, %s) + """, + (chat_id, member_id, position) + ) + + result = chat_json(conn, row) + + return result + + +@app.get("/api/v1/chats") +def list_chats(request: Request): + actor = current_user(request) + limit = validate_limit(request) + cursor = request.query_params.get("cursor") + + resource = "chats:" + str(actor["id"]) + + with db() as conn: + if cursor is None: + rows = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = %s + ORDER BY c.created_at, c.id + LIMIT %s + """, + (actor["id"], limit + 1) + ).fetchall() + else: + timestamp, last_id = read_cursor(cursor, resource) + + rows = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = %s + AND (c.created_at, c.id) > (%s, %s) + ORDER BY c.created_at, c.id + LIMIT %s + """, + (actor["id"], timestamp, last_id, limit + 1) + ).fetchall() + + has_more = len(rows) > limit + page = rows[:limit] + + items = [chat_json(conn, row) for row in page] + + next_cursor = ( + make_cursor(resource, page[-1]) + if has_more else None + ) + + return {"items": items, "next_cursor": next_cursor} + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: str, request: Request): + actor = current_user(request) + + with db() as conn: + row = accessible_chat(conn, chat_id, actor["id"]) + result = chat_json(conn, row) + + return result + + +# ---------- Attachments ---------- + +from src.attachments import ( + MAX_BYTES, + valid_filename, + inspect_image, + attachment_json, + object_key, +) + + +def attachment_for_user(conn, attachment_id, user_id): + if not valid_uuid(attachment_id): + fail(400, "invalid_request", "Invalid attachment ID") + + row = conn.execute( + """ + SELECT a.* + FROM attachments a + JOIN chat_members cm + ON cm.chat_id = a.chat_id + WHERE a.id = %s + AND cm.user_id = %s + """, + (attachment_id, user_id), + ).fetchone() + + if row is None: + fail(404, "not_found", "Attachment not found") + + return row + + +@app.post( + "/api/v1/chats/{chat_id}/attachments", + status_code=202, +) +async def upload_attachment(chat_id: str, request: Request): + actor = current_user(request) + + with db() as conn: + accessible_chat(conn, chat_id, actor["id"]) + + filename = request.query_params.get("filename") + + if not valid_filename(filename): + fail(400, "invalid_request", "Invalid filename") + + content_type = request.headers.get( + "content-type", "" + ).split(";")[0].strip().lower() + + if content_type not in ("image/png", "image/jpeg"): + fail( + 415, + "unsupported_media_type", + "Expected PNG or JPEG", + ) + + content_length = request.headers.get("content-length") + + if content_length is not None: + try: + declared_size = int(content_length) + except ValueError: + fail( + 400, + "invalid_request", + "Invalid Content-Length", + ) + + if declared_size > MAX_BYTES: + fail( + 413, + "payload_too_large", + "Image exceeds 10 MiB", + ) + + # Receive the HTTP body with a strict byte limit. + data = bytearray() + + async for chunk in request.stream(): + if len(data) + len(chunk) > MAX_BYTES: + fail( + 413, + "payload_too_large", + "Image exceeds 10 MiB", + ) + + data.extend(chunk) + + data = bytes(data) + + try: + metadata = inspect_image(data, content_type) + except ValueError as exc: + if str(exc) in ("too_large", "too_many_pixels"): + fail( + 413, + "payload_too_large", + "Image exceeds limits", + ) + + fail( + 415, + "unsupported_media_type", + "Invalid PNG or JPEG", + ) + + attachment_id = str(uuid.uuid4()) + job_id = str(uuid.uuid4()) + + # Server-generated keys, independent of filename. + staging_key = f"staging/{attachment_id}" + original_key = f"originals/{attachment_id}" + + # First persist the accepted original in shared S3. + try: + s3.put_object( + Bucket=S3_BUCKET, + Key=staging_key, + Body=data, + ContentType=metadata["content_type"], + ) + except (BotoCoreError, ClientError, OSError): + fail( + 503, + "unavailable", + "Storage unavailable", + ) + + try: + with db() as conn: + accessible_chat(conn, chat_id, actor["id"]) + + row = conn.execute( + """ + INSERT INTO attachments + ( + id, + chat_id, + uploader_id, + filename, + content_type, + size_bytes, + sha256, + object_key, + staging_key, + width, + height, + status, + error_code + ) + VALUES ( + %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, + %s, 'queued', NULL + ) + RETURNING * + """, + ( + attachment_id, + chat_id, + actor["id"], + filename, + metadata["content_type"], + metadata["size_bytes"], + metadata["sha256"], + original_key, + staging_key, + metadata["width"], + metadata["height"], + ), + ).fetchone() + + # Durable job record in the same DB transaction. + conn.execute( + """ + INSERT INTO attachment_jobs + ( + id, + attachment_id, + status, + attempts, + max_attempts, + next_attempt_at + ) + VALUES ( + %s, %s, 'queued', + 0, %s, NOW() + ) + """, + ( + job_id, + attachment_id, + IMAGE_MAX_ATTEMPTS, + ), + ) + + except Exception: + # The database transaction was not confirmed. + # Try to clean up the unreferenced staging object. + try: + s3.delete_object( + Bucket=S3_BUCKET, + Key=staging_key, + ) + except Exception: + pass + + raise + + # Do not perform resizing or require the worker here. + # The dispatcher will publish this durable DB job. + return attachment_json(row) + + + +@app.get("/api/v1/attachments/{attachment_id}") +def get_attachment(attachment_id: str, request: Request): + actor = current_user(request) + + with db() as conn: + row = attachment_for_user( + conn, attachment_id, actor["id"] + ) + + variant_rows = conn.execute( + """ + SELECT variant_name, content_type, + size_bytes, sha256, width, height + FROM attachment_variants + WHERE attachment_id = %s + ORDER BY variant_name + """, + (attachment_id,), + ).fetchall() + + variants = [ + { + "name": v["variant_name"], + "content_type": v["content_type"], + "size_bytes": v["size_bytes"], + "sha256": v["sha256"], + "width": v["width"], + "height": v["height"], + } + for v in variant_rows + ] + + return attachment_json(row, variants) + + + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment( + attachment_id: str, + request: Request, +): + actor = current_user(request) + + variant = request.query_params.get( + "variant", "original" + ) + + with db() as conn: + row = attachment_for_user( + conn, attachment_id, actor["id"] + ) + + if row["status"] != "ready": + fail( + 409, + "conflict", + "Attachment is not ready", + ) + + if variant == "original": + key = row["object_key"] + content_type = row["content_type"] + expected_size = row["size_bytes"] + expected_sha256 = row["sha256"] + + elif variant == "thumbnail": + variant_row = conn.execute( + """ + SELECT object_key, content_type, + size_bytes, sha256 + FROM attachment_variants + WHERE attachment_id = %s + AND variant_name = 'thumbnail' + """, + (attachment_id,), + ).fetchone() + + if variant_row is None: + fail( + 404, + "not_found", + "Variant not found", + ) + + key = variant_row["object_key"] + content_type = variant_row["content_type"] + expected_size = variant_row["size_bytes"] + expected_sha256 = variant_row["sha256"] + + else: + fail( + 404, + "not_found", + "Variant not found", + ) + + try: + result = s3.get_object( + Bucket=S3_BUCKET, + Key=key, + ) + + body_stream = result["Body"] + + try: + data = body_stream.read(MAX_BYTES + 1) + finally: + body_stream.close() + + except (BotoCoreError, ClientError, OSError): + fail( + 503, + "unavailable", + "Storage unavailable", + ) + + if ( + len(data) != expected_size + or hashlib.sha256(data).hexdigest() + != expected_sha256 + ): + fail( + 503, + "unavailable", + "Stored object is unavailable", + ) + + return Response( + content=data, + media_type=content_type, + headers={ + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + }, + ) + +def get_idempotency_key(request): + key = request.headers.get("Idempotency-Key") + + if ( + key is None + or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key) + ): + fail( + 400, + "invalid_request", + "Invalid or missing Idempotency-Key" + ) + + return key + + +def request_fingerprint(data): + normalized = { + "text": data["text"], + "attachment_ids": data.get("attachment_ids", []), + } + + encoded = json.dumps( + normalized, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + return hashlib.sha256(encoded).hexdigest() + + + +def idempotency_lock_id(user_id, chat_id, key): + scope = f"{user_id}:{chat_id}:{key}" + + digest = hashlib.sha256( + scope.encode("utf-8") + ).digest() + + return int.from_bytes( + digest[:8], + byteorder="big", + signed=True + ) + + +# ---------- Messages ---------- + + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def send_message(chat_id: str, request: Request): + actor = current_user(request) + + data = await json_body( + request, + required=["text"], + allowed=["text", "attachment_ids"], + ) + + message_text = data["text"] + attachment_ids = data.get("attachment_ids", []) + + if ( + not isinstance(message_text, str) + or len(message_text) > 4000 + ): + fail(400, "invalid_request", "Invalid message text") + + if ( + not isinstance(attachment_ids, list) + or len(attachment_ids) > 10 + or not all(valid_uuid(x) for x in attachment_ids) + or len(attachment_ids) != len(set(attachment_ids)) + ): + fail(400, "invalid_request", "Invalid attachment_ids") + + if not message_text.strip() and not attachment_ids: + fail(400, "invalid_request", "Empty message") + + key = get_idempotency_key(request) + fingerprint = request_fingerprint(data) + user_id = actor["id"] + + with db() as conn: + accessible_chat(conn, chat_id, user_id) + + lock_id = idempotency_lock_id( + user_id, chat_id, key + ) + + conn.execute( + "SELECT pg_advisory_xact_lock(%s)", + (lock_id,), + ) + + conn.execute( + """ + DELETE FROM message_idempotency + WHERE user_id = %s + AND chat_id = %s + AND idempotency_key = %s + AND created_at < + clock_timestamp() - INTERVAL '24 hours' + """, + (user_id, chat_id, key), + ) + + previous = conn.execute( + """ + SELECT i.request_hash, m.* + FROM message_idempotency i + JOIN messages m ON m.id = i.message_id + WHERE i.user_id = %s + AND i.chat_id = %s + AND i.idempotency_key = %s + """, + (user_id, chat_id, key), + ).fetchone() + + if previous is not None: + if previous["request_hash"] != fingerprint: + fail( + 409, + "conflict", + "Idempotency-Key already used " + "with different content", + ) + + return message_json(previous, conn) + + for attachment_id in attachment_ids: + attachment = conn.execute( + """ + SELECT id, status + FROM attachments + WHERE id = %s + AND chat_id = %s + FOR KEY SHARE + """, + (attachment_id, chat_id), + ).fetchone() + + if attachment is None: + fail( + 404, + "not_found", + "Attachment not found", + ) + + if attachment["status"] == "failed": + fail( + 409, + "conflict", + "Attachment processing failed", + ) + + if attachment["status"] not in ( + "queued", + "processing", + "ready", + ): + fail( + 409, + "conflict", + "Invalid attachment status", + ) + + message_id = str(uuid.uuid4()) + + row = conn.execute( + """ + INSERT INTO messages + (id, chat_id, sender_id, text) + VALUES (%s, %s, %s, %s) + RETURNING id, chat_id, sender_id, + text, created_at + """, + (message_id, chat_id, user_id, message_text), + ).fetchone() + + for attachment_id in attachment_ids: + conn.execute( + """ + INSERT INTO message_attachments + (message_id, attachment_id) + VALUES (%s, %s) + """, + (message_id, attachment_id), + ) + + conn.execute( + """ + INSERT INTO message_idempotency + ( + user_id, chat_id, idempotency_key, + request_hash, message_id + ) + VALUES (%s, %s, %s, %s, %s) + """, + ( + user_id, chat_id, key, + fingerprint, message_id, + ), + ) + + return message_json(row, conn) + + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(chat_id: str, request: Request): + actor = current_user(request) + limit = validate_limit(request) + cursor = request.query_params.get("cursor") + + resource = "messages:" + chat_id + + with db() as conn: + accessible_chat(conn, chat_id, actor["id"]) + + if cursor is None: + rows = conn.execute( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s + ORDER BY created_at, id + LIMIT %s + """, + (chat_id, limit + 1) + ).fetchall() + else: + timestamp, last_id = read_cursor(cursor, resource) + + rows = conn.execute( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s + AND (created_at, id) > (%s, %s) + ORDER BY created_at, id + LIMIT %s + """, + (chat_id, timestamp, last_id, limit + 1) + ).fetchall() + + has_more = len(rows) > limit + page = rows[:limit] + + next_cursor = ( + make_cursor(resource, page[-1]) + if has_more else None + ) + + items = [message_json(row, conn) for row in page] + + return {"items": items, "next_cursor": next_cursor} + +# Initialize observability after route registration. +configure_api_observability(app) \ No newline at end of file diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 0000000..c13aa99 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,53 @@ + +import os +from pathlib import Path + +import psycopg + + +DATABASE_URL = os.environ["DATABASE_URL"] +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" + + +def run_migrations(): + with psycopg.connect(DATABASE_URL) as conn: + with conn.cursor() as cur: + # Prevent simultaneous migration runs. + cur.execute("SELECT pg_advisory_xact_lock(2026002)") + + cur.execute(""" + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL + DEFAULT clock_timestamp() + ) + """) + + for file in sorted(MIGRATIONS_DIR.glob("*.sql")): + cur.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (file.name,) + ) + + if cur.fetchone(): + print(f"Already applied: {file.name}") + continue + + sql = file.read_text(encoding="utf-8") + + for statement in sql.split(";"): + if statement.strip(): + cur.execute(statement) + + cur.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", + (file.name,) + ) + + print(f"Applied migration: {file.name}") + + print("Migrations completed") + + +if __name__ == "__main__": + run_migrations() diff --git a/src/queue.py b/src/queue.py new file mode 100644 index 0000000..125b517 --- /dev/null +++ b/src/queue.py @@ -0,0 +1,89 @@ + +import json +import os +import uuid + +import pika + + +RABBITMQ_HOST = os.getenv("RABBITMQ_HOST", "rabbitmq") +RABBITMQ_USER = os.environ["RABBITMQ_USER"] +RABBITMQ_PASSWORD = os.environ["RABBITMQ_PASSWORD"] +RABBITMQ_QUEUE = os.getenv( + "RABBITMQ_QUEUE", + "image_jobs", +) + + +def connect(): + credentials = pika.PlainCredentials( + RABBITMQ_USER, + RABBITMQ_PASSWORD, + ) + + parameters = pika.ConnectionParameters( + host=RABBITMQ_HOST, + port=5672, + credentials=credentials, + heartbeat=30, + blocked_connection_timeout=15, + connection_attempts=3, + retry_delay=2, + ) + + return pika.BlockingConnection(parameters) + + +def declare_queue(channel): + channel.queue_declare( + queue=RABBITMQ_QUEUE, + durable=True, + arguments={ + "x-queue-type": "quorum", + }, + ) + + +def publish_job(job_id): + connection = connect() + + try: + channel = connection.channel() + declare_queue(channel) + + # Wait for confirmation from RabbitMQ. + channel.confirm_delivery() + + payload = json.dumps({ + "job_id": str(uuid.UUID(str(job_id))), + }).encode("utf-8") + + channel.basic_publish( + exchange="", + routing_key=RABBITMQ_QUEUE, + body=payload, + properties=pika.BasicProperties( + delivery_mode=2, + content_type="application/json", + message_id=str(job_id), + ), + mandatory=True, + ) + + finally: + if connection.is_open: + connection.close() + + +def parse_job(body): + payload = json.loads(body) + + if not isinstance(payload, dict): + raise ValueError("Invalid job payload") + + job_id = payload.get("job_id") + + if not isinstance(job_id, str): + raise ValueError("Missing job_id") + + return str(uuid.UUID(job_id)) diff --git a/src/s3_setup.py b/src/s3_setup.py new file mode 100644 index 0000000..7798af6 --- /dev/null +++ b/src/s3_setup.py @@ -0,0 +1,60 @@ + +import os +import time + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + + +def main(): + bucket = os.environ["S3_BUCKET"] + + client = boto3.client( + "s3", + endpoint_url=os.environ["S3_ENDPOINT"], + aws_access_key_id=os.environ["S3_ACCESS_KEY"], + aws_secret_access_key=os.environ["S3_SECRET_KEY"], + region_name="us-east-1", + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + ), + ) + + for attempt in range(30): + try: + client.head_bucket(Bucket=bucket) + print(f"Bucket already exists: {bucket}", flush=True) + break + + except ClientError as exc: + code = exc.response["Error"]["Code"] + + if code in ("404", "NoSuchBucket"): + try: + client.create_bucket(Bucket=bucket) + print(f"Bucket created: {bucket}", flush=True) + break + except Exception as create_error: + print( + f"Waiting for S3: {create_error}", + flush=True, + ) + else: + print(f"Waiting for S3: {exc}", flush=True) + + except Exception as exc: + print(f"Waiting for S3: {exc}", flush=True) + + if attempt == 29: + raise RuntimeError("Cannot initialize S3 bucket") + + time.sleep(2) + + # No public bucket policy is configured. + print(f"Private S3 bucket is ready: {bucket}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..a050a41 --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,168 @@ + +import logging +import os +import socket + +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import ( + OTLPLogExporter, +) +from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, +) +from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, +) +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import ( + BatchLogRecordProcessor, +) +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import ( + PeriodicExportingMetricReader, +) +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ( + BatchSpanProcessor, +) + + +SERVICE_VERSION = "lab6-1.0" +ENVIRONMENT = "lab" + +OTLP_ENDPOINT = os.getenv( + "OTEL_EXPORTER_OTLP_ENDPOINT", + "http://otel-collector:4318", +).rstrip("/") + + +class TraceContextFilter(logging.Filter): + def filter(self, record): + context = trace.get_current_span().get_span_context() + + if context.is_valid: + record.trace_id = format( + context.trace_id, "032x" + ) + record.span_id = format( + context.span_id, "016x" + ) + else: + record.trace_id = "-" + record.span_id = "-" + + return True + + +def setup_telemetry(service_name): + """ + Initialize OTLP traces, metrics and logs. + + Do not record request bodies, Authorization + headers, message text or image bytes. + """ + + resource = Resource.create( + { + "service.name": service_name, + "service.version": SERVICE_VERSION, + "service.instance.id": os.getenv( + "HOSTNAME", + socket.gethostname(), + ), + "deployment.environment.name": ENVIRONMENT, + } + ) + + # Distributed traces -> Collector -> Tempo + tracer_provider = TracerProvider(resource=resource) + + trace_exporter = OTLPSpanExporter( + endpoint=f"{OTLP_ENDPOINT}/v1/traces", + timeout=3, + ) + + tracer_provider.add_span_processor( + BatchSpanProcessor( + trace_exporter, + max_queue_size=1024, + ) + ) + + trace.set_tracer_provider(tracer_provider) + + # Metrics -> Collector -> Prometheus + metric_exporter = OTLPMetricExporter( + endpoint=f"{OTLP_ENDPOINT}/v1/metrics", + timeout=3, + ) + + metric_reader = PeriodicExportingMetricReader( + metric_exporter, + export_interval_millis=5000, + export_timeout_millis=3000, + ) + + meter_provider = MeterProvider( + resource=resource, + metric_readers=[metric_reader], + ) + + metrics.set_meter_provider(meter_provider) + + # Structured logs -> Collector -> Loki + logger_provider = LoggerProvider(resource=resource) + + log_exporter = OTLPLogExporter( + endpoint=f"{OTLP_ENDPOINT}/v1/logs", + timeout=3, + ) + + logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + log_exporter, + max_queue_size=1024, + ) + ) + + set_logger_provider(logger_provider) + + # Dedicated application logger. + # We intentionally do not forward arbitrary + # library logs or sensitive request data. + logger = logging.getLogger("messenger.telemetry") + logger.setLevel(logging.INFO) + logger.propagate = False + + for handler in list(logger.handlers): + logger.removeHandler(handler) + + context_filter = TraceContextFilter() + + console_handler = logging.StreamHandler() + console_handler.addFilter(context_filter) + console_handler.setFormatter( + logging.Formatter( + "%(asctime)s %(levelname)s " + "event=%(event)s " + "trace_id=%(trace_id)s " + "span_id=%(span_id)s " + "%(message)s" + ) + ) + + logger.addHandler(console_handler) + + otlp_handler = LoggingHandler( + level=logging.INFO, + logger_provider=logger_provider, + ) + otlp_handler.addFilter(context_filter) + logger.addHandler(otlp_handler) + + tracer = trace.get_tracer(service_name, SERVICE_VERSION) + meter = metrics.get_meter(service_name, SERVICE_VERSION) + + return tracer, meter, logger diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..5bde427 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,576 @@ + +import hashlib +import io +import logging +import os +import time + +import boto3 +import pika +import psycopg +from opentelemetry.trace import SpanKind, Status, StatusCode +from src.telemetry import setup_telemetry +from botocore.config import Config +from PIL import Image, ImageOps +from psycopg.rows import dict_row + +from src.queue import ( + RABBITMQ_QUEUE, + connect, + declare_queue, + parse_job, +) + + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", +) + +logger = logging.getLogger("image-worker") + +worker_tracer, worker_meter, worker_logger = setup_telemetry( + "messenger-worker" +) + +jobs_processed = worker_meter.create_counter( + name="messenger_worker_jobs_processed", + unit="{job}", + description="Image processing job attempts by outcome", +) + +job_duration = worker_meter.create_histogram( + name="messenger_worker_job_duration", + unit="s", + description="Image processing duration in seconds", +) + +DATABASE_URL = os.environ["DATABASE_URL"] +S3_BUCKET = os.environ["S3_BUCKET"] +MAX_BYTES = 10 * 1024 * 1024 + +s3 = boto3.client( + "s3", + endpoint_url=os.environ["S3_ENDPOINT"], + aws_access_key_id=os.environ["S3_ACCESS_KEY"], + aws_secret_access_key=os.environ["S3_SECRET_KEY"], + region_name="us-east-1", + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=3, + read_timeout=15, + retries={"max_attempts": 1}, + ), +) + + +class InvalidImage(Exception): + pass + + +def make_thumbnail(original): + try: + with Image.open(io.BytesIO(original)) as image: + if image.format not in ("PNG", "JPEG"): + raise InvalidImage("Unsupported image") + + if image.width * image.height > 40_000_000: + raise InvalidImage("Too many pixels") + + image = ImageOps.exif_transpose(image) + image.thumbnail( + (256, 256), + Image.Resampling.LANCZOS, + ) + + # JPEG does not support transparency. + rgba = image.convert("RGBA") + + background = Image.new( + "RGB", + rgba.size, + (255, 255, 255), + ) + + background.paste( + rgba, + mask=rgba.getchannel("A"), + ) + + output = io.BytesIO() + + background.save( + output, + format="JPEG", + quality=85, + optimize=True, + ) + + return ( + output.getvalue(), + background.width, + background.height, + ) + + except (OSError, SyntaxError, EOFError) as exc: + raise InvalidImage("Invalid image") from exc + + +def get_staging_object(key): + response = s3.get_object( + Bucket=S3_BUCKET, + Key=key, + ) + + stream = response["Body"] + + try: + data = stream.read(MAX_BYTES + 1) + finally: + stream.close() + + if not data or len(data) > MAX_BYTES: + raise InvalidImage("Invalid staging size") + + return data + + +def process_job(job_id): + """ + Use a PostgreSQL row lock to prevent concurrent + processing of the same logical job. + + The transaction commits before RabbitMQ ACK. + """ + + with psycopg.connect( + DATABASE_URL, + row_factory=dict_row, + ) as conn: + + job = conn.execute( + """ + SELECT j.*, a.staging_key, + a.object_key, a.sha256, + a.size_bytes, a.status AS attachment_status + FROM attachment_jobs j + JOIN attachments a + ON a.id = j.attachment_id + WHERE j.id = %s + FOR UPDATE OF j, a + """, + (job_id,), + ).fetchone() + + if job is None: + logger.info("Unknown job %s, ignoring", job_id) + return + + if job["status"] == "ready": + logger.info("Job %s already completed", job_id) + return + + if job["status"] == "failed": + logger.info("Job %s already failed", job_id) + return + + if job["next_attempt_at"] is not None: + from datetime import datetime, timezone + + if job["next_attempt_at"] > datetime.now(timezone.utc): + logger.info("Job %s is not due yet", job_id) + return + + attempts = job["attempts"] + 1 + + if attempts > job["max_attempts"]: + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', + completed_at = NOW(), + updated_at = NOW() + WHERE id = %s + """, + (job_id,), + ) + + conn.execute( + """ + UPDATE attachments + SET status = 'failed', + error_code = 'processing_failed', + updated_at = NOW() + WHERE id = %s + """, + (job["attachment_id"],), + ) + + return + + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'processing', + attempts = %s, + started_at = NOW(), + updated_at = NOW() + WHERE id = %s + """, + (attempts, job_id), + ) + + conn.execute( + """ + UPDATE attachments + SET status = 'processing', + updated_at = NOW() + WHERE id = %s + """, + (job["attachment_id"],), + ) + + try: + original = get_staging_object( + job["staging_key"] + ) + + if ( + len(original) != job["size_bytes"] + or hashlib.sha256(original).hexdigest() + != job["sha256"] + ): + raise InvalidImage("Staging checksum mismatch") + + thumbnail, width, height = make_thumbnail( + original + ) + + attachment_id = str(job["attachment_id"]) + + # Stable keys make duplicate delivery safe. + original_key = job["object_key"] + thumb_key = ( + f"thumbnails/{attachment_id}.jpg" + ) + + s3.put_object( + Bucket=S3_BUCKET, + Key=original_key, + Body=original, + ContentType="image/png" + if original[:8] == b"\x89PNG\r\n\x1a\n" + else "image/jpeg", + ) + + s3.put_object( + Bucket=S3_BUCKET, + Key=thumb_key, + Body=thumbnail, + ContentType="image/jpeg", + ) + + thumb_sha256 = hashlib.sha256( + thumbnail + ).hexdigest() + + conn.execute( + """ + INSERT INTO attachment_variants + ( + attachment_id, + variant_name, + object_key, + content_type, + size_bytes, + sha256, + width, + height + ) + VALUES ( + %s, 'thumbnail', %s, 'image/jpeg', + %s, %s, %s, %s + ) + ON CONFLICT ( + attachment_id, variant_name + ) + DO UPDATE SET + object_key = EXCLUDED.object_key, + size_bytes = EXCLUDED.size_bytes, + sha256 = EXCLUDED.sha256, + width = EXCLUDED.width, + height = EXCLUDED.height + """, + ( + job["attachment_id"], + thumb_key, + len(thumbnail), + thumb_sha256, + width, + height, + ), + ) + + conn.execute( + """ + UPDATE attachments + SET status = 'ready', + error_code = NULL, + updated_at = NOW() + WHERE id = %s + """, + (job["attachment_id"],), + ) + + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'ready', + completed_at = NOW(), + updated_at = NOW() + WHERE id = %s + """, + (job_id,), + ) + + logger.info( + "Job %s ready, thumbnail %sx%s", + job_id, + width, + height, + ) + + except Exception as exc: + logger.exception( + "Processing failed for job %s: %s", + job_id, + type(exc).__name__, + ) + + if ( + attempts >= job["max_attempts"] + or isinstance(exc, InvalidImage) + ): + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', + completed_at = NOW(), + updated_at = NOW() + WHERE id = %s + """, + (job_id,), + ) + + conn.execute( + """ + UPDATE attachments + SET status = 'failed', + error_code = 'processing_failed', + updated_at = NOW() + WHERE id = %s + """, + (job["attachment_id"],), + ) + + else: + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', + next_attempt_at = + NOW() + INTERVAL '5 seconds', + published_at = NULL, + updated_at = NOW() + WHERE id = %s + """, + (job_id,), + ) + + conn.execute( + """ + UPDATE attachments + SET status = 'queued', + updated_at = NOW() + WHERE id = %s + """, + (job["attachment_id"],), + ) + + +def get_job_status(job_id): + with psycopg.connect(DATABASE_URL) as conn: + row = conn.execute( + """ + SELECT status + FROM attachment_jobs + WHERE id = %s + """, + (job_id,), + ).fetchone() + + return row[0] if row else None + + +def on_message(channel, method, properties, body): + try: + job_id = parse_job(body) + except (ValueError, TypeError, KeyError): + logger.warning("Invalid queue message") + channel.basic_ack( + delivery_tag=method.delivery_tag + ) + return + + attributes = { + "messaging.system": "rabbitmq", + "messaging.destination.name": RABBITMQ_QUEUE, + "messaging.operation.name": "process", + "messenger.job_type": "image_thumbnail", + } + + # A real consumer span around the actual job processing. + with worker_tracer.start_as_current_span( + "image_job.process", + kind=SpanKind.CONSUMER, + attributes=attributes, + ) as span: + + started = time.monotonic() + + try: + previous_status = get_job_status(job_id) + + process_job(job_id) + + current_status = get_job_status(job_id) + duration = time.monotonic() - started + + if previous_status in ("ready", "failed"): + outcome = "duplicate" + + elif current_status == "ready": + outcome = "success" + + elif current_status == "failed": + outcome = "failed" + + elif current_status == "queued": + outcome = "retry" + + else: + outcome = "ignored" + + span.set_attribute( + "messenger.job.outcome", + outcome, + ) + + if outcome == "failed": + span.set_status( + Status(StatusCode.ERROR) + ) + + # Count actual processing attempts, not + # duplicate deliveries of completed jobs. + if outcome in ("success", "failed", "retry"): + metric_attributes = { + "job_type": "image_thumbnail", + "outcome": outcome, + } + + jobs_processed.add( + 1, + attributes=metric_attributes, + ) + + job_duration.record( + duration, + attributes=metric_attributes, + ) + + worker_logger.info( + "Image job handled", + extra={ + "event": "image_job_handled", + "job_type": "image_thumbnail", + "outcome": outcome, + "duration_seconds": round(duration, 4), + }, + ) + + # ACK after process_job has committed the + # PostgreSQL transaction. + channel.basic_ack( + delivery_tag=method.delivery_tag + ) + + except Exception as exc: + span.record_exception(exc) + span.set_status(Status(StatusCode.ERROR)) + + worker_logger.error( + "Image job processing interrupted", + extra={ + "event": "image_job_interrupted", + "job_type": "image_thumbnail", + "error_type": type(exc).__name__, + }, + ) + + logger.exception( + "Unexpected failure for job %s", + job_id, + ) + + channel.basic_nack( + delivery_tag=method.delivery_tag, + requeue=True, + ) + + time.sleep(2) + + + +def main(): + while True: + try: + connection = connect() + channel = connection.channel() + + declare_queue(channel) + + channel.basic_qos( + prefetch_count=1 + ) + + channel.basic_consume( + queue=RABBITMQ_QUEUE, + on_message_callback=on_message, + auto_ack=False, + ) + + logger.info( + "Worker listening on %s", + RABBITMQ_QUEUE, + ) + + channel.start_consuming() + + except Exception: + logger.exception( + "Worker disconnected, reconnecting" + ) + time.sleep(3) + + finally: + if ( + "connection" in locals() + and connection.is_open + ): + connection.close() + + +if __name__ == "__main__": + main()